Merge branch 'develop' into 2.3-yj: this picks up SLIDER-82, Anti-Affinity
# Conflicts:
# app-packages/hbase/appConfig-default.json
# slider-core/src/main/java/org/apache/slider/client/SliderClient.java
# slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
# slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
# slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
# slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
diff --git a/NOTICE b/NOTICE
index 64e2234..aa37418 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache Slider
-Copyright 2015 The Apache Software Foundation
+Copyright 2014-2015 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 11d04b7..8744741 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/accumulo/src/license/THIRD-PARTY.properties b/app-packages/accumulo/src/license/THIRD-PARTY.properties
index 9322b8f..fb20f7e 100644
--- a/app-packages/accumulo/src/license/THIRD-PARTY.properties
+++ b/app-packages/accumulo/src/license/THIRD-PARTY.properties
@@ -2,10 +2,6 @@
#-------------------------------------------------------------------------------
# Already used licenses in project :
# - Apache License
-# - Apache License 2.0
-# - Apache License Version 2
-# - Apache License, Version 2.0
-# - Apache Software License - Version 2.0
# - BSD
# - CDDL + GPLv2 with classpath exception
# - CDDL 1.1
@@ -13,10 +9,10 @@
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - New BSD License
-# - New BSD license
# - Public Domain
# - Revised BSD
# - The Apache Software License, Version 2.0
@@ -26,8 +22,7 @@
# Please fill the missing licenses for dependencies :
#
#
-#Mon Jan 26 16:00:07 GMT 2015
-asm--asm--3.1=The BSD 3-Clause License
+#Thu Oct 15 16:45:03 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml
index 23d355c..82a4c81 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml
index ea66881..0699a80 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/hbase-win/pom.xml b/app-packages/hbase-win/pom.xml
index e641006..424b2c2 100644
--- a/app-packages/hbase-win/pom.xml
+++ b/app-packages/hbase-win/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 3188d60..4d1cea0 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/pom.xml b/app-packages/pom.xml
index 6f9d0d1..f32e71c 100644
--- a/app-packages/pom.xml
+++ b/app-packages/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/storm-win/pom.xml b/app-packages/storm-win/pom.xml
index b6e97c2..d8219db 100644
--- a/app-packages/storm-win/pom.xml
+++ b/app-packages/storm-win/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index b701698..bd293e3 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -38,7 +38,7 @@
nimbus_port = config['configurations']['storm-site']['nimbus.thrift.port']
rest_api_conf_file = format("{conf_dir}/config.yaml")
rest_lib_dir = format("{app_root}/external/storm-rest")
-storm_bin = format("{app_root}/bin/storm.py")
+storm_bin = format("{app_root}/bin/storm")
storm_env_sh_template = config['configurations']['storm-env']['content']
metric_collector_host = default('/configurations/global/metric_collector_host', '')
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
index 6bc95ac..75e918a 100644
--- a/app-packages/storm/pom.xml
+++ b/app-packages/storm/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index 1287daf..93e9540 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
<name>Slider</name>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<description>
@@ -113,7 +113,7 @@
-->
<project.java.src.version>1.7</project.java.src.version>
<enforced.java.version>${project.java.src.version}</enforced.java.version>
- <groovy.version>2.4.0</groovy.version>
+ <groovy.version>2.4.5</groovy.version>
<!--
test options
@@ -234,17 +234,21 @@
<github.site.plugin.version>0.8</github.site.plugin.version>
<maven-site-plugin.skipDeploy>true</maven-site-plugin.skipDeploy>
<github.downloads.plugin.version>0.6</github.downloads.plugin.version>
+
+ <!-- override point for ASF staging/snapshot repos -->
+ <asf.staging>https://repository.apache.org/content/groups/staging/</asf.staging>
+ <asf.snapshots>https://repository.apache.org/content/repositories/snapshots/</asf.snapshots>
</properties>
<repositories>
<repository>
<id>ASF Staging</id>
- <url>https://repository.apache.org/content/groups/staging/</url>
+ <url>${asf.staging}</url>
</repository>
<repository>
<id>ASF Snapshots</id>
- <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ <url>${asf.snapshots}</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
@@ -517,6 +521,13 @@
</dependency>
<dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ <classifier>indy</classifier>
+ <version>${groovy.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
<version>${jcommander.version}</version>
@@ -530,7 +541,7 @@
<!-- hadoop-client includes the following jars, so they do not need to be
included separately:
- hadoop-common, hadoop-hdfs, hadoop-mapreduce-client-app,
+ hadoop-common, hadoop-hdfs (client?), hadoop-mapreduce-client-app,
hadoop-yarn-api, hadoop-mapreduce-client-core,
hadoop-mapreduce-client-jobclient, and hadoop-annotations
-->
@@ -558,6 +569,34 @@
</exclusions>
</dependency>
+ <!--
+ Hadoop HDFS is pulled in to guarantee that any split to hadoop-hdfs-client doesn't
+ break the build.
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<!-- hadoop-minicluster includes the following test-jars, so they do not
need to be included separately:
hadoop-common, hadoop-hdfs, hadoop-yarn-server-tests,
@@ -596,7 +635,7 @@
<artifactId>hadoop-yarn-registry</artifactId>
<version>${hadoop.version}</version>
</dependency>
-
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
@@ -1147,6 +1186,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
@@ -1394,6 +1439,12 @@
<version>1.5</version>
</dependency>
+ <dependency>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ <version>3.3.1</version>
+ </dependency>
+
</dependencies>
</dependencyManagement>
@@ -1464,6 +1515,7 @@
<exclude>DISCLAIMER</exclude>
<exclude>app-packages/hbase/target/**</exclude>
<exclude>target/*</exclude>
+ <exclude>DEPENDENCIES</exclude>
</excludes>
</configuration>
</plugin>
@@ -1627,25 +1679,51 @@
<profile>
<!-- 2.7.0 -->
- <id>release-2.7</id>
+ <id>release-2.7.0</id>
<properties>
<hadoop.version>2.7.0</hadoop.version>
+ <curator.version>2.7.1</curator.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
</properties>
</profile>
<profile>
- <!-- hadoop branch-2.7 builds -->
- <id>branch-2.7</id>
+ <!-- hadoop 2.7.1 -->
+ <id>release-2.7.1</id>
<properties>
- <hadoop.version>2.7.1-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.7.1</hadoop.version>
+ <curator.version>2.7.1</curator.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- hadoop 2.7.1 -->
+ <id>release-2.7</id>
+ <properties>
+ <hadoop.version>2.7.1</hadoop.version>
+ <curator.version>2.7.1</curator.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
</properties>
</profile>
<profile>
<!-- hadoop branch-2 builds -->
- <id>branch-2</id>
+ <id>branch-2.8</id>
<properties>
<hadoop.version>2.8.0-SNAPSHOT</hadoop.version>
+ <curator.version>2.7.1</curator.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <!-- hadoop branch-2 builds -->
+ <id>branch-2</id>
+ <properties>
+ <hadoop.version>2.9.0-SNAPSHOT</hadoop.version>
+ <curator.version>2.7.1</curator.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
</properties>
</profile>
@@ -1654,6 +1732,8 @@
<id>trunk</id>
<properties>
<hadoop.version>3.0.0-SNAPSHOT</hadoop.version>
+ <curator.version>2.7.1</curator.version>
+ <zookeeper.version>3.4.6</zookeeper.version>
</properties>
</profile>
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index c494b77..009a864 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>slider-agent</artifactId>
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index 24f8bf4..0f7340f 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
diff --git a/slider-assembly/src/conf/slider-client.xml b/slider-assembly/src/conf/slider-client.xml
index 3a42bee..9c2e76d 100644
--- a/slider-assembly/src/conf/slider-client.xml
+++ b/slider-assembly/src/conf/slider-client.xml
@@ -22,14 +22,6 @@
-->
<configuration>
- <!--
- The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR.
- Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file
- for Slider client to work. The following list is not an exhaustive list but the minimal config
- needed to interact with a non-secure cluster.
- -->
-
- <!--
<property>
<name>slider.client.resource.origin</name>
<value>conf/slider-client.xml</value>
@@ -37,14 +29,41 @@
</property>
<property>
- <name>yarn.log-aggregation-enable</name>
- <value>true</value>
+ <name>slider.security.protocol.acl</name>
+ <value>*</value>
+ <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
</property>
<property>
- <name>slider.security.protocol.acl</name>
- <value>*</value>
- <description>When security is enabled, set appropriate acl. Default value means allow everyone.</description>
+ <name>slider.yarn.queue</name>
+ <value/>
+ <description>the name of the YARN queue to use.</description>
+ </property>
+
+ <property>
+ <name>slider.yarn.queue.priority</name>
+ <value>1</value>
+ <description>the priority of the application.</description>
+ </property>
+
+ <property>
+ <name>slider.am.login.keytab.required</name>
+ <value>false</value>
+ <description>Declare that a keytab must be provided.</description>
+ </property>
+
+ <!--
+ The recommended approach is to configure slider-env.sh and set HADOOP_CONF_DIR.
+ Otherwise, appropriate configurations from hdfs-site, yarn-site, can be dropped in this file
+ for Slider client to work. The following list is not an exhaustive list but the minimal config
+ needed to interact with a non-secure cluster.
+ -->
+
+ <!--
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
</property>
<property>
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index bea32e4..c7e8933 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
<build>
@@ -225,6 +225,7 @@
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
+ <classifier>indy</classifier>
<scope>test</scope>
</dependency>
@@ -258,6 +259,19 @@
<scope>compile</scope>
</dependency>
+<!-- if the move to hadoop-hdfs-client JAR causes problems insert this
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+-->
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
@@ -369,6 +383,14 @@
<artifactId>curator-framework</artifactId>
</dependency>
+ <!-- not directly needed by the registry, but inserted to
+ retain control over which version is on the classpath.
+ Otherwise, slider-core gets the version Hadoop-common depends on -->
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
@@ -506,7 +528,6 @@
<dependency>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
- <version>3.3.1</version>
</dependency>
</dependencies>
diff --git a/slider-core/src/license/THIRD-PARTY.properties b/slider-core/src/license/THIRD-PARTY.properties
index 662fde5..1abd56e 100644
--- a/slider-core/src/license/THIRD-PARTY.properties
+++ b/slider-core/src/license/THIRD-PARTY.properties
@@ -10,6 +10,7 @@
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - MPL 1.1
@@ -24,8 +25,7 @@
# Please fill the missing licenses for dependencies :
#
#
-#Mon Jan 26 15:27:00 GMT 2015
-asm--asm--3.1=The BSD 3-Clause License
+#Thu Oct 15 16:45:02 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
index 025bd32..8358491 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescription.java
@@ -65,6 +65,12 @@
* As a wire format it is less efficient in both xfer and ser/deser than
* a binary format, but by having one unified format for wire and persistence,
* the code paths are simplified.
+ *
+ * This was the original single-file specification/model used in the Hoya
+ * precursor to Slider. Its now retained primarily as a way to publish
+ * the current state of the application, or at least a fraction thereof ...
+ * the larger set of information from the REST API is beyond the scope of
+ * this structure.
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@@ -165,41 +171,35 @@
* cluster-specific options -to control both
* the Slider AM and the application that it deploys
*/
- public Map<String, String> options =
- new HashMap<String, String>();
+ public Map<String, String> options = new HashMap<>();
/**
* cluster information
* This is only valid when querying the cluster status.
*/
- public Map<String, String> info =
- new HashMap<String, String>();
+ public Map<String, String> info = new HashMap<>();
/**
* Statistics. This is only relevant when querying the cluster status
*/
- public Map<String, Map<String, Integer>> statistics =
- new HashMap<String, Map<String, Integer>>();
+ public Map<String, Map<String, Integer>> statistics = new HashMap<>();
/**
* Instances: role->count
*/
- public Map<String, List<String>> instances =
- new HashMap<String, List<String>>();
+ public Map<String, List<String>> instances = new HashMap<>();
/**
* Role options,
* role -> option -> value
*/
- public Map<String, Map<String, String>> roles =
- new HashMap<String, Map<String, String>>();
+ public Map<String, Map<String, String>> roles = new HashMap<>();
/**
* List of key-value pairs to add to a client config to set up the client
*/
- public Map<String, String> clientProperties =
- new HashMap<String, String>();
+ public Map<String, String> clientProperties = new HashMap<>();
/**
* Status information
@@ -218,7 +218,6 @@
public ClusterDescription() {
}
-
@Override
public String toString() {
try {
@@ -284,8 +283,8 @@
* @param dataOutputStream an outout stream that will always be closed
* @throws IOException any failure
*/
- private void writeJsonAsBytes(DataOutputStream dataOutputStream) throws
- IOException {
+ private void writeJsonAsBytes(DataOutputStream dataOutputStream)
+ throws IOException {
try {
String json = toJsonString();
byte[] b = json.getBytes(UTF_8);
@@ -303,7 +302,7 @@
* @throws IOException IO problems
*/
public static ClusterDescription load(FileSystem fs, Path path)
- throws IOException, JsonParseException, JsonMappingException {
+ throws IOException, JsonParseException, JsonMappingException {
FileStatus status = fs.getFileStatus(path);
byte[] b = new byte[(int) status.getLen()];
FSDataInputStream dataInputStream = fs.open(path);
@@ -389,7 +388,7 @@
try {
return mapper.readValue(jsonFile, ClusterDescription.class);
} catch (IOException e) {
- log.error("Exception while parsing json file {}: {}" , jsonFile, e);
+ log.error("Exception while parsing json file {}" , jsonFile, e);
throw e;
}
}
@@ -528,20 +527,20 @@
}
String val = roleopts.get(option);
if (val == null) {
- throw new BadConfigException("Missing option '%s' in role %s ", option,
- role);
+ throw new BadConfigException("Missing option '%s' in role %s ", option, role);
}
return val;
}
- /**
+
+ /**
* Get a mandatory integer role option
* @param role role to get from
* @param option option name
* @return resolved value
* @throws BadConfigException if the option is not defined
*/
- public int getMandatoryRoleOptInt(String role, String option) throws
- BadConfigException {
+ public int getMandatoryRoleOptInt(String role, String option)
+ throws BadConfigException {
getMandatoryRoleOpt(role, option);
return getRoleOptInt(role, option, 0);
}
@@ -564,7 +563,7 @@
public Map<String, String> getOrAddRole(String role) {
Map<String, String> map = getRole(role);
if (map == null) {
- map = new HashMap<String, String>();
+ map = new HashMap<>();
}
roles.put(role, map);
return map;
@@ -575,7 +574,7 @@
*/
@JsonIgnore
public Set<String> getRoleNames() {
- return new HashSet<String>(roles.keySet());
+ return new HashSet<>(roles.keySet());
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
index 21ece2b..5b95414 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterDescriptionOperations.java
@@ -80,8 +80,7 @@
}
private static void mergeInComponentMap(ClusterDescription cd,
- ConfTree confTree
- ) {
+ ConfTree confTree) {
Map<String, Map<String, String>> components = confTree.components;
for (Map.Entry<String, Map<String, String>> compEntry : components.entrySet()) {
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
index 2608cd7..d255db0 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
@@ -38,11 +38,11 @@
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL )
public final class ClusterNode implements Cloneable {
protected static final Logger
- LOG = LoggerFactory.getLogger(ClusterDescription.class);
+ LOG = LoggerFactory.getLogger(ClusterNode.class);
@JsonIgnore
public ContainerId containerId;
-
+
/**
* server name
*/
@@ -67,8 +67,7 @@
public boolean released;
public String host;
public String hostUrl;
-
-
+
/**
* state from {@link ClusterDescription}
*/
@@ -165,7 +164,7 @@
try {
return mapper.readValue(json, ClusterNode.class);
} catch (IOException e) {
- LOG.error("Exception while parsing json : " + e + "\n" + json, e);
+ LOG.error("Exception while parsing json : {}\n{}", e , json, e);
throw e;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index f481c6a..f92a58d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -167,4 +167,16 @@
*/
String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
+
+ /**
+ * Window of time where application master's failure count
+ * can be reset to 0.
+ */
+ String YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS =
+ "yarn.resourcemanager.am.retry-count-window-ms";
+
+ /**
+ * The default window for Slider.
+ */
+ long DEFAULT_AM_RETRY_COUNT_WINDOW_MS = 300000;
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 4512354..eda01ad 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -65,6 +65,11 @@
String ROLE_PREEMPTED_INSTANCES = "role.failed.preempted.instances";
/**
+ * Number of pending anti-affine instances: {@value}
+ */
+ String ROLE_PENDING_AA_INSTANCES = "role.pending.aa.instances";
+
+ /**
* Status report: number currently being released: {@value}
*/
String ROLE_FAILED_STARTING_INSTANCES = "role.failed.starting.instances";
@@ -98,19 +103,4 @@
*/
String ENV_PREFIX = "env.";
-
- /**
- * Default no. of cores in the AM {@value}
- */
- int DEFAULT_AM_V_CORES = 1;
-
- /**
- * The default memory of the AM: {@value}
- */
- int DEFAULT_AM_MEMORY = 1024;
-
- /**
- * The default heap of the AM: {@value}
- */
- String DEFAULT_AM_HEAP = "512M";
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java
index 7de1915..d21785f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java
+++ b/slider-core/src/main/java/org/apache/slider/api/SliderApplicationApi.java
@@ -21,6 +21,8 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
@@ -36,7 +38,7 @@
/**
* Get the aggregate desired model
* @return the aggregate configuration of what was asked for
- * —before resolution has taken place
+ * -before resolution has taken place
* @throws IOException on any failure
*/
AggregateConf getDesiredModel() throws IOException;
@@ -44,7 +46,7 @@
/**
* Get the desired application configuration
* @return the application configuration asked for
- * —before resolution has taken place
+ * -before resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getDesiredAppconf() throws IOException;
@@ -52,7 +54,7 @@
/**
* Get the desired YARN resources
* @return the resources asked for
- * —before resolution has taken place
+ * -before resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getDesiredResources() throws IOException;
@@ -68,7 +70,7 @@
/**
* Get the aggregate resolved model
* @return the aggregate configuration of what was asked for
- * —after resolution has taken place
+ * -after resolution has taken place
* @throws IOException on any failure
*/
AggregateConf getResolvedModel() throws IOException;
@@ -76,7 +78,7 @@
/**
* Get the resolved application configuration
* @return the application configuration asked for
- * —after resolution has taken place
+ * -after resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getResolvedAppconf() throws IOException;
@@ -84,7 +86,7 @@
/**
* Get the resolved YARN resources
* @return the resources asked for
- * —after resolution has taken place
+ * -after resolution has taken place
* @throws IOException on any failure
*/
ConfTreeOperations getResolvedResources() throws IOException;
@@ -101,8 +103,7 @@
* @return a possibly empty list of serialized containers
* @throws IOException on any failure
*/
- Map<String, ContainerInformation> enumContainers() throws
- IOException;
+ Map<String, ContainerInformation> enumContainers() throws IOException;
/**
* Get a container from the container Id
@@ -110,16 +111,14 @@
* @return the container information
* @throws IOException on any failure
*/
- ContainerInformation getContainer(String containerId) throws
- IOException;
+ ContainerInformation getContainer(String containerId) throws IOException;
/**
* List all components into a map of [name:info]
* @return a possibly empty map of components
* @throws IOException on any failure
*/
- Map<String, ComponentInformation> enumComponents() throws
- IOException;
+ Map<String, ComponentInformation> enumComponents() throws IOException;
/**
* Get information about a component
@@ -127,8 +126,22 @@
* @return the component details
* @throws IOException on any failure
*/
- ComponentInformation getComponent(String componentName) throws
- IOException;
+ ComponentInformation getComponent(String componentName) throws IOException;
+
+ /**
+ * List all nodes into a map of [name:info]
+ * @return a possibly empty list of nodes
+ * @throws IOException on any failure
+ */
+ NodeInformationList getLiveNodes() throws IOException;
+
+ /**
+ * Get information about a node
+ * @param hostname name of the node
+ * @return the node details
+ * @throws IOException on any failure
+ */
+ NodeInformation getLiveNode(String hostname) throws IOException;
/**
* Ping as a GET
diff --git a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
index 910521e..33fce22 100644
--- a/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
+++ b/slider-core/src/main/java/org/apache/slider/api/SliderClusterProtocol.java
@@ -30,8 +30,7 @@
* Cluster protocol. This can currently act as a versioned IPC
* endpoint or be relayed via protobuf
*/
-@KerberosInfo(
- serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL)
+@KerberosInfo(serverPrincipal = SliderXmlConfKeys.KEY_KERBEROS_PRINCIPAL)
public interface SliderClusterProtocol extends VersionedProtocol {
long versionID = 0x01;
@@ -39,10 +38,8 @@
* Stop the cluster
*/
-
Messages.StopClusterResponseProto stopCluster(Messages.StopClusterRequestProto request) throws
IOException, YarnException;
-
/**
* Upgrade the application containers
*
@@ -155,20 +152,26 @@
Messages.GetLiveComponentRequestProto request
) throws IOException;
-
+ Messages.GetLiveNodesResponseProto getLiveNodes(
+ Messages.GetLiveNodesRequestProto request
+ ) throws IOException;
+
+ Messages.NodeInformationProto getLiveNode(
+ Messages.GetLiveNodeRequestProto request
+ ) throws IOException;
+
Messages.WrappedJsonProto getModelDesired(Messages.EmptyPayloadProto request) throws IOException;
-
Messages.WrappedJsonProto getModelDesiredAppconf(Messages.EmptyPayloadProto request) throws IOException;
-
+
Messages.WrappedJsonProto getModelDesiredResources(Messages.EmptyPayloadProto request) throws IOException;
-
+
Messages.WrappedJsonProto getModelResolved(Messages.EmptyPayloadProto request) throws IOException;
-
+
Messages.WrappedJsonProto getModelResolvedAppconf(Messages.EmptyPayloadProto request) throws IOException;
-
+
Messages.WrappedJsonProto getModelResolvedResources(Messages.EmptyPayloadProto request) throws IOException;
-
+
Messages.WrappedJsonProto getLiveResources(Messages.EmptyPayloadProto request) throws IOException;
Messages.GetCertificateStoreResponseProto getClientCertificateStore(Messages.GetCertificateStoreRequestProto request)
diff --git a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
index 6a0a2fa..8a2c4bb 100644
--- a/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/StatusKeys.java
@@ -32,6 +32,7 @@
String STATISTICS_CONTAINERS_PREEMPTED = "containers.failed.preempted";
String STATISTICS_CONTAINERS_LIVE = "containers.live";
String STATISTICS_CONTAINERS_REQUESTED = "containers.requested";
+ String STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING = "containers.anti-affine.pending";
String STATISTICS_CONTAINERS_STARTED = "containers.start.started";
String STATISTICS_CONTAINERS_START_FAILED =
"containers.start.failed";
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
index 6c92f46..373d64d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/Messages.java
@@ -15100,6 +15100,26 @@
* <code>optional int32 preempted = 17;</code>
*/
int getPreempted();
+
+ // optional int32 pendingAntiAffineRequestCount = 18;
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ boolean hasPendingAntiAffineRequestCount();
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ int getPendingAntiAffineRequestCount();
+
+ // optional bool isAARequestOutstanding = 19;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ boolean hasIsAARequestOutstanding();
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ boolean getIsAARequestOutstanding();
}
/**
* Protobuf type {@code org.apache.slider.api.ComponentInformationProto}
@@ -15245,6 +15265,16 @@
preempted_ = input.readInt32();
break;
}
+ case 144: {
+ bitField0_ |= 0x00010000;
+ pendingAntiAffineRequestCount_ = input.readInt32();
+ break;
+ }
+ case 152: {
+ bitField0_ |= 0x00020000;
+ isAARequestOutstanding_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -15628,6 +15658,38 @@
return preempted_;
}
+ // optional int32 pendingAntiAffineRequestCount = 18;
+ public static final int PENDINGANTIAFFINEREQUESTCOUNT_FIELD_NUMBER = 18;
+ private int pendingAntiAffineRequestCount_;
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ public boolean hasPendingAntiAffineRequestCount() {
+ return ((bitField0_ & 0x00010000) == 0x00010000);
+ }
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ public int getPendingAntiAffineRequestCount() {
+ return pendingAntiAffineRequestCount_;
+ }
+
+ // optional bool isAARequestOutstanding = 19;
+ public static final int ISAAREQUESTOUTSTANDING_FIELD_NUMBER = 19;
+ private boolean isAARequestOutstanding_;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean hasIsAARequestOutstanding() {
+ return ((bitField0_ & 0x00020000) == 0x00020000);
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean getIsAARequestOutstanding() {
+ return isAARequestOutstanding_;
+ }
+
private void initFields() {
name_ = "";
priority_ = 0;
@@ -15646,6 +15708,8 @@
failedRecently_ = 0;
nodeFailed_ = 0;
preempted_ = 0;
+ pendingAntiAffineRequestCount_ = 0;
+ isAARequestOutstanding_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -15710,6 +15774,12 @@
if (((bitField0_ & 0x00008000) == 0x00008000)) {
output.writeInt32(17, preempted_);
}
+ if (((bitField0_ & 0x00010000) == 0x00010000)) {
+ output.writeInt32(18, pendingAntiAffineRequestCount_);
+ }
+ if (((bitField0_ & 0x00020000) == 0x00020000)) {
+ output.writeBool(19, isAARequestOutstanding_);
+ }
getUnknownFields().writeTo(output);
}
@@ -15792,6 +15862,14 @@
size += com.google.protobuf.CodedOutputStream
.computeInt32Size(17, preempted_);
}
+ if (((bitField0_ & 0x00010000) == 0x00010000)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(18, pendingAntiAffineRequestCount_);
+ }
+ if (((bitField0_ & 0x00020000) == 0x00020000)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(19, isAARequestOutstanding_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -15897,6 +15975,16 @@
result = result && (getPreempted()
== other.getPreempted());
}
+ result = result && (hasPendingAntiAffineRequestCount() == other.hasPendingAntiAffineRequestCount());
+ if (hasPendingAntiAffineRequestCount()) {
+ result = result && (getPendingAntiAffineRequestCount()
+ == other.getPendingAntiAffineRequestCount());
+ }
+ result = result && (hasIsAARequestOutstanding() == other.hasIsAARequestOutstanding());
+ if (hasIsAARequestOutstanding()) {
+ result = result && (getIsAARequestOutstanding()
+ == other.getIsAARequestOutstanding());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -15978,6 +16066,14 @@
hash = (37 * hash) + PREEMPTED_FIELD_NUMBER;
hash = (53 * hash) + getPreempted();
}
+ if (hasPendingAntiAffineRequestCount()) {
+ hash = (37 * hash) + PENDINGANTIAFFINEREQUESTCOUNT_FIELD_NUMBER;
+ hash = (53 * hash) + getPendingAntiAffineRequestCount();
+ }
+ if (hasIsAARequestOutstanding()) {
+ hash = (37 * hash) + ISAAREQUESTOUTSTANDING_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getIsAARequestOutstanding());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -16126,6 +16222,10 @@
bitField0_ = (bitField0_ & ~0x00008000);
preempted_ = 0;
bitField0_ = (bitField0_ & ~0x00010000);
+ pendingAntiAffineRequestCount_ = 0;
+ bitField0_ = (bitField0_ & ~0x00020000);
+ isAARequestOutstanding_ = false;
+ bitField0_ = (bitField0_ & ~0x00040000);
return this;
}
@@ -16224,6 +16324,14 @@
to_bitField0_ |= 0x00008000;
}
result.preempted_ = preempted_;
+ if (((from_bitField0_ & 0x00020000) == 0x00020000)) {
+ to_bitField0_ |= 0x00010000;
+ }
+ result.pendingAntiAffineRequestCount_ = pendingAntiAffineRequestCount_;
+ if (((from_bitField0_ & 0x00040000) == 0x00040000)) {
+ to_bitField0_ |= 0x00020000;
+ }
+ result.isAARequestOutstanding_ = isAARequestOutstanding_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -16302,6 +16410,12 @@
if (other.hasPreempted()) {
setPreempted(other.getPreempted());
}
+ if (other.hasPendingAntiAffineRequestCount()) {
+ setPendingAntiAffineRequestCount(other.getPendingAntiAffineRequestCount());
+ }
+ if (other.hasIsAARequestOutstanding()) {
+ setIsAARequestOutstanding(other.getIsAARequestOutstanding());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -17032,6 +17146,72 @@
return this;
}
+ // optional int32 pendingAntiAffineRequestCount = 18;
+ private int pendingAntiAffineRequestCount_ ;
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ public boolean hasPendingAntiAffineRequestCount() {
+ return ((bitField0_ & 0x00020000) == 0x00020000);
+ }
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ public int getPendingAntiAffineRequestCount() {
+ return pendingAntiAffineRequestCount_;
+ }
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ public Builder setPendingAntiAffineRequestCount(int value) {
+ bitField0_ |= 0x00020000;
+ pendingAntiAffineRequestCount_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int32 pendingAntiAffineRequestCount = 18;</code>
+ */
+ public Builder clearPendingAntiAffineRequestCount() {
+ bitField0_ = (bitField0_ & ~0x00020000);
+ pendingAntiAffineRequestCount_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // optional bool isAARequestOutstanding = 19;
+ private boolean isAARequestOutstanding_ ;
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean hasIsAARequestOutstanding() {
+ return ((bitField0_ & 0x00040000) == 0x00040000);
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public boolean getIsAARequestOutstanding() {
+ return isAARequestOutstanding_;
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public Builder setIsAARequestOutstanding(boolean value) {
+ bitField0_ |= 0x00040000;
+ isAARequestOutstanding_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool isAARequestOutstanding = 19;</code>
+ */
+ public Builder clearIsAARequestOutstanding() {
+ bitField0_ = (bitField0_ & ~0x00040000);
+ isAARequestOutstanding_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:org.apache.slider.api.ComponentInformationProto)
}
@@ -20132,6 +20312,3363 @@
// @@protoc_insertion_point(class_scope:org.apache.slider.api.PingInformationProto)
}
+ public interface NodeEntryInformationProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required int32 priority = 1;
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ boolean hasPriority();
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ int getPriority();
+
+ // required int32 requested = 2;
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ boolean hasRequested();
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ int getRequested();
+
+ // required int32 starting = 3;
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ boolean hasStarting();
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ int getStarting();
+
+ // required int32 startFailed = 4;
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ boolean hasStartFailed();
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ int getStartFailed();
+
+ // required int32 failed = 5;
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ boolean hasFailed();
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ int getFailed();
+
+ // required int32 failedRecently = 6;
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ boolean hasFailedRecently();
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ int getFailedRecently();
+
+ // required int32 preempted = 7;
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ boolean hasPreempted();
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ int getPreempted();
+
+ // required int32 live = 8;
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ boolean hasLive();
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ int getLive();
+
+ // required int32 releasing = 9;
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ boolean hasReleasing();
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ int getReleasing();
+
+ // required int64 lastUsed = 10;
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ boolean hasLastUsed();
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ long getLastUsed();
+
+ // required string name = 11;
+ /**
+ * <code>required string name = 11;</code>
+ */
+ boolean hasName();
+ /**
+ * <code>required string name = 11;</code>
+ */
+ java.lang.String getName();
+ /**
+ * <code>required string name = 11;</code>
+ */
+ com.google.protobuf.ByteString
+ getNameBytes();
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.NodeEntryInformationProto}
+ */
+ public static final class NodeEntryInformationProto extends
+ com.google.protobuf.GeneratedMessage
+ implements NodeEntryInformationProtoOrBuilder {
+ // Use NodeEntryInformationProto.newBuilder() to construct.
+ private NodeEntryInformationProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private NodeEntryInformationProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final NodeEntryInformationProto defaultInstance;
+ public static NodeEntryInformationProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public NodeEntryInformationProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private NodeEntryInformationProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 8: {
+ bitField0_ |= 0x00000001;
+ priority_ = input.readInt32();
+ break;
+ }
+ case 16: {
+ bitField0_ |= 0x00000002;
+ requested_ = input.readInt32();
+ break;
+ }
+ case 24: {
+ bitField0_ |= 0x00000004;
+ starting_ = input.readInt32();
+ break;
+ }
+ case 32: {
+ bitField0_ |= 0x00000008;
+ startFailed_ = input.readInt32();
+ break;
+ }
+ case 40: {
+ bitField0_ |= 0x00000010;
+ failed_ = input.readInt32();
+ break;
+ }
+ case 48: {
+ bitField0_ |= 0x00000020;
+ failedRecently_ = input.readInt32();
+ break;
+ }
+ case 56: {
+ bitField0_ |= 0x00000040;
+ preempted_ = input.readInt32();
+ break;
+ }
+ case 64: {
+ bitField0_ |= 0x00000080;
+ live_ = input.readInt32();
+ break;
+ }
+ case 72: {
+ bitField0_ |= 0x00000100;
+ releasing_ = input.readInt32();
+ break;
+ }
+ case 80: {
+ bitField0_ |= 0x00000200;
+ lastUsed_ = input.readInt64();
+ break;
+ }
+ case 90: {
+ bitField0_ |= 0x00000400;
+ name_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeEntryInformationProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto.class, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<NodeEntryInformationProto> PARSER =
+ new com.google.protobuf.AbstractParser<NodeEntryInformationProto>() {
+ public NodeEntryInformationProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new NodeEntryInformationProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<NodeEntryInformationProto> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required int32 priority = 1;
+ public static final int PRIORITY_FIELD_NUMBER = 1;
+ private int priority_;
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ public boolean hasPriority() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ public int getPriority() {
+ return priority_;
+ }
+
+ // required int32 requested = 2;
+ public static final int REQUESTED_FIELD_NUMBER = 2;
+ private int requested_;
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ public boolean hasRequested() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ public int getRequested() {
+ return requested_;
+ }
+
+ // required int32 starting = 3;
+ public static final int STARTING_FIELD_NUMBER = 3;
+ private int starting_;
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ public boolean hasStarting() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ public int getStarting() {
+ return starting_;
+ }
+
+ // required int32 startFailed = 4;
+ public static final int STARTFAILED_FIELD_NUMBER = 4;
+ private int startFailed_;
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ public boolean hasStartFailed() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ public int getStartFailed() {
+ return startFailed_;
+ }
+
+ // required int32 failed = 5;
+ public static final int FAILED_FIELD_NUMBER = 5;
+ private int failed_;
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ public boolean hasFailed() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ public int getFailed() {
+ return failed_;
+ }
+
+ // required int32 failedRecently = 6;
+ public static final int FAILEDRECENTLY_FIELD_NUMBER = 6;
+ private int failedRecently_;
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ public boolean hasFailedRecently() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ public int getFailedRecently() {
+ return failedRecently_;
+ }
+
+ // required int32 preempted = 7;
+ public static final int PREEMPTED_FIELD_NUMBER = 7;
+ private int preempted_;
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ public boolean hasPreempted() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ public int getPreempted() {
+ return preempted_;
+ }
+
+ // required int32 live = 8;
+ public static final int LIVE_FIELD_NUMBER = 8;
+ private int live_;
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ public boolean hasLive() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ public int getLive() {
+ return live_;
+ }
+
+ // required int32 releasing = 9;
+ public static final int RELEASING_FIELD_NUMBER = 9;
+ private int releasing_;
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ public boolean hasReleasing() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ public int getReleasing() {
+ return releasing_;
+ }
+
+ // required int64 lastUsed = 10;
+ public static final int LASTUSED_FIELD_NUMBER = 10;
+ private long lastUsed_;
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ public boolean hasLastUsed() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ public long getLastUsed() {
+ return lastUsed_;
+ }
+
+ // required string name = 11;
+ public static final int NAME_FIELD_NUMBER = 11;
+ private java.lang.Object name_;
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ name_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ private void initFields() {
+ priority_ = 0;
+ requested_ = 0;
+ starting_ = 0;
+ startFailed_ = 0;
+ failed_ = 0;
+ failedRecently_ = 0;
+ preempted_ = 0;
+ live_ = 0;
+ releasing_ = 0;
+ lastUsed_ = 0L;
+ name_ = "";
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasPriority()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasRequested()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasStarting()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasStartFailed()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasFailed()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasFailedRecently()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasPreempted()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasLive()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasReleasing()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasLastUsed()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeInt32(1, priority_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeInt32(2, requested_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeInt32(3, starting_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeInt32(4, startFailed_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeInt32(5, failed_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeInt32(6, failedRecently_);
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ output.writeInt32(7, preempted_);
+ }
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ output.writeInt32(8, live_);
+ }
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ output.writeInt32(9, releasing_);
+ }
+ if (((bitField0_ & 0x00000200) == 0x00000200)) {
+ output.writeInt64(10, lastUsed_);
+ }
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeBytes(11, getNameBytes());
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(1, priority_);
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(2, requested_);
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(3, starting_);
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(4, startFailed_);
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(5, failed_);
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(6, failedRecently_);
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(7, preempted_);
+ }
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(8, live_);
+ }
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt32Size(9, releasing_);
+ }
+ if (((bitField0_ & 0x00000200) == 0x00000200)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(10, lastUsed_);
+ }
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(11, getNameBytes());
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.slider.api.proto.Messages.NodeEntryInformationProto)) {
+ return super.equals(obj);
+ }
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto other = (org.apache.slider.api.proto.Messages.NodeEntryInformationProto) obj;
+
+ boolean result = true;
+ result = result && (hasPriority() == other.hasPriority());
+ if (hasPriority()) {
+ result = result && (getPriority()
+ == other.getPriority());
+ }
+ result = result && (hasRequested() == other.hasRequested());
+ if (hasRequested()) {
+ result = result && (getRequested()
+ == other.getRequested());
+ }
+ result = result && (hasStarting() == other.hasStarting());
+ if (hasStarting()) {
+ result = result && (getStarting()
+ == other.getStarting());
+ }
+ result = result && (hasStartFailed() == other.hasStartFailed());
+ if (hasStartFailed()) {
+ result = result && (getStartFailed()
+ == other.getStartFailed());
+ }
+ result = result && (hasFailed() == other.hasFailed());
+ if (hasFailed()) {
+ result = result && (getFailed()
+ == other.getFailed());
+ }
+ result = result && (hasFailedRecently() == other.hasFailedRecently());
+ if (hasFailedRecently()) {
+ result = result && (getFailedRecently()
+ == other.getFailedRecently());
+ }
+ result = result && (hasPreempted() == other.hasPreempted());
+ if (hasPreempted()) {
+ result = result && (getPreempted()
+ == other.getPreempted());
+ }
+ result = result && (hasLive() == other.hasLive());
+ if (hasLive()) {
+ result = result && (getLive()
+ == other.getLive());
+ }
+ result = result && (hasReleasing() == other.hasReleasing());
+ if (hasReleasing()) {
+ result = result && (getReleasing()
+ == other.getReleasing());
+ }
+ result = result && (hasLastUsed() == other.hasLastUsed());
+ if (hasLastUsed()) {
+ result = result && (getLastUsed()
+ == other.getLastUsed());
+ }
+ result = result && (hasName() == other.hasName());
+ if (hasName()) {
+ result = result && getName()
+ .equals(other.getName());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasPriority()) {
+ hash = (37 * hash) + PRIORITY_FIELD_NUMBER;
+ hash = (53 * hash) + getPriority();
+ }
+ if (hasRequested()) {
+ hash = (37 * hash) + REQUESTED_FIELD_NUMBER;
+ hash = (53 * hash) + getRequested();
+ }
+ if (hasStarting()) {
+ hash = (37 * hash) + STARTING_FIELD_NUMBER;
+ hash = (53 * hash) + getStarting();
+ }
+ if (hasStartFailed()) {
+ hash = (37 * hash) + STARTFAILED_FIELD_NUMBER;
+ hash = (53 * hash) + getStartFailed();
+ }
+ if (hasFailed()) {
+ hash = (37 * hash) + FAILED_FIELD_NUMBER;
+ hash = (53 * hash) + getFailed();
+ }
+ if (hasFailedRecently()) {
+ hash = (37 * hash) + FAILEDRECENTLY_FIELD_NUMBER;
+ hash = (53 * hash) + getFailedRecently();
+ }
+ if (hasPreempted()) {
+ hash = (37 * hash) + PREEMPTED_FIELD_NUMBER;
+ hash = (53 * hash) + getPreempted();
+ }
+ if (hasLive()) {
+ hash = (37 * hash) + LIVE_FIELD_NUMBER;
+ hash = (53 * hash) + getLive();
+ }
+ if (hasReleasing()) {
+ hash = (37 * hash) + RELEASING_FIELD_NUMBER;
+ hash = (53 * hash) + getReleasing();
+ }
+ if (hasLastUsed()) {
+ hash = (37 * hash) + LASTUSED_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getLastUsed());
+ }
+ if (hasName()) {
+ hash = (37 * hash) + NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getName().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeEntryInformationProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.slider.api.proto.Messages.NodeEntryInformationProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.NodeEntryInformationProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeEntryInformationProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto.class, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder.class);
+ }
+
+ // Construct using org.apache.slider.api.proto.Messages.NodeEntryInformationProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ priority_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ requested_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000002);
+ starting_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000004);
+ startFailed_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000008);
+ failed_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000010);
+ failedRecently_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000020);
+ preempted_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ live_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000080);
+ releasing_ = 0;
+ bitField0_ = (bitField0_ & ~0x00000100);
+ lastUsed_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000200);
+ name_ = "";
+ bitField0_ = (bitField0_ & ~0x00000400);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor;
+ }
+
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto getDefaultInstanceForType() {
+ return org.apache.slider.api.proto.Messages.NodeEntryInformationProto.getDefaultInstance();
+ }
+
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto build() {
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto buildPartial() {
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto result = new org.apache.slider.api.proto.Messages.NodeEntryInformationProto(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.priority_ = priority_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.requested_ = requested_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.starting_ = starting_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.startFailed_ = startFailed_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.failed_ = failed_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.failedRecently_ = failedRecently_;
+ if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+ to_bitField0_ |= 0x00000040;
+ }
+ result.preempted_ = preempted_;
+ if (((from_bitField0_ & 0x00000080) == 0x00000080)) {
+ to_bitField0_ |= 0x00000080;
+ }
+ result.live_ = live_;
+ if (((from_bitField0_ & 0x00000100) == 0x00000100)) {
+ to_bitField0_ |= 0x00000100;
+ }
+ result.releasing_ = releasing_;
+ if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+ to_bitField0_ |= 0x00000200;
+ }
+ result.lastUsed_ = lastUsed_;
+ if (((from_bitField0_ & 0x00000400) == 0x00000400)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.name_ = name_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.slider.api.proto.Messages.NodeEntryInformationProto) {
+ return mergeFrom((org.apache.slider.api.proto.Messages.NodeEntryInformationProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.slider.api.proto.Messages.NodeEntryInformationProto other) {
+ if (other == org.apache.slider.api.proto.Messages.NodeEntryInformationProto.getDefaultInstance()) return this;
+ if (other.hasPriority()) {
+ setPriority(other.getPriority());
+ }
+ if (other.hasRequested()) {
+ setRequested(other.getRequested());
+ }
+ if (other.hasStarting()) {
+ setStarting(other.getStarting());
+ }
+ if (other.hasStartFailed()) {
+ setStartFailed(other.getStartFailed());
+ }
+ if (other.hasFailed()) {
+ setFailed(other.getFailed());
+ }
+ if (other.hasFailedRecently()) {
+ setFailedRecently(other.getFailedRecently());
+ }
+ if (other.hasPreempted()) {
+ setPreempted(other.getPreempted());
+ }
+ if (other.hasLive()) {
+ setLive(other.getLive());
+ }
+ if (other.hasReleasing()) {
+ setReleasing(other.getReleasing());
+ }
+ if (other.hasLastUsed()) {
+ setLastUsed(other.getLastUsed());
+ }
+ if (other.hasName()) {
+ bitField0_ |= 0x00000400;
+ name_ = other.name_;
+ onChanged();
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasPriority()) {
+
+ return false;
+ }
+ if (!hasRequested()) {
+
+ return false;
+ }
+ if (!hasStarting()) {
+
+ return false;
+ }
+ if (!hasStartFailed()) {
+
+ return false;
+ }
+ if (!hasFailed()) {
+
+ return false;
+ }
+ if (!hasFailedRecently()) {
+
+ return false;
+ }
+ if (!hasPreempted()) {
+
+ return false;
+ }
+ if (!hasLive()) {
+
+ return false;
+ }
+ if (!hasReleasing()) {
+
+ return false;
+ }
+ if (!hasLastUsed()) {
+
+ return false;
+ }
+ if (!hasName()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.slider.api.proto.Messages.NodeEntryInformationProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required int32 priority = 1;
+ private int priority_ ;
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ public boolean hasPriority() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ public int getPriority() {
+ return priority_;
+ }
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ public Builder setPriority(int value) {
+ bitField0_ |= 0x00000001;
+ priority_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 priority = 1;</code>
+ */
+ public Builder clearPriority() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ priority_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 requested = 2;
+ private int requested_ ;
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ public boolean hasRequested() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ public int getRequested() {
+ return requested_;
+ }
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ public Builder setRequested(int value) {
+ bitField0_ |= 0x00000002;
+ requested_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 requested = 2;</code>
+ */
+ public Builder clearRequested() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ requested_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 starting = 3;
+ private int starting_ ;
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ public boolean hasStarting() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ public int getStarting() {
+ return starting_;
+ }
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ public Builder setStarting(int value) {
+ bitField0_ |= 0x00000004;
+ starting_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 starting = 3;</code>
+ */
+ public Builder clearStarting() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ starting_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 startFailed = 4;
+ private int startFailed_ ;
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ public boolean hasStartFailed() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ public int getStartFailed() {
+ return startFailed_;
+ }
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ public Builder setStartFailed(int value) {
+ bitField0_ |= 0x00000008;
+ startFailed_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 startFailed = 4;</code>
+ */
+ public Builder clearStartFailed() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ startFailed_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 failed = 5;
+ private int failed_ ;
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ public boolean hasFailed() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ public int getFailed() {
+ return failed_;
+ }
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ public Builder setFailed(int value) {
+ bitField0_ |= 0x00000010;
+ failed_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 failed = 5;</code>
+ */
+ public Builder clearFailed() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ failed_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 failedRecently = 6;
+ private int failedRecently_ ;
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ public boolean hasFailedRecently() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ public int getFailedRecently() {
+ return failedRecently_;
+ }
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ public Builder setFailedRecently(int value) {
+ bitField0_ |= 0x00000020;
+ failedRecently_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 failedRecently = 6;</code>
+ */
+ public Builder clearFailedRecently() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ failedRecently_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 preempted = 7;
+ private int preempted_ ;
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ public boolean hasPreempted() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ public int getPreempted() {
+ return preempted_;
+ }
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ public Builder setPreempted(int value) {
+ bitField0_ |= 0x00000040;
+ preempted_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 preempted = 7;</code>
+ */
+ public Builder clearPreempted() {
+ bitField0_ = (bitField0_ & ~0x00000040);
+ preempted_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 live = 8;
+ private int live_ ;
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ public boolean hasLive() {
+ return ((bitField0_ & 0x00000080) == 0x00000080);
+ }
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ public int getLive() {
+ return live_;
+ }
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ public Builder setLive(int value) {
+ bitField0_ |= 0x00000080;
+ live_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 live = 8;</code>
+ */
+ public Builder clearLive() {
+ bitField0_ = (bitField0_ & ~0x00000080);
+ live_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int32 releasing = 9;
+ private int releasing_ ;
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ public boolean hasReleasing() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ public int getReleasing() {
+ return releasing_;
+ }
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ public Builder setReleasing(int value) {
+ bitField0_ |= 0x00000100;
+ releasing_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int32 releasing = 9;</code>
+ */
+ public Builder clearReleasing() {
+ bitField0_ = (bitField0_ & ~0x00000100);
+ releasing_ = 0;
+ onChanged();
+ return this;
+ }
+
+ // required int64 lastUsed = 10;
+ private long lastUsed_ ;
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ public boolean hasLastUsed() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ public long getLastUsed() {
+ return lastUsed_;
+ }
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ public Builder setLastUsed(long value) {
+ bitField0_ |= 0x00000200;
+ lastUsed_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int64 lastUsed = 10;</code>
+ */
+ public Builder clearLastUsed() {
+ bitField0_ = (bitField0_ & ~0x00000200);
+ lastUsed_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // required string name = 11;
+ private java.lang.Object name_ = "";
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ name_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public Builder setName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000400;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public Builder clearName() {
+ bitField0_ = (bitField0_ & ~0x00000400);
+ name_ = getDefaultInstance().getName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 11;</code>
+ */
+ public Builder setNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000400;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:org.apache.slider.api.NodeEntryInformationProto)
+ }
+
+ static {
+ defaultInstance = new NodeEntryInformationProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:org.apache.slider.api.NodeEntryInformationProto)
+ }
+
+ public interface NodeInformationProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required string hostname = 1;
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ boolean hasHostname();
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ java.lang.String getHostname();
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getHostnameBytes();
+
+ // required string state = 2;
+ /**
+ * <code>required string state = 2;</code>
+ */
+ boolean hasState();
+ /**
+ * <code>required string state = 2;</code>
+ */
+ java.lang.String getState();
+ /**
+ * <code>required string state = 2;</code>
+ */
+ com.google.protobuf.ByteString
+ getStateBytes();
+
+ // required string httpAddress = 3;
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ boolean hasHttpAddress();
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ java.lang.String getHttpAddress();
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ com.google.protobuf.ByteString
+ getHttpAddressBytes();
+
+ // required string rackName = 4;
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ boolean hasRackName();
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ java.lang.String getRackName();
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ com.google.protobuf.ByteString
+ getRackNameBytes();
+
+ // required string labels = 5;
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ boolean hasLabels();
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ java.lang.String getLabels();
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ com.google.protobuf.ByteString
+ getLabelsBytes();
+
+ // required string healthReport = 6;
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ boolean hasHealthReport();
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ java.lang.String getHealthReport();
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ com.google.protobuf.ByteString
+ getHealthReportBytes();
+
+ // required int64 lastUpdated = 7;
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ boolean hasLastUpdated();
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ long getLastUpdated();
+
+ // repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ java.util.List<org.apache.slider.api.proto.Messages.NodeEntryInformationProto>
+ getEntriesList();
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto getEntries(int index);
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ int getEntriesCount();
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ java.util.List<? extends org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder>
+ getEntriesOrBuilderList();
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder getEntriesOrBuilder(
+ int index);
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.NodeInformationProto}
+ */
+ public static final class NodeInformationProto extends
+ com.google.protobuf.GeneratedMessage
+ implements NodeInformationProtoOrBuilder {
+ // Use NodeInformationProto.newBuilder() to construct.
+ private NodeInformationProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private NodeInformationProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final NodeInformationProto defaultInstance;
+ public static NodeInformationProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public NodeInformationProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private NodeInformationProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ hostname_ = input.readBytes();
+ break;
+ }
+ case 18: {
+ bitField0_ |= 0x00000002;
+ state_ = input.readBytes();
+ break;
+ }
+ case 26: {
+ bitField0_ |= 0x00000004;
+ httpAddress_ = input.readBytes();
+ break;
+ }
+ case 34: {
+ bitField0_ |= 0x00000008;
+ rackName_ = input.readBytes();
+ break;
+ }
+ case 42: {
+ bitField0_ |= 0x00000010;
+ labels_ = input.readBytes();
+ break;
+ }
+ case 50: {
+ bitField0_ |= 0x00000020;
+ healthReport_ = input.readBytes();
+ break;
+ }
+ case 56: {
+ bitField0_ |= 0x00000040;
+ lastUpdated_ = input.readInt64();
+ break;
+ }
+ case 66: {
+ if (!((mutable_bitField0_ & 0x00000080) == 0x00000080)) {
+ entries_ = new java.util.ArrayList<org.apache.slider.api.proto.Messages.NodeEntryInformationProto>();
+ mutable_bitField0_ |= 0x00000080;
+ }
+ entries_.add(input.readMessage(org.apache.slider.api.proto.Messages.NodeEntryInformationProto.PARSER, extensionRegistry));
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000080) == 0x00000080)) {
+ entries_ = java.util.Collections.unmodifiableList(entries_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeInformationProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeInformationProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.NodeInformationProto.class, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<NodeInformationProto> PARSER =
+ new com.google.protobuf.AbstractParser<NodeInformationProto>() {
+ public NodeInformationProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new NodeInformationProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<NodeInformationProto> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required string hostname = 1;
+ public static final int HOSTNAME_FIELD_NUMBER = 1;
+ private java.lang.Object hostname_;
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public boolean hasHostname() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public java.lang.String getHostname() {
+ java.lang.Object ref = hostname_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ hostname_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHostnameBytes() {
+ java.lang.Object ref = hostname_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ hostname_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string state = 2;
+ public static final int STATE_FIELD_NUMBER = 2;
+ private java.lang.Object state_;
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public boolean hasState() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public java.lang.String getState() {
+ java.lang.Object ref = state_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ state_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public com.google.protobuf.ByteString
+ getStateBytes() {
+ java.lang.Object ref = state_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ state_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string httpAddress = 3;
+ public static final int HTTPADDRESS_FIELD_NUMBER = 3;
+ private java.lang.Object httpAddress_;
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public boolean hasHttpAddress() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public java.lang.String getHttpAddress() {
+ java.lang.Object ref = httpAddress_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ httpAddress_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHttpAddressBytes() {
+ java.lang.Object ref = httpAddress_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ httpAddress_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string rackName = 4;
+ public static final int RACKNAME_FIELD_NUMBER = 4;
+ private java.lang.Object rackName_;
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public boolean hasRackName() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public java.lang.String getRackName() {
+ java.lang.Object ref = rackName_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ rackName_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public com.google.protobuf.ByteString
+ getRackNameBytes() {
+ java.lang.Object ref = rackName_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ rackName_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string labels = 5;
+ public static final int LABELS_FIELD_NUMBER = 5;
+ private java.lang.Object labels_;
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public boolean hasLabels() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public java.lang.String getLabels() {
+ java.lang.Object ref = labels_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ labels_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getLabelsBytes() {
+ java.lang.Object ref = labels_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ labels_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required string healthReport = 6;
+ public static final int HEALTHREPORT_FIELD_NUMBER = 6;
+ private java.lang.Object healthReport_;
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public boolean hasHealthReport() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public java.lang.String getHealthReport() {
+ java.lang.Object ref = healthReport_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ healthReport_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHealthReportBytes() {
+ java.lang.Object ref = healthReport_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ healthReport_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ // required int64 lastUpdated = 7;
+ public static final int LASTUPDATED_FIELD_NUMBER = 7;
+ private long lastUpdated_;
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ public boolean hasLastUpdated() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ public long getLastUpdated() {
+ return lastUpdated_;
+ }
+
+ // repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;
+ public static final int ENTRIES_FIELD_NUMBER = 8;
+ private java.util.List<org.apache.slider.api.proto.Messages.NodeEntryInformationProto> entries_;
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public java.util.List<org.apache.slider.api.proto.Messages.NodeEntryInformationProto> getEntriesList() {
+ return entries_;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public java.util.List<? extends org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder>
+ getEntriesOrBuilderList() {
+ return entries_;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public int getEntriesCount() {
+ return entries_.size();
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto getEntries(int index) {
+ return entries_.get(index);
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder getEntriesOrBuilder(
+ int index) {
+ return entries_.get(index);
+ }
+
+ private void initFields() {
+ hostname_ = "";
+ state_ = "";
+ httpAddress_ = "";
+ rackName_ = "";
+ labels_ = "";
+ healthReport_ = "";
+ lastUpdated_ = 0L;
+ entries_ = java.util.Collections.emptyList();
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasHostname()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasState()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasHttpAddress()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasRackName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasLabels()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasHealthReport()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ if (!hasLastUpdated()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ for (int i = 0; i < getEntriesCount(); i++) {
+ if (!getEntries(i).isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getHostnameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeBytes(2, getStateBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ output.writeBytes(3, getHttpAddressBytes());
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ output.writeBytes(4, getRackNameBytes());
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ output.writeBytes(5, getLabelsBytes());
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ output.writeBytes(6, getHealthReportBytes());
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ output.writeInt64(7, lastUpdated_);
+ }
+ for (int i = 0; i < entries_.size(); i++) {
+ output.writeMessage(8, entries_.get(i));
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getHostnameBytes());
+ }
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(2, getStateBytes());
+ }
+ if (((bitField0_ & 0x00000004) == 0x00000004)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(3, getHttpAddressBytes());
+ }
+ if (((bitField0_ & 0x00000008) == 0x00000008)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(4, getRackNameBytes());
+ }
+ if (((bitField0_ & 0x00000010) == 0x00000010)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(5, getLabelsBytes());
+ }
+ if (((bitField0_ & 0x00000020) == 0x00000020)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(6, getHealthReportBytes());
+ }
+ if (((bitField0_ & 0x00000040) == 0x00000040)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(7, lastUpdated_);
+ }
+ for (int i = 0; i < entries_.size(); i++) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(8, entries_.get(i));
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.slider.api.proto.Messages.NodeInformationProto)) {
+ return super.equals(obj);
+ }
+ org.apache.slider.api.proto.Messages.NodeInformationProto other = (org.apache.slider.api.proto.Messages.NodeInformationProto) obj;
+
+ boolean result = true;
+ result = result && (hasHostname() == other.hasHostname());
+ if (hasHostname()) {
+ result = result && getHostname()
+ .equals(other.getHostname());
+ }
+ result = result && (hasState() == other.hasState());
+ if (hasState()) {
+ result = result && getState()
+ .equals(other.getState());
+ }
+ result = result && (hasHttpAddress() == other.hasHttpAddress());
+ if (hasHttpAddress()) {
+ result = result && getHttpAddress()
+ .equals(other.getHttpAddress());
+ }
+ result = result && (hasRackName() == other.hasRackName());
+ if (hasRackName()) {
+ result = result && getRackName()
+ .equals(other.getRackName());
+ }
+ result = result && (hasLabels() == other.hasLabels());
+ if (hasLabels()) {
+ result = result && getLabels()
+ .equals(other.getLabels());
+ }
+ result = result && (hasHealthReport() == other.hasHealthReport());
+ if (hasHealthReport()) {
+ result = result && getHealthReport()
+ .equals(other.getHealthReport());
+ }
+ result = result && (hasLastUpdated() == other.hasLastUpdated());
+ if (hasLastUpdated()) {
+ result = result && (getLastUpdated()
+ == other.getLastUpdated());
+ }
+ result = result && getEntriesList()
+ .equals(other.getEntriesList());
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasHostname()) {
+ hash = (37 * hash) + HOSTNAME_FIELD_NUMBER;
+ hash = (53 * hash) + getHostname().hashCode();
+ }
+ if (hasState()) {
+ hash = (37 * hash) + STATE_FIELD_NUMBER;
+ hash = (53 * hash) + getState().hashCode();
+ }
+ if (hasHttpAddress()) {
+ hash = (37 * hash) + HTTPADDRESS_FIELD_NUMBER;
+ hash = (53 * hash) + getHttpAddress().hashCode();
+ }
+ if (hasRackName()) {
+ hash = (37 * hash) + RACKNAME_FIELD_NUMBER;
+ hash = (53 * hash) + getRackName().hashCode();
+ }
+ if (hasLabels()) {
+ hash = (37 * hash) + LABELS_FIELD_NUMBER;
+ hash = (53 * hash) + getLabels().hashCode();
+ }
+ if (hasHealthReport()) {
+ hash = (37 * hash) + HEALTHREPORT_FIELD_NUMBER;
+ hash = (53 * hash) + getHealthReport().hashCode();
+ }
+ if (hasLastUpdated()) {
+ hash = (37 * hash) + LASTUPDATED_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getLastUpdated());
+ }
+ if (getEntriesCount() > 0) {
+ hash = (37 * hash) + ENTRIES_FIELD_NUMBER;
+ hash = (53 * hash) + getEntriesList().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.NodeInformationProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.slider.api.proto.Messages.NodeInformationProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.NodeInformationProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeInformationProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeInformationProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.NodeInformationProto.class, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder.class);
+ }
+
+ // Construct using org.apache.slider.api.proto.Messages.NodeInformationProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getEntriesFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ hostname_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ state_ = "";
+ bitField0_ = (bitField0_ & ~0x00000002);
+ httpAddress_ = "";
+ bitField0_ = (bitField0_ & ~0x00000004);
+ rackName_ = "";
+ bitField0_ = (bitField0_ & ~0x00000008);
+ labels_ = "";
+ bitField0_ = (bitField0_ & ~0x00000010);
+ healthReport_ = "";
+ bitField0_ = (bitField0_ & ~0x00000020);
+ lastUpdated_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000040);
+ if (entriesBuilder_ == null) {
+ entries_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000080);
+ } else {
+ entriesBuilder_.clear();
+ }
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_NodeInformationProto_descriptor;
+ }
+
+ public org.apache.slider.api.proto.Messages.NodeInformationProto getDefaultInstanceForType() {
+ return org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance();
+ }
+
+ public org.apache.slider.api.proto.Messages.NodeInformationProto build() {
+ org.apache.slider.api.proto.Messages.NodeInformationProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.slider.api.proto.Messages.NodeInformationProto buildPartial() {
+ org.apache.slider.api.proto.Messages.NodeInformationProto result = new org.apache.slider.api.proto.Messages.NodeInformationProto(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.hostname_ = hostname_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.state_ = state_;
+ if (((from_bitField0_ & 0x00000004) == 0x00000004)) {
+ to_bitField0_ |= 0x00000004;
+ }
+ result.httpAddress_ = httpAddress_;
+ if (((from_bitField0_ & 0x00000008) == 0x00000008)) {
+ to_bitField0_ |= 0x00000008;
+ }
+ result.rackName_ = rackName_;
+ if (((from_bitField0_ & 0x00000010) == 0x00000010)) {
+ to_bitField0_ |= 0x00000010;
+ }
+ result.labels_ = labels_;
+ if (((from_bitField0_ & 0x00000020) == 0x00000020)) {
+ to_bitField0_ |= 0x00000020;
+ }
+ result.healthReport_ = healthReport_;
+ if (((from_bitField0_ & 0x00000040) == 0x00000040)) {
+ to_bitField0_ |= 0x00000040;
+ }
+ result.lastUpdated_ = lastUpdated_;
+ if (entriesBuilder_ == null) {
+ if (((bitField0_ & 0x00000080) == 0x00000080)) {
+ entries_ = java.util.Collections.unmodifiableList(entries_);
+ bitField0_ = (bitField0_ & ~0x00000080);
+ }
+ result.entries_ = entries_;
+ } else {
+ result.entries_ = entriesBuilder_.build();
+ }
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.slider.api.proto.Messages.NodeInformationProto) {
+ return mergeFrom((org.apache.slider.api.proto.Messages.NodeInformationProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.slider.api.proto.Messages.NodeInformationProto other) {
+ if (other == org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance()) return this;
+ if (other.hasHostname()) {
+ bitField0_ |= 0x00000001;
+ hostname_ = other.hostname_;
+ onChanged();
+ }
+ if (other.hasState()) {
+ bitField0_ |= 0x00000002;
+ state_ = other.state_;
+ onChanged();
+ }
+ if (other.hasHttpAddress()) {
+ bitField0_ |= 0x00000004;
+ httpAddress_ = other.httpAddress_;
+ onChanged();
+ }
+ if (other.hasRackName()) {
+ bitField0_ |= 0x00000008;
+ rackName_ = other.rackName_;
+ onChanged();
+ }
+ if (other.hasLabels()) {
+ bitField0_ |= 0x00000010;
+ labels_ = other.labels_;
+ onChanged();
+ }
+ if (other.hasHealthReport()) {
+ bitField0_ |= 0x00000020;
+ healthReport_ = other.healthReport_;
+ onChanged();
+ }
+ if (other.hasLastUpdated()) {
+ setLastUpdated(other.getLastUpdated());
+ }
+ if (entriesBuilder_ == null) {
+ if (!other.entries_.isEmpty()) {
+ if (entries_.isEmpty()) {
+ entries_ = other.entries_;
+ bitField0_ = (bitField0_ & ~0x00000080);
+ } else {
+ ensureEntriesIsMutable();
+ entries_.addAll(other.entries_);
+ }
+ onChanged();
+ }
+ } else {
+ if (!other.entries_.isEmpty()) {
+ if (entriesBuilder_.isEmpty()) {
+ entriesBuilder_.dispose();
+ entriesBuilder_ = null;
+ entries_ = other.entries_;
+ bitField0_ = (bitField0_ & ~0x00000080);
+ entriesBuilder_ =
+ com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+ getEntriesFieldBuilder() : null;
+ } else {
+ entriesBuilder_.addAllMessages(other.entries_);
+ }
+ }
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasHostname()) {
+
+ return false;
+ }
+ if (!hasState()) {
+
+ return false;
+ }
+ if (!hasHttpAddress()) {
+
+ return false;
+ }
+ if (!hasRackName()) {
+
+ return false;
+ }
+ if (!hasLabels()) {
+
+ return false;
+ }
+ if (!hasHealthReport()) {
+
+ return false;
+ }
+ if (!hasLastUpdated()) {
+
+ return false;
+ }
+ for (int i = 0; i < getEntriesCount(); i++) {
+ if (!getEntries(i).isInitialized()) {
+
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.slider.api.proto.Messages.NodeInformationProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.slider.api.proto.Messages.NodeInformationProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required string hostname = 1;
+ private java.lang.Object hostname_ = "";
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public boolean hasHostname() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public java.lang.String getHostname() {
+ java.lang.Object ref = hostname_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ hostname_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHostnameBytes() {
+ java.lang.Object ref = hostname_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ hostname_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public Builder setHostname(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ hostname_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public Builder clearHostname() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ hostname_ = getDefaultInstance().getHostname();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string hostname = 1;</code>
+ */
+ public Builder setHostnameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ hostname_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required string state = 2;
+ private java.lang.Object state_ = "";
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public boolean hasState() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public java.lang.String getState() {
+ java.lang.Object ref = state_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ state_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public com.google.protobuf.ByteString
+ getStateBytes() {
+ java.lang.Object ref = state_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ state_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public Builder setState(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ state_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public Builder clearState() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ state_ = getDefaultInstance().getState();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string state = 2;</code>
+ */
+ public Builder setStateBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000002;
+ state_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required string httpAddress = 3;
+ private java.lang.Object httpAddress_ = "";
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public boolean hasHttpAddress() {
+ return ((bitField0_ & 0x00000004) == 0x00000004);
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public java.lang.String getHttpAddress() {
+ java.lang.Object ref = httpAddress_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ httpAddress_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHttpAddressBytes() {
+ java.lang.Object ref = httpAddress_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ httpAddress_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public Builder setHttpAddress(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ httpAddress_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public Builder clearHttpAddress() {
+ bitField0_ = (bitField0_ & ~0x00000004);
+ httpAddress_ = getDefaultInstance().getHttpAddress();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string httpAddress = 3;</code>
+ */
+ public Builder setHttpAddressBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000004;
+ httpAddress_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required string rackName = 4;
+ private java.lang.Object rackName_ = "";
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public boolean hasRackName() {
+ return ((bitField0_ & 0x00000008) == 0x00000008);
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public java.lang.String getRackName() {
+ java.lang.Object ref = rackName_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ rackName_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public com.google.protobuf.ByteString
+ getRackNameBytes() {
+ java.lang.Object ref = rackName_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ rackName_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public Builder setRackName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000008;
+ rackName_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public Builder clearRackName() {
+ bitField0_ = (bitField0_ & ~0x00000008);
+ rackName_ = getDefaultInstance().getRackName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string rackName = 4;</code>
+ */
+ public Builder setRackNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000008;
+ rackName_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required string labels = 5;
+ private java.lang.Object labels_ = "";
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public boolean hasLabels() {
+ return ((bitField0_ & 0x00000010) == 0x00000010);
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public java.lang.String getLabels() {
+ java.lang.Object ref = labels_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ labels_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public com.google.protobuf.ByteString
+ getLabelsBytes() {
+ java.lang.Object ref = labels_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ labels_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public Builder setLabels(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ labels_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public Builder clearLabels() {
+ bitField0_ = (bitField0_ & ~0x00000010);
+ labels_ = getDefaultInstance().getLabels();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string labels = 5;</code>
+ */
+ public Builder setLabelsBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000010;
+ labels_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required string healthReport = 6;
+ private java.lang.Object healthReport_ = "";
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public boolean hasHealthReport() {
+ return ((bitField0_ & 0x00000020) == 0x00000020);
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public java.lang.String getHealthReport() {
+ java.lang.Object ref = healthReport_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ healthReport_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public com.google.protobuf.ByteString
+ getHealthReportBytes() {
+ java.lang.Object ref = healthReport_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ healthReport_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public Builder setHealthReport(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000020;
+ healthReport_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public Builder clearHealthReport() {
+ bitField0_ = (bitField0_ & ~0x00000020);
+ healthReport_ = getDefaultInstance().getHealthReport();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string healthReport = 6;</code>
+ */
+ public Builder setHealthReportBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000020;
+ healthReport_ = value;
+ onChanged();
+ return this;
+ }
+
+ // required int64 lastUpdated = 7;
+ private long lastUpdated_ ;
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ public boolean hasLastUpdated() {
+ return ((bitField0_ & 0x00000040) == 0x00000040);
+ }
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ public long getLastUpdated() {
+ return lastUpdated_;
+ }
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ public Builder setLastUpdated(long value) {
+ bitField0_ |= 0x00000040;
+ lastUpdated_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required int64 lastUpdated = 7;</code>
+ */
+ public Builder clearLastUpdated() {
+ bitField0_ = (bitField0_ & ~0x00000040);
+ lastUpdated_ = 0L;
+ onChanged();
+ return this;
+ }
+
+ // repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;
+ private java.util.List<org.apache.slider.api.proto.Messages.NodeEntryInformationProto> entries_ =
+ java.util.Collections.emptyList();
+ private void ensureEntriesIsMutable() {
+ if (!((bitField0_ & 0x00000080) == 0x00000080)) {
+ entries_ = new java.util.ArrayList<org.apache.slider.api.proto.Messages.NodeEntryInformationProto>(entries_);
+ bitField0_ |= 0x00000080;
+ }
+ }
+
+ private com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder> entriesBuilder_;
+
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public java.util.List<org.apache.slider.api.proto.Messages.NodeEntryInformationProto> getEntriesList() {
+ if (entriesBuilder_ == null) {
+ return java.util.Collections.unmodifiableList(entries_);
+ } else {
+ return entriesBuilder_.getMessageList();
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public int getEntriesCount() {
+ if (entriesBuilder_ == null) {
+ return entries_.size();
+ } else {
+ return entriesBuilder_.getCount();
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto getEntries(int index) {
+ if (entriesBuilder_ == null) {
+ return entries_.get(index);
+ } else {
+ return entriesBuilder_.getMessage(index);
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder setEntries(
+ int index, org.apache.slider.api.proto.Messages.NodeEntryInformationProto value) {
+ if (entriesBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureEntriesIsMutable();
+ entries_.set(index, value);
+ onChanged();
+ } else {
+ entriesBuilder_.setMessage(index, value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder setEntries(
+ int index, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder builderForValue) {
+ if (entriesBuilder_ == null) {
+ ensureEntriesIsMutable();
+ entries_.set(index, builderForValue.build());
+ onChanged();
+ } else {
+ entriesBuilder_.setMessage(index, builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder addEntries(org.apache.slider.api.proto.Messages.NodeEntryInformationProto value) {
+ if (entriesBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureEntriesIsMutable();
+ entries_.add(value);
+ onChanged();
+ } else {
+ entriesBuilder_.addMessage(value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder addEntries(
+ int index, org.apache.slider.api.proto.Messages.NodeEntryInformationProto value) {
+ if (entriesBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureEntriesIsMutable();
+ entries_.add(index, value);
+ onChanged();
+ } else {
+ entriesBuilder_.addMessage(index, value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder addEntries(
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder builderForValue) {
+ if (entriesBuilder_ == null) {
+ ensureEntriesIsMutable();
+ entries_.add(builderForValue.build());
+ onChanged();
+ } else {
+ entriesBuilder_.addMessage(builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder addEntries(
+ int index, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder builderForValue) {
+ if (entriesBuilder_ == null) {
+ ensureEntriesIsMutable();
+ entries_.add(index, builderForValue.build());
+ onChanged();
+ } else {
+ entriesBuilder_.addMessage(index, builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder addAllEntries(
+ java.lang.Iterable<? extends org.apache.slider.api.proto.Messages.NodeEntryInformationProto> values) {
+ if (entriesBuilder_ == null) {
+ ensureEntriesIsMutable();
+ super.addAll(values, entries_);
+ onChanged();
+ } else {
+ entriesBuilder_.addAllMessages(values);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder clearEntries() {
+ if (entriesBuilder_ == null) {
+ entries_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000080);
+ onChanged();
+ } else {
+ entriesBuilder_.clear();
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public Builder removeEntries(int index) {
+ if (entriesBuilder_ == null) {
+ ensureEntriesIsMutable();
+ entries_.remove(index);
+ onChanged();
+ } else {
+ entriesBuilder_.remove(index);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder getEntriesBuilder(
+ int index) {
+ return getEntriesFieldBuilder().getBuilder(index);
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder getEntriesOrBuilder(
+ int index) {
+ if (entriesBuilder_ == null) {
+ return entries_.get(index); } else {
+ return entriesBuilder_.getMessageOrBuilder(index);
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public java.util.List<? extends org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder>
+ getEntriesOrBuilderList() {
+ if (entriesBuilder_ != null) {
+ return entriesBuilder_.getMessageOrBuilderList();
+ } else {
+ return java.util.Collections.unmodifiableList(entries_);
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder addEntriesBuilder() {
+ return getEntriesFieldBuilder().addBuilder(
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto.getDefaultInstance());
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder addEntriesBuilder(
+ int index) {
+ return getEntriesFieldBuilder().addBuilder(
+ index, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.getDefaultInstance());
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeEntryInformationProto entries = 8;</code>
+ */
+ public java.util.List<org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder>
+ getEntriesBuilderList() {
+ return getEntriesFieldBuilder().getBuilderList();
+ }
+ private com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder>
+ getEntriesFieldBuilder() {
+ if (entriesBuilder_ == null) {
+ entriesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.slider.api.proto.Messages.NodeEntryInformationProto, org.apache.slider.api.proto.Messages.NodeEntryInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeEntryInformationProtoOrBuilder>(
+ entries_,
+ ((bitField0_ & 0x00000080) == 0x00000080),
+ getParentForChildren(),
+ isClean());
+ entries_ = null;
+ }
+ return entriesBuilder_;
+ }
+
+ // @@protoc_insertion_point(builder_scope:org.apache.slider.api.NodeInformationProto)
+ }
+
+ static {
+ defaultInstance = new NodeInformationProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:org.apache.slider.api.NodeInformationProto)
+ }
+
public interface GetModelRequestProtoOrBuilder
extends com.google.protobuf.MessageOrBuilder {
}
@@ -28684,6 +32221,1582 @@
// @@protoc_insertion_point(class_scope:org.apache.slider.api.GetCertificateStoreResponseProto)
}
+ public interface GetLiveNodesRequestProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.GetLiveNodesRequestProto}
+ */
+ public static final class GetLiveNodesRequestProto extends
+ com.google.protobuf.GeneratedMessage
+ implements GetLiveNodesRequestProtoOrBuilder {
+ // Use GetLiveNodesRequestProto.newBuilder() to construct.
+ private GetLiveNodesRequestProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private GetLiveNodesRequestProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final GetLiveNodesRequestProto defaultInstance;
+ public static GetLiveNodesRequestProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public GetLiveNodesRequestProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private GetLiveNodesRequestProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesRequestProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesRequestProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.class, org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<GetLiveNodesRequestProto> PARSER =
+ new com.google.protobuf.AbstractParser<GetLiveNodesRequestProto>() {
+ public GetLiveNodesRequestProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new GetLiveNodesRequestProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<GetLiveNodesRequestProto> getParserForType() {
+ return PARSER;
+ }
+
+ private void initFields() {
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto)) {
+ return super.equals(obj);
+ }
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto other = (org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto) obj;
+
+ boolean result = true;
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.GetLiveNodesRequestProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.slider.api.proto.Messages.GetLiveNodesRequestProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesRequestProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesRequestProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.class, org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.Builder.class);
+ }
+
+ // Construct using org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesRequestProto_descriptor;
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto getDefaultInstanceForType() {
+ return org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.getDefaultInstance();
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto build() {
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto buildPartial() {
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto result = new org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto(this);
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto) {
+ return mergeFrom((org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto other) {
+ if (other == org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.getDefaultInstance()) return this;
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:org.apache.slider.api.GetLiveNodesRequestProto)
+ }
+
+ static {
+ defaultInstance = new GetLiveNodesRequestProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:org.apache.slider.api.GetLiveNodesRequestProto)
+ }
+
+ public interface GetLiveNodesResponseProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // repeated .org.apache.slider.api.NodeInformationProto nodes = 1;
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto>
+ getNodesList();
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ org.apache.slider.api.proto.Messages.NodeInformationProto getNodes(int index);
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ int getNodesCount();
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ java.util.List<? extends org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
+ getNodesOrBuilderList();
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder getNodesOrBuilder(
+ int index);
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.GetLiveNodesResponseProto}
+ */
+ public static final class GetLiveNodesResponseProto extends
+ com.google.protobuf.GeneratedMessage
+ implements GetLiveNodesResponseProtoOrBuilder {
+ // Use GetLiveNodesResponseProto.newBuilder() to construct.
+ private GetLiveNodesResponseProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private GetLiveNodesResponseProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final GetLiveNodesResponseProto defaultInstance;
+ public static GetLiveNodesResponseProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public GetLiveNodesResponseProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private GetLiveNodesResponseProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ if (!((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ nodes_ = new java.util.ArrayList<org.apache.slider.api.proto.Messages.NodeInformationProto>();
+ mutable_bitField0_ |= 0x00000001;
+ }
+ nodes_.add(input.readMessage(org.apache.slider.api.proto.Messages.NodeInformationProto.PARSER, extensionRegistry));
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ if (((mutable_bitField0_ & 0x00000001) == 0x00000001)) {
+ nodes_ = java.util.Collections.unmodifiableList(nodes_);
+ }
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesResponseProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.class, org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<GetLiveNodesResponseProto> PARSER =
+ new com.google.protobuf.AbstractParser<GetLiveNodesResponseProto>() {
+ public GetLiveNodesResponseProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new GetLiveNodesResponseProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<GetLiveNodesResponseProto> getParserForType() {
+ return PARSER;
+ }
+
+ // repeated .org.apache.slider.api.NodeInformationProto nodes = 1;
+ public static final int NODES_FIELD_NUMBER = 1;
+ private java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> nodes_;
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> getNodesList() {
+ return nodes_;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public java.util.List<? extends org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
+ getNodesOrBuilderList() {
+ return nodes_;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public int getNodesCount() {
+ return nodes_.size();
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProto getNodes(int index) {
+ return nodes_.get(index);
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder getNodesOrBuilder(
+ int index) {
+ return nodes_.get(index);
+ }
+
+ private void initFields() {
+ nodes_ = java.util.Collections.emptyList();
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ for (int i = 0; i < getNodesCount(); i++) {
+ if (!getNodes(i).isInitialized()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ for (int i = 0; i < nodes_.size(); i++) {
+ output.writeMessage(1, nodes_.get(i));
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ for (int i = 0; i < nodes_.size(); i++) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeMessageSize(1, nodes_.get(i));
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto)) {
+ return super.equals(obj);
+ }
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto other = (org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto) obj;
+
+ boolean result = true;
+ result = result && getNodesList()
+ .equals(other.getNodesList());
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (getNodesCount() > 0) {
+ hash = (37 * hash) + NODES_FIELD_NUMBER;
+ hash = (53 * hash) + getNodesList().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.GetLiveNodesResponseProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.slider.api.proto.Messages.GetLiveNodesResponseProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesResponseProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.class, org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.Builder.class);
+ }
+
+ // Construct using org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ getNodesFieldBuilder();
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ if (nodesBuilder_ == null) {
+ nodes_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000001);
+ } else {
+ nodesBuilder_.clear();
+ }
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor;
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto getDefaultInstanceForType() {
+ return org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance();
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto build() {
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto buildPartial() {
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto result = new org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto(this);
+ int from_bitField0_ = bitField0_;
+ if (nodesBuilder_ == null) {
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ nodes_ = java.util.Collections.unmodifiableList(nodes_);
+ bitField0_ = (bitField0_ & ~0x00000001);
+ }
+ result.nodes_ = nodes_;
+ } else {
+ result.nodes_ = nodesBuilder_.build();
+ }
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto) {
+ return mergeFrom((org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto other) {
+ if (other == org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance()) return this;
+ if (nodesBuilder_ == null) {
+ if (!other.nodes_.isEmpty()) {
+ if (nodes_.isEmpty()) {
+ nodes_ = other.nodes_;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ } else {
+ ensureNodesIsMutable();
+ nodes_.addAll(other.nodes_);
+ }
+ onChanged();
+ }
+ } else {
+ if (!other.nodes_.isEmpty()) {
+ if (nodesBuilder_.isEmpty()) {
+ nodesBuilder_.dispose();
+ nodesBuilder_ = null;
+ nodes_ = other.nodes_;
+ bitField0_ = (bitField0_ & ~0x00000001);
+ nodesBuilder_ =
+ com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders ?
+ getNodesFieldBuilder() : null;
+ } else {
+ nodesBuilder_.addAllMessages(other.nodes_);
+ }
+ }
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ for (int i = 0; i < getNodesCount(); i++) {
+ if (!getNodes(i).isInitialized()) {
+
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // repeated .org.apache.slider.api.NodeInformationProto nodes = 1;
+ private java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> nodes_ =
+ java.util.Collections.emptyList();
+ private void ensureNodesIsMutable() {
+ if (!((bitField0_ & 0x00000001) == 0x00000001)) {
+ nodes_ = new java.util.ArrayList<org.apache.slider.api.proto.Messages.NodeInformationProto>(nodes_);
+ bitField0_ |= 0x00000001;
+ }
+ }
+
+ private com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.slider.api.proto.Messages.NodeInformationProto, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder> nodesBuilder_;
+
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto> getNodesList() {
+ if (nodesBuilder_ == null) {
+ return java.util.Collections.unmodifiableList(nodes_);
+ } else {
+ return nodesBuilder_.getMessageList();
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public int getNodesCount() {
+ if (nodesBuilder_ == null) {
+ return nodes_.size();
+ } else {
+ return nodesBuilder_.getCount();
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProto getNodes(int index) {
+ if (nodesBuilder_ == null) {
+ return nodes_.get(index);
+ } else {
+ return nodesBuilder_.getMessage(index);
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder setNodes(
+ int index, org.apache.slider.api.proto.Messages.NodeInformationProto value) {
+ if (nodesBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureNodesIsMutable();
+ nodes_.set(index, value);
+ onChanged();
+ } else {
+ nodesBuilder_.setMessage(index, value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder setNodes(
+ int index, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder builderForValue) {
+ if (nodesBuilder_ == null) {
+ ensureNodesIsMutable();
+ nodes_.set(index, builderForValue.build());
+ onChanged();
+ } else {
+ nodesBuilder_.setMessage(index, builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder addNodes(org.apache.slider.api.proto.Messages.NodeInformationProto value) {
+ if (nodesBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureNodesIsMutable();
+ nodes_.add(value);
+ onChanged();
+ } else {
+ nodesBuilder_.addMessage(value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder addNodes(
+ int index, org.apache.slider.api.proto.Messages.NodeInformationProto value) {
+ if (nodesBuilder_ == null) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ ensureNodesIsMutable();
+ nodes_.add(index, value);
+ onChanged();
+ } else {
+ nodesBuilder_.addMessage(index, value);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder addNodes(
+ org.apache.slider.api.proto.Messages.NodeInformationProto.Builder builderForValue) {
+ if (nodesBuilder_ == null) {
+ ensureNodesIsMutable();
+ nodes_.add(builderForValue.build());
+ onChanged();
+ } else {
+ nodesBuilder_.addMessage(builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder addNodes(
+ int index, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder builderForValue) {
+ if (nodesBuilder_ == null) {
+ ensureNodesIsMutable();
+ nodes_.add(index, builderForValue.build());
+ onChanged();
+ } else {
+ nodesBuilder_.addMessage(index, builderForValue.build());
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder addAllNodes(
+ java.lang.Iterable<? extends org.apache.slider.api.proto.Messages.NodeInformationProto> values) {
+ if (nodesBuilder_ == null) {
+ ensureNodesIsMutable();
+ super.addAll(values, nodes_);
+ onChanged();
+ } else {
+ nodesBuilder_.addAllMessages(values);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder clearNodes() {
+ if (nodesBuilder_ == null) {
+ nodes_ = java.util.Collections.emptyList();
+ bitField0_ = (bitField0_ & ~0x00000001);
+ onChanged();
+ } else {
+ nodesBuilder_.clear();
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public Builder removeNodes(int index) {
+ if (nodesBuilder_ == null) {
+ ensureNodesIsMutable();
+ nodes_.remove(index);
+ onChanged();
+ } else {
+ nodesBuilder_.remove(index);
+ }
+ return this;
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProto.Builder getNodesBuilder(
+ int index) {
+ return getNodesFieldBuilder().getBuilder(index);
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder getNodesOrBuilder(
+ int index) {
+ if (nodesBuilder_ == null) {
+ return nodes_.get(index); } else {
+ return nodesBuilder_.getMessageOrBuilder(index);
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public java.util.List<? extends org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
+ getNodesOrBuilderList() {
+ if (nodesBuilder_ != null) {
+ return nodesBuilder_.getMessageOrBuilderList();
+ } else {
+ return java.util.Collections.unmodifiableList(nodes_);
+ }
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProto.Builder addNodesBuilder() {
+ return getNodesFieldBuilder().addBuilder(
+ org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance());
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public org.apache.slider.api.proto.Messages.NodeInformationProto.Builder addNodesBuilder(
+ int index) {
+ return getNodesFieldBuilder().addBuilder(
+ index, org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance());
+ }
+ /**
+ * <code>repeated .org.apache.slider.api.NodeInformationProto nodes = 1;</code>
+ */
+ public java.util.List<org.apache.slider.api.proto.Messages.NodeInformationProto.Builder>
+ getNodesBuilderList() {
+ return getNodesFieldBuilder().getBuilderList();
+ }
+ private com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.slider.api.proto.Messages.NodeInformationProto, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>
+ getNodesFieldBuilder() {
+ if (nodesBuilder_ == null) {
+ nodesBuilder_ = new com.google.protobuf.RepeatedFieldBuilder<
+ org.apache.slider.api.proto.Messages.NodeInformationProto, org.apache.slider.api.proto.Messages.NodeInformationProto.Builder, org.apache.slider.api.proto.Messages.NodeInformationProtoOrBuilder>(
+ nodes_,
+ ((bitField0_ & 0x00000001) == 0x00000001),
+ getParentForChildren(),
+ isClean());
+ nodes_ = null;
+ }
+ return nodesBuilder_;
+ }
+
+ // @@protoc_insertion_point(builder_scope:org.apache.slider.api.GetLiveNodesResponseProto)
+ }
+
+ static {
+ defaultInstance = new GetLiveNodesResponseProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:org.apache.slider.api.GetLiveNodesResponseProto)
+ }
+
+ public interface GetLiveNodeRequestProtoOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+ // required string name = 1;
+ /**
+ * <code>required string name = 1;</code>
+ */
+ boolean hasName();
+ /**
+ * <code>required string name = 1;</code>
+ */
+ java.lang.String getName();
+ /**
+ * <code>required string name = 1;</code>
+ */
+ com.google.protobuf.ByteString
+ getNameBytes();
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.GetLiveNodeRequestProto}
+ */
+ public static final class GetLiveNodeRequestProto extends
+ com.google.protobuf.GeneratedMessage
+ implements GetLiveNodeRequestProtoOrBuilder {
+ // Use GetLiveNodeRequestProto.newBuilder() to construct.
+ private GetLiveNodeRequestProto(com.google.protobuf.GeneratedMessage.Builder<?> builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+ }
+ private GetLiveNodeRequestProto(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+ private static final GetLiveNodeRequestProto defaultInstance;
+ public static GetLiveNodeRequestProto getDefaultInstance() {
+ return defaultInstance;
+ }
+
+ public GetLiveNodeRequestProto getDefaultInstanceForType() {
+ return defaultInstance;
+ }
+
+ private final com.google.protobuf.UnknownFieldSet unknownFields;
+ @java.lang.Override
+ public final com.google.protobuf.UnknownFieldSet
+ getUnknownFields() {
+ return this.unknownFields;
+ }
+ private GetLiveNodeRequestProto(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+ boolean done = false;
+ while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+ case 0:
+ done = true;
+ break;
+ default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+ done = true;
+ }
+ break;
+ }
+ case 10: {
+ bitField0_ |= 0x00000001;
+ name_ = input.readBytes();
+ break;
+ }
+ }
+ }
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+ throw new com.google.protobuf.InvalidProtocolBufferException(
+ e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+ this.unknownFields = unknownFields.build();
+ makeExtensionsImmutable();
+ }
+ }
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodeRequestProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.class, org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.Builder.class);
+ }
+
+ public static com.google.protobuf.Parser<GetLiveNodeRequestProto> PARSER =
+ new com.google.protobuf.AbstractParser<GetLiveNodeRequestProto>() {
+ public GetLiveNodeRequestProto parsePartialFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return new GetLiveNodeRequestProto(input, extensionRegistry);
+ }
+ };
+
+ @java.lang.Override
+ public com.google.protobuf.Parser<GetLiveNodeRequestProto> getParserForType() {
+ return PARSER;
+ }
+
+ private int bitField0_;
+ // required string name = 1;
+ public static final int NAME_FIELD_NUMBER = 1;
+ private java.lang.Object name_;
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ return (java.lang.String) ref;
+ } else {
+ com.google.protobuf.ByteString bs =
+ (com.google.protobuf.ByteString) ref;
+ java.lang.String s = bs.toStringUtf8();
+ if (bs.isValidUtf8()) {
+ name_ = s;
+ }
+ return s;
+ }
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof java.lang.String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+
+ private void initFields() {
+ name_ = "";
+ }
+ private byte memoizedIsInitialized = -1;
+ public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized != -1) return isInitialized == 1;
+
+ if (!hasName()) {
+ memoizedIsInitialized = 0;
+ return false;
+ }
+ memoizedIsInitialized = 1;
+ return true;
+ }
+
+ public void writeTo(com.google.protobuf.CodedOutputStream output)
+ throws java.io.IOException {
+ getSerializedSize();
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ output.writeBytes(1, getNameBytes());
+ }
+ getUnknownFields().writeTo(output);
+ }
+
+ private int memoizedSerializedSize = -1;
+ public int getSerializedSize() {
+ int size = memoizedSerializedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ if (((bitField0_ & 0x00000001) == 0x00000001)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBytesSize(1, getNameBytes());
+ }
+ size += getUnknownFields().getSerializedSize();
+ memoizedSerializedSize = size;
+ return size;
+ }
+
+ private static final long serialVersionUID = 0L;
+ @java.lang.Override
+ protected java.lang.Object writeReplace()
+ throws java.io.ObjectStreamException {
+ return super.writeReplace();
+ }
+
+ @java.lang.Override
+ public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto)) {
+ return super.equals(obj);
+ }
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto other = (org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto) obj;
+
+ boolean result = true;
+ result = result && (hasName() == other.hasName());
+ if (hasName()) {
+ result = result && getName()
+ .equals(other.getName());
+ }
+ result = result &&
+ getUnknownFields().equals(other.getUnknownFields());
+ return result;
+ }
+
+ private int memoizedHashCode = 0;
+ @java.lang.Override
+ public int hashCode() {
+ if (memoizedHashCode != 0) {
+ return memoizedHashCode;
+ }
+ int hash = 41;
+ hash = (19 * hash) + getDescriptorForType().hashCode();
+ if (hasName()) {
+ hash = (37 * hash) + NAME_FIELD_NUMBER;
+ hash = (53 * hash) + getName().hashCode();
+ }
+ hash = (29 * hash) + getUnknownFields().hashCode();
+ memoizedHashCode = hash;
+ return hash;
+ }
+
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(
+ com.google.protobuf.ByteString data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(
+ com.google.protobuf.ByteString data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(byte[] data)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(
+ byte[] data,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws com.google.protobuf.InvalidProtocolBufferException {
+ return PARSER.parseFrom(data, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseDelimitedFrom(java.io.InputStream input)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseDelimitedFrom(
+ java.io.InputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseDelimitedFrom(input, extensionRegistry);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(
+ com.google.protobuf.CodedInputStream input)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input);
+ }
+ public static org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parseFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ return PARSER.parseFrom(input, extensionRegistry);
+ }
+
+ public static Builder newBuilder() { return Builder.create(); }
+ public Builder newBuilderForType() { return newBuilder(); }
+ public static Builder newBuilder(org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto prototype) {
+ return newBuilder().mergeFrom(prototype);
+ }
+ public Builder toBuilder() { return newBuilder(this); }
+
+ @java.lang.Override
+ protected Builder newBuilderForType(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ Builder builder = new Builder(parent);
+ return builder;
+ }
+ /**
+ * Protobuf type {@code org.apache.slider.api.GetLiveNodeRequestProto}
+ */
+ public static final class Builder extends
+ com.google.protobuf.GeneratedMessage.Builder<Builder>
+ implements org.apache.slider.api.proto.Messages.GetLiveNodeRequestProtoOrBuilder {
+ public static final com.google.protobuf.Descriptors.Descriptor
+ getDescriptor() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor;
+ }
+
+ protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internalGetFieldAccessorTable() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodeRequestProto_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.class, org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.Builder.class);
+ }
+
+ // Construct using org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.newBuilder()
+ private Builder() {
+ maybeForceBuilderInitialization();
+ }
+
+ private Builder(
+ com.google.protobuf.GeneratedMessage.BuilderParent parent) {
+ super(parent);
+ maybeForceBuilderInitialization();
+ }
+ private void maybeForceBuilderInitialization() {
+ if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) {
+ }
+ }
+ private static Builder create() {
+ return new Builder();
+ }
+
+ public Builder clear() {
+ super.clear();
+ name_ = "";
+ bitField0_ = (bitField0_ & ~0x00000001);
+ return this;
+ }
+
+ public Builder clone() {
+ return create().mergeFrom(buildPartial());
+ }
+
+ public com.google.protobuf.Descriptors.Descriptor
+ getDescriptorForType() {
+ return org.apache.slider.api.proto.Messages.internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor;
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto getDefaultInstanceForType() {
+ return org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.getDefaultInstance();
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto build() {
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto result = buildPartial();
+ if (!result.isInitialized()) {
+ throw newUninitializedMessageException(result);
+ }
+ return result;
+ }
+
+ public org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto buildPartial() {
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto result = new org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto(this);
+ int from_bitField0_ = bitField0_;
+ int to_bitField0_ = 0;
+ if (((from_bitField0_ & 0x00000001) == 0x00000001)) {
+ to_bitField0_ |= 0x00000001;
+ }
+ result.name_ = name_;
+ result.bitField0_ = to_bitField0_;
+ onBuilt();
+ return result;
+ }
+
+ public Builder mergeFrom(com.google.protobuf.Message other) {
+ if (other instanceof org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto) {
+ return mergeFrom((org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto)other);
+ } else {
+ super.mergeFrom(other);
+ return this;
+ }
+ }
+
+ public Builder mergeFrom(org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto other) {
+ if (other == org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.getDefaultInstance()) return this;
+ if (other.hasName()) {
+ bitField0_ |= 0x00000001;
+ name_ = other.name_;
+ onChanged();
+ }
+ this.mergeUnknownFields(other.getUnknownFields());
+ return this;
+ }
+
+ public final boolean isInitialized() {
+ if (!hasName()) {
+
+ return false;
+ }
+ return true;
+ }
+
+ public Builder mergeFrom(
+ com.google.protobuf.CodedInputStream input,
+ com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+ throws java.io.IOException {
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto parsedMessage = null;
+ try {
+ parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry);
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+ parsedMessage = (org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto) e.getUnfinishedMessage();
+ throw e;
+ } finally {
+ if (parsedMessage != null) {
+ mergeFrom(parsedMessage);
+ }
+ }
+ return this;
+ }
+ private int bitField0_;
+
+ // required string name = 1;
+ private java.lang.Object name_ = "";
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public boolean hasName() {
+ return ((bitField0_ & 0x00000001) == 0x00000001);
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public java.lang.String getName() {
+ java.lang.Object ref = name_;
+ if (!(ref instanceof java.lang.String)) {
+ java.lang.String s = ((com.google.protobuf.ByteString) ref)
+ .toStringUtf8();
+ name_ = s;
+ return s;
+ } else {
+ return (java.lang.String) ref;
+ }
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public com.google.protobuf.ByteString
+ getNameBytes() {
+ java.lang.Object ref = name_;
+ if (ref instanceof String) {
+ com.google.protobuf.ByteString b =
+ com.google.protobuf.ByteString.copyFromUtf8(
+ (java.lang.String) ref);
+ name_ = b;
+ return b;
+ } else {
+ return (com.google.protobuf.ByteString) ref;
+ }
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public Builder setName(
+ java.lang.String value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public Builder clearName() {
+ bitField0_ = (bitField0_ & ~0x00000001);
+ name_ = getDefaultInstance().getName();
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>required string name = 1;</code>
+ */
+ public Builder setNameBytes(
+ com.google.protobuf.ByteString value) {
+ if (value == null) {
+ throw new NullPointerException();
+ }
+ bitField0_ |= 0x00000001;
+ name_ = value;
+ onChanged();
+ return this;
+ }
+
+ // @@protoc_insertion_point(builder_scope:org.apache.slider.api.GetLiveNodeRequestProto)
+ }
+
+ static {
+ defaultInstance = new GetLiveNodeRequestProto(true);
+ defaultInstance.initFields();
+ }
+
+ // @@protoc_insertion_point(class_scope:org.apache.slider.api.GetLiveNodeRequestProto)
+ }
+
private static com.google.protobuf.Descriptors.Descriptor
internal_static_org_apache_slider_api_RoleInstanceState_descriptor;
private static
@@ -28820,6 +33933,16 @@
com.google.protobuf.GeneratedMessage.FieldAccessorTable
internal_static_org_apache_slider_api_PingInformationProto_fieldAccessorTable;
private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_org_apache_slider_api_NodeEntryInformationProto_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_org_apache_slider_api_NodeInformationProto_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_org_apache_slider_api_NodeInformationProto_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
internal_static_org_apache_slider_api_GetModelRequestProto_descriptor;
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
@@ -28909,6 +34032,21 @@
private static
com.google.protobuf.GeneratedMessage.FieldAccessorTable
internal_static_org_apache_slider_api_GetCertificateStoreResponseProto_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_org_apache_slider_api_GetLiveNodesRequestProto_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_org_apache_slider_api_GetLiveNodesRequestProto_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_org_apache_slider_api_GetLiveNodesResponseProto_fieldAccessorTable;
+ private static com.google.protobuf.Descriptors.Descriptor
+ internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor;
+ private static
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable
+ internal_static_org_apache_slider_api_GetLiveNodeRequestProto_fieldAccessorTable;
public static com.google.protobuf.Descriptors.FileDescriptor
getDescriptor() {
@@ -28957,7 +34095,7 @@
" \002(\t\022\023\n\013application\030\003 \002(\t\"`\n#Application" +
"LivenessInformationProto\022\034\n\024allRequestsS" +
"atisfied\030\001 \001(\010\022\033\n\023requestsOutstanding\030\002 " +
- "\001(\005\"\347\002\n\031ComponentInformationProto\022\014\n\004nam",
+ "\001(\005\"\256\003\n\031ComponentInformationProto\022\014\n\004nam",
"e\030\001 \001(\t\022\020\n\010priority\030\002 \001(\005\022\017\n\007desired\030\003 \001" +
"(\005\022\016\n\006actual\030\004 \001(\005\022\021\n\treleasing\030\005 \001(\005\022\021\n" +
"\trequested\030\006 \001(\005\022\016\n\006failed\030\007 \001(\005\022\017\n\007star" +
@@ -28966,39 +34104,56 @@
"lureMessage\030\014 \001(\t\022\027\n\017placementPolicy\030\r \001" +
"(\005\022\022\n\ncontainers\030\016 \003(\t\022\026\n\016failedRecently" +
"\030\017 \001(\005\022\022\n\nnodeFailed\030\020 \001(\005\022\021\n\tpreempted\030" +
- "\021 \001(\005\"\210\002\n\031ContainerInformationProto\022\023\n\013c" +
- "ontainerId\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010r",
- "eleased\030\003 \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode" +
- "\030\005 \001(\005\022\023\n\013diagnostics\030\006 \001(\t\022\022\n\ncreateTim" +
- "e\030\007 \001(\003\022\021\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003" +
- "(\t\022\014\n\004host\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tpla" +
- "cement\030\014 \001(\t\022\022\n\nappVersion\030\r \001(\t\"N\n\024Ping" +
- "InformationProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002" +
- " \001(\t\022\014\n\004body\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\026\n\024GetM" +
- "odelRequestProto\"\035\n\033GetModelDesiredReque" +
- "stProto\"$\n\"GetModelDesiredAppconfRequest" +
- "Proto\"&\n$GetModelDesiredResourcesRequest",
- "Proto\"%\n#GetModelResolvedAppconfRequestP" +
- "roto\"\'\n%GetModelResolvedResourcesRequest" +
- "Proto\"#\n!GetModelLiveResourcesRequestPro" +
- "to\"\037\n\035GetLiveContainersRequestProto\"u\n\036G" +
- "etLiveContainersResponseProto\022\r\n\005names\030\001" +
- " \003(\t\022D\n\ncontainers\030\002 \003(\01320.org.apache.sl" +
- "ider.api.ContainerInformationProto\"3\n\034Ge" +
- "tLiveContainerRequestProto\022\023\n\013containerI" +
- "d\030\001 \002(\t\"\037\n\035GetLiveComponentsRequestProto" +
- "\"u\n\036GetLiveComponentsResponseProto\022\r\n\005na",
- "mes\030\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org.apac" +
- "he.slider.api.ComponentInformationProto\"" +
- ",\n\034GetLiveComponentRequestProto\022\014\n\004name\030" +
- "\001 \002(\t\"$\n\"GetApplicationLivenessRequestPr" +
- "oto\"\023\n\021EmptyPayloadProto\" \n\020WrappedJsonP" +
- "roto\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificateStor" +
- "eRequestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013reque" +
- "sterId\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004type\030\004" +
- " \002(\t\"1\n GetCertificateStoreResponseProto" +
- "\022\r\n\005store\030\001 \002(\014B-\n\033org.apache.slider.api",
- ".protoB\010Messages\210\001\001\240\001\001"
+ "\021 \001(\005\022%\n\035pendingAntiAffineRequestCount\030\022" +
+ " \001(\005\022\036\n\026isAARequestOutstanding\030\023 \001(\010\"\210\002\n",
+ "\031ContainerInformationProto\022\023\n\013containerI" +
+ "d\030\001 \001(\t\022\021\n\tcomponent\030\002 \001(\t\022\020\n\010released\030\003" +
+ " \001(\010\022\r\n\005state\030\004 \001(\005\022\020\n\010exitCode\030\005 \001(\005\022\023\n" +
+ "\013diagnostics\030\006 \001(\t\022\022\n\ncreateTime\030\007 \001(\003\022\021" +
+ "\n\tstartTime\030\010 \001(\003\022\016\n\006output\030\t \003(\t\022\014\n\004hos" +
+ "t\030\n \001(\t\022\017\n\007hostURL\030\013 \001(\t\022\021\n\tplacement\030\014 " +
+ "\001(\t\022\022\n\nappVersion\030\r \001(\t\"N\n\024PingInformati" +
+ "onProto\022\014\n\004text\030\001 \001(\t\022\014\n\004verb\030\002 \001(\t\022\014\n\004b" +
+ "ody\030\003 \001(\t\022\014\n\004time\030\004 \001(\003\"\343\001\n\031NodeEntryInf" +
+ "ormationProto\022\020\n\010priority\030\001 \002(\005\022\021\n\treque",
+ "sted\030\002 \002(\005\022\020\n\010starting\030\003 \002(\005\022\023\n\013startFai" +
+ "led\030\004 \002(\005\022\016\n\006failed\030\005 \002(\005\022\026\n\016failedRecen" +
+ "tly\030\006 \002(\005\022\021\n\tpreempted\030\007 \002(\005\022\014\n\004live\030\010 \002" +
+ "(\005\022\021\n\treleasing\030\t \002(\005\022\020\n\010lastUsed\030\n \002(\003\022" +
+ "\014\n\004name\030\013 \002(\t\"\334\001\n\024NodeInformationProto\022\020" +
+ "\n\010hostname\030\001 \002(\t\022\r\n\005state\030\002 \002(\t\022\023\n\013httpA" +
+ "ddress\030\003 \002(\t\022\020\n\010rackName\030\004 \002(\t\022\016\n\006labels" +
+ "\030\005 \002(\t\022\024\n\014healthReport\030\006 \002(\t\022\023\n\013lastUpda" +
+ "ted\030\007 \002(\003\022A\n\007entries\030\010 \003(\01320.org.apache." +
+ "slider.api.NodeEntryInformationProto\"\026\n\024",
+ "GetModelRequestProto\"\035\n\033GetModelDesiredR" +
+ "equestProto\"$\n\"GetModelDesiredAppconfReq" +
+ "uestProto\"&\n$GetModelDesiredResourcesReq" +
+ "uestProto\"%\n#GetModelResolvedAppconfRequ" +
+ "estProto\"\'\n%GetModelResolvedResourcesReq" +
+ "uestProto\"#\n!GetModelLiveResourcesReques" +
+ "tProto\"\037\n\035GetLiveContainersRequestProto\"" +
+ "u\n\036GetLiveContainersResponseProto\022\r\n\005nam" +
+ "es\030\001 \003(\t\022D\n\ncontainers\030\002 \003(\01320.org.apach" +
+ "e.slider.api.ContainerInformationProto\"3",
+ "\n\034GetLiveContainerRequestProto\022\023\n\013contai" +
+ "nerId\030\001 \002(\t\"\037\n\035GetLiveComponentsRequestP" +
+ "roto\"u\n\036GetLiveComponentsResponseProto\022\r" +
+ "\n\005names\030\001 \003(\t\022D\n\ncomponents\030\002 \003(\01320.org." +
+ "apache.slider.api.ComponentInformationPr" +
+ "oto\",\n\034GetLiveComponentRequestProto\022\014\n\004n" +
+ "ame\030\001 \002(\t\"$\n\"GetApplicationLivenessReque" +
+ "stProto\"\023\n\021EmptyPayloadProto\" \n\020WrappedJ" +
+ "sonProto\022\014\n\004json\030\001 \002(\t\"h\n\037GetCertificate" +
+ "StoreRequestProto\022\020\n\010hostname\030\001 \001(\t\022\023\n\013r",
+ "equesterId\030\002 \002(\t\022\020\n\010password\030\003 \002(\t\022\014\n\004ty" +
+ "pe\030\004 \002(\t\"1\n GetCertificateStoreResponseP" +
+ "roto\022\r\n\005store\030\001 \002(\014\"\032\n\030GetLiveNodesReque" +
+ "stProto\"W\n\031GetLiveNodesResponseProto\022:\n\005" +
+ "nodes\030\001 \003(\0132+.org.apache.slider.api.Node" +
+ "InformationProto\"\'\n\027GetLiveNodeRequestPr" +
+ "oto\022\014\n\004name\030\001 \002(\tB-\n\033org.apache.slider.a" +
+ "pi.protoB\010Messages\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -29154,7 +34309,7 @@
internal_static_org_apache_slider_api_ComponentInformationProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_ComponentInformationProto_descriptor,
- new java.lang.String[] { "Name", "Priority", "Desired", "Actual", "Releasing", "Requested", "Failed", "Started", "StartFailed", "Completed", "TotalRequested", "FailureMessage", "PlacementPolicy", "Containers", "FailedRecently", "NodeFailed", "Preempted", });
+ new java.lang.String[] { "Name", "Priority", "Desired", "Actual", "Releasing", "Requested", "Failed", "Started", "StartFailed", "Completed", "TotalRequested", "FailureMessage", "PlacementPolicy", "Containers", "FailedRecently", "NodeFailed", "Preempted", "PendingAntiAffineRequestCount", "IsAARequestOutstanding", });
internal_static_org_apache_slider_api_ContainerInformationProto_descriptor =
getDescriptor().getMessageTypes().get(25);
internal_static_org_apache_slider_api_ContainerInformationProto_fieldAccessorTable = new
@@ -29167,114 +34322,144 @@
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_PingInformationProto_descriptor,
new java.lang.String[] { "Text", "Verb", "Body", "Time", });
- internal_static_org_apache_slider_api_GetModelRequestProto_descriptor =
+ internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor =
getDescriptor().getMessageTypes().get(27);
+ internal_static_org_apache_slider_api_NodeEntryInformationProto_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_org_apache_slider_api_NodeEntryInformationProto_descriptor,
+ new java.lang.String[] { "Priority", "Requested", "Starting", "StartFailed", "Failed", "FailedRecently", "Preempted", "Live", "Releasing", "LastUsed", "Name", });
+ internal_static_org_apache_slider_api_NodeInformationProto_descriptor =
+ getDescriptor().getMessageTypes().get(28);
+ internal_static_org_apache_slider_api_NodeInformationProto_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_org_apache_slider_api_NodeInformationProto_descriptor,
+ new java.lang.String[] { "Hostname", "State", "HttpAddress", "RackName", "Labels", "HealthReport", "LastUpdated", "Entries", });
+ internal_static_org_apache_slider_api_GetModelRequestProto_descriptor =
+ getDescriptor().getMessageTypes().get(29);
internal_static_org_apache_slider_api_GetModelRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetModelDesiredRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(28);
+ getDescriptor().getMessageTypes().get(30);
internal_static_org_apache_slider_api_GetModelDesiredRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelDesiredRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetModelDesiredAppconfRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(29);
+ getDescriptor().getMessageTypes().get(31);
internal_static_org_apache_slider_api_GetModelDesiredAppconfRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelDesiredAppconfRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetModelDesiredResourcesRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(30);
+ getDescriptor().getMessageTypes().get(32);
internal_static_org_apache_slider_api_GetModelDesiredResourcesRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelDesiredResourcesRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetModelResolvedAppconfRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(31);
+ getDescriptor().getMessageTypes().get(33);
internal_static_org_apache_slider_api_GetModelResolvedAppconfRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelResolvedAppconfRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetModelResolvedResourcesRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(32);
+ getDescriptor().getMessageTypes().get(34);
internal_static_org_apache_slider_api_GetModelResolvedResourcesRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelResolvedResourcesRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetModelLiveResourcesRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(33);
+ getDescriptor().getMessageTypes().get(35);
internal_static_org_apache_slider_api_GetModelLiveResourcesRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetModelLiveResourcesRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetLiveContainersRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(34);
+ getDescriptor().getMessageTypes().get(36);
internal_static_org_apache_slider_api_GetLiveContainersRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveContainersRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetLiveContainersResponseProto_descriptor =
- getDescriptor().getMessageTypes().get(35);
+ getDescriptor().getMessageTypes().get(37);
internal_static_org_apache_slider_api_GetLiveContainersResponseProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveContainersResponseProto_descriptor,
new java.lang.String[] { "Names", "Containers", });
internal_static_org_apache_slider_api_GetLiveContainerRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(36);
+ getDescriptor().getMessageTypes().get(38);
internal_static_org_apache_slider_api_GetLiveContainerRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveContainerRequestProto_descriptor,
new java.lang.String[] { "ContainerId", });
internal_static_org_apache_slider_api_GetLiveComponentsRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(37);
+ getDescriptor().getMessageTypes().get(39);
internal_static_org_apache_slider_api_GetLiveComponentsRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveComponentsRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_GetLiveComponentsResponseProto_descriptor =
- getDescriptor().getMessageTypes().get(38);
+ getDescriptor().getMessageTypes().get(40);
internal_static_org_apache_slider_api_GetLiveComponentsResponseProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveComponentsResponseProto_descriptor,
new java.lang.String[] { "Names", "Components", });
internal_static_org_apache_slider_api_GetLiveComponentRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(39);
+ getDescriptor().getMessageTypes().get(41);
internal_static_org_apache_slider_api_GetLiveComponentRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetLiveComponentRequestProto_descriptor,
new java.lang.String[] { "Name", });
internal_static_org_apache_slider_api_GetApplicationLivenessRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(40);
+ getDescriptor().getMessageTypes().get(42);
internal_static_org_apache_slider_api_GetApplicationLivenessRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetApplicationLivenessRequestProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_EmptyPayloadProto_descriptor =
- getDescriptor().getMessageTypes().get(41);
+ getDescriptor().getMessageTypes().get(43);
internal_static_org_apache_slider_api_EmptyPayloadProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_EmptyPayloadProto_descriptor,
new java.lang.String[] { });
internal_static_org_apache_slider_api_WrappedJsonProto_descriptor =
- getDescriptor().getMessageTypes().get(42);
+ getDescriptor().getMessageTypes().get(44);
internal_static_org_apache_slider_api_WrappedJsonProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_WrappedJsonProto_descriptor,
new java.lang.String[] { "Json", });
internal_static_org_apache_slider_api_GetCertificateStoreRequestProto_descriptor =
- getDescriptor().getMessageTypes().get(43);
+ getDescriptor().getMessageTypes().get(45);
internal_static_org_apache_slider_api_GetCertificateStoreRequestProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetCertificateStoreRequestProto_descriptor,
new java.lang.String[] { "Hostname", "RequesterId", "Password", "Type", });
internal_static_org_apache_slider_api_GetCertificateStoreResponseProto_descriptor =
- getDescriptor().getMessageTypes().get(44);
+ getDescriptor().getMessageTypes().get(46);
internal_static_org_apache_slider_api_GetCertificateStoreResponseProto_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_org_apache_slider_api_GetCertificateStoreResponseProto_descriptor,
new java.lang.String[] { "Store", });
+ internal_static_org_apache_slider_api_GetLiveNodesRequestProto_descriptor =
+ getDescriptor().getMessageTypes().get(47);
+ internal_static_org_apache_slider_api_GetLiveNodesRequestProto_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_org_apache_slider_api_GetLiveNodesRequestProto_descriptor,
+ new java.lang.String[] { });
+ internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor =
+ getDescriptor().getMessageTypes().get(48);
+ internal_static_org_apache_slider_api_GetLiveNodesResponseProto_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_org_apache_slider_api_GetLiveNodesResponseProto_descriptor,
+ new java.lang.String[] { "Nodes", });
+ internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor =
+ getDescriptor().getMessageTypes().get(49);
+ internal_static_org_apache_slider_api_GetLiveNodeRequestProto_fieldAccessorTable = new
+ com.google.protobuf.GeneratedMessage.FieldAccessorTable(
+ internal_static_org_apache_slider_api_GetLiveNodeRequestProto_descriptor,
+ new java.lang.String[] { "Name", });
return null;
}
};
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
index c408ed2..feebe1d 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/RestTypeMarshalling.java
@@ -23,6 +23,8 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeEntryInformation;
+import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -36,7 +38,10 @@
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
/**
* Class to handle marshalling of REST
@@ -82,7 +87,12 @@
if (wire.hasFailureMessage()) {
info.failureMessage = wire.getFailureMessage();
}
-
+ if (wire.hasPendingAntiAffineRequestCount()) {
+ info.pendingAntiAffineRequestCount = wire.getPendingAntiAffineRequestCount();
+ }
+ if (wire.hasIsAARequestOutstanding()) {
+ info.isAARequestOutstanding = wire.getIsAARequestOutstanding();
+ }
return info;
}
@@ -132,11 +142,77 @@
if (info.containers != null) {
builder.addAllContainers(info.containers);
}
+ builder.setPendingAntiAffineRequestCount(info.pendingAntiAffineRequestCount);
+ builder.setIsAARequestOutstanding(info.isAARequestOutstanding);
return builder.build();
}
- public static ContainerInformation
- unmarshall(Messages.ContainerInformationProto wire) {
+ public static Messages.NodeInformationProto marshall(NodeInformation info) {
+
+ Messages.NodeInformationProto.Builder builder =
+ Messages.NodeInformationProto.newBuilder();
+ builder.setHostname(info.hostname);
+ builder.setLastUpdated(info.lastUpdated);
+ builder.setState(info.state != null? info.state : "unknown");
+ builder.setRackName(info.rackName != null ? info.rackName : "");
+ builder.setHealthReport(info.healthReport != null ? info.healthReport : "");
+ builder.setHttpAddress(info.httpAddress != null ? info.httpAddress : "");
+ builder.setLabels(info.labels != null ? info.labels: "");
+
+
+ if (info.entries != null) {
+ for (Map.Entry<String, NodeEntryInformation> elt : info.entries.entrySet()) {
+ NodeEntryInformation entry = elt.getValue();
+ Messages.NodeEntryInformationProto.Builder node =
+ Messages.NodeEntryInformationProto.newBuilder();
+ node.setPriority(entry.priority);
+ node.setName(elt.getKey());
+ node.setFailed(entry.failed);
+ node.setFailedRecently(entry.failedRecently);
+ node.setLive(entry.live);
+ node.setLastUsed(entry.lastUsed);
+ node.setPreempted(entry.preempted);
+ node.setRequested(entry.requested);
+ node.setReleasing(entry.releasing);
+ node.setStartFailed(entry.startFailed);
+ node.setStarting(entry.starting);
+ builder.addEntries(node.build());
+ }
+ }
+ return builder.build();
+ }
+
+ public static NodeInformation unmarshall(Messages.NodeInformationProto wire) {
+ NodeInformation info = new NodeInformation();
+ info.healthReport = wire.getHealthReport();
+ info.hostname = wire.getHostname();
+ info.httpAddress = wire.getHttpAddress();
+ info.labels = wire.getLabels();
+ info.lastUpdated = wire.getLastUpdated();
+ info.rackName = wire.getRackName();
+ info.state = wire.getState();
+ List<Messages.NodeEntryInformationProto> entriesList = wire.getEntriesList();
+ if (entriesList != null) {
+ info.entries = new HashMap<>(entriesList.size());
+ for (Messages.NodeEntryInformationProto entry : entriesList) {
+ NodeEntryInformation nei = new NodeEntryInformation();
+ nei.failed = entry.getFailed();
+ nei.failedRecently = entry.getFailedRecently();
+ nei.lastUsed = entry.getLastUsed();
+ nei.live = entry.getLive();
+ nei.preempted = entry.getPreempted();
+ nei.priority = entry.getPriority();
+ nei.requested = entry.getRequested();
+ nei.releasing = entry.getReleasing();
+ nei.startFailed = entry.getStartFailed();
+ nei.starting = entry.getStarting();
+ info.entries.put(entry.getName(), nei);
+ }
+ }
+ return info;
+ }
+
+ public static ContainerInformation unmarshall(Messages.ContainerInformationProto wire) {
ContainerInformation info = new ContainerInformation();
info.containerId = wire.getContainerId();
info.component = wire.getComponent();
@@ -168,19 +244,15 @@
return info;
}
- public static List<ContainerInformation> unmarshall(
- Messages.GetLiveContainersResponseProto wire) {
- List<ContainerInformation> infoList = new ArrayList<ContainerInformation>(
- wire.getContainersList().size());
- for (Messages.ContainerInformationProto container : wire
- .getContainersList()) {
+ public static List<ContainerInformation> unmarshall(Messages.GetLiveContainersResponseProto wire) {
+ List<ContainerInformation> infoList = new ArrayList<>(wire.getContainersList().size());
+ for (Messages.ContainerInformationProto container : wire.getContainersList()) {
infoList.add(unmarshall(container));
}
return infoList;
}
- public static Messages.ContainerInformationProto
- marshall(ContainerInformation info) {
+ public static Messages.ContainerInformationProto marshall(ContainerInformation info) {
Messages.ContainerInformationProto.Builder builder =
Messages.ContainerInformationProto.newBuilder();
@@ -217,8 +289,7 @@
return builder.build();
}
- public static String
- unmarshall(Messages.WrappedJsonProto wire) {
+ public static String unmarshall(Messages.WrappedJsonProto wire) {
return wire.getJson();
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java b/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
index 702c762..081b7fa 100644
--- a/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/api/proto/SliderClusterAPI.java
@@ -203,6 +203,22 @@
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ComponentInformationProto> done);
/**
+ * <code>rpc getLiveNodes(.org.apache.slider.api.GetLiveNodesRequestProto) returns (.org.apache.slider.api.GetLiveNodesResponseProto);</code>
+ */
+ public abstract void getLiveNodes(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done);
+
+ /**
+ * <code>rpc getLiveNode(.org.apache.slider.api.GetLiveNodeRequestProto) returns (.org.apache.slider.api.NodeInformationProto);</code>
+ */
+ public abstract void getLiveNode(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done);
+
+ /**
* <code>rpc getModelDesired(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
*
* <pre>
@@ -428,6 +444,22 @@
}
@java.lang.Override
+ public void getLiveNodes(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done) {
+ impl.getLiveNodes(controller, request, done);
+ }
+
+ @java.lang.Override
+ public void getLiveNode(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done) {
+ impl.getLiveNode(controller, request, done);
+ }
+
+ @java.lang.Override
public void getModelDesired(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
@@ -546,20 +578,24 @@
case 15:
return impl.getLiveComponent(controller, (org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto)request);
case 16:
- return impl.getModelDesired(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getLiveNodes(controller, (org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto)request);
case 17:
- return impl.getModelDesiredAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getLiveNode(controller, (org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto)request);
case 18:
- return impl.getModelDesiredResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getModelDesired(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
case 19:
- return impl.getModelResolved(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getModelDesiredAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
case 20:
- return impl.getModelResolvedAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getModelDesiredResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
case 21:
- return impl.getModelResolvedResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getModelResolved(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
case 22:
- return impl.getLiveResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ return impl.getModelResolvedAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
case 23:
+ return impl.getModelResolvedResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ case 24:
+ return impl.getLiveResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request);
+ case 25:
return impl.getClientCertificateStore(controller, (org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto)request);
default:
throw new java.lang.AssertionError("Can't get here.");
@@ -608,9 +644,9 @@
case 15:
return org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto.getDefaultInstance();
case 16:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.getDefaultInstance();
case 17:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.getDefaultInstance();
case 18:
return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
case 19:
@@ -622,6 +658,10 @@
case 22:
return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
case 23:
+ return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ case 24:
+ return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ case 25:
return org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto.getDefaultInstance();
default:
throw new java.lang.AssertionError("Can't get here.");
@@ -670,9 +710,9 @@
case 15:
return org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance();
case 16:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance();
case 17:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance();
case 18:
return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
case 19:
@@ -684,6 +724,10 @@
case 22:
return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
case 23:
+ return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ case 24:
+ return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ case 25:
return org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance();
default:
throw new java.lang.AssertionError("Can't get here.");
@@ -875,6 +919,22 @@
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.ComponentInformationProto> done);
/**
+ * <code>rpc getLiveNodes(.org.apache.slider.api.GetLiveNodesRequestProto) returns (.org.apache.slider.api.GetLiveNodesResponseProto);</code>
+ */
+ public abstract void getLiveNodes(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done);
+
+ /**
+ * <code>rpc getLiveNode(.org.apache.slider.api.GetLiveNodeRequestProto) returns (.org.apache.slider.api.NodeInformationProto);</code>
+ */
+ public abstract void getLiveNode(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done);
+
+ /**
* <code>rpc getModelDesired(.org.apache.slider.api.EmptyPayloadProto) returns (.org.apache.slider.api.WrappedJsonProto);</code>
*
* <pre>
@@ -1069,41 +1129,51 @@
done));
return;
case 16:
+ this.getLiveNodes(controller, (org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto)request,
+ com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto>specializeCallback(
+ done));
+ return;
+ case 17:
+ this.getLiveNode(controller, (org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto)request,
+ com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.NodeInformationProto>specializeCallback(
+ done));
+ return;
+ case 18:
this.getModelDesired(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 17:
+ case 19:
this.getModelDesiredAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 18:
+ case 20:
this.getModelDesiredResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 19:
+ case 21:
this.getModelResolved(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 20:
+ case 22:
this.getModelResolvedAppconf(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 21:
+ case 23:
this.getModelResolvedResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 22:
+ case 24:
this.getLiveResources(controller, (org.apache.slider.api.proto.Messages.EmptyPayloadProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.WrappedJsonProto>specializeCallback(
done));
return;
- case 23:
+ case 25:
this.getClientCertificateStore(controller, (org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto)request,
com.google.protobuf.RpcUtil.<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto>specializeCallback(
done));
@@ -1155,9 +1225,9 @@
case 15:
return org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto.getDefaultInstance();
case 16:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto.getDefaultInstance();
case 17:
- return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto.getDefaultInstance();
case 18:
return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
case 19:
@@ -1169,6 +1239,10 @@
case 22:
return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
case 23:
+ return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ case 24:
+ return org.apache.slider.api.proto.Messages.EmptyPayloadProto.getDefaultInstance();
+ case 25:
return org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto.getDefaultInstance();
default:
throw new java.lang.AssertionError("Can't get here.");
@@ -1217,9 +1291,9 @@
case 15:
return org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance();
case 16:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance();
case 17:
- return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ return org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance();
case 18:
return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
case 19:
@@ -1231,6 +1305,10 @@
case 22:
return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
case 23:
+ return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ case 24:
+ return org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance();
+ case 25:
return org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance();
default:
throw new java.lang.AssertionError("Can't get here.");
@@ -1493,37 +1571,37 @@
org.apache.slider.api.proto.Messages.ComponentInformationProto.getDefaultInstance()));
}
- public void getModelDesired(
+ public void getLiveNodes(
com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto> done) {
channel.callMethod(
getDescriptor().getMethods().get(16),
controller,
request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance(),
com.google.protobuf.RpcUtil.generalizeCallback(
done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.class,
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance()));
}
- public void getModelDesiredAppconf(
+ public void getLiveNode(
com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
- com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.NodeInformationProto> done) {
channel.callMethod(
getDescriptor().getMethods().get(17),
controller,
request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
+ org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance(),
com.google.protobuf.RpcUtil.generalizeCallback(
done,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
+ org.apache.slider.api.proto.Messages.NodeInformationProto.class,
+ org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance()));
}
- public void getModelDesiredResources(
+ public void getModelDesired(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
@@ -1538,7 +1616,7 @@
org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
}
- public void getModelResolved(
+ public void getModelDesiredAppconf(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
@@ -1553,7 +1631,7 @@
org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
}
- public void getModelResolvedAppconf(
+ public void getModelDesiredResources(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
@@ -1568,7 +1646,7 @@
org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
}
- public void getModelResolvedResources(
+ public void getModelResolved(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
@@ -1583,7 +1661,7 @@
org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
}
- public void getLiveResources(
+ public void getModelResolvedAppconf(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
@@ -1598,12 +1676,42 @@
org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
}
+ public void getModelResolvedResources(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
+ channel.callMethod(
+ getDescriptor().getMethods().get(23),
+ controller,
+ request,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
+ com.google.protobuf.RpcUtil.generalizeCallback(
+ done,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
+ }
+
+ public void getLiveResources(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.EmptyPayloadProto request,
+ com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.WrappedJsonProto> done) {
+ channel.callMethod(
+ getDescriptor().getMethods().get(24),
+ controller,
+ request,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance(),
+ com.google.protobuf.RpcUtil.generalizeCallback(
+ done,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.class,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance()));
+ }
+
public void getClientCertificateStore(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto request,
com.google.protobuf.RpcCallback<org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto> done) {
channel.callMethod(
- getDescriptor().getMethods().get(23),
+ getDescriptor().getMethods().get(25),
controller,
request,
org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance(),
@@ -1700,6 +1808,16 @@
org.apache.slider.api.proto.Messages.GetLiveComponentRequestProto request)
throws com.google.protobuf.ServiceException;
+ public org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto getLiveNodes(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request)
+ throws com.google.protobuf.ServiceException;
+
+ public org.apache.slider.api.proto.Messages.NodeInformationProto getLiveNode(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request)
+ throws com.google.protobuf.ServiceException;
+
public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesired(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
@@ -1940,31 +2058,31 @@
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesired(
+ public org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto getLiveNodes(
com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
+ org.apache.slider.api.proto.Messages.GetLiveNodesRequestProto request)
throws com.google.protobuf.ServiceException {
- return (org.apache.slider.api.proto.Messages.WrappedJsonProto) channel.callBlockingMethod(
+ return (org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto) channel.callBlockingMethod(
getDescriptor().getMethods().get(16),
controller,
request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance());
+ org.apache.slider.api.proto.Messages.GetLiveNodesResponseProto.getDefaultInstance());
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesiredAppconf(
+ public org.apache.slider.api.proto.Messages.NodeInformationProto getLiveNode(
com.google.protobuf.RpcController controller,
- org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
+ org.apache.slider.api.proto.Messages.GetLiveNodeRequestProto request)
throws com.google.protobuf.ServiceException {
- return (org.apache.slider.api.proto.Messages.WrappedJsonProto) channel.callBlockingMethod(
+ return (org.apache.slider.api.proto.Messages.NodeInformationProto) channel.callBlockingMethod(
getDescriptor().getMethods().get(17),
controller,
request,
- org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance());
+ org.apache.slider.api.proto.Messages.NodeInformationProto.getDefaultInstance());
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesiredResources(
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesired(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
throws com.google.protobuf.ServiceException {
@@ -1976,7 +2094,7 @@
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelResolved(
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesiredAppconf(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
throws com.google.protobuf.ServiceException {
@@ -1988,7 +2106,7 @@
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelResolvedAppconf(
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelDesiredResources(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
throws com.google.protobuf.ServiceException {
@@ -2000,7 +2118,7 @@
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelResolvedResources(
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelResolved(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
throws com.google.protobuf.ServiceException {
@@ -2012,7 +2130,7 @@
}
- public org.apache.slider.api.proto.Messages.WrappedJsonProto getLiveResources(
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelResolvedAppconf(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
throws com.google.protobuf.ServiceException {
@@ -2024,12 +2142,36 @@
}
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getModelResolvedResources(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
+ throws com.google.protobuf.ServiceException {
+ return (org.apache.slider.api.proto.Messages.WrappedJsonProto) channel.callBlockingMethod(
+ getDescriptor().getMethods().get(23),
+ controller,
+ request,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance());
+ }
+
+
+ public org.apache.slider.api.proto.Messages.WrappedJsonProto getLiveResources(
+ com.google.protobuf.RpcController controller,
+ org.apache.slider.api.proto.Messages.EmptyPayloadProto request)
+ throws com.google.protobuf.ServiceException {
+ return (org.apache.slider.api.proto.Messages.WrappedJsonProto) channel.callBlockingMethod(
+ getDescriptor().getMethods().get(24),
+ controller,
+ request,
+ org.apache.slider.api.proto.Messages.WrappedJsonProto.getDefaultInstance());
+ }
+
+
public org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto getClientCertificateStore(
com.google.protobuf.RpcController controller,
org.apache.slider.api.proto.Messages.GetCertificateStoreRequestProto request)
throws com.google.protobuf.ServiceException {
return (org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto) channel.callBlockingMethod(
- getDescriptor().getMethods().get(23),
+ getDescriptor().getMethods().get(25),
controller,
request,
org.apache.slider.api.proto.Messages.GetCertificateStoreResponseProto.getDefaultInstance());
@@ -2051,7 +2193,7 @@
java.lang.String[] descriptorData = {
"\n\033SliderClusterProtocol.proto\022\025org.apach" +
"e.slider.api\032\033SliderClusterMessages.prot" +
- "o2\306\026\n\027SliderClusterProtocolPB\022n\n\013stopClu" +
+ "o2\245\030\n\027SliderClusterProtocolPB\022n\n\013stopClu" +
"ster\022..org.apache.slider.api.StopCluster" +
"RequestProto\032/.org.apache.slider.api.Sto" +
"pClusterResponseProto\022\200\001\n\021upgradeContain" +
@@ -2100,31 +2242,37 @@
"api.GetLiveComponentsResponseProto\022y\n\020ge" +
"tLiveComponent\0223.org.apache.slider.api.G",
"etLiveComponentRequestProto\0320.org.apache" +
- ".slider.api.ComponentInformationProto\022d\n" +
- "\017getModelDesired\022(.org.apache.slider.api" +
- ".EmptyPayloadProto\032\'.org.apache.slider.a" +
- "pi.WrappedJsonProto\022k\n\026getModelDesiredAp" +
- "pconf\022(.org.apache.slider.api.EmptyPaylo" +
- "adProto\032\'.org.apache.slider.api.WrappedJ" +
- "sonProto\022m\n\030getModelDesiredResources\022(.o" +
- "rg.apache.slider.api.EmptyPayloadProto\032\'" +
- ".org.apache.slider.api.WrappedJsonProto\022",
- "e\n\020getModelResolved\022(.org.apache.slider." +
+ ".slider.api.ComponentInformationProto\022q\n" +
+ "\014getLiveNodes\022/.org.apache.slider.api.Ge" +
+ "tLiveNodesRequestProto\0320.org.apache.slid" +
+ "er.api.GetLiveNodesResponseProto\022j\n\013getL" +
+ "iveNode\022..org.apache.slider.api.GetLiveN" +
+ "odeRequestProto\032+.org.apache.slider.api." +
+ "NodeInformationProto\022d\n\017getModelDesired\022" +
+ "(.org.apache.slider.api.EmptyPayloadProt" +
+ "o\032\'.org.apache.slider.api.WrappedJsonPro",
+ "to\022k\n\026getModelDesiredAppconf\022(.org.apach" +
+ "e.slider.api.EmptyPayloadProto\032\'.org.apa" +
+ "che.slider.api.WrappedJsonProto\022m\n\030getMo" +
+ "delDesiredResources\022(.org.apache.slider." +
"api.EmptyPayloadProto\032\'.org.apache.slide" +
- "r.api.WrappedJsonProto\022l\n\027getModelResolv" +
- "edAppconf\022(.org.apache.slider.api.EmptyP" +
- "ayloadProto\032\'.org.apache.slider.api.Wrap" +
- "pedJsonProto\022n\n\031getModelResolvedResource" +
- "s\022(.org.apache.slider.api.EmptyPayloadPr" +
- "oto\032\'.org.apache.slider.api.WrappedJsonP" +
- "roto\022e\n\020getLiveResources\022(.org.apache.sl" +
- "ider.api.EmptyPayloadProto\032\'.org.apache.",
- "slider.api.WrappedJsonProto\022\214\001\n\031getClien" +
- "tCertificateStore\0226.org.apache.slider.ap" +
- "i.GetCertificateStoreRequestProto\0327.org." +
- "apache.slider.api.GetCertificateStoreRes" +
- "ponseProtoB5\n\033org.apache.slider.api.prot" +
- "oB\020SliderClusterAPI\210\001\001\240\001\001"
+ "r.api.WrappedJsonProto\022e\n\020getModelResolv" +
+ "ed\022(.org.apache.slider.api.EmptyPayloadP" +
+ "roto\032\'.org.apache.slider.api.WrappedJson" +
+ "Proto\022l\n\027getModelResolvedAppconf\022(.org.a" +
+ "pache.slider.api.EmptyPayloadProto\032\'.org",
+ ".apache.slider.api.WrappedJsonProto\022n\n\031g" +
+ "etModelResolvedResources\022(.org.apache.sl" +
+ "ider.api.EmptyPayloadProto\032\'.org.apache." +
+ "slider.api.WrappedJsonProto\022e\n\020getLiveRe" +
+ "sources\022(.org.apache.slider.api.EmptyPay" +
+ "loadProto\032\'.org.apache.slider.api.Wrappe" +
+ "dJsonProto\022\214\001\n\031getClientCertificateStore" +
+ "\0226.org.apache.slider.api.GetCertificateS" +
+ "toreRequestProto\0327.org.apache.slider.api" +
+ ".GetCertificateStoreResponseProtoB5\n\033org",
+ ".apache.slider.api.protoB\020SliderClusterA" +
+ "PI\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
index 4dd2cb7..9879d05 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ApplicationLivenessInformation.java
@@ -30,9 +30,15 @@
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ApplicationLivenessInformation {
+ /** flag set if the cluster is at size */
public boolean allRequestsSatisfied;
+
+ /** number of outstanding requests: those needed to satisfy */
public int requestsOutstanding;
+ /** number of requests submitted to YARN */
+ public int activeRequests;
+
@Override
public String toString() {
final StringBuilder sb =
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
index 52f6c08..c46a59f 100644
--- a/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
+++ b/slider-core/src/main/java/org/apache/slider/api/types/ComponentInformation.java
@@ -19,6 +19,7 @@
package org.apache.slider.api.types;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.server.appmaster.state.RoleStatus;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -35,13 +36,15 @@
* <p>
* This means that if any fields are added here. they must be added to
* <code>src/main/proto/SliderClusterMessages.proto</code> and
- * the probuf structures rebuilt.
+ * the protobuf structures rebuilt via a {@code mvn generate-sources -Pcompile-protobuf}
+ *
+ * See also {@link RoleStatus#serialize()}
*/
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ComponentInformation {
-
+
public String name;
public int priority;
public int desired, actual, releasing;
@@ -49,6 +52,9 @@
public int requested;
public int failed, started, startFailed, completed, totalRequested;
public int nodeFailed, failedRecently, preempted;
+ public int pendingAntiAffineRequestCount;
+ public boolean isAARequestOutstanding;
+
public String failureMessage;
public List<String> containers;
@@ -59,16 +65,17 @@
public Map<String, Integer> buildStatistics() {
Map<String, Integer> stats = new HashMap<>();
stats.put(StatusKeys.STATISTICS_CONTAINERS_ACTIVE_REQUESTS, requested);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_ANTI_AFFINE_PENDING, pendingAntiAffineRequestCount);
stats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED, completed);
stats.put(StatusKeys.STATISTICS_CONTAINERS_DESIRED, desired);
stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED, failed);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently);
stats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE, actual);
+ stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted);
stats.put(StatusKeys.STATISTICS_CONTAINERS_REQUESTED, totalRequested);
stats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED, started);
stats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED, startFailed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_RECENTLY, failedRecently);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED_NODE, nodeFailed);
- stats.put(StatusKeys.STATISTICS_CONTAINERS_PREEMPTED, preempted);
return stats;
}
@@ -76,22 +83,24 @@
public String toString() {
final StringBuilder sb =
new StringBuilder("ComponentInformation{");
- sb.append("failureMessage='").append(failureMessage).append('\'');
- sb.append(", totalRequested=").append(totalRequested);
- sb.append(", completed=").append(completed);
- sb.append(", startFailed=").append(startFailed);
- sb.append(", started=").append(started);
- sb.append(", failed=").append(failed);
- sb.append(", requested=").append(requested);
- sb.append(", placementPolicy=").append(placementPolicy);
- sb.append(", releasing=").append(releasing);
- sb.append(", actual=").append(actual);
- sb.append(", desired=").append(desired);
- sb.append(", priority=").append(priority);
sb.append(", name='").append(name).append('\'');
+ sb.append(", actual=").append(actual);
+ sb.append(", completed=").append(completed);
+ sb.append(", desired=").append(desired);
+ sb.append(", failed=").append(failed);
+ sb.append(", failureMessage='").append(failureMessage).append('\'');
+ sb.append(", placementPolicy=").append(placementPolicy);
+ sb.append(", isAARequestOutstanding=").append(isAARequestOutstanding);
+ sb.append(", pendingAntiAffineRequestCount=").append(pendingAntiAffineRequestCount);
+ sb.append(", priority=").append(priority);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", requested=").append(requested);
+ sb.append(", started=").append(started);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", totalRequested=").append(totalRequested);
sb.append(", container count='")
- .append(containers== null ? 0: containers.size())
- .append('\'');
+ .append(containers == null ? 0 : containers.size())
+ .append('\'');
sb.append('}');
return sb.toString();
}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
new file mode 100644
index 0000000..8424be2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeEntryInformation.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+/**
+ * Serialized node entry information. Must be kept in sync with the protobuf equivalent.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class NodeEntryInformation {
+
+ /** incrementing counter of instances that failed */
+ public int failed;
+
+ /** Counter of "failed recently" events. */
+ public int failedRecently;
+
+ /** timestamp of last use */
+ public long lastUsed;
+
+ /** Number of live nodes. */
+ public int live;
+
+ /** incrementing counter of instances that have been pre-empted. */
+ public int preempted;
+
+ /** Priority */
+ public int priority;
+
+ /** instance explicitly requested on this node */
+ public int requested;
+
+ /** number of containers being released off this node */
+ public int releasing;
+
+ /** incrementing counter of instances that failed to start */
+ public int startFailed;
+
+ /** number of starting instances */
+ public int starting;
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "NodeEntryInformation{");
+ sb.append("priority=").append(priority);
+ sb.append(", live=").append(live);
+ sb.append(", requested=").append(requested);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", starting=").append(starting);
+ sb.append(", failed=").append(failed);
+ sb.append(", failedRecently=").append(failedRecently);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", preempted=").append(preempted);
+ sb.append(", lastUsed=").append(lastUsed);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
new file mode 100644
index 0000000..4fe5b4c
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformation.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Serialized node information. Must be kept in sync with the protobuf equivalent.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class NodeInformation {
+
+ public String hostname;
+ public String state;
+ public String labels;
+ public String rackName;
+ public String httpAddress;
+ public String healthReport;
+ public long lastUpdated;
+ public Map<String, NodeEntryInformation> entries = new HashMap<>();
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "NodeInformation{");
+ sb.append("hostname='").append(hostname).append('\'');
+ sb.append(", state='").append(state).append('\'');
+ sb.append(", labels='").append(labels).append('\'');
+ sb.append(", rackName='").append(rackName).append('\'');
+ sb.append(", httpAddress='").append(httpAddress).append('\'');
+ sb.append(", healthReport='").append(healthReport).append('\'');
+ sb.append(", lastUpdated=").append(lastUpdated);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
new file mode 100644
index 0000000..741523e
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/NodeInformationList.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.apache.slider.core.persist.JsonSerDeser;
+
+import java.util.ArrayList;
+import java.util.Collection;
+
+public class NodeInformationList extends ArrayList<NodeInformation> {
+ public NodeInformationList() {
+ }
+
+ public NodeInformationList(Collection<? extends NodeInformation> c) {
+ super(c);
+ }
+
+ public NodeInformationList(int initialCapacity) {
+ super(initialCapacity);
+ }
+
+ public static JsonSerDeser<NodeInformationList> createSerializer() {
+ return new JsonSerDeser<>(NodeInformationList.class);
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java b/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
new file mode 100644
index 0000000..c926600
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/api/types/RoleStatistics.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.api.types;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+
+/**
+ * Simple role statistics for state views; can be generated by RoleStatus
+ * instances, and aggregated for summary information.
+ */
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class RoleStatistics {
+ public long activeAA = 0L;
+ public long actual = 0L;
+ public long completed = 0L;
+ public long desired = 0L;
+ public long failed = 0L;
+ public long failedRecently = 0L;
+ public long limitsExceeded = 0L;
+ public long nodeFailed = 0L;
+ public long preempted = 0L;
+ public long releasing = 0L;
+ public long requested = 0L;
+ public long started = 0L;
+ public long startFailed = 0L;
+ public long totalRequested = 0L;
+
+ /**
+ * Add another statistics instance
+ * @param that the other value
+ * @return this entry
+ */
+ public RoleStatistics add(final RoleStatistics that) {
+ activeAA += that.activeAA;
+ actual += that.actual;
+ completed += that.completed;
+ desired += that.desired;
+ failed += that.failed;
+ failedRecently += that.failedRecently;
+ limitsExceeded += that.limitsExceeded;
+ nodeFailed += that.nodeFailed;
+ preempted += that.preempted;
+ releasing += that.releasing;
+ requested += that.requested;
+ started += that.started;
+ startFailed += that.totalRequested;
+ totalRequested += that.totalRequested;
+ return this;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index fe0e038..da94dd4 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -19,8 +19,8 @@
package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import com.google.common.io.Files;
+
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.ArrayUtils;
@@ -34,7 +34,6 @@
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
@@ -65,25 +64,25 @@
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.InternalKeys;
-import org.apache.slider.api.OptionKeys;
-import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.api.SliderClusterProtocol;
import org.apache.slider.api.StateValues;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.SliderInstanceDescription;
+import org.apache.slider.client.ipc.SliderApplicationIpcClient;
import org.apache.slider.client.ipc.SliderClusterOperations;
import org.apache.slider.common.Constants;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionCreateArgs;
import org.apache.slider.common.params.ActionDependencyArgs;
+import org.apache.slider.common.params.ActionDestroyArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionExistsArgs;
@@ -95,6 +94,7 @@
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
import org.apache.slider.common.params.ActionLookupArgs;
+import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionRegistryArgs;
import org.apache.slider.common.params.ActionResolveArgs;
@@ -105,7 +105,6 @@
import org.apache.slider.common.params.ClientArgs;
import org.apache.slider.common.params.CommonArgs;
import org.apache.slider.common.params.LaunchArgsAccessor;
-import org.apache.slider.common.params.SliderActions;
import org.apache.slider.common.tools.ConfigHelper;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -131,7 +130,6 @@
import org.apache.slider.core.exceptions.WaitTimeoutException;
import org.apache.slider.core.launch.AppMasterLauncher;
import org.apache.slider.core.launch.ClasspathConstructor;
-import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.JavaCommandLineBuilder;
import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
@@ -140,6 +138,7 @@
import org.apache.slider.core.persist.AppDefinitionPersister;
import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
+import org.apache.slider.core.persist.JsonSerDeser;
import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.registry.SliderRegistryUtils;
import org.apache.slider.core.registry.YarnAppListClient;
@@ -173,7 +172,6 @@
import java.io.BufferedReader;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -202,38 +200,13 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.ServiceRecordMarshal;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.extractServiceRecords;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.listServiceRecords;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.serviceclassPath;
-import static org.apache.hadoop.registry.client.binding.RegistryUtils.statChildren;
-import static org.apache.slider.common.params.SliderActions.ACTION_AM_SUICIDE;
-import static org.apache.slider.common.params.SliderActions.ACTION_BUILD;
-import static org.apache.slider.common.params.SliderActions.ACTION_CLIENT;
-import static org.apache.slider.common.params.SliderActions.ACTION_CREATE;
-import static org.apache.slider.common.params.SliderActions.ACTION_DEPENDENCY;
-import static org.apache.slider.common.params.SliderActions.ACTION_DESTROY;
-import static org.apache.slider.common.params.SliderActions.ACTION_DIAGNOSTICS;
-import static org.apache.slider.common.params.SliderActions.ACTION_EXISTS;
-import static org.apache.slider.common.params.SliderActions.ACTION_FLEX;
-import static org.apache.slider.common.params.SliderActions.ACTION_FREEZE;
-import static org.apache.slider.common.params.SliderActions.ACTION_HELP;
-import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_KEYTAB;
-import static org.apache.slider.common.params.SliderActions.ACTION_INSTALL_PACKAGE;
-import static org.apache.slider.common.params.SliderActions.ACTION_KEYTAB;
-import static org.apache.slider.common.params.SliderActions.ACTION_KILL_CONTAINER;
-import static org.apache.slider.common.params.SliderActions.ACTION_LIST;
-import static org.apache.slider.common.params.SliderActions.ACTION_LOOKUP;
-import static org.apache.slider.common.params.SliderActions.ACTION_PACKAGE;
-import static org.apache.slider.common.params.SliderActions.ACTION_REGISTRY;
-import static org.apache.slider.common.params.SliderActions.ACTION_RESOLVE;
-import static org.apache.slider.common.params.SliderActions.ACTION_STATUS;
-import static org.apache.slider.common.params.SliderActions.ACTION_THAW;
-import static org.apache.slider.common.params.SliderActions.ACTION_UPDATE;
-import static org.apache.slider.common.params.SliderActions.ACTION_UPGRADE;
-import static org.apache.slider.common.params.SliderActions.ACTION_VERSION;
+import static org.apache.hadoop.registry.client.binding.RegistryUtils.*;
+import static org.apache.slider.api.InternalKeys.*;
+import static org.apache.slider.api.OptionKeys.*;
+import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.common.params.SliderActions.*;
+import static org.apache.slider.common.tools.SliderUtils.*;
+
/**
* Client service for Slider
@@ -286,6 +259,7 @@
/**
* The YARN registry service
*/
+ @SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
private RegistryOperations registryOperations;
/**
@@ -316,19 +290,19 @@
ConfigHelper.injectSliderXMLResource();
// yarn-ify
YarnConfiguration yarnConfiguration = new YarnConfiguration(config);
- return SliderUtils.patchConfiguration(yarnConfiguration);
+ return patchConfiguration(yarnConfiguration);
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- Configuration clientConf = SliderUtils.loadSliderClientXML();
+ Configuration clientConf = loadSliderClientXML();
ConfigHelper.mergeConfigurations(conf, clientConf, SLIDER_CLIENT_XML, true);
serviceArgs.applyDefinitions(conf);
serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
- if (SliderUtils.isHadoopClusterSecure(conf)) {
- SliderUtils.forceLogin();
- SliderUtils.initProcessSecurity(conf);
+ if (isHadoopClusterSecure(conf)) {
+ forceLogin();
+ initProcessSecurity(conf);
}
AbstractActionArgs coreAction = serviceArgs.getCoreAction();
if (coreAction.getHadoopServicesRequired()) {
@@ -338,90 +312,6 @@
}
/**
- * this is where the work is done.
- * @return the exit code
- * @throws Throwable anything that went wrong
- */
-/* JDK7
-
- @Override
- public int runService() throws Throwable {
-
- // choose the action
- String action = serviceArgs.getAction();
- int exitCode = EXIT_SUCCESS;
- String clusterName = serviceArgs.getClusterName();
- // actions
- switch (action) {
- case ACTION_BUILD:
- exitCode = actionBuild(clusterName, serviceArgs.getActionBuildArgs());
- break;
- case ACTION_UPDATE:
- exitCode = actionUpdate(clusterName, serviceArgs.getActionUpdateArgs());
- break;
- case ACTION_UPGRADE:
- exitCode = actionUpgrade(clusterName, serviceArgs.getActionUpgradeArgs());
- break;
- case ACTION_CREATE:
- exitCode = actionCreate(clusterName, serviceArgs.getActionCreateArgs());
- break;
- case ACTION_FREEZE:
- exitCode = actionFreeze(clusterName, serviceArgs.getActionFreezeArgs());
- break;
- case ACTION_THAW:
- exitCode = actionThaw(clusterName, serviceArgs.getActionThawArgs());
- break;
- case ACTION_DESTROY:
- exitCode = actionDestroy(clusterName);
- break;
- case ACTION_EXISTS:
- exitCode = actionExists(clusterName,
- serviceArgs.getActionExistsArgs().live);
- break;
- case ACTION_FLEX:
- exitCode = actionFlex(clusterName, serviceArgs.getActionFlexArgs());
- break;
- case ACTION_GETCONF:
- exitCode =
- actionGetConf(clusterName, serviceArgs.getActionGetConfArgs());
- break;
- case ACTION_HELP:
- case ACTION_USAGE:
- log.info(serviceArgs.usage());
- break;
- case ACTION_KILL_CONTAINER:
- exitCode = actionKillContainer(clusterName,
- serviceArgs.getActionKillContainerArgs());
- break;
- case ACTION_AM_SUICIDE:
- exitCode = actionAmSuicide(clusterName,
- serviceArgs.getActionAMSuicideArgs());
- break;
- case ACTION_LIST:
- exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
- break;
- case ACTION_REGISTRY:
- exitCode = actionRegistry(
- serviceArgs.getActionRegistryArgs());
- break;
- case ACTION_STATUS:
- exitCode = actionStatus(clusterName,
- serviceArgs.getActionStatusArgs());
- break;
- case ACTION_VERSION:
- exitCode = actionVersion();
- break;
- default:
- throw new SliderException(EXIT_UNIMPLEMENTED,
- "Unimplemented: " + action);
- }
-
- return exitCode;
- }
-
-*/
-
- /**
* Launched service execution. This runs {@link #exec()}
* then catches some exceptions and converts them to exit codes
* @return an exit code
@@ -445,11 +335,10 @@
// choose the action
String action = serviceArgs.getAction();
- if (SliderUtils.isUnset(action)) {
- throw new SliderException(EXIT_USAGE,
- serviceArgs.usage());
+ if (isUnset(action)) {
+ throw new SliderException(EXIT_USAGE, serviceArgs.usage());
}
-
+
int exitCode = EXIT_SUCCESS;
String clusterName = serviceArgs.getClusterName();
// actions
@@ -477,7 +366,7 @@
break;
case ACTION_DESTROY:
- exitCode = actionDestroy(clusterName);
+ exitCode = actionDestroy(clusterName, serviceArgs.getActionDestroyArgs());
break;
case ACTION_DIAGNOSTICS:
@@ -507,8 +396,7 @@
break;
case ACTION_INSTALL_KEYTAB:
- exitCode =
- actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
+ exitCode = actionInstallKeytab(serviceArgs.getActionInstallKeytabArgs());
break;
case ACTION_INSTALL_PACKAGE:
@@ -522,11 +410,15 @@
case ACTION_LIST:
exitCode = actionList(clusterName, serviceArgs.getActionListArgs());
break;
-
+
case ACTION_LOOKUP:
exitCode = actionLookup(serviceArgs.getActionLookupArgs());
break;
+ case ACTION_NODES:
+ exitCode = actionNodes("", serviceArgs.getActionNodesArgs());
+ break;
+
case ACTION_PACKAGE:
exitCode = actionPackage(serviceArgs.getActionPackageArgs());
break;
@@ -567,7 +459,7 @@
return exitCode;
}
-/**
+ /**
* Perform everything needed to init the hadoop binding.
* This assumes that the service is already in inited or started state
* @throws IOException
@@ -575,7 +467,7 @@
*/
protected void initHadoopBinding() throws IOException, SliderException {
// validate the client
- SliderUtils.validateSliderClientEnvironment(null);
+ validateSliderClientEnvironment(null);
//create the YARN client
yarnClient = new SliderYarnClientImpl();
yarnClient.init(getConfig());
@@ -586,7 +478,7 @@
yarnAppListClient =
new YarnAppListClient(yarnClient, getUsername(), getConfig());
// create the filesystem
- sliderFileSystem = new SliderFileSystem(getConfig());
+ sliderFileSystem = new SliderFileSystem(getConfig());
}
/**
@@ -620,6 +512,13 @@
/**
* Create the zookeeper node associated with the calling user and the cluster
+ *
+ * @param clusterName slider application name
+ * @param nameOnly should the name only be created (i.e. don't create ZK node)
+ * @return the path, using the policy implemented in
+ * {@link ZKIntegration#mkClusterPath(String, String)}
+ * @throws YarnException
+ * @throws IOException
*/
@VisibleForTesting
public String createZookeeperNode(String clusterName, Boolean nameOnly) throws YarnException, IOException {
@@ -639,7 +538,8 @@
* -throwing exceptions on any failure
* @param clusterName cluster name
* @param nameOnly create the path, not the node
- * @return the path, with the node created
+ * @return the path, using the policy implemented in
+ * {@link ZKIntegration#mkClusterPath(String, String)}
* @throws YarnException
* @throws IOException
* @throws KeeperException
@@ -657,7 +557,7 @@
if (client != null) {
// set up the permissions. This must be done differently on a secure cluster from an insecure
// one
- List<ACL> zkperms = new ArrayList<ACL>();
+ List<ACL> zkperms = new ArrayList<>();
if (UserGroupInformation.isSecurityEnabled()) {
zkperms.add(new ACL(ZooDefs.Perms.ALL, ZooDefs.Ids.AUTH_IDS));
zkperms.add(new ACL(ZooDefs.Perms.READ, ZooDefs.Ids.ANYONE_ID_UNSAFE));
@@ -681,7 +581,8 @@
ZKIntegration client = null;
try {
BlockingZKWatcher watcher = new BlockingZKWatcher();
- client = ZKIntegration.newInstance(registryQuorum, user, clusterName, true, false, watcher);
+ client = ZKIntegration.newInstance(registryQuorum, user, clusterName, true, false, watcher,
+ ZKIntegration.SESSION_TIMEOUT);
client.init();
watcher.waitForZKConnection(2 * 1000);
} catch (InterruptedException e) {
@@ -693,14 +594,28 @@
return client;
}
+ /**
+ * Keep this signature for backward compatibility with
+ * force=true by default.
+ */
@Override
public int actionDestroy(String clustername) throws YarnException,
IOException {
+ ActionDestroyArgs destroyArgs = new ActionDestroyArgs();
+ destroyArgs.force = true;
+ return actionDestroy(clustername, destroyArgs);
+ }
+
+ @Override
+ public int actionDestroy(String clustername,
+ ActionDestroyArgs destroyArgs) throws YarnException, IOException {
// verify that a live cluster isn't there
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
//no=op, it is now mandatory.
verifyBindingsDefined();
verifyNoLiveClusters(clustername, "Destroy");
+ boolean forceDestroy = destroyArgs.force;
+ log.debug("actionDestroy({}, force={})", clustername, forceDestroy);
// create the directory path
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
@@ -709,9 +624,12 @@
boolean exists = fs.exists(clusterDirectory);
if (exists) {
log.debug("Application Instance {} found at {}: destroying", clustername, clusterDirectory);
- boolean deleted =
- fs.delete(clusterDirectory, true);
- if (!deleted) {
+ if (!forceDestroy) {
+ // fail the command if --force is not explicitly specified
+ throw new UsageException("Destroy will permanently delete directories and registries. "
+ + "Reissue this command with the --force option if you want to proceed.");
+ }
+ if (!fs.delete(clusterDirectory, true)) {
log.warn("Filesystem returned false from delete() operation");
}
@@ -807,33 +725,26 @@
// Otherwise the internal app config and resources states of the app will be
// unwantedly modified and the change will take effect to the running app
// immediately.
- if (template != null && resources == null) {
- throw new BadCommandArgumentsException(
+ require(!(template != null && resources == null),
"Option %s must be specified with option %s",
Arguments.ARG_RESOURCES, Arguments.ARG_TEMPLATE);
- }
- if (resources != null && template == null) {
- throw new BadCommandArgumentsException(
+
+ require(!(resources != null && template == null),
"Option %s must be specified with option %s",
Arguments.ARG_TEMPLATE, Arguments.ARG_RESOURCES);
- }
// For upgrade spec, both --template and --resources should be specified
// and neither of --containers or --components should be used
if (template != null && resources != null) {
- if (CollectionUtils.isNotEmpty(containers)) {
- throw new BadCommandArgumentsException(
+ require(CollectionUtils.isEmpty(containers),
"Option %s cannot be specified with %s or %s",
Arguments.ARG_CONTAINERS, Arguments.ARG_TEMPLATE,
Arguments.ARG_RESOURCES);
- }
- if (CollectionUtils.isNotEmpty(components)) {
- throw new BadCommandArgumentsException(
- "Option %s cannot be specified with %s or %s",
- Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE,
- Arguments.ARG_RESOURCES);
- }
-
+ require(CollectionUtils.isEmpty(components),
+ "Option %s cannot be specified with %s or %s",
+ Arguments.ARG_COMPONENTS, Arguments.ARG_TEMPLATE,
+ Arguments.ARG_RESOURCES);
+
// not an error to try to upgrade a stopped cluster, just return success
// code, appropriate log messages have already been dumped
if (!isAppInRunningState(clustername)) {
@@ -858,7 +769,7 @@
private int actionUpgradeContainers(String clustername,
ActionUpgradeArgs upgradeArgs) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
int waittime = upgradeArgs.getWaittime(); // ignored for now
String text = "Upgrade containers";
log.debug("actionUpgradeContainers({}, reason={}, wait={})", clustername,
@@ -956,13 +867,10 @@
return false;
}
log.debug("App to upgrade was found: {}:\n{}", clustername,
- new SliderUtils.OnDemandReportStringifier(app));
- if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED
- .ordinal()) {
- log.info(
- "Cluster {} is in a terminated state {}. Use command '{}' instead.",
- clustername, app.getYarnApplicationState(),
- SliderActions.ACTION_UPDATE);
+ new OnDemandReportStringifier(app));
+ if (app.getYarnApplicationState().ordinal() >= YarnApplicationState.FINISHED.ordinal()) {
+ log.info("Cluster {} is in a terminated state {}. Use command '{}' instead.",
+ clustername, app.getYarnApplicationState(), ACTION_UPDATE);
return false;
}
@@ -979,7 +887,7 @@
private static void checkForCredentials(Configuration conf,
ConfTree tree) throws IOException {
- if (tree.credentials == null || tree.credentials.size()==0) {
+ if (tree.credentials == null || tree.credentials.isEmpty()) {
log.info("No credentials requested");
return;
}
@@ -994,10 +902,8 @@
}
Configuration c = new Configuration(conf);
c.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, provider);
- CredentialProvider credentialProvider =
- CredentialProviderFactory.getProviders(c).get(0);
- Set<String> existingAliases =
- new HashSet<>(credentialProvider.getAliases());
+ CredentialProvider credentialProvider = CredentialProviderFactory.getProviders(c).get(0);
+ Set<String> existingAliases = new HashSet<>(credentialProvider.getAliases());
for (String alias : aliases) {
if (existingAliases.contains(alias.toLowerCase(Locale.ENGLISH))) {
log.info("Credentials for " + alias + " found in " + provider);
@@ -1013,21 +919,13 @@
}
}
} finally {
- if (br != null) {
- br.close();
- }
+ org.apache.hadoop.io.IOUtils.closeStream(br);
}
}
private static char[] readOnePassword(String alias) throws IOException {
- BufferedReader br = null;
- try {
- br = new BufferedReader(new InputStreamReader(System.in));
+ try(BufferedReader br = new BufferedReader(new InputStreamReader(System.in))) {
return readPassword(alias, br);
- } finally {
- if (br != null) {
- br.close();
- }
}
}
@@ -1113,12 +1011,9 @@
Path fileInFs = new Path(pkgPath, keytabInfo.keytab );
log.info("Deleting keytab {}", fileInFs);
- if (!sliderFileSystem.getFileSystem().exists(fileInFs)) {
- throw new BadCommandArgumentsException("No keytab to delete found at " +
- fileInFs.toUri().toString());
- }
-
- sliderFileSystem.getFileSystem().delete(fileInFs, false);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ require(sfs.exists(fileInFs), "No keytab to delete found at %s", fileInFs.toUri());
+ sfs.delete(fileInFs, false);
return EXIT_SUCCESS;
}
@@ -1126,40 +1021,31 @@
private int actionInstallKeytab(ActionKeytabArgs keytabInfo)
throws BadCommandArgumentsException, IOException {
Path srcFile = null;
- if (StringUtils.isEmpty(keytabInfo.folder)) {
- throw new BadCommandArgumentsException(
- "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
- + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- }
+ require(isSet(keytabInfo.folder),
+ "A valid destination keytab sub-folder name is required (e.g. 'security').\n"
+ + CommonArgs.usage(serviceArgs, ACTION_KEYTAB));
- if (StringUtils.isEmpty(keytabInfo.keytab)) {
- throw new BadCommandArgumentsException("A valid local keytab location is required.");
- } else {
- File keytabFile = new File(keytabInfo.keytab);
- if (!keytabFile.exists() || keytabFile.isDirectory()) {
- throw new BadCommandArgumentsException("Unable to access supplied keytab file at " +
- keytabFile.getAbsolutePath());
- } else {
- srcFile = new Path(keytabFile.toURI());
- }
- }
+ requireArgumentSet(Arguments.ARG_KEYTAB, keytabInfo.keytab);
+ File keytabFile = new File(keytabInfo.keytab);
+ require(keytabFile.isFile(),
+ "Unable to access supplied keytab file at %s", keytabFile.getAbsolutePath());
+ srcFile = new Path(keytabFile.toURI());
Path pkgPath = sliderFileSystem.buildKeytabInstallationDirPath(keytabInfo.folder);
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
- sliderFileSystem.getFileSystem().setPermission(pkgPath, new FsPermission(
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ sfs.mkdirs(pkgPath);
+ sfs.setPermission(pkgPath, new FsPermission(
FsAction.ALL, FsAction.NONE, FsAction.NONE));
Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing keytab {} at {} and overwrite is {}.", srcFile, fileInFs, keytabInfo.overwrite);
- if (sliderFileSystem.getFileSystem().exists(fileInFs) && !keytabInfo.overwrite) {
- throw new BadCommandArgumentsException("Keytab exists at " +
- fileInFs.toUri().toString() +
- ". Use --overwrite to overwrite.");
- }
+ log.info("Installing keytab {} at {} and overwrite is {}.",
+ srcFile, fileInFs, keytabInfo.overwrite);
+ require(!(sfs.exists(fileInFs) && !keytabInfo.overwrite),
+ "Keytab exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
- sliderFileSystem.getFileSystem().setPermission(fileInFs, new FsPermission(
- FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+ sfs.copyFromLocalFile(false, keytabInfo.overwrite, srcFile, fileInFs);
+ sfs.setPermission(fileInFs,
+ new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
return EXIT_SUCCESS;
}
@@ -1175,44 +1061,29 @@
public int actionInstallPkg(ActionInstallPackageArgs installPkgInfo) throws
YarnException,
IOException {
- log.warn("The " + SliderActions.ACTION_INSTALL_PACKAGE
+ log.warn("The " + ACTION_INSTALL_PACKAGE
+ " option has been deprecated. Please use '"
- + SliderActions.ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
- Path srcFile = null;
+ + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'.");
if (StringUtils.isEmpty(installPkgInfo.name)) {
throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_TYPE_NAME +"\n"
+ E_INVALID_APPLICATION_TYPE_NAME + "\n"
+ CommonArgs.usage(serviceArgs, ACTION_INSTALL_PACKAGE));
}
-
- if (StringUtils.isEmpty(installPkgInfo.packageURI)) {
- throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- File pkgFile = new File(installPkgInfo.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE +": "
- + pkgFile.getAbsolutePath());
- } else {
- srcFile = new Path(pkgFile.toURI());
- }
- }
+ Path srcFile = extractPackagePath(installPkgInfo.packageURI);
// Do not provide new options to install-package command as it is in
// deprecated mode. So version is kept null here. Use package --install.
Path pkgPath = sliderFileSystem.buildPackageDirPath(installPkgInfo.name,
null);
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ sfs.mkdirs(pkgPath);
Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing package {} at {} and overwrite is {}.", srcFile, fileInFs, installPkgInfo.replacePkg);
- if (sliderFileSystem.getFileSystem().exists(fileInFs) && !installPkgInfo.replacePkg) {
- throw new BadCommandArgumentsException(
- "Package exists at " + fileInFs.toUri().toString() +"."
- + E_USE_REPLACEPKG_TO_OVERWRITE);
- }
-
- sliderFileSystem.getFileSystem().copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
+ log.info("Installing package {} at {} and overwrite is {}.",
+ srcFile, fileInFs, installPkgInfo.replacePkg);
+ require(!(sfs.exists(fileInFs) && !installPkgInfo.replacePkg),
+ "Package exists at %s. : %s", fileInFs.toUri(), E_USE_REPLACEPKG_TO_OVERWRITE);
+ sfs.copyFromLocalFile(false, installPkgInfo.replacePkg, srcFile, fileInFs);
return EXIT_SUCCESS;
}
@@ -1241,11 +1112,7 @@
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
}
- if (clientInfo.name == null) {
- throw new BadCommandArgumentsException("No application name specified\n"
- + CommonArgs.usage(serviceArgs,
- ACTION_CLIENT));
- }
+ requireArgumentSet(Arguments.ARG_NAME, clientInfo.name);
File storeFile = null;
SecurityStore.StoreType type;
@@ -1257,11 +1124,9 @@
type = SecurityStore.StoreType.truststore;
}
- if (storeFile.exists()) {
- throw new BadCommandArgumentsException("File %s already exists. "
- + "Please remove that file or select a different file name.",
- storeFile.getAbsolutePath());
- }
+ require (!storeFile.exists(),
+ "File %s already exists. Please remove that file or select a different file name.",
+ storeFile.getAbsolutePath());
String hostname = null;
if (type == SecurityStore.StoreType.keystore) {
hostname = clientInfo.hostname;
@@ -1296,10 +1161,8 @@
}
}
- byte[]
- keystore =
- createClusterOperations(clientInfo.name).getClientCertificateStore(
- hostname, "client", password, type.name());
+ byte[] keystore = createClusterOperations(clientInfo.name)
+ .getClientCertificateStore(hostname, "client", password, type.name());
// persist to file
IOUtils.write(keystore, new FileOutputStream(storeFile));
@@ -1309,31 +1172,20 @@
private int doClientInstall(ActionClientArgs clientInfo)
throws IOException, SliderException {
- if (clientInfo.installLocation == null) {
- throw new BadCommandArgumentsException(
+ require(clientInfo.installLocation != null,
E_INVALID_INSTALL_LOCATION +"\n"
+ CommonArgs.usage(serviceArgs, ACTION_CLIENT));
- } else {
- if (!clientInfo.installLocation.exists()) {
- throw new BadCommandArgumentsException(E_INSTALL_PATH_DOES_NOT_EXIST
- +": " + clientInfo.installLocation.getAbsolutePath());
- }
- if (!clientInfo.installLocation.isDirectory()) {
- throw new BadCommandArgumentsException(E_INVALID_INSTALL_PATH
- +": " + clientInfo.installLocation.getAbsolutePath());
- }
- }
+ require(clientInfo.installLocation.exists(),
+ E_INSTALL_PATH_DOES_NOT_EXIST + ": " + clientInfo.installLocation.getAbsolutePath());
+
+ require(clientInfo.installLocation.isDirectory(),
+ E_INVALID_INSTALL_PATH + ": " + clientInfo.installLocation.getAbsolutePath());
File pkgFile;
- if (StringUtils.isEmpty(clientInfo.packageURI)) {
- throw new BadCommandArgumentsException(E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- pkgFile = new File(clientInfo.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- +" at " + pkgFile.getAbsolutePath());
- }
- }
+ require(isSet(clientInfo.packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ pkgFile = new File(clientInfo.packageURI);
+ require(pkgFile.isFile(),
+ E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + " at %s", pkgFile.getAbsolutePath());
JSONObject config = null;
if(clientInfo.clientConfig != null) {
@@ -1432,7 +1284,7 @@
Path appDefPath = null;
try {
appDefPath = new Path(
- SliderUtils.getApplicationDefinitionPath(instanceDefinition
+ getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations()));
} catch (BadConfigException e) {
// Invalid cluster state, so move on to next. No need to log anything
@@ -1454,11 +1306,8 @@
println("%-25s %15s %30s %s", clusterName, packageName,
packageVersion, appDefPathStr);
}
- } catch(IOException e) {
- if(log.isDebugEnabled()) {
- log.debug(clusterName + " application definition path "
- + appDefPathStr + " is not found.");
- }
+ } catch (IOException e) {
+ log.debug("{} application definition path {} is not found.", clusterName, appDefPathStr);
}
}
return EXIT_SUCCESS;
@@ -1467,19 +1316,19 @@
private int actionPackageList() throws IOException {
Path pkgPath = sliderFileSystem.buildPackageDirPath(StringUtils.EMPTY,
StringUtils.EMPTY);
- log.info("Package install path : " + pkgPath);
- if (!sliderFileSystem.getFileSystem().isDirectory(pkgPath)) {
+ log.info("Package install path : {}", pkgPath);
+ FileSystem sfs = sliderFileSystem.getFileSystem();
+ if (!sfs.isDirectory(pkgPath)) {
log.info("No package(s) installed");
return EXIT_SUCCESS;
}
- FileStatus[] fileStatus = sliderFileSystem.getFileSystem().listStatus(
- pkgPath);
+ FileStatus[] fileStatus = sfs.listStatus(pkgPath);
boolean hasPackage = false;
StringBuilder sb = new StringBuilder();
sb.append("List of installed packages:\n");
for (FileStatus fstat : fileStatus) {
if (fstat.isDirectory()) {
- sb.append("\t" + fstat.getPath().getName());
+ sb.append("\t").append(fstat.getPath().getName());
sb.append("\n");
hasPackage = true;
}
@@ -1492,53 +1341,31 @@
return EXIT_SUCCESS;
}
- private int actionPackageInstall(ActionPackageArgs actionPackageArgs) throws
- YarnException,
- IOException {
+ private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
+ throws YarnException, IOException {
+ requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
- Path srcFile = null;
- if (StringUtils.isEmpty(actionPackageArgs.name)) {
- throw new BadCommandArgumentsException(
- "A valid application type name is required (e.g. HBASE).\n"
- + CommonArgs.usage(serviceArgs, ACTION_PACKAGE));
- }
-
- if (StringUtils.isEmpty(actionPackageArgs.packageURI)) {
- throw new BadCommandArgumentsException(
- E_INVALID_APPLICATION_PACKAGE_LOCATION);
- } else {
- File pkgFile = new File(actionPackageArgs.packageURI);
- if (!pkgFile.exists() || pkgFile.isDirectory()) {
- throw new BadCommandArgumentsException(
- E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE
- + ": " + pkgFile.getAbsolutePath());
- } else {
- srcFile = new Path(pkgFile.toURI());
- }
- }
+ Path srcFile = extractPackagePath(actionPackageArgs.packageURI);
Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
actionPackageArgs.version);
- if (!sliderFileSystem.getFileSystem().exists(pkgPath)) {
- sliderFileSystem.getFileSystem().mkdirs(pkgPath);
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ if (!fs.exists(pkgPath)) {
+ fs.mkdirs(pkgPath);
}
Path fileInFs = new Path(pkgPath, srcFile.getName());
- if (sliderFileSystem.getFileSystem().exists(fileInFs)
- && !actionPackageArgs.replacePkg) {
- throw new BadCommandArgumentsException(E_PACKAGE_EXISTS +" at " +
- fileInFs.toUri() + ". Use --replacepkg to overwrite.");
- }
+ require(actionPackageArgs.replacePkg || !fs.exists(fileInFs),
+ E_PACKAGE_EXISTS +" at %s. Use --replacepkg to overwrite.", fileInFs.toUri());
log.info("Installing package {} to {} (overwrite set to {})", srcFile,
fileInFs, actionPackageArgs.replacePkg);
- sliderFileSystem.getFileSystem().copyFromLocalFile(false,
- actionPackageArgs.replacePkg, srcFile, fileInFs);
+ fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
String destPathWithHomeDir = Path
.getPathWithoutSchemeAndAuthority(fileInFs).toString();
String destHomeDir = Path.getPathWithoutSchemeAndAuthority(
- sliderFileSystem.getFileSystem().getHomeDirectory()).toString();
+ fs.getHomeDirectory()).toString();
// a somewhat contrived approach to stripping out the home directory and any trailing
// separator; designed to work on windows and unix
String destPathWithoutHomeDir;
@@ -1556,23 +1383,26 @@
return EXIT_SUCCESS;
}
+ private Path extractPackagePath(String packageURI)
+ throws BadCommandArgumentsException {
+ require(isSet(packageURI), E_INVALID_APPLICATION_PACKAGE_LOCATION);
+ File pkgFile = new File(packageURI);
+ require(pkgFile.isFile(),
+ E_UNABLE_TO_READ_SUPPLIED_PACKAGE_FILE + ": " + pkgFile.getAbsolutePath());
+ return new Path(pkgFile.toURI());
+ }
+
private int actionPackageDelete(ActionPackageArgs actionPackageArgs) throws
YarnException, IOException {
- if (StringUtils.isEmpty(actionPackageArgs.name)) {
- throw new BadCommandArgumentsException(
- "A valid application type name is required (e.g. HBASE).\n"
- + CommonArgs.usage(serviceArgs, ACTION_PACKAGE));
- }
+ requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
Path pkgPath = sliderFileSystem.buildPackageDirPath(actionPackageArgs.name,
actionPackageArgs.version);
- if (!sliderFileSystem.getFileSystem().exists(pkgPath)) {
- throw new BadCommandArgumentsException(E_PACKAGE_DOES_NOT_EXIST +": "
- + pkgPath.toUri().toString());
- }
+ FileSystem fs = sliderFileSystem.getFileSystem();
+ require(fs.exists(pkgPath), E_PACKAGE_DOES_NOT_EXIST +": %s ", pkgPath.toUri());
log.info("Deleting package {} at {}.", actionPackageArgs.name, pkgPath);
- if(sliderFileSystem.getFileSystem().delete(pkgPath, true)) {
+ if(fs.delete(pkgPath, true)) {
log.info("Deleted package {} " + actionPackageArgs.name);
return EXIT_SUCCESS;
} else {
@@ -1612,7 +1442,7 @@
boolean liveClusterAllowed, boolean isUpgradeFlow) throws YarnException,
IOException {
// verify that a live cluster isn't there
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyBindingsDefined();
if (!liveClusterAllowed) {
verifyNoLiveClusters(clustername, "Create");
@@ -1691,8 +1521,7 @@
String count = roleEntry.getValue();
String key = roleEntry.getKey();
log.info("{} => {}", key, count);
- resources.getOrAddComponent(key)
- .put(ResourceKeys.COMPONENT_INSTANCES, count);
+ resources.getOrAddComponent(key).put(COMPONENT_INSTANCES, count);
}
//all CLI role options
@@ -1727,7 +1556,7 @@
builder.setQueue(buildInfo.queue);
String quorum = buildInfo.getZKhosts();
- if (SliderUtils.isUnset(quorum)) {
+ if (isUnset(quorum)) {
quorum = registryQuorum;
}
if (isUnset(quorum)) {
@@ -1799,7 +1628,7 @@
if (!SliderKeys.COMPONENT_AM.equals(componentName)) {
clientComponentInstances.put(componentName, clientResources
.getComponentOptInt(componentName,
- ResourceKeys.COMPONENT_INSTANCES, -1));
+ COMPONENT_INSTANCES, -1));
}
}
@@ -1809,8 +1638,7 @@
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on cluster resource : {}", e, e);
throw new BadClusterStateException(
- "Failed to load client resource definition " + clustername + ": "
- + e);
+ "Failed to load client resource definition " + clustername + ": " + e, e);
}
Map<String, Integer> clusterComponentInstances = new HashMap<>();
for (Map.Entry<String, Map<String, String>> component : clusterConf
@@ -1819,7 +1647,7 @@
clusterComponentInstances.put(
component.getKey(),
Integer.decode(component.getValue().get(
- ResourceKeys.COMPONENT_INSTANCES)));
+ COMPONENT_INSTANCES)));
}
}
@@ -1827,16 +1655,13 @@
Iterator<Map.Entry<String, Integer>> clientComponentInstanceIt = clientComponentInstances
.entrySet().iterator();
while (clientComponentInstanceIt.hasNext()) {
- Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt
- .next();
- if (clusterComponentInstances
- .containsKey(clientComponentInstanceEntry.getKey())) {
+ Map.Entry<String, Integer> clientComponentInstanceEntry = clientComponentInstanceIt.next();
+ if (clusterComponentInstances.containsKey(clientComponentInstanceEntry.getKey())) {
// compare instance count now and remove from both maps if they match
if (clusterComponentInstances
.get(clientComponentInstanceEntry.getKey()) == clientComponentInstanceEntry
.getValue()) {
- clusterComponentInstances
- .remove(clientComponentInstanceEntry.getKey());
+ clusterComponentInstances.remove(clientComponentInstanceEntry.getKey());
clientComponentInstanceIt.remove();
}
}
@@ -1910,8 +1735,7 @@
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
String clusterDirPermsOct =
- conf.get(CLUSTER_DIRECTORY_PERMISSIONS,
- DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
+ conf.get(CLUSTER_DIRECTORY_PERMISSIONS, DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS);
return new FsPermission(clusterDirPermsOct);
}
@@ -1921,9 +1745,9 @@
* @throws BadCommandArgumentsException the exception raised on an invalid config
*/
public void verifyBindingsDefined() throws BadCommandArgumentsException {
- InetSocketAddress rmAddr = SliderUtils.getRmAddress(getConfig());
+ InetSocketAddress rmAddr = getRmAddress(getConfig());
if (!getConfig().getBoolean(YarnConfiguration.RM_HA_ENABLED, false)
- && !SliderUtils.isAddressDefined(rmAddr)) {
+ && !isAddressDefined(rmAddr)) {
throw new BadCommandArgumentsException(
E_NO_RESOURCE_MANAGER
+ " in the argument "
@@ -1987,9 +1811,7 @@
* @throws UnknownApplicationInstanceException if the file is not found
*/
public AggregateConf loadInstanceDefinitionUnresolved(String name,
- Path clusterDirectory) throws
- IOException,
- SliderException {
+ Path clusterDirectory) throws IOException, SliderException {
try {
AggregateConf definition =
@@ -2001,7 +1823,8 @@
throw UnknownApplicationInstanceException.unknownInstance(name, e);
}
}
- /**
+
+ /**
* Load the instance definition.
* @param name cluster name
* @param resolved flag to indicate the cluster should be resolved
@@ -2044,38 +1867,32 @@
deployedClusterName = clustername;
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyNoLiveClusters(clustername, "Launch");
Configuration config = getConfig();
lookupZKQuorum();
- boolean clusterSecure = SliderUtils.isHadoopClusterSecure(config);
+ boolean clusterSecure = isHadoopClusterSecure(config);
//create the Slider AM provider -this helps set up the AM
SliderAMClientProvider sliderAM = new SliderAMClientProvider(config);
instanceDefinition.resolve();
launchedInstanceDefinition = instanceDefinition;
- ConfTreeOperations internalOperations =
- instanceDefinition.getInternalOperations();
+ ConfTreeOperations internalOperations = instanceDefinition.getInternalOperations();
MapOperations internalOptions = internalOperations.getGlobalOptions();
- ConfTreeOperations resourceOperations =
- instanceDefinition.getResourceOperations();
- ConfTreeOperations appOperations =
- instanceDefinition.getAppConfOperations();
+ ConfTreeOperations resourceOperations = instanceDefinition.getResourceOperations();
+ ConfTreeOperations appOperations = instanceDefinition.getAppConfOperations();
Path generatedConfDirPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_GENERATED_CONF_PATH));
+ INTERNAL_GENERATED_CONF_PATH));
Path snapshotConfPath =
createPathThatMustExist(internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_SNAPSHOT_CONF_PATH));
+ INTERNAL_SNAPSHOT_CONF_PATH));
// cluster Provider
AbstractClientProvider provider = createClientProvider(
- internalOptions.getMandatoryOption(
- InternalKeys.INTERNAL_PROVIDER_NAME));
- // make sure the conf dir is valid;
-
+ internalOptions.getMandatoryOption(INTERNAL_PROVIDER_NAME));
if (log.isDebugEnabled()) {
log.debug(instanceDefinition.toString());
}
@@ -2085,7 +1902,7 @@
// add the tags if available
Set<String> applicationTags = provider.getApplicationTags(sliderFileSystem,
- SliderUtils.getApplicationDefinitionPath(appOperations));
+ getApplicationDefinitionPath(appOperations));
AppMasterLauncher amLauncher = new AppMasterLauncher(clustername,
SliderKeys.APP_TYPE,
config,
@@ -2110,8 +1927,7 @@
String libdir = "lib";
Path libPath = new Path(tempPath, libdir);
sliderFileSystem.getFileSystem().mkdirs(libPath);
- log.debug("FS={}, tempPath={}, libdir={}",
- sliderFileSystem, tempPath, libPath);
+ log.debug("FS={}, tempPath={}, libdir={}", sliderFileSystem, tempPath, libPath);
// set local resources for the application master
// local files or archives as needed
@@ -2122,9 +1938,8 @@
boolean hasServerLog4jProperties = false;
Path remoteConfPath = null;
String relativeConfDir = null;
- String confdirProp =
- System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
- if (confdirProp == null || confdirProp.isEmpty()) {
+ String confdirProp = System.getProperty(SliderKeys.PROPERTY_CONF_DIR);
+ if (isUnset(confdirProp)) {
log.debug("No local configuration directory provided as system property");
} else {
File confDir = new File(confdirProp);
@@ -2132,11 +1947,11 @@
throw new BadConfigException(E_CONFIGURATION_DIRECTORY_NOT_FOUND,
confDir);
}
- Path localConfDirPath = SliderUtils.createLocalPath(confDir);
+ Path localConfDirPath = createLocalPath(confDir);
remoteConfPath = new Path(clusterDirectory, SliderKeys.SUBMITTED_CONF_DIR);
log.debug("Slider configuration directory is {}; remote to be {}",
localConfDirPath, remoteConfPath);
- SliderUtils.copyDirectory(config, localConfDirPath, remoteConfPath, null);
+ copyDirectory(config, localConfDirPath, remoteConfPath, null);
File log4jserver =
new File(confDir, SliderKeys.LOG4J_SERVER_PROP_FILENAME);
@@ -2156,7 +1971,7 @@
Map<String, LocalResource> submittedConfDir =
sliderFileSystem.submitDirectory(remoteConfPath,
relativeConfDir);
- SliderUtils.mergeMaps(localResources, submittedConfDir);
+ mergeMaps(localResources, submittedConfDir);
}
}
// build up the configuration
@@ -2176,7 +1991,7 @@
Configuration clientConfExtras = new Configuration(false);
// then build up the generated path.
FsPermission clusterPerms = getClusterDirectoryPermissions(config);
- SliderUtils.copyDirectory(config, snapshotConfPath, generatedConfDirPath,
+ copyDirectory(config, snapshotConfPath, generatedConfDirPath,
clusterPerms);
@@ -2229,15 +2044,15 @@
// TODO: consider supporting apps that don't have an image path
Path imagePath =
- SliderUtils.extractImagePath(sliderFileSystem, internalOptions);
+ extractImagePath(sliderFileSystem, internalOptions);
if (sliderFileSystem.maybeAddImagePath(localResources, imagePath)) {
log.debug("Registered image path {}", imagePath);
}
// build the environment
amLauncher.putEnv(
- SliderUtils.buildEnvMap(sliderAMResourceComponent));
- ClasspathConstructor classpath = SliderUtils.buildClasspath(relativeConfDir,
+ buildEnvMap(sliderAMResourceComponent));
+ ClasspathConstructor classpath = buildClasspath(relativeConfDir,
libdir,
getConfig(),
sliderFileSystem,
@@ -2249,13 +2064,13 @@
amLauncher.setEnv("LANGUAGE", "en_US.UTF-8");
amLauncher.putEnv(getAmLaunchEnv(config));
- for (Map.Entry<String, String> envs : SliderUtils.getSystemEnv().entrySet()) {
+ for (Map.Entry<String, String> envs : getSystemEnv().entrySet()) {
log.debug("System env {}={}", envs.getKey(), envs.getValue());
}
if (log.isDebugEnabled()) {
log.debug("AM classpath={}", classpath);
log.debug("Environment Map:\n{}",
- SliderUtils.stringifyMap(amLauncher.getEnv()));
+ stringifyMap(amLauncher.getEnv()));
log.debug("Files in lib path\n{}", sliderFileSystem.listFSDir(libPath));
}
@@ -2263,14 +2078,11 @@
InetSocketAddress rmSchedulerAddress;
try {
- rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(config);
+ rmSchedulerAddress = getRmSchedulerAddress(config);
} catch (IllegalArgumentException e) {
throw new BadConfigException("%s Address invalid: %s",
- YarnConfiguration.RM_SCHEDULER_ADDRESS,
- config.get(
- YarnConfiguration.RM_SCHEDULER_ADDRESS)
- );
-
+ YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ config.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
}
String rmAddr = NetUtils.getHostPortString(rmSchedulerAddress);
@@ -2308,25 +2120,25 @@
commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
}
- /**
- * pass the registry binding
- */
- addConfOptionToCLI(commandLine, config,
- RegistryConstants.KEY_REGISTRY_ZK_ROOT,
+ // pass the registry binding
+ commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
- addMandatoryConfOptionToCLI(commandLine, config,
- RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
+ commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
if (clusterSecure) {
// if the cluster is secure, make sure that
// the relevant security settings go over
-/*
- addConfOptionToCLI(commandLine, config, KEY_SECURITY);
-*/
- addConfOptionToCLI(commandLine,
- config,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
+
+ // copy over any/all YARN RM client values, in case the server-side XML conf file
+ // has the 0.0.0.0 address
+ commandLine.addConfOptions(config,
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.RM_HOSTNAME,
+ YarnConfiguration.RM_PRINCIPAL);
+
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -2341,23 +2153,19 @@
// Set the priority for the application master
-
- int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY,
- DEFAULT_YARN_QUEUE_PRIORITY);
-
-
- amLauncher.setPriority(amPriority);
+ amLauncher.setPriority(config.getInt(KEY_YARN_QUEUE_PRIORITY,
+ DEFAULT_YARN_QUEUE_PRIORITY));
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
String amQueue = config.get(KEY_YARN_QUEUE, DEFAULT_YARN_QUEUE);
- String suppliedQueue = internalOperations.getGlobalOptions().get(InternalKeys.INTERNAL_QUEUE);
- if(!SliderUtils.isUnset(suppliedQueue)) {
+ String suppliedQueue = internalOperations.getGlobalOptions().get(INTERNAL_QUEUE);
+ if(!isUnset(suppliedQueue)) {
amQueue = suppliedQueue;
log.info("Using queue {} for the application instance.", amQueue);
}
- if (amQueue != null) {
+ if (isSet(amQueue)) {
amLauncher.setQueue(amQueue);
}
@@ -2367,15 +2175,15 @@
}
protected Map<String, String> getAmLaunchEnv(Configuration config) {
- String sliderAmLaunchEnv = config.get(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV);
- log.debug("{} = {}", SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
+ String sliderAmLaunchEnv = config.get(KEY_AM_LAUNCH_ENV);
+ log.debug("{} = {}", KEY_AM_LAUNCH_ENV, sliderAmLaunchEnv);
// Multiple env variables can be specified with a comma (,) separator
String[] envs = StringUtils.isEmpty(sliderAmLaunchEnv) ? null
: sliderAmLaunchEnv.split(",");
if (ArrayUtils.isEmpty(envs)) {
return Collections.emptyMap();
}
- Map<String, String> amLaunchEnv = new HashMap<String, String>();
+ Map<String, String> amLaunchEnv = new HashMap<>();
for (String env : envs) {
if (StringUtils.isNotEmpty(env)) {
// Each env name/value is separated by equals sign (=)
@@ -2411,9 +2219,9 @@
if (placeholderMatcher.find()) {
String placeholderKey = placeholderMatcher.group();
String systemKey = placeholderKey
- .substring(2, placeholderKey.length() - 1).toUpperCase()
+ .substring(2, placeholderKey.length() - 1).toUpperCase(Locale.ENGLISH)
.replaceAll("\\.", "_");
- String placeholderValue = SliderUtils.getSystemEnv(systemKey);
+ String placeholderValue = getSystemEnv(systemKey);
log.debug("Placeholder {}={}", placeholderKey, placeholderValue);
placeholderKeyValueMap.put(placeholderKey, placeholderValue);
}
@@ -2423,10 +2231,10 @@
private void propagatePythonExecutable(Configuration config,
AggregateConf instanceDefinition) {
String pythonExec = config.get(
- SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH);
+ PYTHON_EXECUTABLE_PATH);
if (pythonExec != null) {
instanceDefinition.getAppConfOperations().getGlobalOptions().putIfUnset(
- SliderXmlConfKeys.PYTHON_EXECUTABLE_PATH,
+ PYTHON_EXECUTABLE_PATH,
pythonExec);
}
}
@@ -2456,7 +2264,7 @@
new Duration(acceptWaitMillis));
// may have failed, so check that
- if (SliderUtils.hasAppFinished(report)) {
+ if (hasAppFinished(report)) {
exitCode = buildExitCode(report);
} else {
// exit unless there is a wait
@@ -2489,71 +2297,15 @@
*/
private void propagatePrincipals(Configuration config,
AggregateConf clusterSpec) {
- String dfsPrincipal = config.get(
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
- String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+ String siteDfsPrincipal = SITE_XML_PREFIX + DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
}
}
-
- private boolean addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key) {
- String val = conf.get(key);
- return defineIfSet(cmdLine, key, val);
- }
-
- private String addConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key,
- String defVal) {
- String val = conf.get(key, defVal);
- define(cmdLine, key, val);
- return val;
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI
- * @param cmdLine command line
- * @param key key
- * @param val value
- */
- private void define(CommandLineBuilder cmdLine, String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- Preconditions.checkArgument(val != null, "null value");
- cmdLine.add(Arguments.ARG_DEFINE, key + "=" + val);
- }
-
- /**
- * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
- * is not null
- * @param cmdLine command line
- * @param key key
- * @param val value
- */
- private boolean defineIfSet(CommandLineBuilder cmdLine, String key, String val) {
- Preconditions.checkArgument(key != null, "null key");
- if (val != null) {
- define(cmdLine, key, val);
- return true;
- } else {
- return false;
- }
- }
-
- private void addMandatoryConfOptionToCLI(CommandLineBuilder cmdLine,
- Configuration conf,
- String key) throws BadConfigException {
- if (!addConfOptionToCLI(cmdLine, conf, key)) {
- throw new BadConfigException("Missing configuration option: " + key);
- }
- }
-
/**
* Create a path that must exist in the cluster fs
* @param uri uri to create
@@ -2561,8 +2313,7 @@
* @throws FileNotFoundException if the path does not exist
*/
public Path createPathThatMustExist(String uri) throws
- SliderException,
- IOException {
+ SliderException, IOException {
return sliderFileSystem.createPathThatMustExist(uri);
}
@@ -2756,8 +2507,49 @@
@Override
public int actionList(String clustername, ActionListArgs args)
throws IOException, YarnException {
+ Set<String> appInstances = getApplicationList(clustername, args);
+ // getApplicationList never returns null
+ return !appInstances.isEmpty() ? EXIT_SUCCESS
+ : ((appInstances.isEmpty() && isUnset(clustername)) ? EXIT_SUCCESS
+ : EXIT_FALSE);
+ }
+
+ /**
+ * Retrieve a list of all live instances. If clustername is supplied then it
+ * returns this specific cluster, if and only if it exists and is live.
+ *
+ * @param clustername
+ * cluster name (if looking for a specific live cluster)
+ * @return the list of application names which satisfies the list criteria
+ * @throws IOException
+ * @throws YarnException
+ */
+ public Set<String> getApplicationList(String clustername) throws IOException,
+ YarnException {
+ ActionListArgs args = new ActionListArgs();
+ args.live = true;
+ return getApplicationList(clustername, args);
+ }
+
+ /**
+ * Retrieve a list of application instances satisfying the query criteria.
+ *
+ * @param clustername
+ * List out specific instance name (set null for all)
+ * @param args
+ * Action list arguments
+ * @return the list of application names which satisfies the list criteria
+ * @throws IOException
+ * @throws YarnException
+ * @throws UnknownApplicationInstanceException
+ * if a specific instance was named but it was not found
+ */
+ public Set<String> getApplicationList(String clustername, ActionListArgs args)
+ throws IOException, YarnException {
if (args.help) {
- return actionHelp(ACTION_LIST);
+ actionHelp(ACTION_LIST);
+ // the above call throws an exception so the return is not really required
+ return Collections.emptySet();
}
verifyBindingsDefined();
@@ -2807,14 +2599,14 @@
if (persistentInstances.isEmpty() && isUnset(clustername)) {
// an empty listing is a success if no cluster was named
log.debug("No application instances found");
- return EXIT_SUCCESS;
+ return Collections.emptySet();
}
// and those the RM knows about
List<ApplicationReport> instances = listSliderInstances(null);
- SliderUtils.sortApplicationsByMostRecent(instances);
+ sortApplicationsByMostRecent(instances);
Map<String, ApplicationReport> reportMap =
- SliderUtils.buildApplicationReportMap(instances, min, max);
+ buildApplicationReportMap(instances, min, max);
log.debug("Persisted {} deployed {} filtered[{}-{}] & de-duped to {}",
persistentInstances.size(),
instances.size(),
@@ -2839,22 +2631,21 @@
}
// at this point there is either the entire list or a stripped down instance
- int listed = 0;
-
+ Set<String> listedInstances = new HashSet<String>();
for (String name : persistentInstances.keySet()) {
ApplicationReport report = reportMap.get(name);
if (!listOnlyInState || report != null) {
// list the details if all were requested, or the filtering contained
// a report
- listed++;
+ listedInstances.add(name);
// containers will be non-null when only one instance is requested
- String details = SliderUtils.instanceDetailsToString(name, report,
+ String details = instanceDetailsToString(name, report,
containers, version, components, verbose);
print(details);
}
}
- return listed > 0 ? EXIT_SUCCESS: EXIT_FALSE;
+ return listedInstances;
}
public List<ContainerInformation> getContainers(String name)
@@ -2930,11 +2721,15 @@
@Override
@VisibleForTesting
public int actionFlex(String name, ActionFlexArgs args) throws YarnException, IOException {
+ validateClusterName(name);
+ Map<String, String> roleMap = args.getComponentMap();
+ // throw usage exception if no changes proposed
+ if (roleMap.size() == 0) {
+ actionHelp(ACTION_FLEX);
+ }
verifyBindingsDefined();
- SliderUtils.validateClusterName(name);
log.debug("actionFlex({})", name);
Map<String, Integer> roleInstances = new HashMap<>();
- Map<String, String> roleMap = args.getComponentMap();
for (Map.Entry<String, String> roleEntry : roleMap.entrySet()) {
String key = roleEntry.getKey();
String val = roleEntry.getValue();
@@ -2958,7 +2753,7 @@
public int actionExists(String name, ActionExistsArgs args) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(name);
+ validateClusterName(name);
boolean checkLive = args.live;
log.debug("actionExists({}, {}, {})", name, checkLive, args.state);
@@ -2968,7 +2763,7 @@
throw unknownClusterException(name);
}
String state = args.state;
- if (!checkLive && SliderUtils.isUnset(state)) {
+ if (!checkLive && isUnset(state)) {
log.info("Application {} exists", name);
return EXIT_SUCCESS;
}
@@ -3002,8 +2797,8 @@
}
}
- SliderUtils.OnDemandReportStringifier report =
- new SliderUtils.OnDemandReportStringifier(instance);
+ OnDemandReportStringifier report =
+ new OnDemandReportStringifier(instance);
if (!inDesiredState) {
//cluster in the list of apps but not running
log.info("Application {} found but is in wrong state {}", name,
@@ -3023,7 +2818,7 @@
public int actionKillContainer(String name,
ActionKillContainerArgs args) throws YarnException, IOException {
String id = args.id;
- if (SliderUtils.isUnset(id)) {
+ if (isUnset(id)) {
throw new BadCommandArgumentsException("Missing container id");
}
log.info("killingContainer {}:{}", name, id);
@@ -3119,7 +2914,7 @@
YarnException,
IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
String outfile = statusArgs.getOutput();
ClusterDescription status = getClusterDescription(clustername);
String text = status.toJsonString();
@@ -3141,7 +2936,7 @@
public int actionFreeze(String clustername,
ActionFreezeArgs freezeArgs) throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
int waittime = freezeArgs.getWaittime();
String text = freezeArgs.message;
boolean forcekill = freezeArgs.force;
@@ -3160,7 +2955,7 @@
return EXIT_SUCCESS;
}
log.debug("App to stop was found: {}:\n{}", clustername,
- new SliderUtils.OnDemandReportStringifier(app));
+ new OnDemandReportStringifier(app));
if (app.getYarnApplicationState().ordinal() >=
YarnApplicationState.FINISHED.ordinal()) {
log.info("Cluster {} is in a terminated state {}", clustername,
@@ -3227,7 +3022,7 @@
@Override
public int actionThaw(String clustername, ActionThawArgs thaw) throws YarnException, IOException {
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
verifyBindingsDefined();
// see if it is actually running and bail out;
verifyNoLiveClusters(clustername, "Start");
@@ -3247,7 +3042,7 @@
public int flex(String clustername, Map<String, Integer> roleInstances)
throws YarnException, IOException {
verifyBindingsDefined();
- SliderUtils.validateClusterName(clustername);
+ validateClusterName(clustername);
Path clusterDirectory = sliderFileSystem.buildClusterDirPath(clustername);
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername,
@@ -3258,7 +3053,7 @@
for (Map.Entry<String, Integer> entry : roleInstances.entrySet()) {
String role = entry.getKey();
int count = entry.getValue();
- resources.getOrAddComponent(role).put(ResourceKeys.COMPONENT_INSTANCES,
+ resources.getOrAddComponent(role).put(COMPONENT_INSTANCES,
Integer.toString(count));
log.debug("Flexed cluster specification ( {} -> {}) : \n{}",
@@ -3269,7 +3064,7 @@
SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
AbstractClientProvider provider = createClientProvider(
instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption(
- InternalKeys.INTERNAL_PROVIDER_NAME));
+ INTERNAL_PROVIDER_NAME));
// slider provider to validate what there is
validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem);
validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem);
@@ -3580,7 +3375,7 @@
} else {
String filename = RegistryPathUtils.lastPathEntry(name) + ".json";
File jsonFile = new File(destDir, filename);
- SliderUtils.write(jsonFile,
+ write(jsonFile,
serviceRecordMarshal.toBytes(instance),
true);
}
@@ -3593,7 +3388,7 @@
outFile = new File(args.destdir, RegistryPathUtils.lastPathEntry(path));
}
if (outFile != null) {
- SliderUtils.write(outFile, serviceRecordMarshal.toBytes(instance), true);
+ write(outFile, serviceRecordMarshal.toBytes(instance), true);
} else {
println(serviceRecordMarshal.toJson(instance));
}
@@ -3624,12 +3419,12 @@
} else if (registryArgs.listExports) {
// list the exports
actionRegistryListExports(registryArgs);
- } else if (SliderUtils.isSet(registryArgs.getConf)) {
+ } else if (isSet(registryArgs.getConf)) {
// get a configuration
PublishedConfiguration publishedConfiguration =
actionRegistryGetConfig(registryArgs);
outputConfig(publishedConfiguration, registryArgs);
- } else if (SliderUtils.isSet(registryArgs.getExport)) {
+ } else if (isSet(registryArgs.getExport)) {
// get a export group
PublishedExports publishedExports =
actionRegistryGetExport(registryArgs);
@@ -3641,11 +3436,11 @@
}
// JDK7
} catch (FileNotFoundException e) {
- log.info("{}", e);
+ log.info("{}", e.toString());
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
} catch (PathNotFoundException e) {
- log.info("{}", e);
+ log.info("{}", e.toString());
log.debug("{}", e, e);
return EXIT_NOT_FOUND;
}
@@ -3669,10 +3464,7 @@
RegistryOperations operations = getRegistryOperations();
Collection<ServiceRecord> serviceRecords;
if (StringUtils.isEmpty(name)) {
- String path =
- serviceclassPath(
- currentUser(),
- serviceType);
+ String path = serviceclassPath(currentUser(), serviceType);
try {
Map<String, ServiceRecord> recordMap =
@@ -3730,18 +3522,15 @@
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
- throw new BadCommandArgumentsException("application name must be provided with --name option");
- }
-
+ requireArgumentSet(Arguments.ARG_NAME, clusterName);
+
try {
- SliderUtils.validateClientConfigFile();
+ validateClientConfigFile();
log.info("Slider-client.xml is accessible");
} catch (IOException e) {
// we are catching exceptions here because those are indication of
// validation result, and we need to print them here
- log.error(
- "validation of slider-client.xml fails because: " + e.toString(), e);
+ log.error("validation of slider-client.xml fails because: " + e, e);
return;
}
SliderClusterOperations clusterOperations = createClusterOperations(clusterName);
@@ -3754,7 +3543,7 @@
AggregateConf instanceDefinition = clusterOperations
.getInstanceDefinition();
String imagePath = instanceDefinition.getInternalOperations().get(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ INTERNAL_APPLICATION_IMAGE_PATH);
// if null, that means slider uploaded the agent tarball for the user
// and we need to use where slider has put
if (imagePath == null) {
@@ -3765,26 +3554,27 @@
+ "/agent");
imagePath = subPath.toString();
}
+ String pathStr = imagePath + "/" + AGENT_TAR;
try {
- SliderUtils.validateHDFSFile(sliderFileSystem, imagePath + "/" + AGENT_TAR);
+ validateHDFSFile(sliderFileSystem, pathStr);
log.info("Slider agent package is properly installed");
} catch (FileNotFoundException e) {
- log.error("can not find agent package: " + e.toString());
+ log.error("can not find agent package: {}", pathStr);
+ log.debug("can not find agent package: {}", pathStr, e);
return;
} catch (IOException e) {
- log.error("can not open agent package: " + e.toString());
+ log.error("can not open agent package: {}", pathStr, e);
return;
}
- String pkgTarballPath = SliderUtils
- .getApplicationDefinitionPath(instanceDefinition
+ String pkgTarballPath = getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
try {
- SliderUtils.validateHDFSFile(sliderFileSystem, pkgTarballPath);
+ validateHDFSFile(sliderFileSystem, pkgTarballPath);
log.info("Application package is properly installed");
} catch (FileNotFoundException e) {
- log.error("can not find application package: {}", e);
+ log.error("can not find application package: {}", pkgTarballPath, e);
} catch (IOException e) {
- log.error("can not open application package: {} ", e);
+ log.error("can not open application package: {} ", pkgTarballPath, e);
}
}
}
@@ -3801,11 +3591,10 @@
private void actionDiagnosticCredentials() throws BadConfigException,
IOException {
- if (SliderUtils.isHadoopClusterSecure(SliderUtils
- .loadSliderClientXML())) {
+ if (isHadoopClusterSecure(loadSliderClientXML())) {
String credentialCacheFileDescription = null;
try {
- credentialCacheFileDescription = SliderUtils.checkCredentialCacheFile();
+ credentialCacheFileDescription = checkCredentialCacheFile();
} catch (BadConfigException e) {
log.error("The credential config is not valid: " + e.toString());
throw e;
@@ -3867,25 +3656,12 @@
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
+ if(isUnset(clusterName)){
throw new BadCommandArgumentsException("application name must be provided with --name option");
}
- SliderClusterOperations clusterOperations;
- AggregateConf instanceDefinition = null;
- try {
- clusterOperations = createClusterOperations(clusterName);
- instanceDefinition = clusterOperations.getInstanceDefinition();
- } catch (YarnException e) {
- log.error("Exception happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- } catch (IOException e) {
- log.error("Network problem happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- }
+ AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName);
String imagePath = instanceDefinition.getInternalOperations().get(
- InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ INTERNAL_APPLICATION_IMAGE_PATH);
// if null, it will be uploaded by Slider and thus at slider's path
if (imagePath == null) {
ApplicationReport appReport = findInstance(clusterName);
@@ -3897,58 +3673,54 @@
log.info("The path of slider agent tarball on HDFS is: " + imagePath);
}
+ private AggregateConf fetchInstanceDefinition(String clusterName)
+ throws YarnException, IOException {
+ SliderClusterOperations clusterOperations;
+ AggregateConf instanceDefinition = null;
+ try {
+ clusterOperations = createClusterOperations(clusterName);
+ instanceDefinition = clusterOperations.getInstanceDefinition();
+ } catch (YarnException | IOException e) {
+ log.error("Failed to retrieve instance definition from YARN: "
+ + e.toString());
+ throw e;
+ }
+ return instanceDefinition;
+ }
+
private void actionDiagnosticApplication(ActionDiagnosticArgs diagnosticArgs)
throws YarnException, IOException {
// not using member variable clustername because we want to place
// application name after --application option and member variable
// cluster name has to be put behind action
String clusterName = diagnosticArgs.name;
- if(SliderUtils.isUnset(clusterName)){
- throw new BadCommandArgumentsException("application name must be provided with --name option");
- }
- SliderClusterOperations clusterOperations;
- AggregateConf instanceDefinition = null;
- try {
- clusterOperations = createClusterOperations(clusterName);
- instanceDefinition = clusterOperations.getInstanceDefinition();
- } catch (YarnException e) {
- log.error("Exception happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- } catch (IOException e) {
- log.error("Network problem happened when retrieving instance definition from YARN: "
- + e.toString());
- throw e;
- }
+ requireArgumentSet(Arguments.ARG_NAME, clusterName);
+ AggregateConf instanceDefinition = fetchInstanceDefinition(clusterName);
String clusterDir = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(AgentKeys.APP_ROOT);
- String pkgTarball = SliderUtils
- .getApplicationDefinitionPath(instanceDefinition.getAppConfOperations());
+ String pkgTarball = getApplicationDefinitionPath(instanceDefinition.getAppConfOperations());
String runAsUser = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(AgentKeys.RUNAS_USER);
- log.info("The location of the cluster instance directory in HDFS is: "
- + clusterDir);
- log.info("The name of the application package tarball on HDFS is: "
- + pkgTarball);
- log.info("The runas user of the application in the cluster is: "
- + runAsUser);
+ log.info("The location of the cluster instance directory in HDFS is: {}", clusterDir);
+ log.info("The name of the application package tarball on HDFS is: {}",pkgTarball);
+ log.info("The runas user of the application in the cluster is: {}",runAsUser);
if (diagnosticArgs.verbose) {
- log.info("App config of the application: "
- + instanceDefinition.getAppConf().toJson());
- log.info("Resource config of the application: "
- + instanceDefinition.getResources().toJson());
+ log.info("App config of the application:\n{}",
+ instanceDefinition.getAppConf().toJson());
+ log.info("Resource config of the application:\n{}",
+ instanceDefinition.getResources().toJson());
}
}
private void actionDiagnosticClient(ActionDiagnosticArgs diagnosticArgs)
throws SliderException, IOException {
try {
- String currentCommandPath = SliderUtils.getCurrentCommandPath();
+ String currentCommandPath = getCurrentCommandPath();
SliderVersionInfo.loadAndPrintVersionInfo(log);
- String clientConfigPath = SliderUtils.getClientConfigPath();
- String jdkInfo = SliderUtils.getJDKInfo();
+ String clientConfigPath = getClientConfigPath();
+ String jdkInfo = getJDKInfo();
println("The slider command path: %s", currentCommandPath);
println("The slider-client.xml used by current running command path: %s",
clientConfigPath);
@@ -3956,7 +3728,7 @@
// security info
Configuration config = getConfig();
- if (SliderUtils.isHadoopClusterSecure(config)) {
+ if (isHadoopClusterSecure(config)) {
println("Hadoop Cluster is secure");
println("Login user is %s", UserGroupInformation.getLoginUser());
println("Current user is %s", UserGroupInformation.getCurrentUser());
@@ -3968,7 +3740,7 @@
// verbose?
if (diagnosticArgs.verbose) {
// do the environment
- Map<String, String> env = SliderUtils.getSystemEnv();
+ Map<String, String> env = getSystemEnv();
Set<String> envList = ConfigHelper.sortedConfigKeys(env.entrySet());
StringBuilder builder = new StringBuilder("Environment variables:\n");
for (String key : envList) {
@@ -3979,7 +3751,7 @@
// Java properties
builder = new StringBuilder("JVM Properties\n");
Map<String, String> props =
- SliderUtils.sortedMap(SliderUtils.toMap(System.getProperties()));
+ sortedMap(toMap(System.getProperties()));
for (Entry<String, String> entry : props.entrySet()) {
builder.append(entry.getKey()).append("=")
.append(entry.getValue()).append("\n");
@@ -3988,12 +3760,10 @@
println(builder.toString());
// then the config
- println("Slider client configuration:\n"
- + ConfigHelper.dumpConfigToString(config));
-
+ println("Slider client configuration:\n" + ConfigHelper.dumpConfigToString(config));
}
- SliderUtils.validateSliderClientEnvironment(log);
+ validateSliderClientEnvironment(log);
} catch (SliderException | IOException e) {
log.error(e.toString());
throw e;
@@ -4154,8 +3924,7 @@
RegistryRetriever retriever = new RegistryRetriever(getConfig(), instance);
boolean external = !registryArgs.internal;
- PublishedExportsSet exports =
- retriever.getExports(external);
+ PublishedExportsSet exports = retriever.getExports(external);
PublishedExports published = retriever.retrieveExports(exports,
registryArgs.getExport,
@@ -4427,7 +4196,7 @@
return EXIT_FALSE;
}
- String version = SliderUtils.getSliderVersion();
+ String version = getSliderVersion();
Path dependencyLibTarGzip = sliderFileSystem.getDependencyTarGzip();
// Check if dependency has already been uploaded, in which case log
@@ -4441,23 +4210,13 @@
}
String libDir = System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
- if (SliderUtils.isSet(libDir)) {
+ if (isSet(libDir)) {
File srcFolder = new File(libDir);
File tempLibTarGzipFile = File.createTempFile(
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + "_",
SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT);
// copy all jars except slider-core-<version>.jar
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
- SliderUtils.tarGzipFolder(srcFolder, tempLibTarGzipFile, jarFilter);
+ tarGzipFolder(srcFolder, tempLibTarGzipFile, createJarFilter());
log.info("Uploading dependency for AM (version {}) from {} to {}",
version, tempLibTarGzipFile.toURI(), dependencyLibTarGzip.toUri());
@@ -4479,6 +4238,73 @@
throw new UsageException("%s %s", errMsg, CommonArgs.usage(serviceArgs,
actionName));
}
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ *
+ * @param args argument list
+ * @return a possibly empty list of nodes in the cluster
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ @Override
+ public NodeInformationList listYarnClusterNodes(ActionNodesArgs args)
+ throws YarnException, IOException {
+ return yarnClient.listNodes(args.label, args.healthy);
+ }
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ *
+ * @param args argument list
+ * @return a possibly empty list of nodes in the cluster
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ public NodeInformationList listInstanceNodes(String instance, ActionNodesArgs args)
+ throws YarnException, IOException {
+ // TODO
+ log.info("listInstanceNodes {}", instance);
+ SliderClusterOperations clusterOps =
+ new SliderClusterOperations(bondToCluster(instance));
+ return clusterOps.getLiveNodes();
+ }
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ * Prints them to stdout unless the args names a file instead.
+ * @param args argument list
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ public int actionNodes(String instance, ActionNodesArgs args) throws YarnException, IOException {
+
+ args.instance = instance;
+ NodeInformationList nodes;
+ if (SliderUtils.isUnset(instance)) {
+ nodes = listYarnClusterNodes(args);
+ } else {
+ nodes = listInstanceNodes(instance, args);
+ }
+ log.debug("Node listing for {} has {} nodes", args, nodes.size());
+ JsonSerDeser<NodeInformationList> serDeser = NodeInformationList.createSerializer();
+ if (args.outputFile != null) {
+ serDeser.save(nodes, args.outputFile);
+ } else {
+ println(serDeser.toJson(nodes));
+ }
+ return 0;
+ }
+
+ /**
+ * Create a new IPC client for talking to slider via what follows the REST API.
+ * Client must already be bonded to the cluster
+ * @return a new IPC client
+ */
+ public SliderApplicationApi createIpcClient()
+ throws IOException, YarnException {
+ return new SliderApplicationIpcClient(createClusterOperations());
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index bab451f..5c5d96b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -23,11 +23,13 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionDependencyArgs;
+import org.apache.slider.common.params.ActionDestroyArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
@@ -35,6 +37,7 @@
import org.apache.slider.common.params.ActionInstallKeytabArgs;
import org.apache.slider.common.params.ActionInstallPackageArgs;
import org.apache.slider.common.params.ActionKeytabArgs;
+import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.params.ActionPackageArgs;
import org.apache.slider.common.params.ActionKillContainerArgs;
import org.apache.slider.common.params.ActionListArgs;
@@ -62,6 +65,9 @@
* #1 the cluster is started between verifying that there are no live
* clusters of that name.
*/
+ int actionDestroy(String clustername, ActionDestroyArgs destroyArgs)
+ throws YarnException, IOException;
+
int actionDestroy(String clustername) throws YarnException,
IOException;
@@ -99,7 +105,7 @@
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
- * @deperecated use #actionKeytab
+ * @deprecated use #actionKeytab
*/
int actionInstallKeytab(ActionInstallKeytabArgs installKeytabInfo)
throws YarnException, IOException;
@@ -111,7 +117,7 @@
* @throws YarnException Yarn problems
* @throws IOException other problems
* @throws BadCommandArgumentsException bad arguments.
- * @deperecated use #actionKeytab
+ * @deprecated use #actionKeytab
*/
int actionKeytab(ActionKeytabArgs keytabInfo)
throws YarnException, IOException;
@@ -338,4 +344,14 @@
*/
int actionDependency(ActionDependencyArgs dependencyArgs) throws IOException,
YarnException;
+
+ /**
+ * List the nodes
+ * @param args
+ * @return
+ * @throws YarnException
+ * @throws IOException
+ */
+ NodeInformationList listYarnClusterNodes(ActionNodesArgs args)
+ throws YarnException, IOException;
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 3b7a65c..85a582b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -20,6 +20,7 @@
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
@@ -27,12 +28,18 @@
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.ActionNodesArgs;
import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.Duration;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -42,6 +49,8 @@
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.BindException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -55,8 +64,7 @@
* from the slider entry point service
*/
public class SliderYarnClientImpl extends YarnClientImpl {
- protected static final Logger
- log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
+ protected static final Logger log = LoggerFactory.getLogger(SliderYarnClientImpl.class);
/**
* Keyword to use in the {@link #emergencyForceKill(String)}
@@ -65,6 +73,18 @@
*/
public static final String KILL_ALL = "all";
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // address isn't known; fail fast
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS
+ + " value:" + conf.get(YarnConfiguration.RM_ADDRESS)
+ + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
+ }
+ super.serviceInit(conf);
+ }
+
/**
* Get the RM Client RPC interface
* @return an RPC interface valid after initialization and authentication
@@ -96,10 +116,10 @@
public List<ApplicationReport> listDeployedInstances(String user)
throws YarnException, IOException {
Preconditions.checkArgument(user != null, "Null User");
- Set<String> types = new HashSet<String>(1);
+ Set<String> types = new HashSet<>(1);
types.add(SliderKeys.APP_TYPE);
List<ApplicationReport> allApps = getApplications(types);
- List<ApplicationReport> results = new ArrayList<ApplicationReport>();
+ List<ApplicationReport> results = new ArrayList<>();
for (ApplicationReport report : allApps) {
if (StringUtils.isEmpty(user) || user.equals(report.getUser())) {
results.add(report);
@@ -108,7 +128,6 @@
return results;
}
-
/**
* find all instances of a specific app -if there is more than one in the
* YARN cluster,
@@ -124,7 +143,7 @@
List<ApplicationReport> instances = listDeployedInstances(user);
List<ApplicationReport> results =
- new ArrayList<ApplicationReport>(instances.size());
+ new ArrayList<>(instances.size());
for (ApplicationReport report : instances) {
if (report.getName().equals(appname)) {
results.add(report);
@@ -142,8 +161,7 @@
public boolean isApplicationLive(ApplicationReport app) {
Preconditions.checkArgument(app != null, "Null app report");
- return app.getYarnApplicationState().ordinal() <=
- YarnApplicationState.RUNNING.ordinal();
+ return app.getYarnApplicationState().ordinal() <= YarnApplicationState.RUNNING.ordinal();
}
@@ -330,8 +348,6 @@
Preconditions.checkArgument(StringUtils.isNotEmpty(appname),
"Null/empty application name");
Preconditions.checkArgument(desiredState != null, "Null desiredState");
- ApplicationReport found = null;
- ApplicationReport foundAndLive = null;
log.debug("Searching {} records for instance name {} in state '{}'",
instances.size(), appname, desiredState);
for (ApplicationReport app : instances) {
@@ -350,4 +366,45 @@
log.debug("No match");
return null;
}
+
+ /**
+ * List the nodes in the cluster, possibly filtering by node state or label.
+ *
+ * @param label label to filter by -or "" for any
+ * @param live flag to request running nodes only
+ * @return a possibly empty list of nodes in the cluster
+ * @throws IOException IO problems
+ * @throws YarnException YARN problems
+ */
+ public NodeInformationList listNodes(String label, boolean live)
+ throws IOException, YarnException {
+ Preconditions.checkArgument(label != null, "null label");
+ NodeState[] states;
+ if (live) {
+ states = new NodeState[1];
+ states[0] = NodeState.RUNNING;
+ } else {
+ states = new NodeState[0];
+ }
+ List<NodeReport> reports = getNodeReports(states);
+ NodeInformationList results = new NodeInformationList(reports.size());
+ for (NodeReport report : reports) {
+ if (live && report.getNodeState() != NodeState.RUNNING) {
+ continue;
+ }
+ if (!label.isEmpty() && !report.getNodeLabels().contains(label)) {
+ continue;
+ }
+ // build node info from report
+ NodeInformation info = new NodeInformation();
+ info.hostname = report.getNodeId().getHost();
+ info.healthReport = report.getHealthReport();
+ info.httpAddress = report.getHttpAddress();
+ info.labels = SliderUtils.extractNodeLabel(report);
+ info.rackName = report.getRackName();
+ info.state = report.getNodeState().toString();
+ results.add(info);
+ }
+ return results;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java
index bada24e..a007326 100644
--- a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderApplicationIpcClient.java
@@ -23,6 +23,8 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.api.SliderApplicationApi;
import org.apache.slider.core.conf.AggregateConf;
@@ -186,8 +188,7 @@
}
@Override
- public ComponentInformation getComponent(String componentName) throws
- IOException {
+ public ComponentInformation getComponent(String componentName) throws IOException {
try {
return operations.getComponent(componentName);
} catch (IOException e) {
@@ -196,6 +197,24 @@
}
@Override
+ public NodeInformationList getLiveNodes() throws IOException {
+ try {
+ return operations.getLiveNodes();
+ } catch (IOException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public NodeInformation getLiveNode(String hostname) throws IOException {
+ try {
+ return operations.getLiveNode(hostname);
+ } catch (IOException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
public PingInformation ping(String text) throws IOException {
return null;
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
index 69dcb3b..392f451 100644
--- a/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/client/ipc/SliderClusterOperations.java
@@ -19,6 +19,9 @@
package org.apache.slider.client.ipc;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
@@ -30,8 +33,11 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.api.types.PingInformation;
import org.apache.slider.common.tools.Duration;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -102,7 +108,7 @@
*/
public List<ClusterNode> convertNodeWireToClusterNodes(List<Messages.RoleInstanceState> nodes)
throws IOException {
- List<ClusterNode> nodeList = new ArrayList<ClusterNode>(nodes.size());
+ List<ClusterNode> nodeList = new ArrayList<>(nodes.size());
for (Messages.RoleInstanceState node : nodes) {
nodeList.add(ClusterNode.fromProtobuf(node));
}
@@ -116,16 +122,12 @@
* @throws YarnException
* @throws IOException
*/
- public String echo(String text) throws
- YarnException,
- IOException {
+ public String echo(String text) throws YarnException, IOException {
Messages.EchoRequestProto.Builder builder =
Messages.EchoRequestProto.newBuilder();
builder.setText(text);
- Messages.EchoRequestProto req =
- builder.build();
- Messages.EchoResponseProto response =
- appMaster.echo(req);
+ Messages.EchoRequestProto req = builder.build();
+ Messages.EchoResponseProto response = appMaster.echo(req);
return response.getText();
}
@@ -145,9 +147,7 @@
try {
return ClusterDescription.fromJson(statusJson);
} catch (JsonParseException e) {
- log.error(
- "Exception " + e + " parsing:\n" + statusJson,
- e);
+ log.error("Exception " + e + " parsing:\n" + statusJson, e);
throw e;
}
}
@@ -191,10 +191,8 @@
Messages.KillContainerRequestProto.Builder builder =
Messages.KillContainerRequestProto.newBuilder();
builder.setId(id);
- Messages.KillContainerRequestProto req =
- builder.build();
- Messages.KillContainerResponseProto response =
- appMaster.killContainer(req);
+ Messages.KillContainerRequestProto req = builder.build();
+ Messages.KillContainerResponseProto response = appMaster.killContainer(req);
return response.getSuccess();
}
@@ -205,24 +203,19 @@
* @throws IOException
* @throws YarnException
*/
- public String[] listNodeUUIDsByRole(String role) throws
- IOException,
- YarnException {
+ public String[] listNodeUUIDsByRole(String role) throws IOException, YarnException {
Collection<String> uuidList = innerListNodeUUIDSByRole(role);
String[] uuids = new String[uuidList.size()];
return uuidList.toArray(uuids);
}
- public List<String> innerListNodeUUIDSByRole(String role) throws
- IOException,
- YarnException {
+ public List<String> innerListNodeUUIDSByRole(String role) throws IOException, YarnException {
Messages.ListNodeUUIDsByRoleRequestProto req =
Messages.ListNodeUUIDsByRoleRequestProto
.newBuilder()
.setRole(role)
.build();
- Messages.ListNodeUUIDsByRoleResponseProto resp =
- appMaster.listNodeUUIDsByRole(req);
+ Messages.ListNodeUUIDsByRoleResponseProto resp = appMaster.listNodeUUIDsByRole(req);
return resp.getUuidList();
}
@@ -234,9 +227,8 @@
* @throws IOException
* @throws YarnException
*/
- public List<ClusterNode> listClusterNodesInRole(String role) throws
- IOException,
- YarnException {
+ public List<ClusterNode> listClusterNodesInRole(String role)
+ throws IOException, YarnException {
Collection<String> uuidList = innerListNodeUUIDSByRole(role);
Messages.GetClusterNodesRequestProto req =
@@ -250,15 +242,14 @@
/**
* Get the details on a list of uuids
- * @param uuids
+ * @param uuids instance IDs
* @return a possibly empty list of node details
* @throws IOException
* @throws YarnException
*/
@VisibleForTesting
- public List<ClusterNode> listClusterNodes(String[] uuids) throws
- IOException,
- YarnException {
+ public List<ClusterNode> listClusterNodes(String[] uuids)
+ throws IOException, YarnException {
Messages.GetClusterNodesRequestProto req =
Messages.GetClusterNodesRequestProto
@@ -341,8 +332,7 @@
Messages.FlexClusterRequestProto.newBuilder()
.setClusterSpec(resources.toJson())
.build();
- Messages.FlexClusterResponseProto response =
- appMaster.flexCluster(request);
+ Messages.FlexClusterResponseProto response = appMaster.flexCluster(request);
return response.getResponse();
}
@@ -365,10 +355,8 @@
}
builder.setSignal(signal);
builder.setDelay(delay);
- Messages.AMSuicideRequestProto req =
- builder.build();
- Messages.AMSuicideResponseProto response =
- appMaster.amSuicide(req);
+ Messages.AMSuicideRequestProto req = builder.build();
+ appMaster.amSuicide(req);
}
/**
@@ -385,45 +373,38 @@
}
-
public AggregateConf getModelDesired() throws IOException {
return unmarshallToAggregateConf(appMaster.getModelDesired(EMPTY));
}
public ConfTreeOperations getModelDesiredAppconf() throws IOException {
- return unmarshallToCTO(
- appMaster.getModelDesiredAppconf(EMPTY));
+ return unmarshallToCTO(appMaster.getModelDesiredAppconf(EMPTY));
}
public ConfTreeOperations getModelDesiredResources() throws IOException {
- return unmarshallToCTO(
- appMaster.getModelDesiredResources(EMPTY));
+ return unmarshallToCTO(appMaster.getModelDesiredResources(EMPTY));
}
public AggregateConf getModelResolved() throws IOException {
- return unmarshallToAggregateConf(
- appMaster.getModelResolved(EMPTY));
+ return unmarshallToAggregateConf(appMaster.getModelResolved(EMPTY));
}
public ConfTreeOperations getModelResolvedAppconf() throws IOException {
- return unmarshallToCTO(
- appMaster.getModelResolvedAppconf(EMPTY));
+ return unmarshallToCTO(appMaster.getModelResolvedAppconf(EMPTY));
}
public ConfTreeOperations getModelResolvedResources() throws IOException {
- return unmarshallToCTO(
- appMaster.getModelDesiredResources(EMPTY));
+ return unmarshallToCTO(appMaster.getModelDesiredResources(EMPTY));
}
public ConfTreeOperations getLiveResources() throws IOException {
- return unmarshallToCTO(
- appMaster.getLiveResources(EMPTY));
+ return unmarshallToCTO(appMaster.getLiveResources(EMPTY));
}
@@ -439,8 +420,7 @@
+ ") does not match the number of records returned: "
+ records);
}
- Map<String, ContainerInformation> map =
- new HashMap<String, ContainerInformation>(namesCount);
+ Map<String, ContainerInformation> map = new HashMap<>(namesCount);
for (int i = 0; i < namesCount; i++) {
map.put(response.getNames(i), unmarshall(response.getContainers(i)));
}
@@ -473,42 +453,53 @@
int namesCount = response.getNamesCount();
int records = response.getComponentsCount();
if (namesCount != records) {
- throw new IOException("Number of names returned (" + namesCount
- +
- ") does not match the number of records returned: "
- + records);
+ throw new IOException(
+ "Number of names returned (" + namesCount + ")" +
+ " does not match the number of records returned: " + records);
}
- Map<String, ComponentInformation> map =
- new HashMap<String, ComponentInformation>(namesCount);
+ Map<String, ComponentInformation> map = new HashMap<>(namesCount);
for (int i = 0; i < namesCount; i++) {
map.put(response.getNames(i), unmarshall(response.getComponents(i)));
}
return map;
}
-
public ComponentInformation getComponent(String componentName)
throws IOException {
Messages.GetLiveComponentRequestProto.Builder builder =
Messages.GetLiveComponentRequestProto.newBuilder();
builder.setName(componentName);
- Messages.ComponentInformationProto proto =
- appMaster.getLiveComponent(builder.build());
-
+ Messages.ComponentInformationProto proto = appMaster.getLiveComponent(builder.build());
return unmarshall(proto);
}
-
+ public NodeInformationList getLiveNodes() throws IOException {
+ Messages.GetLiveNodesResponseProto response =
+ appMaster.getLiveNodes(Messages.GetLiveNodesRequestProto.newBuilder().build());
+
+ int records = response.getNodesCount();
+ NodeInformationList nil = new NodeInformationList(records);
+ for (int i = 0; i < records; i++) {
+ nil.add(unmarshall(response.getNodes(i)));
+ }
+ return nil;
+ }
+
+ public NodeInformation getLiveNode(String hostname) throws IOException {
+ Messages.GetLiveNodeRequestProto.Builder builder =
+ Messages.GetLiveNodeRequestProto.newBuilder();
+ builder.setName(hostname);
+ return unmarshall(appMaster.getLiveNode(builder.build()));
+ }
+
public PingInformation ping(String text) throws IOException {
return null;
}
-
public void stop(String text) throws IOException {
amSuicide(text, 3, 0);
}
-
public ApplicationLivenessInformation getApplicationLiveness() throws
IOException {
Messages.ApplicationLivenessInformationProto proto =
@@ -535,5 +526,4 @@
return unmarshall(response);
}
-
}
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
index b1aa633..4283ee8 100644
--- a/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/SliderApplicationApiRestClient.java
@@ -21,7 +21,6 @@
import com.google.common.base.Preconditions;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
-import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;
@@ -31,11 +30,12 @@
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
import org.apache.slider.api.SliderApplicationApi;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.ExceptionConverter;
-import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.core.restclient.HttpVerb;
import org.apache.slider.api.types.PingInformation;
import org.slf4j.Logger;
@@ -67,6 +67,19 @@
this.appResource = appResource;
}
+ /**
+ * Create an instance
+ * @param jerseyClient jersey client for operations
+ * @param appmaster URL of appmaster/proxy to AM
+ */
+ public SliderApplicationApiRestClient(Client jerseyClient, String appmaster) {
+ super(jerseyClient);
+ WebResource amResource = jerseyClient.resource(appmaster);
+ amResource.type(MediaType.APPLICATION_JSON);
+ this.appResource = amResource.path(SLIDER_PATH_APPLICATION);
+ }
+
+
@Override
public String toString() {
final StringBuilder sb =
@@ -232,8 +245,7 @@
public Map<String, ComponentInformation> enumComponents() throws
IOException {
return getApplicationResource(LIVE_COMPONENTS,
- new GenericType<Map<String, ComponentInformation>>() {
- });
+ new GenericType<Map<String, ComponentInformation>>() { });
}
@Override
@@ -244,6 +256,17 @@
}
@Override
+ public NodeInformationList getLiveNodes() throws IOException {
+ return getApplicationResource(LIVE_NODES, NodeInformationList.class);
+ }
+
+ @Override
+ public NodeInformation getLiveNode(String hostname) throws IOException {
+ return getApplicationResource(LIVE_NODES + "/" + hostname,
+ NodeInformation.class);
+ }
+
+ @Override
public PingInformation ping(String text) throws IOException {
return pingPost(text);
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 3d2b868..26109a7 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -125,6 +125,8 @@
"hadoop.http.filter.initializers";
String KEY_KEYSTORE_LOCATION = "ssl.server.keystore.location";
String KEY_AM_LOGIN_KEYTAB_NAME = "slider.am.login.keytab.name";
+ /** Declare that a keytab must be provided */
+ String KEY_AM_LOGIN_KEYTAB_REQUIRED = "slider.am.login.keytab.required";
String KEY_HDFS_KEYTAB_DIR = "slider.hdfs.keytab.dir";
String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path";
String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name";
@@ -166,11 +168,39 @@
*/
String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
"slider.ipc.client.retry.enabled";
- public static final boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
- public static final String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
+ boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
+ String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
"slider.ipc.client.retry.policy.spec";
- public static final String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
+ String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
"10000,6,60000,10"; //t1,n1,t2,n2,...
String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
+
+ /**
+ * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
+ */
+ String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
+
+ String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
+
+ //Delegation token related keys
+ String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
+ = "dfs.namenode.delegation.key.update-interval";
+ long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
+ 1000; // 1 day
+ String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
+ = "dfs.namenode.delegation.token.renew-interval";
+ long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
+ 1000; // 1 day
+ String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
+ = "dfs.namenode.delegation.token.max-lifetime";
+ long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
+ 1000; // 7 days
+ String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
+ = "dfs.namenode.delegation.token.always-use"; // for tests
+ boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
+ String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
+ String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
+
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 17c235f..d0b1693 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -56,7 +56,7 @@
* This is the default parameter
*/
@Parameter
- public final List<String> parameters = new ArrayList<String>();
+ public final List<String> parameters = new ArrayList<>();
/**
* get the name: relies on arg 1 being the cluster name in all operations
@@ -77,7 +77,7 @@
*/
@Parameter(names = ARG_DEFINE, arity = 1, description = "Definitions")
- public final List<String> definitions = new ArrayList<String>();
+ public final List<String> definitions = new ArrayList<>();
/**
* System properties
@@ -85,7 +85,7 @@
@Parameter(names = {ARG_SYSPROP}, arity = 1,
description = "system properties in the form name value" +
" These are set after the JVM is started.")
- public final List<String> sysprops = new ArrayList<String>(0);
+ public final List<String> sysprops = new ArrayList<>(0);
@Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index f2e3c61..1c694bd 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -59,7 +59,7 @@
/**
* --image path
- the full path to a .tar or .tar.gz path containing an HBase image.
+ * the full path to a .tar or .tar.gz path containing an HBase image.
*/
@Parameter(names = ARG_IMAGE,
description = "The full path to a .tar or .tar.gz path containing the application",
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
index d6cabf5..5b4cfdc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionAMSuicideArgs.java
@@ -21,7 +21,6 @@
import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
-
@Parameters(commandNames = {SliderActions.ACTION_AM_SUICIDE},
commandDescription = SliderActions.DESCRIBE_ACTION_AM_SUICIDE)
public class ActionAMSuicideArgs extends AbstractActionArgs {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
index 1203d28..4a129ab 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDestroyArgs.java
@@ -18,15 +18,20 @@
package org.apache.slider.common.params;
+import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_DESTROY},
commandDescription = SliderActions.DESCRIBE_ACTION_DESTROY)
public class ActionDestroyArgs extends AbstractActionArgs {
-
+
@Override
public String getActionName() {
return SliderActions.ACTION_DESTROY;
}
+
+ @Parameter(names = {ARG_FORCE},
+ description = "force the operation")
+ public boolean force;
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
index 4b03336..c891873 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionDiagnosticArgs.java
@@ -37,7 +37,7 @@
@Parameter(names = {ARG_CLIENT},
description = "print configuration of the slider client")
public boolean client = false;
-
+
@Parameter(names = {ARG_APPLICATION},
description = "print configuration of the running application")
public boolean application;
@@ -49,15 +49,15 @@
@Parameter(names = {ARG_YARN},
description = "print configuration of the YARN cluster")
public boolean yarn = false;
-
+
@Parameter(names = {ARG_CREDENTIALS},
description = "print credentials of the current user")
public boolean credentials = false;
-
+
@Parameter(names = {ARG_ALL},
description = "print all of the information above")
public boolean all;
-
+
@Parameter(names = {ARG_LEVEL},
description = "diagnose each slider configuration one by one")
public boolean level;
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
index b0f53f8..d05f10b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionEchoArgs.java
@@ -20,15 +20,10 @@
import com.beust.jcommander.Parameter;
-/*
-
-@Parameters(commandNames = {SliderActions.ACTION_KILL_CONTAINER},
- commandDescription = SliderActions.DESCRIBE_ACTION_KILL_CONTAINER)
-*/
public class ActionEchoArgs extends AbstractActionArgs {
@Override
public String getActionName() {
- return SliderActions.ACTION_DESTROY;
+ return SliderActions.ACTION_ECHO;
}
@Parameter(names = {ARG_MESSAGE},
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
index 0bc5792..739b5fc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionListArgs.java
@@ -57,7 +57,7 @@
@Parameter(names = {ARG_COMPONENTS}, variableArity = true,
description = "Filter containers by component names (used with " +
ARG_CONTAINERS + ")")
- public Set<String> components = new HashSet<>(0);;
+ public Set<String> components = new HashSet<>(0);
/**
* Get the min #of params expected
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
index 3b69e74..1b73522 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionLookupArgs.java
@@ -43,7 +43,7 @@
}
@Parameter(names = {ARG_ID},
- description = "ID of the container")
+ description = "ID of the application")
public String id;
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
new file mode 100644
index 0000000..5ddccf6
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionNodesArgs.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_NODES},
+ commandDescription = SliderActions.DESCRIBE_ACTION_NODES)
+public class ActionNodesArgs extends AbstractActionArgs {
+
+ /**
+ * Instance for API use; on CLI the name is derived from {@link #getClusterName()}.
+ */
+ public String instance;
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_NODES;
+ }
+
+ @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
+ description = "Output file for the information")
+ public File outputFile;
+
+ @Parameter(names = {ARG_LABEL})
+ public String label = "";
+
+ @Parameter(names = {ARG_HEALTHY} )
+ public boolean healthy;
+
+ @Override
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ActionNodesArgs{");
+ sb.append("instance='").append(instance).append('\'');
+ sb.append(", outputFile=").append(outputFile);
+ sb.append(", label='").append(label).append('\'');
+ sb.append(", healthy=").append(healthy);
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
index a7edb65..00178df 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionStatusArgs.java
@@ -25,13 +25,14 @@
commandDescription = SliderActions.DESCRIBE_ACTION_STATUS)
public class ActionStatusArgs extends AbstractActionArgs {
+
@Override
public String getActionName() {
return SliderActions.ACTION_STATUS;
}
@Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT},
- description = "Output file for the configuration data")
+ description = "Output file for the status information")
public String output;
public String getOutput() {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
index 832e1cc..6ef51b2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionUpgradeArgs.java
@@ -61,11 +61,11 @@
@Parameter(names={ARG_CONTAINERS}, variableArity = true,
description = "stop specific containers")
- public List<String> containers = new ArrayList<String>(0);
+ public List<String> containers = new ArrayList<>(0);
@Parameter(names={ARG_COMPONENTS}, variableArity = true,
description = "stop all containers of specific components")
- public List<String> components = new ArrayList<String>(0);
+ public List<String> components = new ArrayList<>(0);
@Parameter(names = {ARG_FORCE},
description = "force spec upgrade operation")
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
index 65ebc4b..3ef8e19 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
@@ -34,7 +34,7 @@
arity = 2,
description = "--addon <name> <folder or package>",
splitter = DontSplitArguments.class)
- public List<String> addonTuples = new ArrayList<String>(0);
+ public List<String> addonTuples = new ArrayList<>(0);
/**
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
index 1f07de3..248e4c2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -37,7 +37,7 @@
@Parameter(names = {ARG_OPTION, ARG_OPTION_SHORT}, arity = 2,
description = ARG_OPTION + "<name> <value>",
splitter = DontSplitArguments.class)
- public List<String> optionTuples = new ArrayList<String>(0);
+ public List<String> optionTuples = new ArrayList<>(0);
/**
@@ -47,7 +47,7 @@
description = "Component option " + ARG_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
- public List<String> compOptTriples = new ArrayList<String>(0);
+ public List<String> compOptTriples = new ArrayList<>(0);
/**
* Resource Options
@@ -55,7 +55,7 @@
@Parameter(names = {ARG_RESOURCE_OPT, ARG_RESOURCE_OPT_SHORT}, arity = 2,
description = "Resource option "+ ARG_RESOURCE_OPT + "<name> <value>",
splitter = DontSplitArguments.class)
- public List<String> resOptionTuples = new ArrayList<String>(0);
+ public List<String> resOptionTuples = new ArrayList<>(0);
/**
@@ -65,7 +65,7 @@
description = "Component resource option " + ARG_RES_COMP_OPT +
" <component> <name> <option>",
splitter = DontSplitArguments.class)
- public List<String> resCompOptTriples = new ArrayList<String>(0);
+ public List<String> resCompOptTriples = new ArrayList<>(0);
public Map<String, String> getOptionsMap() throws
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index aeb2979..12a2032 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -44,20 +44,13 @@
* create a 3-tuple
*/
public static List<Object> triple(String msg, int min, int max) {
- List<Object> l = new ArrayList<Object>(3);
+ List<Object> l = new ArrayList<>(3);
l.add(msg);
l.add(min);
l.add(max);
return l;
}
- /**
- * Create a tuple
- */
- public static List<Object> tuple(String msg, int min) {
- return triple(msg, min, min);
- }
-
public static void applyFileSystemBinding(String filesystemBinding,
Configuration conf) {
if (filesystemBinding != null) {
@@ -100,7 +93,7 @@
public static Map<String, String> convertTupleListToMap(String description,
List<String> list) throws
BadCommandArgumentsException {
- Map<String, String> results = new HashMap<String, String>();
+ Map<String, String> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 2 != 0) {
@@ -131,10 +124,9 @@
* @throws BadCommandArgumentsException odd #of arguments received
*/
public static Map<String, Map<String, String>> convertTripleListToMaps(String description,
- List<String> list) throws
- BadCommandArgumentsException {
- Map<String, Map<String, String>> results =
- new HashMap<String, Map<String, String>>();
+ List<String> list) throws BadCommandArgumentsException {
+
+ Map<String, Map<String, String>> results = new HashMap<>();
if (list != null && !list.isEmpty()) {
int size = list.size();
if (size % 3 != 0) {
@@ -149,7 +141,7 @@
Map<String, String> roleMap = results.get(role);
if (roleMap == null) {
//demand create new role map
- roleMap = new HashMap<String, String>();
+ roleMap = new HashMap<>();
results.put(role, roleMap);
}
if (roleMap.get(key) != null) {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index c2fa09c..ca22c24 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -60,6 +60,7 @@
String ARG_GETCONF = "--getconf";
String ARG_GETEXP = "--getexp";
String ARG_GETFILES = "--getfiles";
+ String ARG_HEALTHY= "--healthy";
String ARG_HELP = "--help";
String ARG_HOSTNAME = "--hostname";
String ARG_ID = "--id";
@@ -70,6 +71,7 @@
String ARG_KEYTABINSTALL = "--install";
String ARG_KEYTABDELETE = "--delete";
String ARG_KEYTABLIST = "--list";
+ String ARG_LABEL = "--label";
String ARG_LEVEL = "--level";
String ARG_LIST = "--list";
String ARG_LISTCONF = "--listconf";
@@ -125,10 +127,14 @@
/**
- * Deprecated
+ * Deprecated: use ARG_COMPONENT
*/
@Deprecated
String ARG_ROLE = "--role";
+
+ /**
+ * Deprecated: use ARG_COMP_OPT
+ */
@Deprecated
String ARG_ROLEOPT = "--roleopt";
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 3c430f3..ea1448b 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -66,6 +66,7 @@
new ActionKillContainerArgs();
private final ActionListArgs actionListArgs = new ActionListArgs();
private final ActionLookupArgs actionLookupArgs = new ActionLookupArgs();
+ private final ActionNodesArgs actionNodesArgs = new ActionNodesArgs();
private final ActionRegistryArgs actionRegistryArgs = new ActionRegistryArgs();
private final ActionResolveArgs actionResolveArgs = new ActionResolveArgs();
private final ActionStatusArgs actionStatusArgs = new ActionStatusArgs();
@@ -86,31 +87,32 @@
protected void addActionArguments() {
addActions(
- actionPackageArgs,
- actionKeytabArgs,
+ actionAMSuicideArgs,
actionBuildArgs,
- actionDependencyArgs,
- actionCreateArgs,
- actionListArgs,
- actionStatusArgs,
- actionRegistryArgs,
actionClientArgs,
- actionFlexArgs,
+ actionCreateArgs,
+ actionDependencyArgs,
+ actionDestroyArgs,
actionDiagnosticArgs,
+ actionExistsArgs,
+ actionFlexArgs,
actionFreezeArgs,
+ actionHelpArgs,
+ actionInstallKeytabArgs,
+ actionInstallPackageArgs,
+ actionKeytabArgs,
+ actionKillContainerArgs,
+ actionListArgs,
+ actionLookupArgs,
+ actionNodesArgs,
+ actionPackageArgs,
+ actionRegistryArgs,
+ actionResolveArgs,
+ actionStatusArgs,
actionThawArgs,
actionUpdateArgs,
actionUpgradeArgs,
- actionDestroyArgs,
- actionExistsArgs,
- actionLookupArgs,
- actionResolveArgs,
- actionKillContainerArgs,
- actionAMSuicideArgs,
- actionInstallPackageArgs,
- actionInstallKeytabArgs,
- actionVersionArgs,
- actionHelpArgs
+ actionVersionArgs
);
}
@@ -196,6 +198,10 @@
return actionListArgs;
}
+ public ActionNodesArgs getActionNodesArgs() {
+ return actionNodesArgs;
+ }
+
public ActionLookupArgs getActionLookupArgs() {
return actionLookupArgs;
}
@@ -286,6 +292,9 @@
} else if (SliderActions.ACTION_LOOKUP.equals(action)) {
bindCoreAction(actionLookupArgs);
+ } else if (SliderActions.ACTION_NODES.equals(action)) {
+ bindCoreAction(actionNodesArgs);
+
} else if (SliderActions.ACTION_REGISTRY.equals(action)) {
bindCoreAction(actionRegistryArgs);
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 7e02eec..162a87d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -91,12 +91,12 @@
return coreAction.getClusterName();
}
- public CommonArgs(String[] args) {
+ protected CommonArgs(String[] args) {
this.args = args;
commander = new JCommander(this);
}
- public CommonArgs(Collection args) {
+ protected CommonArgs(Collection args) {
List<String> argsAsStrings = SliderUtils.collectionToStringList(args);
this.args = argsAsStrings.toArray(new String[argsAsStrings.size()]);
commander = new JCommander(this);
@@ -123,7 +123,7 @@
.append("Most commands print help when invoked without parameters or with --help");
result = helperMessage.toString();
} else {
- helperMessage.append("\nUsage: slider " + commandOfInterest);
+ helperMessage.append("\nUsage: slider ").append(commandOfInterest);
helperMessage.append(serviceArgs.coreAction.getMinParams() > 0 ? " <application>" : "");
helperMessage.append("\n");
for (ParameterDescription paramDesc : serviceArgs.commander.getCommands()
@@ -171,7 +171,7 @@
/**
* Add a command
* @param name action
- * @param arg
+ * @param arg value
*/
protected void addAction(String name, Object arg) {
commander.addCommand(name, arg);
@@ -249,7 +249,7 @@
throw new BadCommandArgumentsException(badArgMsgBuilder.toString());
} catch (UsageException e) {
StringBuilder badArgMsgBuilder = new StringBuilder();
- badArgMsgBuilder.append(e.toString() + "\n");
+ badArgMsgBuilder.append(e.toString()).append("\n");
badArgMsgBuilder.append(usage(this, coreAction.getActionName()));
throw new UsageException(badArgMsgBuilder.toString());
}
@@ -289,12 +289,6 @@
return coreAction.manager;
}
-
-// public String getRmAddress() {
-// return rmAddress;
-// }
-
-
public String getAction() {
return commander.getParsedCommand();
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
index 494258e..abda53f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ComponentArgsDelegate.java
@@ -34,7 +34,7 @@
arity = 2,
description = "--component <name> <count>",
splitter = DontSplitArguments.class)
- public List<String> componentTuples = new ArrayList<String>(0);
+ public List<String> componentTuples = new ArrayList<>(0);
/**
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
index 3225133..0344305 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java
@@ -27,7 +27,7 @@
@Override
public List<String> split(String value) {
- List<String> list = new ArrayList<String>(1);
+ List<String> list = new ArrayList<>(1);
list.add(value);
return list;
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index 4d0e04d..1d4e0ea 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -39,6 +39,7 @@
String ACTION_KILL_CONTAINER = "kill-container";
String ACTION_LIST = "list";
String ACTION_LOOKUP = "lookup";
+ String ACTION_NODES = "nodes";
String ACTION_PREFLIGHT = "preflight";
String ACTION_RECONFIGURE = "reconfigure";
String ACTION_REGISTRY = "registry";
@@ -65,7 +66,7 @@
String DESCRIBE_ACTION_UPGRADE =
"Rolling upgrade/downgrade the application to a newer/previous version";
String DESCRIBE_ACTION_DESTROY =
- "Destroy a frozen Slider application)";
+ "Destroy a stopped Slider application";
String DESCRIBE_ACTION_EXISTS =
"Probe for an application running";
String DESCRIBE_ACTION_FLEX = "Flex a Slider application";
@@ -80,6 +81,7 @@
"List running Slider applications";
String DESCRIBE_ACTION_LOOKUP =
"look up a YARN application";
+ String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application";
String DESCRIBE_ACTION_MONITOR =
"Monitor a running application";
String DESCRIBE_ACTION_REGISTRY =
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
index 0ccca0f..6380d0c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java
@@ -21,6 +21,9 @@
import java.io.Serializable;
import java.util.Comparator;
+/**
+ * Some general comparators
+ */
public class Comparators {
public static class LongComparator implements Comparator<Long>, Serializable {
@@ -30,12 +33,20 @@
// need to comparisons with a diff greater than integer size
if (result < 0 ) {
return -1;
- } else if (result >0) {
+ } else if (result > 0) {
return 1;
}
return 0;
}
}
+public static class InvertedLongComparator implements Comparator<Long>, Serializable {
+ private static final LongComparator inner = new LongComparator();
+ @Override
+ public int compare(Long o1, Long o2) {
+ return -inner.compare(o1, o2);
+ }
+ }
+
/**
* Little template class to reverse any comparitor
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index d34dd2a..a1ad690 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -159,7 +159,7 @@
Preconditions.checkNotNull(packageName);
Path path = getBaseApplicationPath();
path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName);
- if (StringUtils.isNotEmpty(packageVersion)) {
+ if (SliderUtils.isSet(packageVersion)) {
path = new Path(path, packageVersion);
}
return path;
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java b/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
index 25b68ae..e5fa424 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/Duration.java
@@ -68,7 +68,7 @@
}
protected long now() {
- return System.currentTimeMillis();
+ return System.nanoTime()/1000000;
}
public long getInterval() {
@@ -103,9 +103,7 @@
builder.append(" - exceeded");
}
}
-
return builder.toString();
}
-
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 8488f71..712eb75 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -35,7 +35,6 @@
import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
@@ -48,7 +47,9 @@
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.Slider;
import org.apache.slider.api.InternalKeys;
@@ -155,6 +156,7 @@
* name of docker program
*/
public static final String DOCKER = "docker";
+ public static final int NODE_LIST_LIMIT = 10;
private SliderUtils() {
}
@@ -501,8 +503,7 @@
//if the fallback option is NOT set, enable it.
//if it is explicitly set to anything -leave alone
- if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) ==
- null) {
+ if (conf.get(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH) == null) {
conf.set(SliderXmlConfKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH, "true");
}
return conf;
@@ -747,10 +748,7 @@
* through
*/
public static boolean filter(String value, String filter) {
- if (StringUtils.isEmpty(filter) || filter.equals(value)) {
- return false;
- }
- return true;
+ return !(StringUtils.isEmpty(filter) || filter.equals(value));
}
/**
@@ -1035,7 +1033,7 @@
* something other than 0.0.0.0
*/
public static boolean isAddressDefined(InetSocketAddress address) {
- return !(address.getHostName().equals("0.0.0.0"));
+ return !(address.getHostString().equals("0.0.0.0"));
}
public static void setRmAddress(Configuration conf, String rmAddr) {
@@ -1225,11 +1223,11 @@
* @param conf configuration to look at
* @return true if the cluster is secure
* @throws IOException cluster is secure
- * @throws BadConfigException the configuration/process is invalid
+ * @throws SliderException the configuration/process is invalid
*/
public static boolean maybeInitSecurity(Configuration conf) throws
IOException,
- BadConfigException {
+ SliderException {
boolean clusterSecure = isHadoopClusterSecure(conf);
if (clusterSecure) {
log.debug("Enabling security");
@@ -1247,7 +1245,7 @@
*/
public static boolean initProcessSecurity(Configuration conf) throws
IOException,
- BadConfigException {
+ SliderException {
if (processSecurityAlreadyInitialized.compareAndSet(true, true)) {
//security is already inited
@@ -1270,27 +1268,26 @@
UserGroupInformation.AuthenticationMethod.KERBEROS, conf);*/
UserGroupInformation.setConfiguration(conf);
UserGroupInformation authUser = UserGroupInformation.getCurrentUser();
- log.debug("Authenticating as " + authUser.toString());
+ log.debug("Authenticating as {}", authUser);
log.debug("Login user is {}", UserGroupInformation.getLoginUser());
if (!UserGroupInformation.isSecurityEnabled()) {
- throw new BadConfigException("Although secure mode is enabled," +
- "the application has already set up its user as an insecure entity %s",
+ throw new SliderException(LauncherExitCodes.EXIT_UNAUTHORIZE,
+ "Although secure mode is enabled," +
+ "the application has already set up its user as an insecure entity %s",
authUser);
}
if (authUser.getAuthenticationMethod() ==
UserGroupInformation.AuthenticationMethod.SIMPLE) {
throw new BadConfigException("Auth User is not Kerberized %s" +
- " -security has already been set up with the wrong authentication method. "
- +
- "This can occur if a file system has already been created prior to the loading of "
- + "the security configuration.",
+ " -security has already been set up with the wrong authentication method. "
+ + "This can occur if a file system has already been created prior to the loading of "
+ + "the security configuration.",
authUser);
}
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
return true;
}
@@ -1354,16 +1351,7 @@
log.info("Loading all dependencies from {}", srcPath);
if (SliderUtils.isSet(srcPath)) {
File srcFolder = new File(srcPath);
- FilenameFilter jarFilter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")) {
- return true;
- } else {
- return false;
- }
- }
- };
+ FilenameFilter jarFilter = createJarFilter();
File[] listOfJars = srcFolder.listFiles(jarFilter);
for (File jarFile : listOfJars) {
LocalResource res = sliderFileSystem.submitFile(jarFile, tempPath, libDir, jarFile.getName());
@@ -1373,6 +1361,18 @@
}
/**
+ * Accept all filenames ending with {@code .jar}
+ * @return a filename filter
+ */
+ public static FilenameFilter createJarFilter() {
+ return new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.toLowerCase(Locale.ENGLISH).endsWith(".jar");
+ }
+ };
+ }
+
+ /**
* Submit the AM tar.gz containing all dependencies and map it
* @param providerResources provider map to build up
* @param sliderFileSystem remote fs
@@ -1775,6 +1775,19 @@
return toTruncate.substring(0, maxSize - pad.length()).concat(pad);
}
+ /**
+ * Get a string node label value from a node report
+ * @param report node report
+ * @return a single trimmed label or ""
+ */
+ public static String extractNodeLabel(NodeReport report) {
+ Set<String> newlabels = report.getNodeLabels();
+ if (newlabels != null && !newlabels.isEmpty()) {
+ return newlabels.iterator().next().trim();
+ } else {
+ return "";
+ }
+ }
/**
* Callable for async/scheduled halt
@@ -2027,10 +2040,10 @@
errorText.append("No native IO library. ");
}
try {
- String path = Shell.getQualifiedBinPath("winutils.exe");
+ String path = Shell.getQualifiedBinPath(WINUTILS);
log.debug("winutils is at {}", path);
} catch (IOException e) {
- errorText.append("No WINUTILS.EXE. ");
+ errorText.append("No " + WINUTILS);
log.warn("No winutils: {}", e, e);
}
try {
@@ -2333,6 +2346,7 @@
public static String getClientConfigPath() {
URL path = ConfigHelper.class.getClassLoader().getResource(
SliderKeys.SLIDER_CLIENT_XML);
+ Preconditions.checkNotNull(path, "Failed to locate resource " + SliderKeys.SLIDER_CLIENT_XML);
return path.toString();
}
@@ -2469,7 +2483,7 @@
* @return +ve if x is less than y, -ve if y is greater than x; 0 for equality
*/
public static int compareTwoLongsReverse(long x, long y) {
- return (x < y) ? +1 : ((x == y) ? 0 : -1);
+ return (x < y) ? 1 : ((x == y) ? 0 : -1);
}
public static String getSystemEnv(String property) {
@@ -2479,4 +2493,34 @@
public static Map<String, String> getSystemEnv() {
return System.getenv();
}
+
+ public static String requestToString(AMRMClient.ContainerRequest request) {
+ Preconditions.checkArgument(request != null, "Null request");
+ StringBuilder buffer = new StringBuilder(request.toString());
+ buffer.append("; ");
+ buffer.append("relaxLocality=").append(request.getRelaxLocality()).append("; ");
+ String labels = request.getNodeLabelExpression();
+ if (labels != null) {
+ buffer.append("nodeLabels=").append(labels).append("; ");
+ }
+ List<String> nodes = request.getNodes();
+ if (nodes != null) {
+ buffer.append("Nodes = [ ");
+ int size = nodes.size();
+ for (int i = 0; i < Math.min(NODE_LIST_LIMIT, size); i++) {
+ buffer.append(nodes.get(i)).append(' ');
+ }
+ if (size > NODE_LIST_LIMIT) {
+ buffer.append(String.format("...(total %d entries)", size));
+ }
+ buffer.append("]; ");
+ }
+ List<String> racks = request.getRacks();
+ if (racks != null) {
+ buffer.append("racks = [")
+ .append(join(racks, ", ", false))
+ .append("]; ");
+ }
+ return buffer.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 0a8dcdb..7756055 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -23,7 +23,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.StatusKeys;
@@ -203,11 +202,10 @@
* Propagate any critical principals from the current site config down to the HBase one.
*/
public void propagatePrincipals() {
- String dfsPrincipal = conf.get(
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ String dfsPrincipal = conf.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+ SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index bc116e7..58896ee 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -432,5 +432,14 @@
public void setComponentOpt(String role, String option, int val) {
setComponentOpt(role, option, Integer.toString(val));
}
+ /**
+ * Set a long role option, creating the role if necessary
+ * @param role role name
+ * @param option option name
+ * @param val long value
+ */
+ public void setComponentOpt(String role, String option, long val) {
+ setComponentOpt(role, option, Long.toString(val));
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
index 5f7b5f0..e58178c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/MapOperations.java
@@ -138,6 +138,21 @@
String val = getOption(option, Integer.toString(defVal));
return Integer.decode(val);
}
+
+ /**
+ * Get a long option; use {@link Long#decode(String)} so as to take hex
+ * oct and bin values too.
+ *
+ * @param option option name
+ * @param defVal default value
+ * @return parsed value
+ * @throws NumberFormatException
+ */
+ public long getOptionLong(String option, long defVal) {
+ String val = getOption(option, Long.toString(defVal));
+ return Long.decode(val);
+ }
+
/**
* Get a mandatory integer option; use {@link Integer#decode(String)} so as to take hex
* oct and bin values too.
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 93aff08..22bf328 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
@@ -279,6 +280,30 @@
}
}
+ /**
+ * Extract the value for option
+ * yarn.resourcemanager.am.retry-count-window-ms
+ * and set it on the ApplicationSubmissionContext. Use the default value
+ * if option is not set.
+ *
+ * @param submissionContext
+ * @param map
+ */
+ public void extractAmRetryCount(ApplicationSubmissionContext submissionContext,
+ Map<String, String> map) {
+
+ if (map != null) {
+ MapOperations options = new MapOperations("", map);
+ long amRetryCountWindow = options.getOptionLong(ResourceKeys
+ .YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
+ ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS);
+ log.info("Setting {} to {}",
+ ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
+ amRetryCountWindow);
+ submissionContext.setAttemptFailuresValidityInterval(amRetryCountWindow);
+ }
+ }
+
public void extractLogAggregationContext(Map<String, String> map) {
if (map != null) {
String logPatternSepStr = "\\|";
@@ -423,7 +448,7 @@
}
/**
- * Suubmit an entire directory
+ * Submit an entire directory
* @param srcDir src path in filesystem
* @param destRelativeDir relative path under destination local dir
* @throws IOException IO problems
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
index b4ca791..c82affa 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AppMasterLauncher.java
@@ -106,6 +106,8 @@
submissionContext.setApplicationTags(applicationTags);
}
submissionContext.setNodeLabelExpression(extractLabelExpression(options));
+
+ extractAmRetryCount(submissionContext, resourceGlobalOptions);
extractResourceRequirements(resource, options);
extractLogAggregationContext(resourceGlobalOptions);
}
@@ -220,8 +222,7 @@
}
Token<? extends TokenIdentifier>[] tokens = null;
- boolean tokensProvided =
- this.getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
+ boolean tokensProvided = getConf().get(MAPREDUCE_JOB_CREDENTIALS_BINARY) != null;
if (!tokensProvided) {
// For now, only getting tokens for the default file-system.
FileSystem fs = coreFileSystem.getFileSystem();
@@ -232,7 +233,7 @@
if (tokens != null && tokens.length > 0) {
AbstractDelegationTokenIdentifier id =
(AbstractDelegationTokenIdentifier)tokens[0].decodeIdentifier();
- Date d = new Date(id.getIssueDate() + 24*60*60*1000);
+ Date d = new Date(id.getIssueDate() + 24 * 60 * 60 * 1000);
log.info("HDFS delegation tokens for AM launch context require renewal by {}",
DateFormat.getDateTimeInstance().format(d));
} else {
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
index 3ba0181..6eb4058 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ClasspathConstructor.java
@@ -38,7 +38,6 @@
public class ClasspathConstructor {
public static final String CLASS_PATH_SEPARATOR = ApplicationConstants.CLASS_PATH_SEPARATOR;
-// public static final String CLASS_PATH_SEPARATOR = File.pathSeparator;
private final List<String> pathElements = new ArrayList<>();
public ClasspathConstructor() {
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index f8ea4ee..69b937d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -55,17 +55,13 @@
public UserGroupInformation setupUGI() {
UserGroupInformation user =
UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr =
- container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress =
- NetUtils.createSocketAddr(cmIpPortStr);
+ String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
+ final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
- org.apache.hadoop.yarn.api.records.Token containerToken =
- container.getContainerToken();
+ org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
if (containerToken != null) {
Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken,
- cmAddress);
+ ConverterUtils.convertFromYarn(containerToken, cmAddress);
user.addToken(token);
}
return user;
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 0b3fa10..dcb322e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -20,11 +20,15 @@
import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
/**
- * Command line builder purely for the Java CLI
+ * Command line builder purely for the Java CLI.
+ * Some of the <code>define</code> methods are designed to work with Hadoop tool and
+ * Slider launcher applications.
*/
public class JavaCommandLineBuilder extends CommandLineBuilder {
@@ -80,4 +84,68 @@
sysprop("java.awt.headless", "true");
return this;
}
+
+ public boolean addConfOption(Configuration conf, String key) {
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String...keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
+ }
+
+ public String addConfOptionToCLI(Configuration conf,
+ String key,
+ String defVal) {
+ String val = conf.get(key, defVal);
+ define(key, val);
+ return val;
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI. This is very Hadoop API
+ * @param key key
+ * @param val value
+ */
+ public void define(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ Preconditions.checkArgument(val != null, "null value");
+ add("-D", key + "=" + val);
+ }
+
+ /**
+ * Add a <code>-D key=val</code> command to the CLI if <code>val</code>
+ * is not null
+ * @param key key
+ * @param val value
+ */
+ public boolean defineIfSet(String key, String val) {
+ Preconditions.checkArgument(key != null, "null key");
+ if (val != null) {
+ define(key, val);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Add a mandatory config option
+ * @param conf configuration
+ * @param key key
+ * @throws BadConfigException if the key is missing
+ */
+ public void addMandatoryConfOption(Configuration conf,
+ String key) throws BadConfigException {
+ if (!addConfOption(conf, key)) {
+ throw new BadConfigException("Missing configuration option: " + key);
+ }
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
index 60717f6..9759205 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/ConfPersister.java
@@ -101,7 +101,7 @@
* Make the persistent directory
* @throws IOException IO failure
*/
- private void mkPersistDir() throws IOException {
+ public void mkPersistDir() throws IOException {
coreFS.getFileSystem().mkdirs(persistDir);
}
@@ -165,13 +165,15 @@
* Acquire the writelock
* @throws IOException IO
* @throws LockAcquireFailedException
- * @throws FileNotFoundException if the target dir does not exist
+ * @throws FileNotFoundException if the target dir does not exist.
*/
@VisibleForTesting
boolean acquireReadLock() throws FileNotFoundException,
IOException,
LockAcquireFailedException {
if (!coreFS.getFileSystem().exists(persistDir)) {
+ // the dir is not there, so the data is not there, so there
+ // is nothing to read
throw new FileNotFoundException(persistDir.toString());
}
long now = System.currentTimeMillis();
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
index c5908bb..4f60c06 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/JsonSerDeser.java
@@ -91,10 +91,11 @@
*/
public T fromFile(File jsonFile)
throws IOException, JsonParseException, JsonMappingException {
+ File absoluteFile = jsonFile.getAbsoluteFile();
try {
- return mapper.readValue(jsonFile, classType);
+ return mapper.readValue(absoluteFile, classType);
} catch (IOException e) {
- log.error("Exception while parsing json file {}: {}", jsonFile, e);
+ log.error("Exception while parsing json file {}", absoluteFile, e);
throw e;
}
}
@@ -106,7 +107,6 @@
* @throws IOException IO problems
* @throws JsonMappingException failure to map from the JSON to this class
*/
-/* JDK7
public T fromResource(String resource)
throws IOException, JsonParseException, JsonMappingException {
try(InputStream resStream = this.getClass().getResourceAsStream(resource)) {
@@ -115,34 +115,9 @@
}
return (T) (mapper.readValue(resStream, classType));
} catch (IOException e) {
- log.error("Exception while parsing json resource {}: {}", resource, e);
+ log.error("Exception while parsing json resource {}", resource, e);
throw e;
}
- }*/
-
- /**
- * Convert from a JSON file
- * @param resource input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
- public synchronized T fromResource(String resource)
- throws IOException, JsonParseException, JsonMappingException {
- InputStream resStream = null;
- try {
- resStream = this.getClass().getResourceAsStream(resource);
- if (resStream == null) {
- throw new FileNotFoundException(resource);
- }
- return (T) (mapper.readValue(resStream, classType));
- } catch (IOException e) {
- log.error("Exception while parsing json resource {}: {}", resource, e);
- throw e;
- } finally {
- IOUtils.closeStream(resStream);
- }
}
/**
@@ -155,7 +130,7 @@
try {
return (T) (mapper.readValue(stream, classType));
} catch (IOException e) {
- log.error("Exception while parsing json input stream: {}", e);
+ log.error("Exception while parsing json input stream", e);
throw e;
} finally {
IOUtils.closeStream(stream);
@@ -201,7 +176,7 @@
FSDataInputStream dataInputStream = fs.open(path);
int count = dataInputStream.read(b);
if (count != len) {
- throw new EOFException("Read finished prematurely");
+ throw new EOFException("Read of " + path +" finished prematurely");
}
return fromBytes(b);
}
@@ -230,9 +205,9 @@
*/
public void save(T instance, File file) throws
IOException {
- writeJsonAsBytes(instance, new FileOutputStream(file));
+ writeJsonAsBytes(instance, new FileOutputStream(file.getAbsoluteFile()));
}
-
+
/**
* Write the json as bytes -then close the file
* @param dataOutputStream an outout stream that will always be closed
@@ -251,7 +226,6 @@
}
}
-
/**
* Convert an object to a JSON string
* @param instance instance to convert
@@ -265,6 +239,5 @@
mapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
return mapper.writeValueAsString(instance);
}
-
-
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
index eac34c0..edc129e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfigSet.java
@@ -41,7 +41,7 @@
RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
public Map<String, PublishedConfiguration> configurations =
- new HashMap<String, PublishedConfiguration>();
+ new HashMap<>();
public PublishedConfigSet() {
}
@@ -84,7 +84,7 @@
}
public Set<String> keys() {
- TreeSet<String> keys = new TreeSet<String>();
+ TreeSet<String> keys = new TreeSet<>();
keys.addAll(configurations.keySet());
return keys;
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
index e453fb1..50b522f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedConfiguration.java
@@ -47,7 +47,7 @@
public String updatedTime;
- public Map<String, String> entries = new HashMap<String, String>();
+ public Map<String, String> entries = new HashMap<>();
public PublishedConfiguration() {
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
index bf96e8c..0759b4e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExports.java
@@ -41,7 +41,7 @@
public String description;
public long updated;
public String updatedTime;
- public Map<String, List<ExportEntry>> entries = new HashMap<String, List<ExportEntry>>();
+ public Map<String, List<ExportEntry>> entries = new HashMap<>();
public PublishedExports() {
}
@@ -68,7 +68,8 @@
}
/**
- * Is the configuration empty. This means either that it has not been given any values, or it is stripped down copy
+ * Is the configuration empty. This means either that it has not been given any values,
+ * or it is stripped down copy
* set down over the wire.
*
* @return true if it is empty
@@ -87,8 +88,8 @@
}
/**
- * Set the values from an iterable (this includes a Hadoop Configuration and Java properties object). Any existing
- * value set is discarded
+ * Set the values from an iterable (this includes a Hadoop Configuration and Java properties
+ * object). Any existing value set is discarded
*
* @param entries entries to put
*/
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
index cdd35de..339d3d6 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/PublishedExportsSet.java
@@ -40,8 +40,7 @@
private static final PatternValidator validator = new PatternValidator(
RestPaths.PUBLISHED_CONFIGURATION_REGEXP);
- public Map<String, PublishedExports> exports =
- new HashMap<String, PublishedExports>();
+ public Map<String, PublishedExports> exports = new HashMap<>();
public PublishedExportsSet() {
}
@@ -84,15 +83,14 @@
}
public Set<String> keys() {
- TreeSet<String> keys = new TreeSet<String>();
+ TreeSet<String> keys = new TreeSet<>();
keys.addAll(exports.keySet());
return keys;
}
public PublishedExportsSet shallowCopy() {
PublishedExportsSet that = new PublishedExportsSet();
- for (Map.Entry<String, PublishedExports> entry :
- exports.entrySet()) {
+ for (Map.Entry<String, PublishedExports> entry : exports.entrySet()) {
that.put(entry.getKey(), entry.getValue().shallowCopy());
}
return that;
diff --git a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
index a76e28d..120966f 100644
--- a/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
+++ b/slider-core/src/main/java/org/apache/slider/core/registry/docstore/UriMap.java
@@ -29,7 +29,7 @@
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class UriMap {
- public Map<String, String> uris = new HashMap<String, String>();
+ public Map<String, String> uris = new HashMap<>();
@JsonIgnore
public void put(String key, String value) {
diff --git a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 9a9cae4..ca41e4b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -57,7 +57,7 @@
ZK_USERS_PATH_LIST.add(ZK_USERS);
}
- public static int SESSION_TIMEOUT = 5000;
+ public static int SESSION_TIMEOUT = 30000;
protected static final Logger log =
LoggerFactory.getLogger(ZKIntegration.class);
private ZooKeeper zookeeper;
@@ -80,7 +80,8 @@
String clustername,
boolean canBeReadOnly,
boolean createClusterPath,
- Watcher watchEventHandler
+ Watcher watchEventHandler,
+ int sessionTimeout
) throws IOException {
this.username = username;
this.clustername = clustername;
@@ -88,6 +89,7 @@
this.zkConnection = zkConnection;
this.canBeReadOnly = canBeReadOnly;
this.createClusterPath = createClusterPath;
+ this.sessionTimeout = sessionTimeout;
this.userPath = mkSliderUserPath(username);
}
@@ -107,14 +109,21 @@
* @return the new instance
* @throws IOException
*/
- public static ZKIntegration newInstance(String zkConnection, String username, String clustername, boolean createClusterPath, boolean canBeReadOnly, Watcher watchEventHandler) throws IOException {
+ public static ZKIntegration newInstance(String zkConnection,
+ String username,
+ String clustername,
+ boolean createClusterPath,
+ boolean canBeReadOnly,
+ Watcher watchEventHandler,
+ int sessionTimeout) throws IOException {
return new ZKIntegration(zkConnection,
username,
clustername,
canBeReadOnly,
createClusterPath,
- watchEventHandler);
+ watchEventHandler,
+ sessionTimeout);
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 7cba840..c701a55 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -81,7 +81,7 @@
protected YarnRegistryViewForProviders yarnRegistry;
protected QueueAccess queueAccess;
- public AbstractProviderService(String name) {
+ protected AbstractProviderService(String name) {
super(name);
setStopIfNoChildServicesAtStartup(false);
}
@@ -343,12 +343,6 @@
return details;
}
-
- protected String getInfoAvoidingNull(ClusterDescription clusterDesc, String key) {
- String value = clusterDesc.getInfo(key);
-
- return null == value ? "N/A" : value;
- }
@Override
public void buildEndpointDetails(Map<String, String> details) {
@@ -363,10 +357,8 @@
if (!urls.isEmpty()) {
details.put(endpoint.api, urls.get(0).toString());
}
- } catch (InvalidRecordException ignored) {
+ } catch (InvalidRecordException | MalformedURLException ignored) {
// Ignored
- } catch (MalformedURLException ignored) {
- // ignored
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
index 4e85a93..71d7566 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/PlacementPolicy.java
@@ -27,7 +27,12 @@
/**
* Default value: history used, anti-affinity hinted at on rebuild/flex up
*/
- public static final int DEFAULT = 0;
+ public static final int NONE = 0;
+
+ /**
+ * Default value: history used, anti-affinity hinted at on rebuild/flex up
+ */
+ public static final int DEFAULT = NONE;
/**
* Strict placement: when asking for an instance for which there is
@@ -41,7 +46,7 @@
public static final int NO_DATA_LOCALITY = 2;
/**
- * Anti-affinity is mandatory. This is not supported in YARN
+ * Anti-affinity is mandatory.
*/
public static final int ANTI_AFFINITY_REQUIRED = 4;
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 3009f50..1b95b42 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -32,13 +32,15 @@
public int placementPolicy;
public int nodeFailureThreshold;
public final long placementTimeoutSeconds;
+ public final String labelExpression;
public ProviderRole(String name, int id) {
this(name,
id,
PlacementPolicy.DEFAULT,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+ ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION);
}
/**
@@ -49,18 +51,20 @@
* @param nodeFailureThreshold threshold for node failures (within a reset interval)
* after which a node failure is considered an app failure
* @param placementTimeoutSeconds for lax placement, timeout in seconds before
- * a relaxed placement request is generated.
+ * @param labelExpression label expression for requests; may be null
*/
public ProviderRole(String name,
int id,
int policy,
int nodeFailureThreshold,
- long placementTimeoutSeconds) {
+ long placementTimeoutSeconds,
+ String labelExpression) {
this.name = name;
this.id = id;
this.placementPolicy = policy;
this.nodeFailureThreshold = nodeFailureThreshold;
this.placementTimeoutSeconds = placementTimeoutSeconds;
+ this.labelExpression = labelExpression;
}
@Override
@@ -83,11 +87,14 @@
@Override
public String toString() {
- return "ProviderRole {" +
- "name='" + name + '\'' +
- ", id=" + id +
- ", policy=" + placementPolicy +
- ", nodeFailureThreshold=" + nodeFailureThreshold +
- '}';
+ final StringBuilder sb = new StringBuilder("ProviderRole{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", id=").append(id);
+ sb.append(", placementPolicy=").append(placementPolicy);
+ sb.append(", nodeFailureThreshold=").append(nodeFailureThreshold);
+ sb.append(", placementTimeoutSeconds=").append(placementTimeoutSeconds);
+ sb.append(", labelExpression='").append(labelExpression).append('\'');
+ sb.append('}');
+ return sb.toString();
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index 5967a3a..3a445f4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -16,10 +16,10 @@
*/
package org.apache.slider.providers.agent.application.metadata;
-import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonProperty;
import java.util.ArrayList;
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index e1dc4f9..e5430f2 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -90,7 +90,7 @@
new ProviderRole(COMPONENT_AM, KEY_AM,
PlacementPolicy.EXCLUDE_FROM_FLEXING,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
- 0);
+ 0, "");
/**
* Initialize role list
@@ -174,8 +174,7 @@
Path tempPath, boolean miniClusterTestRun)
throws IOException, SliderException {
- Map<String, LocalResource> providerResources =
- new HashMap<String, LocalResource>();
+ Map<String, LocalResource> providerResources = new HashMap<>();
ProviderUtils.addProviderJar(providerResources,
this,
@@ -219,11 +218,12 @@
* @param instanceDescription
* @param providerResources
* @throws IOException
+ * @throws BadConfigException if there's no keytab and it is explicitly required.
*/
protected void addKeytabResourceIfNecessary(SliderFileSystem fileSystem,
AggregateConf instanceDescription,
Map<String, LocalResource> providerResources)
- throws IOException {
+ throws IOException, BadConfigException {
if (UserGroupInformation.isSecurityEnabled()) {
String keytabPathOnHost = instanceDescription.getAppConfOperations()
.getComponent(SliderKeys.COMPONENT_AM).get(
@@ -244,10 +244,16 @@
providerResources.put(SliderKeys.KEYTAB_DIR + "/" +
amKeytabName, keytabRes);
} else {
- log.warn("No keytab file was found at {}. The AM will be "
- + "started without a kerberos authenticated identity. "
- + "The application is therefore not guaranteed to remain "
- + "operational beyond 24 hours.", keytabPath);
+ log.warn("No keytab file was found at {}.", keytabPath);
+ if (getConf().getBoolean(KEY_AM_LOGIN_KEYTAB_REQUIRED, false)) {
+ throw new BadConfigException("No keytab file was found at %s.", keytabPath);
+
+ } else {
+ log.warn("The AM will be "
+ + "started without a kerberos authenticated identity. "
+ + "The application is therefore not guaranteed to remain "
+ + "operational beyond 24 hours.");
+ }
}
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index cee7a97..e382058 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -88,12 +88,11 @@
MapOperations resourceComponent,
MapOperations appComponent,
Path containerTmpDirPath) throws IOException, SliderException {
-
}
@Override
public List<ProviderRole> getRoles() {
- return new ArrayList<ProviderRole>(0);
+ return new ArrayList<>(0);
}
@Override
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
deleted file mode 100644
index 32684c6..0000000
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AMUtils.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster;
-
-import org.apache.slider.common.SliderExitCodes;
-import org.apache.slider.core.main.LauncherExitCodes;
-
-public class AMUtils {
- /**
- * Map an exit code from a process
- * @param exitCode
- * @return an exit code
- */
- public static int mapProcessExitCodeToYarnExitCode(int exitCode) {
- switch (exitCode) {
- case LauncherExitCodes.EXIT_SUCCESS:
- return LauncherExitCodes.EXIT_SUCCESS;
- //remap from a planned shutdown to a failure
- case LauncherExitCodes.EXIT_CLIENT_INITIATED_SHUTDOWN:
- return SliderExitCodes.EXIT_SUCCESS;
- default:
- return exitCode;
- }
- }
-
- public static boolean isMappedExitAFailure(int mappedExitCode) {
- return mappedExitCode != 0;
- }
-
-}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
index d78b718..288f25a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/AppMasterActionOperations.java
@@ -18,11 +18,8 @@
package org.apache.slider.server.appmaster;
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.appmaster.operations.RMOperationHandlerActions;
-import java.util.List;
-
/**
* Interface of AM operations
*/
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
similarity index 75%
rename from slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
rename to slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
index d7f79f1..5d52441 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufRecordFactory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/ProtobufClusterServices.java
@@ -20,10 +20,17 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.slider.server.appmaster.state.AbstractClusterServices;
-public class ProtobufRecordFactory extends AbstractRecordFactory {
+public class ProtobufClusterServices extends AbstractClusterServices {
+
public Resource newResource() {
return Records.newRecord(Resource.class);
}
+
+ @Override
+ public Resource newResource(int memory, int cores) {
+ return Resources.createResource(memory, cores);
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 45e2ff9..18d5bfa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -20,6 +20,9 @@
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.health.HealthCheckRegistry;
+import com.codahale.metrics.jvm.GarbageCollectorMetricSet;
+import com.codahale.metrics.jvm.MemoryUsageGaugeSet;
+import com.codahale.metrics.jvm.ThreadStatesGaugeSet;
import com.google.common.base.Preconditions;
import com.google.protobuf.BlockingService;
@@ -29,7 +32,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
@@ -55,6 +57,7 @@
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -62,6 +65,7 @@
import org.apache.hadoop.yarn.client.api.async.NMClientAsync;
import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.*;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -82,6 +86,7 @@
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.proto.SliderClusterAPI;
+import org.apache.slider.client.SliderYarnClientImpl;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractActionArgs;
@@ -141,6 +146,7 @@
import org.apache.slider.server.appmaster.rpc.SliderIPCService;
import org.apache.slider.server.appmaster.security.SecurityConfiguration;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ContainerAssignment;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.operations.RMOperationHandler;
@@ -166,6 +172,7 @@
import java.io.File;
import java.io.IOException;
+import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
@@ -208,8 +215,7 @@
*/
protected static final Logger LOG_YARN = log;
- public static final String SERVICE_CLASSNAME_SHORT =
- "SliderAppMaster";
+ public static final String SERVICE_CLASSNAME_SHORT = "SliderAppMaster";
public static final String SERVICE_CLASSNAME =
"org.apache.slider.server.appmaster." + SERVICE_CLASSNAME_SHORT;
@@ -221,10 +227,13 @@
* Deployed in {@link #serviceInit(Configuration)}
*/
private final MetricsAndMonitoring metricsAndMonitoring = new MetricsAndMonitoring();
+
/**
* metrics registry
*/
public MetricRegistry metrics;
+
+ /** Error string on chaos monkey launch failure action: {@value} */
public static final String E_TRIGGERED_LAUNCH_FAILURE =
"Chaos monkey triggered launch failure";
@@ -244,7 +253,6 @@
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
public NMClientAsync nmClientAsync;
-// YarnConfiguration conf;
/**
* token blob
*/
@@ -291,7 +299,7 @@
* live on, etc.
*/
private final AppState appState =
- new AppState(new ProtobufRecordFactory(), metricsAndMonitoring);
+ new AppState(new ProtobufClusterServices(), metricsAndMonitoring);
/**
* App state for external objects. This is almost entirely
@@ -301,7 +309,6 @@
private final ProviderAppState stateForProviders =
new ProviderAppState("undefined", appState);
-
/**
* model the state using locks and conditions
*/
@@ -353,16 +360,6 @@
private RegistryOperations registryOperations;
/**
- * Record of the max no. of cores allowed in this cluster
- */
- private int containerMaxCores;
-
- /**
- * limit container memory
- */
- private int containerMaxMemory;
-
- /**
* The stop request received...the exit details are extracted
* from this
*/
@@ -412,6 +409,12 @@
*/
private boolean securityEnabled;
private ContentCache contentCache;
+ private SliderYarnClientImpl yarnClient;
+
+ /**
+ * resource limits
+ */
+ private Resource maximumResourceCapability;
/**
* Service Constructor
@@ -434,10 +437,8 @@
// Load in the server configuration - if it is actually on the Classpath
URL serverXmlUrl = ConfigHelper.getResourceUrl(SLIDER_SERVER_XML);
if (serverXmlUrl != null) {
-
log.info("Loading {} at {}", SLIDER_SERVER_XML, serverXmlUrl);
- Configuration serverConf =
- ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
+ Configuration serverConf = ConfigHelper.loadFromResource(SLIDER_SERVER_XML);
ConfigHelper.mergeConfigurations(customConf, serverConf,
SLIDER_SERVER_XML, true);
}
@@ -469,8 +470,7 @@
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
log.debug("Authenticating as {}", ugi);
- SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ SliderUtils.verifyPrincipalSet(conf, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
} else {
log.info("Cluster is insecure");
}
@@ -485,22 +485,24 @@
false);
SliderUtils.validateSliderServerEnvironment(log, dependencyChecks);
- // create app state and monitoring
+ // create and register monitoring services
addService(metricsAndMonitoring);
metrics = metricsAndMonitoring.getMetrics();
+/* TODO: turn these one once the metrics testing is more under control
+ metrics.registerAll(new ThreadStatesGaugeSet());
+ metrics.registerAll(new MemoryUsageGaugeSet());
+ metrics.registerAll(new GarbageCollectorMetricSet());
+*/
+ contentCache = ApplicationResouceContentCacheFactory.createContentCache(stateForProviders);
- contentCache = ApplicationResouceContentCacheFactory.createContentCache(
- stateForProviders);
-
-
- executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor",
+ executorService = new WorkflowExecutorService<>("AmExecutor",
Executors.newFixedThreadPool(2,
new ServiceThreadFactory("AmExecutor", true)));
addService(executorService);
addService(actionQueues);
-
+
//init all child services
super.serviceInit(conf);
}
@@ -531,8 +533,7 @@
* @param args argument list
*/
@Override // RunService
- public Configuration bindArgs(Configuration config, String... args) throws
- Exception {
+ public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
// add the slider XML config
@@ -559,19 +560,17 @@
//dump the system properties if in debug mode
if (log.isDebugEnabled()) {
- log.debug("System properties:\n" +
- SliderUtils.propertiesToString(System.getProperties()));
+ log.debug("System properties:\n" + SliderUtils.propertiesToString(System.getProperties()));
}
//choose the action
String action = serviceArgs.getAction();
List<String> actionArgs = serviceArgs.getActionArgs();
int exitCode;
-/* JDK7
- switch (action) {
+ switch (action) {
case SliderActions.ACTION_HELP:
- log.info(getName() + serviceArgs.usage());
- exitCode = LauncherExitCodes.EXIT_USAGE;
+ log.info("{}: {}", getName(), serviceArgs.usage());
+ exitCode = SliderExitCodes.EXIT_USAGE;
break;
case SliderActions.ACTION_CREATE:
exitCode = createAndRunCluster(actionArgs.get(0));
@@ -579,20 +578,10 @@
default:
throw new SliderException("Unimplemented: " + action);
}
- */
- if (action.equals(SliderActions.ACTION_HELP)) {
- log.info("{}: {}", getName(), serviceArgs.usage());
- exitCode = SliderExitCodes.EXIT_USAGE;
- } else if (action.equals(SliderActions.ACTION_CREATE)) {
- exitCode = createAndRunCluster(actionArgs.get(0));
- } else {
- throw new SliderException("Unimplemented: " + action);
- }
log.info("Exiting AM; final exit code = {}", exitCode);
return exitCode;
}
-
/**
* Initialize a newly created service then add it.
* Because the service is not started, this MUST be done before
@@ -636,13 +625,11 @@
Configuration serviceConf = getConfig();
- securityConfiguration = new SecurityConfiguration(
- serviceConf, instanceDefinition, clustername);
+ securityConfiguration = new SecurityConfiguration(serviceConf, instanceDefinition, clustername);
// obtain security state
securityEnabled = securityConfiguration.isSecurityEnabled();
// set the global security flag for the instance definition
- instanceDefinition.getAppConfOperations().set(
- KEY_SECURITY_ENABLED, securityEnabled);
+ instanceDefinition.getAppConfOperations().set(KEY_SECURITY_ENABLED, securityEnabled);
// triggers resolution and snapshotting for agent
appState.setInitialInstanceDefinition(instanceDefinition);
@@ -660,61 +647,68 @@
InternalKeys.INTERNAL_PROVIDER_NAME);
log.info("Cluster provider type is {}", providerType);
SliderProviderFactory factory =
- SliderProviderFactory.createSliderProviderFactory(
- providerType);
+ SliderProviderFactory.createSliderProviderFactory(providerType);
providerService = factory.createServerProvider();
// init the provider BUT DO NOT START IT YET
initAndAddService(providerService);
- providerRMOperationHandler =
- new ProviderNotifyingOperationHandler(providerService);
+ providerRMOperationHandler = new ProviderNotifyingOperationHandler(providerService);
// create a slider AM provider
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", address);
+ InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // client addr is being unset. We can lift it from the other RM APIs
+ log.warn("Yarn RM address was unbound; attempting to fix up");
+ serviceConf.set(YarnConfiguration.RM_ADDRESS,
+ String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
+ }
+ initAndAddService(yarnClient = new SliderYarnClientImpl());
+ yarnClient.start();
+
/*
* Extract the container ID. This is then
* turned into an (incompete) container
*/
appMasterContainerID = ConverterUtils.toContainerId(
- SliderUtils.mandatoryEnvVariable(
- ApplicationConstants.Environment.CONTAINER_ID.name())
- );
+ SliderUtils.mandatoryEnvVariable(ApplicationConstants.Environment.CONTAINER_ID.name()));
appAttemptID = appMasterContainerID.getApplicationAttemptId();
ApplicationId appid = appAttemptID.getApplicationId();
log.info("AM for ID {}", appid.getId());
- appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID,
- appMasterContainerID.toString());
- appInformation.put(StatusKeys.INFO_AM_APP_ID,
- appid.toString());
- appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID,
- appAttemptID.toString());
+ appInformation.put(StatusKeys.INFO_AM_CONTAINER_ID, appMasterContainerID.toString());
+ appInformation.put(StatusKeys.INFO_AM_APP_ID, appid.toString());
+ appInformation.put(StatusKeys.INFO_AM_ATTEMPT_ID, appAttemptID.toString());
Map<String, String> envVars;
List<Container> liveContainers;
-
- /**
+
+ List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
+ log.info("Yarn node report count: {}", nodeReports.size());
+
+ /*
* It is critical this section is synchronized, to stop async AM events
* arriving while registering a restarting AM.
*/
synchronized (appState) {
int heartbeatInterval = HEARTBEAT_INTERVAL;
- //add the RM client -this brings the callbacks in
- asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval,
- this);
+ // add the RM client -this brings the callbacks in
+ asyncRMClient = AMRMClientAsync.createAMRMClientAsync(heartbeatInterval, this);
addService(asyncRMClient);
//now bring it up
deployChildService(asyncRMClient);
- //nmclient relays callbacks back to this class
+ // nmclient relays callbacks back to this class
nmClientAsync = new NMClientAsyncImpl("nmclient", this);
deployChildService(nmClientAsync);
@@ -723,7 +717,7 @@
if (securityEnabled) {
// fix up the ACLs if they are not set
- String acls = getConfig().get(KEY_PROTOCOL_ACL);
+ String acls = serviceConf.get(KEY_PROTOCOL_ACL);
if (acls == null) {
getConfig().set(KEY_PROTOCOL_ACL, "*");
}
@@ -739,8 +733,7 @@
appMasterHostname = rpcServiceAddress.getHostName();
appMasterRpcPort = rpcServiceAddress.getPort();
appMasterTrackingUrl = null;
- log.info("AM Server is listening at {}:{}", appMasterHostname,
- appMasterRpcPort);
+ log.info("AM Server is listening at {}:{}", appMasterHostname, appMasterRpcPort);
appInformation.put(StatusKeys.INFO_AM_HOSTNAME, appMasterHostname);
appInformation.set(StatusKeys.INFO_AM_RPC_PORT, appMasterRpcPort);
@@ -749,8 +742,7 @@
log.info(registryOperations.toString());
//build the role map
- List<ProviderRole> providerRoles =
- new ArrayList<ProviderRole>(providerService.getRoles());
+ List<ProviderRole> providerRoles = new ArrayList<>(providerService.getRoles());
providerRoles.addAll(SliderAMClientProvider.ROLES);
// Start up the WebApp and track the URL for it
@@ -785,25 +777,27 @@
// *****************************************************
log.info("Connecting to RM at {},address tracking URL={}",
appMasterRpcPort, appMasterTrackingUrl);
- amRegistrationData = asyncRMClient
- .registerApplicationMaster(appMasterHostname,
+ amRegistrationData = asyncRMClient.registerApplicationMaster(appMasterHostname,
appMasterRpcPort,
appMasterTrackingUrl);
- Resource maxResources =
- amRegistrationData.getMaximumResourceCapability();
- containerMaxMemory = maxResources.getMemory();
- containerMaxCores = maxResources.getVirtualCores();
- appState.setContainerLimits(maxResources.getMemory(),
- maxResources.getVirtualCores());
+ maximumResourceCapability = amRegistrationData.getMaximumResourceCapability();
+
+ int minMemory = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ // validate scheduler vcores allocation setting
+ int minCores = serviceConf.getInt(RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ int maxMemory = maximumResourceCapability.getMemory();
+ int maxCores = maximumResourceCapability.getVirtualCores();
+ appState.setContainerLimits(minMemory,maxMemory, minCores, maxCores );
// build the handler for RM request/release operations; this uses
// the max value as part of its lookup
- rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient,
- maxResources);
+ rmOperationHandler = new AsyncRMOperationHandler(asyncRMClient, maximumResourceCapability);
// set the RM-defined maximum cluster values
- appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(containerMaxCores));
- appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(containerMaxMemory));
+ appInformation.put(ResourceKeys.YARN_CORES, Integer.toString(maxCores));
+ appInformation.put(ResourceKeys.YARN_MEMORY, Integer.toString(maxMemory));
processAMCredentials(securityConfiguration);
@@ -820,8 +814,7 @@
// principal. Can do so now since AM registration with RM above required
// tokens associated to principal
String principal = securityConfiguration.getPrincipal();
- File localKeytabFile =
- securityConfiguration.getKeytabFile(instanceDefinition);
+ File localKeytabFile = securityConfiguration.getKeytabFile(instanceDefinition);
// Now log in...
login(principal, localKeytabFile);
// obtain new FS reference that should be kerberos based and different
@@ -838,8 +831,7 @@
Configuration providerConf =
providerService.loadProviderConfigurationInformation(confDir);
- providerService
- .initializeApplicationConfiguration(instanceDefinition, fs);
+ providerService.initializeApplicationConfiguration(instanceDefinition, fs);
providerService.validateApplicationConfiguration(instanceDefinition,
confDir,
@@ -849,15 +841,18 @@
Path historyDir = new Path(clusterDirPath, HISTORY_DIR_NAME);
//build the instance
- appState.buildInstance(instanceDefinition,
- serviceConf,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation,
- providerService.createContainerReleaseSelector());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = instanceDefinition;
+ binding.serviceConfig = serviceConf;
+ binding.publishedProviderConf = providerConf;
+ binding.roles = providerRoles;
+ binding.fs = fs.getFileSystem();
+ binding.historyPath = historyDir;
+ binding.liveContainers = liveContainers;
+ binding.applicationInfo = appInformation;
+ binding.releaseSelector = providerService.createContainerReleaseSelector();
+ binding.nodeReports = nodeReports;
+ appState.buildInstance(binding);
providerService.rebuildContainerDetails(liveContainers,
instanceDefinition.getName(), appState.getRolePriorityMap());
@@ -1000,7 +995,7 @@
.start(webApp);
WebAppService<SliderAMWebApp> webAppService =
- new WebAppService<SliderAMWebApp>("slider", webApp);
+ new WebAppService<>("slider", webApp);
deployChildService(webAppService);
}
@@ -1013,7 +1008,7 @@
// up to date token for container launches (getContainerCredentials()).
UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
Credentials credentials = currentUser.getCredentials();
- List<Text> filteredTokens = new ArrayList<Text>();
+ List<Text> filteredTokens = new ArrayList<>();
filteredTokens.add(AMRMTokenIdentifier.KIND_NAME);
boolean keytabProvided = securityConfiguration.isKeytabProvided();
@@ -1086,6 +1081,7 @@
protected void login(String principal, File localKeytabFile)
throws IOException, SliderException {
+ log.info("Logging in as {} with keytab {}", principal, localKeytabFile);
UserGroupInformation.loginUserFromKeytab(principal,
localKeytabFile.getAbsolutePath());
validateLoginUser(UserGroupInformation.getLoginUser());
@@ -1648,7 +1644,7 @@
// If components are specified as well, then grab all the containers of
// each of the components (roles)
if (CollectionUtils.isNotEmpty(components)) {
- Map<ContainerId, RoleInstance> liveContainers = appState.getLiveNodes();
+ Map<ContainerId, RoleInstance> liveContainers = appState.getLiveContainers();
if (CollectionUtils.isNotEmpty(liveContainers.keySet())) {
Map<String, Set<String>> roleContainerMap = prepareRoleContainerMap(liveContainers);
for (String component : components) {
@@ -1888,7 +1884,15 @@
LOG_YARN.info("onNodesUpdated({})", updatedNodes.size());
log.info("Updated nodes {}", updatedNodes);
// Check if any nodes are lost or revived and update state accordingly
- appState.onNodesUpdated(updatedNodes);
+
+ AppState.NodeUpdatedOutcome outcome = appState.onNodesUpdated(updatedNodes);
+ if (!outcome.operations.isEmpty()) {
+ execute(outcome.operations);
+ }
+ // trigger a review if the cluster changed
+ if (outcome.clusterChanged) {
+ reviewRequestAndReleaseNodes("nodes updated");
+ }
}
/**
@@ -1948,15 +1952,6 @@
/* =================================================================== */
/**
- * Update the cluster description with anything interesting
- */
- public synchronized ClusterDescription updateClusterStatus() {
- Map<String, String> providerStatus = providerService.buildProviderStatus();
- assert providerStatus != null : "null provider status";
- return appState.refreshClusterStatus(providerStatus);
- }
-
- /**
* Launch the provider service
*
* @param instanceDefinition definition of the service
@@ -1967,7 +1962,7 @@
protected synchronized void launchProviderService(AggregateConf instanceDefinition,
File confDir)
throws IOException, SliderException {
- Map<String, String> env = new HashMap<String, String>();
+ Map<String, String> env = new HashMap<>();
boolean execStarted = providerService.exec(instanceDefinition, confDir, env,
this);
if (execStarted) {
@@ -2249,7 +2244,7 @@
internals.getOptionBool(InternalKeys.CHAOS_MONKEY_ENABLED,
InternalKeys.DEFAULT_CHAOS_MONKEY_ENABLED);
if (!enabled) {
- log.info("Chaos monkey disabled");
+ log.debug("Chaos monkey disabled");
return false;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
index 1aa9088..7446e82 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/ActionKillContainer.java
@@ -77,7 +77,7 @@
public void execute(SliderAppMaster appMaster,
QueueAccess queueService,
AppState appState) throws Exception {
- List<AbstractRMOperation> opsList = new LinkedList<AbstractRMOperation>();
+ List<AbstractRMOperation> opsList = new LinkedList<>();
ContainerReleaseOperation release = new ContainerReleaseOperation(containerId);
opsList.add(release);
//now apply the operations
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
index 146dea4..34acade 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/QueueService.java
@@ -58,20 +58,20 @@
* Immediate actions.
*/
public final BlockingDeque<AsyncAction> actionQueue =
- new LinkedBlockingDeque<AsyncAction>();
+ new LinkedBlockingDeque<>();
/**
* Actions to be scheduled in the future
*/
- public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<AsyncAction>();
+ public final DelayQueue<AsyncAction> scheduledActions = new DelayQueue<>();
/**
* Map of renewing actions by name ... this is to allow them to
* be cancelled by name
*/
private final Map<String, RenewingAction<? extends AsyncAction>> renewingActions
- = new ConcurrentHashMap<String, RenewingAction<? extends AsyncAction>>();
-
+ = new ConcurrentHashMap<>();
+
/**
* Create a queue instance with a single thread executor
*/
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
new file mode 100644
index 0000000..33f8d85
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetric.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * A bool metric, mapped to an integer. true maps to 1, false to zero,
+ */
+public class BoolMetric implements Metric, Gauge<Integer> {
+
+ private final AtomicBoolean value;
+
+ public BoolMetric(boolean b) {
+ value = new AtomicBoolean(b);
+ }
+
+ public void set(boolean b) {
+ value.set(b);
+ }
+
+ public boolean get() {
+ return value.get();
+ }
+
+ @Override
+ public Integer getValue() {
+ return value.get() ? 1 : 0;
+ }
+
+ /**
+ * Evaluate from a string. Returns true if the string is considered to match 'true',
+ * false otherwise.
+ * @param s source
+ * @return true if the input parses to an integer other than 0. False if it doesn't parse
+ * or parses to 0.
+ */
+ public static boolean fromString(String s) {
+ try {
+ return Integer.valueOf(s) != 0;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return value.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ BoolMetric that = (BoolMetric) o;
+ return get() == that.get();
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
new file mode 100644
index 0000000..82bcd3a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/BoolMetricPredicate.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+/**
+ * A metric which takes a predicate and returns 1 if the predicate evaluates
+ * to true. The predicate is evaluated whenever the metric is read.
+ */
+public class BoolMetricPredicate implements Metric, Gauge<Integer> {
+
+ private final Eval predicate;
+
+ public BoolMetricPredicate(Eval predicate) {
+ this.predicate = predicate;
+ }
+
+ @Override
+ public Integer getValue() {
+ return predicate.eval() ? 1: 0;
+ }
+
+ public interface Eval {
+ boolean eval();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
new file mode 100644
index 0000000..c93467b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongGauge.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This is a {@link AtomicLong} which acts as a metrics gauge: its state can be exposed as
+ * a metrics.
+ * It also exposes some of the same method names as the Codahale Counter class, so that
+ * it's easy to swap in.
+ *
+ */
+public class LongGauge extends AtomicLong implements Metric, Gauge<Long> {
+
+ /**
+ * Instantiate
+ * @param val current value
+ */
+ public LongGauge(long val) {
+ super(val);
+ }
+
+ /**
+ * Instantiate with value 0
+ */
+ public LongGauge() {
+ this(0);
+ }
+
+ /**
+ * Get the value as a metric
+ * @return current value
+ */
+ @Override
+ public Long getValue() {
+ return get();
+ }
+
+ /**
+ * Method from {@Code counter}; used here for drop-in replacement
+ * without any recompile
+ * @return current value
+ */
+ public Long getCount() {
+ return get();
+ }
+
+ /**
+ * {@code ++}
+ */
+ public void inc() {
+ incrementAndGet();
+ }
+
+ /**
+ * {@code --}
+ */
+ public void dec() {
+ decrementAndGet();
+ }
+
+ /**
+ * Decrement to the floor of 0. Operations in parallel may cause confusion here,
+ * but it will still never go below zero
+ * @param delta delta
+ * @return the current value
+ */
+ public long decToFloor(long delta) {
+ long l = get();
+ long r = l - delta;
+ if (r < 0) {
+ r = 0;
+ }
+ // if this fails, the decrement has been lost
+ compareAndSet(l, r);
+ return get();
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
similarity index 60%
copy from slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
copy to slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
index f7d353f..1de7345 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/LongMetricFunction.java
@@ -16,15 +16,29 @@
* limitations under the License.
*/
-package org.apache.slider.server.appmaster.model.mock
+package org.apache.slider.server.appmaster.management;
-import org.apache.hadoop.yarn.api.records.Resource
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import com.codahale.metrics.Gauge;
+import com.codahale.metrics.Metric;
-class MockRecordFactory extends AbstractRecordFactory {
+/**
+ * A metric which takes a function to generate a long value.
+ * The function is evaluated whenever the metric is read.
+ */
+public class LongMetricFunction implements Metric, Gauge<Long> {
+
+ private final Eval function;
+
+ public LongMetricFunction(Eval function) {
+ this.function = function;
+ }
@Override
- Resource newResource() {
- return new MockResource()
+ public Long getValue() {
+ return function.eval();
+ }
+
+ public interface Eval {
+ long eval();
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java
index 58d9eb3..02ab7bc 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MeterAndCounter.java
@@ -74,6 +74,11 @@
meter.mark();
}
+ public void inc() {
+ mark();
+ }
+
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
index 5b96256..37a8935 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsAndMonitoring.java
@@ -18,13 +18,17 @@
package org.apache.slider.server.appmaster.management;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
import com.codahale.metrics.health.HealthCheckRegistry;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -32,7 +36,8 @@
* Class for all metrics and monitoring
*/
public class MetricsAndMonitoring extends CompositeService {
-
+ protected static final Logger log =
+ LoggerFactory.getLogger(MetricsAndMonitoring.class);
public MetricsAndMonitoring(String name) {
super(name);
}
@@ -49,9 +54,17 @@
final HealthCheckRegistry health = new HealthCheckRegistry();
private final Map<String, MeterAndCounter> meterAndCounterMap
- = new ConcurrentHashMap<String, MeterAndCounter>();
-
-
+ = new ConcurrentHashMap<>();
+
+ private final List<MetricSet> metricSets = new ArrayList<>();
+
+ /**
+ * List of recorded events
+ */
+ private final List<RecordedEvent> eventHistory = new ArrayList<>(100);
+
+ public static final int EVENT_LIMIT = 1000;
+
public MetricRegistry getMetrics() {
return metrics;
}
@@ -67,6 +80,14 @@
super.serviceInit(conf);
}
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ for (MetricSet set : metricSets) {
+ unregister(set);
+ }
+ }
+
public MeterAndCounter getMeterAndCounter(String name) {
return meterAndCounterMap.get(name);
}
@@ -92,7 +113,7 @@
}
/**
- * Get a specific meter and mark it
+ * Get a specific meter and mark it. This will create and register it on demand.
* @param name name of meter/counter
*/
public void markMeterAndCounter(String name) {
@@ -100,4 +121,75 @@
meter.mark();
}
+ /**
+ * Given a {@link Metric}, registers it under the given name.
+ *
+ * @param name the name of the metric
+ * @param metric the metric
+ * @param <T> the type of the metric
+ * @return {@code metric}
+ * @throws IllegalArgumentException if the name is already registered
+ */
+ public <T extends Metric> T register(String name, T metric) throws IllegalArgumentException {
+ return metrics.register(name, metric);
+ }
+
+ public <T extends Metric> T register(Class<?> klass, T metric, String... names)
+ throws IllegalArgumentException {
+ return register(MetricRegistry.name(klass, names), metric);
+ }
+
+
+ /**
+ * Add an event (synchronized)
+ * @param event event
+ */
+ public synchronized void noteEvent(RecordedEvent event) {
+ if (eventHistory.size() > EVENT_LIMIT) {
+ eventHistory.remove(0);
+ }
+ eventHistory.add(event);
+ }
+
+ /**
+ * Clone the event history; blocks for the duration of the copy operation.
+ * @return a new list
+ */
+ public synchronized List<RecordedEvent> cloneEventHistory() {
+ return new ArrayList<>(eventHistory);
+ }
+
+ /**
+ * Add a metric set for registering and deregistration on service stop
+ * @param metricSet metric set
+ */
+ public void addMetricSet(MetricSet metricSet) {
+ metricSets.add(metricSet);
+ metrics.registerAll(metricSet);
+ }
+
+ /**
+ * add a metric set, giving each entry a prefix
+ * @param prefix prefix (a trailing "." is automatically added)
+ * @param metricSet the metric set to register
+ */
+ public void addMetricSet(String prefix, MetricSet metricSet) {
+ addMetricSet(new PrefixedMetricsSet(prefix, metricSet));
+ }
+
+ /**
+ * Unregister a metric set; robust
+ * @param metricSet metric set to unregister
+ */
+ public void unregister(MetricSet metricSet) {
+ for (String s : metricSet.getMetrics().keySet()) {
+ try {
+ metrics.remove(s);
+ } catch (IllegalArgumentException e) {
+ // log but continue
+ log.info("Exception when trying to unregister {}", s, e);
+ }
+ }
+ }
}
+
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
index f8646bf..864a1cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsBindingService.java
@@ -19,7 +19,9 @@
package org.apache.slider.server.appmaster.management;
import com.codahale.metrics.JmxReporter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.MetricSet;
import com.codahale.metrics.ScheduledReporter;
import com.codahale.metrics.Slf4jReporter;
import com.google.common.base.Preconditions;
@@ -29,6 +31,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
@@ -79,7 +84,7 @@
JmxReporter jmxReporter;
jmxReporter = JmxReporter.forRegistry(metrics).build();
jmxReporter.start();
- addService(new ClosingService<JmxReporter>(jmxReporter));
+ addService(new ClosingService<>(jmxReporter));
// Ganglia
@@ -128,7 +133,7 @@
.convertDurationsTo(TimeUnit.MILLISECONDS)
.build();
reporter.start(interval, TimeUnit.MINUTES);
- addService(new ClosingService<ScheduledReporter>(reporter));
+ addService(new ClosingService<>(reporter));
summary.append(String.format(", SLF4J to log %s interval=%d",
logName, interval));
}
@@ -136,8 +141,11 @@
log.info(reportingDetails);
}
+
@Override
public String toString() {
return super.toString() + " " + reportingDetails;
}
+
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
index 31a82a3..fa6bfc0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/MetricsConstants.java
@@ -53,4 +53,6 @@
*/
public static final String CONTAINERS_START_FAILED = "containers.start-failed";
+ public static final String PREFIX_SLIDER_ROLES = "slider.roles.";
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
new file mode 100644
index 0000000..e9ad46a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/PrefixedMetricsSet.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * From an existing metrics set, generate a new metrics set with the
+ * prefix in front of every key.
+ *
+ * The prefix is added directly: if you want a '.' between prefix and metric
+ * keys, include it in the prefix.
+ */
+public class PrefixedMetricsSet implements MetricSet {
+
+ private final String prefix;
+ private final MetricSet source;
+
+ public PrefixedMetricsSet(String prefix, MetricSet source) {
+ this.prefix = prefix;
+ this.source = source;
+ }
+
+ @Override
+ public Map<String, Metric> getMetrics() {
+ Map<String, Metric> sourceMetrics = source.getMetrics();
+ Map<String, Metric> metrics = new HashMap<>(sourceMetrics.size());
+ for (Map.Entry<String, Metric> entry : sourceMetrics.entrySet()) {
+ metrics.put(prefix + "." + entry.getKey(), entry.getValue());
+ }
+ return metrics;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java
new file mode 100644
index 0000000..80e88fc
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RangeLimitedCounter.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import com.codahale.metrics.Counter;
+import com.codahale.metrics.Counting;
+import com.codahale.metrics.Metric;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * This is a counter whose range can be given a min and a max
+ */
+public class RangeLimitedCounter implements Metric, Counting {
+
+ private final AtomicLong value;
+ private final long min, max;
+
+ /**
+ * Instantiate
+ * @param val current value
+ * @param min minimum value
+ * @param max max value (or 0 for no max)
+ */
+ public RangeLimitedCounter(long val, long min, long max) {
+ this.value = new AtomicLong(val);
+ this.min = min;
+ this.max = max;
+ }
+
+ /**
+ * Set to a new value. If below the min, set to the minimum. If the max is non
+ * zero and the value is above that maximum, set it to the maximum instead.
+ * @param val value
+ */
+ public synchronized void set(long val) {
+ if (val < min) {
+ val = min;
+ } else if (max > 0 && val > max) {
+ val = max;
+ }
+ value.set(val);
+ }
+
+ public void inc() {
+ inc(1);
+ }
+
+ public void dec() {
+ dec(1);
+ }
+
+ public synchronized void inc(int delta) {
+ set(value.get() + delta);
+ }
+
+ public synchronized void dec(int delta) {
+ set(value.get() - delta);
+ }
+
+ public long get() {
+ return value.get();
+ }
+
+ @Override
+ public long getCount() {
+ return value.get();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
new file mode 100644
index 0000000..d48d337
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/RecordedEvent.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.management;
+
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import java.text.DateFormat;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+public class RecordedEvent {
+ private static final DateFormat dateFormat = DateFormat.getDateInstance();
+ public long id;
+ public String name;
+ public long timestamp;
+ public String time;
+ public String category;
+ public String host;
+ public int role;
+ public String text;
+
+ public RecordedEvent() {
+ }
+
+ /**
+ * Create an event. The timestamp is also converted to a time string
+ * @param id id counter
+ * @param name event name
+ * @param timestamp timestamp. If non-zero, is used to build the {@code time} text field.
+ * @param category even category
+ * @param text arbitrary text
+ */
+ public RecordedEvent(long id, String name, long timestamp, String category, String text) {
+ this.id = id;
+ this.name = name;
+ this.timestamp = timestamp;
+ this.time = timestamp > 0 ? dateFormat.format(timestamp) : "";
+ this.category = category;
+ this.text = text;
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java
similarity index 74%
rename from slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
rename to slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java
index e9655f0..c30e749 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractRecordFactory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/management/Timestamp.java
@@ -16,15 +16,18 @@
* limitations under the License.
*/
-package org.apache.slider.server.appmaster.state;
-
-import org.apache.hadoop.yarn.api.records.Resource;
+package org.apache.slider.server.appmaster.management;
/**
- * Factory supplying records created by the App state; entry point
- * for mock code.
+ * A timestamp metric
*/
-public abstract class AbstractRecordFactory {
-
- public abstract Resource newResource();
+public class Timestamp extends LongGauge {
+
+ public Timestamp(long val) {
+ super(val);
+ }
+
+ public Timestamp() {
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
index a5cdfc6..ae38e4c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosKillContainer.java
@@ -72,15 +72,13 @@
}
}
int size = liveContainers.size();
- if (size <= 0) {
- log.info("No containers to kill");
- return;
- }
- int target = random.nextInt(size);
- RoleInstance roleInstance = liveContainers.get(target);
- log.info("Killing {}", roleInstance);
+ if (size > 0) {
+ int target = random.nextInt(size);
+ RoleInstance roleInstance = liveContainers.get(target);
+ log.info("Killing {}", roleInstance);
- queues.schedule(new ActionKillContainer(roleInstance.getId(),
- DELAY, TimeUnit.MILLISECONDS, operationHandler));
+ queues.schedule(new ActionKillContainer(roleInstance.getId(),
+ DELAY, TimeUnit.MILLISECONDS, operationHandler));
+ }
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index 27219e4..8948f0d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -60,7 +60,8 @@
public synchronized void addTarget(String name,
ChaosTarget target, long probability) {
if (probability > 0) {
- log.info("Adding {} with probability {}", name, probability / InternalKeys.PROBABILITY_PERCENT_1);
+ log.info("Adding {} with probability {}", name,
+ ((double)probability) / InternalKeys.PROBABILITY_PERCENT_1);
chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
} else {
log.debug("Action {} not enabled", name);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
index da8d646..ed3f197 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AbstractRMOperation.java
@@ -23,8 +23,8 @@
/**
* Execute the operation
* @param asyncRMClient client
- * @param handler
+ * @param handler handler to perform the execution
*/
public abstract void execute(RMOperationHandlerActions handler);
-
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
index 11afc0e..03231ef 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/AsyncRMOperationHandler.java
@@ -63,6 +63,7 @@
* @param count count to cancel
* @return number of requests cancelled
*/
+ @SuppressWarnings("unchecked")
protected int cancelSinglePriorityRequests(Priority priority,
int count) {
List<Collection<AMRMClient.ContainerRequest>> requestSets =
@@ -88,6 +89,7 @@
}
@Override
+ @SuppressWarnings("unchecked")
public void cancelSingleRequest(AMRMClient.ContainerRequest request) {
// a single release
client.removeContainerRequest(request);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
index 08eb5bc..d7673d3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/CancelSingleRequest.java
@@ -18,7 +18,9 @@
package org.apache.slider.server.appmaster.operations;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
/**
* Cancel a container request
@@ -28,6 +30,7 @@
private final AMRMClient.ContainerRequest request;
public CancelSingleRequest(AMRMClient.ContainerRequest request) {
+ Preconditions.checkArgument(request != null, "Null container request");
this.request = request;
}
@@ -42,7 +45,9 @@
@Override
public String toString() {
- return "cancel single request for container at " + request.getPriority().toString();
+ return "Cancel container request"
+ + " for :" + ContainerPriority.toString(request.getPriority())
+ + " request " + request;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
index 46da536..4271d50 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerReleaseOperation.java
@@ -18,13 +18,16 @@
package org.apache.slider.server.appmaster.operations;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.slider.server.appmaster.state.ContainerPriority;
public class ContainerReleaseOperation extends AbstractRMOperation {
private final ContainerId containerId;
public ContainerReleaseOperation(ContainerId containerId) {
+ Preconditions.checkArgument(containerId != null, "Null containerId");
this.containerId = containerId;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
index b8120ca..e29ddd0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/ContainerRequestOperation.java
@@ -18,14 +18,20 @@
package org.apache.slider.server.appmaster.operations;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.slider.server.appmaster.state.ContainerPriority;
+/**
+ * A container request operation
+ */
public class ContainerRequestOperation extends AbstractRMOperation {
private final AMRMClient.ContainerRequest request;
public ContainerRequestOperation(AMRMClient.ContainerRequest request) {
+ Preconditions.checkArgument(request != null, "Null container request");
this.request = request;
}
@@ -33,6 +39,14 @@
return request;
}
+ public Priority getPriority() {
+ return request.getPriority();
+ }
+
+ public boolean getRelaxLocality() {
+ return request.getRelaxLocality();
+ }
+
@Override
public void execute(RMOperationHandlerActions handler) {
handler.addContainerRequest(request);
@@ -40,6 +54,9 @@
@Override
public String toString() {
- return "request container for " + ContainerPriority.toString(request.getPriority());
+ return "request container for role "
+ + ContainerPriority.toString(getPriority())
+ + " request " + request
+ + " relaxLocality=" + getRelaxLocality();
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
index 594ee47..b7794ed 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/operations/RMOperationHandlerActions.java
@@ -25,6 +25,11 @@
import java.util.List;
public interface RMOperationHandlerActions {
+
+ /**
+ * Release an assigned container
+ * @param containerId container
+ */
void releaseAssignedContainer(ContainerId containerId);
/**
@@ -38,7 +43,7 @@
* @param request request to cancel
*/
void cancelSingleRequest(AMRMClient.ContainerRequest request);
-
+
/**
* Remove a container request
* @param priority1 priority to remove at
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
index 9d4ea99..7d237de 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPB.java
@@ -23,5 +23,5 @@
public interface SliderClusterProtocolPB extends SliderClusterAPI.SliderClusterProtocolPB.BlockingInterface{
- public static final long versionID = SliderClusterProtocol.versionID;
+ long versionID = SliderClusterProtocol.versionID;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
index c597626..f0d9063 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolPBImpl.java
@@ -26,7 +26,7 @@
import java.io.IOException;
/**
- * Relay from Protobuf to internal RPC.
+ * Server-side Relay from Protobuf to internal RPC.
*
*/
public class SliderClusterProtocolPBImpl implements SliderClusterProtocolPB {
@@ -44,15 +44,14 @@
return new ServiceException(e);
}
- public long getProtocolVersion(String protocol, long clientVersion) throws
- IOException {
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
return SliderClusterProtocol.versionID;
}
-
+
@Override
public Messages.StopClusterResponseProto stopCluster(RpcController controller,
- Messages.StopClusterRequestProto request) throws
- ServiceException {
+ Messages.StopClusterRequestProto request) throws ServiceException {
try {
return real.stopCluster(request);
} catch (Exception e) {
@@ -62,8 +61,7 @@
@Override
public Messages.UpgradeContainersResponseProto upgradeContainers(RpcController controller,
- Messages.UpgradeContainersRequestProto request) throws
- ServiceException {
+ Messages.UpgradeContainersRequestProto request) throws ServiceException {
try {
return real.upgradeContainers(request);
} catch (Exception e) {
@@ -73,8 +71,7 @@
@Override
public Messages.FlexClusterResponseProto flexCluster(RpcController controller,
- Messages.FlexClusterRequestProto request) throws
- ServiceException {
+ Messages.FlexClusterRequestProto request) throws ServiceException {
try {
return real.flexCluster(request);
} catch (Exception e) {
@@ -97,8 +94,8 @@
@Override
public Messages.GetInstanceDefinitionResponseProto getInstanceDefinition(
RpcController controller,
- Messages.GetInstanceDefinitionRequestProto request) throws
- ServiceException {
+ Messages.GetInstanceDefinitionRequestProto request)
+ throws ServiceException {
try {
return real.getInstanceDefinition(request);
} catch (Exception e) {
@@ -119,8 +116,7 @@
@Override
public Messages.GetNodeResponseProto getNode(RpcController controller,
- Messages.GetNodeRequestProto request) throws
- ServiceException {
+ Messages.GetNodeRequestProto request) throws ServiceException {
try {
return real.getNode(request);
} catch (Exception e) {
@@ -130,8 +126,7 @@
@Override
public Messages.GetClusterNodesResponseProto getClusterNodes(RpcController controller,
- Messages.GetClusterNodesRequestProto request) throws
- ServiceException {
+ Messages.GetClusterNodesRequestProto request) throws ServiceException {
try {
return real.getClusterNodes(request);
} catch (Exception e) {
@@ -141,8 +136,7 @@
@Override
public Messages.EchoResponseProto echo(RpcController controller,
- Messages.EchoRequestProto request) throws
- ServiceException {
+ Messages.EchoRequestProto request) throws ServiceException {
try {
return real.echo(request);
} catch (Exception e) {
@@ -152,8 +146,7 @@
@Override
public Messages.KillContainerResponseProto killContainer(RpcController controller,
- Messages.KillContainerRequestProto request) throws
- ServiceException {
+ Messages.KillContainerRequestProto request) throws ServiceException {
try {
return real.killContainer(request);
} catch (Exception e) {
@@ -163,8 +156,7 @@
@Override
public Messages.AMSuicideResponseProto amSuicide(RpcController controller,
- Messages.AMSuicideRequestProto request) throws
- ServiceException {
+ Messages.AMSuicideRequestProto request) throws ServiceException {
try {
return real.amSuicide(request);
} catch (Exception e) {
@@ -175,8 +167,7 @@
@Override
public Messages.ApplicationLivenessInformationProto getLivenessInformation(
RpcController controller,
- Messages.GetApplicationLivenessRequestProto request) throws
- ServiceException {
+ Messages.GetApplicationLivenessRequestProto request) throws ServiceException {
try {
return real.getLivenessInformation(request);
} catch (Exception e) {
@@ -225,6 +216,26 @@
}
@Override
+ public Messages.GetLiveNodesResponseProto getLiveNodes(RpcController controller,
+ Messages.GetLiveNodesRequestProto request) throws ServiceException {
+ try {
+ return real.getLiveNodes(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
+ public Messages.NodeInformationProto getLiveNode(RpcController controller,
+ Messages.GetLiveNodeRequestProto request) throws ServiceException {
+ try {
+ return real.getLiveNode(request);
+ } catch (Exception e) {
+ throw wrap(e);
+ }
+ }
+
+ @Override
public Messages.WrappedJsonProto getModelDesired(RpcController controller,
Messages.EmptyPayloadProto request) throws ServiceException {
try {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
index 2d927c6..b230816 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderClusterProtocolProxy.java
@@ -263,6 +263,26 @@
}
@Override
+ public Messages.GetLiveNodesResponseProto getLiveNodes(Messages.GetLiveNodesRequestProto request)
+ throws IOException {
+ try {
+ return endpoint.getLiveNodes(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
+ public Messages.NodeInformationProto getLiveNode(Messages.GetLiveNodeRequestProto request)
+ throws IOException {
+ try {
+ return endpoint.getLiveNode(NULL_CONTROLLER, request);
+ } catch (ServiceException e) {
+ throw convert(e);
+ }
+ }
+
+ @Override
public Messages.WrappedJsonProto getModelDesired(Messages.EmptyPayloadProto request) throws IOException {
try {
return endpoint.getModelDesired(NULL_CONTROLLER, request);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
index 1581482..fda23aa 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/rpc/SliderIPCService.java
@@ -30,9 +30,10 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.ServiceNotReadyException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.main.LauncherExitCodes;
@@ -51,7 +52,6 @@
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.services.security.CertificateManager;
-import org.apache.slider.server.services.security.KeystoreGenerator;
import org.apache.slider.server.services.security.SecurityStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +65,7 @@
import static org.apache.slider.api.proto.RestTypeMarshalling.marshall;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_NODES;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF;
@@ -273,7 +274,7 @@
String role = request.getRole();
Messages.ListNodeUUIDsByRoleResponseProto.Builder builder =
Messages.ListNodeUUIDsByRoleResponseProto.newBuilder();
- List<RoleInstance> nodes = state.enumLiveNodesInRole(role);
+ List<RoleInstance> nodes = state.enumLiveInstancesInRole(role);
for (RoleInstance node : nodes) {
builder.addUuid(node.id);
}
@@ -373,13 +374,11 @@
Messages.GetLiveContainersRequestProto request)
throws IOException {
Map<String, ContainerInformation> infoMap =
- (Map<String, ContainerInformation>) cache.lookupWithIOE(
- LIVE_CONTAINERS);
+ (Map<String, ContainerInformation>) cache.lookupWithIOE(LIVE_CONTAINERS);
Messages.GetLiveContainersResponseProto.Builder builder =
Messages.GetLiveContainersResponseProto.newBuilder();
- for (Map.Entry<String, ContainerInformation> entry : infoMap
- .entrySet()) {
+ for (Map.Entry<String, ContainerInformation> entry : infoMap.entrySet()) {
builder.addNames(entry.getKey());
builder.addContainers(marshall(entry.getValue()));
}
@@ -387,8 +386,8 @@
}
@Override
- public Messages.ContainerInformationProto getLiveContainer(Messages.GetLiveContainerRequestProto request) throws
- IOException {
+ public Messages.ContainerInformationProto getLiveContainer(Messages.GetLiveContainerRequestProto request)
+ throws IOException {
String containerId = request.getContainerId();
RoleInstance id = state.getLiveInstanceByContainerID(containerId);
ContainerInformation containerInformation = id.serialize();
@@ -396,22 +395,58 @@
}
@Override
- public Messages.GetLiveComponentsResponseProto getLiveComponents(Messages.GetLiveComponentsRequestProto request) throws
- IOException {
+ public Messages.GetLiveComponentsResponseProto getLiveComponents(Messages.GetLiveComponentsRequestProto request)
+ throws IOException {
Map<String, ComponentInformation> infoMap =
- (Map<String, ComponentInformation>) cache.lookupWithIOE(
- LIVE_COMPONENTS);
+ (Map<String, ComponentInformation>) cache.lookupWithIOE(LIVE_COMPONENTS);
Messages.GetLiveComponentsResponseProto.Builder builder =
Messages.GetLiveComponentsResponseProto.newBuilder();
- for (Map.Entry<String, ComponentInformation> entry : infoMap
- .entrySet()) {
+ for (Map.Entry<String, ComponentInformation> entry : infoMap.entrySet()) {
builder.addNames(entry.getKey());
builder.addComponents(marshall(entry.getValue()));
}
return builder.build();
}
+
+ @Override
+ public Messages.ComponentInformationProto getLiveComponent(Messages.GetLiveComponentRequestProto request)
+ throws IOException {
+ String name = request.getName();
+ try {
+ return marshall(state.getComponentInformation(name));
+ } catch (YarnRuntimeException e) {
+ throw new FileNotFoundException("Unknown component: " + name);
+ }
+ }
+
+ @Override
+ public Messages.GetLiveNodesResponseProto getLiveNodes(Messages.GetLiveNodesRequestProto request)
+ throws IOException {
+ NodeInformationList info = (NodeInformationList) cache.lookupWithIOE(LIVE_NODES);
+ Messages.GetLiveNodesResponseProto.Builder builder =
+ Messages.GetLiveNodesResponseProto.newBuilder();
+
+ for (NodeInformation nodeInformation : info) {
+ builder.addNodes(marshall(nodeInformation));
+ }
+ return builder.build();
+ }
+
+
+ @Override
+ public Messages.NodeInformationProto getLiveNode(Messages.GetLiveNodeRequestProto request)
+ throws IOException {
+ String name = request.getName();
+ NodeInformation nodeInformation = state.getNodeInformation(name);
+ if (nodeInformation != null) {
+ return marshall(nodeInformation);
+ } else {
+ throw new FileNotFoundException("Unknown host: " + name);
+ }
+ }
+
@Override
public Messages.WrappedJsonProto getModelDesired(Messages.EmptyPayloadProto request) throws IOException {
return lookupAggregateConf(MODEL_DESIRED);
@@ -447,17 +482,6 @@
return lookupConfTree(LIVE_RESOURCES);
}
- @Override
- public Messages.ComponentInformationProto getLiveComponent(Messages.GetLiveComponentRequestProto request) throws
- IOException {
- String name = request.getName();
- try {
- return marshall(state.getComponentInformation(name));
- } catch (YarnRuntimeException e) {
- throw new FileNotFoundException("Unknown component: " + name);
- }
- }
-
/**
* Helper method; look up an aggregate configuration in the cache from
* a key, or raise an exception
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
index 4ff6916..a01fb18 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/security/SecurityConfiguration.java
@@ -17,18 +17,11 @@
package org.apache.slider.server.appmaster.security;
import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.slider.common.SliderExitCodes;
+import static org.apache.slider.core.main.LauncherExitCodes.EXIT_UNAUTHORIZE;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.exceptions.SliderException;
@@ -39,7 +32,7 @@
import java.io.IOException;
/**
- *
+ * Class keeping code security information
*/
public class SecurityConfiguration {
@@ -71,7 +64,7 @@
try {
loginUser = getLoginUser();
} catch (IOException e) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_STATE, e,
+ throw new SliderException(EXIT_UNAUTHORIZE, e,
"No principal configured for the application and "
+ "exception raised during retrieval of login user. "
+ "Unable to proceed with application "
@@ -81,7 +74,7 @@
SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL);
}
if (loginUser == null) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ throw new SliderException(EXIT_UNAUTHORIZE,
"No principal configured for the application "
+ "and no login user found. "
+ "Unable to proceed with application "
@@ -100,7 +93,7 @@
.getComponent(SliderKeys.COMPONENT_AM)
.get(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
if (SliderUtils.isSet(keytabFullPath) && SliderUtils.isSet(keytabName)) {
- throw new SliderException(SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ throw new SliderException(EXIT_UNAUTHORIZE,
"Both a keytab on the cluster host (%s) and a"
+ " keytab to be retrieved from HDFS (%s) are"
+ " specified. Please configure only one keytab"
@@ -160,8 +153,7 @@
// download keytab to local, protected directory
localKeytabFile = new File(SliderKeys.KEYTAB_DIR, keytabName);
} else {
- log.info("Leveraging host keytab file {} for login",
- keytabFullPath);
+ log.info("Using host keytab file {} for login", keytabFullPath);
localKeytabFile = new File(keytabFullPath);
}
return localKeytabFile;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
new file mode 100644
index 0000000..54f384b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AbstractClusterServices.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+
+/**
+ * Cluster services offered by the YARN infrastructure.
+ */
+public abstract class AbstractClusterServices {
+
+ private final DefaultResourceCalculator
+ defaultResourceCalculator = new DefaultResourceCalculator();
+
+ /**
+ * Create a resource for requests
+ * @return a resource which can be built up.
+ */
+ public abstract Resource newResource();
+
+ public abstract Resource newResource(int memory, int cores);
+
+ /**
+ * Normalise memory, CPU and other resources according to the YARN AM-supplied
+ * values and the resource calculator in use (currently hard-coded to the
+ * {@link DefaultResourceCalculator}.
+ * Those resources which aren't normalized (currently: CPU) are left
+ * as is.
+ * @param resource resource requirements of a role
+ * @param minR minimum values of this queue
+ * @param maxR max values of this queue
+ * @return a normalized value.
+ */
+ public Resource normalize(Resource resource, Resource minR, Resource maxR) {
+ Preconditions.checkArgument(resource != null, "null resource");
+ Preconditions.checkArgument(minR != null, "null minR");
+ Preconditions.checkArgument(maxR != null, "null maxR");
+
+ Resource normalize = defaultResourceCalculator.normalize(resource, minR,
+ maxR, minR);
+ return newResource(normalize.getMemory(), resource.getVirtualCores());
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 18eb578..4b4d7a7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -18,12 +18,11 @@
package org.apache.slider.server.appmaster.state;
-import com.codahale.metrics.Counter;
+import com.codahale.metrics.Metric;
import com.codahale.metrics.MetricRegistry;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -36,6 +35,7 @@
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterDescriptionOperations;
@@ -46,6 +46,7 @@
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -64,6 +65,7 @@
import org.apache.slider.core.persist.ConfTreeSerDeser;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.LongGauge;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
import org.apache.slider.server.appmaster.management.MetricsConstants;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
@@ -85,16 +87,27 @@
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_INSTANCES;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY;
+import static org.apache.slider.api.ResourceKeys.COMPONENT_PRIORITY;
+import static org.apache.slider.api.ResourceKeys.CONTAINER_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS;
import static org.apache.slider.api.ResourceKeys.DEF_YARN_CORES;
import static org.apache.slider.api.ResourceKeys.DEF_YARN_LABEL_EXPRESSION;
import static org.apache.slider.api.ResourceKeys.DEF_YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.NODE_FAILURE_THRESHOLD;
+import static org.apache.slider.api.ResourceKeys.PLACEMENT_ESCALATE_DELAY;
import static org.apache.slider.api.ResourceKeys.YARN_CORES;
import static org.apache.slider.api.ResourceKeys.YARN_LABEL_EXPRESSION;
import static org.apache.slider.api.ResourceKeys.YARN_MEMORY;
+import static org.apache.slider.api.ResourceKeys.YARN_RESOURCE_MAX;
import static org.apache.slider.api.RoleKeys.ROLE_FAILED_INSTANCES;
-import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES;
import static org.apache.slider.api.RoleKeys.ROLE_FAILED_RECENTLY_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_FAILED_STARTING_INSTANCES;
import static org.apache.slider.api.RoleKeys.ROLE_NODE_FAILED_INSTANCES;
+import static org.apache.slider.api.RoleKeys.ROLE_PENDING_AA_INSTANCES;
import static org.apache.slider.api.RoleKeys.ROLE_PREEMPTED_INSTANCES;
import static org.apache.slider.api.RoleKeys.ROLE_RELEASING_INSTANCES;
import static org.apache.slider.api.RoleKeys.ROLE_REQUESTED_INSTANCES;
@@ -115,7 +128,7 @@
protected static final Logger log =
LoggerFactory.getLogger(AppState.class);
- private final AbstractRecordFactory recordFactory;
+ private final AbstractClusterServices recordFactory;
private final MetricsAndMonitoring metricsAndMonitoring;
@@ -156,7 +169,7 @@
private ConfTreeOperations resourcesSnapshot;
private ConfTreeOperations appConfSnapshot;
private ConfTreeOperations internalsSnapshot;
-
+
/**
* This is the status, the live model
*/
@@ -166,7 +179,7 @@
* Metadata provided by the AM for use in filling in status requests
*/
private Map<String, String> applicationInfo;
-
+
/**
* Client properties created via the provider -static for the life
* of the application
@@ -179,13 +192,13 @@
private ClusterDescription clusterStatusTemplate = new ClusterDescription();
private final Map<Integer, RoleStatus> roleStatusMap =
- new ConcurrentHashMap<Integer, RoleStatus>();
+ new ConcurrentHashMap<>();
private final Map<String, ProviderRole> roles =
- new ConcurrentHashMap<String, ProviderRole>();
+ new ConcurrentHashMap<>();
private final Map<Integer, ProviderRole> rolePriorityMap =
- new ConcurrentHashMap<Integer, ProviderRole>();
+ new ConcurrentHashMap<>();
/**
* The master node.
@@ -210,62 +223,61 @@
/**
* Counter for completed containers ( complete denotes successful or failed )
*/
- private final Counter completedContainerCount = new Counter();
+ private final LongGauge completedContainerCount = new LongGauge();
/**
* Count of failed containers
-
*/
- private final Counter failedContainerCount = new Counter();
+ private final LongGauge failedContainerCount = new LongGauge();
/**
* # of started containers
*/
- private final Counter startedContainers = new Counter();
+ private final LongGauge startedContainers = new LongGauge();
/**
* # of containers that failed to start
*/
- private final Counter startFailedContainerCount = new Counter();
+ private final LongGauge startFailedContainerCount = new LongGauge();
/**
* Track the number of surplus containers received and discarded
*/
- private final Counter surplusContainers = new Counter();
-
+ private final LongGauge surplusContainers = new LongGauge();
/**
- * Track the number of requested Containers
+ * Track the number of requested containers.
+ * Important: this does not include AA requests which are yet to be issued.
*/
- private final Counter outstandingContainerRequests = new Counter();
+ private final LongGauge outstandingContainerRequests = new LongGauge();
/**
* Map of requested nodes. This records the command used to start it,
* resources, etc. When container started callback is received,
* the node is promoted from here to the containerMap
*/
- private final Map<ContainerId, RoleInstance> startingNodes =
+ private final Map<ContainerId, RoleInstance> startingContainers =
new ConcurrentHashMap<>();
/**
* List of completed nodes. This isn't kept in the CD as it gets too
* big for the RPC responses. Indeed, we should think about how deep to get this
*/
- private final Map<ContainerId, RoleInstance> completedNodes
+ private final Map<ContainerId, RoleInstance> completedContainers
= new ConcurrentHashMap<>();
/**
* Nodes that failed to start.
* Again, kept out of the CD
*/
- private final Map<ContainerId, RoleInstance> failedNodes =
+ private final Map<ContainerId, RoleInstance> failedContainers =
new ConcurrentHashMap<>();
/**
* Nodes that came assigned to a role above that
* which were asked for -this appears to happen
*/
- private final Set<ContainerId> surplusNodes = new HashSet<ContainerId>();
+ private final Set<ContainerId> surplusNodes = new HashSet<>();
/**
* Map of containerID to cluster nodes, for status reports.
@@ -280,36 +292,42 @@
/**
- * Record of the max no. of cores allowed in this cluster
+ * limits of container core numbers in this queue
*/
private int containerMaxCores;
+ private int containerMinCores;
/**
- * limit container memory
+ * limits of container memory in this queue
*/
private int containerMaxMemory;
-
+ private int containerMinMemory;
+
private RoleHistory roleHistory;
private Configuration publishedProviderConf;
private long startTimeThreshold;
-
+
private int failureThreshold = 10;
private int nodeFailureThreshold = 3;
-
+
private String logServerURL = "";
/**
* Selector of containers to release; application wide.
*/
private ContainerReleaseSelector containerReleaseSelector;
+ private Resource minResource;
+ private Resource maxResource;
/**
* Create an instance
* @param recordFactory factory for YARN records
* @param metricsAndMonitoring metrics and monitoring services
*/
- public AppState(AbstractRecordFactory recordFactory,
+ public AppState(AbstractClusterServices recordFactory,
MetricsAndMonitoring metricsAndMonitoring) {
+ Preconditions.checkArgument(recordFactory != null, "null recordFactory");
+ Preconditions.checkArgument(metricsAndMonitoring != null, "null metricsAndMonitoring");
this.recordFactory = recordFactory;
this.metricsAndMonitoring = metricsAndMonitoring;
@@ -322,10 +340,9 @@
register(MetricsConstants.CONTAINERS_START_FAILED, startFailedContainerCount);
}
- private void register(String name, Counter counter) {
+ private void register(String name, Metric counter) {
this.metricsAndMonitoring.getMetrics().register(
- MetricRegistry.name(AppState.class,
- name), counter);
+ MetricRegistry.name(AppState.class, name), counter);
}
public long getFailedCountainerCount() {
@@ -382,21 +399,19 @@
return rolePriorityMap;
}
- private Map<ContainerId, RoleInstance> getStartingNodes() {
- return startingNodes;
+ private Map<ContainerId, RoleInstance> getStartingContainers() {
+ return startingContainers;
}
- private Map<ContainerId, RoleInstance> getCompletedNodes() {
- return completedNodes;
+ private Map<ContainerId, RoleInstance> getCompletedContainers() {
+ return completedContainers;
}
-
- public Map<ContainerId, RoleInstance> getFailedNodes() {
- return failedNodes;
+ public Map<ContainerId, RoleInstance> getFailedContainers() {
+ return failedContainers;
}
-
- public Map<ContainerId, RoleInstance> getLiveNodes() {
+ public Map<ContainerId, RoleInstance> getLiveContainers() {
return liveNodes;
}
@@ -464,27 +479,31 @@
}
/**
- * Set the container limits -the max that can be asked for,
- * which are used when the "max" values are requested
+ * Set the container limits -the min and max values for
+ * resource requests. All requests must be multiples of the min
+ * values.
+ * @param minMemory min memory MB
* @param maxMemory maximum memory
+ * @param minCores min v core count
* @param maxCores maximum cores
*/
- public void setContainerLimits(int maxMemory, int maxCores) {
+ public void setContainerLimits(int minMemory,int maxMemory, int minCores, int maxCores) {
+ containerMinCores = minCores;
containerMaxCores = maxCores;
+ containerMinMemory = minMemory;
containerMaxMemory = maxMemory;
+ minResource = recordFactory.newResource(containerMinMemory, containerMinCores);
+ maxResource = recordFactory.newResource(containerMaxMemory, containerMaxCores);
}
-
public ConfTreeOperations getResourcesSnapshot() {
return resourcesSnapshot;
}
-
public ConfTreeOperations getAppConfSnapshot() {
return appConfSnapshot;
}
-
public ConfTreeOperations getInternalsSnapshot() {
return internalsSnapshot;
}
@@ -505,38 +524,17 @@
return unresolvedInstanceDefinition;
}
- /**
- * Build up the application state
- * @param instanceDefinition definition of the applicatin instance
- * @param appmasterConfig
- * @param publishedProviderConf any configuration info to be published by a provider
- * @param providerRoles roles offered by a provider
- * @param fs filesystem
- * @param historyDir directory containing history files
- * @param liveContainers list of live containers supplied on an AM restart
- * @param applicationInfo app info to retain for web views
- * @param releaseSelector selector of containers to release
- */
- public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration appmasterConfig,
- Configuration publishedProviderConf,
- List<ProviderRole> providerRoles,
- FileSystem fs,
- Path historyDir,
- List<Container> liveContainers,
- Map<String, String> applicationInfo,
- ContainerReleaseSelector releaseSelector)
- throws BadClusterStateException, BadConfigException, IOException {
- Preconditions.checkArgument(instanceDefinition != null);
- Preconditions.checkArgument(releaseSelector != null);
+ public synchronized void buildInstance(AppStateBindingInfo binding)
+ throws BadClusterStateException, BadConfigException, IOException {
+ binding.validate();
log.debug("Building application state");
- this.publishedProviderConf = publishedProviderConf;
- this.applicationInfo = applicationInfo != null ? applicationInfo
- : new HashMap<String, String>();
+ publishedProviderConf = binding.publishedProviderConf;
+ applicationInfo = binding.applicationInfo != null ? binding.applicationInfo
+ : new HashMap<String, String>();
clientProperties = new HashMap<>();
- containerReleaseSelector = releaseSelector;
+ containerReleaseSelector = binding.releaseSelector;
Set<String> confKeys = ConfigHelper.sortedConfigKeys(publishedProviderConf);
@@ -549,15 +547,15 @@
// set the cluster specification (once its dependency the client properties
// is out the way
- setInitialInstanceDefinition(instanceDefinition);
+ setInitialInstanceDefinition(binding.instanceDefinition);
//build the initial role list
- for (ProviderRole providerRole : providerRoles) {
+ List<ProviderRole> roleList = new ArrayList<>(binding.roles);
+ for (ProviderRole providerRole : roleList) {
buildRole(providerRole);
}
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ ConfTreeOperations resources = instanceDefinition.getResourceOperations();
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
@@ -565,43 +563,43 @@
// this is a new value
log.info("Adding role {}", name);
MapOperations resComponent = resources.getComponent(name);
- ProviderRole dynamicRole =
- createDynamicProviderRole(name, resComponent);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
buildRole(dynamicRole);
- providerRoles.add(dynamicRole);
+ roleList.add(dynamicRole);
}
}
//then pick up the requirements
buildRoleRequirementsFromResources();
-
//set the livespan
- MapOperations globalResOpts =
- instanceDefinition.getResourceOperations().getGlobalOptions();
-
+ MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions();
+
startTimeThreshold = globalResOpts.getOptionInt(
InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
failureThreshold = globalResOpts.getOptionInt(
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_CONTAINER_FAILURE_THRESHOLD);
+ CONTAINER_FAILURE_THRESHOLD,
+ DEFAULT_CONTAINER_FAILURE_THRESHOLD);
nodeFailureThreshold = globalResOpts.getOptionInt(
- ResourceKeys.NODE_FAILURE_THRESHOLD,
- ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD);
+ NODE_FAILURE_THRESHOLD,
+ DEFAULT_NODE_FAILURE_THRESHOLD);
initClusterStatus();
- // add the roles
- roleHistory = new RoleHistory(providerRoles);
- roleHistory.onStart(fs, historyDir);
+ // set up the role history
+ roleHistory = new RoleHistory(roleStatusMap.values(), recordFactory);
+ roleHistory.register(metricsAndMonitoring);
+ roleHistory.onStart(binding.fs, binding.historyPath);
+ // trigger first node update
+ roleHistory.onNodesUpdated(binding.nodeReports);
+
//rebuild any live containers
- rebuildModelFromRestart(liveContainers);
+ rebuildModelFromRestart(binding.liveContainers);
// any am config options to pick up
- logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
-
+ logServerURL = binding.serviceConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL, "");
//mark as live
applicationLive = true;
}
@@ -641,32 +639,31 @@
* @return a new provider role
* @throws BadConfigException bad configuration
*/
- public ProviderRole createDynamicProviderRole(String name,
- MapOperations component) throws
- BadConfigException {
- String priOpt = component.getMandatoryOption(ResourceKeys.COMPONENT_PRIORITY);
- int priority = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PRIORITY,
- priOpt, 0, 1, -1);
- String placementOpt = component.getOption(
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ public ProviderRole createDynamicProviderRole(String name, MapOperations component)
+ throws BadConfigException {
+ String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY);
+ int priority = SliderUtils.parseAndValidate(
+ "value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1);
+
+ String placementOpt = component.getOption(COMPONENT_PLACEMENT_POLICY,
Integer.toString(PlacementPolicy.DEFAULT));
- int placement = SliderUtils.parseAndValidate("value of " + name + " " +
- ResourceKeys.COMPONENT_PLACEMENT_POLICY,
- placementOpt, 0, 0, -1);
- int placementTimeout =
- component.getOptionInt(ResourceKeys.PLACEMENT_ESCALATE_DELAY,
- ResourceKeys.DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+
+ int placement = SliderUtils.parseAndValidate(
+ "value of " + name + " " + COMPONENT_PLACEMENT_POLICY, placementOpt, 0, 0, -1);
+
+ int placementTimeout = component.getOptionInt(PLACEMENT_ESCALATE_DELAY,
+ DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
+
ProviderRole newRole = new ProviderRole(name,
priority,
placement,
getNodeFailureThresholdForRole(name),
- placementTimeout);
+ placementTimeout,
+ component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION));
log.info("New {} ", newRole);
return newRole;
}
-
/**
* Actions to perform when an instance definition is updated
* Currently:
@@ -681,56 +678,52 @@
*
* @throws BadConfigException
*/
- private synchronized void onInstanceDefinitionUpdated() throws
- BadConfigException,
- IOException {
+ private synchronized void onInstanceDefinitionUpdated()
+ throws BadConfigException, IOException {
+
log.debug("Instance definition updated");
//note the time
snapshotTime = now();
-
+
// resolve references if not already done
instanceDefinition.resolve();
// force in the AM desired state values
- ConfTreeOperations resources =
- instanceDefinition.getResourceOperations();
+ ConfTreeOperations resources = instanceDefinition.getResourceOperations();
+
if (resources.getComponent(SliderKeys.COMPONENT_AM) != null) {
resources.setComponentOpt(
- SliderKeys.COMPONENT_AM, ResourceKeys.COMPONENT_INSTANCES, "1");
+ SliderKeys.COMPONENT_AM, COMPONENT_INSTANCES, "1");
}
//snapshot all three sectons
- resourcesSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getResources());
- appConfSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
- internalsSnapshot =
- ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
+ resourcesSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getResources());
+ appConfSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getAppConf());
+ internalsSnapshot = ConfTreeOperations.fromInstance(instanceDefinition.getInternal());
//build a new aggregate from the snapshots
instanceDefinitionSnapshot = new AggregateConf(resourcesSnapshot.confTree,
appConfSnapshot.confTree,
internalsSnapshot.confTree);
instanceDefinitionSnapshot.setName(instanceDefinition.getName());
- clusterStatusTemplate =
- ClusterDescriptionOperations.buildFromInstanceDefinition(
+ clusterStatusTemplate = ClusterDescriptionOperations.buildFromInstanceDefinition(
instanceDefinition);
-
-// Add the -site configuration properties
+ // Add the -site configuration properties
for (Map.Entry<String, String> prop : clientProperties.entrySet()) {
clusterStatusTemplate.clientProperties.put(prop.getKey(), prop.getValue());
}
-
+
}
-
+
/**
* The resource configuration is updated -review and update state.
* @param resources updated resources specification
* @return a list of any dynamically added provider roles
* (purely for testing purposes)
*/
+ @VisibleForTesting
public synchronized List<ProviderRole> updateResourceDefinitions(ConfTree resources)
throws BadConfigException, IOException {
log.debug("Updating resources to {}", resources);
@@ -757,9 +750,9 @@
private List<ProviderRole> buildRoleRequirementsFromResources() throws BadConfigException {
List<ProviderRole> newRoles = new ArrayList<>(0);
-
- //now update every role's desired count.
- //if there are no instance values, that role count goes to zero
+
+ // now update every role's desired count.
+ // if there are no instance values, that role count goes to zero
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
@@ -770,10 +763,8 @@
// skip inflexible roles, e.g AM itself
continue;
}
- int currentDesired = roleStatus.getDesired();
+ long currentDesired = roleStatus.getDesired();
String role = roleStatus.getName();
- MapOperations comp =
- resources.getComponent(role);
int desiredInstanceCount = getDesiredInstanceCount(resources, role);
if (desiredInstanceCount == 0) {
log.info("Role {} has 0 instances specified", role);
@@ -785,23 +776,25 @@
}
}
- //now the dynamic ones. Iterate through the the cluster spec and
- //add any role status entries not in the role status
+ // now the dynamic ones. Iterate through the the cluster spec and
+ // add any role status entries not in the role status
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
if (!roles.containsKey(name)) {
// this is a new value
log.info("Adding new role {}", name);
MapOperations component = resources.getComponent(name);
- ProviderRole dynamicRole = createDynamicProviderRole(name,
- component);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, component);
RoleStatus roleStatus = buildRole(dynamicRole);
roleStatus.setDesired(getDesiredInstanceCount(resources, name));
log.info("New role {}", roleStatus);
- roleHistory.addNewProviderRole(dynamicRole);
+ roleHistory.addNewRole(roleStatus);
newRoles.add(dynamicRole);
}
}
+ // and fill in all those roles with their requirements
+ buildRoleResourceRequirements();
+
return newRoles;
}
@@ -815,7 +808,7 @@
private int getDesiredInstanceCount(ConfTreeOperations resources,
String role) throws BadConfigException {
int desiredInstanceCount =
- resources.getComponentOptInt(role, ResourceKeys.COMPONENT_INSTANCES, 0);
+ resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0);
if (desiredInstanceCount < 0) {
log.error("Role {} has negative desired instances : {}", role,
@@ -837,7 +830,7 @@
* @throws BadConfigException if a role of that priority already exists
*/
public RoleStatus buildRole(ProviderRole providerRole) throws BadConfigException {
- //build role status map
+ // build role status map
int priority = providerRole.id;
if (roleStatusMap.containsKey(priority)) {
throw new BadConfigException("Duplicate Provider Key: %s and %s",
@@ -846,12 +839,26 @@
}
RoleStatus roleStatus = new RoleStatus(providerRole);
roleStatusMap.put(priority, roleStatus);
- roles.put(providerRole.name, providerRole);
+ String name = providerRole.name;
+ roles.put(name, providerRole);
rolePriorityMap.put(priority, providerRole);
+ // register its entries
+ metricsAndMonitoring.addMetricSet(MetricsConstants.PREFIX_SLIDER_ROLES + name, roleStatus);
return roleStatus;
}
/**
+ * Build up the requirements of every resource
+ */
+ private void buildRoleResourceRequirements() {
+ roleStatusMap.values();
+ for (RoleStatus role : roleStatusMap.values()) {
+ role.setResourceRequirements(
+ buildResourceRequirements(role, recordFactory.newResource()));
+ }
+ }
+
+ /**
* build up the special master node, which lives
* in the live node set but has a lifecycle bonded to the AM
* @param containerId the AM master
@@ -871,16 +878,15 @@
RoleInstance am = new RoleInstance(container);
am.role = SliderKeys.COMPONENT_AM;
am.roleId = SliderKeys.ROLE_AM_PRIORITY_INDEX;
- am.createTime = System.currentTimeMillis();
- am.startTime = System.currentTimeMillis();
+ am.createTime =now();
+ am.startTime = am.createTime;
appMasterNode = am;
//it is also added to the set of live nodes
- getLiveNodes().put(containerId, am);
+ getLiveContainers().put(containerId, am);
putOwnedContainer(containerId, am);
-
+
// patch up the role status
- RoleStatus roleStatus = roleStatusMap.get(
- (SliderKeys.ROLE_AM_PRIORITY_INDEX));
+ RoleStatus roleStatus = roleStatusMap.get(SliderKeys.ROLE_AM_PRIORITY_INDEX);
roleStatus.setDesired(1);
roleStatus.incActual();
roleStatus.incStarted();
@@ -894,7 +900,7 @@
* and it will just cause confusion
*/
public void noteAMLaunched() {
- getLiveNodes().put(appMasterNode.getContainerId(), appMasterNode);
+ getLiveContainers().put(appMasterNode.getContainerId(), appMasterNode);
}
/**
@@ -906,11 +912,12 @@
appMasterNode.state = STATE_LIVE;
}
- public RoleInstance getAppMasterNode() {
- return appMasterNode;
- }
-
-
+ /**
+ * Look up the status entry of a role or raise an exception
+ * @param key role ID
+ * @return the status entry
+ * @throws RuntimeException if the role cannot be found
+ */
public RoleStatus lookupRoleStatus(int key) {
RoleStatus rs = getRoleStatusMap().get(key);
if (rs == null) {
@@ -918,8 +925,15 @@
}
return rs;
}
-
- public RoleStatus lookupRoleStatus(Container c) throws YarnRuntimeException {
+
+ /**
+ * Look up the status entry of a container or raise an exception
+ *
+ * @param c container
+ * @return the status entry
+ * @throws RuntimeException if the role cannot be found
+ */
+ public RoleStatus lookupRoleStatus(Container c) {
return lookupRoleStatus(ContainerPriority.extractRole(c));
}
@@ -930,7 +944,7 @@
*/
public List<RoleStatus> cloneRoleStatusList() {
Collection<RoleStatus> statuses = roleStatusMap.values();
- List<RoleStatus> statusList = new ArrayList<RoleStatus>(statuses.size());
+ List<RoleStatus> statusList = new ArrayList<>(statuses.size());
try {
for (RoleStatus status : statuses) {
statusList.add((RoleStatus)(status.clone()));
@@ -942,6 +956,12 @@
}
+ /**
+ * Look up a role in the map
+ * @param name role name
+ * @return the instance
+ * @throws YarnRuntimeException if not found
+ */
public RoleStatus lookupRoleStatus(String name) throws YarnRuntimeException {
ProviderRole providerRole = roles.get(name);
if (providerRole == null) {
@@ -957,7 +977,7 @@
*/
public synchronized List<RoleInstance> cloneOwnedContainerList() {
Collection<RoleInstance> values = ownedContainers.values();
- return new ArrayList<RoleInstance>(values);
+ return new ArrayList<>(values);
}
/**
@@ -1002,7 +1022,7 @@
*/
public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
List<RoleInstance> allRoleInstances;
- Collection<RoleInstance> values = getLiveNodes().values();
+ Collection<RoleInstance> values = getLiveContainers().values();
allRoleInstances = new ArrayList<>(values);
return allRoleInstances;
}
@@ -1015,7 +1035,7 @@
*/
public synchronized RoleInstance getLiveInstanceByContainerID(String containerId)
throws NoSuchNodeException {
- Collection<RoleInstance> nodes = getLiveNodes().values();
+ Collection<RoleInstance> nodes = getLiveContainers().values();
return findNodeInCollection(containerId, nodes);
}
@@ -1056,13 +1076,12 @@
}
}
-
public synchronized List<RoleInstance> getLiveInstancesByContainerIDs(
Collection<String> containerIDs) {
//first, a hashmap of those containerIDs is built up
Set<String> uuidSet = new HashSet<String>(containerIDs);
List<RoleInstance> nodes = new ArrayList<RoleInstance>(uuidSet.size());
- Collection<RoleInstance> clusterNodes = getLiveNodes().values();
+ Collection<RoleInstance> clusterNodes = getLiveContainers().values();
for (RoleInstance node : clusterNodes) {
if (uuidSet.contains(node.id)) {
@@ -1080,7 +1099,7 @@
*/
public synchronized List<RoleInstance> enumLiveNodesInRole(String role) {
List<RoleInstance> nodes = new ArrayList<RoleInstance>();
- Collection<RoleInstance> allRoleInstances = getLiveNodes().values();
+ Collection<RoleInstance> allRoleInstances = getLiveContainers().values();
for (RoleInstance node : allRoleInstances) {
if (role.isEmpty() || role.equals(node.role)) {
nodes.add(node);
@@ -1110,14 +1129,13 @@
return nodes;
}
-
/**
* Build an instance map.
* @return the map of Role name to list of role instances
*/
private synchronized Map<String, List<String>> createRoleToInstanceMap() {
Map<String, List<String>> map = new HashMap<String, List<String>>();
- for (RoleInstance node : getLiveNodes().values()) {
+ for (RoleInstance node : getLiveContainers().values()) {
List<String> containers = map.get(node.role);
if (containers == null) {
containers = new ArrayList<String>();
@@ -1127,8 +1145,7 @@
}
return map;
}
-
-
+
/**
* Build a map of role->nodename->node-info
*
@@ -1136,7 +1153,7 @@
*/
public synchronized Map<String, Map<String, ClusterNode>> createRoleToClusterNodeMap() {
Map<String, Map<String, ClusterNode>> map = new HashMap<>();
- for (RoleInstance node : getLiveNodes().values()) {
+ for (RoleInstance node : getLiveContainers().values()) {
Map<String, ClusterNode> containers = map.get(node.role);
if (containers == null) {
@@ -1160,7 +1177,7 @@
instance.state = STATE_SUBMITTED;
instance.container = container;
instance.createTime = now();
- getStartingNodes().put(container.getId(), instance);
+ getStartingContainers().put(container.getId(), instance);
putOwnedContainer(container.getId(), instance);
roleHistory.onContainerStartSubmitted(container, instance);
}
@@ -1195,43 +1212,39 @@
roleHistory.onContainerReleaseSubmitted(container);
}
-
/**
- * Set up the resource requirements with all that this role needs,
- * then create the container request itself.
- * @param role role to ask an instance of
- * @param capability a resource to set up
- * @return the request for a new container
+ * Create a container request.
+ * Update internal state, such as the role request count.
+ * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
+ * This is where role history information will be used for placement decisions.
+ * @param role role
+ * @return the container request to submit or null if there is none
*/
- public AMRMClient.ContainerRequest buildContainerResourceAndRequest(
- RoleStatus role,
- Resource capability) {
- buildResourceRequirements(role, capability);
- String labelExpression = getLabelExpression(role);
- //get the role history to select a suitable node, if available
- AMRMClient.ContainerRequest containerRequest =
- createContainerRequest(role, capability, labelExpression);
- return containerRequest;
+ private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role) {
+ if (role.isAntiAffinePlacement()) {
+ return createAAContainerRequest(role);
+ } else {
+ incrementRequestCount(role);
+ return roleHistory.requestContainerForRole(role).getIssuedRequest();
+ }
}
/**
* Create a container request.
- * Update internal state, such as the role request count
- * This is where role history information will be used for placement decisions -
+ * Update internal state, such as the role request count.
+ * Anti-Affine: the {@link RoleStatus#outstandingAArequest} is set here.
+ * This is where role history information will be used for placement decisions.
* @param role role
- * @param resource requirements
- * @param labelExpression label expression to satisfy
- * @return the container request to submit
+ * @return the container request to submit or null if there is none
*/
- private AMRMClient.ContainerRequest createContainerRequest(RoleStatus role,
- Resource resource,
- String labelExpression) {
-
-
- AMRMClient.ContainerRequest request;
- request = roleHistory.requestNode(role, resource, labelExpression);
+ private AMRMClient.ContainerRequest createAAContainerRequest(RoleStatus role) {
+ OutstandingRequest request = roleHistory.requestContainerForAARole(role);
+ if (request == null) {
+ return null;
+ }
incrementRequestCount(role);
- return request;
+ role.setOutstandingAArequest(request);
+ return request.getIssuedRequest();
}
/**
@@ -1245,24 +1258,11 @@
incOutstandingContainerRequests();
}
-
- /**
- * dec requested count of a role
- * <p>
- * Also updates application state counters.
- * @param role role to decrement
- */
- protected synchronized void decrementRequestCount(RoleStatus role) {
- role.decRequested();
- }
-
/**
* Inc #of outstanding requests.
*/
private void incOutstandingContainerRequests() {
- synchronized (outstandingContainerRequests) {
- outstandingContainerRequests.inc();
- }
+ outstandingContainerRequests.inc();
}
/**
@@ -1295,11 +1295,11 @@
String option,
int defVal,
int maxVal) {
-
+
String val = resources.getComponentOpt(name, option,
Integer.toString(defVal));
Integer intVal;
- if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
+ if (YARN_RESOURCE_MAX.equals(val)) {
intVal = maxVal;
} else {
intVal = Integer.decode(val);
@@ -1307,15 +1307,15 @@
return intVal;
}
-
/**
* Build up the resource requirements for this role from the
* cluster specification, including substituing max allowed values
* if the specification asked for it.
* @param role role
- * @param capability capability to set up
+ * @param capability capability to set up. A new one may be created
+ * during normalization
*/
- public void buildResourceRequirements(RoleStatus role, Resource capability) {
+ public Resource buildResourceRequirements(RoleStatus role, Resource capability) {
// Set up resource requirements from role values
String name = role.getName();
ConfTreeOperations resources = getResourcesSnapshot();
@@ -1330,17 +1330,16 @@
DEF_YARN_MEMORY,
containerMaxMemory);
capability.setMemory(ram);
- }
-
- /**
- * Extract the label expression for this role.
- * @param role role
- */
- public String getLabelExpression(RoleStatus role) {
- // Set up resource requirements from role values
- String name = role.getName();
- ConfTreeOperations resources = getResourcesSnapshot();
- return resources.getComponentOpt(name, YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION);
+ log.debug("Component {} has RAM={}, vCores ={}", name, ram, cores);
+ Resource normalized = recordFactory.normalize(capability, minResource,
+ maxResource);
+ if (!Resources.equals(normalized, capability)) {
+ // resource requirements normalized to something other than asked for.
+ // LOG @ WARN so users can see why this is happening.
+ log.warn("Resource requirements of {} normalized" +
+ " from {} to {}", name, capability, normalized);
+ }
+ return normalized;
}
/**
@@ -1354,7 +1353,7 @@
throw new RuntimeException(
"Unknown role for node " + node);
}
- getLiveNodes().put(node.getContainerId(), node);
+ getLiveContainers().put(node.getContainerId(), node);
//tell role history
roleHistory.onContainerStarted(container);
}
@@ -1395,10 +1394,10 @@
instance);
}
instance.startTime = now();
- RoleInstance starting = getStartingNodes().remove(containerId);
+ RoleInstance starting = getStartingContainers().remove(containerId);
if (null == starting) {
throw new YarnRuntimeException(
- "Container "+ containerId +"%s is already started");
+ "Container "+ containerId +" is already started");
}
instance.state = STATE_LIVE;
RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
@@ -1423,7 +1422,7 @@
removeOwnedContainer(containerId);
incFailedCountainerCount();
incStartFailedCountainerCount();
- RoleInstance instance = getStartingNodes().remove(containerId);
+ RoleInstance instance = getStartingContainers().remove(containerId);
if (null != instance) {
RoleStatus roleStatus = lookupRoleStatus(instance.roleId);
String text;
@@ -1434,16 +1433,40 @@
}
instance.diagnostics = text;
roleStatus.noteFailed(true, text, ContainerOutcome.Failed);
- getFailedNodes().put(containerId, instance);
+ getFailedContainers().put(containerId, instance);
roleHistory.onNodeManagerContainerStartFailed(instance.container);
}
}
- public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
- roleHistory.onNodesUpdated(updatedNodes);
+ /**
+ * Handle node update from the RM. This syncs up the node map with the RM's view
+ * @param updatedNodes updated nodes
+ */
+ public synchronized NodeUpdatedOutcome onNodesUpdated(List<NodeReport> updatedNodes) {
+ boolean changed = roleHistory.onNodesUpdated(updatedNodes);
+ if (changed) {
+ log.info("YARN cluster changed —cancelling current AA requests");
+ List<AbstractRMOperation> operations = cancelOutstandingAARequests();
+ log.debug("Created {} cancel requests", operations.size());
+ return new NodeUpdatedOutcome(true, operations);
+ }
+ return new NodeUpdatedOutcome(false, new ArrayList<AbstractRMOperation>(0));
}
/**
+ * Return value of the {@link #onNodesUpdated(List)} call.
+ */
+ public static class NodeUpdatedOutcome {
+ public final boolean clusterChanged;
+ public final List<AbstractRMOperation> operations;
+
+ public NodeUpdatedOutcome(boolean clusterChanged,
+ List<AbstractRMOperation> operations) {
+ this.clusterChanged = clusterChanged;
+ this.operations = operations;
+ }
+ }
+ /**
* Is a role short lived by the threshold set for this application
* @param instance instance
* @return true if the instance is considered short lived
@@ -1487,7 +1510,6 @@
public int exitStatus = 0;
public boolean unknownNode = false;
-
public String toString() {
final StringBuilder sb =
new StringBuilder("NodeCompletionResult{");
@@ -1501,10 +1523,10 @@
return sb.toString();
}
}
-
+
/**
* handle completed node in the CD -move something from the live
- * server list to the completed server list
+ * server list to the completed server list.
* @param status the node that has just completed
* @return NodeCompletionResult
*/
@@ -1519,9 +1541,9 @@
log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- int releasing = roleStatus.decReleasing();
- int actual = roleStatus.decActual();
- int completedCount = roleStatus.incCompleted();
+ long releasing = roleStatus.decReleasing();
+ long actual = roleStatus.decActual();
+ long completedCount = roleStatus.incCompleted();
log.info("decrementing role count for role {} to {}; releasing={}, completed={}",
roleStatus.getName(),
actual,
@@ -1543,11 +1565,11 @@
if (roleInstance != null) {
//it was active, move it to failed
incFailedCountainerCount();
- failedNodes.put(containerId, roleInstance);
+ failedContainers.put(containerId, roleInstance);
} else {
// the container may have been noted as failed already, so look
// it up
- roleInstance = failedNodes.get(containerId);
+ roleInstance = failedContainers.get(containerId);
}
if (roleInstance != null) {
int roleId = roleInstance.roleId;
@@ -1564,7 +1586,7 @@
if (failedContainer != null) {
String completedLogsUrl = getLogsURLForContainer(failedContainer);
message = String.format("Failure %s on host %s (%d): %s",
- roleInstance.getContainerId().toString(),
+ roleInstance.getContainerId(),
failedContainer.getNodeId().getHost(),
exitStatus,
completedLogsUrl);
@@ -1601,26 +1623,26 @@
//remove the node
ContainerId id = status.getContainerId();
log.info("Removing node ID {}", id);
- RoleInstance node = getLiveNodes().remove(id);
+ RoleInstance node = getLiveContainers().remove(id);
if (node != null) {
node.state = STATE_DESTROYED;
node.exitCode = exitStatus;
node.diagnostics = status.getDiagnostics();
- getCompletedNodes().put(id, node);
+ getCompletedContainers().put(id, node);
result.roleInstance = node;
} else {
// not in the list
log.warn("Received notification of completion of unknown node {}", id);
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
-
+
// and the active node list if present
removeOwnedContainer(containerId);
-
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
containerId) : "container still in release queue";
- assert !getLiveNodes().containsKey(
+ assert !getLiveContainers().containsKey(
containerId) : " container still in live nodes";
assert getOwnedContainer(containerId) ==
null : "Container still in active container list";
@@ -1651,9 +1673,6 @@
return completedLogsUrl;
}
-
-
-
/**
* Return the percentage done that Slider is to have YARN display in its
* Web UI
@@ -1661,7 +1680,7 @@
*/
public synchronized float getApplicationProgressPercentage() {
float percentage;
- int desired = 0;
+ long desired = 0;
float actual = 0;
for (RoleStatus role : getRoleStatusMap().values()) {
desired += role.getDesired();
@@ -1682,7 +1701,7 @@
public ClusterDescription refreshClusterStatus() {
return refreshClusterStatus(null);
}
-
+
/**
* Update the cluster description with the current application state
* @param providerStatus status from the provider for the cluster info section
@@ -1695,13 +1714,13 @@
now);
if (providerStatus != null) {
for (Map.Entry<String, String> entry : providerStatus.entrySet()) {
- cd.setInfo(entry.getKey(),entry.getValue());
+ cd.setInfo(entry.getKey(), entry.getValue());
}
}
MapOperations infoOps = new MapOperations("info", cd.info);
infoOps.mergeWithoutOverwrite(applicationInfo);
SliderUtils.addBuildInfo(infoOps, "status");
- cd.statistics = new HashMap<String, Map<String, Integer>>();
+ cd.statistics = new HashMap<>();
// build the map of node -> container IDs
Map<String, List<String>> instanceMap = createRoleToInstanceMap();
@@ -1710,7 +1729,7 @@
//build the map of node -> containers
Map<String, Map<String, ClusterNode>> clusterNodes =
createRoleToClusterNodeMap();
- cd.status = new HashMap<String, Object>();
+ cd.status = new HashMap<>();
cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes);
@@ -1718,7 +1737,7 @@
String rolename = role.getName();
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
- cd.setRoleOpt(rolename, ResourceKeys.COMPONENT_INSTANCES,
+ cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
role.getDesired());
cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
@@ -1728,17 +1747,19 @@
cd.setRoleOpt(rolename, ROLE_FAILED_RECENTLY_INSTANCES, role.getFailedRecently());
cd.setRoleOpt(rolename, ROLE_NODE_FAILED_INSTANCES, role.getNodeFailed());
cd.setRoleOpt(rolename, ROLE_PREEMPTED_INSTANCES, role.getPreempted());
+ if (role.isAntiAffinePlacement()) {
+ cd.setRoleOpt(rolename, ROLE_PENDING_AA_INSTANCES, role.getPendingAntiAffineRequests());
+ }
Map<String, Integer> stats = role.buildStatistics();
cd.statistics.put(rolename, stats);
}
-
Map<String, Integer> sliderstats = getLiveStatistics();
cd.statistics.put(SliderKeys.COMPONENT_AM, sliderstats);
-
+
// liveness
cd.liveness = getApplicationLivenessInformation();
-
+
return cd;
}
@@ -1748,12 +1769,14 @@
*/
public ApplicationLivenessInformation getApplicationLivenessInformation() {
ApplicationLivenessInformation li = new ApplicationLivenessInformation();
- int outstanding = (int) outstandingContainerRequests.getCount();
+ RoleStatistics stats = getRoleStatistics();
+ int outstanding = (int)(stats.desired - stats.actual);
li.requestsOutstanding = outstanding;
li.allRequestsSatisfied = outstanding <= 0;
+ li.activeRequests = (int)stats.requested;
return li;
}
-
+
/**
* Get the live statistics map
* @return a map of statistics values, defined in the {@link StatusKeys}
@@ -1764,21 +1787,33 @@
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_LIVE,
liveNodes.size());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_COMPLETED,
- (int)completedContainerCount.getCount());
+ completedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_FAILED,
- (int)failedContainerCount.getCount());
+ failedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_STARTED,
- (int)startedContainers.getCount());
+ startedContainers.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_START_FAILED,
- (int) startFailedContainerCount.getCount());
+ startFailedContainerCount.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_SURPLUS,
- (int)surplusContainers.getCount());
+ surplusContainers.intValue());
sliderstats.put(StatusKeys.STATISTICS_CONTAINERS_UNKNOWN_COMPLETED,
completionOfUnknownContainerEvent.get());
return sliderstats;
}
/**
+ * Get the aggregate statistics across all roles
+ * @return role statistics
+ */
+ public RoleStatistics getRoleStatistics() {
+ RoleStatistics stats = new RoleStatistics();
+ for (RoleStatus role : getRoleStatusMap().values()) {
+ stats.add(role.getStatistics());
+ }
+ return stats;
+ }
+
+ /**
* Get a snapshot of component information.
* <p>
* This does <i>not</i> include any container list, which
@@ -1788,8 +1823,7 @@
public Map<String, ComponentInformation> getComponentInfoSnapshot() {
Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
- Map<String, ComponentInformation> results =
- new HashMap<String, ComponentInformation>(
+ Map<String, ComponentInformation> results = new HashMap<>(
statusMap.size());
for (RoleStatus status : statusMap.values()) {
@@ -1806,7 +1840,7 @@
public synchronized List<AbstractRMOperation> reviewRequestAndReleaseNodes()
throws SliderInternalStateException, TriggerClusterTeardownException {
log.debug("in reviewRequestAndReleaseNodes()");
- List<AbstractRMOperation> allOperations = new ArrayList<AbstractRMOperation>();
+ List<AbstractRMOperation> allOperations = new ArrayList<>();
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
if (!roleStatus.isExcludeFromFlexing()) {
List<AbstractRMOperation> operations = reviewOneRole(roleStatus);
@@ -1826,8 +1860,10 @@
throws TriggerClusterTeardownException {
long failures = role.getFailedRecently();
int threshold = getFailureThresholdForRole(role);
- log.debug("Failure count of component: {}: {}, threshold={}",
- role.getName(), failures, threshold);
+ if (log.isDebugEnabled() && failures > 0) {
+ log.debug("Failure count of component: {}: {}, threshold={}",
+ role.getName(), failures, threshold);
+ }
if (failures > threshold) {
throw new TriggerClusterTeardownException(
@@ -1853,7 +1889,7 @@
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleStatus.getName(),
- ResourceKeys.CONTAINER_FAILURE_THRESHOLD,
+ CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
@@ -1867,7 +1903,7 @@
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
return resources.getComponentOptInt(roleName,
- ResourceKeys.NODE_FAILURE_THRESHOLD,
+ NODE_FAILURE_THRESHOLD,
nodeFailureThreshold);
}
@@ -1891,7 +1927,25 @@
public List<AbstractRMOperation> escalateOutstandingRequests() {
return roleHistory.escalateOutstandingRequests();
}
-
+
+ /**
+ * Cancel any outstanding AA Requests, building up the list of ops to
+ * cancel, removing them from RoleHistory structures and the RoleStatus
+ * entries.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public synchronized List<AbstractRMOperation> cancelOutstandingAARequests() {
+ // get the list of cancel operations
+ List<AbstractRMOperation> operations = roleHistory.cancelOutstandingAARequests();
+ for (RoleStatus roleStatus : roleStatusMap.values()) {
+ if (roleStatus.isAARequestOutstanding()) {
+ log.info("Cancelling outstanding AA request for {}", roleStatus);
+ roleStatus.cancelOutstandingAARequest();
+ }
+ }
+ return operations;
+ }
+
/**
* Look at the allocation status of one role, and trigger add/release
* actions if the number of desired role instances doesn't equal
@@ -1908,15 +1962,16 @@
private List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List<AbstractRMOperation> operations = new ArrayList<>();
- int delta;
- int expected;
+ long delta;
+ long expected;
String name = role.getName();
synchronized (role) {
delta = role.getDelta();
expected = role.getDesired();
}
- log.info("Reviewing {} : expected {}", role, expected);
+ log.info("Reviewing {} : ", role);
+ log.debug("Expected {}, Delta: {}", expected, delta);
checkFailureThreshold(role);
if (expected < 0 ) {
@@ -1929,38 +1984,53 @@
}
if (delta > 0) {
- log.info("{}: Asking for {} more nodes(s) for a total of {} ", name,
- delta, expected);
- //more workers needed than we have -ask for more
- for (int i = 0; i < delta; i++) {
- Resource capability = recordFactory.newResource();
- AMRMClient.ContainerRequest containerAsk =
- buildContainerResourceAndRequest(role, capability);
- log.info("Container ask is {} and label = {}", containerAsk,
- containerAsk.getNodeLabelExpression());
- int askMemory = containerAsk.getCapability().getMemory();
- if (askMemory > this.containerMaxMemory) {
- log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
+ // more workers needed than we have -ask for more
+ log.info("{}: Asking for {} more nodes(s) for a total of {} ", name, delta, expected);
+
+ if (role.isAntiAffinePlacement()) {
+ long pending = delta;
+ if (roleHistory.canPlaceAANodes()) {
+ // build one only if there is none outstanding, the role history knows
+ // enough about the cluster to ask, and there is somewhere to place
+ // the node
+ if (!role.isAARequestOutstanding()) {
+ // no outstanding AA; try to place things
+ AMRMClient.ContainerRequest request = createAAContainerRequest(role);
+ if (request != null) {
+ pending--;
+ log.info("Starting an anti-affine request sequence for {} nodes; pending={}",
+ delta, pending);
+ addContainerRequest(operations, request);
+ } else {
+ log.info("No location for anti-affine request");
+ }
+ }
+ } else {
+ log.warn("Awaiting node map before generating anti-affinity requests");
}
- operations.add(new ContainerRequestOperation(containerAsk));
+ log.info("Setting pending to {}", pending);
+ role.setPendingAntiAffineRequests(pending);
+ } else {
+
+ for (int i = 0; i < delta; i++) {
+ //get the role history to select a suitable node, if available
+ addContainerRequest(operations, createContainerRequest(role));
+ }
}
} else if (delta < 0) {
log.info("{}: Asking for {} fewer node(s) for a total of {}", name,
-delta,
expected);
- //reduce the number expected (i.e. subtract the delta)
+ // reduce the number expected (i.e. subtract the delta)
+ long excess = -delta;
- //then pick some containers to kill
- int excess = -delta;
-
- // how many requests are outstanding
- int outstandingRequests = role.getRequested();
+ // how many requests are outstanding? for AA roles, this includes pending
+ long outstandingRequests = role.getRequested() + role.getPendingAntiAffineRequests();
if (outstandingRequests > 0) {
// outstanding requests.
- int toCancel = Math.min(outstandingRequests, excess);
+ int toCancel = (int)Math.min(outstandingRequests, excess);
// Delegate to Role History
-
List<AbstractRMOperation> cancellations = roleHistory.cancelRequestsForRole(role, toCancel);
log.info("Found {} outstanding requests to cancel", cancellations.size());
operations.addAll(cancellations);
@@ -1980,7 +2050,6 @@
}
}
-
// after the cancellation there may be no excess
if (excess > 0) {
@@ -1994,7 +2063,7 @@
log.info("No containers for component {}", roleId);
}
- // cut all release-in-progress nodes
+ // filter out all release-in-progress nodes
ListIterator<RoleInstance> li = containersToRelease.listIterator();
while (li.hasNext()) {
RoleInstance next = li.next();
@@ -2014,16 +2083,13 @@
// ask the release selector to sort the targets
containersToRelease = containerReleaseSelector.sortCandidates(
roleId,
- containersToRelease,
- excess);
+ containersToRelease);
- //crop to the excess
-
- List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
- ? containersToRelease.subList(0, excess)
+ // crop to the excess
+ List<RoleInstance> finalCandidates = (excess < numberAvailableForRelease)
+ ? containersToRelease.subList(0, (int)excess)
: containersToRelease;
-
// then build up a release operation, logging each container as released
for (RoleInstance possible : finalCandidates) {
log.info("Targeting for release: {}", possible);
@@ -2032,13 +2098,43 @@
}
}
+ } else {
+ // actual + requested == desired
+ // there's a special case here: clear all pending AA requests
+ if (role.getPendingAntiAffineRequests() > 0) {
+ log.debug("Clearing outstanding pending AA requests");
+ role.setPendingAntiAffineRequests(0);
+ }
}
- // list of operations to execute
+ // there's now a list of operations to execute
+ log.debug("operations scheduled: {}; updated role: {}", operations.size(), role);
return operations;
}
/**
+ * Add a container request if the request is non-null
+ * @param operations operations to add the entry to
+ * @param containerAsk what to ask for
+ * @return true if a request was added
+ */
+ private boolean addContainerRequest(List<AbstractRMOperation> operations,
+ AMRMClient.ContainerRequest containerAsk) {
+ if (containerAsk != null) {
+ log.info("Container ask is {} and label = {}", containerAsk,
+ containerAsk.getNodeLabelExpression());
+ int askMemory = containerAsk.getCapability().getMemory();
+ if (askMemory > this.containerMaxMemory) {
+ log.warn("Memory requested: {} > max of {}", askMemory, containerMaxMemory);
+ }
+ operations.add(new ContainerRequestOperation(containerAsk));
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /**
* Releases a container based on container id
* @param containerId
* @return
@@ -2117,53 +2213,42 @@
* a list of operations to perform
* @param allocatedContainers the containers allocated
* @param assignments the assignments of roles to containers
- * @param releaseOperations any release operations
+ * @param operations any allocation or release operations
*/
public synchronized void onContainersAllocated(List<Container> allocatedContainers,
List<ContainerAssignment> assignments,
- List<AbstractRMOperation> releaseOperations) {
+ List<AbstractRMOperation> operations) {
assignments.clear();
- releaseOperations.clear();
+ operations.clear();
List<Container> ordered = roleHistory.prepareAllocationList(allocatedContainers);
+ log.debug("onContainersAllocated(): Total containers allocated = {}", ordered.size());
for (Container container : ordered) {
- String containerHostInfo = container.getNodeId().getHost()
- + ":" +
- container.getNodeId().getPort();
+ final NodeId nodeId = container.getNodeId();
+ String containerHostInfo = nodeId.getHost() + ":" + nodeId.getPort();
//get the role
final ContainerId cid = container.getId();
final RoleStatus role = lookupRoleStatus(container);
-
//dec requested count
- decrementRequestCount(role);
+ role.decRequested();
//inc allocated count -this may need to be dropped in a moment,
// but us needed to update the logic below
- final int allocated = role.incActual();
- final int desired = role.getDesired();
+ final long allocated = role.incActual();
+ final long desired = role.getDesired();
final String roleName = role.getName();
- final ContainerAllocation allocation =
+ final ContainerAllocationResults allocation =
roleHistory.onContainerAllocated(container, desired, allocated);
final ContainerAllocationOutcome outcome = allocation.outcome;
- // cancel an allocation request which granted this, so as to avoid repeated
- // requests
- if (allocation.origin != null && allocation.origin.getIssuedRequest() != null) {
- releaseOperations.add(allocation.origin.createCancelOperation());
- } else {
- // there's a request, but no idea what to cancel.
- // rather than try to recover from it inelegantly, (and cause more confusion),
- // log the event, but otherwise continue
- log.warn("Unexpected allocation of container "
- + SliderUtils.containerToString(container));
- }
+ // add all requests to the operations list
+ operations.addAll(allocation.operations);
//look for condition where we get more back than we asked
if (allocated > desired) {
- log.info("Discarding surplus {} container {} on {}", roleName, cid,
- containerHostInfo);
- releaseOperations.add(new ContainerReleaseOperation(cid));
+ log.info("Discarding surplus {} container {} on {}", roleName, cid, containerHostInfo);
+ operations.add(new ContainerReleaseOperation(cid));
//register as a surplus node
surplusNodes.add(cid);
surplusContainers.inc();
@@ -2179,13 +2264,34 @@
" on {}:{},",
roleName,
cid,
- container.getNodeId().getHost(),
- container.getNodeId().getPort()
- );
+ nodeId.getHost(),
+ nodeId.getPort());
assignments.add(new ContainerAssignment(container, role, outcome));
//add to the history
roleHistory.onContainerAssigned(container);
+ // now for AA requests, add some more
+ if (role.isAntiAffinePlacement()) {
+ role.completeOutstandingAARequest();
+ // check invariants. The new node must become unavailable.
+ NodeInstance node = roleHistory.getOrCreateNodeInstance(container);
+ if (node.canHost(role.getKey(), role.getLabelExpression())) {
+ log.error("Assigned node still declares as available {}", node.toFullString() );
+ }
+ if (role.getPendingAntiAffineRequests() > 0) {
+ // still an outstanding AA request: need to issue a new one.
+ log.info("Asking for next container for AA role {}", roleName);
+ if (!addContainerRequest(operations, createAAContainerRequest(role))) {
+ log.info("No capacity in cluster for new requests");
+ } else {
+ role.decPendingAntiAffineRequests();
+ }
+ log.debug("Current AA role status {}", role);
+ } else {
+ log.info("AA request sequence completed for role {}", role);
+ }
+ }
+
}
}
}
@@ -2248,7 +2354,7 @@
cid,
roleName,
containerHostInfo);
-
+
//update app state internal structures and maps
RoleInstance instance = new RoleInstance(container);
@@ -2268,4 +2374,32 @@
// now pretend it has just started
innerOnNodeManagerContainerStarted(cid);
}
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("AppState{");
+ sb.append("applicationLive=").append(applicationLive);
+ sb.append(", live nodes=").append(liveNodes.size());
+ sb.append(", startedContainers=").append(startedContainers);
+ sb.append(", startFailedContainerCount=").append(startFailedContainerCount);
+ sb.append(", surplusContainers=").append(surplusContainers);
+ sb.append(", failedContainerCount=").append(failedContainerCount);
+ sb.append(", outstanding non-AA Container Requests=")
+ .append(outstandingContainerRequests);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Build map of role ID-> name
+ * @return
+ */
+ public Map<Integer, String> buildNamingMap() {
+ Map<Integer, RoleStatus> statusMap = getRoleStatusMap();
+ Map<Integer, String> naming = new HashMap<>(statusMap.size());
+ for (Map.Entry<Integer, RoleStatus> entry : statusMap.entrySet()) {
+ naming.put(entry.getKey(), entry.getValue().getName());
+ }
+ return naming;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
new file mode 100644
index 0000000..a8aa1a2
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.slider.core.conf.AggregateConf;
+import org.apache.slider.providers.ProviderRole;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Binding information for application states; designed to be extensible
+ * so that tests don't have to be massivley reworked when new arguments
+ * are added.
+ */
+public class AppStateBindingInfo {
+ public AggregateConf instanceDefinition;
+ public Configuration serviceConfig = new Configuration();
+ public Configuration publishedProviderConf = new Configuration(false);
+ public List<ProviderRole> roles = new ArrayList<>();
+ public FileSystem fs;
+ public Path historyPath;
+ public List<Container> liveContainers = new ArrayList<>(0);
+ public Map<String, String> applicationInfo = new HashMap<>();
+ public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
+ /** node reports off the RM. */
+ public List<NodeReport> nodeReports = new ArrayList<>(0);
+
+ public void validate() throws IllegalArgumentException {
+ Preconditions.checkArgument(instanceDefinition != null, "null instanceDefinition");
+ Preconditions.checkArgument(serviceConfig != null, "null appmasterConfig");
+ Preconditions.checkArgument(publishedProviderConf != null, "null publishedProviderConf");
+ Preconditions.checkArgument(releaseSelector != null, "null releaseSelector");
+ Preconditions.checkArgument(roles != null, "null providerRoles");
+ Preconditions.checkArgument(fs != null, "null fs");
+ Preconditions.checkArgument(historyPath != null, "null historyDir");
+ Preconditions.checkArgument(nodeReports != null, "null nodeReports");
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
similarity index 77%
rename from slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
rename to slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
index 306ffb2..e80639e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocation.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerAllocationResults.java
@@ -18,10 +18,15 @@
package org.apache.slider.server.appmaster.state;
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
+
+import java.util.ArrayList;
+import java.util.List;
+
/**
* This is just a tuple of the outcome of a container allocation
*/
-public class ContainerAllocation {
+public class ContainerAllocationResults {
/**
* What was the outcome of this allocation: placed, escalated, ...
@@ -35,12 +40,11 @@
*/
public OutstandingRequest origin;
- public ContainerAllocation(ContainerAllocationOutcome outcome,
- OutstandingRequest origin) {
- this.outcome = outcome;
- this.origin = origin;
- }
+ /**
+ * A possibly empty list of requests to add to the follow-up actions
+ */
+ public List<AbstractRMOperation> operations = new ArrayList<>(0);
- public ContainerAllocation() {
+ public ContainerAllocationResults() {
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
index 3cc2106..0322f83 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerPriority.java
@@ -57,10 +57,9 @@
locationSpecified));
return pri;
}
-
-
+
public static int extractRole(int priority) {
- return priority >= NOLOCATION ? priority^NOLOCATION : priority;
+ return priority >= NOLOCATION ? priority ^ NOLOCATION : priority;
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
index 0cbc134..fafbada 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerReleaseSelector.java
@@ -30,9 +30,8 @@
* Given a list of candidate containers, return a sorted version of the priority
* in which they should be released.
* @param candidates candidate list ... everything considered suitable
- * @return
+ * @return the list of candidates
*/
List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates,
- int minimumToSelect);
+ List<RoleInstance> candidates);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
index 9d936a1..38c5b8e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/MostRecentContainerReleaseSelector.java
@@ -32,8 +32,7 @@
@Override
public List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates,
- int minimumToSelect) {
+ List<RoleInstance> candidates) {
Collections.sort(candidates, new newerThan());
return candidates;
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
index 0f46054..cf3881e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java
@@ -18,25 +18,27 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.slider.api.types.NodeEntryInformation;
+
/**
* Information about the state of a role on a specific node instance.
* No fields are synchronized; sync on the instance to work with it
- <p>
- The two fields `releasing` and `requested` are used to track the ongoing
- state of YARN requests; they do not need to be persisted across stop/start
- cycles. They may be relevant across AM restart, but without other data
- structures in the AM, not enough to track what the AM was up to before
- it was restarted. The strategy will be to ignore unexpected allocation
- responses (which may come from pre-restart) requests, while treating
- unexpected container release responses as failures.
- <p>
- The `active` counter is only decremented after a container release response
- has been received.
- <p>
-
- Accesses are synchronized.
+ * <p>
+ * The two fields `releasing` and `requested` are used to track the ongoing
+ * state of YARN requests; they do not need to be persisted across stop/start
+ * cycles. They may be relevant across AM restart, but without other data
+ * structures in the AM, not enough to track what the AM was up to before
+ * it was restarted. The strategy will be to ignore unexpected allocation
+ * responses (which may come from pre-restart) requests, while treating
+ * unexpected container release responses as failures.
+ * <p>
+ * The `active` counter is only decremented after a container release response
+ * has been received.
+ * <p>
+ *
*/
-public class NodeEntry {
+public class NodeEntry implements Cloneable {
public final int rolePriority;
@@ -47,33 +49,57 @@
/**
* instance explicitly requested on this node: it's OK if an allocation
* comes in that has not been (and when that happens, this count should
- * not drop)
+ * not drop).
*/
private int requested;
+
+ /** number of starting instances */
private int starting;
+
+ /** incrementing counter of instances that failed to start */
private int startFailed;
+
+ /** incrementing counter of instances that failed */
private int failed;
- private int preempted;
+
/**
* Counter of "failed recently" events. These are all failures
* which have happened since it was last reset.
*/
private int failedRecently;
+
+ /** incrementing counter of instances that have been pre-empted. */
+ private int preempted;
+
/**
* Number of live nodes.
*/
private int live;
+
+ /** number of containers being released off this node */
private int releasing;
+
+ /** timestamp of last use */
private long lastUsed;
-
+
/**
- * Is the node available for assignments. This does not track
- * whether or not there are any outstanding requests for this node
- * @return true if there are no role instances here
- * other than some being released.
+ * Is the node available for assignments? That is, it is
+ * not running any instances of this type, nor are there
+ * any requests oustanding for it.
+ * @return true if a new request could be issued without taking
+ * the number of instances > 1.
*/
public synchronized boolean isAvailable() {
- return getActive() == 0 && (requested == 0) && starting == 0;
+ return live + requested + starting - releasing <= 0;
+ }
+
+ /**
+ * Are the anti-affinity constraints held. That is, zero or one
+ * node running or starting
+ * @return true if the constraint holds.
+ */
+ public synchronized boolean isAntiAffinityConstraintHeld() {
+ return (live - releasing + starting) <= 1;
}
/**
@@ -240,10 +266,16 @@
return failedRecently;
}
+ @VisibleForTesting
+ public synchronized void setFailedRecently(int failedRecently) {
+ this.failedRecently = failedRecently;
+ }
+
public synchronized int getPreempted() {
return preempted;
}
+
/**
* Reset the failed recently count.
*/
@@ -266,4 +298,28 @@
sb.append('}');
return sb.toString();
}
+
+ /**
+ * Produced a serialized form which can be served up as JSON
+ * @return a summary of the current role status.
+ */
+ public synchronized NodeEntryInformation serialize() {
+ NodeEntryInformation info = new NodeEntryInformation();
+ info.priority = rolePriority;
+ info.requested = requested;
+ info.releasing = releasing;
+ info.starting = starting;
+ info.startFailed = startFailed;
+ info.failed = failed;
+ info.failedRecently = failedRecently;
+ info.preempted = preempted;
+ info.live = live;
+ info.lastUsed = lastUsed;
+ return info;
+ }
+
+ @Override
+ public Object clone() throws CloneNotSupportedException {
+ return super.clone();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
index fb80b5f..cc17cf0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeInstance.java
@@ -18,11 +18,20 @@
package org.apache.slider.server.appmaster.state;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.common.tools.Comparators;
+import org.apache.slider.common.tools.SliderUtils;
+
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
+import java.util.Map;
/**
* A node instance -stores information about a node in the cluster.
@@ -33,6 +42,34 @@
public final String hostname;
+ /**
+ * last state of node. Starts off as {@link NodeState#RUNNING},
+ * on the assumption that it is live.
+ */
+ private NodeState nodeState = NodeState.RUNNING;
+
+ /**
+ * Last node report. If null: none
+ */
+ private NodeReport nodeReport = null;
+
+ /**
+ * time of state update
+ */
+ private long nodeStateUpdateTime = 0;
+
+ /**
+ * Node labels.
+ *
+ * IMPORTANT: we assume that there is one label/node, which is the policy
+ * for Hadoop as of November 2015
+ */
+ private String nodeLabels = "";
+
+ /**
+ * An unordered list of node entries of specific roles. There's nothing
+ * indexed so as to support sparser datastructures.
+ */
private final List<NodeEntry> nodeEntries;
/**
@@ -45,6 +82,37 @@
}
/**
+ * Update the node status.
+ * The return code is true if the node state changed enough to
+ * trigger a re-evaluation of pending requests. That is, either a node
+ * became available when it was previously not, or the label changed
+ * on an available node.
+ *
+ * Transitions of a node from live to dead aren't treated as significant,
+ * nor label changes on a dead node.
+ *
+ * @param report latest node report
+ * @return true if the node state changed enough for a request evaluation.
+ */
+ public synchronized boolean updateNode(NodeReport report) {
+ nodeStateUpdateTime = report.getLastHealthReportTime();
+ nodeReport = report;
+ NodeState oldState = nodeState;
+ boolean oldStateUnusable = oldState.isUnusable();
+ nodeState = report.getNodeState();
+ boolean newUsable = !nodeState.isUnusable();
+ boolean nodeNowAvailable = oldStateUnusable && newUsable;
+ String labels = this.nodeLabels;
+ nodeLabels = SliderUtils.extractNodeLabel(report);
+ return nodeNowAvailable
+ || newUsable && !this.nodeLabels.equals(labels);
+ }
+
+ public String getNodeLabels() {
+ return nodeLabels;
+ }
+
+ /**
* Get the entry for a role -if present
* @param role role index
* @return the entry
@@ -75,6 +143,15 @@
}
/**
+ * Get the node entry matching a container on this node
+ * @param container container
+ * @return matching node instance for the role
+ */
+ public NodeEntry getOrCreate(Container container) {
+ return getOrCreate(ContainerPriority.extractRole(container));
+ }
+
+ /**
* Count the number of active role instances on this node
* @param role role index
* @return 0 if there are none, otherwise the #of nodes that are running and
@@ -96,6 +173,14 @@
}
/**
+ * Is the node considered online
+ * @return the node
+ */
+ public boolean isOnline() {
+ return !nodeState.isUnusable();
+ }
+
+ /**
* Query for a node being considered unreliable
* @param role role key
* @param threshold threshold above which a node is considered unreliable
@@ -103,7 +188,6 @@
*/
public boolean isConsideredUnreliable(int role, int threshold) {
NodeEntry entry = get(role);
-
return entry != null && entry.getFailedRecently() > threshold;
}
@@ -125,7 +209,6 @@
nodeEntries.add(nodeEntry);
}
-
/**
* run through each entry; gc'ing & removing old ones that don't have
* a recent failure count (we care about those)
@@ -168,11 +251,12 @@
public String toFullString() {
final StringBuilder sb =
new StringBuilder(toString());
- int i = 0;
+ sb.append("{ ");
for (NodeEntry entry : nodeEntries) {
sb.append(String.format("\n [%02d] ", entry.rolePriority));
sb.append(entry.toString());
}
+ sb.append("} ");
return sb.toString();
}
@@ -215,16 +299,60 @@
}
/**
- * A comparator for sorting entries where the node is preferred over another.
- * <p>
- * The exact algorithm may change
- *
- * @return +ve int if left is preferred to right; -ve if right over left, 0 for equal
+ * Produced a serialized form which can be served up as JSON
+ * @param naming map of priority -> value for naming entries
+ * @return a summary of the current role status.
*/
- public static class Preferred implements Comparator<NodeInstance>,
- Serializable {
+ public synchronized NodeInformation serialize(Map<Integer, String> naming) {
+ NodeInformation info = new NodeInformation();
+ info.hostname = hostname;
+ // null-handling state constructor
+ info.state = "" + nodeState;
+ info.lastUpdated = nodeStateUpdateTime;
+ info.labels = nodeLabels;
+ if (nodeReport != null) {
+ info.httpAddress = nodeReport.getHttpAddress();
+ info.rackName = nodeReport.getRackName();
+ info.healthReport = nodeReport.getHealthReport();
+ }
+ info.entries = new HashMap<>(nodeEntries.size());
+ for (NodeEntry nodeEntry : nodeEntries) {
+ String name = naming.get(nodeEntry.rolePriority);
+ if (name == null) {
+ name = Integer.toString(nodeEntry.rolePriority);
+ }
+ info.entries.put(name, nodeEntry.serialize());
+ }
+ return info;
+ }
- final int role;
+ /**
+ * Is this node instance a suitable candidate for the specific role?
+ * @param role role ID
+ * @param label label which must match, or "" for no label checks
+ * @return true if the node has space for this role, is running and the labels
+ * match.
+ */
+ public boolean canHost(int role, String label) {
+ return isOnline()
+ && (SliderUtils.isUnset(label) || label.equals(nodeLabels)) // label match
+ && getOrCreate(role).isAvailable(); // no live role
+ }
+
+ /**
+ * A comparator for sorting entries where the node is preferred over another.
+ *
+ * The exact algorithm may change: current policy is "most recent first", so sorted
+ * on the lastUsed
+ *
+ * the comparision is a positive int if left is preferred to right;
+ * negative if right over left, 0 for equal
+ */
+ public static class Preferred implements Comparator<NodeInstance>, Serializable {
+
+ private static final Comparators.InvertedLongComparator comparator =
+ new Comparators.InvertedLongComparator();
+ private final int role;
public Preferred(int role) {
this.role = role;
@@ -234,16 +362,9 @@
public int compare(NodeInstance o1, NodeInstance o2) {
NodeEntry left = o1.get(role);
NodeEntry right = o2.get(role);
- long ageL = left != null ? left.getLastUsed() : 0;
- long ageR = right != null ? right.getLastUsed() : 0;
-
- if (ageL > ageR) {
- return -1;
- } else if (ageL < ageR) {
- return 1;
- }
- // equal
- return 0;
+ long ageL = left != null ? left.getLastUsed() : -1;
+ long ageR = right != null ? right.getLastUsed() : -1;
+ return comparator.compare(ageL, ageR);
}
}
@@ -256,7 +377,7 @@
public static class MoreActiveThan implements Comparator<NodeInstance>,
Serializable {
- final int role;
+ private final int role;
public MoreActiveThan(int role) {
this.role = role;
@@ -269,5 +390,20 @@
return activeRight - activeLeft;
}
}
+ /**
+ * A comparator for sorting entries alphabetically
+ */
+ public static class CompareNames implements Comparator<NodeInstance>,
+ Serializable {
+
+ public CompareNames() {
+ }
+
+ @Override
+ public int compare(NodeInstance left, NodeInstance right) {
+ return left.hostname.compareTo(right.hostname);
+ }
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
index aa50baa..3858b68 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeMap.java
@@ -19,6 +19,7 @@
package org.apache.slider.server.appmaster.state;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -26,9 +27,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
-import java.util.ListIterator;
import java.util.Map;
/**
@@ -84,27 +83,6 @@
}
/**
- * purge the history of all nodes that have been inactive since the absolute time
- * @param absoluteTime time
- * @return the number purged
- */
- public int purgeUnusedEntries(long absoluteTime) {
- int purged = 0;
- Iterator<Map.Entry<String, NodeInstance>> iterator =
- entrySet().iterator();
- while (iterator.hasNext()) {
- Map.Entry<String, NodeInstance> entry = iterator.next();
- NodeInstance ni = entry.getValue();
- if (!ni.purgeUnusedEntries(absoluteTime)) {
- iterator.remove();
- purged ++;
- }
- }
- return purged;
- }
-
-
- /**
* reset the failed recently counters
*/
public void resetFailedRecently() {
@@ -115,8 +93,23 @@
}
/**
+ * Update the node state. Return true if the node state changed: either by
+ * being created, or by changing its internal state as defined
+ * by {@link NodeInstance#updateNode(NodeReport)}.
+ *
+ * @param hostname host name
+ * @param report latest node report
+ * @return true if the node state changed enough for a request evaluation.
+ */
+ public boolean updateNode(String hostname, NodeReport report) {
+ boolean nodeExisted = get(hostname) != null;
+ boolean updated = getOrCreate(hostname).updateNode(report);
+ return updated || !nodeExisted;
+ }
+
+ /**
* Clone point
- * @return
+ * @return a shallow clone
*/
@Override
public Object clone() {
@@ -134,4 +127,48 @@
put(node.hostname, node);
}
}
+
+ /**
+ * Test helper: build or update a cluster from a list of node reports
+ * @param reports the list of reports
+ * @return true if this has been considered to have changed the cluster
+ */
+ @VisibleForTesting
+ public boolean buildOrUpdate(List<NodeReport> reports) {
+ boolean updated = false;
+ for (NodeReport report : reports) {
+ updated |= getOrCreate(report.getNodeId().getHost()).updateNode(report);
+ }
+ return updated;
+ }
+
+ /**
+ * Scan the current node map for all nodes capable of hosting an instance
+ * @param role role ID
+ * @param label label which must match, or "" for no label checks
+ * @return a possibly empty list of node instances matching the criteria.
+ */
+ public List<NodeInstance> findAllNodesForRole(int role, String label) {
+ List<NodeInstance> nodes = new ArrayList<>(size());
+ for (NodeInstance instance : values()) {
+ if (instance.canHost(role, label)) {
+ nodes.add(instance);
+ }
+ }
+ Collections.sort(nodes, new NodeInstance.CompareNames());
+ return nodes;
+ }
+
+ @Override
+ public synchronized String toString() {
+ final StringBuilder sb = new StringBuilder("NodeMap{");
+ List<String> keys = new ArrayList<>(keySet());
+ Collections.sort(keys);
+ for (String key : keys) {
+ sb.append(key).append(": ");
+ sb.append(get(key).toFullString()).append("\n");
+ }
+ sb.append('}');
+ return sb.toString();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index 2d31ffd..3a75f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -24,10 +24,12 @@
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
+import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.operations.CancelSingleRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -42,24 +44,21 @@
* instance constructed with (role, hostname) can be used to look up
* a complete request instance in the {@link OutstandingRequestTracker} map
*/
-public final class OutstandingRequest {
+public final class OutstandingRequest extends RoleHostnamePair {
protected static final Logger log =
LoggerFactory.getLogger(OutstandingRequest.class);
/**
- * requested role
- */
- public final int roleId;
-
- /**
* Node the request is for -may be null
*/
public final NodeInstance node;
-
+
/**
- * hostname -will be null if node==null
+ * A list of all possible nodes to list in an AA request. For a non-AA
+ * request where {@link #node} is set, element 0 of the list is the same
+ * value.
*/
- public final String hostname;
+ public final List<NodeInstance> nodes = new ArrayList<>(1);
/**
* Optional label. This is cached as the request option (explicit-location + label) is forbidden,
@@ -70,21 +69,21 @@
/**
* Requested time in millis.
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private AMRMClient.ContainerRequest issuedRequest;
/**
* Requested time in millis.
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private long requestedTimeMillis;
/**
* Time in millis after which escalation should be triggered..
* <p>
- * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long, String)}
+ * Only valid after {@link #buildContainerRequest(Resource, RoleStatus, long)}
*/
private long escalationTimeoutMillis;
@@ -104,15 +103,21 @@
private int priority = -1;
/**
+ * Is this an Anti-affine request which should be cancelled on
+ * a cluster resize?
+ */
+ private boolean antiAffine = false;
+
+ /**
* Create a request
* @param roleId role
* @param node node -can be null
*/
public OutstandingRequest(int roleId,
NodeInstance node) {
- this.roleId = roleId;
+ super(roleId, node != null ? node.hostname : null);
this.node = node;
- this.hostname = node != null ? node.hostname : null;
+ nodes.add(node);
}
/**
@@ -124,9 +129,21 @@
* @param hostname hostname
*/
public OutstandingRequest(int roleId, String hostname) {
+ super(roleId, hostname);
this.node = null;
- this.roleId = roleId;
- this.hostname = hostname;
+ }
+
+ /**
+ * Create an Anti-affine reques, including all listed nodes (there must be one)
+ * as targets.
+ * @param roleId role
+ * @param nodes list of nodes
+ */
+ public OutstandingRequest(int roleId, List<NodeInstance> nodes) {
+ super(roleId, nodes.get(0).hostname);
+ this.node = null;
+ this.antiAffine = true;
+ this.nodes.addAll(nodes);
}
/**
@@ -161,6 +178,14 @@
return priority;
}
+ public boolean isAntiAffine() {
+ return antiAffine;
+ }
+
+ public void setAntiAffine(boolean antiAffine) {
+ this.antiAffine = antiAffine;
+ }
+
/**
* Build a container request.
* <p>
@@ -177,16 +202,15 @@
* @param resource resource
* @param role role
* @param time time in millis to record as request time
- * @param labelExpression label to satisfy
* @return the request to raise
*/
public synchronized AMRMClient.ContainerRequest buildContainerRequest(
- Resource resource, RoleStatus role, long time, String labelExpression) {
+ Resource resource, RoleStatus role, long time) {
Preconditions.checkArgument(resource != null, "null `resource` arg");
Preconditions.checkArgument(role != null, "null `role` arg");
// cache label for escalation
- label = labelExpression;
+ label = role.getLabelExpression();
requestedTimeMillis = time;
escalationTimeoutMillis = time + role.getPlacementTimeoutSeconds() * 1000;
String[] hosts;
@@ -195,7 +219,23 @@
NodeInstance target = this.node;
String nodeLabels;
- if (target != null) {
+ if (isAntiAffine()) {
+ int size = nodes.size();
+ log.info("Creating anti-affine request across {} nodes; first node = {}",
+ size, hostname);
+ hosts = new String[size];
+ StringBuilder builder = new StringBuilder(size * 16);
+ int c = 0;
+ for (NodeInstance nodeInstance : nodes) {
+ hosts[c++] = nodeInstance.hostname;
+ builder.append(nodeInstance.hostname).append(" ");
+ }
+ log.debug("Full host list: [ {}]", builder);
+ escalated = false;
+ mayEscalate = false;
+ relaxLocality = false;
+ nodeLabels = null;
+ } else if (target != null) {
// placed request. Hostname is used in request
hosts = new String[1];
hosts[0] = target.hostname;
@@ -217,7 +257,7 @@
escalated = true;
// and forbid it happening
mayEscalate = false;
- nodeLabels = labelExpression;
+ nodeLabels = label;
}
Priority pri = ContainerPriority.createPriority(roleId, !relaxLocality);
priority = pri.getPriority();
@@ -227,7 +267,6 @@
pri,
relaxLocality,
nodeLabels);
-
validate();
return issuedRequest;
}
@@ -245,23 +284,26 @@
escalated = true;
// this is now the priority
- Priority pri = ContainerPriority.createPriority(roleId, true);
+ // it is tagged as unlocated because it needs to go into a different
+ // set of outstanding requests from the strict placements
+ Priority pri = ContainerPriority.createPriority(roleId, false);
+ // update the field
+ priority = pri.getPriority();
+
String[] nodes;
List<String> issuedRequestNodes = issuedRequest.getNodes();
- if (label == null && issuedRequestNodes != null) {
+ if (SliderUtils.isUnset(label) && issuedRequestNodes != null) {
nodes = issuedRequestNodes.toArray(new String[issuedRequestNodes.size()]);
} else {
nodes = null;
}
- AMRMClient.ContainerRequest newRequest =
- new AMRMClient.ContainerRequest(issuedRequest.getCapability(),
- nodes,
- null,
- pri,
- true,
- label);
- issuedRequest = newRequest;
+ issuedRequest = new AMRMClient.ContainerRequest(issuedRequest.getCapability(),
+ nodes,
+ null,
+ pri,
+ true,
+ label);
validate();
return issuedRequest;
}
@@ -298,55 +340,22 @@
return issuedRequest != null && issuedRequest.getCapability().equals(resource);
}
- /**
- * Equality is on hostname and role
- * @param o other
- * @return true on a match
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- OutstandingRequest request = (OutstandingRequest) o;
-
- if (roleId != request.roleId) {
- return false;
- }
- if (hostname != null
- ? !hostname.equals(request.hostname)
- : request.hostname != null) {
- return false;
- }
- return true;
- }
-
- /**
- * hash on hostname and role
- * @return hash code
- */
- @Override
- public int hashCode() {
- int result = roleId;
- result = 31 * result + (hostname != null ? hostname.hashCode() : 0);
- return result;
- }
-
@Override
public String toString() {
+ boolean requestHasLocation = ContainerPriority.hasLocation(getPriority());
final StringBuilder sb = new StringBuilder("OutstandingRequest{");
sb.append("roleId=").append(roleId);
+ if (hostname != null) {
+ sb.append(", hostname='").append(hostname).append('\'');
+ }
sb.append(", node=").append(node);
- sb.append(", hostname='").append(hostname).append('\'');
- sb.append(", issuedRequest=").append(issuedRequest);
+ sb.append(", hasLocation=").append(requestHasLocation);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
sb.append(", mayEscalate=").append(mayEscalate);
sb.append(", escalated=").append(escalated);
sb.append(", escalationTimeoutMillis=").append(escalationTimeoutMillis);
+ sb.append(", issuedRequest=").append(
+ issuedRequest != null ? SliderUtils.requestToString(issuedRequest) : "(null)");
sb.append('}');
return sb.toString();
}
@@ -360,14 +369,13 @@
return new CancelSingleRequest(issuedRequest);
}
-
/**
* Valid if a node label expression specified on container request is valid or
* not. Mimics the logic in AMRMClientImpl, so can be used for preflight checking
* and in mock tests
*
*/
- public void validate() throws InvalidContainerRequestException {
+ public void validate() throws InvalidContainerRequestException {
Preconditions.checkNotNull(issuedRequest, "request has not yet been built up");
AMRMClient.ContainerRequest containerRequest = issuedRequest;
String exp = containerRequest.getNodeLabelExpression();
@@ -380,18 +388,35 @@
if (exp.contains("&&") || exp.contains("||")) {
throw new InvalidContainerRequestException(
"Cannot specify more than two node labels"
- + " in a single node label expression");
+ + " in a single node label expression: " + this);
}
- // Don't allow specify node label against ANY request
+ // Don't allow specify node label against ANY request listing hosts or racks
if ((containerRequest.getRacks() != null &&
(!containerRequest.getRacks().isEmpty()))
||
(containerRequest.getNodes() != null &&
(!containerRequest.getNodes().isEmpty()))) {
throw new InvalidContainerRequestException(
- "Cannot specify node label with rack and node");
+ "Cannot specify node label with rack and node: " + this);
}
+
+ // relax priority
+ boolean hasLocation = ContainerPriority.hasLocation(priority);
+ if (containerRequest.getRelaxLocality() != !hasLocation) {
+ throw new InvalidContainerRequestException(
+ "relax location flag doesn't match container priority: "
+ + this);
+
+ }
+ }
+
+ /**
+ * Create a new role/hostname pair for indexing.
+ * @return a new index.
+ */
+ public RoleHostnamePair getIndex() {
+ return new RoleHostnamePair(roleId, hostname);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
index 97d321c..64698f2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequestTracker.java
@@ -35,10 +35,12 @@
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
+import java.util.Set;
/**
* Tracks outstanding requests made with a specific placement option.
@@ -62,8 +64,7 @@
*/
private final List<AbstractRMOperation> NO_REQUESTS = new ArrayList<>(0);
- private Map<OutstandingRequest, OutstandingRequest> placedRequests =
- new HashMap<>();
+ private Map<RoleHostnamePair, OutstandingRequest> placedRequests = new HashMap<>();
/**
* List of open requests; no specific details on them.
@@ -82,10 +83,9 @@
* @return a new request
*/
public synchronized OutstandingRequest newRequest(NodeInstance instance, int role) {
- OutstandingRequest request =
- new OutstandingRequest(role, instance);
+ OutstandingRequest request = new OutstandingRequest(role, instance);
if (request.isLocated()) {
- placedRequests.put(request, request);
+ placedRequests.put(request.getIndex(), request);
} else {
openRequests.add(request);
}
@@ -93,6 +93,31 @@
}
/**
+ * Create a new Anti-affine request for the specific role
+ * <p>
+ * It is added to {@link #openRequests}
+ * <p>
+ * This does not update the node instance's role's request count
+ * @param role role index
+ * @param nodes list of suitable nodes
+ * @param label label to use
+ * @return a new request
+ */
+ public synchronized OutstandingRequest newAARequest(int role,
+ List<NodeInstance> nodes,
+ String label) {
+ Preconditions.checkArgument(!nodes.isEmpty());
+ // safety check to verify the allocation will hold
+ for (NodeInstance node : nodes) {
+ Preconditions.checkState(node.canHost(role, label),
+ "Cannot allocate role ID %d to node %s", role, node);
+ }
+ OutstandingRequest request = new OutstandingRequest(role, nodes);
+ openRequests.add(request);
+ return request;
+ }
+
+ /**
* Look up any oustanding request to a (role, hostname).
* @param role role index
* @param hostname hostname
@@ -101,7 +126,7 @@
@VisibleForTesting
public synchronized OutstandingRequest lookupPlacedRequest(int role, String hostname) {
Preconditions.checkArgument(hostname != null, "null hostname");
- return placedRequests.get(new OutstandingRequest(role, hostname));
+ return placedRequests.get(new RoleHostnamePair(role, hostname));
}
/**
@@ -115,25 +140,30 @@
}
/**
- * Notification that a container has been allocated -drop it
- * from the {@link #placedRequests} structure.
+ * Notification that a container has been allocated
+ *
+ * <ol>
+ * <li>drop it from the {@link #placedRequests} structure.</li>
+ * <li>generate the cancellation request</li>
+ * <li>for AA placement, any actions needed</li>
+ * </ol>
+ *
* @param role role index
* @param hostname hostname
* @return the allocation outcome
*/
- public synchronized ContainerAllocation onContainerAllocated(int role,
+ public synchronized ContainerAllocationResults onContainerAllocated(int role,
String hostname,
Container container) {
final String containerDetails = SliderUtils.containerToString(container);
log.debug("Processing allocation for role {} on {}", role,
containerDetails);
- ContainerAllocation allocation = new ContainerAllocation();
+ ContainerAllocationResults allocation = new ContainerAllocationResults();
ContainerAllocationOutcome outcome;
- OutstandingRequest request =
- placedRequests.remove(new OutstandingRequest(role, hostname));
+ OutstandingRequest request = placedRequests.remove(new OutstandingRequest(role, hostname));
if (request != null) {
//satisfied request
- log.debug("Found placed request for container: {}", request);
+ log.debug("Found oustanding placed request for container: {}", request);
request.completed();
// derive outcome from status of tracked request
outcome = request.isEscalated()
@@ -144,16 +174,25 @@
// scan through all containers in the open request list
request = removeOpenRequest(container);
if (request != null) {
- log.debug("Found open request for container: {}", request);
+ log.debug("Found open outstanding request for container: {}", request);
request.completed();
outcome = ContainerAllocationOutcome.Open;
} else {
- log.warn("No open request found for container {}, outstanding queue has {} entries ",
+ log.warn("No oustanding request found for container {}, outstanding queue has {} entries ",
containerDetails,
openRequests.size());
outcome = ContainerAllocationOutcome.Unallocated;
}
}
+ if (request != null && request.getIssuedRequest() != null) {
+ allocation.operations.add(request.createCancelOperation());
+ } else {
+ // there's a request, but no idea what to cancel.
+ // rather than try to recover from it inelegantly, (and cause more confusion),
+ // log the event, but otherwise continue
+ log.warn("Unexpected allocation of container " + SliderUtils.containerToString(container));
+ }
+
allocation.origin = request;
allocation.outcome = outcome;
return allocation;
@@ -280,12 +319,12 @@
*/
public synchronized List<NodeInstance> resetOutstandingRequests(int role) {
List<NodeInstance> hosts = new ArrayList<>();
- Iterator<Map.Entry<OutstandingRequest,OutstandingRequest>> iterator =
+ Iterator<Map.Entry<RoleHostnamePair, OutstandingRequest>> iterator =
placedRequests.entrySet().iterator();
while (iterator.hasNext()) {
- Map.Entry<OutstandingRequest, OutstandingRequest> next =
+ Map.Entry<RoleHostnamePair, OutstandingRequest> next =
iterator.next();
- OutstandingRequest request = next.getKey();
+ OutstandingRequest request = next.getValue();
if (request.roleId == role) {
iterator.remove();
request.completed();
@@ -340,8 +379,7 @@
// time to escalate
CancelSingleRequest cancel = outstandingRequest.createCancelOperation();
operations.add(cancel);
- AMRMClient.ContainerRequest escalated =
- outstandingRequest.escalate();
+ AMRMClient.ContainerRequest escalated = outstandingRequest.escalate();
operations.add(new ContainerRequestOperation(escalated));
}
}
@@ -351,6 +389,47 @@
}
/**
+ * Cancel all outstanding AA requests from the lists of requests.
+ *
+ * This does not remove them from the role status; they must be reset
+ * by the caller.
+ *
+ */
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
+ public synchronized List<AbstractRMOperation> cancelOutstandingAARequests() {
+
+ log.debug("Looking for AA request to cancel");
+ List<AbstractRMOperation> operations = new ArrayList<>();
+
+ // first, all placed requests
+ for (Map.Entry<RoleHostnamePair, OutstandingRequest> entry : placedRequests.entrySet()) {
+ OutstandingRequest outstandingRequest = entry.getValue();
+ synchronized (outstandingRequest) {
+ if (outstandingRequest.isAntiAffine()) {
+ // time to escalate
+ operations.add(outstandingRequest.createCancelOperation());
+ placedRequests.remove(entry.getKey());
+ }
+ }
+ }
+ // second, all open requests
+ ListIterator<OutstandingRequest> orit = openRequests.listIterator();
+ while (orit.hasNext()) {
+ OutstandingRequest outstandingRequest = orit.next();
+ synchronized (outstandingRequest) {
+ if (outstandingRequest.isAntiAffine()) {
+ // time to escalate
+ operations.add(outstandingRequest.createCancelOperation());
+ orit.remove();
+ }
+ }
+ }
+ log.info("Cancelling {} outstanding AA requests", operations.size());
+
+ return operations;
+ }
+
+ /**
* Extract a specific number of open requests for a role
* @param roleId role Id
* @param count count to extract
@@ -369,6 +448,7 @@
}
return results;
}
+
/**
* Extract a specific number of placed requests for a role
* @param roleId role Id
@@ -377,9 +457,10 @@
*/
public synchronized List<OutstandingRequest> extractPlacedRequestsForRole(int roleId, int count) {
List<OutstandingRequest> results = new ArrayList<>();
- Iterator<OutstandingRequest> iterator = placedRequests.keySet().iterator();
+ Iterator<Map.Entry<RoleHostnamePair, OutstandingRequest>>
+ iterator = placedRequests.entrySet().iterator();
while (iterator.hasNext() && count > 0) {
- OutstandingRequest request = iterator.next();
+ OutstandingRequest request = iterator.next().getValue();
if (request.roleId == roleId) {
results.add(request);
count--;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 0508579..c409114 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -25,6 +25,8 @@
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -118,13 +120,13 @@
@Override
- public Map<ContainerId, RoleInstance> getFailedNodes() {
- return appState.getFailedNodes();
+ public Map<ContainerId, RoleInstance> getFailedContainers() {
+ return appState.getFailedContainers();
}
@Override
- public Map<ContainerId, RoleInstance> getLiveNodes() {
- return appState.getLiveNodes();
+ public Map<ContainerId, RoleInstance> getLiveContainers() {
+ return appState.getLiveContainers();
}
@Override
@@ -249,10 +251,10 @@
}
@Override
- public List<RoleInstance> enumLiveNodesInRole(String role) {
- List<RoleInstance> nodes = new ArrayList<RoleInstance>();
+ public List<RoleInstance> enumLiveInstancesInRole(String role) {
+ List<RoleInstance> nodes = new ArrayList<>();
Collection<RoleInstance> allRoleInstances = cloneLiveContainerInfoList();
- getLiveNodes().values();
+ getLiveContainers().values();
for (RoleInstance node : allRoleInstances) {
if (role.isEmpty() || role.equals(node.role)) {
nodes.add(node);
@@ -265,8 +267,7 @@
public List<RoleInstance> lookupRoleContainers(String component) {
RoleStatus roleStatus = lookupRoleStatus(component);
List<RoleInstance> ownedContainerList = cloneOwnedContainerList();
- List<RoleInstance> matching =
- new ArrayList<>(ownedContainerList.size());
+ List<RoleInstance> matching = new ArrayList<>(ownedContainerList.size());
int roleId = roleStatus.getPriority();
for (RoleInstance instance : ownedContainerList) {
if (instance.roleId == roleId) {
@@ -286,7 +287,22 @@
info.containers.add(container.id);
}
return info;
-
}
+ @Override
+ public Map<String, NodeInformation> getNodeInformationSnapshot() {
+ return appState.getRoleHistory()
+ .getNodeInformationSnapshot(appState.buildNamingMap());
+ }
+
+ @Override
+ public NodeInformation getNodeInformation(String hostname) {
+ return appState.getRoleHistory()
+ .getNodeInformation(hostname, appState.buildNamingMap());
+ }
+
+ @Override
+ public RoleStatistics getRoleStatistics() {
+ return appState.getRoleStatistics();
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 926d440..0584d30 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -19,15 +19,20 @@
package org.apache.slider.server.appmaster.state;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.slider.api.types.NodeInformation;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.BoolMetric;
+import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.management.Timestamp;
import org.apache.slider.server.appmaster.operations.AbstractRMOperation;
import org.apache.slider.server.avro.LoadedRoleHistory;
import org.apache.slider.server.avro.NodeEntryRecord;
@@ -41,11 +46,10 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
/**
* The Role History.
@@ -65,106 +69,110 @@
protected static final Logger log =
LoggerFactory.getLogger(RoleHistory.class);
private final List<ProviderRole> providerRoles;
+ /** the roles in here are shared with App State */
+ private final Map<Integer, RoleStatus> roleStatusMap = new HashMap<>();
+ private final AbstractClusterServices recordFactory;
+
private long startTime;
- /**
- * Time when saved
- */
- private long saveTime;
- /**
- * If the history was loaded, the time at which the history was saved
- *
- */
- private long thawedDataTime;
+
+ /** Time when saved */
+ private final Timestamp saveTime = new Timestamp(0);
+
+ /** If the history was loaded, the time at which the history was saved.
+ * That is: the time the data was valid */
+ private final Timestamp thawedDataTime = new Timestamp(0);
private NodeMap nodemap;
private int roleSize;
- private boolean dirty;
+ private final BoolMetric dirty = new BoolMetric(false);
private FileSystem filesystem;
private Path historyPath;
private RoleHistoryWriter historyWriter = new RoleHistoryWriter();
+ /**
+ * When were the nodes updated in a {@link #onNodesUpdated(List)} call?
+ * If zero: never.
+ */
+ private final Timestamp nodesUpdatedTime = new Timestamp(0);
+ private final BoolMetric nodeUpdateReceived = new BoolMetric(false);
+
private OutstandingRequestTracker outstandingRequests =
new OutstandingRequestTracker();
/**
* For each role, lists nodes that are available for data-local allocation,
- ordered by more recently released - To accelerate node selection
+ * ordered by more recently released - to accelerate node selection.
+ * That is, they are "recently used nodes"
*/
- private Map<Integer, LinkedList<NodeInstance>> availableNodes;
+ private Map<Integer, LinkedList<NodeInstance>> recentNodes;
/**
- * Track the failed nodes. Currently used to make wiser decision of container
- * ask with/without locality. Has other potential uses as well.
+ * Instantiate
+ * @param roles initial role list
+ * @param recordFactory yarn record factory
+ * @throws BadConfigException
*/
- private Set<String> failedNodes = new HashSet<>();
-
-
- public RoleHistory(List<ProviderRole> providerRoles) throws
- BadConfigException {
- this.providerRoles = providerRoles;
- roleSize = providerRoles.size();
+ public RoleHistory(Collection<RoleStatus> roles, AbstractClusterServices recordFactory) throws BadConfigException {
+ this.recordFactory = recordFactory;
+ roleSize = roles.size();
+ providerRoles = new ArrayList<>(roleSize);
+ for (RoleStatus role : roles) {
+ addNewRole(role);
+ }
reset();
}
/**
* Reset the variables -this does not adjust the fixed attributes
- * of the history
+ * of the history, but the nodemap and failed node map are cleared.
*/
protected synchronized void reset() throws BadConfigException {
nodemap = new NodeMap(roleSize);
resetAvailableNodeLists();
-
outstandingRequests = new OutstandingRequestTracker();
-
- Map<Integer, RoleStatus> roleStats = new HashMap<>();
- for (ProviderRole providerRole : providerRoles) {
- checkProviderRole(roleStats, providerRole);
- }
}
/**
- * safety check: make sure the provider role is unique amongst
+ * Register all metrics with the metrics infra
+ * @param metrics metrics
+ */
+ public void register(MetricsAndMonitoring metrics) {
+ metrics.register(RoleHistory.class, dirty, "dirty");
+ metrics.register(RoleHistory.class, nodesUpdatedTime, "nodes-updated.time");
+ metrics.register(RoleHistory.class, nodeUpdateReceived, "nodes-updated.flag");
+ metrics.register(RoleHistory.class, thawedDataTime, "thawed.time");
+ metrics.register(RoleHistory.class, saveTime, "saved.time");
+ }
+
+ /**
+ * safety check: make sure the role is unique amongst
* the role stats...which is extended with the new role
- * @param roleStats role stats
- * @param providerRole role
+ * @param roleStatus role
* @throws ArrayIndexOutOfBoundsException
* @throws BadConfigException
*/
- protected void checkProviderRole(Map<Integer, RoleStatus> roleStats,
- ProviderRole providerRole)
- throws BadConfigException {
- int index = providerRole.id;
+ protected void putRole(RoleStatus roleStatus) throws BadConfigException {
+ int index = roleStatus.getKey();
if (index < 0) {
- throw new BadConfigException("Provider " + providerRole
- + " id is out of range");
+ throw new BadConfigException("Provider " + roleStatus + " id is out of range");
}
- if (roleStats.get(index) != null) {
+ if (roleStatusMap.get(index) != null) {
throw new BadConfigException(
- providerRole.toString() + " id duplicates that of " +
- roleStats.get(index));
+ roleStatus.toString() + " id duplicates that of " +
+ roleStatusMap.get(index));
}
- roleStats.put(index, new RoleStatus(providerRole));
+ roleStatusMap.put(index, roleStatus);
}
-
/**
- * Add a new provider role to the map
- * @param providerRole new provider role
+ * Add a new role
+ * @param roleStatus new role
*/
- public void addNewProviderRole(ProviderRole providerRole)
- throws BadConfigException {
- log.debug("Validating/adding new provider role to role history: {} ",
- providerRole);
- Map<Integer, RoleStatus> roleStats = new HashMap<>();
-
- for (ProviderRole role : providerRoles) {
- roleStats.put(role.id, new RoleStatus(role));
- }
-
- checkProviderRole(roleStats, providerRole);
- log.debug("Check successful; adding role");
- this.providerRoles.add(providerRole);
+ public void addNewRole(RoleStatus roleStatus) throws BadConfigException {
+ log.debug("Validating/adding new role to role history: {} ", roleStatus);
+ putRole(roleStatus);
+ this.providerRoles.add(roleStatus.getProviderRole());
}
/**
@@ -185,12 +193,11 @@
* Clear the lists of available nodes
*/
private synchronized void resetAvailableNodeLists() {
- availableNodes = new HashMap<>(roleSize);
+ recentNodes = new ConcurrentHashMap<>(roleSize);
}
/**
- * Reset the variables -this does not adjust the fixed attributes
- * of the history.
+ * Prepare the history for re-reading its state.
* <p>
* This intended for use by the RoleWriter logic.
* @throws BadConfigException if there is a problem rebuilding the state
@@ -245,21 +252,20 @@
return discarded;
}
-
public synchronized long getStartTime() {
return startTime;
}
public synchronized long getSaveTime() {
- return saveTime;
+ return saveTime.get();
}
public long getThawedDataTime() {
- return thawedDataTime;
+ return thawedDataTime.get();
}
public void setThawedDataTime(long thawedDataTime) {
- this.thawedDataTime = thawedDataTime;
+ this.thawedDataTime.set(thawedDataTime);
}
public synchronized int getRoleSize() {
@@ -275,11 +281,11 @@
}
public synchronized boolean isDirty() {
- return dirty;
+ return dirty.get();
}
public synchronized void setDirty(boolean dirty) {
- this.dirty = dirty;
+ this.dirty.set(dirty);
}
/**
@@ -287,8 +293,8 @@
* @param timestamp timestamp -updates the savetime field
*/
public synchronized void saved(long timestamp) {
- dirty = false;
- saveTime = timestamp;
+ setDirty(false);
+ saveTime.set(timestamp);
}
/**
@@ -301,6 +307,34 @@
}
/**
+ * Get snapshot of the node map
+ * @return a snapshot of the current node state
+ * @param naming naming map of priority to enty name; entries must be unique.
+ * It's OK to be incomplete, for those the list falls back to numbers.
+ */
+ public synchronized Map<String, NodeInformation> getNodeInformationSnapshot(
+ Map<Integer, String> naming) {
+ Map<String, NodeInformation> result = new HashMap<>(nodemap.size());
+ for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
+ result.put(entry.getKey(), entry.getValue().serialize(naming));
+ }
+ return result;
+ }
+
+ /**
+ * Get the information on a node
+ * @param hostname hostname
+ * @param naming naming map of priority to enty name; entries must be unique.
+ * It's OK to be incomplete, for those the list falls back to numbers.
+ * @return the information about that host, or null if there is none
+ */
+ public NodeInformation getNodeInformation(String hostname,
+ Map<Integer, String> naming) {
+ NodeInstance nodeInstance = nodemap.get(hostname);
+ return nodeInstance != null ? nodeInstance.serialize(naming) : null;
+ }
+
+ /**
* Get the node instance for the specific node -creating it if needed
* @param hostname node address
* @return the instance
@@ -321,7 +355,7 @@
public synchronized void insert(Collection<NodeInstance> nodes) {
nodemap.insert(nodes);
}
-
+
/**
* Get current time. overrideable for test subclasses
* @return current time in millis
@@ -331,18 +365,6 @@
}
/**
- * Garbage collect the structure -this will drop
- * all nodes that have been inactive since the (relative) age.
- * This will drop the failure counts of the nodes too, so it will
- * lose information that matters.
- * @param age relative age
- */
- public void gc(long age) {
- long absoluteTime = now() - age;
- purgeUnusedEntries(absoluteTime);
- }
-
- /**
* Mark ourselves as dirty
*/
public void touch() {
@@ -355,15 +377,6 @@
}
/**
- * purge the history of
- * all nodes that have been inactive since the absolute time
- * @param absoluteTime time
- */
- public synchronized void purgeUnusedEntries(long absoluteTime) {
- nodemap.purgeUnusedEntries(absoluteTime);
- }
-
- /**
* reset the failed recently counters
*/
public synchronized void resetFailedRecently() {
@@ -402,8 +415,7 @@
*/
public synchronized Path saveHistoryIfDirty() throws IOException {
if (isDirty()) {
- long time = now();
- return saveHistory(time);
+ return saveHistory(now());
} else {
return null;
}
@@ -415,8 +427,7 @@
* @param historyDir path in FS for history
* @return true if the history was thawed
*/
- public boolean onStart(FileSystem fs, Path historyDir) throws
- BadConfigException {
+ public boolean onStart(FileSystem fs, Path historyDir) throws BadConfigException {
assert filesystem == null;
filesystem = fs;
historyPath = historyDir;
@@ -426,7 +437,7 @@
}
/**
- * Handler for bootstrap event
+ * Handler for bootstrap event: there was no history to thaw
*/
public void onBootstrap() {
log.debug("Role history bootstrapped");
@@ -451,7 +462,7 @@
if (loadedRoleHistory != null) {
rebuild(loadedRoleHistory);
thawSuccessful = true;
- Path loadPath = loadedRoleHistory.getPath();;
+ Path loadPath = loadedRoleHistory.getPath();
log.debug("loaded history from {}", loadPath);
// delete any old entries
try {
@@ -463,7 +474,7 @@
}
//start is then completed
- buildAvailableNodeLists();
+ buildRecentNodeLists();
} else {
//fallback to bootstrap procedure
onBootstrap();
@@ -476,7 +487,7 @@
* (After the start), rebuild the availability data structures
*/
@VisibleForTesting
- public synchronized void buildAvailableNodeLists() {
+ public synchronized void buildRecentNodeLists() {
resetAvailableNodeLists();
// build the list of available nodes
for (Map.Entry<String, NodeInstance> entry : nodemap.entrySet()) {
@@ -485,13 +496,13 @@
NodeEntry nodeEntry = ni.get(i);
if (nodeEntry != null && nodeEntry.isAvailable()) {
log.debug("Adding {} for role {}", ni, i);
- getOrCreateNodesForRoleId(i).add(ni);
+ listRecentNodesForRoleId(i).add(ni);
}
}
}
// sort the resulting arrays
for (int i = 0; i < roleSize; i++) {
- sortAvailableNodeList(i);
+ sortRecentNodeList(i);
}
}
@@ -501,30 +512,35 @@
* @return potentially null list
*/
@VisibleForTesting
- public List<NodeInstance> getNodesForRoleId(int id) {
- return availableNodes.get(id);
+ public List<NodeInstance> getRecentNodesForRoleId(int id) {
+ return recentNodes.get(id);
}
-
+
/**
- * Get the nodes for an ID -may be null
+ * Get a possibly empty list of suggested nodes for a role.
* @param id role ID
* @return list
*/
- private LinkedList<NodeInstance> getOrCreateNodesForRoleId(int id) {
- LinkedList<NodeInstance> instances = availableNodes.get(id);
+ private LinkedList<NodeInstance> listRecentNodesForRoleId(int id) {
+ LinkedList<NodeInstance> instances = recentNodes.get(id);
if (instances == null) {
- instances = new LinkedList<>();
- availableNodes.put(id, instances);
+ synchronized (this) {
+ // recheck in the synchronized block and recreate
+ if (recentNodes.get(id) == null) {
+ recentNodes.put(id, new LinkedList<NodeInstance>());
+ }
+ instances = recentNodes.get(id);
+ }
}
return instances;
}
-
+
/**
- * Sort an available node list
+ * Sort a the recent node list for a single role
* @param role role to sort
*/
- private void sortAvailableNodeList(int role) {
- List<NodeInstance> nodesForRoleId = getNodesForRoleId(role);
+ private void sortRecentNodeList(int role) {
+ List<NodeInstance> nodesForRoleId = getRecentNodesForRoleId(role);
if (nodesForRoleId != null) {
Collections.sort(nodesForRoleId, new NodeInstance.Preferred(role));
}
@@ -536,7 +552,7 @@
* @return the instance, or null for none
*/
@VisibleForTesting
- public synchronized NodeInstance findNodeForNewInstance(RoleStatus role) {
+ public synchronized NodeInstance findRecentNodeForNewInstance(RoleStatus role) {
if (!role.isPlacementDesired()) {
// no data locality policy
return null;
@@ -546,7 +562,7 @@
NodeInstance nodeInstance = null;
// Get the list of possible targets.
// This is a live list: changes here are preserved
- List<NodeInstance> targets = getNodesForRoleId(roleId);
+ List<NodeInstance> targets = getRecentNodesForRoleId(roleId);
if (targets == null) {
// nothing to allocate on
return null;
@@ -558,7 +574,8 @@
NodeInstance candidate = targets.get(i);
if (candidate.getActiveRoleInstances(roleId) == 0) {
// no active instances: check failure statistics
- if (strictPlacement || !candidate.exceedsFailureThreshold(role)) {
+ if (strictPlacement
+ || (candidate.isOnline() && !candidate.exceedsFailureThreshold(role))) {
targets.remove(i);
// exit criteria for loop is now met
nodeInstance = candidate;
@@ -577,6 +594,18 @@
}
/**
+ * Find a node for use
+ * @param role role
+ * @return the instance, or null for none
+ */
+ @VisibleForTesting
+ public synchronized List<NodeInstance> findNodeForNewAAInstance(RoleStatus role) {
+ // all nodes that are live and can host the role; no attempt to exclude ones
+ // considered failing
+ return nodemap.findAllNodesForRole(role.getKey(), role.getLabelExpression());
+ }
+
+ /**
* Request an instance on a given node.
* An outstanding request is created & tracked, with the
* relevant node entry for that role updated.
@@ -586,56 +615,66 @@
* Returns the request that is now being tracked.
* If the node instance is not null, it's details about the role is incremented
*
- *
* @param node node to target or null for "any"
* @param role role to request
- * @param labelExpression label to satisfy
- * @return the container priority
+ * @return the request
*/
- public synchronized AMRMClient.ContainerRequest requestInstanceOnNode(
- NodeInstance node, RoleStatus role, Resource resource, String labelExpression) {
+ public synchronized OutstandingRequest requestInstanceOnNode(
+ NodeInstance node, RoleStatus role, Resource resource) {
OutstandingRequest outstanding = outstandingRequests.newRequest(node, role.getKey());
- return outstanding.buildContainerRequest(resource, role, now(), labelExpression);
- }
-
- /**
- * Find a node for a role and request an instance on that (or a location-less
- * instance) with a label expression
- * @param role role status
- * @param resource resource capabilities
- * @param labelExpression label to satisfy
- * @return a request ready to go
- */
- public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
- Resource resource,
- String labelExpression) {
- NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource, labelExpression);
+ outstanding.buildContainerRequest(resource, role, now());
+ return outstanding;
}
/**
* Find a node for a role and request an instance on that (or a location-less
* instance)
* @param role role status
- * @param resource resource capabilities
- * @return a request ready to go
+ * @return a request ready to go, or null if this is an AA request and no
+ * location can be found.
*/
- public synchronized AMRMClient.ContainerRequest requestNode(RoleStatus role,
- Resource resource) {
- NodeInstance node = findNodeForNewInstance(role);
- return requestInstanceOnNode(node, role, resource, null);
+ public synchronized OutstandingRequest requestContainerForRole(RoleStatus role) {
+
+ if (role.isAntiAffinePlacement()) {
+ return requestContainerForAARole(role);
+ } else {
+ Resource resource = recordFactory.newResource();
+ role.copyResourceRequirements(resource);
+ NodeInstance node = findRecentNodeForNewInstance(role);
+ return requestInstanceOnNode(node, role, resource);
+ }
}
/**
- * Get the list of active nodes ... walks the node map so
- * is O(nodes)
+ * Find a node for an AA role and request an instance on that (or a location-less
+ * instance)
+ * @param role role status
+ * @return a request ready to go, or null if no location can be found.
+ */
+ public synchronized OutstandingRequest requestContainerForAARole(RoleStatus role) {
+ List<NodeInstance> nodes = findNodeForNewAAInstance(role);
+ if (!nodes.isEmpty()) {
+ OutstandingRequest outstanding = outstandingRequests.newAARequest(
+ role.getKey(), nodes, role.getLabelExpression());
+ Resource resource = recordFactory.newResource();
+ role.copyResourceRequirements(resource);
+ outstanding.buildContainerRequest(resource, role, now());
+ return outstanding;
+ } else {
+ log.warn("No suitable location for {}", role.getName());
+ return null;
+ }
+ }
+ /**
+ * Get the list of active nodes ... walks the node map so
+ * is {@code O(nodes)}
* @param role role index
* @return a possibly empty list of nodes with an instance of that node
*/
public synchronized List<NodeInstance> listActiveNodes(int role) {
return nodemap.listActiveNodes(role);
}
-
+
/**
* Get the node entry of a container
* @param container container to look up
@@ -643,8 +682,7 @@
* @throws RuntimeException if the container has no hostname
*/
public NodeEntry getOrCreateNodeEntry(Container container) {
- NodeInstance node = getOrCreateNodeInstance(container);
- return node.getOrCreate(ContainerPriority.extractRole(container));
+ return getOrCreateNodeInstance(container).getOrCreate(container);
}
/**
@@ -654,8 +692,7 @@
* @throws RuntimeException if the container has no hostname
*/
public synchronized NodeInstance getOrCreateNodeInstance(Container container) {
- String hostname = RoleHistoryUtils.hostnameOf(container);
- return nodemap.getOrCreate(hostname);
+ return nodemap.getOrCreate(RoleHistoryUtils.hostnameOf(container));
}
/**
@@ -686,7 +723,7 @@
* @return list of containers potentially reordered
*/
public synchronized List<Container> prepareAllocationList(List<Container> allocatedContainers) {
-
+
//partition into requested and unrequested
List<Container> requested =
new ArrayList<>(allocatedContainers.size());
@@ -698,7 +735,7 @@
requested.addAll(unrequested);
return requested;
}
-
+
/**
* A container has been allocated on a node -update the data structures
* @param container container
@@ -706,24 +743,24 @@
* @param actualCount current count of instances
* @return The allocation outcome
*/
- public synchronized ContainerAllocation onContainerAllocated(Container container,
- int desiredCount,
- int actualCount) {
+ public synchronized ContainerAllocationResults onContainerAllocated(Container container,
+ long desiredCount,
+ long actualCount) {
int role = ContainerPriority.extractRole(container);
+
String hostname = RoleHistoryUtils.hostnameOf(container);
- List<NodeInstance> nodeInstances = getOrCreateNodesForRoleId(role);
- ContainerAllocation outcome =
+ List<NodeInstance> nodeInstances = listRecentNodesForRoleId(role);
+ ContainerAllocationResults outcome =
outstandingRequests.onContainerAllocated(role, hostname, container);
if (desiredCount <= actualCount) {
// all outstanding requests have been satisfied
// clear all the lists, so returning nodes to the available set
- List<NodeInstance>
- hosts = outstandingRequests.resetOutstandingRequests(role);
+ List<NodeInstance> hosts = outstandingRequests.resetOutstandingRequests(role);
if (!hosts.isEmpty()) {
//add the list
log.info("Adding {} hosts for role {}", hosts.size(), role);
nodeInstances.addAll(hosts);
- sortAvailableNodeList(role);
+ sortRecentNodeList(role);
}
}
return outcome;
@@ -734,20 +771,20 @@
* @param container container
*/
public void onContainerAssigned(Container container) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
+ NodeInstance node = getOrCreateNodeInstance(container);
+ NodeEntry nodeEntry = node.getOrCreate(container);
nodeEntry.onStarting();
+ log.debug("Node {} has updated NodeEntry {}", node, nodeEntry);
}
/**
- * Event: a container start has been submitter
+ * Event: a container start has been submitted
* @param container container being started
* @param instance instance bound to the container
*/
public void onContainerStartSubmitted(Container container,
RoleInstance instance) {
- NodeEntry nodeEntry = getOrCreateNodeEntry(container);
- int role = ContainerPriority.extractRole(container);
- // any actions we want here
+ // no actions here
}
/**
@@ -771,26 +808,54 @@
}
/**
- * Update failedNodes and nodemap based on the node state
- *
- * @param updatedNodes list of updated nodes
+ * Does the RoleHistory have enough information about the YARN cluster
+ * to start placing AA requests? That is: has it the node map and
+ * any label information needed?
+ * @return true if the caller can start requesting AA nodes
*/
- public synchronized void onNodesUpdated(List<NodeReport> updatedNodes) {
+ public boolean canPlaceAANodes() {
+ return nodeUpdateReceived.get();
+ }
+
+ /**
+ * Get the last time the nodes were updated from YARN
+ * @return the update time or zero if never updated.
+ */
+ public long getNodesUpdatedTime() {
+ return nodesUpdatedTime.get();
+ }
+
+ /**
+ * Update failedNodes and nodemap based on the node state
+ *
+ * @param updatedNodes list of updated nodes
+ * @return true if a review should be triggered.
+ */
+ public synchronized boolean onNodesUpdated(List<NodeReport> updatedNodes) {
+ log.debug("Updating {} nodes", updatedNodes.size());
+ nodesUpdatedTime.set(now());
+ nodeUpdateReceived.set(true);
+ int printed = 0;
+ boolean triggerReview = false;
for (NodeReport updatedNode : updatedNodes) {
- String hostname = updatedNode.getNodeId() == null
- ? null
- : updatedNode.getNodeId().getHost();
- if (hostname == null) {
+ String hostname = updatedNode.getNodeId() == null
+ ? ""
+ : updatedNode.getNodeId().getHost();
+ NodeState nodeState = updatedNode.getNodeState();
+ if (hostname.isEmpty() || nodeState == null) {
+ log.warn("Ignoring incomplete update");
continue;
}
- if (updatedNode.getNodeState() != null
- && updatedNode.getNodeState().isUnusable()) {
- failedNodes.add(hostname);
- nodemap.remove(hostname);
- } else {
- failedNodes.remove(hostname);
+ if (log.isDebugEnabled() && printed++ < 10) {
+ // log the first few, but avoid overloading the logs for a full cluster
+ // update
+ log.debug("Node \"{}\" is in state {}", hostname, nodeState);
}
+ // update the node; this also creates an instance if needed
+ boolean updated = nodemap.updateNode(hostname, updatedNode);
+ triggerReview |= updated;
}
+ return triggerReview;
}
/**
@@ -828,7 +893,7 @@
/**
* Mark a container finished; if it was released then that is treated
- * differently. history is touch()ed
+ * differently. history is {@code touch()}-ed
*
*
* @param container completed container
@@ -858,7 +923,7 @@
/**
* If the node is marked as available; queue it for assignments.
- * Unsynced: expects caller to be in a sync block.
+ * Unsynced: requires caller to be in a sync block.
* @param container completed container
* @param nodeEntry node
* @param available available flag
@@ -873,7 +938,7 @@
NodeInstance ni = getOrCreateNodeInstance(container);
int roleId = ContainerPriority.extractRole(container);
log.debug("Node {} is now available for role id {}", ni, roleId);
- getOrCreateNodesForRoleId(roleId).addFirst(ni);
+ listRecentNodesForRoleId(roleId).addFirst(ni);
}
return available;
}
@@ -884,8 +949,7 @@
public synchronized void dump() {
for (ProviderRole role : providerRoles) {
log.info(role.toString());
- List<NodeInstance> instances =
- getOrCreateNodesForRoleId(role.id);
+ List<NodeInstance> instances = listRecentNodesForRoleId(role.id);
log.info(" available: " + instances.size()
+ " " + SliderUtils.joinWithInnerSeparator(" ", instances));
}
@@ -894,9 +958,6 @@
for (NodeInstance node : nodemap.values()) {
log.info(node.toFullString());
}
-
- log.info("Failed nodes: {}",
- SliderUtils.joinWithInnerSeparator(" ", failedNodes));
}
/**
@@ -917,8 +978,8 @@
* @return a clone of the list
*/
@VisibleForTesting
- public List<NodeInstance> cloneAvailableList(int role) {
- return new LinkedList<>(getOrCreateNodesForRoleId(role));
+ public List<NodeInstance> cloneRecentNodeList(int role) {
+ return new LinkedList<>(listRecentNodesForRoleId(role));
}
/**
@@ -930,7 +991,6 @@
return outstandingRequests.listPlacedRequests();
}
-
/**
* Get a snapshot of the outstanding placed request list
* @return a list of the requests outstanding at the time of requesting
@@ -941,32 +1001,42 @@
}
/**
- * Get a clone of the failedNodes
- *
- * @return the list
- */
- public List<String> cloneFailedNodes() {
- List<String> lst = new ArrayList<>();
- lst.addAll(failedNodes);
- return lst;
- }
-
- /**
* Escalate operation as triggered by external timer.
* @return a (usually empty) list of cancel/request operations.
*/
public List<AbstractRMOperation> escalateOutstandingRequests() {
- long now = now();
- return outstandingRequests.escalateOutstandingRequests(now);
+ return outstandingRequests.escalateOutstandingRequests(now());
+ }
+ /**
+ * Escalate operation as triggered by external timer.
+ * @return a (usually empty) list of cancel/request operations.
+ */
+ public List<AbstractRMOperation> cancelOutstandingAARequests() {
+ return outstandingRequests.cancelOutstandingAARequests();
+ }
+
+ /**
+ * Cancel a number of outstanding requests for a role -that is, not
+ * actual containers, just requests for new ones.
+ * @param role role
+ * @param toCancel number to cancel
+ * @return a list of cancellable operations.
+ */
+ public List<AbstractRMOperation> cancelRequestsForRole(RoleStatus role, int toCancel) {
+ return role.isAntiAffinePlacement() ?
+ cancelRequestsForAARole(role, toCancel)
+ : cancelRequestsForSimpleRole(role, toCancel);
}
/**
* Build the list of requests to cancel from the outstanding list.
- * @param role
- * @param toCancel
+ * @param role role
+ * @param toCancel number to cancel
* @return a list of cancellable operations.
*/
- public synchronized List<AbstractRMOperation> cancelRequestsForRole(RoleStatus role, int toCancel) {
+ private synchronized List<AbstractRMOperation> cancelRequestsForSimpleRole(RoleStatus role, int toCancel) {
+ Preconditions.checkArgument(toCancel > 0,
+ "trying to cancel invalid number of requests: " + toCancel);
List<AbstractRMOperation> results = new ArrayList<>(toCancel);
// first scan through the unplaced request list to find all of a role
int roleId = role.getKey();
@@ -985,4 +1055,47 @@
return results;
}
+ /**
+ * Build the list of requests to cancel for an AA role. This reduces the number
+ * of outstanding pending requests first, then cancels any active request,
+ * before finally asking for any placed containers
+ * @param role role
+ * @param toCancel number to cancel
+ * @return a list of cancellable operations.
+ */
+ private synchronized List<AbstractRMOperation> cancelRequestsForAARole(RoleStatus role, int toCancel) {
+ List<AbstractRMOperation> results = new ArrayList<>(toCancel);
+ int roleId = role.getKey();
+ List<OutstandingRequest> requests = new ArrayList<>(toCancel);
+ // there may be pending requests which can be cancelled here
+ long pending = role.getPendingAntiAffineRequests();
+ if (pending > 0) {
+ // there are some pending ones which can be cancelled first
+ long pendingToCancel = Math.min(pending, toCancel);
+ log.info("Cancelling {} pending AA allocations, leaving {}", toCancel,
+ pendingToCancel);
+ role.setPendingAntiAffineRequests(pending - pendingToCancel);
+ toCancel -= pendingToCancel;
+ }
+ if (toCancel > 0 && role.isAARequestOutstanding()) {
+ // not enough
+ log.info("Cancelling current AA request");
+ // find the single entry which may be running
+ requests = outstandingRequests.extractOpenRequestsForRole(roleId, toCancel);
+ role.cancelOutstandingAARequest();
+ toCancel--;
+ }
+
+ // ask for some excess nodes
+ if (toCancel > 0) {
+ requests.addAll(outstandingRequests.extractPlacedRequestsForRole(roleId, toCancel));
+ }
+
+ // build cancellations
+ for (OutstandingRequest request : requests) {
+ results.add(request.createCancelOperation());
+ }
+ return results;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
new file mode 100644
index 0000000..920887a
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHostnamePair.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.state;
+
+import java.util.Objects;
+
+public class RoleHostnamePair {
+
+ /**
+ * requested role
+ */
+ public final int roleId;
+
+ /**
+ * hostname -will be null if node==null
+ */
+ public final String hostname;
+
+ public RoleHostnamePair(int roleId, String hostname) {
+ this.roleId = roleId;
+ this.hostname = hostname;
+ }
+
+ public int getRoleId() {
+ return roleId;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof RoleHostnamePair)) {
+ return false;
+ }
+ RoleHostnamePair that = (RoleHostnamePair) o;
+ return Objects.equals(roleId, that.roleId) &&
+ Objects.equals(hostname, that.hostname);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(roleId, hostname);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "RoleHostnamePair{");
+ sb.append("roleId=").append(roleId);
+ sb.append(", hostname='").append(hostname).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index e2974fc..656f96c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -18,21 +18,32 @@
package org.apache.slider.server.appmaster.state;
+import com.codahale.metrics.Metric;
+import com.codahale.metrics.MetricSet;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
+import org.apache.slider.server.appmaster.management.BoolMetric;
+import org.apache.slider.server.appmaster.management.BoolMetricPredicate;
+import org.apache.slider.server.appmaster.management.LongGauge;
import java.io.Serializable;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-
/**
- * Models the ongoing status of all nodes in
- * Nothing here is synchronized: grab the whole instance to update.
+ * Models the ongoing status of all nodes in an application.
+ *
+ * These structures are shared across the {@link AppState} and {@link RoleHistory} structures,
+ * and must be designed for synchronous access. Atomic counters are preferred to anything which
+ * requires synchronization. Where synchronized access is good is that it allows for
+ * the whole instance to be locked, for updating multiple entries.
*/
-public final class RoleStatus implements Cloneable {
+public final class RoleStatus implements Cloneable, MetricSet {
private final String name;
@@ -40,16 +51,31 @@
* Role priority
*/
private final int key;
-
private final ProviderRole providerRole;
- private int desired, actual, requested, releasing;
- private int failed, startFailed;
- private int started, completed, totalRequested;
- private final AtomicLong preempted = new AtomicLong(0);
- private final AtomicLong nodeFailed = new AtomicLong(0);
- private final AtomicLong failedRecently = new AtomicLong(0);
- private final AtomicLong limitsExceeded = new AtomicLong(0);
+ private final LongGauge actual = new LongGauge();
+ private final LongGauge completed = new LongGauge();
+ private final LongGauge desired = new LongGauge();
+ private final LongGauge failed = new LongGauge();
+ private final LongGauge failedRecently = new LongGauge(0);
+ private final LongGauge limitsExceeded = new LongGauge(0);
+ private final LongGauge nodeFailed = new LongGauge(0);
+ /** Number of AA requests queued. */
+ private final LongGauge pendingAntiAffineRequests = new LongGauge(0);
+ private final LongGauge preempted = new LongGauge(0);
+ private final LongGauge releasing = new LongGauge();
+ private final LongGauge requested = new LongGauge();
+ private final LongGauge started = new LongGauge();
+ private final LongGauge startFailed = new LongGauge();
+ private final LongGauge totalRequested = new LongGauge();
+
+ /** resource requirements */
+ private Resource resourceRequirements;
+
+
+ /** any pending AA request */
+ private volatile OutstandingRequest outstandingAArequest = null;
+
private String failureMessage = "";
@@ -58,7 +84,37 @@
this.name = providerRole.name;
this.key = providerRole.id;
}
-
+
+ @Override
+ public Map<String, Metric> getMetrics() {
+ Map<String, Metric> metrics = new HashMap<>(15);
+ metrics.put("actual", actual);
+ metrics.put("completed", completed );
+ metrics.put("desired", desired);
+ metrics.put("failed", failed);
+ metrics.put("limitsExceeded", limitsExceeded);
+ metrics.put("nodeFailed", nodeFailed);
+ metrics.put("preempted", preempted);
+ metrics.put("pendingAntiAffineRequests", pendingAntiAffineRequests);
+ metrics.put("releasing", releasing);
+ metrics.put("requested", requested);
+ metrics.put("preempted", preempted);
+ metrics.put("releasing", releasing );
+ metrics.put("requested", requested);
+ metrics.put("started", started);
+ metrics.put("startFailed", startFailed);
+ metrics.put("totalRequested", totalRequested);
+
+ metrics.put("outstandingAArequest",
+ new BoolMetricPredicate(new BoolMetricPredicate.Eval() {
+ @Override
+ public boolean eval() {
+ return isAARequestOutstanding();
+ }
+ }));
+ return metrics;
+ }
+
public String getName() {
return name;
}
@@ -113,63 +169,64 @@
return !hasPlacementPolicy(PlacementPolicy.NO_DATA_LOCALITY);
}
- public synchronized int getDesired() {
- return desired;
+ public long getDesired() {
+ return desired.get();
}
- public synchronized void setDesired(int desired) {
- this.desired = desired;
+ public void setDesired(long desired) {
+ this.desired.set(desired);
}
- public synchronized int getActual() {
- return actual;
+ public long getActual() {
+ return actual.get();
}
- public synchronized int incActual() {
- return ++actual;
+ public long incActual() {
+ return actual.incrementAndGet();
}
- public synchronized int decActual() {
- actual = Math.max(0, actual - 1);
- return actual;
+ public long decActual() {
+ return actual.decToFloor(1);
}
- public synchronized int getRequested() {
- return requested;
+ /**
+ * Get the request count.
+ * @return a count of requested containers
+ */
+ public long getRequested() {
+ return requested.get();
}
- public synchronized int incRequested() {
- totalRequested++;
- return ++requested;
+ public long incRequested() {
+ totalRequested.incrementAndGet();
+ return requested.incrementAndGet();
}
- public synchronized int cancel(int count) {
- requested = Math.max(0, requested - count);
- return requested;
- }
-
- public synchronized int decRequested() {
- return cancel(1);
+ public void cancel(long count) {
+ requested.decToFloor(count);
}
- public synchronized int getReleasing() {
- return releasing;
+ public void decRequested() {
+ cancel(1);
}
- public synchronized int incReleasing() {
- return ++releasing;
+ public long getReleasing() {
+ return releasing.get();
}
- public synchronized int decReleasing() {
- releasing = Math.max(0, releasing - 1);
- return releasing;
+ public long incReleasing() {
+ return releasing.incrementAndGet();
}
- public synchronized int getFailed() {
- return failed;
+ public long decReleasing() {
+ return releasing.decToFloor(1);
}
- public synchronized long getFailedRecently() {
+ public long getFailed() {
+ return failed.get();
+ }
+
+ public long getFailedRecently() {
return failedRecently.get();
}
@@ -185,6 +242,26 @@
return limitsExceeded.get();
}
+ public long incPendingAntiAffineRequests(long v) {
+ return pendingAntiAffineRequests.addAndGet(v);
+ }
+
+ /**
+ * Probe for an outstanding AA request being true
+ * @return true if there is an outstanding AA Request
+ */
+ public boolean isAARequestOutstanding() {
+ return outstandingAArequest != null;
+ }
+
+ /**
+ * expose the predicate {@link #isAARequestOutstanding()} as an integer,
+ * which is very convenient in tests
+ * @return 1 if there is an outstanding request; 0 if not
+ */
+ public int getOutstandingAARequestCount() {
+ return isAARequestOutstanding()? 1: 0;
+ }
/**
* Note that a role failed, text will
* be used in any diagnostics if an exception
@@ -205,7 +282,7 @@
case Node_failure:
nodeFailed.incrementAndGet();
- failed++;
+ failed.incrementAndGet();
break;
case Failed_limits_exceeded: // exceeded memory or CPU; app/configuration related
@@ -213,7 +290,7 @@
// fall through
case Failed: // application failure, possibly node related, possibly not
default: // anything else (future-proofing)
- failed++;
+ failed.incrementAndGet();
failedRecently.incrementAndGet();
//have a look to see if it short lived
if (startupFailure) {
@@ -223,39 +300,39 @@
}
}
- public synchronized int getStartFailed() {
- return startFailed;
+ public long getStartFailed() {
+ return startFailed.get();
}
public synchronized void incStartFailed() {
- startFailed++;
+ startFailed.getAndIncrement();
}
public synchronized String getFailureMessage() {
return failureMessage;
}
- public synchronized int getCompleted() {
- return completed;
+ public long getCompleted() {
+ return completed.get();
}
public synchronized void setCompleted(int completed) {
- this.completed = completed;
+ this.completed.set(completed);
}
- public synchronized int incCompleted() {
- return completed ++;
+ public long incCompleted() {
+ return completed.incrementAndGet();
}
- public synchronized int getStarted() {
- return started;
+ public long getStarted() {
+ return started.get();
}
public synchronized void incStarted() {
- started++;
+ started.incrementAndGet();
}
- public synchronized int getTotalRequested() {
- return totalRequested;
+ public long getTotalRequested() {
+ return totalRequested.get();
}
public long getPreempted() {
@@ -266,19 +343,58 @@
return nodeFailed.get();
}
+ public long getPendingAntiAffineRequests() {
+ return pendingAntiAffineRequests.get();
+ }
+
+ public void setPendingAntiAffineRequests(long pendingAntiAffineRequests) {
+ this.pendingAntiAffineRequests.set(pendingAntiAffineRequests);
+ }
+
+ public long decPendingAntiAffineRequests() {
+ return pendingAntiAffineRequests.decToFloor(1);
+ }
+
+ public OutstandingRequest getOutstandingAArequest() {
+ return outstandingAArequest;
+ }
+
+ public void setOutstandingAArequest(OutstandingRequest outstandingAArequest) {
+ this.outstandingAArequest = outstandingAArequest;
+ }
+
+ /**
+ * Complete the outstanding AA request (there's no check for one in progress, caller
+ * expected to have done that).
+ */
+ public void completeOutstandingAARequest() {
+ setOutstandingAArequest(null);
+ }
+
+ /**
+ * Cancel any outstanding AA request. Harmless if the role is non-AA, or
+ * if there are no outstanding requests.
+ */
+ public void cancelOutstandingAARequest() {
+ if (outstandingAArequest != null) {
+ setOutstandingAArequest(null);
+ setPendingAntiAffineRequests(0);
+ decRequested();
+ }
+ }
+
/**
* Get the number of roles we are short of.
* nodes released are ignored.
* @return the positive or negative number of roles to add/release.
* 0 means "do nothing".
*/
- public synchronized int getDelta() {
- int inuse = getActualAndRequested();
- //don't know how to view these. Are they in-use or not?
- int delta = desired - inuse;
+ public long getDelta() {
+ long inuse = getActualAndRequested();
+ long delta = desired.get() - inuse;
if (delta < 0) {
//if we are releasing, remove the number that are already released.
- delta += releasing;
+ delta += releasing.get();
//but never switch to a positive
delta = Math.min(delta, 0);
}
@@ -286,31 +402,41 @@
}
/**
- * Get count of actual and requested containers
- * @return the size of the application when outstanding requests are included
+ * Get count of actual and requested containers. This includes pending ones
+ * @return the size of the application when outstanding requests are included.
*/
- public synchronized int getActualAndRequested() {
- return actual + requested;
+ public long getActualAndRequested() {
+ return actual.get() + requested.get();
}
@Override
- public synchronized String toString() {
- return "RoleStatus{" +
- "name='" + name + '\'' +
- ", key=" + key +
- ", desired=" + desired +
- ", actual=" + actual +
- ", requested=" + requested +
- ", releasing=" + releasing +
- ", failed=" + failed +
- ", failed recently=" + failedRecently.get() +
- ", node failed=" + nodeFailed.get() +
- ", pre-empted=" + preempted.get() +
- ", started=" + started +
- ", startFailed=" + startFailed +
- ", completed=" + completed +
- ", failureMessage='" + failureMessage + '\'' +
- '}';
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("RoleStatus{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", key=").append(key);
+ sb.append(", desired=").append(desired);
+ sb.append(", actual=").append(actual);
+ sb.append(", requested=").append(requested);
+ sb.append(", releasing=").append(releasing);
+ sb.append(", failed=").append(failed);
+ sb.append(", startFailed=").append(startFailed);
+ sb.append(", started=").append(started);
+ sb.append(", completed=").append(completed);
+ sb.append(", totalRequested=").append(totalRequested);
+ sb.append(", preempted=").append(preempted);
+ sb.append(", nodeFailed=").append(nodeFailed);
+ sb.append(", failedRecently=").append(failedRecently);
+ sb.append(", limitsExceeded=").append(limitsExceeded);
+ sb.append(", resourceRequirements=").append(resourceRequirements);
+ sb.append(", isAntiAffinePlacement=").append(isAntiAffinePlacement());
+ if (isAntiAffinePlacement()) {
+ sb.append(", pendingAntiAffineRequests=").append(pendingAntiAffineRequests);
+ sb.append(", outstandingAArequest=").append(outstandingAArequest);
+ }
+ sb.append(", failureMessage='").append(failureMessage).append('\'');
+ sb.append(", providerRole=").append(providerRole);
+ sb.append('}');
+ return sb.toString();
}
@Override
@@ -343,21 +469,39 @@
ComponentInformation info = new ComponentInformation();
info.name = name;
info.priority = getPriority();
- info.desired = desired;
- info.actual = actual;
- info.requested = requested;
- info.releasing = releasing;
- info.failed = failed;
- info.startFailed = startFailed;
+ info.desired = desired.intValue();
+ info.actual = actual.intValue();
+ info.requested = requested.intValue();
+ info.releasing = releasing.intValue();
+ info.failed = failed.intValue();
+ info.startFailed = startFailed.intValue();
info.placementPolicy = getPlacementPolicy();
info.failureMessage = failureMessage;
- info.totalRequested = totalRequested;
+ info.totalRequested = totalRequested.intValue();
info.failedRecently = failedRecently.intValue();
info.nodeFailed = nodeFailed.intValue();
info.preempted = preempted.intValue();
+ info.pendingAntiAffineRequestCount = pendingAntiAffineRequests.intValue();
+ info.isAARequestOutstanding = isAARequestOutstanding();
return info;
}
-
+
+ /**
+ * Get the (possibly null) label expression for this role
+ * @return a string or null
+ */
+ public String getLabelExpression() {
+ return providerRole.labelExpression;
+ }
+
+ public Resource getResourceRequirements() {
+ return resourceRequirements;
+ }
+
+ public void setResourceRequirements(Resource resourceRequirements) {
+ this.resourceRequirements = resourceRequirements;
+ }
+
/**
* Compare two role status entries by name
*/
@@ -379,5 +523,35 @@
return (o1.getKey() < o2.getKey() ? -1 : (o1.getKey() == o2.getKey() ? 0 : 1));
}
}
-
+
+ /**
+ * Given a resource, set its requirements to those this role needs
+ * @param resource resource to configure
+ * @return the resource
+ */
+ public Resource copyResourceRequirements(Resource resource) {
+ Preconditions.checkNotNull(resourceRequirements,
+ "Role resource requirements have not been set");
+ resource.setMemory(resourceRequirements.getMemory());
+ resource.setVirtualCores(resourceRequirements.getVirtualCores());
+ return resource;
+ }
+
+ public synchronized RoleStatistics getStatistics() {
+ RoleStatistics stats = new RoleStatistics();
+ stats.activeAA = getOutstandingAARequestCount();
+ stats.actual = actual.get();
+ stats.desired = desired.get();
+ stats.failed = failed.get();
+ stats.limitsExceeded = limitsExceeded.get();
+ stats.nodeFailed = nodeFailed.get();
+ stats.preempted = preempted.get();
+ stats.releasing = releasing.get();
+ stats.requested = requested.get();
+ stats.started = started.get();
+ stats.startFailed = startFailed.get();
+ stats.totalRequested = totalRequested.get();
+ return stats;
+ }
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
index b7f0e05..b848096 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/SimpleReleaseSelector.java
@@ -27,8 +27,7 @@
@Override
public List<RoleInstance> sortCandidates(int roleId,
- List<RoleInstance> candidates,
- int minimumToSelect) {
+ List<RoleInstance> candidates) {
return candidates;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index eef178a..ad91183 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -26,6 +26,8 @@
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -37,7 +39,8 @@
import java.util.Map;
/**
- * The methods to offer state access to the providers
+ * The methods to offer state access to the providers and other parts of
+ * the system which want read-only access to the state.
*/
public interface StateAccessForProviders {
@@ -86,17 +89,17 @@
List<String> listConfigSets();
/**
- * Get a map of all the failed nodes
- * @return map of recorded failed notes
+ * Get a map of all the failed containers
+ * @return map of recorded failed containers
*/
- Map<ContainerId, RoleInstance> getFailedNodes();
+ Map<ContainerId, RoleInstance> getFailedContainers();
/**
- * Get the live nodes.
+ * Get the live containers.
*
* @return the live nodes
*/
- Map<ContainerId, RoleInstance> getLiveNodes();
+ Map<ContainerId, RoleInstance> getLiveContainers();
/**
* Get the current cluster description
@@ -267,11 +270,11 @@
Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping();
/**
- * Enum all nodes by role.
+ * Enum all role instances by role.
* @param role role, or "" for all roles
- * @return a list of nodes, may be empty
+ * @return a list of instances, may be empty
*/
- List<RoleInstance> enumLiveNodesInRole(String role);
+ List<RoleInstance> enumLiveInstancesInRole(String role);
/**
* Look up all containers of a specific component name
@@ -280,5 +283,31 @@
*/
List<RoleInstance> lookupRoleContainers(String component);
+ /**
+ * Get the JSON serializable information about a component
+ * @param component component to look up
+ * @return a structure describing the component.
+ */
ComponentInformation getComponentInformation(String component);
+
+
+ /**
+ * Get a clone of the nodemap.
+ * The instances inside are not cloned
+ * @return a possibly empty map of hostname top info
+ */
+ Map<String, NodeInformation> getNodeInformationSnapshot();
+
+ /**
+ * get information on a node
+ * @param hostname hostname to look up
+ * @return the information, or null if there is no information held.
+ */
+ NodeInformation getNodeInformation(String hostname);
+
+ /**
+ * Get the aggregate statistics across all roles
+ * @return role statistics
+ */
+ RoleStatistics getRoleStatistics();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
index 84f0eba..7ecc00c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/SliderAMWebApp.java
@@ -92,7 +92,7 @@
String regex = "(?!/ws)";
serveRegex(regex).with(SliderDefaultWrapperServlet.class);
- Map<String, String> params = new HashMap<String, String>();
+ Map<String, String> params = new HashMap<>();
params.put(ResourceConfig.FEATURE_IMPLICIT_VIEWABLES, "true");
params.put(ServletContainer.FEATURE_FILTER_FORWARD_ON_404, "true");
params.put(ResourceConfig.FEATURE_XMLROOTELEMENT_PROCESSING, "true");
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
index 179ae8c..0f99d6d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApi.java
@@ -81,7 +81,15 @@
*/
QueueAccess getQueues();
+ /**
+ * API for AM operations
+ * @return current operations implementation
+ */
AppMasterActionOperations getAMOperations();
+ /**
+ * Local cache of content
+ * @return the cache
+ */
ContentCache getContentCache();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
index 0730a21..a0fe310 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/WebAppApiImpl.java
@@ -89,8 +89,7 @@
@Override
public Map<String,RoleStatus> getRoleStatusByName() {
List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
- Map<String, RoleStatus> map =
- new TreeMap<String, RoleStatus>();
+ Map<String, RoleStatus> map = new TreeMap<>();
for (RoleStatus status : roleStatuses) {
map.put(status.getName(), status);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
index cfddb12..7ff83b6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/AbstractSliderResource.java
@@ -45,12 +45,11 @@
protected final WebAppApi slider;
protected final MetricsAndMonitoring metricsAndMonitoring;
- public AbstractSliderResource(WebAppApi slider) {
+ protected AbstractSliderResource(WebAppApi slider) {
this.slider = slider;
metricsAndMonitoring = slider.getMetricsAndMonitoring();
}
-
/**
* Generate a redirect to the WASL
* @param request to base the URL on
@@ -105,6 +104,7 @@
protected void mark(String verb, String path) {
metricsAndMonitoring.markMeterAndCounter(verb + "-" + path);
}
+
/**
* Mark an GET operation on a path
* @param verb HTTP Verb
@@ -129,6 +129,7 @@
protected void markGet(String path, String subpath) {
mark("GET", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
@@ -136,6 +137,7 @@
protected void markPost(String path, String subpath) {
mark("POST", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
@@ -143,6 +145,7 @@
protected void markPut(String path, String subpath) {
mark("PUT", path, subpath);
}
+
/**
* Mark a GET operation on a path
* @param path path relative to slider API
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
index 06b7ba2..b90eb62 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java
@@ -116,7 +116,6 @@
*/
public static final String SYSTEM_THREADS = SYSTEM + "/threads";
-
/**
* application subpath
*/
@@ -134,7 +133,7 @@
public static final String LIVE_RESOURCES = "/live/resources";
public static final String LIVE_CONTAINERS = "/live/containers";
public static final String LIVE_COMPONENTS = "/live/components";
- public static final String LIVE_NODES = "/live/";
+ public static final String LIVE_NODES = "/live/nodes";
public static final String LIVE_LIVENESS = "/live/liveness";
public static final String LIVE_STATISTICS = "/live/statistics";
public static final String MODEL = "/model";
@@ -149,4 +148,15 @@
public static final String ACTION = "/action";
public static final String ACTION_PING = ACTION + "/ping";
public static final String ACTION_STOP = ACTION + "/stop";
+
+ /**
+ * Path to a role
+ * @param name role name
+ * @return a path to it
+ */
+ public String pathToRole(String name) {
+
+ // ws/v1/slider/application/live/components/$name
+ return SLIDER_PATH_APPLICATION + LIVE_COMPONENTS + "/" + name;
+ }
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java
index e7b8fc7..2facf16 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResouceContentCacheFactory.java
@@ -18,10 +18,6 @@
package org.apache.slider.server.appmaster.web.rest.application;
-import org.apache.slider.api.types.ComponentInformation;
-import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.core.conf.AggregateConf;
-import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.rest.application.resources.AggregateModelRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.AppconfRefresher;
@@ -29,63 +25,39 @@
import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveComponentsRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveContainersRefresher;
+import org.apache.slider.server.appmaster.web.rest.application.resources.LiveNodesRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveResourcesRefresher;
import org.apache.slider.server.appmaster.web.rest.application.resources.LiveStatisticsRefresher;
-import java.util.Map;
-
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_CONTAINERS;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_RESOURCES;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_STATISTICS;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_APPCONF;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_DESIRED_RESOURCES;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_APPCONF;
-import static org.apache.slider.server.appmaster.web.rest.RestPaths.MODEL_RESOLVED_RESOURCES;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
public class ApplicationResouceContentCacheFactory {
public static final int LIFESPAN = 500;
/**
* Build the content cache
- * @param cache cache to construct
* @param state state view
*/
public static ContentCache createContentCache(
StateAccessForProviders state) {
ContentCache cache = new ContentCache();
- cache.put(LIVE_RESOURCES,
- new CachedContent<ConfTree>(LIFESPAN,
- new LiveResourcesRefresher(state)));
- cache.put(LIVE_CONTAINERS,
- new CachedContent<Map<String, ContainerInformation>>(LIFESPAN,
- new LiveContainersRefresher(state)));
- cache.put(LIVE_COMPONENTS,
- new CachedContent<Map<String, ComponentInformation>>(LIFESPAN,
- new LiveComponentsRefresher(state)));
+ cache.put(LIVE_RESOURCES, new CachedContent<>(LIFESPAN, new LiveResourcesRefresher(state)));
+ cache.put(LIVE_CONTAINERS, new CachedContent<>(LIFESPAN, new LiveContainersRefresher(state)));
+ cache.put(LIVE_COMPONENTS, new CachedContent<>(LIFESPAN, new LiveComponentsRefresher(state)));
+ cache.put(LIVE_NODES, new CachedContent<>(LIFESPAN, new LiveNodesRefresher(state)));
cache.put(MODEL_DESIRED,
- new CachedContent<AggregateConf>(LIFESPAN,
- new AggregateModelRefresher(state, false)));
+ new CachedContent<>(LIFESPAN, new AggregateModelRefresher(state, false)));
cache.put(MODEL_RESOLVED,
- new CachedContent<AggregateConf>(LIFESPAN,
- new AggregateModelRefresher(state, true)));
+ new CachedContent<>(LIFESPAN, new AggregateModelRefresher(state, true)));
cache.put(MODEL_RESOLVED_APPCONF,
- new CachedContent<ConfTree>(LIFESPAN,
- new AppconfRefresher(state, false, false)));
+ new CachedContent<>(LIFESPAN, new AppconfRefresher(state, false, false)));
cache.put(MODEL_RESOLVED_RESOURCES,
- new CachedContent<ConfTree>(LIFESPAN,
- new AppconfRefresher(state, false, true)));
+ new CachedContent<>(LIFESPAN, new AppconfRefresher(state, false, true)));
cache.put(MODEL_DESIRED_APPCONF,
- new CachedContent<ConfTree>(LIFESPAN,
- new AppconfRefresher(state, true, false)));
+ new CachedContent<>(LIFESPAN, new AppconfRefresher(state, true, false)));
cache.put(MODEL_DESIRED_RESOURCES,
- new CachedContent<ConfTree>(LIFESPAN,
- new AppconfRefresher(state, true, true)));
- cache.put(LIVE_STATISTICS,
- new CachedContent<Map<String, Integer>>(LIFESPAN,
- new LiveStatisticsRefresher(state)));
+ new CachedContent<>(LIFESPAN, new AppconfRefresher(state, true, true)));
+ cache.put(LIVE_STATISTICS, new CachedContent<>(LIFESPAN, new LiveStatisticsRefresher(state)));
return cache;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
index 383bc5a..52068d6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java
@@ -25,6 +25,8 @@
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.ContainerInformation;
+import org.apache.slider.api.types.NodeInformation;
+import org.apache.slider.api.types.NodeInformationList;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
import org.apache.slider.core.exceptions.NoSuchNodeException;
@@ -285,8 +287,7 @@
public Map<String, ComponentInformation> getLiveComponents() {
markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS);
try {
- return (Map<String, ComponentInformation>) cache.lookup(
- LIVE_COMPONENTS);
+ return (Map<String, ComponentInformation>) cache.lookup(LIVE_COMPONENTS);
} catch (Exception e) {
throw buildException(LIVE_COMPONENTS, e);
}
@@ -346,6 +347,38 @@
}
*/
+
+ @GET
+ @Path(LIVE_NODES)
+ @Produces({APPLICATION_JSON})
+ public NodeInformationList getLiveNodes() {
+ markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS);
+ try {
+ return (NodeInformationList) cache.lookup(LIVE_NODES);
+ } catch (Exception e) {
+ throw buildException(LIVE_COMPONENTS, e);
+ }
+ }
+
+ @GET
+ @Path(LIVE_NODES + "/{hostname}")
+ @Produces({APPLICATION_JSON})
+ public NodeInformation getLiveNode(@PathParam("hostname") String hostname) {
+ markGet(SLIDER_SUBPATH_APPLICATION, LIVE_COMPONENTS);
+ try {
+ NodeInformation ni = state.getNodeInformation(hostname);
+ if (ni != null) {
+ return ni;
+ } else {
+ throw new NotFoundException("Unknown node: " + hostname);
+ }
+ } catch (NotFoundException e) {
+ throw e;
+ } catch (Exception e) {
+ throw buildException(LIVE_COMPONENTS + "/" + hostname, e);
+ }
+ }
+
/**
* Statistics of the application
* @return snapshot statistics
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
new file mode 100644
index 0000000..aeb7a11
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/LiveNodesRefresher.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.rest.application.resources;
+
+import org.apache.slider.api.types.NodeInformationList;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+
+/**
+ * Update the live nodes map
+ */
+public class LiveNodesRefresher
+ implements ResourceRefresher<NodeInformationList> {
+
+ private final StateAccessForProviders state;
+
+ public LiveNodesRefresher(StateAccessForProviders state) {
+ this.state = state;
+ }
+
+ @Override
+ public NodeInformationList refresh() {
+
+ return new NodeInformationList(state.getNodeInformationSnapshot().values());
+ }
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
index 75d417b..794daf9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/management/resources/AggregateConfResource.java
@@ -37,23 +37,13 @@
public AggregateConfResource(AggregateConf conf, UriBuilder uriBuilder) {
if (uriBuilder != null) {
- this.href =
- uriBuilder.build().toASCIIString();
- resources =
- ResourceFactory.createConfTreeResource(conf.getAppConf(),
- uriBuilder.clone().path(
- "configurations").path(
- "resources"));
- internal =
- ResourceFactory.createConfTreeResource(conf.getInternal(),
- uriBuilder.clone().path(
- "configurations").path(
- "internal"));
- appConf =
- ResourceFactory.createConfTreeResource(conf.getAppConf(),
- uriBuilder.clone().path(
- "configurations").path(
- "appConf"));
+ this.href = uriBuilder.build().toASCIIString();
+ resources = ResourceFactory.createConfTreeResource(conf.getResources(),
+ uriBuilder.clone().path("configurations").path("resources"));
+ internal = ResourceFactory.createConfTreeResource(conf.getInternal(),
+ uriBuilder.clone().path("configurations").path("internal"));
+ appConf = ResourceFactory.createConfTreeResource(conf.getAppConf(),
+ uriBuilder.clone().path("configurations").path("appConf"));
initConfMap();
} else {
resources = null;
@@ -63,7 +53,7 @@
}
private void initConfMap() {
- confMap = new HashMap<String, ConfTreeResource>();
+ confMap = new HashMap<>();
confMap.put("internal", internal);
confMap.put("resources", resources);
confMap.put("appConf", appConf);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
index 137c476..2f02f27 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ClusterSpecificationBlock.java
@@ -18,23 +18,16 @@
import com.google.inject.Inject;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
*
*/
-public class ClusterSpecificationBlock extends HtmlBlock {
- private static final Logger log = LoggerFactory.getLogger(ClusterSpecificationBlock.class);
-
- private StateAccessForProviders appState;
+public class ClusterSpecificationBlock extends SliderHamletBlock {
@Inject
public ClusterSpecificationBlock(WebAppApi slider) {
- this.appState = slider.getAppState();
+ super(slider);
}
@Override
@@ -50,7 +43,7 @@
pre().
_(getJson())._()._();
}
-
+
/**
* Get the JSON, catching any exceptions and returning error text instead
* @return
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
index 0896e2b..56285c2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/ContainerStatsBlock.java
@@ -26,12 +26,10 @@
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import java.io.Serializable;
@@ -45,19 +43,18 @@
/**
*
*/
-public class ContainerStatsBlock extends HtmlBlock {
+public class ContainerStatsBlock extends SliderHamletBlock {
private static final String EVEN = "even", ODD = "odd", BOLD = "bold", SCHEME = "http://", PATH = "/node/container/";
// Some functions that help transform the data into an object we can use to abstract presentation specifics
protected static final Function<Entry<String,Integer>,Entry<TableContent,Integer>> stringIntPairFunc = toTableContentFunction();
+ protected static final Function<Entry<String,Long>,Entry<TableContent,Long>> stringLongPairFunc = toTableContentFunction();
protected static final Function<Entry<String,String>,Entry<TableContent,String>> stringStringPairFunc = toTableContentFunction();
- private WebAppApi slider;
-
@Inject
public ContainerStatsBlock(WebAppApi slider) {
- this.slider = slider;
+ super(slider);
}
/**
@@ -92,7 +89,6 @@
@Override
protected void render(Block html) {
- StateAccessForProviders appState = slider.getAppState();
final Map<String,RoleInstance> containerInstances = getContainerInstances(
appState.cloneOwnedContainerList());
@@ -108,7 +104,7 @@
DIV<Hamlet> div = html.div("role-info ui-widget-content ui-corner-all");
List<ClusterNode> nodesInRole =
- new ArrayList<ClusterNode>(clusterNodesInRole.values());
+ new ArrayList<>(clusterNodesInRole.values());
div.h2(BOLD, StringUtils.capitalize(name));
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
index 5131b5e..b3be3bf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java
@@ -21,50 +21,52 @@
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
+import org.apache.slider.api.types.RoleStatistics;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.providers.ProviderService;
import org.apache.slider.server.appmaster.state.RoleStatus;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.LIVE_COMPONENTS;
+
/**
- *
+ * The main content on the Slider AM web page
*/
-public class IndexBlock extends HtmlBlock {
+public class IndexBlock extends SliderHamletBlock {
private static final Logger log = LoggerFactory.getLogger(IndexBlock.class);
- private StateAccessForProviders appView;
- private ProviderService providerService;
+ /**
+ * Message printed when application is at full size.
+ *
+ * {@value}
+ */
+ public static final String ALL_CONTAINERS_ALLOCATED = "all containers allocated";
@Inject
public IndexBlock(WebAppApi slider) {
- this.appView = slider.getAppState();
- this.providerService = slider.getProviderService();
+ super(slider);
}
@Override
protected void render(Block html) {
- final String providerName = getProviderName();
-
- doIndex(html, providerName);
+ doIndex(html, getProviderName());
}
// An extra method to make testing easier since you can't make an instance of Block
@VisibleForTesting
protected void doIndex(Hamlet html, String providerName) {
- ClusterDescription clusterStatus = appView.getClusterStatus();
+ ClusterDescription clusterStatus = appState.getClusterStatus();
String name = clusterStatus.name;
if (name != null && (name.startsWith(" ") || name.endsWith(" "))) {
name = "'" + name + "'";
@@ -74,10 +76,9 @@
"Application: " + name);
ApplicationLivenessInformation liveness =
- appView.getApplicationLivenessInformation();
- String livestatus =
- liveness.allRequestsSatisfied
- ? "all containers allocated"
+ appState.getApplicationLivenessInformation();
+ String livestatus = liveness.allRequestsSatisfied
+ ? ALL_CONTAINERS_ALLOCATED
: String.format("Awaiting %d containers", liveness.requestsOutstanding);
Hamlet.TABLE<DIV<Hamlet>> table1 = div.table();
table1.tr()
@@ -86,7 +87,7 @@
._();
table1.tr()
.td("Total number of containers")
- .td(Integer.toString(appView.getNumOwnedContainers()))
+ .td(Integer.toString(appState.getNumOwnedContainers()))
._();
table1.tr()
.td("Create time: ")
@@ -108,44 +109,132 @@
.td("Application configuration path: ")
.td(clusterStatus.originConfigurationPath)
._();
- table1._()._();
+ table1._();
+ div._();
+ div = null;
+ DIV<Hamlet> containers = html.div("container_instances")
+ .h3("Component Instances");
- html.div("container_instances").h3("Component Instances");
+ int aaRoleWithNoSuitableLocations = 0;
+ int aaRoleWithOpenRequest = 0;
+ int roleWithOpenRequest = 0;
- Hamlet.TABLE<DIV<Hamlet>> table = div.table();
- table.tr()
- .td("Component")
- .td("Desired")
- .td("Actual")
- .td("Outstanding Requests")
- .td("Failed")
- .td("Failed to start")
- ._();
+ Hamlet.TABLE<DIV<Hamlet>> table = containers.table();
+ Hamlet.TR<Hamlet.THEAD<Hamlet.TABLE<DIV<Hamlet>>>> header = table.thead().tr();
+ trb(header, "Component");
+ trb(header, "Desired");
+ trb(header, "Actual");
+ trb(header, "Outstanding Requests");
+ trb(header, "Failed");
+ trb(header, "Failed to start");
+ trb(header, "Placement");
+ header._()._(); // tr & thead
- List<RoleStatus> roleStatuses = appView.cloneRoleStatusList();
+ List<RoleStatus> roleStatuses = appState.cloneRoleStatusList();
Collections.sort(roleStatuses, new RoleStatus.CompareByName());
for (RoleStatus status : roleStatuses) {
+ String roleName = status.getName();
+ String nameUrl = apiPath(LIVE_COMPONENTS) + "/" + roleName;
+ String aatext;
+ if (status.isAntiAffinePlacement()) {
+ boolean aaRequestOutstanding = status.isAARequestOutstanding();
+ int pending = (int)status.getPendingAntiAffineRequests();
+ aatext = buildAADetails(aaRequestOutstanding, pending);
+ if (SliderUtils.isSet(status.getLabelExpression())) {
+ aatext += " (label: " + status.getLabelExpression() + ")";
+ }
+ if (pending > 0 && !aaRequestOutstanding) {
+ aaRoleWithNoSuitableLocations ++;
+ } else if (aaRequestOutstanding) {
+ aaRoleWithOpenRequest++;
+ }
+ } else {
+ if (SliderUtils.isSet(status.getLabelExpression())) {
+ aatext = "label: " + status.getLabelExpression();
+ } else {
+ aatext = "";
+ }
+ if (status.getRequested() > 0) {
+ roleWithOpenRequest ++;
+ }
+ }
table.tr()
- .td(status.getName())
- .td(String.format("%d", status.getDesired()))
- .td(String.format("%d", status.getActual()))
- .td(String.format("%d", status.getRequested()))
- .td(String.format("%d", status.getFailed()))
- .td(String.format("%d", status.getStartFailed()))
- ._();
+ .td().a(nameUrl, roleName)._()
+ .td(String.format("%d", status.getDesired()))
+ .td(String.format("%d", status.getActual()))
+ .td(String.format("%d", status.getRequested()))
+ .td(String.format("%d", status.getFailed()))
+ .td(String.format("%d", status.getStartFailed()))
+ .td(aatext)
+ ._();
}
- table._()._();
+ // empty row for some more spacing
+ table.tr()._();
+ // close table
+ table._();
+
+ containers._();
+ containers = null;
// some spacing
- html.p()._();
- html.p()._();
+ html.div()._();
+ html.div()._();
- html.div("provider_info").h3(providerName + " information");
- UL<DIV<Hamlet>> ul = div.ul();
+ DIV<Hamlet> diagnostics = html.div("diagnostics");
+
+ List<String> statusEntries = new ArrayList<>(0);
+ if (roleWithOpenRequest > 0) {
+ statusEntries.add(String.format("%d %s with requests unsatisfiable by cluster",
+ roleWithOpenRequest, plural(roleWithOpenRequest, "component")));
+ }
+ if (aaRoleWithNoSuitableLocations > 0) {
+ statusEntries.add(String.format("%d anti-affinity %s no suitable nodes in the cluster",
+ aaRoleWithNoSuitableLocations,
+ plural(aaRoleWithNoSuitableLocations, "component has", "components have")));
+ }
+ if (aaRoleWithOpenRequest > 0) {
+ statusEntries.add(String.format("%d anti-affinity %s with requests unsatisfiable by cluster",
+ aaRoleWithOpenRequest,
+ plural(aaRoleWithOpenRequest, "component has", "components have")));
+
+ }
+ if (!statusEntries.isEmpty()) {
+ diagnostics.h3("Diagnostics");
+ Hamlet.TABLE<DIV<Hamlet>> diagnosticsTable = diagnostics.table();
+ for (String entry : statusEntries) {
+ diagnosticsTable.tr().td(entry)._();
+ }
+ diagnosticsTable._();
+ }
+ diagnostics._();
+
+ DIV<Hamlet> provider_info = html.div("provider_info");
+ provider_info.h3(providerName + " information");
+ UL<Hamlet> ul = html.ul();
addProviderServiceOptions(providerService, ul, clusterStatus);
- ul._()._();
+ ul._();
+ provider_info._();
+ }
+
+ @VisibleForTesting
+ String buildAADetails(boolean outstanding, int pending) {
+ return String.format("Anti-affinity:%s %d pending %s",
+ (outstanding ? " 1 active request and" : ""),
+ pending, plural(pending, "request"));
+ }
+
+ private String plural(int n, String singular) {
+ return plural(n, singular, singular + "s");
+ }
+ private String plural(int n, String singular, String plural) {
+ return n == 1 ? singular : plural;
+ }
+
+ private void trb(Hamlet.TR tr,
+ String text) {
+ tr.td().b(text)._();
}
private String getProviderName() {
@@ -153,14 +242,14 @@
}
private String getInfoAvoidingNulls(String key) {
- String createTime = appView.getClusterStatus().getInfo(key);
+ String createTime = appState.getClusterStatus().getInfo(key);
return null == createTime ? "N/A" : createTime;
}
- protected void addProviderServiceOptions(ProviderService providerService,
- UL<DIV<Hamlet>> ul, ClusterDescription clusterStatus) {
- Map<String, String> details = providerService.buildMonitorDetails(
+ protected void addProviderServiceOptions(ProviderService provider,
+ UL ul, ClusterDescription clusterStatus) {
+ Map<String, String> details = provider.buildMonitorDetails(
clusterStatus);
if (null == details) {
return;
@@ -176,4 +265,5 @@
}
}
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
index b2327ba..069d386 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/NavBlock.java
@@ -16,16 +16,21 @@
*/
package org.apache.slider.server.appmaster.web.view;
-import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import com.google.inject.Inject;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+
import static org.apache.slider.server.appmaster.web.SliderAMWebApp.*;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*;
-import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-
/**
*
*/
-public class NavBlock extends HtmlBlock {
+public class NavBlock extends SliderHamletBlock {
+
+ @Inject
+ public NavBlock(WebAppApi slider) {
+ super(slider);
+ }
@Override
protected void render(Block html) {
@@ -36,7 +41,6 @@
li().a(this.prefix(), "Overview")._().
li().a(relPath(CONTAINER_STATS), "Statistics")._().
li().a(relPath(CLUSTER_SPEC), "Specification")._().
- li().a(relPath(CLUSTER_SPEC), "Specification")._().
li().a(rootPath(SYSTEM_METRICS_JSON), "Metrics")._().
li().a(rootPath(SYSTEM_HEALTHCHECK), "Health")._().
li().a(rootPath(SYSTEM_THREADS), "Threads")._().
@@ -48,20 +52,11 @@
li().a(apiPath(LIVE_RESOURCES), "Resources")._().
li().a(apiPath(LIVE_COMPONENTS), "Components")._().
li().a(apiPath(LIVE_CONTAINERS), "Containers")._().
+ li().a(apiPath(LIVE_NODES), "Nodes")._().
+ li().a(apiPath(LIVE_STATISTICS), "Statistics")._().
li().a(apiPath(LIVE_LIVENESS), "Liveness")._()
._()
._();
}
- private String rootPath(String absolutePath) {
- return root_url(absolutePath);
- }
-
- private String relPath(String... args) {
- return ujoin(this.prefix(), args);
- }
- private String apiPath(String api) {
- return root_url(SLIDER_PATH_APPLICATION, api);
- }
-
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
new file mode 100644
index 0000000..82d7c8f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/SliderHamletBlock.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.web.view;
+
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.slider.providers.ProviderService;
+import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.WebAppApi;
+import org.apache.slider.server.appmaster.web.rest.RestPaths;
+
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
+
+/**
+ * Anything we want to share across slider hamlet blocks
+ */
+public abstract class SliderHamletBlock extends HtmlBlock {
+
+ protected final StateAccessForProviders appState;
+ protected final ProviderService providerService;
+ protected final RestPaths restPaths = new RestPaths();
+
+ public SliderHamletBlock(WebAppApi slider) {
+ this.appState = slider.getAppState();
+ this.providerService = slider.getProviderService();
+ }
+
+ protected String rootPath(String absolutePath) {
+ return root_url(absolutePath);
+ }
+
+ protected String relPath(String... args) {
+ return ujoin(this.prefix(), args);
+ }
+
+ protected String apiPath(String api) {
+ return root_url(SLIDER_PATH_APPLICATION, api);
+ }
+
+}
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index 01cc5e5..63f66a3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -17,14 +17,15 @@
package org.apache.slider.server.services.security;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.Time;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.actions.AsyncAction;
@@ -63,15 +64,15 @@
private void createRemoteUser(Configuration configuration) throws IOException {
Configuration loginConfig = new Configuration(configuration);
- loginConfig.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,
+ loginConfig.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
// using HDFS principal...
this.remoteUser = UserGroupInformation
.loginUserFromKeytabAndReturnUGI(
SecurityUtil.getServerPrincipal(
- loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
+ loginConfig.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
InetAddress.getLocalHost().getCanonicalHostName()),
- loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
+ loginConfig.get(SliderXmlConfKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
log.info("Created remote user {}. UGI reports current user is {}",
this.remoteUser, UserGroupInformation.getCurrentUser());
}
@@ -84,8 +85,8 @@
if (SliderUtils.isHadoopClusterSecure(configuration) &&
renewingAction == null) {
renewInterval = configuration.getLong(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+ SliderXmlConfKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ SliderXmlConfKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
// constructor of action will retrieve initial token. One may already be
// associated with user, but its lifecycle/management is not clear so let's
// create and manage a token explicitly
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
index 7f3931b..1622309 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/utility/AbstractSliderLaunchedService.java
@@ -98,11 +98,23 @@
*/
protected static void requireArgumentSet(String argname, String value)
throws BadCommandArgumentsException {
- if (isUnset(value)) {
- throw new BadCommandArgumentsException(
- "Required argument " + argname + " missing");
- }
+ require(isSet(value), "Required argument %s missing", argname );
}
+ /**
+ * Require a condition to hold; throw {@link BadCommandArgumentsException} if not.
+ * The exception text is the formatted message.
+ * @param condition condition
+ * @param message string to format
+ * @param args list of arguments to format.
+ * @throws BadCommandArgumentsException
+ */
+ protected static void require(boolean condition, String message,
+ Object... args)
+ throws BadCommandArgumentsException {
+ if (!condition) {
+ throw new BadCommandArgumentsException(message, args);
+ }
+ }
}
diff --git a/slider-core/src/main/proto/SliderClusterMessages.proto b/slider-core/src/main/proto/SliderClusterMessages.proto
index caca87b..b8bdc59 100644
--- a/slider-core/src/main/proto/SliderClusterMessages.proto
+++ b/slider-core/src/main/proto/SliderClusterMessages.proto
@@ -24,21 +24,25 @@
//import "Security.proto";
+/*
+ Look at SliderClusterProtocol.proto to see how to build this
+*/
+
message RoleInstanceState {
- required string name = 1;
- optional string role = 2;
- required uint32 state = 4;
- required uint32 exitCode = 5;
- optional string command = 6;
+ required string name = 1;
+ optional string role = 2;
+ required uint32 state = 4;
+ required uint32 exitCode = 5;
+ optional string command = 6;
optional string diagnostics = 7;
- repeated string output = 8;
+ repeated string output = 8;
repeated string environment = 9;
- required uint32 roleId = 10;
- required bool released = 11;
- required int64 createTime = 12;
- required int64 startTime = 13;
- required string host = 14;
- required string hostURL = 15;
+ required uint32 roleId = 10;
+ required bool released = 11;
+ required int64 createTime = 12;
+ required int64 startTime = 13;
+ required string host = 14;
+ required string hostURL = 15;
optional string appVersion = 16;
}
@@ -65,9 +69,9 @@
/**
message to include
*/
- required string message = 1;
- repeated string container = 2;
- repeated string component = 3;
+ required string message = 1;
+ repeated string container = 2;
+ repeated string component = 3;
}
/**
@@ -181,9 +185,9 @@
* AM suicide
*/
message AMSuicideRequestProto {
- required string text = 1;
- required int32 signal = 2;
- required int32 delay = 3;
+ required string text = 1;
+ required int32 signal = 2;
+ required int32 delay = 3;
}
/**
@@ -206,9 +210,9 @@
* Get the definition back as three separate JSON strings
*/
message GetInstanceDefinitionResponseProto {
- required string internal = 1;
- required string resources = 2;
- required string application = 3;
+ required string internal = 1;
+ required string resources = 2;
+ required string application = 3;
}
@@ -233,42 +237,44 @@
* see org.apache.slider.api.types.ComponentInformation
*/
message ComponentInformationProto {
- optional string name = 1;
- optional int32 priority = 2;
- optional int32 desired = 3;
- optional int32 actual = 4;
- optional int32 releasing = 5;
- optional int32 requested = 6;
- optional int32 failed = 7;
- optional int32 started = 8;
- optional int32 startFailed = 9;
- optional int32 completed = 10;
- optional int32 totalRequested = 11;
- optional string failureMessage = 12 ;
- optional int32 placementPolicy = 13;
- repeated string containers = 14;
+ optional string name = 1;
+ optional int32 priority = 2;
+ optional int32 desired = 3;
+ optional int32 actual = 4;
+ optional int32 releasing = 5;
+ optional int32 requested = 6;
+ optional int32 failed = 7;
+ optional int32 started = 8;
+ optional int32 startFailed = 9;
+ optional int32 completed = 10;
+ optional int32 totalRequested = 11;
+ optional string failureMessage =12;
+ optional int32 placementPolicy =13;
+ repeated string containers = 14;
optional int32 failedRecently = 15;
- optional int32 nodeFailed = 16;
- optional int32 preempted = 17;
+ optional int32 nodeFailed = 16;
+ optional int32 preempted = 17;
+ optional int32 pendingAntiAffineRequestCount = 18;
+ optional bool isAARequestOutstanding = 19;
}
/*
* see org.apache.slider.api.types.ContainerInformation
*/
message ContainerInformationProto {
- optional string containerId = 1;
- optional string component = 2;
- optional bool released = 3;
- optional int32 state = 4;
- optional int32 exitCode = 5;
- optional string diagnostics = 6;
- optional int64 createTime = 7;
- optional int64 startTime = 8;
- repeated string output = 9;
- optional string host = 10;
- optional string hostURL = 11;
- optional string placement = 12;
- optional string appVersion = 13;
+ optional string containerId = 1;
+ optional string component = 2;
+ optional bool released = 3;
+ optional int32 state = 4;
+ optional int32 exitCode = 5;
+ optional string diagnostics = 6;
+ optional int64 createTime = 7;
+ optional int64 startTime = 8;
+ repeated string output = 9;
+ optional string host = 10;
+ optional string hostURL = 11;
+ optional string placement = 12;
+ optional string appVersion = 13;
}
@@ -279,10 +285,33 @@
optional string text = 1;
optional string verb = 2;
optional string body = 3;
- optional int64 time = 4;
+ optional int64 time = 4;
}
+message NodeEntryInformationProto {
+ required int32 priority = 1;
+ required int32 requested = 2;
+ required int32 starting = 3;
+ required int32 startFailed = 4;
+ required int32 failed = 5;
+ required int32 failedRecently= 6;
+ required int32 preempted = 7;
+ required int32 live = 8;
+ required int32 releasing = 9;
+ required int64 lastUsed = 10;
+ required string name = 11;
+}
+message NodeInformationProto {
+ required string hostname = 1;
+ required string state = 2;
+ required string httpAddress = 3;
+ required string rackName = 4;
+ required string labels = 5;
+ required string healthReport= 6;
+ required int64 lastUpdated = 7;
+ repeated NodeEntryInformationProto entries = 8;
+}
message GetModelRequestProto {
}
@@ -345,12 +374,23 @@
}
message GetCertificateStoreRequestProto {
- optional string hostname = 1;
+ optional string hostname = 1;
required string requesterId = 2;
- required string password = 3;
- required string type = 4;
+ required string password = 3;
+ required string type = 4;
}
message GetCertificateStoreResponseProto {
required bytes store = 1;
+}
+
+message GetLiveNodesRequestProto {
+}
+
+message GetLiveNodesResponseProto {
+ repeated NodeInformationProto nodes = 1;
+}
+
+message GetLiveNodeRequestProto {
+ required string name = 1;
}
\ No newline at end of file
diff --git a/slider-core/src/main/proto/SliderClusterProtocol.proto b/slider-core/src/main/proto/SliderClusterProtocol.proto
index aa59bb4..4221b1d 100644
--- a/slider-core/src/main/proto/SliderClusterProtocol.proto
+++ b/slider-core/src/main/proto/SliderClusterProtocol.proto
@@ -129,18 +129,24 @@
rpc getLivenessInformation(GetApplicationLivenessRequestProto)
returns(ApplicationLivenessInformationProto);
-
+
rpc getLiveContainers(GetLiveContainersRequestProto)
returns(GetLiveContainersResponseProto);
rpc getLiveContainer(GetLiveContainerRequestProto)
returns(ContainerInformationProto);
-
+
rpc getLiveComponents(GetLiveComponentsRequestProto)
returns(GetLiveComponentsResponseProto);
-
+
rpc getLiveComponent(GetLiveComponentRequestProto)
returns(ComponentInformationProto);
+
+ rpc getLiveNodes(GetLiveNodesRequestProto)
+ returns(GetLiveNodesResponseProto);
+
+ rpc getLiveNode(GetLiveNodeRequestProto)
+ returns(NodeInformationProto);
// AggregateConf getModelDesired()
rpc getModelDesired(EmptyPayloadProto)
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig.json
new file mode 100644
index 0000000..6e3ef14
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig.json
@@ -0,0 +1,7 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ }
+}
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
index c8ada2c..073d1ff 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
@@ -1,16 +1,24 @@
{
- "schemaVersion": "2.1",
- "application": {
- "name": "SLEEPER",
- "components": [
- {
- "name": "SLEEP_100",
- "commands": [
- {
- "exec": "sleep 180"
- }
- ]
- }
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "SLEEPER",
+ "components": [
+ {
+ "name": "SLEEP_100",
+ "commands": [
+ {
+ "exec": "sleep 180"
+ }
]
- }
+ },
+ {
+ "name": "SLEEP_LONG",
+ "commands": [
+ {
+ "exec": "sleep 180000"
+ }
+ ]
+ }
+ ]
+ }
}
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
index be7b962..e0ed16a 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
@@ -11,7 +11,13 @@
"SLEEP_100": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "128"
+ },
+ "SLEEP_LONG": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "0",
+ "yarn.memory": "128",
+ "yarn.component.placement.policy": "4"
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index a7bc0a3..b37ee3a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -27,15 +27,13 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXMLConfKeysForTesting
import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.agent.AgentKeys
+import org.apache.slider.test.KeysForTests
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.AfterClass
import org.junit.BeforeClass
import org.junit.rules.TemporaryFolder
-import org.slf4j.Logger
-import org.slf4j.LoggerFactory
/**
* test base for agent clusters
@@ -43,8 +41,7 @@
@CompileStatic
@Slf4j
public abstract class AgentMiniClusterTestBase
-extends YarnZKMiniClusterTestBase {
- private static Logger LOG = LoggerFactory.getLogger(AgentMiniClusterTestBase)
+extends YarnZKMiniClusterTestBase implements KeysForTests {
protected static File agentConf
protected static File agentDef
protected static Map<String, String> agentDefOptions
@@ -92,7 +89,7 @@
@AfterClass
public static void cleanSubConfFiles() {
- def tempRoot
+ def tempRoot = ""
try {
tempRoot = tempFolder.root
if (tempRoot.exists()) {
@@ -101,7 +98,7 @@
} catch (IOException e) {
log.info("Failed to delete $tempRoot :$e", e)
} catch (IllegalStateException e) {
- log.warn("Temp folder deletion failed: $e")
+ log.warn("Temp folder deletion failed: $e", e)
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
index 4d27c37..a25d0f2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionList.groovy
@@ -174,14 +174,14 @@
new ActionListArgs(state: YarnApplicationState.FINISHED.toString(),
verbose: true));
- assert -1 == sliderClient.actionList("", new ActionListArgs(live: true));
+ assert 0 == sliderClient.actionList("", new ActionListArgs(live: true));
assert -1 == sliderClient.actionList(clustername,
new ActionListArgs(live: true));
assert -1 == sliderClient.actionList(clustername,
new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
- assert -1 == sliderClient.actionList("",
+ assert 0 == sliderClient.actionList("",
new ActionListArgs(state: YarnApplicationState.RUNNING.toString()));
// now look for finished app state
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
index 8245dae..7e3f55a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
@@ -41,8 +41,7 @@
class TestActionPackage extends AgentMiniClusterTestBase {
- public static final String E_INVALID_APP_TYPE =
- "A valid application type name is required (e.g. HBASE)"
+ public static final String E_NO_NAME = "Required argument --name missing"
String s = File.separator
File packageFile = new File("src${s}test${s}resources${s}log4j.properties")
@@ -70,7 +69,7 @@
} catch (BadCommandArgumentsException e) {
assertExceptionDetails(e,
LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
- E_INVALID_APP_TYPE)
+ E_NO_NAME)
}
}
@@ -285,7 +284,7 @@
} catch (BadCommandArgumentsException e) {
assertExceptionDetails(e,
LauncherExitCodes.EXIT_COMMAND_ARGUMENT_ERROR,
- E_INVALID_APP_TYPE)
+ E_NO_NAME)
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
index 0473e02..b2fcf8e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/freezethaw/TestFreezeCommands.groovy
@@ -144,7 +144,7 @@
ServiceLauncher<SliderClient> destroy1 = execSliderCommand(conf,
[
SliderActions.ACTION_DESTROY, clustername,
- Arguments.ARG_FILESYSTEM, fsDefaultName
+ Arguments.ARG_FILESYSTEM, fsDefaultName, Arguments.ARG_FORCE
]);
fail(
"expected a failure from the destroy, got error code ${destroy1.serviceExitCode}");
@@ -168,7 +168,7 @@
ServiceLauncher<SliderClient> destroy2 = execSliderCommand(conf,
[
SliderActions.ACTION_DESTROY, clustername,
- Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_FILESYSTEM, fsDefaultName, Arguments.ARG_FORCE
]);
assertSucceeded(destroy2)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy
index 58eb49e..6727a29 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/AbstractAppApiTestDelegates.groovy
@@ -203,6 +203,17 @@
assert !liveness.requestsOutstanding
}
+
+ public void testListNodes() throws Throwable {
+ describe "Node listing via $appAPI"
+ def liveNodes = appAPI.liveNodes
+ assert liveNodes.size() > 0
+ prettyPrintAsJson(liveNodes)
+ def h = liveNodes[0].hostname;
+ def localhost = appAPI.getLiveNode(h)
+ assert localhost.httpAddress == liveNodes[0].httpAddress
+ }
+
/**
* Probe that spins until the liveness query fails
* @param args argument map
@@ -226,6 +237,7 @@
testLiveContainers();
testRESTModel()
testAppLiveness()
+ testListNodes();
}
public void testFlexOperation() {
@@ -263,9 +275,10 @@
String key = args["key"]
String val = args["val"]
def resolved = appAPI.getResolvedResources()
- return Outcome.fromBool(resolved.get(key)==val)
+ return Outcome.fromBool(resolved.get(key) == val)
}
+
/**
* Get the resolved value and push that out as the new state
*
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java b/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java
index 9411c3c..f339f6d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/IpcApiClientTestDelegates.java
@@ -25,5 +25,5 @@
public IpcApiClientTestDelegates(SliderApplicationApi appAPI) {
super(true, appAPI);
}
-
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
index 857d15c..ff01dc5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/JerseyTestDelegates.groovy
@@ -177,7 +177,6 @@
public void testCodahaleOperations() throws Throwable {
describe "Codahale operations"
-
jerseyGet("/")
jerseyGet(SYSTEM_THREADS)
jerseyGet(SYSTEM_HEALTHCHECK)
@@ -288,14 +287,9 @@
// two components
assert components.size() >= 1
log.info "${components}"
-
- ComponentInformation amComponentInfo =
- (ComponentInformation) components[COMPONENT_AM]
-
ComponentInformation amFullInfo = jFetchType(
LIVE_COMPONENTS + "/${COMPONENT_AM}",
- ComponentInformation
- )
+ ComponentInformation)
assert amFullInfo.containers.size() == 1
assert amFullInfo.containers[0] == amContainerId
@@ -352,8 +346,7 @@
def unresolvedAppConf = unresolvedConf.appConfOperations
def sam = "slider-appmaster"
- assert unresolvedAppConf.getComponentOpt(sam,
- TEST_GLOBAL_OPTION, "") == ""
+ assert unresolvedAppConf.getComponentOpt(sam, TEST_GLOBAL_OPTION, "") == ""
def resolvedConf = jFetchType(MODEL_RESOLVED, AggregateConf)
// log.info "Resolved \n$resolvedConf"
assert resolvedConf.appConfOperations.getComponentOpt(
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy
index c0f123d..ed10479 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/LowLevelRestTestDelegates.groovy
@@ -59,26 +59,26 @@
application = appendToURL(appmaster, SLIDER_PATH_APPLICATION)
}
-
public void testCodahaleOperations() throws Throwable {
describe "Codahale operations $this"
getWebPage(appmaster)
getWebPage(appmaster, SYSTEM_THREADS)
getWebPage(appmaster, SYSTEM_HEALTHCHECK)
getWebPage(appmaster, SYSTEM_PING)
- getWebPage(appmaster, SYSTEM_METRICS_JSON)
+ def page = getWebPage(appmaster, SYSTEM_METRICS_JSON)
+ validateCodahaleJson(parseMetrics(page))
}
public void logCodahaleMetrics() {
// query Coda Hale metrics
- log.info getWebPage(appmaster, SYSTEM_HEALTHCHECK)
- log.info getWebPage(appmaster, SYSTEM_METRICS)
+ log.info prettyPrintJson(getWebPage(appmaster, SYSTEM_HEALTHCHECK))
+ log.info prettyPrintJson(getWebPage(appmaster, SYSTEM_METRICS))
}
public void testMimeTypes() throws Throwable {
describe "Mime Types $this"
- HttpOperationResponse response= executeGet(
+ HttpOperationResponse response = executeGet(
appendToURL(appmaster,
SLIDER_PATH_APPLICATION, LIVE_RESOURCES))
response.headers.each { key, val -> log.info("$key $val")}
@@ -150,9 +150,6 @@
assert components.size() >= 1
log.info "${components}"
- ComponentInformation amComponentInfo =
- (ComponentInformation) components[COMPONENT_AM]
-
ComponentInformation amFullInfo = fetchType(
ComponentInformation,
appmaster,
@@ -278,7 +275,6 @@
MediaType.TEXT_PLAIN)
log.info "Stopped: $outcome"
-
// now a ping is expected to fail
String ping = appendToURL(appmaster, SLIDER_PATH_APPLICATION, ACTION_PING)
@@ -317,7 +313,6 @@
@Override
public void testSuiteGetOperations() {
-
testCodahaleOperations()
testMimeTypes()
testLiveResources()
@@ -369,10 +364,8 @@
* @return the outcome
*/
Outcome probeForResolveConfValues(Map args) {
- assert args["key"]
- assert args["val"]
- String key = args["key"]
- String val = args["val"]
+ String key = requiredMapValue(args, "key")
+ String val = requiredMapValue(args, "val")
ConfTreeOperations resolved = modelDesiredResolvedResources
return Outcome.fromBool(resolved.get(key) == val)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 7e10d30..b0ae102 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -18,16 +18,8 @@
package org.apache.slider.agent.rest
-import com.sun.jersey.api.client.Client
-import com.sun.jersey.api.client.config.ClientConfig
-import com.sun.jersey.api.json.JSONConfiguration
-import com.sun.jersey.client.apache.ApacheHttpClient
-import com.sun.jersey.client.apache.ApacheHttpClientHandler
-import com.sun.jersey.client.apache.config.DefaultApacheHttpClientConfig
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.commons.httpclient.HttpClient
-import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
import org.apache.hadoop.registry.client.api.RegistryOperations
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.slider.agent.AgentMiniClusterTestBase
@@ -40,16 +32,15 @@
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.restclient.HttpOperationResponse
import org.apache.slider.server.appmaster.rpc.RpcBinder
+import org.apache.slider.test.KeysForTests
import org.junit.Test
-import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED
-import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL
+import static org.apache.slider.server.appmaster.management.MetricsKeys.*
import static org.apache.slider.server.appmaster.web.rest.RestPaths.*
@CompileStatic
@Slf4j
-class TestStandaloneREST extends AgentMiniClusterTestBase {
-
+class TestStandaloneREST extends AgentMiniClusterTestBase {
@Test
public void testStandaloneREST() throws Throwable {
@@ -74,21 +65,19 @@
ApplicationReport report = waitForClusterLive(client)
def proxyAM = report.trackingUrl
def directAM = report.originalTrackingUrl
-
-
+
// set up url config to match
initHttpTestSupport(launcher.configuration)
-
execOperation(WEB_STARTUP_TIME) {
GET(directAM)
}
-
+
execOperation(WEB_STARTUP_TIME) {
- def metrics = GET(directAM, SYSTEM_METRICS)
- log.info metrics
+ def metrics = GET(directAM, SYSTEM_METRICS_JSON)
+ log.info prettyPrintJson(metrics)
}
-
+
GET(proxyAM)
log.info GET(proxyAM, SYSTEM_PING)
@@ -96,35 +85,35 @@
log.info GET(proxyAM, SYSTEM_HEALTHCHECK)
log.info GET(proxyAM, SYSTEM_METRICS_JSON)
- /*
- Is the back door required? If so, don't test complex verbs via the proxy
- */
+ // using the metrics, await the first node status update.
+ // this should be from AM launch itself
+ awaitGaugeValue(proxyAM,
+ NODES_UPDATED_FLAG_METRIC,
+ 1,
+ WEB_STARTUP_TIME * 2, 500)
+
+ // Is the back door required? If so, don't test complex verbs via the proxy
def proxyComplexVerbs = !SliderXmlConfKeys.X_DEV_INSECURE_REQUIRED
- /*
- * Only do direct complex verbs if the no back door is needed, or if
- * it is enabled
- */
+ // Only do direct complex verbs if the no back door is needed, or if
+ // it is enabled
def directComplexVerbs = proxyComplexVerbs || SLIDER_CONFIG.getBoolean(
SliderXmlConfKeys.X_DEV_INSECURE_WS,
SliderXmlConfKeys.X_DEV_INSECURE_DEFAULT)
describe "Direct response headers from AM Web resources"
- def liveResUrl = appendToURL(directAM,
- SLIDER_PATH_APPLICATION, LIVE_RESOURCES);
+ def liveResUrl = appendToURL(directAM, SLIDER_PATH_APPLICATION, LIVE_RESOURCES);
HttpOperationResponse response = executeGet(liveResUrl)
response.headers.each { key, val -> log.info("$key $val") }
log.info "Content type: ${response.contentType}"
describe "proxied response headers from AM Web resources"
- response = executeGet(appendToURL(proxyAM,
- SLIDER_PATH_APPLICATION, LIVE_RESOURCES))
+ response = executeGet(appendToURL(proxyAM, SLIDER_PATH_APPLICATION, LIVE_RESOURCES))
response.headers.each { key, val -> log.info("$key $val") }
log.info "Content type: ${response.contentType}"
-
def ugiClient = createUGIJerseyClient();
-
+
describe "Proxy SliderRestClient Tests"
RestAPIClientTestDelegates proxySliderRestAPI =
new RestAPIClientTestDelegates(proxyAM, ugiClient, proxyComplexVerbs)
@@ -134,8 +123,7 @@
RestAPIClientTestDelegates directSliderRestAPI =
new RestAPIClientTestDelegates(directAM, ugiClient, directComplexVerbs)
directSliderRestAPI.testSuiteAll()
-
-
+
describe "Proxy Jersey Tests"
JerseyTestDelegates proxyJerseyTests =
new JerseyTestDelegates(proxyAM, ugiClient, proxyComplexVerbs)
@@ -143,14 +131,12 @@
describe "Direct Jersey Tests"
- JerseyTestDelegates directJerseyTests =
- new JerseyTestDelegates(directAM, ugiClient)
+ JerseyTestDelegates directJerseyTests = new JerseyTestDelegates(directAM, ugiClient)
directJerseyTests.testSuiteAll()
describe "Direct Tests"
- LowLevelRestTestDelegates direct =
- new LowLevelRestTestDelegates(directAM, directComplexVerbs)
+ LowLevelRestTestDelegates direct = new LowLevelRestTestDelegates(directAM, directComplexVerbs)
direct.testSuiteAll()
describe "Proxy Tests"
@@ -176,13 +162,10 @@
describe( "IPC equivalent operations")
def sliderClusterProtocol = RpcBinder.getProxy(conf, report, 1000)
- SliderApplicationIpcClient ipcClient =
- new SliderApplicationIpcClient(sliderClusterProtocol)
- IpcApiClientTestDelegates ipcDelegates =
- new IpcApiClientTestDelegates(ipcClient)
+ SliderApplicationIpcClient ipcClient = new SliderApplicationIpcClient(sliderClusterProtocol)
+ IpcApiClientTestDelegates ipcDelegates = new IpcApiClientTestDelegates(ipcClient)
ipcDelegates.testSuiteAll()
-
-
+
// log the metrics to show what's up
direct.logCodahaleMetrics();
@@ -193,23 +176,4 @@
}
}
- /**
- * Create Jersey client with URL handling by way
- * of the Apache HttpClient classes.
- * @return a Jersey client
- */
- public static Client createJerseyClientHttpClient() {
-
- def httpclient = new HttpClient(new MultiThreadedHttpConnectionManager());
- httpclient.httpConnectionManager.params.connectionTimeout = 10000;
- ClientConfig clientConfig = new DefaultApacheHttpClientConfig();
- clientConfig.features[JSONConfiguration.FEATURE_POJO_MAPPING] = Boolean.TRUE;
-
- def handler = new ApacheHttpClientHandler(httpclient, clientConfig);
-
- def client = new ApacheHttpClient(handler)
- client.followRedirects = true
- return client;
- }
-
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 5495e62..6637c73 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -169,9 +169,5 @@
Map<String, String> masterRole = cd.getRole(master)
assert masterRole != null, "Role hbase-master must exist"
assert cd.roleNames.contains(master), "Role names must contain hbase-master"
-
- // and check liveness
- assert cd.liveness.allRequestsSatisfied
- assert 0 == cd.liveness.requestsOutstanding
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
index 164f609..e3eb229 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMDestroy.groovy
@@ -25,12 +25,15 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.ActionEchoArgs
+import org.apache.slider.common.params.ActionDestroyArgs
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.exceptions.SliderException
import org.apache.slider.core.exceptions.UnknownApplicationInstanceException
+import org.apache.slider.core.exceptions.UsageException
+import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.core.main.ServiceLauncher
import org.junit.Test
@@ -113,13 +116,29 @@
describe "END EXPECTED WARNINGS"
describe "destroying $clustername"
- //now: destroy it
-
- int exitCode = sliderClient.actionDestroy(clustername);
+ //now: destroy it without the --force option and
+ //expect it to fail
+
+ def destroyEx = launchExpectingException(SliderClient,
+ configuration,
+ "",
+ [
+ SliderActions.ACTION_DESTROY,
+ clustername,
+ Arguments.ARG_FILESYSTEM, fsDefaultName,
+ Arguments.ARG_MANAGER, RMAddr,
+ ])
+ assert destroyEx instanceof UsageException
+
+ // destroy again but with --force option
+ describe "destroying $clustername --force"
+ ActionDestroyArgs destroyArgs = new ActionDestroyArgs()
+ destroyArgs.force = true
+ int exitCode = sliderClient.actionDestroy(clustername, destroyArgs);
assert 0 == exitCode
sleep(1000)
// twice, not expecting an error the second time
- exitCode = sliderClient.actionDestroy(clustername);
+ exitCode = sliderClient.actionDestroy(clustername, destroyArgs);
assert 0 == exitCode
describe "post destroy checks"
@@ -171,14 +190,14 @@
"Successful echo of a text document ${echoed.size()} characters long")
//try to destroy it while live
try {
- int ec = cluster2.actionDestroy(clustername)
+ int ec = cluster2.actionDestroy(clustername, destroyArgs)
fail("expected a failure from the destroy, got error code $ec")
} catch (SliderException e) {
assertFailureClusterInUse(e);
}
//and try to destroy a completely different cluster just for the fun of it
- assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name")
+ assert 0 == sliderClient.actionDestroy("no-cluster-of-this-name", destroyArgs)
maybeStopCluster(cluster2, "", "Teardown at end of test case", false);
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
index bdcf615..6947156 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAMRestart.groovy
@@ -20,10 +20,13 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+
import org.apache.hadoop.yarn.api.records.ApplicationReport
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ResourceKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.ActionAMSuicideArgs
@@ -94,6 +97,125 @@
assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
}
+
+ @Test
+ public void testStandaloneAMRestartWithRetryWindow() throws Throwable {
+ describe "kill a Standalone AM and verify that the AM failure count " +
+ "is reset after the AM retry-count-window elapses"
+ // patch the configuration for AM restart
+ YarnConfiguration conf = getRestartableConfiguration(5)
+
+ int restartLimit = 3;
+ int amRetryWindow = 60000;
+ String amRetryWindowStr = amRetryWindow.toString()
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_DEFINE,
+ SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=" + restartLimit,
+ Arguments.ARG_RESOURCE_OPT,
+ ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
+ amRetryWindowStr
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ logReport(report)
+ waitUntilClusterLive(sliderClient, 30000)
+
+ def diagnosticArgs = new ActionDiagnosticArgs()
+ diagnosticArgs.client = true
+ diagnosticArgs.yarn = true
+ sliderClient.actionDiagnostic(diagnosticArgs)
+
+ describe "kill AM #1"
+ int iteration = 1;
+ killAMAndWaitForRestart(sliderClient, iteration, clustername)
+
+ describe "kill AM #2"
+ killAMAndWaitForRestart(sliderClient, iteration++, clustername)
+
+ // app should be running here
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+ // make sure the am reset window has elapsed
+ describe "sleeping to ensure reset window elapsed"
+ sleep (amRetryWindow)
+
+ // kill again & expect the app to still be running
+ describe "kill AM #3 after window elapsed"
+ killAMAndWaitForRestart(sliderClient, iteration++, clustername)
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+ report = sliderClient.applicationReport
+ assert report.getYarnApplicationState() == YarnApplicationState.RUNNING
+
+ logReport(report)
+ describe("stopping the cluster")
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+
+ report = sliderClient.applicationReport
+ assert report.finalApplicationStatus == FinalApplicationStatus.KILLED
+ }
+
+
+ @Test
+ public void testStandaloneAMRestartWithDefaultRetryWindow() throws Throwable {
+ describe "kill AM more than the max limit allowed within the AM " +
+ "retry-count-window and expect the app to fail"
+ // patch the configuration for AM restart
+ YarnConfiguration conf = getRestartableConfiguration(5)
+
+ int restartLimit = 3;
+ String clustername = createMiniCluster("", conf, 1, true)
+ ServiceLauncher<SliderClient> launcher =
+ createStandaloneAMWithArgs(clustername,
+ [
+ Arguments.ARG_DEFINE,
+ SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=" + restartLimit,
+ ],
+ true,
+ false)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ ApplicationReport report = waitForClusterLive(sliderClient)
+ logReport(report)
+ waitUntilClusterLive(sliderClient, 30000)
+
+ def diagnosticArgs = new ActionDiagnosticArgs()
+ diagnosticArgs.client = true
+ diagnosticArgs.yarn = true
+ sliderClient.actionDiagnostic(diagnosticArgs)
+
+ describe "kill AM #1"
+ int iteration = 1;
+ killAMAndWaitForRestart(sliderClient, iteration, clustername)
+
+ describe "kill AM #2"
+ killAMAndWaitForRestart(sliderClient, iteration++, clustername)
+
+ // app should be running here
+ assert 0 == sliderClient.actionExists(clustername, true)
+
+ // kill again & expect the app to fail
+ describe "kill AM #3"
+ killAmAndWaitForDeath(sliderClient, iteration++, clustername)
+ sleep(40000)
+
+ report = sliderClient.applicationReport
+ assert report.finalApplicationStatus == FinalApplicationStatus.FAILED
+
+ logReport(report)
+ describe("stopping the cluster")
+ assert 0 == clusterActionFreeze(sliderClient, clustername, "force", true)
+ }
+
+
/**
* Kill an AM. take an iteration count for the message sent to the
* AM (hence its logs)
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
index 3707b85..eaf4386 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentAM.groovy
@@ -30,6 +30,7 @@
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.slider.common.params.ActionDestroyArgs
import org.apache.slider.common.tools.Duration
import org.apache.slider.core.build.InstanceBuilder
import org.apache.slider.core.conf.AggregateConf
@@ -192,7 +193,9 @@
assert instance3.yarnApplicationState >= YarnApplicationState.FINISHED
// destroy it
- client.actionDestroy(newcluster)
+ ActionDestroyArgs args = new ActionDestroyArgs()
+ args.force = true;
+ client.actionDestroy(newcluster, args)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
index 12736e3..5cac590 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestClientBadArgs.groovy
@@ -21,14 +21,12 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
-import org.apache.slider.common.params.ActionRegistryArgs
+import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.core.exceptions.BadCommandArgumentsException
import org.apache.slider.core.exceptions.ErrorStrings
import org.apache.slider.core.exceptions.UsageException
-import org.apache.slider.core.exceptions.BadConfigException
-import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.junit.Test
@@ -38,13 +36,11 @@
@CompileStatic
@Slf4j
class TestClientBadArgs extends ServiceLauncherBaseTest {
-
- static String TEST_FILES = "./src/test/resources/org/apache/slider/providers/agent/tests/"
-
+
@Test
public void testNoAction() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Usage: slider COMMAND",
[])
@@ -53,7 +49,7 @@
@Test
public void testUnknownAction() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"not-a-known-action",
["not-a-known-action"])
}
@@ -61,7 +57,7 @@
@Test
public void testActionWithoutOptions() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Usage: slider build <application>",
[SliderActions.ACTION_BUILD])
}
@@ -69,7 +65,7 @@
@Test
public void testActionWithoutEnoughArgs() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS,
[SliderActions.ACTION_THAW])
}
@@ -77,7 +73,7 @@
@Test
public void testActionWithTooManyArgs() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
ErrorStrings.ERROR_TOO_MANY_ARGUMENTS,
[SliderActions.ACTION_HELP,
"hello, world"])
@@ -86,7 +82,7 @@
@Test
public void testBadImageArg() throws Throwable {
launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Unknown option: --image",
[SliderActions.ACTION_HELP,
Arguments.ARG_IMAGE])
@@ -95,7 +91,7 @@
@Test
public void testRegistryUsage() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"org.apache.slider.core.exceptions.UsageException: Argument --name missing",
[SliderActions.ACTION_REGISTRY])
assert exception instanceof UsageException
@@ -105,7 +101,7 @@
@Test
public void testRegistryExportBadUsage1() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Expected a value after parameter --getexp",
[SliderActions.ACTION_REGISTRY,
Arguments.ARG_NAME,
@@ -118,7 +114,7 @@
@Test
public void testRegistryExportBadUsage2() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Expected a value after parameter --getexp",
[SliderActions.ACTION_REGISTRY,
Arguments.ARG_NAME,
@@ -132,7 +128,7 @@
@Test
public void testRegistryExportBadUsage3() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Usage: registry",
[SliderActions.ACTION_REGISTRY,
Arguments.ARG_NAME,
@@ -147,7 +143,7 @@
@Test
public void testUpgradeUsage() throws Throwable {
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"org.apache.slider.core.exceptions.BadCommandArgumentsException: Not enough arguments for action: upgrade Expected minimum 1 but got 0",
[SliderActions.ACTION_UPGRADE])
assert exception instanceof BadCommandArgumentsException
@@ -158,7 +154,7 @@
public void testUpgradeWithTemplateOptionOnly() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --resources must be specified with option --template",
[SliderActions.ACTION_UPGRADE,
appName,
@@ -169,11 +165,17 @@
log.info(exception.toString())
}
+ public Configuration createTestConfig() {
+ def configuration = new Configuration()
+ configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032")
+ return configuration
+ }
+
@Test
public void testUpgradeWithResourcesOptionOnly() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --template must be specified with option --resources",
[SliderActions.ACTION_UPGRADE,
appName,
@@ -188,7 +190,7 @@
public void testUpgradeWithTemplateResourcesAndContainersOption() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --containers cannot be "
+ "specified with --template or --resources",
[SliderActions.ACTION_UPGRADE,
@@ -208,7 +210,7 @@
public void testUpgradeWithTemplateResourcesAndComponentsOption() throws Throwable {
String appName = "test_hbase"
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"BadCommandArgumentsException: Option --components cannot be "
+ "specified with --template or --resources",
[SliderActions.ACTION_UPGRADE,
@@ -228,7 +230,7 @@
public void testCreateAppWithAddonPkgBadArg1() throws Throwable {
//add on package without specifying add on package name
def exception = launchExpectingException(SliderClient,
- new Configuration(),
+ createTestConfig(),
"Expected 2 values after --addon",
[SliderActions.ACTION_CREATE,
"cl1",
@@ -237,4 +239,27 @@
assert exception instanceof BadCommandArgumentsException
log.info(exception.toString())
}
+
+ @Test
+ public void testNodesMissingFile() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ createTestConfig(),
+ "after parameter --out",
+ [SliderActions.ACTION_NODES, Arguments.ARG_OUTPUT])
+ assert exception instanceof BadCommandArgumentsException
+ }
+
+ @Test
+ public void testFlexWithNoComponents() throws Throwable {
+ def exception = launchExpectingException(SliderClient,
+ new Configuration(),
+ "Usage: slider flex <application>",
+ [
+ SliderActions.ACTION_FLEX,
+ "flex1",
+ Arguments.ARG_DEFINE, YarnConfiguration.RM_ADDRESS + "=127.0.0.1:8032"
+ ])
+ assert exception instanceof UsageException
+ log.info(exception.toString())
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
index a6d8538..72e7565 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -23,7 +23,6 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
import org.apache.slider.api.RoleKeys
@@ -132,7 +131,7 @@
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
SliderUtils.verifyPrincipalSet(
conf,
- DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
+ SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
}
@@ -150,7 +149,7 @@
assert ca.clusterName == CLUSTERNAME
assert conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH) == "/projects/slider/clusters"
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
+ SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
index 168415c..e244923 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestSliderClientMethods.groovy
@@ -30,6 +30,7 @@
import org.apache.slider.core.launch.LaunchedApplication
import org.apache.slider.core.main.ServiceLauncherBaseTest
import org.apache.slider.core.persist.LockAcquireFailedException
+import org.apache.slider.server.appmaster.model.mock.MockApplicationId
import org.easymock.EasyMock
import org.junit.Assert
import org.junit.Test
@@ -137,7 +138,7 @@
AggregateConf instanceDefinition,
boolean debugAM)
throws YarnException, IOException {
- return new LaunchedApplication(clustername, new SliderYarnClientImpl());
+ return new LaunchedApplication(new MockApplicationId(1), new SliderYarnClientImpl());
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy
index 8d5aef1..5295f9b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestUpgradeCommandOptions.groovy
@@ -82,10 +82,8 @@
new File(tmpDir, "resources.json").toURI()
])
fail("Upgrade command should have failed")
- } catch (SliderException e) {
- log.info(e.toString())
- assert e instanceof UnknownApplicationInstanceException
- assert e.getMessage().contains("Unknown application instance")
+ } catch (UnknownApplicationInstanceException e) {
+ assertExceptionDetails(e, SliderExitCodes.EXIT_UNKNOWN_INSTANCE, "Unknown application instance")
}
}
@@ -301,10 +299,6 @@
assert launcher.serviceExitCode == 0
}
- private File getTempLocation () {
- return new File(System.getProperty("user.dir") + "/target/_")
- }
-
static class TestSliderClient extends SliderClient {
public TestSliderClient() {
super()
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
index f71257a..1ea9001 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
@@ -41,6 +41,7 @@
// as the static compiler doesn't resolve consistently
public static final String USER = KeysForTests.USERNAME
+ public static final int CONNECT_TIMEOUT = 5000
private ZKIntegration zki
@Before
@@ -61,7 +62,7 @@
public ZKIntegration initZKI() {
zki = createZKIntegrationInstance(
- getZKBinding(), methodName.methodName, true, false, 5000)
+ getZKBinding(), methodName.methodName, true, false, CONNECT_TIMEOUT)
return zki
}
@@ -114,7 +115,7 @@
assert zkPath == "/services/slider/users/" + USER + "/cl1", "zkPath must be as expected"
assert path == zkPath
assert zki == null, "ZKIntegration should be null."
- zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, 5000);
+ zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, CONNECT_TIMEOUT);
assert !zki.exists(zkPath)
path = client.createZookeeperNodeInner("cl1", false)
@@ -145,7 +146,7 @@
@Override
protected ZKIntegration getZkClient(String clusterName, String user) {
- zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, false, 5000)
+ zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, false, CONNECT_TIMEOUT)
return zki;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
index 5bf1c1f..e5951ac 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy
@@ -32,6 +32,7 @@
import org.junit.Before
import org.junit.Rule
import org.junit.rules.TemporaryFolder
+import org.slf4j.Logger
import static org.apache.slider.common.SliderXMLConfKeysForTesting.*
import static org.apache.slider.providers.agent.AgentKeys.CONF_RESOURCE
@@ -53,7 +54,7 @@
*/
public static void assumeValidServerEnv() {
try {
- SliderUtils.validateSliderServerEnvironment(log, true)
+ SliderUtils.validateSliderServerEnvironment(log as Logger, true)
} catch (Exception e) {
skip(e.toString())
}
@@ -135,12 +136,8 @@
boolean create,
boolean blockUntilRunning) {
-
YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
create ? SliderActions.ACTION_CREATE : SliderActions.ACTION_BUILD,
@@ -166,11 +163,7 @@
List<String> extraArgs,
boolean deleteExistingData) {
- YarnConfiguration conf = testConfiguration
-
- def clusterOps = [
- :
- ]
+ def clusterOps = [:]
return createOrBuildCluster(
SliderActions.ACTION_UPDATE,
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
new file mode 100644
index 0000000..855ed36
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/DemoAgentAAEcho.groovy
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import org.apache.slider.client.SliderClient
+
+/**
+ * Overridde the test actions with a sleep command, so that developers
+ * can see what the application is up to
+ */
+class DemoAgentAAEcho extends TestAgentAAEcho {
+
+ @Override
+ protected void postLaunchActions(
+ SliderClient sliderClient,
+ String clustername,
+ String rolename,
+ Map<String, Integer> roles,
+ String proxyAM) {
+
+ def url = proxyAM
+ // spin repeating the URl in the logs so YARN chatter doesn't lose it
+ describe("Web UI is at $url")
+
+ // run the superclass rest tests
+ // queryRestAPI(sliderClient, roles, proxyAM)
+
+ 5.times {
+ describe("Web UI is at $url")
+ sleep(60 *1000)
+ }
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
new file mode 100644
index 0000000..e193563
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAAEcho.groovy
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.client.rest.SliderApplicationApiRestClient
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionNodesArgs
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.server.appmaster.management.MetricsConstants
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+import static org.apache.slider.providers.agent.AgentKeys.*
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_ENABLED
+import static org.apache.slider.server.appmaster.management.MetricsKeys.METRICS_LOGGING_LOG_INTERVAL
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_METRICS_JSON
+
+/**
+ * Tests an echo command
+ */
+@CompileStatic
+@Slf4j
+class TestAgentAAEcho extends TestAgentEcho {
+
+ @Test
+ public void testAgentEcho() throws Throwable {
+ assumeValidServerEnv()
+ def conf = configuration
+ conf.setBoolean(METRICS_LOGGING_ENABLED, true)
+ conf.setInt(METRICS_LOGGING_LOG_INTERVAL, 1)
+ String clustername = createMiniCluster("testaaecho",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ validatePaths()
+
+ def echo = "echo"
+ Map<String, Integer> roles = buildRoleMap(echo)
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+ roles,
+ [
+ ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath,
+ ARG_OPTION, APP_DEF, toURIArg(app_def_path),
+ ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path),
+ ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path),
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ "" + PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ ARG_COMP_OPT, echo, SCRIPT_PATH, echo_py,
+ ARG_COMP_OPT, echo, SERVICE_NAME, "Agent",
+ ARG_DEFINE, SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false",
+ ARG_COMP_OPT, echo, TEST_RELAX_VERIFICATION, "true",
+ ],
+ true, true,
+ true)
+ SliderClient sliderClient = launcher.service
+ initHttpTestSupport(sliderClient.config)
+ def applicationReport = sliderClient.applicationReport
+ def proxyAM = applicationReport.trackingUrl
+ try {
+ postLaunchActions(sliderClient, clustername, echo, roles, proxyAM)
+ } catch (Exception ex) {
+ logMetricsQuietly(proxyAM)
+ throw ex;
+ }
+ }
+
+ /**
+ * retrieve cluster metrics and log quietly
+ * @param proxyAM
+ */
+ protected void logMetricsQuietly(String proxyAM) {
+ try {
+ log.error prettyPrintAsJson(GET(proxyAM, SYSTEM_METRICS_JSON));
+ } catch (Exception ex) {
+ log.warn("failed to get AM", ex)
+ }
+ }
+
+ /**
+ * Build the role map to use when creating teh cluster
+ * @param roleName the name used for the echo role
+ * @return the map
+ */
+ protected Map<String, Integer> buildRoleMap(String roleName) {
+ [
+ (roleName): 3,
+ ];
+ }
+
+ /**
+ * Any actions to perform after starting the agent cluster.
+ * HTTP client operations will have been set up already.
+ * @param sliderClient client for the cluster
+ * @param clustername cluster name
+ * @param rolename name of the echo role
+ * @param roles original set of roles
+ * @param proxyAM URl to proxy AM.
+ */
+ protected void postLaunchActions(
+ SliderClient sliderClient,
+ String clustername,
+ String rolename,
+ Map<String, Integer> roles,
+ String proxyAM) {
+ def onlyOneEcho = [(rolename): 1]
+ def requested = roles[rolename]
+
+ waitForRoleCount(sliderClient, onlyOneEcho, AGENT_CLUSTER_STARTUP_TIME)
+ //sleep a bit
+ sleep(5000)
+ //expect the role count to be the same
+ waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+ def cd = sliderClient.getClusterDescription()
+ assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == requested - 1;
+ assert !cd.liveness.allRequestsSatisfied
+ assert cd.liveness.requestsOutstanding == requested - 1
+ def ipcClient = sliderClient.createIpcClient()
+
+ def echoInstances = sliderClient.listNodeUUIDsByRole(rolename)
+ queryRestAPI(sliderClient, roles, proxyAM)
+ // flex size
+ // while running, ask for many more, expect them to still be outstanding
+ sleep(5000)
+
+ requested = 50
+ def expectedPending = requested - 1
+
+ sliderClient.flex(clustername, [(rolename): requested]);
+ waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+ sleep(4000)
+ def now = System.currentTimeMillis();
+ sleep(1000)
+
+ def componentInformation = ipcClient.getComponent(rolename)
+ assert !ipcClient.getComponent(rolename).isAARequestOutstanding
+
+ assert componentInformation.pendingAntiAffineRequestCount == expectedPending
+
+ cd = sliderClient.getClusterDescription()
+ assert !cd.liveness.allRequestsSatisfied
+ assert cd.liveness.requestsOutstanding == requested - 1
+ assert now <= Long.valueOf(cd.info.get(StatusKeys.INFO_STATUS_TIME_MILLIS))
+ assert expectedPending == cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1)
+
+ // while running, flex it to size = 1
+ sleep(1000)
+ sliderClient.flex(clustername, onlyOneEcho);
+ waitForRoleCount(sliderClient, onlyOneEcho, 1000)
+
+ def echoInstances2 = sliderClient.listNodeUUIDsByRole(rolename)
+ assertArrayEquals(echoInstances, echoInstances2)
+
+ assert !ipcClient.getComponent(rolename).isAARequestOutstanding
+ cd = sliderClient.getClusterDescription()
+ assert cd.liveness.allRequestsSatisfied
+
+
+ assert cd.getRoleOptInt(rolename, RoleKeys.ROLE_PENDING_AA_INSTANCES, -1) == 0;
+
+ def nodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs())
+ assert nodes.size() == 1
+ def activeNodes = sliderClient.listInstanceNodes(clustername, new ActionNodesArgs())
+ assert activeNodes[0].entries[rolename] && activeNodes[0].entries[rolename].live == 1
+ }
+
+ protected void queryRestAPI(SliderClient sliderClient, Map<String, Integer> roles, String proxyAM) {
+ GET(proxyAM)
+ describe "Proxy SliderRestClient Tests"
+ SliderApplicationApiRestClient restAPI =
+ new SliderApplicationApiRestClient(createUGIJerseyClient(), proxyAM)
+ awaitGaugeValue(proxyAM,
+ MetricsConstants.PREFIX_SLIDER_ROLES + "echo.pendingAntiAffineRequests",
+ 2,
+ WEB_STARTUP_TIME * 2, 500)
+
+ def echoInfo = restAPI.getComponent(ECHO)
+ assert echoInfo.pendingAntiAffineRequestCount == 2
+ // no active requests ... there's no capacity
+ assert !echoInfo.isAARequestOutstanding
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 973114d..23a7bbb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -22,11 +22,14 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.ActionNodesArgs
import org.apache.slider.core.exceptions.BadClusterStateException
import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.core.persist.JsonSerDeser
import org.junit.Before
import org.junit.Test
@@ -40,6 +43,7 @@
@Slf4j
class TestAgentEcho extends AgentTestBase {
+ protected static final String ECHO = "echo"
File slider_core
String echo_py
File echo_py_path
@@ -59,7 +63,6 @@
agt_ver_path = new File(slider_core, agt_ver)
agt_conf = "agent.ini"
agt_conf_path = new File(slider_core, agt_conf)
-
}
@Override
@@ -68,7 +71,7 @@
}
@Test
- public void testEchoOperation() throws Throwable {
+ public void testAgentEcho() throws Throwable {
assumeValidServerEnv()
String clustername = createMiniCluster("",
@@ -79,12 +82,9 @@
true,
false)
- assert echo_py_path.exists()
- assert app_def_path.exists()
- assert agt_ver_path.exists()
- assert agt_conf_path.exists()
+ validatePaths()
- def role = "echo"
+ def role = ECHO
Map<String, Integer> roles = [
(role): 2,
];
@@ -127,5 +127,41 @@
assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_STATE, "negative")
}
+
+ runNodemapTests(sliderClient)
+
+ }
+
+ /**
+ * do some nodemap checks, currently cluster-wide
+ * @param sliderClient
+ */
+ protected void runNodemapTests(SliderClient sliderClient) {
+ describe "slider nodes"
+ sliderClient.actionNodes("", new ActionNodesArgs())
+
+ def allNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs()).collect { it.httpAddress }
+ assert !allNodes.empty
+
+ // healthy only
+ def healthyNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs(healthy: true))
+ assert healthyNodes.collect { it.httpAddress }.containsAll(allNodes)
+ // look for an unknown label and expect none
+ def gpuNodes = sliderClient.listYarnClusterNodes(new ActionNodesArgs(label: "gpu"))
+ assert gpuNodes.empty
+ File t1 = createTempJsonFile()
+ sliderClient.actionNodes("", new ActionNodesArgs(outputFile: t1))
+ assert t1.exists()
+ JsonSerDeser<NodeInformationList> serDeser = new JsonSerDeser<>(NodeInformationList.class);
+ NodeInformationList loaded = serDeser.fromFile(t1)
+ assert allNodes.containsAll(loaded.collect { it.httpAddress })
+
+ }
+
+ protected void validatePaths() {
+ assert echo_py_path.exists()
+ assert app_def_path.exists()
+ assert agt_ver_path.exists()
+ assert agt_conf_path.exists()
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 264d260..60e9035 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -94,7 +94,9 @@
1,
true,
false)
- buildAgentCluster("test_build_basic_agent_node_only",
+
+ def cluster01 = clustername + "_01"
+ buildAgentCluster(cluster01,
[(ROLE_NODE): 1],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -110,7 +112,9 @@
def master = "hbase-master"
def rs = "hbase-rs"
- ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
+
+ def cluster02 = clustername + "_02"
+ ServiceLauncher<SliderClient> launcher = buildAgentCluster(cluster02,
[
(ROLE_NODE): 1,
(master): 1,
@@ -149,10 +153,11 @@
def rscomponent = resource.getMandatoryComponent(rs)
assert "5" == rscomponent.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
- // now create an instance with no role priority for the newnode role
+ describe "build a cluster with no role priority for the newnode role"
+
try {
- def name2 = clustername + "-2"
- buildAgentCluster(name2,
+ def cluster03 = clustername + "_03"
+ buildAgentCluster(cluster03,
[
(ROLE_NODE): 2,
"role3": 1,
@@ -166,14 +171,16 @@
],
true, false,
false)
- failWithBuildSucceeding(name2, "no priority for one role")
+ failWithBuildSucceeding(cluster03, "no priority for one role")
} catch (BadConfigException expected) {
}
+ describe "build a cluster with the number of agents out of range"
try {
- launcher = buildAgentCluster(clustername + "-10",
+ def cluster04 = clustername + "_04"
+ launcher = buildAgentCluster(cluster04,
[
- (ROLE_NODE): 4,
+ (ROLE_NODE): 2000,
],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
@@ -185,15 +192,18 @@
],
true, false,
false)
- failWithBuildSucceeding(ROLE_NODE, "too many instances")
+ failWithBuildSucceeding(cluster04, "too many instances")
} catch (BadConfigException expected) {
- assert expected.message.contains("Expected minimum is 1 and maximum is 2")
- assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "Expected minimum is 1 and maximum is")
+ assertExceptionDetails(expected, SliderExitCodes.EXIT_BAD_CONFIGURATION,
+ "out of range.")
}
- //duplicate priorities
+
+ describe "build a cluster with duplicate priorities for roles"
try {
- def name3 = clustername + "-3"
- buildAgentCluster(name3,
+ def cluster05 = clustername + "_05"
+ buildAgentCluster(cluster05,
[
(ROLE_NODE): 5,
(master): 1,
@@ -206,16 +216,16 @@
true, false,
false)
- failWithBuildSucceeding(name3, "duplicate priorities")
+ failWithBuildSucceeding(cluster05, "duplicate priorities")
} catch (BadConfigException expected) {
}
- def cluster4 = clustername + "-4"
+ def cluster06 = clustername + "_06"
def jvmopts = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=5005"
- buildAgentCluster(cluster4,
+ buildAgentCluster(cluster06,
[
(master): 1,
(rs): 5
@@ -235,7 +245,7 @@
false)
//now we want to look at the value
- AggregateConf instanceDefinition = loadInstanceDefinition(cluster4)
+ AggregateConf instanceDefinition = loadInstanceDefinition(cluster06)
def opt = instanceDefinition.getAppConfOperations().getComponentOpt(
SliderKeys.COMPONENT_AM,
RoleKeys.JVM_OPTS,
@@ -245,8 +255,8 @@
// now create an instance with no component options, hence no
// entry in the app config
- def name5 = clustername + "-5"
- buildAgentCluster(name5,
+ def name07 = clustername + "_07"
+ buildAgentCluster(name07,
[
"hbase-rs": 1,
],
@@ -403,9 +413,8 @@
def rscomponent2 = resource2.getMandatoryComponent(rs)
assert "6" == rscomponent2.getMandatoryOption(ResourceKeys.COMPONENT_INSTANCES)
}
-
+
public AggregateConf loadInstanceDefinition(String name) {
- def cluster4
def sliderFS = createSliderFileSystem()
def dirPath = sliderFS.buildClusterDirPath(name)
ConfPersister persister = new ConfPersister(sliderFS, dirPath)
@@ -697,10 +706,8 @@
}
public void failWithBuildSucceeding(String name, String reason) {
- def badArgs1
AggregateConf instanceDefinition = loadInstanceDefinition(name)
- log.error(
- "Build operation should have failed from $reason : \n$instanceDefinition")
+ log.error("Build operation should have failed from $reason : \n$instanceDefinition")
fail("Build operation should have failed from $reason")
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
new file mode 100644
index 0000000..7b3a65d
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.groovy
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
+import org.apache.slider.server.appmaster.state.RoleStatus
+
+/**
+ * class for basis of Anti-affine placement tests; sets up role2
+ * for anti-affinity
+ */
+@CompileStatic
+@Slf4j
+class BaseMockAppStateAATest extends BaseMockAppStateTest
+ implements MockRoles {
+
+ /** Role status for the base AA role */
+ RoleStatus aaRole
+
+ /** Role status for the AA role requiring a node with the gpu label */
+ RoleStatus gpuRole
+
+ @Override
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.roles = [
+ MockFactory.PROVIDER_ROLE0,
+ MockFactory.AAROLE_1_GPU,
+ MockFactory.AAROLE_2,
+ ]
+ bindingInfo
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ aaRole = lookupRole(MockFactory.AAROLE_2.name)
+ gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name)
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
new file mode 100644
index 0000000..a8c50d1
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAOvercapacity.groovy
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement with a cluster of size 1
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateAAOvercapacity extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 1
+
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 1)
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(aaRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testOvercapacityRecovery() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ "verify the state. kill the allocated container and review")
+ //more than expected
+ long desired = 3
+ aaRole.desired = desired
+ assert appState.roleHistory.canPlaceAANodes()
+
+ //first request
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert desired - 1 == aaRole.pendingAntiAffineRequests
+ List<AbstractRMOperation > operationsOut = []
+ // allocate and re-submit
+ def instances = submitOperations(operations, [], operationsOut)
+ assert 1 == instances.size()
+ assertAllContainersAA()
+
+ // expect an outstanding AA request to be unsatisfied
+ assert aaRole.actual < aaRole.desired
+ assert !aaRole.requested
+ assert !aaRole.AARequestOutstanding
+ assert desired - 1 == aaRole.pendingAntiAffineRequests
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+
+ // now lets trigger a failure
+ def nodemap = cloneNodemap()
+ assert nodemap.size() == 1
+
+ def instance = instances[0]
+ def cid = instance.containerId
+
+ AppState.NodeCompletionResult result = appState.onCompletedNode(containerStatus(cid,
+ LauncherExitCodes.EXIT_TASK_LAUNCH_FAILURE))
+ assert result.containerFailed
+
+ assert aaRole.failed == 1
+ assert aaRole.actual == 0
+ def availablePlacements = appState.getRoleHistory().findNodeForNewAAInstance(aaRole)
+ assert availablePlacements.size() == 1
+ describe "expecting a successful review with available placements of $availablePlacements"
+ operations = appState.reviewRequestAndReleaseNodes()
+ assert operations.size() == 1
+ }
+
+ }
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
new file mode 100644
index 0000000..e43d894
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -0,0 +1,340 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.providers.PlacementPolicy
+import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 3
+
+ /**
+ * The YARN engine has a cluster with very few nodes (3) and lots of containers, so
+ * if AA placement isn't working, there will be affine placements surfacing.
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 8)
+ }
+
+ /**
+ * This is the simplest AA allocation: no lables, so allocate anywhere
+ * @throws Throwable
+ */
+ @Test
+ public void testAllocateAANoLabel() throws Throwable {
+ assert cloneNodemap().size() > 0
+
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 2
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ AMRMClient.ContainerRequest request = getSingleRequest(ops)
+ assert !request.relaxLocality
+ assert request.nodes.size() == engine.cluster.clusterSize
+ assert request.racks == null
+ assert request.capability
+
+ Container allocated = engine.allocateContainer(request)
+
+ // notify the container ane expect
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> operations = []
+ appState.onContainersAllocated([allocated], assignments, operations)
+
+ def host = allocated.nodeId.host
+ def hostInstance = cloneNodemap().get(host)
+ assert hostInstance.get(aaRole.key).starting == 1
+ assert !hostInstance.canHost(aaRole.key, "")
+ assert !hostInstance.canHost(aaRole.key, null)
+
+ // assignment
+ assert assignments.size() == 1
+
+ // verify the release matches the allocation
+ assert operations.size() == 2
+ assert getCancel(operations, 0).capability.equals(allocated.resource)
+
+ // we also expect a new allocation request to have been issued
+
+ def req2 = getRequest(operations, 1)
+ assert req2.nodes.size() == engine.cluster.clusterSize - 1
+
+ assert !req2.nodes.contains(host)
+ assert !request.relaxLocality
+
+ // verify the pending couner is down
+ assert 0L == aaRole.pendingAntiAffineRequests
+ Container allocated2 = engine.allocateContainer(req2)
+
+ // placement must be on a different host
+ assert allocated2.nodeId != allocated.nodeId
+
+ ContainerAssignment assigned = assignments[0]
+ Container container = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+ //tell the app it arrived
+ appState.containerStartSubmitted(container, ri);
+ assert appState.onNodeManagerContainerStarted(container.id)
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.size() == 0
+ assertAllContainersAA();
+
+ // identify those hosts with an aa role on
+ def naming = appState.buildNamingMap()
+ assert naming.size() == 3
+
+ def name = aaRole.name
+ assert name == naming[aaRole.key]
+ def info = appState.roleHistory.getNodeInformationSnapshot(naming);
+ assert info
+
+ def nodeInformation = info[host]
+ assert nodeInformation
+ assert nodeInformation.entries
+ assert nodeInformation.entries[name]
+ assert nodeInformation.entries[name].live
+ }
+
+ @Test
+ public void testAllocateFlexUp() throws Throwable {
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 2
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ getSingleRequest(ops)
+ assert aaRole.requested == 1
+ assert aaRole.pendingAntiAffineRequests == 1
+ assert aaRole.actualAndRequested + aaRole.pendingAntiAffineRequests == aaRole.desired
+
+ // now trigger that flex up
+ aaRole.desired = 3
+
+ // expect: no new reqests, pending count ++
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ assert aaRole.actual + aaRole.pendingAntiAffineRequests + aaRole.outstandingAARequestCount ==
+ aaRole.desired
+
+ // 1 outstanding
+ assert aaRole.actual == 0
+ assert aaRole.AARequestOutstanding
+ // and one AA
+ assert aaRole.pendingAntiAffineRequests == 2
+ assertAllContainersAA()
+
+ // next iter
+ assert 1 == submitOperations(ops, [], ops2).size()
+ assert 2 == ops2.size()
+ assert aaRole.pendingAntiAffineRequests == 1
+ assertAllContainersAA()
+
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
+ // now trigger the next execution cycle
+ List<AbstractRMOperation> ops3 = []
+ assert 1 == submitOperations(ops2, [], ops3).size()
+ assert 2 == ops3.size()
+ assert aaRole.pendingAntiAffineRequests == 0
+ assertAllContainersAA()
+
+ }
+
+ @Test
+ public void testAllocateFlexDownDecrementsPending() throws Throwable {
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 2
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ getSingleRequest(ops)
+ assert aaRole.pendingAntiAffineRequests == 1
+ assert aaRole.AARequestOutstanding
+
+ // flex down so that the next request should be cancelled
+ aaRole.desired = 1
+
+ // expect: no new requests, pending count --
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ assert aaRole.AARequestOutstanding
+ assert aaRole.pendingAntiAffineRequests == 0
+ assertAllContainersAA()
+
+ // next iter
+ submitOperations(ops, [], ops2).size()
+ assert 1 == ops2.size()
+ assertAllContainersAA()
+ }
+
+ /**
+ * Here flex down while there is only one outstanding request.
+ * The outstanding flex should be cancelled
+ * @throws Throwable
+ */
+ @Test
+ public void testAllocateFlexDownForcesCancel() throws Throwable {
+ // want multiple instances, so there will be iterations
+ aaRole.desired = 1
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ getSingleRequest(ops)
+ assert aaRole.pendingAntiAffineRequests == 0
+ assert aaRole.AARequestOutstanding
+
+ // flex down so that the next request should be cancelled
+ aaRole.desired = 0
+ // expect: no new reqests, pending count --
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.pendingAntiAffineRequests == 0
+ assert !aaRole.AARequestOutstanding
+ assert ops2.size() == 1
+ getSingleCancel(ops2)
+
+ // next iter
+ submitOperations(ops, [], ops2).size()
+ assert 1 == ops2.size()
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(aaRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testAskForTooMany() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ " expect the final request to be unsatisfied until the cluster changes size")
+ //more than expected
+ aaRole.desired = NODES + 1
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert NODES == aaRole.pendingAntiAffineRequests
+ for (int i = 0; i < NODES; i++) {
+ def iter = "Iteration $i role = $aaRole"
+ log.info(iter)
+ List<AbstractRMOperation > operationsOut = []
+ assert 1 == submitOperations(operations, [], operationsOut).size(), iter
+ operations = operationsOut
+ if (i + 1 < NODES) {
+ assert operations.size() == 2
+ } else {
+ assert operations.size() == 1
+ }
+ assertAllContainersAA()
+ }
+ // expect an outstanding AA request to be unsatisfied
+ assert aaRole.actual < aaRole.desired
+ assert !aaRole.requested
+ assert !aaRole.AARequestOutstanding
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+ // in a review now, no more requests can be generated, as there is no space for AA placements,
+ // even though there is cluster capacity
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
+
+ // now do a node update (this doesn't touch the YARN engine; the node isn't really there)
+ def outcome = addNewNode()
+ assert cloneNodemap().size() == NODES + 1
+ assert outcome.clusterChanged
+ // no active calls to empty
+ assert outcome.operations.empty
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+ protected AppState.NodeUpdatedOutcome addNewNode() {
+ updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu"))
+ }
+
+ @Test
+ public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
+ describe("Change the cluster size where the cluster size changes during a test sequence.")
+ aaRole.desired = NODES + 1
+ appState.reviewRequestAndReleaseNodes()
+ assert aaRole.AARequestOutstanding
+ assert NODES == aaRole.pendingAntiAffineRequests
+ def outcome = addNewNode()
+ assert outcome.clusterChanged
+ // one call to cancel
+ assert 1 == outcome.operations.size()
+ // and on a review, one more to rebuild
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+ @Test
+ public void testBindingInfoMustHaveNodeMap() throws Throwable {
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.nodeReports = null;
+ try {
+ def state = new MockAppState(bindingInfo)
+ fail("Expected an exception, got $state")
+ } catch (IllegalArgumentException expected) {
+ }
+ }
+
+ @Test
+ public void testAMRestart() throws Throwable {
+ def desiredAA = 3
+ aaRole.desired = desiredAA
+ List<RoleInstance> instances = createAndStartNodes()
+ List<Container> containers = instances.collect { it.container }
+
+ // now destroy the app state
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.instanceDefinition = factory.newInstanceDefinition(0, 0, desiredAA)
+ ConfTreeOperations cto = new ConfTreeOperations(bindingInfo.instanceDefinition.resources)
+ cto.setComponentOpt(ROLE2,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED)
+ bindingInfo.liveContainers = containers
+ appState = new MockAppState(bindingInfo)
+
+ def aaRole = lookupRole(MockFactory.AAROLE_2.name)
+ def gpuRole = lookupRole(MockFactory.AAROLE_1_GPU.name)
+ appState.reviewRequestAndReleaseNodes()
+ assert aaRole.antiAffinePlacement
+ assert aaRole.AARequestOutstanding
+
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
index 7838886..6739623 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy
@@ -41,16 +41,10 @@
@Slf4j
class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements MockRoles {
- @Override
- String getTestName() {
- return "TestMockAppStateAppRestIntegration"
- }
-
@Test
public void testCachedIntDocument() throws Throwable {
ContentCache cache = new ContentCache()
-
def refresher = new IntRefresher()
assert 0 == refresher.count
def entry = new CachedContentManagedTimer(refresher)
@@ -103,8 +97,8 @@
new ApplicationResource(webAppApi)
def containers = applicationResource.liveContainers
assert containers.size() == instances.size()
-
}
+
/**
* Get a state accessor for the appState field
* @return something to hand down to refreshers and resources
@@ -146,7 +140,7 @@
class CachedContentManagedTimer extends CachedContent {
int time = 0;
-
+
@Override
protected long now() {
return time++;
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
index 1b79115..3235827 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateContainerFailure.groovy
@@ -58,7 +58,7 @@
*/
@Override
MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(8000, 4)
+ return new MockYarnEngine(4, 8000)
}
@Override
@@ -95,7 +95,7 @@
//view the world
appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ List<NodeInstance> queue = appState.roleHistory.cloneRecentNodeList(0)
assert queue.size() == 0
}
@@ -123,7 +123,7 @@
//view the world
appState.getRoleHistory().dump();
- List<NodeInstance> queue = appState.roleHistory.cloneAvailableList(0)
+ List<NodeInstance> queue = appState.roleHistory.cloneRecentNodeList(0)
assert queue.size() == 1
}
@@ -148,7 +148,7 @@
RoleHistory history = appState.roleHistory
history.dump();
- List<NodeInstance> queue = history.cloneAvailableList(0)
+ List<NodeInstance> queue = history.cloneRecentNodeList(0)
assert queue.size() == 0
NodeInstance ni = history.getOrCreateNodeInstance(instance.container)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
index aa7bb11..e57f341 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicHistory.groovy
@@ -20,7 +20,6 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
import org.apache.slider.core.conf.ConfTreeOperations
@@ -28,7 +27,6 @@
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
@@ -36,7 +34,7 @@
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleInstance
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
/**
@@ -47,11 +45,6 @@
class TestMockAppStateDynamicHistory extends BaseMockAppStateTest
implements MockRoles {
- @Override
- String getTestName() {
- return "TestMockAppStateDynamicHistory"
- }
-
/**
* Small cluster with multiple containers per node,
* to guarantee many container allocations on each node
@@ -62,26 +55,6 @@
return new MockYarnEngine(8, 1)
}
- @Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
-
- def instance = factory.newInstanceDefinition(0,0,0)
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null, new SimpleReleaseSelector())
- }
-
-
@Test
public void testDynamicRoleHistory() throws Throwable {
@@ -199,7 +172,7 @@
assert !entry.live
- def nodesForRoleId = roleHistory.getNodesForRoleId(role_priority_8)
+ def nodesForRoleId = roleHistory.getRecentNodesForRoleId(role_priority_8)
assert nodesForRoleId
// make sure new nodes will default to a different host in the engine
@@ -215,21 +188,16 @@
@Test(expected = BadConfigException.class)
public void testRoleHistoryRoleAdditions() throws Throwable {
MockRoleHistory roleHistory = new MockRoleHistory([])
- roleHistory.addNewProviderRole(new ProviderRole("one", 1))
- roleHistory.addNewProviderRole(new ProviderRole("two", 1))
+ roleHistory.addNewRole(new RoleStatus(new ProviderRole("one", 1)))
+ roleHistory.addNewRole(new RoleStatus(new ProviderRole("two", 1)))
roleHistory.dump()
- fail("should have raised an exception")
}
-
-
+
@Test(expected = BadConfigException.class)
public void testRoleHistoryRoleStartupConflict() throws Throwable {
MockRoleHistory roleHistory = new MockRoleHistory([
new ProviderRole("one", 1), new ProviderRole("two", 1)
])
roleHistory.dump()
- fail("should have raised an exception")
}
-
-
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
index ee4abd6..d0163d2 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateDynamicRoles.groovy
@@ -20,21 +20,17 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
-import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
-import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.RoleHistoryUtils
import org.apache.slider.server.appmaster.state.RoleInstance
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
import org.junit.Test
/**
@@ -65,40 +61,27 @@
}
@Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- def instance = factory.newInstanceDefinition(0,0,0)
-
+ AggregateConf buildInstanceDefinition() {
+ def instance = factory.newInstanceDefinition(0, 0, 0)
def opts = [
- (ResourceKeys.COMPONENT_PRIORITY): ROLE4,
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE4,
(ResourceKeys.COMPONENT_INSTANCES): "1",
]
- instance.resourceOperations.components[ROLE4]= opts
+ instance.resourceOperations.components[ROLE4] = opts
def opts5 = [
- (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
- (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY) : ROLE5,
+ (ResourceKeys.COMPONENT_INSTANCES) : "1",
(ResourceKeys.COMPONENT_PLACEMENT_POLICY):
Integer.toString(PlacementPolicy.STRICT),
- (ResourceKeys.NODE_FAILURE_THRESHOLD):
+ (ResourceKeys.NODE_FAILURE_THRESHOLD) :
Integer.toString(2),
]
- instance.resourceOperations.components[ROLE5]= opts5
-
- appState.buildInstance(
- instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null,
- null, new SimpleReleaseSelector())
+ instance.resourceOperations.components[ROLE5] = opts5
+ instance
}
@Test
@@ -115,21 +98,16 @@
* @param actions source list
* @return found list
*/
- List<ContainerRequestOperation> findAllocationsForRole(int role,
+ Collection<ContainerRequestOperation> findAllocationsForRole(int role,
List<AbstractRMOperation> actions) {
- List <ContainerRequestOperation > results = []
- actions.each { AbstractRMOperation operation ->
- if (operation instanceof ContainerRequestOperation) {
- def req = (ContainerRequestOperation) operation;
- def reqrole = ContainerPriority.extractRole(req.request.priority)
- if (role == reqrole) {
- results << req
- }
- }
+ def requests = actions.findAll {
+ it instanceof ContainerRequestOperation}.collect {it as ContainerRequestOperation}
+
+ requests.findAll {
+ role == ContainerPriority.extractRole(it.request.priority)
}
- return results
- }
-
+ }
+
@Test
public void testStrictPlacementInitialRequest() throws Throwable {
log.info("Initial engine state = $engine")
@@ -141,7 +119,6 @@
assertRelaxLocalityFlag(ID5, null, true, actions)
}
-
@Test
public void testPolicyPropagation() throws Throwable {
assert !(appState.lookupRoleStatus(ROLE4).placementPolicy & PlacementPolicy.STRICT)
@@ -153,7 +130,6 @@
public void testNodeFailureThresholdPropagation() throws Throwable {
assert (appState.lookupRoleStatus(ROLE4).nodeFailureThreshold == 3)
assert (appState.lookupRoleStatus(ROLE5).nodeFailureThreshold == 2)
-
}
@Test
@@ -173,7 +149,6 @@
assert instanceA
def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
-
log.info("Allocated engine state = $engine")
assert engine.containerCount() == 1
@@ -183,8 +158,7 @@
role4.desired = 0
appState.lookupRoleStatus(ROLE4).desired = 0
def completionResults = []
- def containersToRelease = []
- instances = createStartAndStopNodes(completionResults)
+ createStartAndStopNodes(completionResults)
assert engine.containerCount() == 0
assert completionResults.size() == 1
@@ -215,19 +189,16 @@
}
assert instanceA
def hostname = RoleHistoryUtils.hostnameOf(instanceA.container)
-
-
log.info("Allocated engine state = $engine")
assert engine.containerCount() == 1
assert role5.actual == 1
- // shrinking cluster
+ // shrinking cluster
role5.desired = 0
def completionResults = []
- def containersToRelease = []
- instances = createStartAndStopNodes(completionResults)
+ createStartAndStopNodes(completionResults)
assert engine.containerCount() == 0
assert completionResults.size() == 1
assert role5.actual == 0
@@ -240,16 +211,15 @@
def nodes = cro.request.nodes
assert nodes.size() == 1
assert hostname == nodes[0]
-
}
public void assertRelaxLocalityFlag(
- int id,
+ int role,
String expectedHost,
boolean expectedRelaxFlag,
List<AbstractRMOperation> actions) {
def requests
- requests = findAllocationsForRole(id, actions)
+ requests = findAllocationsForRole(role, actions)
assert requests.size() == 1
def req = requests[0]
assert expectedRelaxFlag == req.request.relaxLocality
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
index 015c6a3..7bc6fe4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexDynamicRoles.groovy
@@ -20,23 +20,22 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
-import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
/**
- * Test that if you have >1 role, the right roles are chosen for release.
+ * Test that if you have more than one role, the right roles are chosen for release.
*/
@CompileStatic
@Slf4j
@@ -59,32 +58,25 @@
}
@Override
- void initApp() {
- super.initApp()
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector()
+ bindingInfo
+ }
+ @Override
+ AggregateConf buildInstanceDefinition() {
def instance = factory.newInstanceDefinition(0, 0, 0)
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
- (ResourceKeys.COMPONENT_PRIORITY): "6",
+ (ResourceKeys.COMPONENT_PRIORITY) : "6",
]
instance.resourceOperations.components["dynamic-6"] = opts
-
-
- appState.buildInstance(instance,
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null,
- new MostRecentContainerReleaseSelector())
+ instance
}
-
private ConfTreeOperations init() {
createAndStartNodes();
def resources = appState.instanceDefinition.resources;
@@ -183,16 +175,10 @@
def historyWorkDir2 = new File("target/history" + testName + "-0002")
def historyPath2 = new Path(historyWorkDir2.toURI())
appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath2,
- null, null,
- new MostRecentContainerReleaseSelector())
+ def binding2 = buildBindingInfo()
+ binding2.instanceDefinition = factory.newInstanceDefinition(0, 0, 0)
+ binding2.historyPath = historyPath2
+ appState.buildInstance(binding2)
// on this read there won't be the right number of roles
def loadedRoleHistory = historyWriter.read(fs, history)
assert 0 == appState.roleHistory.rebuild(loadedRoleHistory)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
index 257092a..548842c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateFlexing.groovy
@@ -20,7 +20,6 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
-import org.apache.slider.api.ClusterDescription
import org.apache.slider.core.exceptions.TriggerClusterTeardownException
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockRoles
@@ -76,7 +75,7 @@
ops = appState.reviewRequestAndReleaseNodes()
assert ops.empty
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ appState.innerOnNodeManagerContainerStarted(target.id)
}
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
index 9ac6fcf..ba7588a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.groovy
@@ -110,7 +110,6 @@
// four outstanding
assert role0.requested == 4
-
// flex cluster to 3
role0.desired = 3
ops = appState.reviewRequestAndReleaseNodes()
@@ -226,8 +225,7 @@
public void testFlexUpNoSpace() throws Throwable {
// engine only has two nodes, so > 2 will be outstanding
engine = new MockYarnEngine(1, 2)
- List<AbstractRMOperation> ops
- // role: desired = 2, requested = 1, actual=1
+ // role: desired = 2, requested = 1, actual=1
def role0 = role0Status
role0.desired = 4
createAndSubmitNodes()
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
index c310583..0e526b6 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRebuildOnAMRestart.groovy
@@ -20,14 +20,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.yarn.api.records.Container
import org.apache.slider.api.StatusKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
-import org.apache.slider.server.appmaster.operations.AbstractRMOperation
-import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.NodeMap
@@ -35,12 +32,11 @@
import org.junit.Test
/**
- * Test that if you have >1 role, the right roles are chosen for release.
+ * Test that app state is rebuilt on a restart
*/
@CompileStatic
@Slf4j
-class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest
- implements MockRoles {
+class TestMockAppStateRebuildOnAMRestart extends BaseMockAppStateTest implements MockRoles {
@Override
String getTestName() {
@@ -62,28 +58,15 @@
assert instances.size() == clusterSize
//clone the list
- List<RoleInstance> cloned = [];
- List<Container> containers = []
- instances.each { RoleInstance elt ->
- cloned.add(elt.clone() as RoleInstance)
- containers.add(elt.container)
- }
+ List<Container> containers = instances.collect { it.container }
NodeMap nodemap = appState.roleHistory.cloneNodemap()
- // now destroy the app state
- appState = new MockAppState()
-
//and rebuild
- appState.buildInstance(
- factory.newInstanceDefinition(r0, r1, r2),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- containers,
- null,
- new MostRecentContainerReleaseSelector())
+
+ def bindingInfo = buildBindingInfo()
+ bindingInfo.instanceDefinition = factory.newInstanceDefinition(r0, r1, r2)
+ bindingInfo.liveContainers = containers
+ appState = new MockAppState(bindingInfo)
assert appState.startedCountainerCount == clusterSize
@@ -107,22 +90,18 @@
assertNotNull("Null entry in original nodemap for " + hostname, orig)
for (int i = 0; i < ROLE_COUNT; i++) {
-
- assert (nodeInstance.getActiveRoleInstances(i) ==
- orig.getActiveRoleInstances(i))
+ assert (nodeInstance.getActiveRoleInstances(i) == orig.getActiveRoleInstances(i))
NodeEntry origRE = orig.getOrCreate(i)
NodeEntry newRE = nodeInstance.getOrCreate(i)
assert origRE.live == newRE.live
- assert newRE.starting == 0
+ assert 0 == newRE.starting
}
}
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert ops.size() == 0
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
- def status = appState.getClusterStatus()
+ def status = appState.clusterStatus
// verify the AM restart container count was set
- String restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ String restarted = status.getInfo(StatusKeys.INFO_CONTAINERS_AM_RESTART)
assert restarted != null;
//and that the count == 1 master + the region servers
assert Integer.parseInt(restarted) == containers.size()
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
index 4ba0afd..93cce95 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockContainerResourceAllocations.groovy
@@ -25,6 +25,7 @@
import org.apache.slider.core.conf.ConfTree
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
@@ -37,11 +38,6 @@
@Slf4j
class TestMockContainerResourceAllocations extends BaseMockAppStateTest {
- @Override
- String getTestName() {
- "TestMockContainerResourceAllocations"
- }
-
@Test
public void testNormalAllocations() throws Throwable {
ConfTree clusterSpec = factory.newConfTree(1, 0, 0)
@@ -71,7 +67,7 @@
assert ops.size() == 1
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
Resource requirements = operation.request.capability
- assert requirements.memory == RM_MAX_RAM
+ assert requirements.memory == MockAppState.RM_MAX_RAM
assert requirements.virtualCores == 2
}
@@ -89,7 +85,7 @@
ContainerRequestOperation operation = (ContainerRequestOperation) ops[0]
Resource requirements = operation.request.capability
assert requirements.memory == 512
- assert requirements.virtualCores == RM_MAX_CORES
+ assert requirements.virtualCores == MockAppState.RM_MAX_CORES
}
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
new file mode 100644
index 0000000..f0fed95
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.junit.Test
+
+/**
+ * Test Anti-affine placement
+ */
+@CompileStatic
+@Slf4j
+class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
+ implements MockRoles {
+
+ private int NODES = 3
+ private int GPU_NODES = 2
+ private String HOST0 = "00000000"
+ private String HOST1 = "00000001"
+
+
+ @Override
+ void setup() {
+ super.setup()
+ // node 1 is GPU
+
+ updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ }
+
+ @Override
+ MockYarnEngine createYarnEngine() {
+ new MockYarnEngine(NODES, 8)
+ }
+
+ void assertAllContainersAA() {
+ assertAllContainersAA(gpuRole.key)
+ }
+
+ /**
+ *
+ * @throws Throwable
+ */
+ @Test
+ public void testAskForTooMany() throws Throwable {
+
+ describe("Ask for 1 more than the no of available nodes;" +
+ " expect the final request to be unsatisfied until the cluster changes size")
+ //more than expected
+ int size = GPU_NODES
+ gpuRole.desired = size + 1
+
+ List<AbstractRMOperation > operations = appState.reviewRequestAndReleaseNodes()
+ assert gpuRole.AARequestOutstanding
+
+ assert gpuRole.pendingAntiAffineRequests == size
+ for (int i = 0; i < size; i++) {
+ def iter = "Iteration $i role = $aaRole"
+ describe iter
+ List<AbstractRMOperation > operationsOut = []
+
+ def roleInstances = submitOperations(operations, [], operationsOut)
+ // one instance per request
+ assert 1 == roleInstances.size()
+ appState.onNodeManagerContainerStarted(roleInstances[0].containerId)
+ assertAllContainersAA()
+ // there should be none left
+ log.debug(nodeInformationSnapshotAsString())
+ operations = operationsOut
+ if (i + 1 < size) {
+ assert operations.size() == 2
+ } else {
+ assert operations.size() == 1
+ }
+ }
+ // expect an outstanding AA request to be unsatisfied
+ assert gpuRole.actual < gpuRole.desired
+ assert !gpuRole.requested
+ assert !gpuRole.AARequestOutstanding
+ List<Container> allocatedContainers = engine.execute(operations, [])
+ assert 0 == allocatedContainers.size()
+ // in a review now, no more requests can be generated, as there is no space for AA placements,
+ // even though there is cluster capacity
+ assert 0 == appState.reviewRequestAndReleaseNodes().size()
+
+ // switch node 2 into being labelled
+ def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu"))
+
+ assert cloneNodemap().size() == NODES
+ assert outcome.clusterChanged
+ // no active calls to empty
+ assert outcome.operations.empty
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+ protected AppState.NodeUpdatedOutcome addNewNode() {
+ updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu"))
+ }
+
+ @Test
+ public void testClusterSizeChangesDuringRequestSequence() throws Throwable {
+ describe("Change the cluster size where the cluster size changes during a test sequence.")
+ gpuRole.desired = GPU_NODES + 1
+ List<AbstractRMOperation> operations = appState.reviewRequestAndReleaseNodes()
+ assert gpuRole.AARequestOutstanding
+ assert GPU_NODES == gpuRole.pendingAntiAffineRequests
+ def outcome = addNewNode()
+ assert outcome.clusterChanged
+ // one call to cancel
+ assert 1 == outcome.operations.size()
+ // and on a review, one more to rebuild
+ assert 1 == appState.reviewRequestAndReleaseNodes().size()
+ }
+
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
new file mode 100644
index 0000000..db84b0b
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.history
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.slider.api.proto.Messages
+import org.apache.slider.api.proto.RestTypeMarshalling
+import org.apache.slider.api.types.NodeInformation
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
+import org.apache.slider.server.appmaster.state.NodeEntry
+import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+/**
+ * Test anti-affine
+ */
+//@CompileStatic
+@Slf4j
+class TestRoleHistoryAA extends SliderTestBase {
+
+ List<String> hostnames = ["1", "2", "3"]
+ NodeMap nodeMap, gpuNodeMap
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
+
+
+ @Override
+ void setup() {
+ super.setup()
+ nodeMap = createNodeMap(hostnames, NodeState.RUNNING)
+ gpuNodeMap = createNodeMap(hostnames, NodeState.RUNNING, "GPU")
+ }
+
+ @Test
+ public void testFindNodesInFullCluster() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, nodeMap.findAllNodesForRole(1, ""))
+ }
+
+ @Test
+ public void testFindNodesInUnhealthyCluster() throws Throwable {
+ // all three will surface at first
+ markNodeOneUnhealthy()
+ verifyResultSize(2, nodeMap.findAllNodesForRole(1, ""))
+ }
+
+ public boolean markNodeOneUnhealthy() {
+ return setNodeState(nodeMap.get("1"), NodeState.UNHEALTHY)
+ }
+
+ protected boolean setNodeState(NodeInstance node, NodeState state) {
+ node.updateNode(new MockNodeReport(node.hostname, state))
+ }
+
+ @Test
+ public void testFindNoNodesWrongLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"))
+ }
+
+ @Test
+ public void testFindSomeNodesSomeLabel() throws Throwable {
+ // all three will surface at first
+ update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")])
+ def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU")
+ verifyResultSize(1, gpuNodes)
+ def instance = gpuNodes[0]
+ instance.getOrCreate(1).onStarting()
+ assert !instance.canHost(1, "GPU")
+ assert !instance.canHost(1, "")
+ verifyResultSize(0, nodeMap.findAllNodesForRole(1, "GPU"))
+
+ }
+
+ @Test
+ public void testFindNoNodesRightLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, "GPU"))
+ }
+
+ @Test
+ public void testFindNoNodesNoLabel() throws Throwable {
+ // all three will surface at first
+ verifyResultSize(3, gpuNodeMap.findAllNodesForRole(1, ""))
+ }
+
+ @Test
+ public void testFindNoNodesClusterRequested() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.request()
+ }
+ assertNoAvailableNodes(1)
+ }
+
+ @Test
+ public void testFindNoNodesClusterBusy() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.request()
+ }
+ assertNoAvailableNodes(1)
+ }
+
+ /**
+ * Tag all nodes as starting, then walk one through a bit
+ * more of its lifecycle
+ */
+ @Test
+ public void testFindNoNodesLifecycle() throws Throwable {
+ // all three will surface at first
+ applyToNodeEntries(nodeMap) {
+ NodeEntry it -> it.onStarting()
+ }
+ assertNoAvailableNodes(1)
+
+ // walk one of the nodes through the lifecycle
+ def node1 = nodeMap.get("1")
+ assert !node1.canHost(1,"")
+ node1.get(1).onStartCompleted()
+ assert !node1.canHost(1,"")
+ assertNoAvailableNodes()
+ node1.get(1).release()
+ assert node1.canHost(1,"")
+ def list2 = verifyResultSize(1, nodeMap.findAllNodesForRole(1, ""))
+ assert list2[0].hostname == "1"
+
+ // now tag that node as unhealthy and expect it to go away
+ markNodeOneUnhealthy()
+ assertNoAvailableNodes()
+ }
+
+ @Test
+ public void testRolesIndependent() throws Throwable {
+ def node1 = nodeMap.get("1")
+ def role1 = node1.getOrCreate(1)
+ def role2 = node1.getOrCreate(2)
+ nodeMap.values().each {
+ it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY))
+ }
+ assertNoAvailableNodes(1)
+ assertNoAvailableNodes(2)
+ assert setNodeState(node1, NodeState.RUNNING)
+ // tag role 1 as busy
+ role1.onStarting()
+ assertNoAvailableNodes(1)
+
+ verifyResultSize(1, nodeMap.findAllNodesForRole(2, ""))
+ assert node1.canHost(2,"")
+ }
+
+ @Test
+ public void testNodeEntryAvailablity() throws Throwable {
+ def entry = new NodeEntry(1)
+ assert entry.available
+ entry.onStarting()
+ assert !entry.available
+ entry.onStartCompleted()
+ assert !entry.available
+ entry.release()
+ assert entry.available
+ entry.onStarting()
+ assert !entry.available
+ entry.onStartFailed()
+ assert entry.available
+ }
+
+ @Test
+ public void testNodeInstanceSerialization() throws Throwable {
+ def rh2 = new MockRoleHistory([])
+ rh2.getOrCreateNodeInstance("localhost")
+ def instance = rh2.getOrCreateNodeInstance("localhost")
+ instance.getOrCreate(1).onStartCompleted()
+ def Map<Integer, String> naming = [(1):"manager"]
+ def ni = instance.serialize(naming)
+ assert 1 == ni.entries["manager"].live
+ def ni2 = rh2.getNodeInformation("localhost", naming)
+ assert 1 == ni2.entries["manager"].live
+ def info = rh2.getNodeInformationSnapshot(naming)
+ assert 1 == info["localhost"].entries["manager"].live
+ def nil = new NodeInformationList(info.values());
+ assert 1 == nil[0].entries["manager"].live
+
+ def nodeInformationProto = RestTypeMarshalling.marshall(ni)
+ def entryProto = nodeInformationProto.getEntries(0)
+ assert entryProto && entryProto.getPriority() == 1
+ def unmarshalled = RestTypeMarshalling.unmarshall(nodeInformationProto)
+ assert unmarshalled.hostname == ni.hostname
+ assert unmarshalled.entries.keySet().containsAll(ni.entries.keySet())
+
+ }
+
+ @Test
+ public void testBuildRolenames() throws Throwable {
+
+ }
+ public List<NodeInstance> assertNoAvailableNodes(int role = 1, String label = "") {
+ return verifyResultSize(0, nodeMap.findAllNodesForRole(role, label))
+ }
+
+ List<NodeInstance> verifyResultSize(int size, List<NodeInstance> list) {
+ if (list.size() != size) {
+ list.each { log.error(it.toFullString()) }
+ }
+ assert size == list.size()
+ list
+ }
+
+ def applyToNodeEntries(Collection<NodeInstance> list, Closure cl) {
+ list.each { it -> cl(it.getOrCreate(1)) }
+ }
+
+ def applyToNodeEntries(NodeMap nodeMap, Closure cl) {
+ applyToNodeEntries(nodeMap.values(), cl)
+ }
+
+ def NodeMap createNodeMap(List<NodeReport> nodeReports) {
+ NodeMap nodeMap = new NodeMap(1)
+ update(nodeMap, nodeReports)
+ nodeMap
+ }
+
+ protected boolean update(NodeMap nodeMap, List<NodeReport> nodeReports) {
+ nodeMap.buildOrUpdate(nodeReports)
+ }
+
+ def NodeMap createNodeMap(List<String> hosts, NodeState state,
+ String label = "") {
+ createNodeMap(MockNodeReport.createInstances(hosts, state, label))
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 4fdf39c..ca42546 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -28,13 +28,11 @@
import org.apache.hadoop.yarn.api.records.Resource
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockNodeId
import org.apache.slider.server.appmaster.state.*
-import org.junit.Before
import org.junit.Test
/**
@@ -60,15 +58,20 @@
String roleName = "test"
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory
Resource resource
- @Before
- public void setupRH() {
- roleHistory.onStart(fs, historyPath)
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest
+ }
+
+ @Override
+ void setup() {
+ super.setup()
+ roleHistory = appState.roleHistory
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
resource = Resource.newInstance(ResourceKeys.DEF_YARN_CORES,
ResourceKeys.DEF_YARN_MEMORY);
}
@@ -76,11 +79,10 @@
@Test
public void testFindAndCreate() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
List<String> nodes = request.getNodes()
assert nodes != null
@@ -113,25 +115,21 @@
@Test
public void testCreateAndRelease() throws Throwable {
int role = 1
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
//verify it is empty
assert roleHistory.listActiveNodes(role).empty
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
- List<String> nodes = request.nodes
- assert nodes == null
+ assert request.nodes == null
//pick an idle host
String hostname = age3Active0.hostname;
//build a container
- MockContainer container = factory.newContainer()
- container.nodeId = new MockNodeId(hostname, 0)
- container.priority = request.priority
+ MockContainer container = factory.newContainer(new MockNodeId(hostname, 0), request.priority)
roleHistory.onContainerAssigned(container);
NodeMap nodemap = roleHistory.cloneNodemap();
@@ -172,7 +170,7 @@
// ask for a container and expect to get the recently released one
AMRMClient.ContainerRequest request2 =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
List<String> nodes2 = request2.nodes
assert nodes2 != null
@@ -189,15 +187,14 @@
//pick an idle host
String hostname = age3Active0.hostname;
//build a container
- MockContainer container = factory.newContainer()
- container.nodeId = new MockNodeId(hostname, 0)
- container.priority = ContainerPriority.createPriority(0, false)
+ MockContainer container = factory.newContainer(
+ new MockNodeId(hostname, 0),
+ ContainerPriority.createPriority(0, false))
NodeMap nodemap = roleHistory.cloneNodemap();
NodeInstance allocated = nodemap.get(hostname)
NodeEntry roleEntry = allocated.get(role)
- RoleInstance ri = new RoleInstance(container);
//tell RH that it started
roleHistory.onContainerStarted(container)
assert roleEntry.starting == 0
@@ -209,19 +206,16 @@
@Test
public void testStartFailed() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
//build a container
- MockContainer container = factory.newContainer()
- container.nodeId = new MockNodeId(hostname, 0)
- container.priority = request.getPriority()
+ MockContainer container = factory.newContainer(new MockNodeId(hostname, 0), request.priority)
roleHistory.onContainerAssigned(container);
NodeMap nodemap = roleHistory.cloneNodemap();
@@ -245,10 +239,10 @@
@Test
public void testStartFailedWithoutWarning() throws Throwable {
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -276,10 +270,10 @@
describe("fail a container without declaring it as starting")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -315,10 +309,10 @@
public void testContainerFailedWithoutWarning() throws Throwable {
describe( "fail a container without declaring it as starting")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -348,10 +342,10 @@
public void testAllocationListPrep() throws Throwable {
describe("test prepareAllocationList")
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ requestContainer(roleStatus);
String hostname = request.getNodes()[0]
assert hostname == age3Active0.hostname
@@ -378,24 +372,23 @@
@Test
public void testNodeUpdated() throws Throwable {
describe("fail a node")
-
+
int role = 0
- ProviderRole provRole = new ProviderRole(roleName, role)
- RoleStatus roleStatus = new RoleStatus(provRole)
+ RoleStatus roleStatus = appState.lookupRoleStatus(role)
+
AMRMClient.ContainerRequest request =
- roleHistory.requestNode(roleStatus, resource);
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest;
String hostname = request.getNodes()[0]
- assert hostname == age3Active0.hostname
+ assert age3Active0.hostname == hostname
// build a container
- MockContainer container = factory.newContainer()
- container.nodeId = new MockNodeId(hostname, 0)
- container.priority = request.getPriority()
+ MockContainer container = factory.newContainer(new MockNodeId(hostname, 0), request.priority)
+
roleHistory.onContainerAssigned(container);
NodeMap nodemap = roleHistory.cloneNodemap();
- NodeInstance allocated = nodemap.get(hostname)
+ NodeInstance allocated = nodemap[hostname]
NodeEntry roleEntry = allocated.get(role)
assert roleEntry.starting == 1
assert !roleEntry.available
@@ -407,21 +400,34 @@
int startSize = nodemap.size()
// now send a list of updated (failed) nodes event
- List<NodeReport> nodesUpdated = new ArrayList<NodeReport>();
- NodeId nodeId = NodeId.newInstance(hostname, 0)
- NodeReport nodeReport = NodeReport.newInstance(nodeId, NodeState.LOST, null, null, null, null, 1, null, 0)
+ List<NodeReport> nodesUpdated = new ArrayList<>();
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance(hostname, 0),
+ NodeState.LOST,
+ null, null, null, null, 1, null, 0)
nodesUpdated.add(nodeReport)
roleHistory.onNodesUpdated(nodesUpdated)
nodemap = roleHistory.cloneNodemap()
int endSize = nodemap.size()
- if (startSize == 0) {
- assert endSize == 0
- } else {
- assert startSize - endSize == 1
- }
- assert nodemap.get(hostname) == null
- List<String> failedNodes = roleHistory.cloneFailedNodes()
- assert failedNodes.contains(hostname)
+ // as even unused nodes are added to the list, we expect the map size to be >1
+ assert startSize <= endSize
+ assert nodemap[hostname] != null
+ assert !nodemap[hostname].online
+
+ // add a failure of a node we've never head of
+ def newhost = "newhost"
+ nodesUpdated = [
+ NodeReport.newInstance(
+ NodeId.newInstance(newhost, 0),
+ NodeState.LOST,
+ null, null, null, null, 1, null, 0)
+ ]
+ roleHistory.onNodesUpdated(nodesUpdated)
+
+ def nodemap2 = roleHistory.cloneNodemap()
+ assert nodemap2[newhost]
+ assert !nodemap2[newhost].online
+
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
index 79d23e5..f36724e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryFindNodesForNewInstances.groovy
@@ -23,6 +23,7 @@
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.ContainerOutcome
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
@@ -32,10 +33,8 @@
/**
* Testing finding nodes for new instances.
- * These tests validate the (currently) suboptimal
- * behavior of not listing any known nodes when there
- * are none in the available list -even if there are nodes
- * known to be not running live instances in the cluster.
+ *
+ * This stresses the non-AA codepath
*/
@Slf4j
@CompileStatic
@@ -54,7 +53,7 @@
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
String roleName = "test"
RoleStatus roleStat = new RoleStatus(new ProviderRole(roleName, 0))
@@ -63,14 +62,14 @@
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
}
public List<NodeInstance> findNodes(int count, RoleStatus roleStatus = roleStat) {
List < NodeInstance > found = [];
for (int i = 0; i < count; i++) {
- NodeInstance f = roleHistory.findNodeForNewInstance(roleStatus)
+ NodeInstance f = roleHistory.findRecentNodeForNewInstance(roleStatus)
if (f) {
found << f
};
@@ -80,17 +79,17 @@
@Test
public void testFind1NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info("found: $found")
assert [age3Active0].contains(found)
}
@Test
public void testFind2NodeR0() throws Throwable {
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info("found: $found")
assert [age2Active0, age3Active0].contains(found)
- NodeInstance found2 = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found2 = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info("found: $found2")
assert [age2Active0, age3Active0].contains(found2)
assert found != found2;
@@ -99,7 +98,7 @@
@Test
public void testFind3NodeR0ReturnsNull() throws Throwable {
assert 2== findNodes(2).size()
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
assert found == null;
}
@@ -123,7 +122,7 @@
assert age2Active0.getActiveRoleInstances(0) != 0
age3Active0.get(0).onStartCompleted()
assert age3Active0.getActiveRoleInstances(0) != 0
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
log.info(found ?.toFullString())
assert found == null
}
@@ -147,7 +146,7 @@
assert age2Active0.exceedsFailureThreshold(roleStat)
// get the role & expect age3 to be picked up, even though it is older
- NodeInstance found = roleHistory.findNodeForNewInstance(roleStat)
+ NodeInstance found = roleHistory.findRecentNodeForNewInstance(roleStat)
assert age3Active0.is(found)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
index b26b2f0..bcd8f9f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryNIComparators.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.transform.CompileStatic
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -27,17 +28,20 @@
/**
* Unit test to verify the comparators sort as expected
*/
+@CompileStatic
class TestRoleHistoryNIComparators extends BaseMockAppStateTest {
- NodeInstance age1Active4 = nodeInstance(1000, 4, 0, 0)
- NodeInstance age2Active2 = nodeInstance(1001, 2, 0, 0)
- NodeInstance age3Active0 = nodeInstance(1002, 0, 0, 0)
- NodeInstance age4Active1 = nodeInstance(1005, 0, 0, 0)
+ NodeInstance age1Active4 = nodeInstance(1001, 4, 0, 0)
+ NodeInstance age2Active2 = nodeInstance(1002, 2, 0, 0)
+ NodeInstance age3Active0 = nodeInstance(1003, 0, 0, 0)
+ NodeInstance age4Active1 = nodeInstance(1004, 1, 0, 0)
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
NodeInstance age6failing = nodeInstance(1006, 0, 0, 0)
- NodeInstance age1failing = nodeInstance(1000, 0, 0, 0)
+ NodeInstance age1failing = nodeInstance(1001, 0, 0, 0)
List<NodeInstance> nodes = [age2Active2, age4Active1, age1Active4, age3Active0]
+ List<NodeInstance> nodesPlusEmpty = [age2Active2, age4Active1, age1Active4, age3Active0, empty]
+ List<NodeInstance> allnodes = [age6failing, age2Active2, age4Active1, age1Active4, age3Active0, age1failing]
@Before
public void setup() {
@@ -51,43 +55,48 @@
}
@Test
- public void testNewerThan() throws Throwable {
-
+ public void testPreferred() throws Throwable {
Collections.sort(nodes, new NodeInstance.Preferred(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4])
+ assertListEquals(nodes, [age4Active1, age3Active0, age2Active2, age1Active4])
+ }
+
+ /**
+ * The preferred sort still includes failures; up to next phase in process
+ * to handle that
+ * @throws Throwable
+ */
+ @Test
+ public void testPreferredWithFailures() throws Throwable {
+ Collections.sort(allnodes, new NodeInstance.Preferred(0))
+ assert allnodes[0] == age6failing
+ assert allnodes[1] == age4Active1
}
@Test
- public void testFailureCountFirst() throws Throwable {
+ public void testPreferredComparatorDowngradesFailures() throws Throwable {
def preferred = new NodeInstance.Preferred(0)
assert preferred.compare(age6failing, age1failing) == -1
assert preferred.compare(age1failing, age6failing) == 1
}
-
+
@Test
public void testNewerThanNoRole() throws Throwable {
-
- nodes << empty
- Collections.sort(nodes, new NodeInstance.Preferred(0))
- assertListEquals(nodes,
- [age4Active1, age3Active0, age2Active2, age1Active4, empty])
+ Collections.sort(nodesPlusEmpty, new NodeInstance.Preferred(0))
+ assertListEquals(nodesPlusEmpty, [age4Active1, age3Active0, age2Active2, age1Active4, empty])
}
@Test
public void testMoreActiveThan() throws Throwable {
Collections.sort(nodes, new NodeInstance.MoreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0],)
+ assertListEquals(nodes, [age1Active4, age2Active2, age4Active1, age3Active0])
}
@Test
public void testMoreActiveThanEmpty() throws Throwable {
- nodes << empty
- Collections.sort(nodes, new NodeInstance.MoreActiveThan(0))
- assertListEquals(nodes,
- [age1Active4, age2Active2, age4Active1, age3Active0, empty])
+
+ Collections.sort(nodesPlusEmpty, new NodeInstance.MoreActiveThan(0))
+ assertListEquals(nodesPlusEmpty, [age1Active4, age2Active2, age4Active1, age3Active0, empty])
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 653af84..6969b38 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -27,6 +27,7 @@
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.ContainerPriority
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -35,16 +36,28 @@
import org.apache.slider.server.appmaster.state.RoleStatus
import org.junit.Test
-class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
+ public static final String WORKERS_LABEL = "workers"
NodeInstance host1 = new NodeInstance("host1", 3)
NodeInstance host2 = new NodeInstance("host2", 3)
+ def resource = factory.newResource(48, 1)
OutstandingRequestTracker tracker = new OutstandingRequestTracker()
-
+
+ public static final ProviderRole WORKER = new ProviderRole(
+ "worker",
+ 5,
+ PlacementPolicy.NONE,
+ 2,
+ 1,
+ WORKERS_LABEL)
+
@Override
- String getTestName() {
- return "TestOutstandingRequestTracker"
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.roles = [ WORKER ] + bindingInfo.roles
+ bindingInfo
}
@Test
@@ -57,10 +70,16 @@
@Test
public void testAddCompleteEntry() throws Throwable {
- tracker.newRequest(host1, 0)
- tracker.newRequest(host2, 0)
- tracker.newRequest(host1, 1)
- assert tracker.onContainerAllocated(1, "host1", null).outcome == ContainerAllocationOutcome.Placed
+ def req1 = tracker.newRequest(host1, 0)
+ req1.buildContainerRequest(resource, role0Status, 0)
+
+ tracker.newRequest(host2, 0).buildContainerRequest(resource, role0Status, 0)
+ tracker.newRequest(host1, 1).buildContainerRequest(resource, role0Status, 0)
+
+ def allocation = tracker.onContainerAllocated(1, "host1", null)
+ assert allocation.outcome == ContainerAllocationOutcome.Placed
+ assert allocation.operations[0] instanceof CancelSingleRequest
+
assert !tracker.lookupPlacedRequest(1, "host1")
assert tracker.lookupPlacedRequest(0, "host1")
}
@@ -80,46 +99,40 @@
@Test
public void testRemoveOpenRequestUnissued() throws Throwable {
def req1 = tracker.newRequest(null, 0)
+ req1.buildContainerRequest(resource, role0Status, 0)
assert tracker.listOpenRequests().size() == 1
- def c1 = factory.newContainer()
- c1.setPriority(new MockPriority(0))
+ def c1 = factory.newContainer(null, new MockPriority(0))
+ c1.resource = resource
- def resource = factory.newResource()
- resource.virtualCores=1
- resource.memory = 48;
- c1.setResource(resource)
- ContainerAllocationOutcome outcome = tracker.onContainerAllocated(0, "host1", c1).outcome
+ def allocation = tracker.onContainerAllocated(0, "host1", c1)
+ ContainerAllocationOutcome outcome = allocation.outcome
assert outcome == ContainerAllocationOutcome.Unallocated
+ assert allocation.operations.empty
assert tracker.listOpenRequests().size() == 1
}
@Test
public void testIssuedOpenRequest() throws Throwable {
def req1 = tracker.newRequest(null, 0)
- def resource = factory.newResource()
- resource.virtualCores = 1
- resource.memory = 48;
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, "")
+ req1.buildContainerRequest(resource, role0Status, 0)
assert tracker.listOpenRequests().size() == 1
- def c1 = factory.newContainer()
-
- def nodeId = factory.newNodeId()
- c1.nodeId = nodeId
- nodeId.host ="hostname-1"
def pri = ContainerPriority.buildPriority(0, false)
assert pri > 0
- c1.setPriority(new MockPriority(pri))
+ def nodeId = factory.newNodeId("hostname-1")
+ def c1 = factory.newContainer(nodeId, new MockPriority(pri))
- c1.setResource(resource)
+ c1.resource = resource
def issued = req1.issuedRequest
assert issued.capability == resource
- assert issued.priority.priority == c1.getPriority().getPriority()
+ assert issued.priority.priority == c1.priority.priority
assert req1.resourceRequirementsMatch(resource)
def allocation = tracker.onContainerAllocated(0, nodeId.host, c1)
assert tracker.listOpenRequests().size() == 0
+ assert allocation.operations[0] instanceof CancelSingleRequest
+
assert allocation.outcome == ContainerAllocationOutcome.Open
assert allocation.origin.is(req1)
}
@@ -146,7 +159,7 @@
// first request: default placement
assert role0Status.placementPolicy == PlacementPolicy.DEFAULT;
final def (res0, outstanding0) = newRequest(role0Status)
- final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0, null)
+ final def initialRequest = outstanding0.buildContainerRequest(res0, role0Status, 0)
assert outstanding0.issuedRequest != null;
assert outstanding0.located
assert !outstanding0.escalated
@@ -157,7 +170,6 @@
// process skips entries which are in the list but have not been issued.
// ...which can be a race condition between request issuance & escalation.
// (not one observed outside test authoring, but retained for completeness)
- assert role2Status.placementPolicy == PlacementPolicy.ANTI_AFFINITY_REQUIRED
def (res2, outstanding2) = newRequest(role2Status)
// simulate some time escalation of role 1 MUST now be triggered
@@ -182,7 +194,7 @@
// build that second request from an anti-affine entry
// these get placed as well
now += interval
- final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now, null)
+ final def containerReq2 = outstanding2.buildContainerRequest(res2, role2Status, now)
// escalate a little bit more
final List<AbstractRMOperation> escalations2 = tracker.escalateOutstandingRequests(now)
// and expect no new entries
@@ -223,18 +235,19 @@
resource.virtualCores = 1
resource.memory = 48;
- def label = "workers"
+ def workerRole = lookupRole(WORKER.name)
// initial request
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label)
+ def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0)
assert (yarnRequest.nodeLabelExpression == null)
assert (!yarnRequest.relaxLocality)
+ // escalation
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodeLabelExpression == label)
+ assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
assert (yarnRequest2.relaxLocality)
}
/**
- * If the placement doesnt include a lablel, then the escalation request
+ * If the placement doesnt include a label, then the escalation request
* retains the node list, but sets relaxLocality==true
* @throws Throwable
*/
@@ -246,17 +259,44 @@
resource.virtualCores = 1
resource.memory = 48;
- def label = null
// initial request
- def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0, label)
- assert (yarnRequest.nodes != null)
- assert (yarnRequest.nodeLabelExpression == null)
- assert (!yarnRequest.relaxLocality)
+ def yarnRequest = req1.buildContainerRequest(resource, role0Status, 0)
+ assert yarnRequest.nodes != null
+ assert !yarnRequest.nodeLabelExpression
+ assert !yarnRequest.relaxLocality
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodes != null)
- assert (yarnRequest2.relaxLocality)
+ assert yarnRequest2.nodes != null
+ assert yarnRequest2.relaxLocality
}
+ @Test(expected = IllegalArgumentException)
+ public void testAARequestNoNodes() throws Throwable {
+ tracker.newAARequest(role0Status.key, [], "")
+ }
+
+ @Test
+ public void testAARequest() throws Throwable {
+ def role0 = role0Status.key
+ OutstandingRequest request = tracker.newAARequest(role0, [host1], "")
+ assert host1.hostname == request.hostname
+ assert !request.located
+ }
+
+ @Test
+ public void testAARequestPair() throws Throwable {
+ def role0 = role0Status.key
+ OutstandingRequest request = tracker.newAARequest(role0, [host1, host2], "")
+ assert host1.hostname == request.hostname
+ assert !request.located
+ def yarnRequest = request.buildContainerRequest(
+ role0Status.copyResourceRequirements(new MockResource(0, 0)),
+ role0Status,
+ 0)
+ assert !yarnRequest.relaxLocality
+ assert !request.mayEscalate()
+
+ assert yarnRequest.nodes.size() == 2
+ }
/**
* Create a new request (always against host1)
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
index c81c686..72e4240 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRW.groovy
@@ -22,15 +22,15 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.FSDataOutputStream
import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
-import org.apache.slider.server.avro.LoadedRoleHistory
import org.apache.slider.server.avro.RoleHistoryWriter
import org.junit.Test
@@ -48,7 +48,8 @@
3,
PlacementPolicy.STRICT,
3,
- 3)
+ 3,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
@Override
String getTestName() {
@@ -57,7 +58,7 @@
@Test
public void testWriteReadEmpty() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
Path history = roleHistory.saveHistory(time++)
assert fs.isFile(history)
@@ -67,7 +68,7 @@
@Test
public void testWriteReadData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert !roleHistory.onStart(fs, historyPath)
String addr = "localhost"
NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
@@ -77,7 +78,7 @@
Path history = roleHistory.saveHistory(time++)
assert fs.isFile(history)
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
def loadedRoleHistory = historyWriter.read(fs, history)
@@ -92,7 +93,7 @@
@Test
public void testWriteReadActiveData() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
String addr = "localhost"
String addr2 = "rack1server5"
@@ -117,7 +118,7 @@
describe("Loaded")
log.info("testWriteReadActiveData in $history")
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
def loadedRoleHistory = historyWriter.read(fs, history)
assert 3 == loadedRoleHistory.size()
rh2.rebuild(loadedRoleHistory)
@@ -136,16 +137,16 @@
assert rh2.thawedDataTime == savetime
// now start it
- rh2.buildAvailableNodeLists();
+ rh2.buildRecentNodeLists();
describe("starting")
rh2.dump();
- List<NodeInstance> available0 = rh2.cloneAvailableList(0)
+ List<NodeInstance> available0 = rh2.cloneRecentNodeList(0)
assert available0.size() == 1
NodeInstance entry = available0.get(0)
assert entry.hostname == "localhost"
assert entry == localhost
- List<NodeInstance> available1 = rh2.cloneAvailableList(1)
+ List<NodeInstance> available1 = rh2.cloneRecentNodeList(1)
assert available1.size() == 2
//and verify that even if last used was set, the save time is picked up
assert entry.get(1).lastUsed == roleHistory.saveTime
@@ -154,7 +155,7 @@
@Test
public void testWriteThaw() throws Throwable {
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert !roleHistory.onStart(fs, historyPath)
String addr = "localhost"
NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
@@ -164,7 +165,7 @@
Path history = roleHistory.saveHistory(time++)
long savetime =roleHistory.saveTime;
assert fs.isFile(history)
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
assert rh2.onStart(fs, historyPath)
NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
assert ni2 != null
@@ -211,7 +212,7 @@
@Test
public void testSkipEmptyFileOnRead() throws Throwable {
describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
time = 0
Path oldhistory = roleHistory.saveHistory(time++)
@@ -226,7 +227,7 @@
RoleHistoryWriter historyWriter = new RoleHistoryWriter();
Path touched = touch(historyWriter, time++)
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
assert rh2.onStart(fs, historyPath)
NodeInstance ni2 = rh2.getExistingNodeInstance(addr)
assert ni2 != null
@@ -240,7 +241,7 @@
@Test
public void testSkipBrokenFileOnRead() throws Throwable {
describe "verify that empty histories are skipped on read; old histories purged"
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
roleHistory.onStart(fs, historyPath)
time = 0
Path oldhistory = roleHistory.saveHistory(time++)
@@ -258,7 +259,7 @@
out.writeBytes("{broken:true}")
out.close()
- RoleHistory rh2 = new RoleHistory(MockFactory.ROLES)
+ RoleHistory rh2 = new MockRoleHistory(MockFactory.ROLES)
describe("IGNORE STACK TRACE BELOW")
assert rh2.onStart(fs, historyPath)
@@ -285,7 +286,7 @@
def loadedRoleHistory = writer.read(source)
assert 4 == loadedRoleHistory.size()
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert 0 == roleHistory.rebuild(loadedRoleHistory)
}
@@ -300,7 +301,7 @@
def loadedRoleHistory = writer.read(source)
assert 6 == loadedRoleHistory.size()
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert 3 == roleHistory.rebuild(loadedRoleHistory)
}
@@ -318,7 +319,7 @@
assert 4 == loadedRoleHistory.size()
def expandedRoles = new ArrayList(MockFactory.ROLES)
expandedRoles << PROVIDER_ROLE3
- RoleHistory roleHistory = new RoleHistory(expandedRoles)
+ RoleHistory roleHistory = new MockRoleHistory(expandedRoles)
assert 0 == roleHistory.rebuild(loadedRoleHistory)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
index aef22fb..8a0c1ca 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRWOrdering.groovy
@@ -18,11 +18,13 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.common.SliderKeys
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
import org.apache.slider.server.appmaster.state.RoleHistory
@@ -34,14 +36,15 @@
import java.util.regex.Pattern
@Slf4j
+@CompileStatic
class TestRoleHistoryRWOrdering extends BaseMockAppStateTest {
- def paths = pathlist(
+ List<Path> paths = pathlist(
[
- "hdfs://localhost/history-0406c.json",
- "hdfs://localhost/history-5fffa.json",
- "hdfs://localhost/history-0001a.json",
- "hdfs://localhost/history-0001f.json",
+ "hdfs://localhost/history-0406c.json",
+ "hdfs://localhost/history-5fffa.json",
+ "hdfs://localhost/history-0001a.json",
+ "hdfs://localhost/history-0001f.json",
]
)
Path h_0406c = paths[0]
@@ -50,9 +53,7 @@
List<Path> pathlist(List<String> pathnames) {
- def result = []
- pathnames.each { result << new Path(new URI(it as String)) }
- result
+ pathnames.collect{ new Path(new URI(it as String)) }
}
@Override
@@ -60,7 +61,6 @@
return "TestHistoryRWOrdering"
}
-
/**
* This tests regexp pattern matching. It uses the current time so isn't
* repeatable -but it does test a wider range of values in the process
@@ -79,13 +79,12 @@
}
}
-
@Test
public void testWriteSequenceReadData() throws Throwable {
describe "test that if multiple entries are written, the newest is picked up"
long time = System.currentTimeMillis();
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
assert !roleHistory.onStart(fs, historyPath)
String addr = "localhost"
NodeInstance instance = roleHistory.getOrCreateNodeInstance(addr)
@@ -94,7 +93,7 @@
Path history1 = roleHistory.saveHistory(time++)
Path history2 = roleHistory.saveHistory(time++)
- Path history3 = roleHistory.saveHistory(time++)
+ Path history3 = roleHistory.saveHistory(time)
//inject a later file with a different name
sliderFileSystem.cat(new Path(historyPath, "file.json"), true, "hello, world")
@@ -116,7 +115,7 @@
public void testPathStructure() throws Throwable {
assert h_5fffa.getName() == "history-5fffa.json"
}
-
+
@Test
public void testPathnameComparator() throws Throwable {
@@ -137,10 +136,10 @@
RoleHistoryWriter.sortHistoryPaths(paths2)
assertListEquals(paths2,
[
- paths[1],
- paths[0],
- paths[3],
- paths[2]
+ paths[1],
+ paths[0],
+ paths[3],
+ paths[2]
])
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
index 9847992..2f160cb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryRequestTracking.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.server.appmaster.model.history
+import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.Resource
@@ -27,6 +28,7 @@
import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
import org.apache.slider.server.appmaster.model.mock.MockContainer
import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.ContainerAllocationOutcome
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -41,6 +43,7 @@
* get removed and added
*/
@Slf4j
+@CompileStatic
class TestRoleHistoryRequestTracking extends BaseMockAppStateTest {
String roleName = "test"
@@ -53,12 +56,17 @@
NodeInstance empty = new NodeInstance("empty", MockFactory.ROLE_COUNT)
List<NodeInstance> nodes = [age2Active2, age2Active0, age4Active1, age1Active4, age3Active0]
- RoleHistory roleHistory = new RoleHistory(MockFactory.ROLES)
+ RoleHistory roleHistory = new MockRoleHistory(MockFactory.ROLES)
+ /** 1MB, 1 vcore*/
Resource resource = Resource.newInstance(1, 1)
ProviderRole provRole = new ProviderRole(roleName, 0)
RoleStatus roleStatus = new RoleStatus(provRole)
+ AMRMClient.ContainerRequest requestContainer(RoleStatus roleStatus) {
+ roleHistory.requestContainerForRole(roleStatus).issuedRequest
+ }
+
@Override
String getTestName() {
return "TestRoleHistoryAvailableList"
@@ -67,46 +75,44 @@
@Before
public void setupNodeMap() {
roleHistory.insert(nodes)
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
+ roleStatus.setResourceRequirements(Resource.newInstance(1, 1))
}
@Test
public void testAvailableListBuiltForRoles() throws Throwable {
- List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
+ List<NodeInstance> available0 = roleHistory.cloneRecentNodeList(0)
assertListEquals([age3Active0, age2Active0], available0)
}
@Test
public void testRequestedNodeOffList() throws Throwable {
- List<NodeInstance> available0 = roleHistory.cloneAvailableList(0)
- NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus)
assert age3Active0 == ni
- AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
+ assertListEquals([age2Active0], roleHistory.cloneRecentNodeList(0))
+ roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource,
- "")
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
- assertListEquals([age2Active0], a2)
+ resource
+ )
}
@Test
public void testRequestedNodeOffListWithFailures() throws Throwable {
assert 0 == roleStatus.key
- assert !roleHistory.cloneAvailableList(0).isEmpty()
+ assert !roleHistory.cloneRecentNodeList(0).empty
NodeEntry age3role0 = recordAsFailed(age3Active0, 0, 4)
assert age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
recordAsFailed(age2Active0, 0, 4)
assert age2Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
// expect to get a null node back
- NodeInstance ni = roleHistory.findNodeForNewInstance(roleStatus)
+ NodeInstance ni = roleHistory.findRecentNodeForNewInstance(roleStatus)
assert !ni
// which is translated to a no-location request
AMRMClient.ContainerRequest req = roleHistory.requestInstanceOnNode(ni,
roleStatus,
- resource,
- "")
+ resource).issuedRequest
assertNull(req.nodes)
@@ -115,57 +121,61 @@
roleHistory.dump()
assert 0 == age3role0.failedRecently
assert !age3Active0.isConsideredUnreliable(0, roleStatus.nodeFailureThreshold)
- assert !roleHistory.cloneAvailableList(0).isEmpty()
+ assert !roleHistory.cloneRecentNodeList(0).empty
// looking for a node should now find one
- ni = roleHistory.findNodeForNewInstance(roleStatus)
+ ni = roleHistory.findRecentNodeForNewInstance(roleStatus)
assert ni == age3Active0
- req = roleHistory.requestInstanceOnNode(ni,
- roleStatus,
- resource,
- "")
+ req = roleHistory.requestInstanceOnNode(ni, roleStatus, resource).issuedRequest
assert 1 == req.nodes.size()
}
+ /**
+ * verify that strict placement policies generate requests for nodes irrespective
+ * of their failed status
+ * @throws Throwable
+ */
@Test
public void testStrictPlacementIgnoresFailures() throws Throwable {
def targetRole = role1Status
final ProviderRole providerRole1 = targetRole.providerRole
assert providerRole1.placementPolicy == PlacementPolicy.STRICT
- int key = targetRole.key
+ int key1 = targetRole.key
+ def key0 = role0Status.key
- recordAsFailed(age1Active4, key, 4)
- recordAsFailed(age2Active0, key, 4)
- recordAsFailed(age2Active2, key, 4)
- recordAsFailed(age3Active0, key, 4)
- recordAsFailed(age4Active1, key, 4)
+ def nodes = [age1Active4, age2Active0, age2Active2, age3Active0, age4Active1]
+ recordAllFailed(key0, 4, nodes)
+ recordAllFailed(key1, 4, nodes)
// trigger a list rebuild
- roleHistory.buildAvailableNodeLists();
+ roleHistory.buildRecentNodeLists();
+ def recentRole0 = roleHistory.cloneRecentNodeList(key0)
+ assert recentRole0.indexOf(age3Active0) < recentRole0.indexOf(age2Active0)
- assert !roleHistory.cloneAvailableList(key).isEmpty()
+ // the non-strict role has no suitable nodes
+ assert null == roleHistory.findRecentNodeForNewInstance(role0Status)
- NodeInstance ni = roleHistory.findNodeForNewInstance(targetRole)
- assert ni == age4Active1!= null
- // next lookup returns next node
- ni = roleHistory.findNodeForNewInstance(roleStatus)
- assert ni == age3Active0
+ def ni = roleHistory.findRecentNodeForNewInstance(targetRole)
+ assert ni
+
+ def ni2 = roleHistory.findRecentNodeForNewInstance(targetRole)
+ assert ni2
+ assert ni != ni2
}
-
@Test
public void testFindAndRequestNode() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
assert age3Active0.hostname == req.nodes[0]
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0)
assertListEquals([age2Active0], a2)
}
@Test
public void testRequestedNodeIntoReqList() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
assert age3Active0.hostname == requests[0].hostname
@@ -173,7 +183,7 @@
@Test
public void testCompletedRequestDropsNode() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 1
String hostname = requests[0].hostname
@@ -202,8 +212,8 @@
@Test
public void testTwoRequests() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
@@ -216,9 +226,9 @@
@Test
public void testThreeRequestsOneUnsatisified() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus)
List<OutstandingRequest> requests = roleHistory.listPlacedRequests()
assert requests.size() == 2
MockContainer container = factory.newContainer(req, req.nodes[0])
@@ -239,15 +249,15 @@
roleHistory.listOpenRequests().empty
// and the remainder goes onto the available list
- List<NodeInstance> a2 = roleHistory.cloneAvailableList(0)
+ List<NodeInstance> a2 = roleHistory.cloneRecentNodeList(0)
assertListEquals([age2Active0], a2)
}
@Test
public void testThreeRequests() throws Throwable {
- AMRMClient.ContainerRequest req = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req2 = roleHistory.requestNode(roleStatus, resource)
- AMRMClient.ContainerRequest req3 = roleHistory.requestNode(roleStatus, resource)
+ AMRMClient.ContainerRequest req = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req2 = requestContainer(roleStatus)
+ AMRMClient.ContainerRequest req3 = requestContainer(roleStatus)
assertOutstandingPlacedRequests(2)
assert req3.nodes == null
MockContainer container = factory.newContainer(req, req.nodes[0])
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 29eefa5..69a98eb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -20,35 +20,38 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.ContainerState
import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
+import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
import org.apache.slider.server.appmaster.state.ContainerAssignment
import org.apache.slider.server.appmaster.state.ContainerOutcome
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
+import org.apache.slider.server.appmaster.state.NodeMap
+import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.state.RoleInstance
import org.apache.slider.server.appmaster.state.RoleStatus
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector
+import org.apache.slider.server.appmaster.state.StateAccessForProviders
import org.apache.slider.test.SliderTestBase
-import org.junit.Before
@CompileStatic
@Slf4j
abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles {
- public static final int RM_MAX_RAM = 4096
- public static final int RM_MAX_CORES = 64
MockFactory factory = new MockFactory()
MockAppState appState
MockYarnEngine engine
@@ -58,6 +61,16 @@
protected Path historyPath;
protected MockApplicationId applicationId;
protected MockApplicationAttemptId applicationAttemptId;
+ protected StateAccessForProviders stateAccess
+
+ /**
+ * Override point: called in setup() to create the YARN engine; can
+ * be changed for different sizes and options
+ * @return
+ */
+ public MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(8, 8)
+ }
@Override
void setup() {
@@ -66,49 +79,46 @@
fs = HadoopFS.get(new URI("file:///"), conf)
sliderFileSystem = new SliderFileSystem(fs, conf)
engine = createYarnEngine()
+ initApp()
}
/**
- * Override point: called in setup() to create the YARN engine; can
- * be changed for different sizes and options
- * @return
+ * Initialize the application.
+ * This uses the binding information supplied by {@link #buildBindingInfo()}.
*/
- public MockYarnEngine createYarnEngine() {
- return new MockYarnEngine(64, 1)
- }
-
- @Before
void initApp(){
-
String historyDirName = testName;
-
-
- YarnConfiguration conf = SliderUtils.createConfiguration()
applicationId = new MockApplicationId(id: 1, clusterTimestamp: 0)
applicationAttemptId = new MockApplicationAttemptId(
applicationId: applicationId,
attemptId: 1)
- fs = HadoopFS.get(new URI("file:///"), conf)
historyWorkDir = new File("target/history", historyDirName)
historyPath = new Path(historyWorkDir.toURI())
fs.delete(historyPath, true)
- appState = new MockAppState()
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
- appState.buildInstance(
- buildInstanceDefinition(),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null,
- new SimpleReleaseSelector())
+ appState = new MockAppState(buildBindingInfo())
+ stateAccess = new ProviderAppState(testName, appState)
+ }
+
+ /**
+ * Build the binding info from the default constructor values,
+ * the roles from {@link #factory}, and an instance definition
+ * from {@link #buildInstanceDefinition()}
+ * @return
+ */
+ AppStateBindingInfo buildBindingInfo() {
+ AppStateBindingInfo binding = new AppStateBindingInfo()
+ binding.instanceDefinition = buildInstanceDefinition();
+ binding.roles = factory.ROLES
+ binding.fs = fs
+ binding.historyPath = historyPath
+ binding.nodeReports = engine.nodeReports as List<NodeReport>
+ binding
}
/**
* Override point, define the instance definition
- * @return
+ * @return the instance definition
*/
public AggregateConf buildInstanceDefinition() {
factory.newInstanceDefinition(0, 0, 0)
@@ -116,22 +126,26 @@
/**
* Get the test name ... defaults to method name
- * @return
+ * @return the method name
*/
String getTestName() {
methodName.methodName;
}
public RoleStatus getRole0Status() {
- return appState.lookupRoleStatus(ROLE0)
+ lookupRole(ROLE0)
+ }
+
+ public RoleStatus lookupRole(String role) {
+ appState.lookupRoleStatus(role)
}
public RoleStatus getRole1Status() {
- return appState.lookupRoleStatus(ROLE1)
+ lookupRole(ROLE1)
}
public RoleStatus getRole2Status() {
- return appState.lookupRoleStatus(ROLE2)
+ lookupRole(ROLE2)
}
/**
@@ -147,7 +161,6 @@
return ri
}
-
public NodeInstance nodeInstance(long age, int live0, int live1=0, int live2=0) {
NodeInstance ni = new NodeInstance("age${age}-[${live0},${live1},$live2]",
MockFactory.ROLE_COUNT)
@@ -200,7 +213,6 @@
return createStartAndStopNodes([])
}
-
/**
* Create, Start and stop nodes
* @param completionResults List filled in with the status on all completed nodes
@@ -266,7 +278,7 @@
* @return a list of roles
*/
public List<RoleInstance> createAndSubmitNodes() {
- return createAndSubmitNodes([])
+ return createAndSubmitNodes([], [])
}
/**
@@ -275,35 +287,36 @@
* @return a list of roles allocated
*/
public List<RoleInstance> createAndSubmitNodes(
- List<ContainerId> released) {
+ List<ContainerId> containerIds,
+ List<AbstractRMOperation> operationsOut = []) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- return submitOperations(ops, released)
+ return submitOperations(ops, containerIds, operationsOut)
}
/**
* Process the RM operations and send <code>onContainersAllocated</code>
* events to the app state
- * @param ops
- * @param released
- * @return
+ * @param operationsIn list of incoming ops
+ * @param released released containers
+ * @return list of outbound operations
*/
public List<RoleInstance> submitOperations(
- List<AbstractRMOperation> ops,
- List<ContainerId> released) {
- List<Container> allocatedContainers = engine.execute(ops, released)
+ List<AbstractRMOperation> operationsIn,
+ List<ContainerId> released,
+ List<AbstractRMOperation> operationsOut = []) {
+ List<Container> allocatedContainers = engine.execute(operationsIn, released)
List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> operations = []
- appState.onContainersAllocated(allocatedContainers, assignments, operations)
- List<RoleInstance> instances = []
- for (ContainerAssignment assigned : assignments) {
+ appState.onContainersAllocated(allocatedContainers, assignments, operationsOut)
+
+ assignments.collect {
+ ContainerAssignment assigned ->
Container container = assigned.container
RoleInstance ri = roleInstance(assigned)
- instances << ri
//tell the app it arrived
log.debug("Start submitted ${ri.role} on ${container.id} ")
appState.containerStartSubmitted(container, ri);
+ ri
}
- return instances
}
/**
@@ -326,13 +339,7 @@
List<ContainerId> extractContainerIds(
List<RoleInstance> instances,
int role) {
- List<ContainerId> cids = []
- instances.each { RoleInstance instance ->
- if (instance.roleId == role) {
- cids << instance.id
- }
- }
- return cids
+ instances.findAll { it.roleId == role }.collect { RoleInstance instance -> instance.id }
}
/**
@@ -351,4 +358,99 @@
}
entry
}
+
+ def recordAllFailed(int id, int count, List<NodeInstance> nodes) {
+ nodes.each { NodeInstance node -> recordAsFailed(node, id, count)}
+ }
+
+ /**
+ * Get the container request of an indexed entry. Includes some assertions for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ AMRMClient.ContainerRequest getRequest(List<AbstractRMOperation> ops, int index) {
+ assert index < ops.size()
+ def op = ops[index]
+ assert op instanceof ContainerRequestOperation
+ ((ContainerRequestOperation) op).request
+ }
+
+ /**
+ * Get the cancel request of an indexed entry. Includes some assertions for better diagnostics
+ * @param ops operation list
+ * @param index index in the list
+ * @return the request.
+ */
+ AMRMClient.ContainerRequest getCancel(List<AbstractRMOperation> ops, int index) {
+ assert index < ops.size()
+ def op = ops[index]
+ assert op instanceof CancelSingleRequest
+ ((CancelSingleRequest) op).request
+ }
+
+ /**
+ * Get the single request of a list of operations; includes the check for the size
+ * @param ops operations list of size 1
+ * @return the request within the first ContainerRequestOperation
+ */
+ public AMRMClient.ContainerRequest getSingleRequest(List<AbstractRMOperation> ops) {
+ assert 1 == ops.size()
+ getRequest(ops, 0)
+ }
+
+ /**
+ * Get the node information as a large JSON String
+ * @return
+ */
+ String nodeInformationSnapshotAsString() {
+ prettyPrintAsJson(stateAccess.nodeInformationSnapshot)
+ }
+
+ /**
+ * Scan through all containers and assert that the assignment is AA
+ * @param index role index
+ */
+ void assertAllContainersAA(int index) {
+ cloneNodemap().each { name, info ->
+ def nodeEntry = info.get(index)
+ assert nodeEntry == null || nodeEntry.antiAffinityConstraintHeld
+ "too many instances on node $name"
+ }
+ }
+
+ List<NodeInstance> verifyNodeInstanceCount(int size, List<NodeInstance> list) {
+ if (list.size() != size) {
+ list.each { log.error(it.toFullString()) }
+ }
+ assert size == list.size()
+ list
+ }
+
+ /**
+ * Get the single request of a list of operations; includes the check for the size
+ * @param ops operations list of size 1
+ * @return the request within the first operation
+ */
+ public AMRMClient.ContainerRequest getSingleCancel(List<AbstractRMOperation> ops) {
+ assert 1 == ops.size()
+ getCancel(ops, 0)
+ }
+
+ /**
+ * Get a snapshot of the nodemap of the application state
+ * @return a cloned nodemap
+ */
+ protected NodeMap cloneNodemap() {
+ appState.roleHistory.cloneNodemap()
+ }
+
+ /**
+ * Issue a nodes updated event
+ * @param report report to notify
+ * @return response of AM
+ */
+ protected AppState.NodeUpdatedOutcome updateNodes(NodeReport report) {
+ appState.onNodesUpdated([report])
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
index 2cc13c2..c041ce5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockAppState.groovy
@@ -19,16 +19,20 @@
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.state.AbstractClusterServices
import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
/**
* Extended app state that makes more things public
*/
class MockAppState extends AppState {
+ public static final int RM_MAX_RAM = 4096
+ public static final int RM_MAX_CORES = 64
- public MockAppState(AbstractRecordFactory recordFactory) {
+ public MockAppState(AbstractClusterServices recordFactory) {
super(recordFactory, new MetricsAndMonitoring());
+ setContainerLimits(1, RM_MAX_RAM, 1, RM_MAX_CORES)
}
long time = 0;
@@ -37,7 +41,12 @@
* Instance with a mock record factory
*/
public MockAppState() {
- super(new MockRecordFactory(), new MetricsAndMonitoring());
+ this(new MockClusterServices());
+ }
+
+ MockAppState(AppStateBindingInfo bindingInfo) {
+ this()
+ buildInstance(bindingInfo)
}
public Map<String, ProviderRole> getRoleMap() {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
similarity index 80%
rename from slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
rename to slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
index f7d353f..dfecc94 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockClusterServices.groovy
@@ -19,12 +19,17 @@
package org.apache.slider.server.appmaster.model.mock
import org.apache.hadoop.yarn.api.records.Resource
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.state.AbstractClusterServices
-class MockRecordFactory extends AbstractRecordFactory {
+class MockClusterServices extends AbstractClusterServices {
@Override
Resource newResource() {
return new MockResource()
}
+
+ @Override
+ Resource newResource(int memory, int cores) {
+ return new MockResource(memory, cores)
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
index 9f5d939..ad586cd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainer.groovy
@@ -26,7 +26,7 @@
import org.apache.hadoop.yarn.api.records.Token
class MockContainer extends Container {
-
+
ContainerId id;
NodeId nodeId
String nodeHttpAddress;
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
index bfb00b0..4237f1e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
@@ -21,6 +21,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.ContainerState
import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.Resource
class MockContainerStatus extends ContainerStatus {
@@ -28,4 +29,6 @@
ContainerState state
String diagnostics
int exitStatus
+
+ Resource capability;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index fca3376..4bbfbd8 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -23,6 +23,8 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
@@ -38,26 +40,64 @@
@Slf4j
class MockFactory implements MockRoles {
+ /*
+ Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
+ */
+
+ /**
+ * basic role
+ */
public static final ProviderRole PROVIDER_ROLE0 = new ProviderRole(
MockRoles.ROLE0,
0,
PlacementPolicy.DEFAULT,
2,
- 1)
- // role 1 is strict. timeout should be irrelevant; same as failures
+ 1,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+ /**
+ * role 1 is strict. timeout should be irrelevant; same as failures
+ */
public static final ProviderRole PROVIDER_ROLE1 = new ProviderRole(
MockRoles.ROLE1,
1,
PlacementPolicy.STRICT,
2,
- 1)
- // role 2: longer delay
+ 1,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+
+ /**
+ * role 2: longer delay
+ */
public static final ProviderRole PROVIDER_ROLE2 = new ProviderRole(
MockRoles.ROLE2,
2,
+ PlacementPolicy.NO_DATA_LOCALITY,
+ 2,
+ 2,
+ ResourceKeys.DEF_YARN_LABEL_EXPRESSION)
+
+ /**
+ * Patch up a "role2" role to have anti-affinity set
+ */
+ public static final ProviderRole AAROLE_2 = new ProviderRole(
+ MockRoles.ROLE2,
+ 2,
PlacementPolicy.ANTI_AFFINITY_REQUIRED,
2,
- 2)
+ 2,
+ null)
+
+ /**
+ * Patch up a "role1" role to have anti-affinity set and GPI as the label
+ */
+ public static final ProviderRole AAROLE_1_GPU = new ProviderRole(
+ MockRoles.ROLE1,
+ 1,
+ PlacementPolicy.ANTI_AFFINITY_REQUIRED,
+ 2,
+ 1,
+ MockRoles.LABEL_GPU)
+
int appIdCount;
int attemptIdCount;
int containerIdCount;
@@ -73,7 +113,7 @@
PROVIDER_ROLE1,
PROVIDER_ROLE2,
]
-
+
public static final int ROLE_COUNT = ROLES.size();
MockContainerId newContainerId() {
@@ -100,10 +140,10 @@
return id;
}
- MockNodeId newNodeId() {
- MockNodeId nodeId = new MockNodeId()
+ MockNodeId newNodeId(String host = null) {
+ new MockNodeId(host: host)
}
-
+
MockContainer newContainer(ContainerId cid) {
MockContainer c = new MockContainer()
c.id = cid
@@ -114,6 +154,13 @@
newContainer(newContainerId())
}
+ MockContainer newContainer(NodeId nodeId, Priority priority) {
+ def container = newContainer(newContainerId())
+ container.nodeId = nodeId
+ container.priority = priority
+ container
+ }
+
/**
* Build a new container using the request to suppy priority and resource
* @param req request
@@ -188,12 +235,11 @@
]
}
- MockResource newResource() {
- return new MockResource()
+ MockResource newResource(int memory = 0, int vcores = 0) {
+ return new MockResource(memory, vcores)
}
MockContainerStatus newContainerStatus() {
return new MockContainerStatus()
-
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
new file mode 100644
index 0000000..8c3b712
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.Resource
+
+/**
+ * Node report for testing
+ */
+@CompileStatic
+class MockNodeReport extends NodeReport {
+ NodeId nodeId;
+ NodeState nodeState;
+ String httpAddress;
+ String rackName;
+ Resource used;
+ Resource capability;
+ int numContainers;
+ String healthReport;
+ long lastHealthReportTime;
+ Set<String> nodeLabels;
+
+ MockNodeReport() {
+ }
+
+ /**
+ * Create a single instance
+ * @param hostname
+ * @param nodeState
+ * @param label
+ */
+ MockNodeReport(String hostname, NodeState nodeState, String label ="") {
+ nodeId = NodeId.newInstance(hostname, 80)
+ Integer.valueOf(hostname, 16)
+ this.nodeState = nodeState
+ this.httpAddress = "http$hostname:80"
+ this.nodeLabels = new HashSet<>()
+ nodeLabels.add(label)
+ }
+
+ /**
+ * Create a list of instances -one for each hostname
+ * @param hostnames hosts
+ * @param nodeState state of all of them
+ * @param label label for all of them
+ * @return
+ */
+ static List<MockNodeReport> createInstances(
+ List<String> hostnames,
+ NodeState nodeState = NodeState.RUNNING,
+ String label = "") {
+ hostnames.collect { String name ->
+ new MockNodeReport(name, nodeState, label)}
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
index a68ce02..c803b54 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRMOperationHandler.groovy
@@ -69,10 +69,10 @@
cancelled++;
}
}
-
-/**
- * clear the history
- */
+
+ /**
+ * clear the history
+ */
public void clear() {
operations.clear()
releases = 0;
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
index f4c54f3..91ab43b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockResource.groovy
@@ -24,7 +24,7 @@
int memory
int virtualCores
- MockResource(int memory=0, int vcores=0) {
+ MockResource(int memory = 0, int vcores = 0) {
this.memory = memory
this.virtualCores = vcores
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
index c521697..4553e22 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoleHistory.groovy
@@ -21,15 +21,22 @@
import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.providers.ProviderRole
import org.apache.slider.server.appmaster.state.RoleHistory
+import org.apache.slider.server.appmaster.state.RoleStatus
/**
* subclass to enable access to some of the protected methods
*/
class MockRoleHistory extends RoleHistory {
+ /**
+ * Take a list of provider roles and build the history from them, dynamically creating
+ * the role status entries on the way
+ * @param providerRoles provider role list
+ * @throws BadConfigException configuration problem with the role list
+ */
MockRoleHistory(List<ProviderRole> providerRoles) throws BadConfigException {
- super(providerRoles)
+ super(providerRoles.collect { new RoleStatus(it) },
+ new MockClusterServices())
}
-
-
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
index b44482d..5ee8f24 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRoles.groovy
@@ -24,4 +24,6 @@
String ROLE1 = "role1"
String ROLE2 = "role2"
int ROLE_COUNT = 3
+ String LABEL_GPU = "gpu"
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 6056e3a..265a796 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeState
/**
* Models the cluster itself: a set of mock cluster nodes.
@@ -143,21 +144,30 @@
}
return total;
}
-
+
+ /**
+ * Get the list of node reports. These are not cloned; updates will persist in the nodemap
+ * @return current node report list
+ */
+ List<MockNodeReport> getNodeReports() {
+ nodes.collect { MockYarnClusterNode n -> n.nodeReport }
+ }
/**
* Model cluster nodes on the simpler "slot" model than the YARN-era
- * resource allocation model. Why? Makes it easier to implement.
+ * resource allocation model. Why? Easier to implement scheduling.
+ * Of course, if someone does want to implement the full process...
*
- * When a cluster is offline,
*/
public static class MockYarnClusterNode {
public final int nodeIndex
public final String hostname;
+ public List<String> labels = []
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
+ public MockNodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -170,6 +180,10 @@
MockContainerId mci = new MockContainerId(containerId: cid)
containers[i] = new MockYarnClusterContainer(mci)
}
+
+ nodeReport = new MockNodeReport()
+ nodeReport.nodeId = nodeId
+ nodeReport.nodeState = NodeState.RUNNING
}
/**
@@ -230,8 +244,6 @@
}
return result
}
-
-
/**
* Release a container
@@ -291,8 +303,8 @@
return (hostIndex << 8) | containerIndex & 0xff;
}
- public static final int extractHost(int cid) {
- return (cid >>> 8);
+ public static final int extractHost(long cid) {
+ return (cid >>> 8) & 0xffff;
}
public static final int extractContainer(int cid) {
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index e3d509a..7ab97fa 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -26,8 +26,10 @@
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
+import org.apache.slider.server.appmaster.operations.CancelSingleRequest
import org.apache.slider.server.appmaster.operations.ContainerReleaseOperation
import org.apache.slider.server.appmaster.operations.ContainerRequestOperation
+import org.junit.Assert
/**
* This is an evolving engine to mock YARN operations
@@ -64,13 +66,13 @@
allocator = new Allocator(cluster)
}
-/**
- * Allocate a container from a request. The containerID will be
- * unique, nodeId and other fields chosen internally with
- * no such guarantees; resource and priority copied over
- * @param request request
- * @return container
- */
+ /**
+ * Allocate a container from a request. The containerID will be
+ * unique, nodeId and other fields chosen internally with
+ * no such guarantees; resource and priority copied over
+ * @param request request
+ * @return container
+ */
Container allocateContainer(AMRMClient.ContainerRequest request) {
MockContainer allocated = allocator.allocate(request)
if (allocated != null) {
@@ -104,6 +106,7 @@
*/
List<Container> execute(List<AbstractRMOperation> ops,
List<ContainerId> released) {
+ validateRequests(ops)
List<Container> allocation = [];
ops.each { AbstractRMOperation op ->
if (op instanceof ContainerReleaseOperation) {
@@ -111,7 +114,9 @@
ContainerId cid = cro.containerId
assert releaseContainer(cid);
released.add(cid)
- } else {
+ } else if (op instanceof CancelSingleRequest) {
+ // no-op
+ } else if (op instanceof ContainerRequestOperation) {
ContainerRequestOperation req = (ContainerRequestOperation) op
Container container = allocateContainer(req.request)
if (container != null) {
@@ -121,9 +126,47 @@
log.debug("Unsatisfied allocation $req")
pending.add(req)
}
+ } else {
+ log.warn("Unsupported operation $op")
}
}
return allocation
}
+ /**
+ * Try and mimic some of the logic of <code>AMRMClientImpl.checkLocalityRelaxationConflict</code>
+ * @param ops operations list
+ */
+ void validateRequests(List<AbstractRMOperation> ops) {
+ // run through the requests and verify that they are all consistent.
+ List<ContainerRequestOperation> outstandingRequests = []
+ for (AbstractRMOperation operation : ops) {
+ if (operation instanceof ContainerRequestOperation) {
+ ContainerRequestOperation containerRequest = (ContainerRequestOperation) operation
+ def amRequest = containerRequest.request
+ def priority = amRequest.priority
+ def relax = amRequest.relaxLocality
+
+ outstandingRequests.each { req ->
+
+ if (req.priority == priority && req.relaxLocality != relax) {
+ // mismatch in values
+ Assert.fail("operation $operation has incompatible request priority" +
+ " from outsanding request")
+ }
+ outstandingRequests << containerRequest
+
+ }
+
+ }
+ }
+ }
+
+ /**
+ * Get the list of node reports. These are not cloned; updates will persist in the nodemap
+ * @return current node report list
+ */
+ List<MockNodeReport> getNodeReports() {
+ cluster.nodeReports
+ }
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
index 82192b9..ec202b0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/monkey/TestMockMonkey.groovy
@@ -148,7 +148,7 @@
public void testContainerKillerIgnoresAM() throws Throwable {
addAppMastertoAppState()
- assert 1 == appState.liveNodes.size()
+ assert 1 == appState.liveContainers.size()
def chaos = new ChaosKillContainer(appState,
queues,
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index 29f0510..e301e6f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -107,7 +107,7 @@
webResource = client.resource(sliderConfigset + "dummy-site");
- execOperation(30000) {
+ execOperation(WEB_STARTUP_TIME) {
GET(sliderConfigset)
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
index e1964f5..5a7a891 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestClusterSpecificationBlock.groovy
@@ -19,15 +19,14 @@
import com.google.inject.AbstractModule
import com.google.inject.Guice
import com.google.inject.Injector
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
import org.apache.slider.api.ClusterDescription
import org.apache.slider.providers.ProviderService
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockProviderService
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory
-import org.apache.slider.server.appmaster.state.AbstractRecordFactory
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices
+import org.apache.slider.server.appmaster.state.AbstractClusterServices
import org.apache.slider.server.appmaster.state.AppState
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
@@ -43,7 +42,7 @@
@Before
public void setup() {
- AppState appState = new MyAppState(new MockRecordFactory());
+ AppState appState = new MyAppState(new MockClusterServices());
ProviderAppState providerAppState = new ProviderAppState(
"undefined",
appState)
@@ -79,7 +78,7 @@
}
private static class MyAppState extends MockAppState {
- public MyAppState(AbstractRecordFactory recordFactory) {
+ public MyAppState(AbstractClusterServices recordFactory) {
super(recordFactory);
this.clusterStatus = new MockClusterDescription();
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
index c2ea837..a818e53 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/view/TestIndexBlock.groovy
@@ -24,8 +24,10 @@
import org.apache.hadoop.yarn.api.records.Priority
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet
import org.apache.slider.providers.ProviderService
+import org.apache.slider.server.appmaster.model.appstate.BaseMockAppStateAATest
import org.apache.slider.server.appmaster.model.mock.*
import org.apache.slider.server.appmaster.state.ContainerOutcome
+import org.apache.slider.server.appmaster.state.OutstandingRequest
import org.apache.slider.server.appmaster.state.ProviderAppState
import org.apache.slider.server.appmaster.web.WebAppApi
import org.apache.slider.server.appmaster.web.WebAppApiImpl
@@ -34,7 +36,7 @@
@Slf4j
//@CompileStatic
-public class TestIndexBlock extends BaseMockAppStateTest {
+public class TestIndexBlock extends BaseMockAppStateAATest {
private IndexBlock indexBlock;
@@ -81,42 +83,80 @@
public void testIndex() {
def role0 = role0Status
def role1 = role1Status
- role0.desired = 8
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role0.incActual()
- role1.incRequested()
- role1.incRequested()
- role1.incRequested()
+ def role2 = role2Status
+
+ def role0_desired = 8
+
+ role0.desired = role0_desired
+ int role0_actual = 5
+ int role0_requested = role0_desired - role0_actual
+ role0_actual.times {
+ role0.incActual()
+ }
+ assert role0.getActual() == role0_actual
+ role0_requested.times {
+ role0.incRequested()
+ }
+ assert role0.getRequested() == role0_requested
+
+ def role0_failures = 2
+
role0.noteFailed(false, "", ContainerOutcome.Failed)
role0.noteFailed(true, "", ContainerOutcome.Failed)
+ // all aa roles fields are in the
+ def aarole_desired = 200
+ aaRole.desired = aarole_desired
+ def aarole_actual = 90
+ def aarole_active = 1
+ def aarole_requested = aarole_desired - aarole_actual
+ def aarole_pending = aarole_requested - 1
+ def aarole_failures = 0
+ aarole_actual.times {
+ aaRole.incActual()
+ }
+ assert aaRole.actual == aarole_actual
+ aaRole.outstandingAArequest = new OutstandingRequest(2, "")
+ // add a requested
+ aaRole.incRequested()
+ aaRole.setPendingAntiAffineRequests(aarole_pending)
+ assert aaRole.pendingAntiAffineRequests == aarole_pending
+
+ assert aaRole.actualAndRequested == aarole_actual + 1
StringWriter sw = new StringWriter(64);
PrintWriter pw = new PrintWriter(sw);
Hamlet hamlet = new Hamlet(pw, 0, false);
-
- int level = hamlet.nestLevel();
+
indexBlock.doIndex(hamlet, "accumulo");
def body = sw.toString()
log.info(body)
- assertEquals(body, level, hamlet.nestLevel())
// verify role data came out
assert body.contains("role0")
- //
- assert body.contains("8")
- assert body.contains("5")
- assert body.contains("3")
- assert body.contains("2")
- assert body.contains("1")
-
+ assertContains(role0_desired, body)
+ assertContains(role0_actual, body)
+ assertContains(role0_requested, body)
+ assertContains(role0_failures, body)
+
assert body.contains("role1")
assert body.contains("role2")
+
+ assertContains(aarole_desired, body)
+ assertContains(aarole_actual, body)
+// assertContains(aarole_requested, body)
+ assertContains(aarole_failures, body)
+ assert body.contains(indexBlock.buildAADetails(true, aarole_pending))
+
// verify that the sorting took place
assert body.indexOf("role0") < body.indexOf("role1")
assert body.indexOf("role1") < body.indexOf("role2")
+
+ assert !body.contains(IndexBlock.ALL_CONTAINERS_ALLOCATED)
+ // role
+ }
+
+ def assertContains(int ex, String html) {
+ assertStringContains(Integer.toString(ex), html)
}
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy b/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
new file mode 100644
index 0000000..451bdae
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/management/TestGauges.groovy
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.management
+
+import org.apache.slider.server.appmaster.management.LongGauge
+import org.apache.slider.test.SliderTestBase
+import org.junit.Test
+
+class TestGauges extends SliderTestBase {
+
+ @Test
+ public void testLongGaugeOperations() throws Throwable {
+ LongGauge gauge = new LongGauge();
+ assert gauge.get() == 0
+ gauge.inc()
+ assert gauge.get() == 1
+ gauge.inc()
+ assert gauge.get() == 2
+ gauge.inc()
+ assert gauge.get() == 3
+ assert gauge.getValue() == gauge.get()
+ assert gauge.count == gauge.get()
+
+ gauge.dec()
+ assert gauge.get() == 2
+ assert gauge.decToFloor(1) == 1
+ assert gauge.get() == 1
+ assert gauge.decToFloor(1) == 0
+ assert gauge.decToFloor(1) == 0
+ assert gauge.decToFloor(0) == 0
+
+ gauge.set(4)
+ assert gauge.decToFloor(8) == 0
+
+ }
+}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
index 3de3c67..d82a79c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
@@ -20,9 +20,9 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.RawLocalFileSystem
-import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.Credentials
@@ -32,6 +32,7 @@
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager
import org.apache.hadoop.service.ServiceOperations
import org.apache.hadoop.util.Time
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.tools.CoreFileSystem
import org.apache.slider.server.appmaster.actions.ActionStopQueue
import org.apache.slider.server.appmaster.actions.QueueExecutor
@@ -60,10 +61,10 @@
conf = new Configuration()
conf.set(
- DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"TOKEN")
conf.setLong(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ SliderXmlConfKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
1000)
queues.init(conf)
queues.start();
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
index 4721552..b4a7ac9 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy
@@ -33,8 +33,9 @@
String USERNAME = "bigdataborat"
int WAIT_TIME = 120;
- String WAIT_TIME_ARG = WAIT_TIME.toString()
+ String WAIT_TIME_ARG = Integer.toString(WAIT_TIME)
String SLIDER_TEST_XML = "slider-test.xml"
+ String NODES_UPDATED_FLAG_METRIC = "org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag"
}
\ No newline at end of file
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
index 7a935dc..2b75c26 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestBase.groovy
@@ -44,7 +44,6 @@
*/
public static final MetricsAndMonitoring metrics = new MetricsAndMonitoring()
public static final int WEB_STARTUP_TIME = 30000
- public static final byte[] NO_BYTES = new byte[0]
@Rule
public TestName methodName = new TestName();
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
index 9d14815..cb6ce0e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy
@@ -24,6 +24,7 @@
import com.sun.jersey.api.json.JSONConfiguration
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler
import groovy.json.JsonOutput
+import groovy.json.JsonSlurper
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.commons.httpclient.HttpClient
@@ -73,6 +74,7 @@
import java.util.concurrent.TimeoutException
import static Arguments.ARG_OPTION
+import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_METRICS_JSON
/**
* Static utils for tests in this package and in other test projects.
@@ -98,15 +100,38 @@
log.info("");
}
- public static String prettyPrint(String json) {
+ /**
+ * Convert a JSON string to something readable
+ * @param json
+ * @return a string for printing
+ */
+ public static String prettyPrintJson(String json) {
JsonOutput.prettyPrint(json)
}
+ /**
+ * Convert a JSON string to something readable
+ * @param json
+ * @return a string for printing
+ */
+ public static String prettyPrintAsJson(Object src) {
+ JsonOutput.prettyPrint(JsonOutput.toJson(src))
+ }
+
+ /**
+ * Skip the test with a message
+ * @param message message logged and thrown
+ */
public static void skip(String message) {
log.warn("Skipping test: {}", message)
Assume.assumeTrue(message, false);
}
+ /**
+ * Skip the test with a message if condition holds
+ * @param condition predicate
+ * @param message message logged and thrown
+ */
public static void assume(boolean condition, String message) {
if (!condition) {
skip(message)
@@ -116,18 +141,25 @@
/**
* Skip a test if not running on Windows
*/
- public assumeWindows() {
+ public static void assumeWindows() {
assume(Shell.WINDOWS, "not windows")
}
/**
* Skip a test if running on Windows
*/
- public assumeNotWindows() {
+ public static void assumeNotWindows() {
assume(!Shell.WINDOWS, "windows")
}
/**
+ * skip a test on windows
+ */
+ public static void skipOnWindows() {
+ assumeNotWindows();
+ }
+
+ /**
* Equality size for a list
* @param left
* @param right
@@ -227,15 +259,6 @@
}
/**
- * skip a test on windows
- */
- public static void skipOnWindows() {
- if (Shell.WINDOWS) {
- skip("Not supported on windows")
- }
- }
-
- /**
* Assert that any needed libraries being present. On Unix none are needed;
* on windows they must be present
*/
@@ -368,7 +391,7 @@
if (timedOut) {
duration.finish();
describe("$operation: role count not met after $duration: $details")
- log.info(prettyPrint(status.toJsonString()))
+ log.info(prettyPrintJson(status.toJsonString()))
fail("$operation: role counts not met after $duration: " +
details.toString() +
" in \n$status ")
@@ -414,7 +437,7 @@
String text,
ClusterDescription status) {
describe(text)
- log.info(prettyPrint(status.toJsonString()))
+ log.info(prettyPrintJson(status.toJsonString()))
}
@@ -614,7 +637,6 @@
static UrlConnectionOperations connectionOperations
static UgiJerseyBinding jerseyBinding;
-
/**
* Static initializer of the connection operations
* @param conf config
@@ -655,7 +677,6 @@
clientConfig);
}
-
/**
* Create a jersey client config with the settings needed for tests
* (e.g. POJO mappings)
@@ -677,14 +698,18 @@
* to have been called.
*
* @param path path to page
- * @param connectionChecks optional closure to run against an open connection
* @return body of response
*/
- public static String getWebPage(String path, Closure connectionChecks = null) {
+ public static String getWebPage(String path) {
HttpOperationResponse outcome = executeGet(path)
return new String(outcome.data);
}
+ /**
+ * Execute a GET operation
+ * @param path path to GET
+ * @return the response
+ */
public static HttpOperationResponse executeGet(String path) {
assert path
assertHttpSupportInitialized()
@@ -707,7 +732,7 @@
HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE
}
-/**
+ /**
* Assert that a service operation succeeded
* @param service service
*/
@@ -738,6 +763,7 @@
int actual = instances != null ? instances.size() : 0
return actual
}
+
/**
* Exec a set of commands, wait a few seconds for it to finish.
* @param status code
@@ -752,11 +778,12 @@
assert status == exitCode
return process
}
+
/**
- * Exec a set of commands, wait a few seconds for it to finish.
- * @param commands
- * @return
- */
+ * Exec a set of commands, wait a few seconds for it to finish.
+ * @param commands
+ * @return
+ */
public static ForkedProcessService exec(List<String> commands) {
ForkedProcessService process;
process = new ForkedProcessService(
@@ -857,6 +884,29 @@
log.info("$realkey = $val")
return val
}
+ /**
+ * Create a temp JSON file. After coming up with the name, the file
+ * is deleted
+ * @return the filename
+ */
+ public static File createTempJsonFile() {
+ return tmpFile(".json")
+ }
+
+ /**
+ * Create a temp file with the specific name. It's deleted after creation,
+ * to avoid "file exists exceptions"
+ * @param suffix suffix, e.g. ".txt"
+ * @return a path to a file which may be created
+ */
+ public static File tmpFile(String suffix) {
+ File reportFile = File.createTempFile(
+ "temp",
+ suffix,
+ new File("target"))
+ reportFile.delete()
+ return reportFile
+ }
/**
* Execute a closure, assert it fails with a given exit code and text
@@ -1313,6 +1363,17 @@
}
/**
+ * Get a value from a map; raise an assertion if it is not there
+ * @param map map to look up
+ * @param key key
+ * @return the string value
+ */
+ String requiredMapValue(Map map, String key) {
+ assert map[key] != null
+ map[key].toString()
+ }
+
+ /**
* Get a web page and deserialize the supplied JSON into
* an instance of the specific class.
* @param clazz class to deserialize to
@@ -1380,4 +1441,99 @@
assert list.size() == entries.size()
assert entries.containsAll(list)
}
+
+ public Map parseMetrics(String metrics) {
+ new JsonSlurper().parse(metrics.bytes) as Map
+ }
+
+ public void validateCodahaleJson(Map metricsMap) {
+ assert metricsMap["version"] == "3.0.0"
+ assert metricsMap["gauges"] instanceof Map
+ assert metricsMap["histograms"] instanceof Map
+ assert metricsMap["timers"] instanceof Map
+ }
+
+ public int getGaugeValue(Map metricsMap, String gauge, int defVal) {
+ def entry = metricsMap["gauges"][gauge]
+ if (entry != null) {
+ return entry["value"] as int
+ } else {
+ return defVal
+ }
+ }
+
+ public boolean getGaugeAsBool(Map metricsMap, String gauge, boolean defVal) {
+ return 0 != getGaugeValue(metricsMap, gauge, defVal ? 1 : 0)
+ }
+
+ /**
+ * Fetch and parse the JSON codahale metrics under a path
+ * @param baseUrl base path
+ * @return the fetch, parsed and partially validated JSON mapping
+ */
+ public Map getMetrics(String baseUrl) {
+ def raw = GET(baseUrl, SYSTEM_METRICS_JSON)
+ def metrics = parseMetrics(raw)
+ validateCodahaleJson(metrics)
+ return metrics;
+ }
+
+ /**
+ * Await a specific gauge being of the desired value
+ * @param am URL of appmaster
+ * @param gauge gauge name
+ * @param desiredValue desired value
+ * @param timeout timeout in millis
+ * @param sleepDur sleep in millis
+ */
+ public void awaitGaugeValue(String am, String gauge, int desiredValue,
+ int timeout,
+ int sleepDur) {
+ String target = appendToURL(am, SYSTEM_METRICS_JSON)
+ def text = "Probe $target for gauge $gauge == $desiredValue"
+ repeatUntilSuccess(text,
+ this.&probeMetricGaugeValue,
+ timeout, sleepDur,
+ [
+ url : target,
+ gauge: gauge,
+ desiredValue: desiredValue.toString()
+ ],
+ true, text) {
+ log.error(prettyPrintJson(GET(target)))
+ }
+ }
+
+ /**
+ * Probe for a metric gauge holding a value.
+ *
+ * Keys: "url:String", "gauge:String", "desiredValue:int"
+ * @param args argument map
+ * @return success on the desired value, retry if not; fail on IOE
+ */
+ Outcome probeMetricGaugeValue(Map args) {
+ String url = requiredMapValue(args, "url")
+ String gauge = requiredMapValue(args, "gauge")
+ String vstr = requiredMapValue(args, "desiredValue")
+ assert vstr != null, "null desired value in $args"
+ assert vstr != "", "empty desired value in $args"
+ int desiredValue = Integer.decode(vstr)
+ try {
+ def metrics = parseMetrics(GET(url))
+ def gaugeValue = getGaugeValue(metrics, gauge, -1)
+ return gaugeValue == desiredValue ? Outcome.Success : Outcome.Retry
+ } catch (IOException e) {
+ return Outcome.Fail
+ }
+ }
+
+ public static void assertStringContains(String expected, String text) {
+ assertNotNull("null text", text)
+ if (!text.contains(expected)) {
+ def message = "id not find $expected in \"$text\""
+ log.error(message)
+ fail(message)
+ }
+
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 9fb75df..faaf571 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.hdfs.MiniDFSCluster
import org.apache.hadoop.service.ServiceOperations
import org.apache.hadoop.util.Shell
@@ -303,7 +302,7 @@
conf.setFloat(
YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
99.0f)
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 2 * 1024 * 1024)
+ conf.setInt(SliderXmlConfKeys.DFS_NAMENODE_DU_RESERVED_KEY, 2 * 1024 * 1024)
}
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
index e907209..313ab46 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy
@@ -46,17 +46,20 @@
}
public ZKIntegration createZKIntegrationInstance(String zkQuorum,
- String clusterName,
- boolean createClusterPath,
- boolean canBeReadOnly,
- int timeout) {
+ String clusterName,
+ boolean createClusterPath,
+ boolean canBeReadOnly,
+ int timeout,
+ int sessionTimeout = ZKIntegration.SESSION_TIMEOUT) {
BlockingZKWatcher watcher = new BlockingZKWatcher();
ZKIntegration zki = ZKIntegration.newInstance(zkQuorum,
- USERNAME,
- clusterName,
- createClusterPath,
- canBeReadOnly, watcher)
+ USERNAME,
+ clusterName,
+ createClusterPath,
+ canBeReadOnly,
+ watcher,
+ sessionTimeout)
zki.init()
//here the callback may or may not have occurred.
//optionally wait for it
diff --git a/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
new file mode 100644
index 0000000..cc64cab
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/core/launch/TestAppMasterLauncherWithAmReset.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.launch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.slider.api.ResourceKeys;
+import org.apache.slider.client.SliderYarnClientImpl;
+import org.apache.slider.common.SliderKeys;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestAppMasterLauncherWithAmReset {
+ SliderYarnClientImpl mockYarnClient;
+ YarnClientApplication yarnClientApp;
+ ApplicationSubmissionContext appSubmissionContext;
+ GetNewApplicationResponse newApp;
+ Set<String> tags = Collections.emptySet();
+ AppMasterLauncher appMasterLauncher = null;
+ boolean isOldApi = true;
+
+ @Before
+ public void initialize() throws Exception {
+ mockYarnClient = EasyMock.createNiceMock(SliderYarnClientImpl.class);
+ yarnClientApp = EasyMock.createNiceMock(YarnClientApplication.class);
+ newApp = EasyMock.createNiceMock(GetNewApplicationResponse.class);
+ EasyMock.expect(mockYarnClient.createApplication())
+ .andReturn(new YarnClientApplication(newApp,
+ Records.newRecord(ApplicationSubmissionContext.class)));
+ }
+
+ @Test
+ public void testExtractYarnResourceManagerAmRetryCountWindowMs() throws
+ Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ final String expectedInterval = Integer.toString (120000);
+ options.put(ResourceKeys.YARN_RESOURCEMANAGER_AM_RETRY_COUNT_WINDOW_MS,
+ expectedInterval);
+ EasyMock.replay(mockYarnClient, yarnClientApp);
+
+ appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags);
+
+ ApplicationSubmissionContext ctx = appMasterLauncher.application
+ .getApplicationSubmissionContext();
+ String retryIntervalWindow = Long.toString(ctx
+ .getAttemptFailuresValidityInterval());
+ Assert.assertEquals(expectedInterval, retryIntervalWindow);
+ }
+
+ @Test
+ public void testExtractYarnResourceManagerAmRetryCountWindowMsDefaultValue()
+ throws Exception {
+ Map<String, String> options = new HashMap<String, String>();
+ EasyMock.replay(mockYarnClient, yarnClientApp);
+
+ appMasterLauncher = new AppMasterLauncher("am1", SliderKeys.APP_TYPE, null,
+ null, mockYarnClient, false, null, options, tags);
+
+ ApplicationSubmissionContext ctx = appMasterLauncher.application
+ .getApplicationSubmissionContext();
+ long retryIntervalWindow = ctx.getAttemptFailuresValidityInterval();
+ Assert.assertEquals(ResourceKeys.DEFAULT_AM_RETRY_COUNT_WINDOW_MS,
+ retryIntervalWindow);
+ }
+
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
index 1fbbfce..b919fcf 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
@@ -44,6 +44,7 @@
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.powermock.api.easymock.PowerMock;
+import org.powermock.core.classloader.annotations.PowerMockIgnore;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
@@ -62,6 +63,7 @@
*
*/
@RunWith(PowerMockRunner.class)
+@PowerMockIgnore("org.apache.log4j.*")
@PrepareForTest({ProviderUtils.class, ProcessBuilder.class, AgentClientProvider.class, RegistryUtils.class})
public class TestAgentClientProvider2 extends SliderTestUtils {
protected static final Logger log =
@@ -235,6 +237,8 @@
@Test
public void testSliderClientForInstallFailures() throws Exception {
+ describe(" IGNORE ANY STACK TRACES BELOW ");
+
SliderClient client = new SliderClient();
client.bindArgs(new Configuration(), "client", "--dest", "a_random_path/none", "--package", "a_random_pkg.zip");
ActionClientArgs args = new ActionClientArgs();
@@ -274,6 +278,7 @@
assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_CONFIGURATION,
SliderClient.E_MUST_BE_A_VALID_JSON_FILE);
}
+ describe(" END IGNORE ");
args.clientConfig = null;
try {
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index d37fcea..7237ff4 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -24,10 +24,10 @@
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
-import junit.framework.Assert;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
@@ -35,10 +35,12 @@
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
@@ -71,26 +73,19 @@
public boolean verify(String hostname,
javax.net.ssl.SSLSession sslSession) {
- if (hostname.equals("localhost")) {
- return true;
- }
- return false;
+ return hostname.equals("localhost");
}
});
-
}
protected static final Logger log =
LoggerFactory.getLogger(TestAMAgentWebServices.class);
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
public static final String AGENT_URL =
"https://localhost:${PORT}/ws/v1/slider/agents/";
static MockFactory factory = new MockFactory();
- private static Configuration conf = new Configuration();
private static WebAppApi slider;
private static FileSystem fs;
@@ -117,28 +112,16 @@
YarnConfiguration conf = SliderUtils.createConfiguration();
fs = FileSystem.get(new URI("file:///"), conf);
AppState appState = null;
- try {
- fs = FileSystem.get(new URI("file:///"), conf);
- File
- historyWorkDir =
- new File("target/history", "TestAMAgentWebServices");
- org.apache.hadoop.fs.Path
- historyPath =
- new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
- fs.delete(historyPath, true);
- appState = new AppState(new MockRecordFactory(), new MetricsAndMonitoring());
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null, new SimpleReleaseSelector());
- } catch (Exception e) {
- log.error("Failed to set up app {}", e, e);
- }
+ File historyWorkDir = new File("target/history", "TestAMAgentWebServices");
+ Path historyPath = new Path(historyWorkDir.toURI());
+ fs.delete(historyPath, true);
+ appState = new MockAppState(new MockClusterServices());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0);
+ binding.roles = MockFactory.ROLES;
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ appState.buildInstance(binding);
ProviderAppState providerAppState = new ProviderAppState("undefined",
appState);
@@ -173,7 +156,7 @@
WebResource webResource = client.resource(base_url + "test/register");
response = webResource.type(MediaType.APPLICATION_JSON)
.post(RegistrationResponse.class, createDummyJSONRegister());
- Assert.assertEquals(RegistrationStatus.OK, response.getResponseStatus());
+ assertEquals(RegistrationStatus.OK, response.getResponseStatus());
}
protected Client createTestClient() {
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 90d40c6..49ad71a 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -31,7 +31,6 @@
import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.AggregateConf;
@@ -39,13 +38,13 @@
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.persist.JsonSerDeser;
-import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
+import org.apache.slider.server.appmaster.model.mock.MockAppState;
+import org.apache.slider.server.appmaster.model.mock.MockClusterServices;
import org.apache.slider.server.appmaster.model.mock.MockFactory;
import org.apache.slider.server.appmaster.model.mock.MockProviderService;
-import org.apache.slider.server.appmaster.model.mock.MockRecordFactory;
import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo;
import org.apache.slider.server.appmaster.state.ProviderAppState;
-import org.apache.slider.server.appmaster.state.SimpleReleaseSelector;
import org.apache.slider.server.appmaster.web.WebAppApi;
import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.AMWebServices;
@@ -67,15 +66,12 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.assertTrue;
public class TestAMManagementWebServices extends JerseyTest {
protected static final Logger log =
LoggerFactory.getLogger(TestAMManagementWebServices.class);
- public static final int RM_MAX_RAM = 4096;
- public static final int RM_MAX_CORES = 64;
- public static final String EXAMPLES =
- "/org/apache/slider/core/conf/examples/";
+ public static final String EXAMPLES = "/org/apache/slider/core/conf/examples/";
static MockFactory factory = new MockFactory();
private static Configuration conf = new Configuration();
private static WebAppApi slider;
@@ -116,40 +112,27 @@
}
protected AggregateConf getAggregateConf() {
- JsonSerDeser<ConfTree> confTreeJsonSerDeser =
- new JsonSerDeser<ConfTree>(ConfTree.class);
- ConfTree internal = null;
- ConfTree app_conf = null;
- ConfTree resources = null;
try {
- internal =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES +"internal.json");
- app_conf =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES + "app_configuration.json");
- resources =
- confTreeJsonSerDeser.fromResource(
- EXAMPLES + "resources.json");
+ JsonSerDeser<ConfTree> confTreeJsonSerDeser = new JsonSerDeser<>(ConfTree.class);
+ AggregateConf aggregateConf = new AggregateConf(
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "resources.json"),
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "app_configuration.json"),
+ confTreeJsonSerDeser.fromResource(EXAMPLES + "internal.json")
+ );
+ aggregateConf.setName("test");
+ return aggregateConf;
} catch (IOException e) {
- fail(e.getMessage());
+ throw new AssertionError(e.getMessage(), e);
}
- AggregateConf aggregateConf = new AggregateConf(
- resources,
- app_conf,
- internal);
- aggregateConf.setName("test");
- return aggregateConf;
}
-
}
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
injector = createInjector();
- YarnConfiguration conf = SliderUtils.createConfiguration();
- fs = FileSystem.get(new URI("file:///"), conf);
+ fs = FileSystem.get(new URI("file:///"), SliderUtils.createConfiguration());
}
private static Injector createInjector() {
@@ -160,31 +143,18 @@
AppState appState = null;
try {
fs = FileSystem.get(new URI("file:///"), conf);
- File
- historyWorkDir =
- new File("target/history", "TestAMManagementWebServices");
- org.apache.hadoop.fs.Path
- historyPath =
+ File historyWorkDir = new File("target/history", "TestAMManagementWebServices");
+ org.apache.hadoop.fs.Path historyPath =
new org.apache.hadoop.fs.Path(historyWorkDir.toURI());
fs.delete(historyPath, true);
- appState = new AppState(new MockRecordFactory(), new MetricsAndMonitoring());
- appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
- appState.buildInstance(
- factory.newInstanceDefinition(0, 0, 0),
- new Configuration(),
- new Configuration(false),
- factory.ROLES,
- fs,
- historyPath,
- null, null, new SimpleReleaseSelector());
-// JDK7 } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
- } catch (IOException e) {
- log.error("{}", e, e);
- } catch (BadClusterStateException e) {
- log.error("{}", e, e);
- } catch (URISyntaxException e) {
- log.error("{}", e, e);
- } catch (BadConfigException e) {
+ appState = new MockAppState(new MockClusterServices());
+ AppStateBindingInfo binding = new AppStateBindingInfo();
+ binding.instanceDefinition = factory.newInstanceDefinition(0, 0, 0);
+ binding.roles = MockFactory.ROLES;
+ binding.fs = fs;
+ binding.historyPath = historyPath;
+ appState.buildInstance(binding);
+ } catch (IOException | BadClusterStateException | URISyntaxException | BadConfigException e) {
log.error("{}", e, e);
}
ProviderAppState providerAppState = new ProviderAppState("undefined",
@@ -246,9 +216,19 @@
assertEquals("wrong href",
"http://localhost:9998/slideram/ws/v1/slider/mgmt/app/configurations/internal",
json.getHref());
- assertEquals("wrong description",
- "Internal configuration DO NOT EDIT",
- json.getMetadata().get("description"));
+
+ assertDescriptionContains("org/apache/slider/core/conf/examples/internal.json", json);
+ }
+
+ private void assertDescriptionContains(String expected, ConfTreeResource json) {
+
+ Map<String, Object> metadata = json.getMetadata();
+ assertNotNull("No metadata", metadata);
+ Object actual = metadata.get("description");
+ assertNotNull("No description", actual);
+
+ assertTrue(String.format("Did not find \"%s\" in \"%s\"", expected, actual),
+ actual.toString().contains(expected));
}
@Test
@@ -270,6 +250,7 @@
assertNotNull("no components", components);
assertEquals("incorrect number of components", 2, components.size());
assertNotNull("wrong component", components.get("worker"));
+ assertDescriptionContains("org/apache/slider/core/conf/examples/resources.json", json);
}
@Test
@@ -290,5 +271,7 @@
assertNotNull("no components", components);
assertEquals("incorrect number of components", 2, components.size());
assertNotNull("wrong component", components.get("worker"));
+ assertDescriptionContains("org/apache/slider/core/conf/examples/app_configuration.json", json);
+
}
}
diff --git a/slider-core/src/test/python/agent.py b/slider-core/src/test/python/agent.py
index 4177074..1c9ff5b 100644
--- a/slider-core/src/test/python/agent.py
+++ b/slider-core/src/test/python/agent.py
@@ -24,14 +24,21 @@
from optparse import OptionParser
import os
+
# A representative Agent code for the embedded agent
def main():
- print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ print "Executing src/test/python/agent.py"
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
parser.add_option("--config", dest="conf_folder", help="conf folder")
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
+
(options, args) = parser.parse_args()
if options.log_folder:
@@ -43,9 +50,17 @@
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 30
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
diff --git a/slider-core/src/test/python/agent/main.py b/slider-core/src/test/python/agent/main.py
index 116d179..2eacc89 100755
--- a/slider-core/src/test/python/agent/main.py
+++ b/slider-core/src/test/python/agent/main.py
@@ -26,7 +26,7 @@
def main():
- print "Executing echo"
+ print "Executing src/test/python/agent/main.py"
try:
print 'Argument List: {0}'.format(str(sys.argv))
except AttributeError:
@@ -37,6 +37,8 @@
parser.add_option("--config", dest="conf_folder", help="conf folder")
parser.add_option('--command', dest='command', help='command to execute')
parser.add_option('--label', dest='label', help='label')
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
parser.add_option('--zk-quorum', dest='host:2181', help='zookeeper quorum')
parser.add_option('--zk-reg-path', dest='/register/org-apache-slider/cl1', help='zookeeper registry path')
@@ -51,9 +53,17 @@
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 300
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
diff --git a/slider-core/src/test/python/echo.py b/slider-core/src/test/python/echo.py
index ea5e8ce..1dabf1c 100644
--- a/slider-core/src/test/python/echo.py
+++ b/slider-core/src/test/python/echo.py
@@ -26,13 +26,18 @@
def main():
- print "Executing echo"
- print 'Argument List: {0}'.format(str(sys.argv))
+ print "Executing src/test/python/echo.py"
+ try:
+ print 'Argument List: {0}'.format(str(sys.argv))
+ except AttributeError:
+ pass
parser = OptionParser()
parser.add_option("--log", dest="log_folder", help="log destination")
parser.add_option("--config", dest="conf_folder", help="conf folder")
parser.add_option('--command', dest='command', help='command to execute')
+ parser.add_option('--sleep', dest='sleep', help='sleep time')
+ parser.add_option('--exitcode', dest='exitcode', help='exit code to return')
(options, args) = parser.parse_args()
if options.log_folder:
@@ -44,9 +49,17 @@
logging.info("Number of arguments: %s arguments.", str(len(sys.argv)))
logging.info("Argument List: %s", str(sys.argv))
- time.sleep(30)
+ sleeptime = 300
+ if options.sleep:
+ sleeptime = int(options.sleep)
+ if sleeptime > 0:
+ logging.info("Sleeping for %d seconds", sleeptime)
+ time.sleep(sleeptime)
+ exitcode = 0
+ if options.exitcode:
+ exitcode = int(options.exitcode)
+ return exitcode
if __name__ == "__main__":
- main()
- sys.exit(0)
+ sys.exit(main())
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index cf4afe1..b7251d3 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -24,7 +24,7 @@
</comment>
<version>0.1</version>
<type>YARN-APP</type>
- <minHadoopVersion>2.1.0</minHadoopVersion>
+ <minHadoopVersion>2.6.0</minHadoopVersion>
<components>
<component>
<name>hbase-rs</name>
@@ -51,7 +51,7 @@
<name>echo</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
+ <maxInstanceCount>100</maxInstanceCount>
<commandScript>
<script>echo.py</script>
<scriptType>PYTHON</scriptType>
diff --git a/slider-core/src/test/resources/example-slider-test.xml b/slider-core/src/test/resources/example-slider-test.xml
index abf42f9..ee9fc59 100644
--- a/slider-core/src/test/resources/example-slider-test.xml
+++ b/slider-core/src/test/resources/example-slider-test.xml
@@ -33,12 +33,6 @@
</property>
<property>
- <name>slider.test.teardown.killall</name>
- <description>Kill all hbase/accumulo, etc processes on test teardown</description>
- <value>true</value>
- </property>
-
- <property>
<name>slider.test.thaw.wait.seconds</name>
<description>Time to wait for a start to work</description>
<value>60</value>
@@ -50,68 +44,4 @@
<value>60</value>
</property>
-
- <!-- Properties for the slider-hbase-provider only -not HBase-under-agent- -->
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.hbase.launch.wait.seconds</name>
- <description>Time to wait for the HBase application to be live</description>
- <value>180</value>
- </property>
-
- <property>
- <name>slider.test.hbase.home</name>
- <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1</value>
- <description>HBASE Home</description>
- </property>
-
- <property>
- <name>slider.test.hbase.tar</name>
- <value>/home/slider/Projects/hbase/hbase-assembly/target/hbase-0.98.1-bin.tar.gz</value>
- <description>HBASE archive URI</description>
- </property>
-
- <!-- Properties for the slider-accumulo-provider only -not HBase-under-agent- -->
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.launch.wait.seconds</name>
- <description>Time to wait for the accumulo application to be live</description>
- <value>180</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.home</name>
- <value>/home/slider/accumulo</value>
- <description>Accumulo Home</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.tar</name>
- <value>/home/slider/Projects/accumulo/accumulo-1.6.0-bin.tar.gz</value>
- <description>Accumulo archive URI</description>
- </property>
-
- <property>
- <name>zk.home</name>
- <value>/home/slider/zookeeper</value>
- <description>Zookeeper home dir on target systems</description>
- </property>
-
- <property>
- <name>hadoop.home</name>
- <value>/home/slider/hadoop/</value>
- <description>Hadoop home dir on target systems</description>
- </property>
-
</configuration>
diff --git a/slider-core/src/test/resources/log4j.properties b/slider-core/src/test/resources/log4j.properties
index ed19a5b..5858f55 100644
--- a/slider-core/src/test/resources/log4j.properties
+++ b/slider-core/src/test/resources/log4j.properties
@@ -38,6 +38,7 @@
#crank back on some noise
log4j.logger.org.apache.hadoop.util.Shell=ERROR
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager=FATAL
log4j.logger.org.apache.hadoop.hdfs.server.datanode.BlockPoolSliceScanner=WARN
log4j.logger.org.apache.hadoop.hdfs.server.blockmanagement=WARN
log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=WARN
@@ -49,6 +50,7 @@
log4j.logger.org.apache.zookeeper=WARN
log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitorImpl=ERROR
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
log4j.logger.org.apache.hadoop.metrics2=ERROR
log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
index 489acda..5690225 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/app_configuration.json
@@ -1,6 +1,8 @@
{
"schema": "http://example.org/specification/v2.0.0",
-
+ "metadata": {
+ "description": "org/apache/slider/core/conf/examples/app_configuration.json"
+ },
"global": {
"zookeeper.port": "2181",
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
index 592b4dc..da53b94 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal-resolved.json
@@ -2,7 +2,7 @@
"schema": "http://example.org/specification/v2.0.0",
"metadata": {
- "description": "Internal configuration DO NOT EDIT"
+ "description": "Internal resolved - org/apache/slider/core/conf/examples/internal-resolved.json"
},
"global": {
"application.name": "small_cluster",
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
index 4c782fb..b628d10 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/internal.json
@@ -2,7 +2,7 @@
"schema": "http://example.org/specification/v2.0.0",
"metadata": {
- "description": "Internal configuration DO NOT EDIT"
+ "description": "Internal unresolved - org/apache/slider/core/conf/examples/internal.json"
},
"global": {
"application.name": "small_cluster",
diff --git a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
index 9d1e916..206424d 100644
--- a/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
+++ b/slider-core/src/test/resources/org/apache/slider/core/conf/examples/resources.json
@@ -2,7 +2,7 @@
"schema": "http://example.org/specification/v2.0.0",
"metadata": {
- "description": "example of a resources file"
+ "description": "example of a resources file: org/apache/slider/core/conf/examples/resources.json"
},
"global": {
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index cfbc401..2d0ac58 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
<properties>
<work.dir>package-tmp</work.dir>
diff --git a/slider-funtest/src/license/THIRD-PARTY.properties b/slider-funtest/src/license/THIRD-PARTY.properties
index 2279ca6..4a2ba24 100644
--- a/slider-funtest/src/license/THIRD-PARTY.properties
+++ b/slider-funtest/src/license/THIRD-PARTY.properties
@@ -8,6 +8,7 @@
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - New BSD License
@@ -20,8 +21,7 @@
# Please fill the missing licenses for dependencies :
#
#
-#Mon Jan 26 15:56:57 GMT 2015
-asm--asm--3.1=The BSD 3-Clause License
+#Thu Oct 15 16:45:03 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
index 60d3ee2..12890b7 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentCommandTestBase.groovy
@@ -192,7 +192,8 @@
shell = slider([
ACTION_DESTROY,
- applicationName])
+ applicationName,
+ ARG_FORCE])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
assertExitCode(shell, 0)
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 4c02289..5fa4c2a 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -23,22 +23,23 @@
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.HdfsConfiguration
import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.hadoop.util.ExitUtil
import org.apache.hadoop.util.Shell
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.StatusKeys
-import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.launch.SerializedApplicationReport
-import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
+import org.apache.slider.common.tools.ConfigHelper
import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.client.SliderClient
+import org.apache.slider.core.launch.SerializedApplicationReport
+import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.core.persist.ApplicationReportSerDeser
+import org.apache.slider.core.persist.JsonSerDeser
import org.apache.slider.test.SliderTestUtils
import org.apache.slider.test.Outcome;
@@ -89,7 +90,6 @@
* Keytab for secure cluster
*/
public static final String TEST_AM_KEYTAB
- static File keytabFile
/**
* shell-escaped ~ symbol. On windows this does
@@ -106,7 +106,6 @@
(which it may) the class will not be instantiable.
*/
static {
- new HdfsConfiguration()
ConfigHelper.injectSliderXMLResource()
ConfigHelper.registerDeprecatedConfigItems();
SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true);
@@ -209,7 +208,7 @@
return false;
}
}
-
+
/**
* Add a jar to the slider classpath by looking up a class and determining
* its containing JAR
@@ -291,13 +290,13 @@
static SliderShell destroy(String name) {
slider([
- ACTION_DESTROY, name
+ ACTION_DESTROY, name, ARG_FORCE
])
}
static SliderShell destroy(int result, String name) {
slider(result, [
- ACTION_DESTROY, name
+ ACTION_DESTROY, name, ARG_FORCE
])
}
@@ -328,7 +327,6 @@
])
}
-
static SliderShell freeze(
int exitCode,
String name,
@@ -362,9 +360,9 @@
static SliderShell killContainer(String name, String containerID) {
slider(0,
[
- ACTION_KILL_CONTAINER,
- name,
- containerID
+ ACTION_KILL_CONTAINER,
+ name,
+ containerID
])
}
@@ -443,9 +441,7 @@
}
static SliderShell registry(Collection<String> commands) {
- slider(0,
- [ACTION_REGISTRY] + commands
- )
+ slider(0, [ACTION_REGISTRY] + commands)
}
/**
@@ -575,7 +571,7 @@
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
- SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
+ SLIDER_CONFIG.getTrimmed(RegistryConstants.KEY_REGISTRY_ZK_QUORUM)
if (blockUntilRunning) {
@@ -777,24 +773,6 @@
}
/**
- * Create a temp JSON file. After coming up with the name, the file
- * is deleted
- * @return the filename
- */
- public static File createTempJsonFile() {
- return tmpFile(".json")
- }
-
- public static File tmpFile(String suffix) {
- File reportFile = File.createTempFile(
- "launch",
- suffix,
- new File("target"))
- reportFile.delete()
- return reportFile
- }
-
- /**
* If the option is not null/empty, add the command and the option
* @param args arg list being built up
* @param command command to add option
@@ -815,20 +793,20 @@
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
return report
- }
+ }
return null;
- }
-
+ }
+
public static SerializedApplicationReport loadAppReport(File reportFile) {
- if (reportFile.exists() && reportFile.length()> 0) {
+ if (reportFile.exists() && reportFile.length() > 0) {
ApplicationReportSerDeser serDeser = new ApplicationReportSerDeser()
def report = serDeser.fromFile(reportFile)
return report
- } else {
+ }else {
throw new FileNotFoundException(reportFile.absolutePath)
- }
- }
-
+ }
+ }
+
public static SerializedApplicationReport maybeLookupFromLaunchReport(File launchReport) {
def report = maybeLoadAppReport(launchReport)
if (report) {
@@ -859,7 +837,50 @@
}
}
-
+ /**
+ * Lookup an application, return null if loading failed
+ * @param id application ID
+ * @return an application report or null
+ */
+ public static NodeInformationList listNodes(String name = "", boolean healthy = false, String label = "") {
+ File reportFile = createTempJsonFile();
+ try {
+ def shell = nodes(name, reportFile, healthy, label)
+ if (log.isDebugEnabled()) {
+ shell.dumpOutput()
+ }
+ JsonSerDeser<NodeInformationList> serDeser = NodeInformationList.createSerializer();
+ serDeser.fromFile(reportFile)
+ } finally {
+ reportFile.delete()
+ }
+ }
+
+ /**
+ * List cluster nodes
+ * @param name of cluster or null
+ * @param out output file (or null)
+ * @param healthy list healthy nodes only
+ * @param label label to filter on
+ * @return output
+ */
+ static SliderShell nodes(String name, File out = null, boolean healthy = false, String label = "") {
+ def commands = [ACTION_NODES]
+ if (label) {
+ commands += [ ARG_LABEL, label]
+ }
+ if (name) {
+ commands << name
+ }
+ if (out) {
+ commands += [ARG_OUTPUT, out.absolutePath]
+ }
+ if (healthy) {
+ commands << ARG_HEALTHY
+ }
+ slider(0, commands)
+ }
+
public Path buildClusterPath(String clustername) {
return new Path(
clusterFS.homeDirectory,
@@ -933,11 +954,7 @@
}
// trigger a failure on registry lookup
- SliderShell shell = registry(0, [
- ARG_NAME,
- application,
- ARG_LISTEXP
- ])
+ registry(0, [ARG_NAME, application, ARG_LISTEXP])
}
}
@@ -1208,7 +1225,6 @@
}
public ClusterDescription execStatus(String application) {
- ClusterDescription cd
File statusFile = File.createTempFile("status", ".json")
try {
slider(EXIT_SUCCESS,
@@ -1282,6 +1298,13 @@
}
}
+ /**
+ * Assert that exactly the number of containers are live
+ * @param clustername name of cluster
+ * @param component component to probe
+ * @param count count
+ * @return
+ */
public ClusterDescription assertContainersLive(String clustername,
String component,
int count) {
@@ -1428,8 +1451,8 @@
}
/**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
+ * probe for the output {@code command: List} containing {@code text}
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome commandOutputContains(Map args) {
@@ -1438,9 +1461,10 @@
SliderShell shell = slider(0, command)
return Outcome.fromBool(shell.outputContains(text))
}
+
/**
- * Is the registry accessible for an application?
- * @param args argument map containing <code>"application"</code>
+ * probe for a command {@code command: List} succeeeding
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome commandSucceeds(Map args) {
@@ -1448,9 +1472,11 @@
SliderShell shell = slider(command)
return Outcome.fromBool(shell.ret == 0)
}
+
/**
- * Is the registry accessible for an application?
- * @param args argument map
+ * probe for a command {@code command: List} generating a file 'filename'
+ * which must contain the text 'text'
+ * @param args argument map containing the required parameters
* @return probe outcome
*/
protected Outcome generatedFileContains(Map args) {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
new file mode 100644
index 0000000..5de2b8e
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest
+
+/**
+ * The various resources used for test runs
+ */
+interface ResourcePaths {
+
+ String SLIDER_CORE_TEST_SRC= "../slider-core/src/test"
+ String SLIDER_CORE_APP_PACKAGES = "$SLIDER_CORE_TEST_SRC/app_packages"
+ String COMMAND_LOG_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources.json"
+ String COMMAND_LOG_RESOURCES_QUEUE_LABELS = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources_queue_labels.json"
+ String COMMAND_LOG_RESOURCES_NO_ROLE = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources_no_role.json"
+ String COMMAND_LOG_APPCONFIG_NO_HB = "$SLIDER_CORE_APP_PACKAGES/test_command_log/appConfig_no_hb.json"
+ String COMMAND_LOG_APPCONFIG_FAST_NO_REG = "$SLIDER_CORE_APP_PACKAGES/test_command_log/appConfig_fast_no_reg.json"
+
+ String PING_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/nc_ping_cmd/resources.json"
+ String PING_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/nc_ping_cmd/metainfo.json"
+ String PING_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/nc_ping_cmd/appConfig.json"
+
+ String SLEEP_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources.json"
+ String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json"
+ String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json"
+
+}
\ No newline at end of file
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
index 9826e97..5d069bc 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ClusterConnectivityIT.groovy
@@ -37,7 +37,6 @@
*/
class ClusterConnectivityIT extends CommandTestBase {
-
public static final int CONNECT_TIMEOUT = 2000
@Test
@@ -60,7 +59,7 @@
tuples.each {
telnet(it.hostText, it.port)
}
-
+
}
@Test
@@ -73,7 +72,7 @@
telnet(rmAddr.hostName, rmAddr.port)
}
}
-
+
@Test
public void testRMBinding() throws Throwable {
SliderYarnClientImpl yarnClient = new SliderYarnClientImpl()
@@ -83,10 +82,10 @@
YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,5000)
SLIDER_CONFIG.setInt(
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,50)
-
+
yarnClient.init(SLIDER_CONFIG)
yarnClient.start();
- def instances = yarnClient.listInstances("")
+ def instances = yarnClient.listDeployedInstances("")
instances.each {it -> log.info("Instance $it.applicationId")}
} finally {
yarnClient.stop()
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
index cf74bbf..91797f9 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/ComponentConfigsInAppConfigShowUpOnAgentIT.groovy
@@ -23,9 +23,6 @@
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
import org.junit.Before
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
index 728920e..e53c875 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/basic/SyspropsIT.groovy
@@ -36,11 +36,10 @@
@Slf4j
class SyspropsIT extends SliderTestUtils{
-
@Test
public void testDumpSysprops() throws Throwable {
def sysprops = System.properties
- TreeSet<String> sorted = new TreeSet<String>();
+ TreeSet<String> sorted = new TreeSet<>();
sysprops.keys().each { String it -> sorted.add(it)}
sorted.each { String key ->
log.info("$key=\"${sysprops[key]}\"")
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
index 73912e6..666342e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/CommandExitCodesIT.groovy
@@ -20,7 +20,6 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderExitCodes
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.Test
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
index 916117c..b378110 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/ListCommandIT.groovy
@@ -22,7 +22,6 @@
import groovy.util.logging.Slf4j
import org.apache.slider.core.main.LauncherExitCodes
import org.apache.slider.funtest.framework.CommandTestBase
-import org.junit.BeforeClass
import org.junit.Test
@CompileStatic
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
index 75d0634..e22c5f2 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/commands/SimpleCommandsIT.groovy
@@ -20,9 +20,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.bigtop.itest.shell.Shell
import org.apache.slider.funtest.framework.CommandTestBase
-import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.common.params.SliderActions
import org.junit.Test
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
index 43275e6..b32275e 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
@@ -18,28 +18,9 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-
-import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderUtils
-import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.apache.slider.funtest.framework.FileUploader
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.CommandTestBase
import org.junit.After
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
new file mode 100644
index 0000000..c42edf8
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AASleepIT.groovy
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.types.NodeEntryInformation
+import org.apache.slider.api.types.NodeInformation
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.launch.SerializedApplicationReport
+import org.apache.slider.funtest.ResourcePaths
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AASleepIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ static String NAME = "test-aa-sleep"
+
+ static String TEST_RESOURCE = ResourcePaths.SLEEP_RESOURCES
+ static String TEST_METADATA = ResourcePaths.SLEEP_META
+ public static final String SLEEP_100 = "SLEEP_100"
+ public static final int SLEEP_LONG_PRIORITY = 3
+ public static final String SLEEP_LONG_PRIORITY_S = Integer.toString(SLEEP_LONG_PRIORITY)
+
+ public static final String SLEEP_LONG = "SLEEP_LONG"
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(NAME)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(NAME)
+ }
+
+ @Test
+ public void testAASleepIt() throws Throwable {
+ describe("Test Anti-Affinity Placement")
+
+ describe "diagnostics"
+
+ slider([ACTION_DIAGNOSTICS, ARG_VERBOSE, ARG_CLIENT, ARG_YARN, ARG_CREDENTIALS]).dumpOutput()
+
+ describe "list nodes"
+
+ def healthyNodes = listNodes("", true)
+
+ def healthyNodeCount = healthyNodes.size()
+ describe("Cluster nodes : ${healthyNodeCount}")
+ log.info(NodeInformationList.createSerializer().toJson(healthyNodes))
+
+ File launchReportFile = createTempJsonFile();
+
+ int desired = buildDesiredCount(healthyNodeCount)
+
+ SliderShell shell = createSliderApplicationMinPkg(NAME,
+ TEST_METADATA,
+ TEST_RESOURCE,
+ ResourcePaths.SLEEP_APPCONFIG,
+ [
+ ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, Integer.toString( desired),
+ ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_PRIORITY, SLEEP_LONG_PRIORITY_S
+ ],
+ launchReportFile)
+
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+
+ status(0, NAME)
+
+ def expected = buildExpectedCount(desired)
+ expectLiveContainerCountReached(NAME, SLEEP_100, expected,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ operations(NAME, loadAppReport(launchReportFile), desired, expected, healthyNodes)
+
+ //stop
+ freeze(0, NAME,
+ [
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
+
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
+ destroy(0, NAME)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, NAME)
+ }
+
+ protected int buildExpectedCount(int desired) {
+ desired - 1
+ }
+
+ protected int buildDesiredCount(int clustersize) {
+ clustersize + 1
+ }
+
+ protected void operations(String name,
+ SerializedApplicationReport appReport,
+ int desired,
+ int expected,
+ NodeInformationList healthyNodes) {
+
+ // now here await for the cluster size to grow: if it does, there's a problem
+ // spin for a while and fail if the number ever goes above it.
+ ClusterDescription cd = null
+ 5.times {
+ cd = assertContainersLive(NAME, SLEEP_LONG, expected)
+ sleep(1000 * 10)
+ }
+
+ // here cluster is still 1 below expected
+ def role = cd.getRole(SLEEP_LONG)
+ assert "1" == role.get(RoleKeys.ROLE_PENDING_AA_INSTANCES)
+
+ // look through the nodes
+ def currentNodes = listNodes(name)
+ // assert that there is no entry of the sleep long priority on any node
+ currentNodes.each { NodeInformation it ->
+ def entry = it.entries[SLEEP_LONG]
+ assert entry == null || entry.live <= 1
+ }
+
+ // now reduce the cluster size and assert that the size stays the same
+
+
+ }
+}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
index 950dc68..05adebd 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMClientCertStoreRetrievalIT.groovy
@@ -33,7 +33,6 @@
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.junit.After
-import org.junit.Assert
import org.junit.Test
import javax.net.ssl.TrustManager
@@ -47,7 +46,6 @@
import java.security.cert.CertificateException
import java.security.cert.X509Certificate
import com.google.common.io.Files
-import java.io.File
@CompileStatic
@Slf4j
@@ -107,7 +105,8 @@
String password = "welcome";
// ensure file doesn't exist
- new File(filename).delete();
+ def keystoreFile = new File(filename)
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -118,7 +117,7 @@
ARG_PASSWORD, password
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
KeyStore keystore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -126,7 +125,7 @@
filename = myTempDir.canonicalPath + File.separator + "test.truststore"
// ensure file doesn't exist
- new File(filename).delete();
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -137,7 +136,7 @@
ARG_PASSWORD, password
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
KeyStore truststore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -164,7 +163,7 @@
log.info("Created credential provider $providerString for test")
// ensure file doesn't exist
- new File(filename).delete();
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -176,7 +175,7 @@
ARG_PROVIDER, providerString
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
keystore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -184,7 +183,7 @@
filename = myTempDir.canonicalPath + File.separator + "test.truststore"
// ensure file doesn't exist
- new File(filename).delete();
+ keystoreFile.delete();
shell = slider(EXIT_SUCCESS,
[
@@ -196,7 +195,7 @@
ARG_PROVIDER, providerString
])
- assert new File(filename).exists()
+ assert keystoreFile.exists()
truststore = loadKeystoreFromFile(filename, password.toCharArray())
@@ -207,7 +206,7 @@
private static void validateKeystore(KeyStore keystore) {
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
+ assert certificate
String hostname = InetAddress.localHost.canonicalHostName;
@@ -217,9 +216,7 @@
// Get subject
Principal principal = x509cert.getSubjectDN();
String subjectDn = principal.getName();
- Assert.assertEquals("wrong DN",
- "CN=" + hostname + ", OU=" + APPLICATION_NAME + ", OU=client",
- subjectDn);
+ assert subjectDn == "CN=" + hostname + ", OU=" + APPLICATION_NAME + ", OU=client"
}
}
@@ -229,7 +226,7 @@
// obtain server cert
Certificate certificate = keystore.getCertificate(
keystore.aliases().nextElement());
- Assert.assertNotNull(certificate);
+ assert certificate
// validate keystore cert using trust store
TrustManagerFactory trustManagerFactory =
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
index deb2d97..6d1f759 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
@@ -21,19 +21,12 @@
import groovy.io.FileType
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.yarn.api.records.YarnApplicationState
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
-import org.junit.After
-import org.junit.Before
import org.junit.Test
@CompileStatic
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
index ca9f71b..faeb0a1 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClusterLifecycleIT.groovy
@@ -28,6 +28,7 @@
import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -43,7 +44,7 @@
static String CLUSTER = "test-agent-cluster-lifecycle"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
index 39b5d6c..d73eb76 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailures2IT.groovy
@@ -23,6 +23,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -36,8 +37,7 @@
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "two-container-fail-heartbeat"
- private static String APP_TEMPLATE3 =
- "../slider-core/src/test/app_packages/test_command_log/appConfig_no_hb.json"
+ private static String APP_TEMPLATE3 = ResourcePaths.COMMAND_LOG_APPCONFIG_NO_HB
@After
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
index 7d1be89..cfdff75 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentFailuresIT.groovy
@@ -23,6 +23,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -36,8 +37,7 @@
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "one-container-fail-register"
- private static String APP_TEMPLATE2 =
- "../slider-core/src/test/app_packages/test_command_log/appConfig_fast_no_reg.json"
+ private static String APP_TEMPLATE2 = ResourcePaths.COMMAND_LOG_APPCONFIG_FAST_NO_REG
@After
public void destroyCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy
index 17eaf04..ee22360 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentLaunchFailureIT_Disabled.groovy
@@ -21,6 +21,7 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.server.appmaster.SliderAppMaster
import static org.apache.slider.api.InternalKeys.*
@@ -41,7 +42,7 @@
static String CLUSTER = "test-agent-launchfail"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
public void prepareCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
index e7b0454..779316b 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
@@ -20,18 +20,11 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.Endpoint
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.Outcome
-
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -47,9 +40,8 @@
static String CLUSTER = "test-agent-sleep-100"
- static String APP_RESOURCE11 = "../slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json"
- static String APP_META11 = "../slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json"
-
+ static String TEST_RESOURCE = ResourcePaths.SLEEP_RESOURCES
+ static String TEST_METADATA = ResourcePaths.SLEEP_META
@Before
public void prepareCluster() {
@@ -62,14 +54,14 @@
}
@Test
- public void testAgentRegistry() throws Throwable {
+ public void testAgentMinSleepIt() throws Throwable {
describe("Create a cluster using metainfo and resources only that executes sleep 100")
def clusterpath = buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
- APP_META11,
- APP_RESOURCE11,
+ TEST_METADATA,
+ TEST_RESOURCE,
null,
[],
launchReportFile)
@@ -91,7 +83,7 @@
CONTAINER_LAUNCH_TIMEOUT)
// sleep for some manual test
- describe("You may quickly perform manual tests against the application instance " + CLUSTER)
+ describe("You may quickly perform manual tests against the application instance $CLUSTER")
sleep(1000 * 30)
//stop
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
index ce2ed92..b10e792 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
@@ -22,18 +22,11 @@
import groovy.util.logging.Slf4j
import groovy.json.*
import org.apache.hadoop.net.NetUtils
-import org.apache.hadoop.registry.client.binding.RegistryUtils
-import org.apache.hadoop.registry.client.types.Endpoint
-import org.apache.hadoop.registry.client.types.ServiceRecord
import org.apache.hadoop.yarn.api.records.YarnApplicationState
import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.test.Outcome
-
-import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -49,9 +42,9 @@
static String CLUSTER = "test-agent-ping-port"
- static String APP_RESOURCE12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json"
- static String APP_META12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json"
- static String APP_TEMPLATE12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json"
+ static String APP_RESOURCE12 = ResourcePaths.PING_RESOURCES
+ static String APP_META12 = ResourcePaths.PING_META
+ static String APP_TEMPLATE12 = ResourcePaths.PING_APPCONFIG
@Before
@@ -68,7 +61,7 @@
public void testAgentRegistry() throws Throwable {
describe("Create a cluster using metainfo, resources, and appConfig that calls nc to listen on a port")
assumeNotWindows()
- def clusterpath = buildClusterPath(CLUSTER)
+ buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
index 4f9701c..7a03a05 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentRegistryIT.groovy
@@ -28,6 +28,7 @@
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.test.Outcome
import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
@@ -46,8 +47,7 @@
static String CLUSTER = "test-agent-registry"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
-
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
public void prepareCluster() {
@@ -160,7 +160,7 @@
Outcome probeForEntryMissing(Map args) {
- String path = args["path"]
+ String path = requiredMapValue(args, "path")
def shell = slider([ACTION_RESOLVE, ARG_PATH, path])
return Outcome.fromBool(shell.ret == EXIT_NOT_FOUND)
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
index 62be615..d1f8f20 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy
@@ -23,13 +23,9 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.registry.client.api.RegistryOperations
import org.apache.hadoop.security.UserGroupInformation
-import org.apache.hadoop.yarn.api.ApplicationClientProtocol
-import org.apache.hadoop.yarn.client.ClientRMProxy
-import org.apache.hadoop.yarn.webapp.ForbiddenException
import org.apache.slider.agent.rest.IpcApiClientTestDelegates
import org.apache.slider.agent.rest.JerseyTestDelegates
import org.apache.slider.agent.rest.AbstractRestTestDelegate
-import org.apache.slider.agent.rest.LowLevelRestTestDelegates
import org.apache.slider.agent.rest.RestAPIClientTestDelegates
import org.apache.slider.client.SliderClient
import org.apache.slider.client.ipc.SliderApplicationIpcClient
@@ -40,13 +36,13 @@
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
import org.apache.slider.server.appmaster.rpc.RpcBinder
import org.junit.After
import org.junit.Before
-import org.junit.Ignore
import org.junit.Test
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SYSTEM_HEALTHCHECK
@@ -59,7 +55,7 @@
static String CLUSTER = "test-agent-web"
- static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json"
+ static String APP_RESOURCE2 = ResourcePaths.COMMAND_LOG_RESOURCES_NO_ROLE
@Before
public void prepareCluster() {
@@ -72,12 +68,11 @@
}
@Test
- @Ignore("SLIDER-907")
public void testAgentWeb() throws Throwable {
describe("Web queries & REST operations against an AM")
// verify the ws/ path is open for all HTTP verbs
- def sliderConfiguration = ConfigHelper.loadSliderConfiguration();
+ ConfigHelper.loadSliderConfiguration();
/*
Is the back door required? If so, don't test complex verbs via the proxy
@@ -91,7 +86,7 @@
def directComplexVerbs = proxyComplexVerbs || SLIDER_CONFIG.getBoolean(
SliderXmlConfKeys.X_DEV_INSECURE_WS,
SliderXmlConfKeys.X_DEV_INSECURE_DEFAULT)
- def clusterpath = buildClusterPath(CLUSTER)
+ buildClusterPath(CLUSTER)
File launchReportFile = createTempJsonFile();
SliderShell shell = createTemplatedSliderApplication(CLUSTER,
APP_TEMPLATE,
@@ -110,77 +105,86 @@
def appId = ensureYarnApplicationIsUp(launchReportFile)
assert appId
awaitApplicationURLPublished(appId, instanceLaunchTime)
- File liveReportFile = createTempJsonFile();
- lookup(appId, liveReportFile)
- def report = loadAppReport(liveReportFile)
- assert report.url
+ def report = lookupApplication(appId)
+ assert report && report.url
def proxyAM = report.url
+ // decide whether or not to use https
+ def proxyTests = !proxyAM.toLowerCase(Locale.ENGLISH).startsWith("https:")
+
// here the URL has been published, but it may not be live yet, due to the time
// it takes the AM to build app state and boostrap the Web UI
def directAM = report.origTrackingUrl;
- describe "Proxy Jersey Tests"
-
Client jerseyClient = createUGIJerseyClient()
- JerseyTestDelegates proxyJerseyTests =
- new JerseyTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
+ if (proxyTests) {
+ describe "Proxy Jersey Tests"
- // wait it coming up
- awaitRestEndpointLive(proxyJerseyTests, instanceLaunchTime)
+ JerseyTestDelegates proxyJerseyTests =
+ new JerseyTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
- proxyJerseyTests.testSuiteGetOperations()
+ // wait it coming up
+ awaitRestEndpointLive(proxyJerseyTests, instanceLaunchTime)
+
+ proxyJerseyTests.testSuiteGetOperations()
+
+ describe "Proxy SliderRestClient Tests"
+ RestAPIClientTestDelegates proxySliderRestAPI =
+ new RestAPIClientTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
+ proxySliderRestAPI.testSuiteAll()
+
+
+ if (UserGroupInformation.securityEnabled) {
+ describe "Insecure Proxy Tests against a secure cluster"
+
+ // these tests use the Jersey client without the Hadoop-specific
+ // SPNEGO
+ JerseyTestDelegates basicJerseyClientTests =
+ new JerseyTestDelegates(proxyAM, createBasicJerseyClient())
+ basicJerseyClientTests.testSuiteGetOperations()
+ }
+
+ // create the Rest client via the registry
+
+ //get a slider client against the cluster
+
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
+ RegistryOperations operations = sliderClient.registryOperations;
+ def restClientFactory = new RestClientFactory(
+ operations, jerseyClient,
+ "~", SliderKeys.APP_TYPE, CLUSTER)
+ def sliderApplicationApi = restClientFactory.createSliderAppApiClient();
+ sliderApplicationApi.desiredModel
+ sliderApplicationApi.resolvedModel
+ if (proxyComplexVerbs) {
+ sliderApplicationApi.ping("registry located")
+ }
+
+ } else {
+ describe "skipping tests against HTTPS RM proxy $proxyAM"
+ }
+
describe "Direct Jersey Tests"
JerseyTestDelegates directJerseyTests =
new JerseyTestDelegates(directAM, jerseyClient, directComplexVerbs)
directJerseyTests.testSuiteAll()
- describe "Proxy SliderRestClient Tests"
- RestAPIClientTestDelegates proxySliderRestAPI =
- new RestAPIClientTestDelegates(proxyAM, jerseyClient, proxyComplexVerbs)
- proxySliderRestAPI.testSuiteAll()
-
describe "Direct SliderRestClient Tests"
RestAPIClientTestDelegates directSliderRestAPI =
new RestAPIClientTestDelegates(directAM, jerseyClient, directComplexVerbs)
directSliderRestAPI.testSuiteAll()
- if (UserGroupInformation.securityEnabled) {
- describe "Insecure Proxy Tests against a secure cluster"
-
- // these tests use the Jersey client without the Hadoop-specific
- // SPNEGO
- JerseyTestDelegates basicJerseyClientTests =
- new JerseyTestDelegates(proxyAM, createBasicJerseyClient())
- basicJerseyClientTests.testSuiteGetOperations()
- }
-
- // create the Rest client via the registry
-
- //get a slider client against the cluster
-
- SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
- RegistryOperations operations = sliderClient.registryOperations;
- def restClientFactory = new RestClientFactory(
- operations, jerseyClient,
- "~", SliderKeys.APP_TYPE, CLUSTER)
- def sliderApplicationApi = restClientFactory.createSliderAppApiClient();
- sliderApplicationApi.desiredModel
- sliderApplicationApi.resolvedModel
- if (proxyComplexVerbs) {
- sliderApplicationApi.ping("registry located")
- }
// maybe execute IPC operations.
// these are skipped when security is enabled, until
// there's some code set up to do the tokens properly
- if (!UserGroupInformation.isSecurityEnabled()) {
+ if (!UserGroupInformation.securityEnabled) {
describe("IPC equivalent operations")
def sliderClusterProtocol =
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
index 0db775b..060af5d 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentIT.groovy
@@ -76,7 +76,8 @@
application,
ARG_COMPONENT,
COMMAND_LOGGER,
- "2"])
+ "2"
+ ])
// sleep till the new instance starts
sleep(1000 * 10)
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
index 5758feb..666efff 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentQueueAndLabelsIT.groovy
@@ -25,6 +25,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -67,8 +68,7 @@
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "happy-path-with-queue-labels"
private static String TARGET_QUEUE = "labeled"
- private static String APP_RESOURCE4 =
- "../slider-core/src/test/app_packages/test_command_log/resources_queue_labels.json"
+ private static String APP_RESOURCE4 = ResourcePaths.COMMAND_LOG_RESOURCES_QUEUE_LABELS
@After
public void destroyCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy
index 9fc8e6a..fa8b7a5 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsUpgradeIT.groovy
@@ -25,6 +25,7 @@
import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
import org.apache.slider.funtest.framework.AgentCommandTestBase
import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.funtest.framework.SliderShell
@@ -49,8 +50,7 @@
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "app-upgrade-happy-path"
- private static String APP_RESOURCE =
- "../slider-core/src/test/app_packages/test_command_log/resources.json"
+ private static String APP_RESOURCE = ResourcePaths.COMMAND_LOG_RESOURCES
@After
public void destroyCluster() {
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
index f03fb63..66bc10b 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/ClusterBuildDestroyIT.groovy
@@ -37,13 +37,10 @@
public class ClusterBuildDestroyIT extends AgentCommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
-
static String CLUSTER = "test-cluster-build-destroy"
-
@BeforeClass
public static void prepareCluster() {
-
setupCluster(CLUSTER)
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAASleep.groovy
similarity index 60%
copy from slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
copy to slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAASleep.groovy
index 3dd7857..7a8e1a6 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAASleep.groovy
@@ -20,26 +20,24 @@
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.funtest.framework.AgentCommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.funtest.framework.SliderShell
-import org.junit.Before
-import org.junit.Test
+import org.apache.slider.api.types.NodeInformationList
+import org.apache.slider.core.launch.SerializedApplicationReport
-/**
- * For a quick demo of a slider app; this starts the apps through agent test but
- * neglects to tear it down afterwards
- */
@CompileStatic
@Slf4j
-public class AppsThroughAgentDemo extends AppsThroughAgentIT {
+class DemoAASleep extends AASleepIT {
@Override
- void destroyCluster() {
-// super.destroyCluster()
+ protected void operations(
+ String name,
+ SerializedApplicationReport appReport,
+ int desired,
+ int expected,
+ NodeInformationList healthyNodes) {
+ super.operations(name, appReport, desired, expected, healthyNodes)
+
+ describe("cluster is live at ${appReport.url}")
+
+ sleep(10 * 60 * 1000)
}
-
}
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAppsThroughAgent.groovy
similarity index 93%
rename from slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
rename to slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAppsThroughAgent.groovy
index 3dd7857..ea10390 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AppsThroughAgentDemo.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/DemoAppsThroughAgent.groovy
@@ -35,11 +35,11 @@
*/
@CompileStatic
@Slf4j
-public class AppsThroughAgentDemo extends AppsThroughAgentIT {
+public class DemoAppsThroughAgent extends AppsThroughAgentIT {
@Override
void destroyCluster() {
-// super.destroyCluster()
+
}
-
+
}
