Merge branch 'develop' into feature/SLIDER-1107_AM_config_generation
Conflicts:
slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 193245d..5f81e1c 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -224,6 +224,26 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>rat</id>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/license/THIRD-PARTY.properties</exclude>
+ <exclude>**/*.json</exclude>
+ <exclude>src/test/resources/test_password_file</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
</profiles>
<build>
diff --git a/app-packages/accumulo/src/test/resources/resources.json b/app-packages/accumulo/src/test/resources/resources.json
index 6afbc74..e64de3f 100644
--- a/app-packages/accumulo/src/test/resources/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources.json
@@ -10,10 +10,10 @@
"ACCUMULO_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "64"
+ "yarn.memory": "256"
},
"slider-appmaster": {
- "yarn.memory": "128"
+ "yarn.memory": "384"
},
"ACCUMULO_TSERVER": {
"yarn.role.priority": "2",
diff --git a/app-packages/accumulo/src/test/resources/resources_with_proxy.json b/app-packages/accumulo/src/test/resources/resources_with_proxy.json
index b424f09..43707a6 100644
--- a/app-packages/accumulo/src/test/resources/resources_with_proxy.json
+++ b/app-packages/accumulo/src/test/resources/resources_with_proxy.json
@@ -10,10 +10,10 @@
"ACCUMULO_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "64"
+ "yarn.memory": "256"
},
"slider-appmaster": {
- "yarn.memory": "128"
+ "yarn.memory": "384"
},
"ACCUMULO_TSERVER": {
"yarn.role.priority": "2",
diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml
index 33a0517..f95f2b2 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml
index 89d4582..4ea77d5 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/hbase-win/pom.xml b/app-packages/hbase-win/pom.xml
index d4946ef..52f3242 100644
--- a/app-packages/hbase-win/pom.xml
+++ b/app-packages/hbase-win/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 662cd88..c149d77 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/kafka/pom.xml b/app-packages/kafka/pom.xml
index 8e54455..c1e9604 100644
--- a/app-packages/kafka/pom.xml
+++ b/app-packages/kafka/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/pom.xml b/app-packages/pom.xml
index aa9632b..016bdaa 100644
--- a/app-packages/pom.xml
+++ b/app-packages/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -54,6 +54,7 @@
<module>hbase</module>
<module>kafka</module>
<module>storm</module>
+ <module>tomcat-test-war</module>
<module>tomcat</module>
</modules>
</profile>
@@ -88,6 +89,7 @@
<module>storm</module>
<module>hbase-win</module>
<module>storm-win</module>
+ <module>tomcat-test-war</module>
<module>tomcat</module>
</modules>
</profile>
@@ -96,28 +98,30 @@
<id>rat</id>
<!-- RAT profile -->
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/test_password_file</exclude>
- <exclude>command-logger/**</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/test_password_file</exclude>
+ <exclude>command-logger/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
</profile>
</profiles>
diff --git a/app-packages/storm-win/pom.xml b/app-packages/storm-win/pom.xml
index c64eb10..a52fe4c 100644
--- a/app-packages/storm-win/pom.xml
+++ b/app-packages/storm-win/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/storm/package/files/storm-slider b/app-packages/storm/package/files/storm-slider
index af492bd..7a9203e 100644
--- a/app-packages/storm/package/files/storm-slider
+++ b/app-packages/storm/package/files/storm-slider
@@ -1,7 +1,5 @@
#!/bin/bash
#
-# Copyright 2014 The Apache Software Foundation
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
index ad495c5..73ceefa 100644
--- a/app-packages/storm/pom.xml
+++ b/app-packages/storm/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/tomcat-test-war/pom.xml b/app-packages/tomcat-test-war/pom.xml
new file mode 100644
index 0000000..f89eab2
--- /dev/null
+++ b/app-packages/tomcat-test-war/pom.xml
@@ -0,0 +1,68 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <parent>
+ <groupId>org.apache.slider.packages</groupId>
+ <artifactId>slider-app-packages</artifactId>
+ <version>0.92.0-incubating-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-tomcat-app-package-test-war</artifactId>
+ <packaging>war</packaging>
+ <name>Slider Tomcat App Package Testing WAR file</name>
+ <description>A WAR file that can be used to test the Tomcat app package.</description>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <artifactId>maven-war-plugin</artifactId>
+ <configuration>
+ <classifier>test</classifier>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>javax.ws.rs-api</artifactId>
+ <version>2.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet-core</artifactId>
+ <version>2.13</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/app-packages/tomcat-test-war/src/main/java/org/apache/slider/war/RootResource.java b/app-packages/tomcat-test-war/src/main/java/org/apache/slider/war/RootResource.java
new file mode 100644
index 0000000..fe098c8
--- /dev/null
+++ b/app-packages/tomcat-test-war/src/main/java/org/apache/slider/war/RootResource.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.packages.tomcat.war;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+
+/**
+ * Root resource
+ */
+@Path("/")
+public class RootResource {
+
+ /**
+ * Method handling HTTP GET requests. The returned object will be sent
+ * to the client as "text/plain" media type.
+ *
+ * @return String that will be returned as a text/plain response.
+ */
+ @GET
+ @Produces(MediaType.TEXT_PLAIN)
+ public String getIt() {
+ return "Success";
+ }
+}
diff --git a/app-packages/tomcat-test-war/src/main/webapp/WEB-INF/web.xml b/app-packages/tomcat-test-war/src/main/webapp/WEB-INF/web.xml
new file mode 100644
index 0000000..3c0b696
--- /dev/null
+++ b/app-packages/tomcat-test-war/src/main/webapp/WEB-INF/web.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to you under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!-- This web.xml file is not required when using Servlet 3.0 container,
+ see implementation details http://jersey.java.net/nonav/documentation/latest/jax-rs.html -->
+<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd">
+ <display-name>Apache Slider (incubating) Test WAR</display-name>
+ <servlet>
+ <servlet-name>Jersey Web Application</servlet-name>
+ <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
+ <init-param>
+ <param-name>jersey.config.server.provider.packages</param-name>
+ <param-value>org.apache.slider.packages.tomcat.war</param-value>
+ </init-param>
+ <load-on-startup>1</load-on-startup>
+ </servlet>
+ <servlet-mapping>
+ <servlet-name>Jersey Web Application</servlet-name>
+ <url-pattern>/*</url-pattern>
+ </servlet-mapping>
+</web-app>
diff --git a/app-packages/tomcat/README.md b/app-packages/tomcat/README.md
index ff77c80..bfabc07 100644
--- a/app-packages/tomcat/README.md
+++ b/app-packages/tomcat/README.md
@@ -57,8 +57,3 @@
Check the Slider Application Master page on the Hadoop YARN status page. You should see
the application running and without failures. The address of the Tomcat servers will be listed
as exports on the AppMaster's web UI.
-
-# Slider Automated Testing
-
-The provided slider-test.war is built using https://github.com/joshelser/simple-rest-war. Modifications
-of that WAR file can be made using that codebase and including the binary test resource here.
diff --git a/app-packages/tomcat/appConfig-default.json b/app-packages/tomcat/appConfig-default.json
index 84183ca..5004b68 100644
--- a/app-packages/tomcat/appConfig-default.json
+++ b/app-packages/tomcat/appConfig-default.json
@@ -13,7 +13,8 @@
"site.server-xml.http.port":"${TOMCAT.ALLOCATED_PORT}{PER_CONTAINER}",
"site.server-xml.connection.timeout":"20000",
"site.tomcat-users-xml.ui_user": "tomcat",
- "site.tomcat-users-xml.ui_password": "password"
+ "site.tomcat-users-xml.ui_password": "password",
+ "agent.python.exec.path": "/usr/local/bin/python"
},
"components": {
"slider-appmaster": {
diff --git a/app-packages/tomcat/pom.xml b/app-packages/tomcat/pom.xml
index 4d3a718..7b943ee 100644
--- a/app-packages/tomcat/pom.xml
+++ b/app-packages/tomcat/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
@@ -112,6 +112,7 @@
<systemPropertyVariables>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
<java.awt.headless>true</java.awt.headless>
+ <project.build.directory>${project.build.directory}</project.build.directory>
<!-- this property must be supplied-->
<slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
<slider.bin.dir>${slider.bin.dir}</slider.bin.dir>
@@ -147,7 +148,6 @@
<configuration>
<excludes>
<exclude>**/*.json</exclude>
- <exclude>src/test/resources/slider-test.war</exclude>
<exclude>**/*.iml</exclude>
</excludes>
</configuration>
@@ -192,6 +192,31 @@
</dependency>
</dependencies>
</plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-test-war</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.slider.packages</groupId>
+ <artifactId>slider-tomcat-app-package-test-war</artifactId>
+ <version>${project.version}</version>
+ <classifier>test</classifier>
+ <type>war</type>
+ <destFileName>slider-test.war</destFileName>
+ </artifactItem>
+ </artifactItems>
+ <outputDirectory>${project.build.directory}</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<dependencies>
@@ -237,5 +262,12 @@
<artifactId>slf4j-api</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.slider.packages</groupId>
+ <artifactId>slider-tomcat-app-package-test-war</artifactId>
+ <version>${project.version}</version>
+ <type>war</type>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/app-packages/tomcat/src/test/groovy/org/apache/slider/funtest/tomcat/TomcatBasicIT.groovy b/app-packages/tomcat/src/test/groovy/org/apache/slider/funtest/tomcat/TomcatBasicIT.groovy
index d77897f..369c96f 100644
--- a/app-packages/tomcat/src/test/groovy/org/apache/slider/funtest/tomcat/TomcatBasicIT.groovy
+++ b/app-packages/tomcat/src/test/groovy/org/apache/slider/funtest/tomcat/TomcatBasicIT.groovy
@@ -74,7 +74,7 @@
Configuration conf = new Configuration()
FileSystem fs = FileSystem.getLocal(conf)
// The path from the local filesystem of the war (copied to target/test-classes from src/test/resources)
- Path sourcePath = fs.makeQualified(new Path(sysprop("test-classes.dir") + "/slider-test.war"))
+ Path sourcePath = fs.makeQualified(new Path(sysprop("project.build.directory"), "slider-test.war"))
// Path in HDFS we'll put the WAR
Path targetPath = clusterFS.makeQualified(new Path("/tmp/slider-test.war"))
if (clusterFS.exists(targetPath)) {
@@ -185,7 +185,7 @@
return httpAddress
} catch (Exception e) {
caught = e;
- log.info("Got exception trying to read quicklinks")
+ log.info("Got exception trying to read quicklinks", e)
if (tries-- == 0) {
break
}
diff --git a/app-packages/tomcat/src/test/resources/appConfig-default.json b/app-packages/tomcat/src/test/resources/appConfig-default.json
deleted file mode 100644
index 009838c..0000000
--- a/app-packages/tomcat/src/test/resources/appConfig-default.json
+++ /dev/null
@@ -1,23 +0,0 @@
-{
- "schema": "http://example.org/specification/v2.0.0",
- "metadata": {
- },
- "global": {
- "application.def": ".slider/package/tomcat/slider-tomcat-app-package.zip",
- "java_home": "/usr/jdk64/jdk1.7.0_67",
- "application.resources": "hdfs://localhost:8020/user/jelser/my_application.war",
-
- "site.global.xmx_val": "1g",
- "site.global.xms_val": "128m",
- "site.global.memory_val": "200M",
- "site.server-xml.http.port":"${TOMCAT.ALLOCATED_PORT}",
- "site.server-xml.connection.timeout":"20000",
- "site.tomcat-users-xml.ui.user": "tomcat",
- "site.tomcat-users-xml.ui.password": "password"
- },
- "components": {
- "slider-appmaster": {
- "jvm.heapsize": "128M"
- }
- }
-}
diff --git a/app-packages/tomcat/src/test/resources/slider-test.war b/app-packages/tomcat/src/test/resources/slider-test.war
deleted file mode 100644
index 7410435..0000000
--- a/app-packages/tomcat/src/test/resources/slider-test.war
+++ /dev/null
Binary files differ
diff --git a/bin/release.xml b/bin/release.xml
index 603d1fc..518caf7 100644
--- a/bin/release.xml
+++ b/bin/release.xml
@@ -99,7 +99,7 @@
<!-- SVN properties-->
<property name="incubator.svn.url"
- value="https://dist.apache.org/repos/dist/release/incubator" />
+ value="https://dist.apache.org/repos/dist/dev/incubator" />
<!-- RC release properties-->
<property name="svn.rc.dir" value="${svn.publish.dir}/${rc.name}" />
diff --git a/pom.xml b/pom.xml
index 64a78e4..7c21e07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
<name>Slider</name>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<description>
@@ -123,7 +123,6 @@
<test.forkedProcessTimeoutInSeconds>18000</test.forkedProcessTimeoutInSeconds>
<test.argLine>-Xmx1024m -XX:+HeapDumpOnOutOfMemoryError</test.argLine>
<test.reuseForks>false</test.reuseForks>
- <test.failIfNoTests>true</test.failIfNoTests>
<test.funtests.failIfNoTests>false</test.funtests.failIfNoTests>
<test.forkMode>always</test.forkMode>
<!-- path environment variable -->
@@ -153,8 +152,6 @@
<guava.version>11.0.2</guava.version>
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
- <httpclient.version>3.1</httpclient.version>
- <httpclient4.version>4.2.5</httpclient4.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -522,12 +519,6 @@
</exclusions>
</dependency>
- <!-- hadoop-client includes the following jars, so they do not need to be
- included separately:
- hadoop-common, hadoop-hdfs (client?), hadoop-mapreduce-client-app,
- hadoop-yarn-api, hadoop-mapreduce-client-core,
- hadoop-mapreduce-client-jobclient, and hadoop-annotations
- -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -541,14 +532,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -569,14 +552,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -632,6 +607,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-azure</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
@@ -697,30 +678,6 @@
<version>${commons-lang.version}</version>
</dependency>
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>${httpclient.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${httpclient4.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>${httpclient4.version}</version>
- </dependency>
-
<!-- ======================================================== -->
<!-- HBASE -->
<!-- ======================================================== -->
@@ -1449,40 +1406,42 @@
<profile>
<id>rat</id>
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/*.tar</exclude>
- <exclude>**/THIRD-PARTY.properties</exclude>
- <exclude>**/build.properties</exclude>
- <exclude>**/regionservers</exclude>
- <exclude>**/slaves</exclude>
- <exclude>**/httpfs-signature.secret</exclude>
- <exclude>**/dfs.exclude</exclude>
- <exclude>**/*.iml</exclude>
- <exclude>**/rat.txt</exclude>
- <exclude>**/test_password_file</exclude>
- <exclude>DISCLAIMER</exclude>
- <exclude>app-packages/hbase/target/**</exclude>
- <exclude>target/*</exclude>
- <exclude>DEPENDENCIES</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.tar</exclude>
+ <exclude>**/THIRD-PARTY.properties</exclude>
+ <exclude>**/build.properties</exclude>
+ <exclude>**/regionservers</exclude>
+ <exclude>**/slaves</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/dfs.exclude</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>**/rat.txt</exclude>
+ <exclude>**/test_password_file</exclude>
+ <exclude>DISCLAIMER</exclude>
+ <exclude>app-packages/hbase/target/**</exclude>
+ <exclude>target/*</exclude>
+ <exclude>DEPENDENCIES</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
</profile>
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index 351ee86..3f695d5 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>slider-agent</artifactId>
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 6b16767..e973337 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -72,8 +72,8 @@
self.customServiceOrchestrator = CustomServiceOrchestrator(config,
controller,
self.queueOutAgentToggleLogger)
- self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
- self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
+ self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator)
+ self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator)
def stop(self):
@@ -164,7 +164,7 @@
if 'commandParams' in command and ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
store_command = False
- if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
+ if 'roleParams' in command and command['roleParams'] is not None and ActionQueue.AUTO_RESTART in command['roleParams']:
store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
if store_command:
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index 6ede66e..524db6a 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -34,7 +34,7 @@
ZK_REG_PATH="zk_reg_path"
AUTO_GENERATED="auto_generated"
MAX_AM_CONNECT_RETRIES = 10
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application_"
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application"
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = ".out"
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application_"
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application"
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = ".err"
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index b40a8b5..5b73fa4 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -63,6 +63,8 @@
self.stored_command = {}
self.allocated_ports = {}
self.log_folders = {}
+
+ self.allocated_ports_set = set()
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -300,7 +302,6 @@
allocated_for_any = ".ALLOCATED_PORT}"
port_allocation_req = allocated_for_this_component_format.format(component)
- allowed_ports = self.get_allowed_ports(command)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -311,7 +312,7 @@
value = value.replace("${AGENT_LOG_ROOT}",
self.config.getLogPath())
if port_allocation_req in value:
- value = self.allocate_ports(value, port_allocation_req, allowed_ports)
+ value = self.allocate_ports(value, port_allocation_req, self.get_allowed_ports(command))
allocated_ports[key + "." + k] = value
elif allocated_for_any in value:
## All unallocated ports should be set to 0
@@ -413,6 +414,10 @@
value = value.replace(replaced_pattern, str(port), 1)
logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
index = value.find(port_req_pattern)
+
+ if allowed_ports != None:
+ allowed_ports.remove(port)
+ self.allocated_ports_set.add(port)
pass
return value
pass
@@ -488,10 +493,12 @@
except:
# not an int and not a range...
invalid.add(i)
+ selection = selection - self.allocated_ports_set
selection = random.sample(selection, min (len(selection), num_values))
# Report invalid tokens before returning valid selection
logger.info("Allowed port values: " + str(selection))
- logger.warning("Invalid port range values: " + str(invalid))
+ if len(invalid):
+ logger.warning("Invalid port range values: " + str(invalid))
return selection
diff --git a/slider-agent/src/main/python/agent/DockerManager.py b/slider-agent/src/main/python/agent/DockerManager.py
index 023a163..48085fe 100644
--- a/slider-agent/src/main/python/agent/DockerManager.py
+++ b/slider-agent/src/main/python/agent/DockerManager.py
@@ -31,9 +31,10 @@
stored_command = ''
container_id = ''
- def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator):
self.tmpdir = tmpdir
self.workroot = workroot
+ self.logDir = logDir
self.customServiceOrchestrator = customServiceOrchestrator
def execute_command(self, command, store_command=False):
@@ -128,14 +129,11 @@
docker_command = self.add_additional_param_to_command(docker_command, additional_param)
#adding redirecting stdout stderr to file
logger.info("docker run command: " + str(docker_command))
- outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+ outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
return self.execute_command_on_linux(docker_command, stdoutFile, stderrFile)
def add_docker_run_options_to_command(self, docker_command, options):
diff --git a/slider-agent/src/main/python/agent/NetUtil.py b/slider-agent/src/main/python/agent/NetUtil.py
index eb658f7..88c129a 100644
--- a/slider-agent/src/main/python/agent/NetUtil.py
+++ b/slider-agent/src/main/python/agent/NetUtil.py
@@ -18,6 +18,7 @@
import time
import logging
import httplib
+import sys
from ssl import SSLError
logger = logging.getLogger()
@@ -36,9 +37,16 @@
(like unreachable server or wrong HTTP code) result will be False
"""
logger.info("Connecting to the following url " + url);
+
try:
parsedurl = urlparse(url)
- ca_connection = httplib.HTTPSConnection(parsedurl[1])
+
+ if sys.version_info >= (2,7,9):
+ import ssl
+ ca_connection = httplib.HTTPSConnection(parsedurl[1], context=ssl._create_unverified_context())
+ else:
+ ca_connection = httplib.HTTPSConnection(parsedurl[1])
+
ca_connection.request("GET", parsedurl[2])
response = ca_connection.getresponse()
status = response.status
diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py
index abdb2fb..4a4d6de 100644
--- a/slider-agent/src/main/python/agent/YarnDockerManager.py
+++ b/slider-agent/src/main/python/agent/YarnDockerManager.py
@@ -24,6 +24,7 @@
import Constants
import time
import traceback
+from AgentConfig import AgentConfig
from resource_management import *
logger = logging.getLogger()
@@ -33,9 +34,10 @@
stored_command = ''
container_id = ''
- def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator):
self.tmpdir = tmpdir
self.workroot = workroot
+ self.logDir = logDir
self.customServiceOrchestrator = customServiceOrchestrator
def execute_command(self, command, store_command=False):
@@ -168,14 +170,11 @@
#extracting param needed by docker run from the command passed from AM
startCommand = self.extract_config_from_command(command, 'docker.startCommand')
#adding redirecting stdout stderr to file
- outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+ outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
returncode,out,err = self.execute_command_on_linux(startCommand, False,
stdoutFile, stderrFile)
return returncode,out,err
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index 39de1ae..1932a37 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -60,7 +60,7 @@
if docker_mode:
tmpdir = controller.actionQueue.dockerManager.stop_container()
- if controller is not None and hasattr(controller, 'stopCommand'):
+ if controller is not None and hasattr(controller, 'stopCommand') and controller.stopCommand is not None:
controller.stopCommand = _increment_task_id(controller.stopCommand)
controller.appGracefulStopQueued = True
controller.actionQueue.execute_command(controller.stopCommand)
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index b85c020..a3d9126 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -30,9 +30,9 @@
import shutil
from resource_management.libraries.script import Script
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log'
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err'
def _merge_env(env1, env2, merge_keys=['PYTHONPATH']):
@@ -74,14 +74,12 @@
# TODO implement user
Logger.info("Executing %s" % (command))
#adding redirecting stdout stderr to file
- outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
- errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+ errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True,
cwd=cwd, env=env, shell=False)
code = None
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index f21dbbf..4a383c7 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -31,9 +31,9 @@
from resource_management.core.logger import Logger
import time
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log'
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err'
def checked_call(command, logoutput=False,
@@ -69,14 +69,11 @@
"""
command = ["/bin/bash","--login","-c", command]
#adding redirecting stdout stderr to file
- outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+ outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True,
cwd=cwd, env=env, shell=False,
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
index 8c11d93..5d6cf07 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
@@ -18,6 +18,7 @@
limitations under the License.
'''
+import re
import os
import sys
import platform
@@ -26,15 +27,42 @@
'OSCheck',
]
+_IS_REDHAT_LINUX = os.path.exists('/etc/redhat-release')
+
+SYSTEM_RELEASE_FILE = "/etc/system-release"
+
+def _is_redhat_linux():
+ return _IS_REDHAT_LINUX
+
+def advanced_check(distribution):
+ distribution = list(distribution)
+ if os.path.exists(SYSTEM_RELEASE_FILE):
+ with open(SYSTEM_RELEASE_FILE, "rb") as fp:
+ issue_content = fp.read()
+
+ if "Amazon" in issue_content:
+ distribution[0] = "amazon"
+ search_groups = re.search('(\d+\.\d+)', issue_content)
+
+ if search_groups:
+ distribution[1] = search_groups.group(1)
+
+ return tuple(distribution)
def linux_distribution():
PYTHON_VER = sys.version_info[0] * 10 + sys.version_info[1]
if PYTHON_VER < 26:
- (distname, version, id) = platform.dist()
+ (distname, version, id) = platform.dist()
+ elif _is_redhat_linux():
+ (distname, version, id) = platform.dist()
else:
(distname, version, id) = platform.linux_distribution()
+ if distname == '':
+ (distname, version) = advanced_check((distname, version))
+ id = ' '
+
return (platform.system(), os.name, distname, version, id)
def windows_distribution():
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index aaef3f9..a6fa054 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -674,6 +674,38 @@
self.assertTrue(set(allowed_ports).issubset(port_range_full_list))
+ def test_finalize_command_when_set_allowed_ports(self):
+ dummy_controller = MagicMock()
+ tempdir = tempfile.gettempdir()
+ tempWorkDir = tempdir + "W"
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempWorkDir
+ config.getLogPath.return_value = tempdir
+
+ allowed_ports = "6700-6701"
+ allowed_ports_full_list = [6700, 6701]
+
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
+ command = {}
+ command['componentName'] = "HBASE_MASTER"
+ command['configurations'] = {}
+ command['configurations']['global'] = {}
+ command['configurations']['global']['slider.allowed.ports'] = allowed_ports
+ command['configurations']['hbase-site'] = {}
+ command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
+ command['configurations']['hbase-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
+ command['configurations']['hbase-site']['b_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
+
+ orchestrator.finalize_command(command, False, {})
+ a_port = int(command['configurations']['hbase-site']['a_port'])
+ b_port = int(command['configurations']['hbase-site']['b_port'])
+
+ self.assertTrue((a_port in allowed_ports_full_list) and (b_port in allowed_ports_full_list))
+ self.assertTrue(a_port != b_port)
+
+
def tearDown(self):
# enable stdout
sys.stdout = sys.__stdout__
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap
index 88a8c55..d95cf07 100755
--- a/slider-agent/src/test/python/python-wrap
+++ b/slider-agent/src/test/python/python-wrap
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-# Copyright 2011 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index a64dd71..0b2f077 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
</parent>
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 8b19e7f..57507d6 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
</parent>
<build>
@@ -114,7 +114,7 @@
</forkedProcessTimeoutInSeconds>
<threadCount>1</threadCount>
<argLine>${test.argLine}</argLine>
- <failIfNoTests>${test.failIfNoTests}</failIfNoTests>
+ <failIfNoTests>true</failIfNoTests>
<redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
<environmentVariables>
<PATH>${test.env.path}</PATH>
@@ -334,21 +334,6 @@
</dependency>
<dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </dependency>
-
- <dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
</dependency>
diff --git a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
index 1747a2b..8c572b3 100644
--- a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
+++ b/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.security;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -316,11 +317,11 @@
}
/**
- * Dump a keytab: list all principals
+ * Dump a keytab: list all principals.
* @param keytabFile the keytab file
* @throws IOException IO problems
*/
- private void dumpKeytab(File keytabFile) throws IOException {
+ public void dumpKeytab(File keytabFile) throws IOException {
title("Examining keytab %s", keytabFile);
File kt = keytabFile.getCanonicalFile();
failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
@@ -513,15 +514,27 @@
}
/**
- * Print a line of output. This goes to any output file, or
+ * Format and print a line of output.
+ * This goes to any output file, or
* is logged at info. The output is flushed before and after, to
* try and stay in sync with JRE logging.
* @param format format string
* @param args any arguments
*/
- private void println(String format, Object... args) {
+ @VisibleForTesting
+ public void println(String format, Object... args) {
+ println(format(format, args));
+ }
+
+ /**
+ * Print a line of output. This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param msg message string
+ */
+ @VisibleForTesting
+ private void println(String msg) {
flush();
- String msg = String.format(format, args);
if (out != null) {
out.println(msg);
} else {
@@ -538,7 +551,7 @@
private void title(String format, Object... args) {
println("");
println("");
- String msg = "== " + String.format(format, args) + " ==";
+ String msg = "== " + format(format, args) + " ==";
println(msg);
println("");
}
@@ -575,10 +588,10 @@
* @param file file to dump
* @throws IOException IO problems
*/
- private void dump(File file) throws IOException {
+ public void dump(File file) throws IOException {
try (FileInputStream in = new FileInputStream(file)) {
for (String line : IOUtils.readLines(in)) {
- println(line);
+ println("%s", line);
}
}
println("");
@@ -617,6 +630,22 @@
}
/**
+ * Format a string, treating a call where there are no varags values
+ * as a string to pass through unformatted.
+ * @param message message, which is either a format string + args, or
+ * a general string
+ * @param args argument array
+ * @return a string for printing.
+ */
+ public static String format(String message, Object... args) {
+ if (args.length == 0) {
+ return message;
+ } else {
+ return String.format(message, args);
+ }
+ }
+
+ /**
* Diagnostics failures return the exit code 41, "unauthorized".
*
* They have a category, initially for testing: the category can be
@@ -631,7 +660,7 @@
}
public KerberosDiagsFailure(String category, String message, Object... args) {
- this(category, String.format(message, args));
+ this(category, format(message, args));
}
public KerberosDiagsFailure(String category, Throwable throwable,
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 647ac23..812a6b3 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -30,6 +30,11 @@
String ROLE_NAME = "role.name";
/**
+ * The group of a role: {@value}
+ */
+ String ROLE_GROUP = "role.group";
+
+ /**
* Status report: number actually granted : {@value}
*/
String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index e06c243..ba3effc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -267,11 +267,6 @@
* {@value}
*/
String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports";
-
- /**
- * Allowed port range
- */
- String KEY_AM_ALLOWED_PORT_RANGE = "slider.am.allowed.port.range";
/**
* env var for custom JVM options.
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 1c694bd..2a5eedc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -87,9 +87,13 @@
public File template;
@Parameter(names = {ARG_METAINFO},
- description = "Application meta info")
+ description = "Application meta info file")
public File appMetaInfo;
+ @Parameter(names = {ARG_METAINFO_JSON},
+ description = "Application meta info JSON blob")
+ public String appMetaInfoJson;
+
@Parameter(names = {ARG_APPDEF},
description = "Application def (folder or a zip package)")
public File appDef;
@@ -208,12 +212,6 @@
ConfTree confTree = new ConfTree();
ConfTreeOperations ops = new ConfTreeOperations(confTree);
confTree.global.putAll(optionsMap);
- Map<String, Map<String, String>> roleOptionMap = getCompOptionMap();
- for (Map.Entry<String, Map<String, String>> entry : roleOptionMap.entrySet()) {
- String key = entry.getKey();
- Map<String, String> value = entry.getValue();
- ops.getOrAddComponent(key).putAll(value);
- }
return confTree;
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
index 248e4c2..f171708 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -95,7 +95,7 @@
*/
public Map<String, Map<String, String>> getResourceCompOptionMap() throws
BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, resCompOptTriples);
+ return convertTripleListToMaps(ARG_RES_COMP_OPT, resCompOptTriples);
}
public void setOption(String key, String value) {
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 63c75ee..aec4e26 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -87,6 +87,7 @@
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
String ARG_METAINFO = "--metainfo";
+ String ARG_METAINFO_JSON = "--metainfojson";
String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
index b5b21ce..64783b6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -17,8 +17,10 @@
package org.apache.slider.common.tools;
import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -39,27 +41,38 @@
public PortScanner() {
}
- int nextPort = 1024;
-
- public void setPortRange(String input) {
+ public void setPortRange(String input) throws BadConfigException {
// first split based on commas
Set<Integer> inputPorts= new TreeSet<Integer>();
String[] ranges = input.split(",");
for ( String range : ranges ) {
+ if (range.trim().isEmpty()) {
+ continue;
+ }
Matcher m = SINGLE_NUMBER.matcher(range.trim());
if (m.find()) {
inputPorts.add(Integer.parseInt(m.group()));
- } else {
- m = NUMBER_RANGE.matcher(range.trim());
- if (m.find()) {
- String[] boundaryValues = m.group(0).split("-");
- int start = Integer.parseInt(boundaryValues[0].trim());
- int end = Integer.parseInt(boundaryValues[1].trim());
- for (int i = start; i < end + 1; i++) {
- inputPorts.add(i);
- }
- }
+ continue;
}
+ m = NUMBER_RANGE.matcher(range.trim());
+ if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ if (end < start) {
+ throw new BadConfigException("End of port range is before start: "
+ + range + " in input: " + input);
+ }
+ for (int i = start; i < end + 1; i++) {
+ inputPorts.add(i);
+ }
+ continue;
+ }
+ throw new BadConfigException("Bad port range: " + range + " in input: "
+ + input);
+ }
+ if (inputPorts.size() == 0) {
+ throw new BadConfigException("No ports found in range: " + input);
}
this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
}
@@ -68,23 +81,14 @@
return remainingPortsToCheck;
}
- public int getAvailablePort() throws SliderException {
+ public int getAvailablePort() throws SliderException, IOException {
if (remainingPortsToCheck != null) {
return getAvailablePortViaPortArray();
} else {
- return getAvailablePortViaCounter();
+ return SliderUtils.getOpenPort();
}
}
- private int getAvailablePortViaCounter() throws SliderException {
- int port;
- do {
- port = nextPort;
- nextPort++;
- } while (!SliderUtils.isPortAvailable(port));
- return port;
- }
-
private int getAvailablePortViaPortArray() throws SliderException {
boolean found = false;
int availablePort = -1;
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index eae80f5..746e468 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1114,6 +1114,22 @@
}
/**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
* See if a port is available for listening on by trying to listen
* on it and seeing if that works or fails.
* @param port port to listen to
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
index e96baef..7fb3158 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -35,6 +36,7 @@
import java.io.File;
import java.io.IOException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -109,32 +111,61 @@
ConfTreeOperations appConf)
throws BadConfigException, IOException, BadCommandArgumentsException {
// if metainfo is provided add to the app instance
- if (buildInfo.appMetaInfo != null) {
-
- if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
- throw new BadConfigException("--metainfo file cannot be read.");
+ if (buildInfo.appMetaInfo != null || buildInfo.appMetaInfoJson != null) {
+ if (buildInfo.appMetaInfo != null && buildInfo.appMetaInfoJson != null) {
+ throw new BadConfigException("Both %s and %s cannot be specified",
+ Arguments.ARG_METAINFO, Arguments.ARG_METAINFO_JSON);
}
+ // Now we know that only one of either file or JSON is used
+ boolean isFileUsed = buildInfo.appMetaInfo != null ? true : false;
+ String argUsed = isFileUsed ? Arguments.ARG_METAINFO
+ : Arguments.ARG_METAINFO_JSON;
+
if (buildInfo.appDef != null) {
- throw new BadConfigException("both --metainfo and --appdef may not be specified.");
+ throw new BadConfigException("Both %s and %s cannot be specified",
+ argUsed, Arguments.ARG_APPDEF);
}
if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
- throw new BadConfigException("application.def must not be set if --metainfo is provided.");
+ throw new BadConfigException(
+ "%s cannot not be set if %s is specified in the cmd line ",
+ AgentKeys.APP_DEF, argUsed);
+ }
+
+ if (isFileUsed) {
+ if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
+ throw new BadConfigException(
+ "Path specified with %s either cannot be read or is not a file",
+ Arguments.ARG_METAINFO);
+ }
+ } else {
+ if (StringUtils.isEmpty(buildInfo.appMetaInfoJson.trim())) {
+ throw new BadConfigException("Empty string specified with %s",
+ Arguments.ARG_METAINFO_JSON);
+ }
}
File tempDir = Files.createTempDir();
File pkgSrcDir = new File(tempDir, "default");
pkgSrcDir.mkdirs();
- if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
- Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml"));
+ File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
+ if (isFileUsed) {
+ if (buildInfo.appMetaInfo.getName().endsWith(".xml")) {
+ Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.xml"));
+ } else {
+ Files.copy(buildInfo.appMetaInfo, destMetaInfo);
+ }
} else {
- Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json"));
+ Files.write(
+ buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")),
+ destMetaInfo);
}
Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
log.info("Using default app def path {}", appDirPath.toString());
- appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, SliderKeys.DEFAULT_APP_PKG));
+ appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir,
+ SliderKeys.DEFAULT_APP_PKG));
Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
log.info("Setting app package to {}.", appDefPath);
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 2c087ce..3eef0b0 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -93,7 +93,7 @@
public static final String E_COULD_NOT_READ_METAINFO
= "Not a valid app package. Could not read metainfo.";
- protected static Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
+ protected Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
protected AgentClientProvider(Configuration conf) {
super(conf);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8d30da7..b767059 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -404,11 +404,6 @@
private SecurityConfiguration securityConfiguration;
/**
- * The port for the web application
- */
- private int webAppPort;
-
- /**
* Is security enabled?
* Set early on in the {@link #createAndRunCluster(String)} operation.
*/
@@ -776,11 +771,23 @@
uploadServerCertForLocalization(clustername, fs);
}
- webAppPort = getPortToRequest();
- if (webAppPort == 0) {
- // failure to find a port
- throw new BadConfigException("Failed to fix a web application port");
- }
+ // Web service endpoints: initialize
+ WebAppApiImpl webAppApi =
+ new WebAppApiImpl(
+ stateForProviders,
+ providerService,
+ certificateManager,
+ registryOperations,
+ metricsAndMonitoring,
+ actionQueues,
+ this,
+ contentCache);
+ initAMFilterOptions(serviceConf);
+
+ // start the agent web app
+ startAgentWebApp(appInformation, serviceConf, webAppApi);
+ int webAppPort = deployWebApplication(webAppApi);
+
String scheme = WebAppUtils.HTTP_PREFIX;
appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort;
@@ -926,7 +933,7 @@
Path tmpDirPath = new Path(amTmpDir);
Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
fs.getFileSystem().mkdirs(launcherTmpDirPath);
-
+
//launcher service
launchService = new RoleLaunchService(actionQueues,
providerService,
@@ -972,25 +979,6 @@
scheduleEscalation(instanceDefinition.getInternal());
try {
-
- // Web service endpoints: initialize
-
- WebAppApiImpl webAppApi =
- new WebAppApiImpl(
- stateForProviders,
- providerService,
- certificateManager,
- registryOperations,
- metricsAndMonitoring,
- actionQueues,
- this,
- contentCache);
- initAMFilterOptions(serviceConf);
-
- // start the agent web app
- startAgentWebApp(appInformation, serviceConf, webAppApi);
- deployWebApplication(webAppPort, webAppApi);
-
// schedule YARN Registry registration
queue(new ActionRegisterServiceInstance(clustername, appid));
@@ -1051,7 +1039,7 @@
}
/**
- * List the node reports: uses {@link #yarnClient} as the login user
+ * List the node reports: uses {@link SliderYarnClientImpl} as the login user
* @param yarnClient client to the RM
* @return the node reports
* @throws IOException
@@ -1083,17 +1071,18 @@
* Creates and starts the web application, and adds a
* <code>WebAppService</code> service under the AM, to ensure
* a managed web application shutdown.
- * @param port port to deploy the web application on
* @param webAppApi web app API instance
+ * @return port the web application is deployed on
* @throws IOException general problems starting the webapp (network, etc)
* @throws WebAppException other issues
*/
- private void deployWebApplication(int port, WebAppApiImpl webAppApi)
- throws IOException {
+ private int deployWebApplication(WebAppApiImpl webAppApi)
+ throws IOException, SliderException {
try {
webApp = new SliderAMWebApp(webAppApi);
HttpConfig.Policy policy = HttpConfig.Policy.HTTP_ONLY;
+ int port = getPortToRequest();
log.info("Launching web application at port {} with policy {}", port, policy);
WebApps.$for(SliderAMWebApp.BASE_PATH,
@@ -1101,7 +1090,7 @@
webAppApi,
RestPaths.WS_CONTEXT)
.withHttpPolicy(getConfig(), policy)
- .at(port)
+ .at("0.0.0.0", port, true)
.inDevMode()
.start(webApp);
@@ -1109,6 +1098,7 @@
new WebAppService<>("slider", webApp);
deployChildService(webAppService);
+ return webApp.port();
} catch (WebAppException e) {
if (e.getCause() instanceof IOException) {
throw (IOException)e.getCause();
@@ -1147,7 +1137,8 @@
/**
* Build up the port scanner. This may include setting a port range.
*/
- private void buildPortScanner(AggregateConf instanceDefinition) {
+ private void buildPortScanner(AggregateConf instanceDefinition)
+ throws BadConfigException {
portScanner = new PortScanner();
String portRange = instanceDefinition.
getAppConfOperations().getGlobalOptions().
@@ -1167,8 +1158,7 @@
* @return the port to request.
* @throws SliderException
*/
- private int getPortToRequest()
- throws SliderException {
+ private int getPortToRequest() throws SliderException, IOException {
return portScanner.getAvailablePort();
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index ed87b89..3213d93 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -42,7 +42,6 @@
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
@@ -802,6 +801,7 @@
continue;
}
if (hasUniqueNames(resources, name)) {
+ // THIS NAME IS A GROUP
int desiredInstanceCount = getDesiredInstanceCount(resources, name);
Integer groupCount = 0;
if (groupCounts.containsKey(name)) {
@@ -853,21 +853,21 @@
/**
* Get the desired instance count of a role, rejecting negative values
* @param resources resource map
- * @param role role name
+ * @param roleGroup role group
* @return the instance count
* @throws BadConfigException if the count is negative
*/
private int getDesiredInstanceCount(ConfTreeOperations resources,
- String role) throws BadConfigException {
+ String roleGroup) throws BadConfigException {
int desiredInstanceCount =
- resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0);
+ resources.getComponentOptInt(roleGroup, COMPONENT_INSTANCES, 0);
if (desiredInstanceCount < 0) {
- log.error("Role {} has negative desired instances : {}", role,
+ log.error("Role {} has negative desired instances : {}", roleGroup,
desiredInstanceCount);
throw new BadConfigException(
"Negative instance count (%) requested for component %s",
- desiredInstanceCount, role);
+ desiredInstanceCount, roleGroup);
}
return desiredInstanceCount;
}
@@ -1345,7 +1345,7 @@
* These are returned as integers, but there is special handling of the
* string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers
* the return of the maximum value.
- * @param name component to get from
+ * @param group component to get from
* @param option option name
* @param defVal default value
* @param maxVal value to return if the max val is requested
@@ -1353,12 +1353,12 @@
* @throws NumberFormatException if the role could not be parsed.
*/
private int getResourceRequirement(ConfTreeOperations resources,
- String name,
+ String group,
String option,
int defVal,
int maxVal) {
- String val = resources.getComponentOpt(name, option,
+ String val = resources.getComponentOpt(group, option,
Integer.toString(defVal));
Integer intVal;
if (YARN_RESOURCE_MAX.equals(val)) {
@@ -1380,14 +1380,15 @@
public Resource buildResourceRequirements(RoleStatus role, Resource capability) {
// Set up resource requirements from role values
String name = role.getName();
+ String group = role.getGroup();
ConfTreeOperations resources = getResourcesSnapshot();
int cores = getResourceRequirement(resources,
- name,
+ group,
YARN_CORES,
DEF_YARN_CORES,
containerMaxCores);
capability.setVirtualCores(cores);
- int ram = getResourceRequirement(resources, name,
+ int ram = getResourceRequirement(resources, group,
YARN_MEMORY,
DEF_YARN_MEMORY,
containerMaxMemory);
@@ -1798,11 +1799,20 @@
for (RoleStatus role : getRoleStatusMap().values()) {
String rolename = role.getName();
+ if (hasUniqueNames(instanceDefinition.getResourceOperations(),
+ role.getGroup())) {
+ cd.setRoleOpt(rolename, COMPONENT_PRIORITY, role.getPriority());
+ cd.setRoleOpt(rolename, ROLE_GROUP, role.getGroup());
+ MapOperations groupOptions = instanceDefinition.getResourceOperations()
+ .getComponent(role.getGroup());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename),
+ groupOptions.options);
+ }
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
role.getDesired());
- cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
+ cd.setRoleOpt(rolename, ROLE_ACTUAL_INSTANCES, nodeCount);
cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
cd.setRoleOpt(rolename, ROLE_RELEASING_INSTANCES, role.getReleasing());
cd.setRoleOpt(rolename, ROLE_FAILED_INSTANCES, role.getFailed());
@@ -1959,13 +1969,13 @@
/**
* Get the node failure threshold for a specific role, falling back to
* the global one if not
- * @param roleName role name
+ * @param roleGroup role group
* @return the threshold for failures
*/
- private int getNodeFailureThresholdForRole(String roleName) {
+ private int getNodeFailureThresholdForRole(String roleGroup) {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
- return resources.getComponentOptInt(roleName,
+ return resources.getComponentOptInt(roleGroup,
NODE_FAILURE_THRESHOLD,
nodeFailureThreshold);
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index e0b01a1..c6524be 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -283,6 +283,7 @@
*/
public synchronized AMRMClient.ContainerRequest escalate() {
Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued " + this);
+ log.debug("Escalating {}", this.toString());
escalated = true;
// this is now the priority
@@ -352,6 +353,7 @@
}
sb.append(", node=").append(node);
sb.append(", hasLocation=").append(requestHasLocation);
+ sb.append(", label=").append(label);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
sb.append(", mayEscalate=").append(mayEscalate);
sb.append(", escalated=").append(escalated);
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index 200fbc2..3a3b0c0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -25,7 +25,6 @@
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.SecurityUtils;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
@@ -40,6 +39,7 @@
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import java.net.BindException;
import java.util.Set;
/**
@@ -91,6 +91,7 @@
new QueuedThreadPool(
configsMap.getOptionInt("agent.threadpool.size.max", 25)));
agentServer.setStopAtShutdown(true);
+ agentServer.setGracefulShutdown(1000);
SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port);
SslSelectChannelConnector ssl2WayConnector =
@@ -115,6 +116,7 @@
agentRoot.addServlet(agent, "/*");
try {
+ openListeners();
agentServer.start();
} catch (IOException e) {
LOG.error("Unable to start agent server", e);
@@ -131,6 +133,37 @@
}
+ private void openListeners() throws Exception {
+ // from HttpServer2.openListeners()
+ for (Connector listener : agentServer.getConnectors()) {
+ if (listener.getLocalPort() != -1) {
+ // This listener is either started externally or has been bound
+ continue;
+ }
+ int port = listener.getPort();
+ while (true) {
+ // jetty has a bug where you can't reopen a listener that previously
+ // failed to open w/o issuing a close first, even if the port is changed
+ try {
+ listener.close();
+ listener.open();
+ LOG.info("Jetty bound to port " + listener.getLocalPort());
+ break;
+ } catch (BindException ex) {
+ if (port == 0) {
+ BindException be = new BindException("Port in use: "
+ + listener.getHost() + ":" + listener.getPort());
+ be.initCause(ex);
+ throw be;
+ }
+ }
+ // try the next port number
+ listener.setPort(++port);
+ Thread.sleep(100);
+ }
+ }
+ }
+
private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) {
SslSelectChannelConnector sslConnector = new
SslSelectChannelConnector();
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index 9bd4d3d..9e9e7ac 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -53,8 +53,8 @@
* <li>Output is streamed to the output logger provided</li>.
* <li>the input stream is closed as soon as the process starts.</li>
* <li>The most recent lines of output are saved to a linked list</li>.
- * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start
- * and finish of a process.</li>
+ * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent},
+ * is raised on the start and finish of a process.</li>
* </ol>
*
*/
@@ -65,12 +65,12 @@
public static final int RECENT_LINE_LOG_LIMIT = 64;
/**
- * Const defining the time in millis between polling for new text
+ * Const defining the time in millis between polling for new text.
*/
private static final int STREAM_READER_SLEEP_TIME = 200;
/**
- * limit on the length of a stream before it triggers an automatic newline
+ * limit on the length of a stream before it triggers an automatic newline.
*/
private static final int LINE_LENGTH = 256;
private final ProcessBuilder processBuilder;
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
index ec0c61d..dc591df 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
@@ -32,7 +32,7 @@
private Exception exception;
/**
- * Create an instance
+ * Create an instance.
* @param owner owning service
* @param action action to execute before terminating the service
*/
@@ -44,7 +44,7 @@
}
/**
- * Get the owning service
+ * Get the owning service.
* @return the service to receive notification when
* the runnable completes.
*/
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
index 99a2c28..737197b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
@@ -42,9 +42,10 @@
*/
public class ServiceThreadFactory implements ThreadFactory {
- private static AtomicInteger counter = new AtomicInteger(1);
+ private static final AtomicInteger counter = new AtomicInteger(1);
+
/**
- * Default format for thread names: {@value}
+ * Default format for thread names: {@value}.
*/
public static final String DEFAULT_NAMING_FORMAT = "%s-%03d";
private final String name;
@@ -68,8 +69,7 @@
}
/**
- *
- * Create an instance with the default naming format
+ * Create an instance with the default naming format.
* @param name base thread name
* @param daemons flag to indicate the threads should be marked as daemons
*/
@@ -89,7 +89,7 @@
}
/**
- * Create a single thread executor using this naming policy
+ * Create a single thread executor using this naming policy.
* @param name base thread name
* @param daemons flag to indicate the threads should be marked as daemons
* @return an executor
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
index 74d5623..65d14b7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
@@ -30,7 +30,9 @@
/**
* A service that calls the supplied callback when it is started -after the
- * given delay. It can be configured to stop itself after the callback has
+ * given delay.
+ *
+ * It can be configured to stop itself after the callback has
* completed, marking any exception raised as the exception of this service.
* The notifications come in on a callback thread -a thread that is only
* started in this service's <code>start()</code> operation.
@@ -38,9 +40,10 @@
public class WorkflowCallbackService<V> extends
WorkflowScheduledExecutorService<ScheduledExecutorService> {
protected static final Logger LOG =
- LoggerFactory.getLogger(WorkflowCallbackService.class);
+ LoggerFactory.getLogger(WorkflowCallbackService.class);
+
/**
- * This is the callback
+ * This is the callback.
*/
private final Callable<V> callback;
private final int delay;
diff --git a/slider-core/src/main/resources/webapps/static/busy.gif b/slider-core/src/main/resources/webapps/static/busy.gif
deleted file mode 100644
index 058ce1f..0000000
--- a/slider-core/src/main/resources/webapps/static/busy.gif
+++ /dev/null
Binary files differ
diff --git a/slider-core/src/main/resources/webapps/static/hadoop-st.png b/slider-core/src/main/resources/webapps/static/hadoop-st.png
deleted file mode 100644
index b481c04..0000000
--- a/slider-core/src/main/resources/webapps/static/hadoop-st.png
+++ /dev/null
Binary files differ
diff --git a/slider-core/src/test/app_packages/test_command_log/resources.json b/slider-core/src/test/app_packages/test_command_log/resources.json
index 18c505f..f6a401a 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources.json
@@ -11,7 +11,7 @@
"yarn.component.instances": "1"
},
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "512"
}
}
}
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
index 5255f0e..25e090e 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
@@ -6,12 +6,12 @@
},
"components": {
"COMMAND_LOGGER": {
- "yarn.memory": "128",
+ "yarn.memory": "256",
"yarn.role.priority": "1",
"yarn.component.instances": "1"
},
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "512"
}
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 6637c73..4a97e69 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -170,4 +170,54 @@
assert masterRole != null, "Role hbase-master must exist"
assert cd.roleNames.contains(master), "Role names must contain hbase-master"
}
+
+ @Test
+ public void testBuildClusterWithCompOpt() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "verify that a cluster can be built with compots"
+
+ String echo = "echo"
+ ServiceLauncher<SliderClient> launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [(echo): 1],
+ [ARG_COMP_OPT, echo, "non.resource.property", "value",
+ ARG_COMP_OPT, echo, "yarn.some.property", "yarn",
+ ARG_COMP_OPT, echo, "component.some.property", "component",
+ ARG_COMP_OPT, echo, "role.some.property", "role",
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PLACEMENT_POLICY, "4"],
+ true,
+ false,
+ agentDefOptions)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(clustername, false)
+
+ // verify that global component options propagate from the CLI
+ def aggregateConf = sliderClient.loadPersistedClusterDescription(clustername)
+ assert "value" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "non.resource.property", null)
+ assert null == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "non.resource.property", null)
+ assert "yarn" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "yarn.some.property", null)
+ assert "yarn" == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "yarn.some.property", null)
+ assert "component" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "component.some.property", null)
+ assert "component" == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "component.some.property", null)
+ assert "role" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "role.some.property", null)
+ assert "role" == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "role.some.property", null)
+ assert 1 == aggregateConf.resourceOperations.getComponentOptInt(echo,
+ ResourceKeys.COMPONENT_PRIORITY, -1)
+ assert 4 == aggregateConf.resourceOperations.getComponentOptInt(echo,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY, -1)
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
index a20ab23..81c4daf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
@@ -18,8 +18,10 @@
package org.apache.slider.client
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.security.KerberosDiags
import org.apache.hadoop.yarn.conf.YarnConfiguration
import static org.apache.slider.common.Constants.SUN_SECURITY_KRB5_DEBUG
import org.apache.slider.common.params.ActionDiagnosticArgs
@@ -28,8 +30,6 @@
import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.test.SliderTestBase
-import org.apache.slider.test.YarnMiniClusterTestBase
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Test
@@ -72,4 +72,50 @@
assert 0 == launcher.serviceExitCode
}
+ @Test
+ public void testKDiagExceptionConstruction() throws Throwable {
+ assert new KerberosDiags.KerberosDiagsFailure("CAT", "%02d", 3).toString().contains("03")
+ assert new KerberosDiags.KerberosDiagsFailure("CAT", "%w").toString().contains("%w")
+ assert new KerberosDiags.KerberosDiagsFailure("CAT", new Exception(), "%w")
+ .toString().contains("%w")
+ }
+
+ @Test
+ public void testKDiagPrintln() throws Throwable {
+ assert "%w" == KerberosDiags.format("%w")
+ assert "%s" == KerberosDiags.format("%s")
+ assert "false" == KerberosDiags.format("%s", false)
+ def sw = new StringWriter()
+ def kdiag = new KerberosDiags(new Configuration(),
+ new PrintWriter(sw), [], null, "self", 16, false)
+ try {
+ kdiag.println("%02d", 3)
+ kdiag.println("%s")
+ kdiag.println("%w")
+ } finally {
+ kdiag.close()
+ }
+ def output = sw.toString()
+ assert output.contains("03")
+ assert output.contains("%s")
+ assert output.contains("%w")
+ }
+
+ @Test
+ public void testKDiagDumpFile() throws Throwable {
+ def file1 = new File("./target/kdiaginput.txt")
+
+ def s = 'invalid %w string %s'
+ file1 << s
+ def sw = new StringWriter()
+ def kdiag = new KerberosDiags(new Configuration(),
+ new PrintWriter(sw), [], null, "self", 16, false)
+ try {
+ kdiag.dump(file1)
+ } finally {
+ kdiag.close()
+ }
+ def output = sw.toString()
+ assert output.contains(s)
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
index f009e25..4d87c75 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.core.exceptions.SliderException
import org.junit.Test
@@ -125,4 +126,47 @@
server.close()
}
}
+
+ @Test(expected = BadConfigException.class)
+ public void testBadRange() {
+ PortScanner portScanner = new PortScanner()
+ // note the em dash
+ portScanner.setPortRange("2000–2010")
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testEndBeforeStart() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("2001-2000")
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testEmptyRange() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("")
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testBlankRange() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange(" ")
+ }
+
+ @Test
+ public void testExtraComma() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("2000-2001, ")
+ List<Integer> ports = portScanner.remainingPortsToCheck
+ def expectedPorts = [2000, 2001]
+ assert ports == expectedPorts
+ }
+
+ @Test
+ public void testExtraCommas() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("2000-2001,, ,2003,")
+ List<Integer> ports = portScanner.remainingPortsToCheck
+ def expectedPorts = [2000, 2001, 2003]
+ assert ports == expectedPorts
+ }
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index e43d894..125b1f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -24,12 +24,10 @@
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -59,7 +57,7 @@
}
/**
- * This is the simplest AA allocation: no lables, so allocate anywhere
+ * This is the simplest AA allocation: no labels, so allocate anywhere
* @throws Throwable
*/
@Test
@@ -220,7 +218,7 @@
// flex down so that the next request should be cancelled
aaRole.desired = 0
- // expect: no new reqests, pending count --
+ // expect: no new requests, pending count --
List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
assert aaRole.pendingAntiAffineRequests == 0
assert !aaRole.AARequestOutstanding
@@ -283,7 +281,7 @@
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("4", NodeState.RUNNING, "gpu"))
}
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
index 5256163..84d6bc7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
@@ -22,6 +22,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
@@ -71,6 +72,8 @@
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "1024",
+ (ResourceKeys.YARN_CORES) : "2",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -84,12 +87,33 @@
return new ConfTreeOperations(resources)
}
+ private static void checkRole(MockAppState appState,
+ String roleName,
+ String roleGroup,
+ Map<String, String> expectedOpts) {
+
+ for (String key : expectedOpts.keySet()) {
+ if (ResourceKeys.COMPONENT_PRIORITY.equals(key) ||
+ ResourceKeys.COMPONENT_INSTANCES.equals(key)) {
+ continue
+ }
+ assert expectedOpts.get(key).equals(appState.getClusterStatus()
+ .getMandatoryRoleOpt(roleName, key))
+ }
+ assert 1 == appState.getClusterStatus().getMandatoryRoleOptInt(
+ roleName, ResourceKeys.COMPONENT_INSTANCES)
+ assert roleGroup.equals(appState.getClusterStatus().getMandatoryRoleOpt(
+ roleName, RoleKeys.ROLE_GROUP))
+ }
+
@Test
public void testDynamicFlexAddRole() throws Throwable {
def cd = init()
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "2",
(ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.YARN_MEMORY) : "384",
+ (ResourceKeys.YARN_CORES) : "4",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -103,6 +127,16 @@
assert 6 == appState.lookupRoleStatus("group11").priority
assert 7 == appState.lookupRoleStatus("group21").priority
assert 8 == appState.lookupRoleStatus("group22").priority
+ assert 1024 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 384 == appState.lookupRoleStatus("group21").resourceRequirements.memory
+ assert 384 == appState.lookupRoleStatus("group22").resourceRequirements.memory
+ assert 2 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
+ assert 4 == appState.lookupRoleStatus("group21").resourceRequirements.virtualCores
+ assert 4 == appState.lookupRoleStatus("group22").resourceRequirements.virtualCores
+
+ appState.refreshClusterStatus()
+ checkRole(appState, "group21", "group2", opts)
+ checkRole(appState, "group22", "group2", opts)
}
@Test
@@ -111,6 +145,8 @@
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "0",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "384",
+ (ResourceKeys.YARN_CORES) : "4",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -121,6 +157,8 @@
appState.lookupRoleStatus(6)
assert 0 == appState.lookupRoleStatus("group11").desired
assert 6 == appState.lookupRoleStatus("group11").priority
+ assert 384 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 4 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
}
@Test
@@ -129,6 +167,8 @@
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "3",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "640",
+ (ResourceKeys.YARN_CORES) : "8",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -145,6 +185,17 @@
assert 6 == appState.lookupRoleStatus("group11").priority
assert 7 == appState.lookupRoleStatus("group12").priority
assert 8 == appState.lookupRoleStatus("group13").priority
+ assert 640 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 640 == appState.lookupRoleStatus("group12").resourceRequirements.memory
+ assert 640 == appState.lookupRoleStatus("group13").resourceRequirements.memory
+ assert 8 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
+ assert 8 == appState.lookupRoleStatus("group12").resourceRequirements.virtualCores
+ assert 8 == appState.lookupRoleStatus("group13").resourceRequirements.virtualCores
+
+ appState.refreshClusterStatus()
+ checkRole(appState, "group11", "group1", opts)
+ checkRole(appState, "group12", "group1", opts)
+ checkRole(appState, "group13", "group1", opts)
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index f0fed95..e159dbb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -22,7 +22,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -48,8 +48,8 @@
super.setup()
// node 1 is GPU
- updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
- updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.instance.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.instance.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
@@ -108,7 +108,8 @@
assert 0 == appState.reviewRequestAndReleaseNodes().size()
// switch node 2 into being labelled
- def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu"))
+ def outcome = updateNodes(MockFactory.instance.
+ newNodeReport("00000002", NodeState.RUNNING, "gpu"))
assert cloneNodemap().size() == NODES
assert outcome.clusterChanged
@@ -118,7 +119,7 @@
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("00000004", NodeState.RUNNING, "gpu"))
}
@Test
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
index 9202b5c..fb9d976 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
@@ -55,21 +55,6 @@
createAndValidate(H1, "", true)
}
- @Test
- public void testLabelAndHostRelaxed() throws Throwable {
- expectValidationFailure(H1, "gpu", true)
- }
-
- @Test
- public void testLabelAndHostNonRelaxed() throws Throwable {
- expectValidationFailure(H1, "gpu", false)
- }
-
- @Test
- public void testComplexLabelExpression() throws Throwable {
- expectValidationFailure(null, "gpu||ps4", true)
- }
-
/**
* Use varargs for simple list to array conversion
* @param hostnames host names
@@ -79,17 +64,6 @@
hostnames
}
- void expectValidationFailure(
- String[] hosts,
- String labels,
- boolean relaxLocality) {
- try {
- def result = createAndValidate(hosts, labels, relaxLocality)
- fail("Expected an exception, got $result")
- } catch (InvalidContainerRequestException expected) {
- }
- }
-
void expectCreationFailure(
String[] hosts,
String labels,
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
index db84b0b..9c60ea5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -21,12 +21,9 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.api.proto.Messages
import org.apache.slider.api.proto.RestTypeMarshalling
-import org.apache.slider.api.types.NodeInformation
import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -72,7 +69,7 @@
}
protected boolean setNodeState(NodeInstance node, NodeState state) {
- node.updateNode(new MockNodeReport(node.hostname, state))
+ node.updateNode(MockFactory.instance.newNodeReport(node.hostname, state))
}
@Test
@@ -84,7 +81,8 @@
@Test
public void testFindSomeNodesSomeLabel() throws Throwable {
// all three will surface at first
- update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")])
+ update(nodeMap,
+ [MockFactory.instance.newNodeReport("1", NodeState.RUNNING, "GPU")])
def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU")
verifyResultSize(1, gpuNodes)
def instance = gpuNodes[0]
@@ -159,7 +157,7 @@
def role1 = node1.getOrCreate(1)
def role2 = node1.getOrCreate(2)
nodeMap.values().each {
- it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY))
+ it.updateNode(MockFactory.instance.newNodeReport("0", NodeState.UNHEALTHY))
}
assertNoAvailableNodes(1)
assertNoAvailableNodes(2)
@@ -249,6 +247,6 @@
def NodeMap createNodeMap(List<String> hosts, NodeState state,
String label = "") {
- createNodeMap(MockNodeReport.createInstances(hosts, state, label))
+ createNodeMap(MockFactory.instance.createNodeReports(hosts, state, label))
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 6969b38..0969824 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -238,12 +238,15 @@
def workerRole = lookupRole(WORKER.name)
// initial request
def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0)
+ assert (req1.label == WORKERS_LABEL)
+
assert (yarnRequest.nodeLabelExpression == null)
assert (!yarnRequest.relaxLocality)
// escalation
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
+ assert yarnRequest2.nodes == null
assert (yarnRequest2.relaxLocality)
+ assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
}
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index d873390..3ba6e31 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -23,8 +23,13 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
@@ -40,6 +45,8 @@
@Slf4j
class MockFactory implements MockRoles {
+ public static MockFactory instance = new MockFactory();
+
/*
Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
*/
@@ -239,7 +246,56 @@
return new MockResource(memory, vcores)
}
- MockContainerStatus newContainerStatus() {
- return new MockContainerStatus()
+ ContainerStatus newContainerStatus() {
+ return newContainerStatus(null, null, "", 0)
}
+
+ ContainerStatus newContainerStatus(ContainerId containerId,
+ ContainerState containerState, String diagnostics, int exitStatus) {
+ ContainerStatus.newInstance(containerId, containerState, diagnostics, exitStatus)
+ }
+
+ /**
+ * Create a single instance
+ * @param hostname
+ * @param nodeState
+ * @param label
+ */
+ NodeReport newNodeReport(String hostname, NodeState nodeState, String label = "") {
+ def nodeId = NodeId.newInstance(hostname, 80)
+ Integer.valueOf(hostname, 16)
+ newNodeReport(hostname, nodeId, nodeState, label)
+ }
+
+ NodeReport newNodeReport(
+ String hostname,
+ NodeId nodeId,
+ NodeState nodeState,
+ String label) {
+ def report = new NodeReportPBImpl();
+ def nodeLabels = new HashSet<>()
+ nodeLabels.add(label)
+ report.nodeId = nodeId
+ report.nodeLabels = nodeLabels
+ report.nodeState = nodeState
+ report.httpAddress = "http$hostname:80"
+ report
+ }
+
+ /**
+ * Create a list of instances -one for each hostname
+ * @param hostnames hosts
+ * @param nodeState state of all of them
+ * @param label label for all of them
+ * @return
+ */
+ List<NodeReport> createNodeReports(
+ List<String> hostnames,
+ NodeState nodeState = NodeState.RUNNING,
+ String label = "") {
+ hostnames.collect { String name ->
+ newNodeReport(name, nodeState, label)
+ }
+ }
+
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
deleted file mode 100644
index 8c3b712..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import groovy.transform.CompileStatic
-import org.apache.hadoop.yarn.api.records.NodeId
-import org.apache.hadoop.yarn.api.records.NodeReport
-import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.hadoop.yarn.api.records.Resource
-
-/**
- * Node report for testing
- */
-@CompileStatic
-class MockNodeReport extends NodeReport {
- NodeId nodeId;
- NodeState nodeState;
- String httpAddress;
- String rackName;
- Resource used;
- Resource capability;
- int numContainers;
- String healthReport;
- long lastHealthReportTime;
- Set<String> nodeLabels;
-
- MockNodeReport() {
- }
-
- /**
- * Create a single instance
- * @param hostname
- * @param nodeState
- * @param label
- */
- MockNodeReport(String hostname, NodeState nodeState, String label ="") {
- nodeId = NodeId.newInstance(hostname, 80)
- Integer.valueOf(hostname, 16)
- this.nodeState = nodeState
- this.httpAddress = "http$hostname:80"
- this.nodeLabels = new HashSet<>()
- nodeLabels.add(label)
- }
-
- /**
- * Create a list of instances -one for each hostname
- * @param hostnames hosts
- * @param nodeState state of all of them
- * @param label label for all of them
- * @return
- */
- static List<MockNodeReport> createInstances(
- List<String> hostnames,
- NodeState nodeState = NodeState.RUNNING,
- String label = "") {
- hostnames.collect { String name ->
- new MockNodeReport(name, nodeState, label)}
- }
-}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
similarity index 75%
rename from slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
rename to slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
index 4237f1e..a608e5a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
@@ -18,17 +18,20 @@
package org.apache.slider.server.appmaster.model.mock
+import groovy.transform.CompileStatic
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.ContainerState
import org.apache.hadoop.yarn.api.records.ContainerStatus
-import org.apache.hadoop.yarn.api.records.Resource
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
-class MockContainerStatus extends ContainerStatus {
+/**
+ * Node report for testing
+ */
+@CompileStatic
+class MockRecordFactory {
- ContainerId containerId
- ContainerState state
- String diagnostics
- int exitStatus
- Resource capability;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 265a796..f0a1494 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
/**
@@ -149,7 +150,7 @@
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
nodes.collect { MockYarnClusterNode n -> n.nodeReport }
}
@@ -167,7 +168,7 @@
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
- public MockNodeReport nodeReport
+ public NodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -181,9 +182,7 @@
containers[i] = new MockYarnClusterContainer(mci)
}
- nodeReport = new MockNodeReport()
- nodeReport.nodeId = nodeId
- nodeReport.nodeState = NodeState.RUNNING
+ nodeReport = MockFactory.instance.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
}
/**
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 7ab97fa..d946c6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -24,6 +24,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
@@ -166,7 +167,7 @@
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
cluster.nodeReports
}
-}
\ No newline at end of file
+}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
index eaf496c..dedf4f6 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
@@ -85,7 +85,9 @@
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("--metainfo file cannot be read"));
+ Assert.assertTrue(bce.getMessage().contains(
+ "Path specified with "
+ + "--metainfo either cannot be read or is not a file"));
}
try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
@@ -98,18 +100,53 @@
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("both --metainfo and --appdef may not be specified"));
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfo and --appdef cannot be specified"));
+ }
+
+ // both --metainfojson and --appdef cannot be specified
+ buildInfo.appMetaInfo = null;
+ buildInfo.appMetaInfoJson = "{}";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfojson and --appdef cannot be specified"));
}
buildInfo.appDef = null;
+ buildInfo.appMetaInfoJson = "";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Empty string specified with --metainfojson"));
+ }
+ buildInfo.appMetaInfo = metainfo;
+
+ // both --metainfo and --metainfojson cannot be specified
+ buildInfo.appMetaInfoJson = "{}";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfo and --metainfojson cannot be specified"));
+ }
+ buildInfo.appMetaInfoJson = null;
+
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --metainfo is provided"));
+ Assert.assertTrue(bce.getMessage().contains(
+ "application.def cannot "
+ + "not be set if --metainfo is specified in the cmd line"));
}
appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index e1d9ad1..153f965 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.91.0-incubating-SNAPSHOT</version>
+ <version>0.92.0-incubating-SNAPSHOT</version>
</parent>
<properties>
<work.dir>package-tmp</work.dir>
