Merge branch 'develop' into feature/SLIDER-906_docker_support
diff --git a/app-packages/kafka/README.md b/app-packages/kafka/README.md
index 0a3de5a..0341093 100644
--- a/app-packages/kafka/README.md
+++ b/app-packages/kafka/README.md
@@ -18,8 +18,6 @@
Kafka On YARN (KOYA)
====================
-[](https://gitter.im/DataTorrent/koya?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
-
### Goals
* Use capabilities of YARN for Kafka broker management
@@ -36,24 +34,23 @@
Checkout Slider code (https://github.com/apache/incubator-slider)
```sh
-git clone git@github.com:apache/incubator-slider.git
-git checkout -b slider-0.80.0-incubating remotes/origin/releases/slider-0.80.0-incubating
+git clone https://git-wip-us.apache.org/repos/asf/incubator-slider.git
+cd incubator-slider
+git checkout -b develop remotes/origin/develop
```
-The Slider version you checked out needs to match ${slider.version} in pom.xml
-Create symbolic link to Slider source code within the KOYA repository:
+Download Kafka binary package (http://kafka.apache.org/downloads.html), e.g.
```sh
-ln -s /path/to/repo/incubator-slider/ slider
+wget -O /tmp/kafka_2.10-0.8.2.1.tgz https://dist.apache.org/repos/dist/release/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz
```
-Download Kafka binary package (http://kafka.apache.org/downloads.html)
+Build Kafka app package
```sh
-mvn clean install -DskipTests -Dkafka.src=path/to/kafka_2.10-0.8.1.1.tgz -Dkafka.version=kafka_2.10-0.8.1.1
+mvn clean package -DskipTests -Pkafka-app-package -pkg.src=/tmp -Dpkg.name=kafka_2.10-0.8.2.1.tgz
```
Artifacts:
- - Archive with embedded Slider: __`target/koya-with-slider.zip`__
- - Separate Slider application package: __`target/slider-kafka-app-package-0.90.0-incubating-SNAPSHOT.zip`__
+ - Slider application package: __`app-packages/kafka/target/slider-kafka-app-package-0.90.0-incubating-SNAPSHOT.zip`__
###Installation
@@ -61,14 +58,14 @@
To use the archive with embedded Slider, copy it to the machine from which you launch YARN applications (Hadoop client, gateway or edge node). Extract the file and configure Slider:
-If the environment variables `HADOOP_CONF_DIR` or `JAVA_HOME` are not already defined through your Hadoop installation, you can export them in `slider-0.80.0-incubating/conf/slider-env.sh`
+If the environment variables `HADOOP_CONF_DIR` or `JAVA_HOME` are not already defined through your Hadoop installation, you can export them in `slider-0.90.0-incubating/conf/slider-env.sh`
-Example for CDH 5.4:
+Example:
```
export HADOOP_CONF_DIR=/etc/hadoop/conf
-export JAVA_HOME=/usr/java/jdk1.7.0_45-cloudera
+export JAVA_HOME=/usr/jdk64/jdk1.8.0_60
```
-If the registry ZooKeeper quorum was not already configured through Hadoop, modify `slider-0.80.0-incubating/conf/slider-client.xml`:
+If the registry ZooKeeper quorum was not already configured through Hadoop, modify `slider-0.90.0-incubating/conf/slider-client.xml`:
```
<property>
<name>hadoop.registry.zk.quorum</name>
diff --git a/app-packages/kafka/pom.xml b/app-packages/kafka/pom.xml
index d491109..8e54455 100644
--- a/app-packages/kafka/pom.xml
+++ b/app-packages/kafka/pom.xml
@@ -28,85 +28,6 @@
<name>Slider Kafka App Package</name>
<description>Slider Kafka App Package</description>
- <properties>
- <kafka.src>${project.build.directory}/kafka_2.10-0.8.2.1.tgz</kafka.src>
- <kafka.version>kafka_2.10-0.8.2.1</kafka.version>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>${maven-assembly-plugin.version}</version>
- <executions>
- <execution>
- <id>app-package</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptor>src/assembly/koya-app-package.xml</descriptor>
- <appendAssemblyId>false</appendAssemblyId>
- <finalName>${app.package.name}</finalName>
- </configuration>
- </execution>
- <execution>
- <id>package-with-slider</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <descriptor>src/assembly/koya-with-slider.xml</descriptor>
- <appendAssemblyId>false</appendAssemblyId>
- <finalName>slider-kafka-app-package-with-slider-${project.version}</finalName>
- <attach>false</attach>
- </configuration>
- </execution>
-
- </executions>
- </plugin>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>${maven-compiler-plugin.version}</version>
- <dependencies>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-eclipse-compiler</artifactId>
- <version>${groovy-eclipse-compiler.version}</version>
- </dependency>
- <dependency>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-eclipse-batch</artifactId>
- <version>${groovy-eclipse-batch.version}</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>wagon-maven-plugin</artifactId>
- <version>${wagon-maven-plugin.version}</version>
- <executions>
- <execution>
- <id>download-kafka</id>
- <phase>prepare-package</phase>
- <goals>
- <goal>download-single</goal>
- </goals>
- <configuration>
- <url>https://dist.apache.org</url>
- <fromFile>repos/dist/release/kafka/0.8.2.1/kafka_2.10-0.8.2.1.tgz</fromFile>
- <toDir>${project.build.directory}</toDir>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
-
- </build>
-
<dependencies>
<dependency>
<groupId>org.apache.slider</groupId>
@@ -116,6 +37,64 @@
<profiles>
<profile>
+ <id>kafka-app-package</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven-assembly-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>app-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptor>src/assembly/koya-app-package.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ <finalName>${app.package.name}</finalName>
+ </configuration>
+ </execution>
+ <execution>
+ <id>package-with-slider</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <descriptor>src/assembly/koya-with-slider.xml</descriptor>
+ <appendAssemblyId>false</appendAssemblyId>
+ <finalName>slider-kafka-app-package-with-slider-${project.version}</finalName>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>${maven-antrun-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>validate</phase>
+ <configuration>
+ <target name="copy and rename file">
+ <copy file="${pkg.src}/${pkg.name}" tofile="${project.build.directory}/${pkg.name}" />
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>rat</id>
<build>
<plugins>
@@ -142,5 +121,24 @@
</build>
</profile>
</profiles>
-
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven-compiler-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/app-packages/kafka/src/assembly/koya-app-package.xml b/app-packages/kafka/src/assembly/koya-app-package.xml
index 9d984c1..10f807f 100644
--- a/app-packages/kafka/src/assembly/koya-app-package.xml
+++ b/app-packages/kafka/src/assembly/koya-app-package.xml
@@ -48,8 +48,8 @@
<fileMode>0755</fileMode>
</file>
<file>
- <source>${kafka.src}</source>
- <destName>${kafka.version}.tgz</destName>
+ <source>${pkg.src}/${pkg.name}</source>
+ <destName>${pkg.name}</destName>
<outputDirectory>/package/files/</outputDirectory>
<fileMode>0755</fileMode>
</file>
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 7baf659..dd90e46 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -25,6 +25,7 @@
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -180,6 +181,7 @@
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.InterruptedIOException;
import java.io.PrintStream;
@@ -1356,6 +1358,36 @@
return EXIT_SUCCESS;
}
+ private void createSummaryMetainfoFile(Path srcFile, Path destFile,
+ boolean overwrite) throws IOException {
+ FileSystem srcFs = srcFile.getFileSystem(getConfig());
+ try (InputStream inputStreamJson = SliderUtils
+ .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.json");
+ InputStream inputStreamXml = SliderUtils
+ .getApplicationResourceInputStream(srcFs, srcFile, "metainfo.xml");) {
+ InputStream inputStream = null;
+ Path summaryFileInFs = null;
+ if (inputStreamJson != null) {
+ inputStream = inputStreamJson;
+ summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
+ + ".metainfo.json");
+ log.info("Found JSON metainfo file in package");
+ } else if (inputStreamXml != null) {
+ inputStream = inputStreamXml;
+ summaryFileInFs = new Path(destFile.getParent(), destFile.getName()
+ + ".metainfo.xml");
+ log.info("Found XML metainfo file in package");
+ }
+ if (inputStream != null) {
+ try (FSDataOutputStream dataOutputStream = sliderFileSystem
+ .getFileSystem().create(summaryFileInFs, overwrite)) {
+ log.info("Creating summary metainfo file");
+ IOUtils.copy(inputStream, dataOutputStream);
+ }
+ }
+ }
+ }
+
private int actionPackageInstall(ActionPackageArgs actionPackageArgs)
throws YarnException, IOException {
requireArgumentSet(Arguments.ARG_NAME, actionPackageArgs.name);
@@ -1376,6 +1408,7 @@
log.info("Installing package {} to {} (overwrite set to {})", srcFile,
fileInFs, actionPackageArgs.replacePkg);
fs.copyFromLocalFile(false, actionPackageArgs.replacePkg, srcFile, fileInFs);
+ createSummaryMetainfoFile(srcFile, fileInFs, actionPackageArgs.replacePkg);
String destPathWithHomeDir = Path
.getPathWithoutSchemeAndAuthority(fileInFs).toString();
diff --git a/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java b/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
index 9c18036..4286596 100644
--- a/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
+++ b/slider-core/src/main/java/org/apache/slider/client/rest/RestClientFactory.java
@@ -30,7 +30,7 @@
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_APPLICATION;
/**
- * Factory for the Rest cilent; hides the lookup and instantiation.
+ * Factory for the Rest client; hides the lookup and instantiation.
* <p>
*
*/
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 18cc779..eae80f5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1902,12 +1902,19 @@
}
/**
- * Retrieve the HDP version if it is an HDP cluster, or null otherwise
+ * Retrieve the HDP version if it is an HDP cluster, or null otherwise. It
+ * first checks if system property HDP_VERSION is defined. If not it checks if
+ * system env HDP_VERSION is defined.
*
- * @return HDP version
+ * @return HDP version (if defined) or null otherwise
*/
public static String getHdpVersion() {
- return System.getenv(SliderKeys.HDP_VERSION_PROP_NAME);
+ String hdpVersion = System
+ .getProperty(SliderKeys.HDP_VERSION_PROP_NAME);
+ if (StringUtils.isEmpty(hdpVersion)) {
+ hdpVersion = System.getenv(SliderKeys.HDP_VERSION_PROP_NAME);
+ }
+ return hdpVersion;
}
/**
diff --git a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
index cf6d21f..f192ec8 100644
--- a/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/main/ServiceLauncher.java
@@ -87,7 +87,7 @@
private volatile S service;
private int serviceExitCode;
@SuppressWarnings("MismatchedQueryAndUpdateOfCollection")
- private final List<IrqHandler> interruptHandlers = new ArrayList<IrqHandler>(1);
+ private final List<IrqHandler> interruptHandlers = new ArrayList<>(1);
private Configuration configuration;
private String serviceClassName;
private static AtomicBoolean signalAlreadyReceived = new AtomicBoolean(false);
@@ -447,7 +447,7 @@
try {
conf.addResource(file.toURI().toURL());
} catch (MalformedURLException e) {
- LOG.debug("File {} cannot be converted to URL: {}", e);
+ LOG.debug("File {} cannot be converted to URL", file, e);
exitWithMessage(EXIT_COMMAND_ARGUMENT_ERROR,
ARG_CONF + ": configuration file path invalid: " + file);
}
@@ -626,7 +626,7 @@
Thread.setDefaultUncaughtExceptionHandler(
new YarnUncaughtExceptionHandler());
- ServiceLauncher serviceLauncher = new ServiceLauncher<Service>(serviceClassName);
+ ServiceLauncher serviceLauncher = new ServiceLauncher<>(serviceClassName);
serviceLauncher.launchServiceAndExit(argsList);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index b38dc6e..cfcfc5d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -40,30 +41,55 @@
public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
+ public static Metainfo getApplicationMetainfoFromSummaryFile(
+ SliderFileSystem fileSystem, String metainfoPath, boolean metainfoForAddon) {
+ FileSystem fs = fileSystem.getFileSystem();
+ Path appPathXML = new Path(metainfoPath + ".metainfo.xml");
+ Path appPathJson = new Path(metainfoPath + ".metainfo.json");
+ Path appPathUsed = null;
+ try {
+ FSDataInputStream appStream = null;
+ if (fs.exists(appPathXML)) {
+ appPathUsed = appPathXML;
+ appStream = fs.open(appPathXML);
+ return parseMetainfo(appStream, metainfoForAddon, "xml");
+ } else if (fs.exists(appPathJson)) {
+ appPathUsed = appPathJson;
+ appStream = fs.open(appPathJson);
+ return parseMetainfo(appStream, metainfoForAddon, "json");
+ }
+ } catch (IOException e) {
+ log.info("Failed to get metainfo from summary file {} - {}", appPathUsed,
+ e.getMessage());
+ log.debug("Failed to get metainfo", e);
+ }
+ return null;
+ }
+
public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
String metainfoPath, boolean metainfoForAddon) throws IOException,
BadConfigException {
log.info("Reading metainfo at {}", metainfoPath);
+ Metainfo metainfo = getApplicationMetainfoFromSummaryFile(fileSystem,
+ metainfoPath, metainfoForAddon);
+ if (metainfo != null) {
+ log.info("Got metainfo from summary file");
+ return metainfo;
+ }
+
FileSystem fs = fileSystem.getFileSystem();
Path appPath = new Path(metainfoPath);
- Metainfo metainfo = null;
- AbstractMetainfoParser metainfoParser = null;
- if (metainfoForAddon) {
- metainfoParser = new AddonPackageMetainfoParser();
- } else {
- metainfoParser = new MetainfoParser();
- }
InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.json");
if (metainfoJsonStream == null) {
InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
fs, appPath, "metainfo.xml");
if (metainfoXMLStream != null) {
- metainfo = metainfoParser.fromXmlStream(metainfoXMLStream);
+ metainfo = parseMetainfo(metainfoXMLStream, metainfoForAddon, "xml");
}
} else {
- metainfo = metainfoParser.fromJsonStream(metainfoJsonStream);
+ metainfo = parseMetainfo(metainfoJsonStream, metainfoForAddon, "json");
}
if (metainfo == null) {
@@ -74,6 +100,22 @@
return metainfo;
}
+ private static Metainfo parseMetainfo(InputStream stream,
+ boolean metainfoForAddon, String type) throws IOException {
+ AbstractMetainfoParser metainfoParser = null;
+ if (metainfoForAddon) {
+ metainfoParser = new AddonPackageMetainfoParser();
+ } else {
+ metainfoParser = new MetainfoParser();
+ }
+ if (type.equals("xml")) {
+ return metainfoParser.fromXmlStream(stream);
+ } else if (type.equals("json")) {
+ return metainfoParser.fromJsonStream(stream);
+ }
+ return null;
+ }
+
static DefaultConfig getDefaultConfig(SliderFileSystem fileSystem,
String appDef, String configFileName)
throws IOException {
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
index bce53b8..67d1f15 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/AbstractMetainfoParser.java
@@ -109,7 +109,9 @@
log.debug("SAXException in metainfoparser during fromXmlStream: "
+ e.getMessage());
} finally {
- metainfoStream.close();
+ if (metainfoStream != null) {
+ metainfoStream.close();
+ }
}
return null;
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 0759c5a..73f24c7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1235,7 +1235,7 @@
* @throws IOException
*/
private void startAgentWebApp(MapOperations appInformation,
- Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException {
+ Configuration serviceConf, WebAppApiImpl webAppApi) throws IOException, SliderException {
URL[] urls = ((URLClassLoader) AgentWebApp.class.getClassLoader() ).getURLs();
StringBuilder sb = new StringBuilder("AM classpath:");
for (URL url : urls) {
@@ -1252,7 +1252,9 @@
webAppApi,
RestPaths.AGENT_WS_CONTEXT)
.withComponentConfig(appMasterConfig)
- .start();
+ .withPort(getPortToRequest())
+ .withSecuredPort(getPortToRequest())
+ .start();
agentOpsUrl =
"https://" + appMasterHostname + ":" + agentWebApp.getSecuredPort();
agentStatusUrl =
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index f9ea06d..200fbc2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -56,6 +56,8 @@
final String name;
final String wsName;
final WebAppApi application;
+ int port;
+ int securedPort;
MapOperations configsMap;
public Builder(String name, String wsName, WebAppApi application) {
@@ -69,6 +71,16 @@
return this;
}
+ public Builder withPort (int port) {
+ this.port = port;
+ return this;
+ }
+
+ public Builder withSecuredPort (int securedPort) {
+ this.securedPort = securedPort;
+ return this;
+ }
+
public AgentWebApp start() throws IOException {
if (configsMap == null) {
throw new IllegalStateException("No SSL Configuration Available");
@@ -80,11 +92,11 @@
configsMap.getOptionInt("agent.threadpool.size.max", 25)));
agentServer.setStopAtShutdown(true);
- SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false);
+ SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port);
SslSelectChannelConnector ssl2WayConnector =
createSSLConnector(Boolean.valueOf(
configsMap.getOption(AgentKeys.KEY_AGENT_TWO_WAY_SSL_ENABLED,
- "false")));
+ "false")), securedPort);
agentServer.setConnectors(new Connector[]{ssl1WayConnector,
ssl2WayConnector});
@@ -119,7 +131,7 @@
}
- private SslSelectChannelConnector createSSLConnector(boolean needClientAuth) {
+ private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) {
SslSelectChannelConnector sslConnector = new
SslSelectChannelConnector();
@@ -135,6 +147,7 @@
sslConnector.setTruststoreType("PKCS12");
sslConnector.setNeedClientAuth(needClientAuth);
+ sslConnector.setPort(port);
sslConnector.setAcceptors(2);
return sslConnector;
}
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
new file mode 100644
index 0000000..5e1dc7f
--- /dev/null
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentUtils.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.tools.TestUtility;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestAgentUtils {
+ protected static final Logger log =
+ LoggerFactory.getLogger(TestAgentUtils.class);
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+ private static final String metainfo_str = "<metainfo>\n"
+ + " <schemaVersion>2.0</schemaVersion>\n"
+ + " <application>\n"
+ + " <name>MYTESTAPPLICATION</name>\n"
+ + " <comment>\n"
+ + " My Test Application\n"
+ + " </comment>\n"
+ + " <version>1.0</version>\n"
+ + " <type>YARN-APP</type>\n"
+ + " <components>\n"
+ + " <component>\n"
+ + " <name>REST</name>\n"
+ + " <category>MASTER</category>\n"
+ + " <commandScript>\n"
+ + " <script>scripts/rest.py</script>\n"
+ + " <scriptType>PYTHON</scriptType>\n"
+ + " <timeout>600</timeout>\n"
+ + " </commandScript>\n"
+ + " </component>\n"
+ + " </components>\n"
+ + " </application>\n"
+ + "</metainfo>";
+
+ @Test
+ public void testGetApplicationMetainfo() throws Exception {
+ String zipFileName = TestUtility.createAppPackage(
+ folder,
+ "testpkg",
+ "test.zip",
+ "target/test-classes/org/apache/slider/common/tools/test");
+ Configuration configuration = new Configuration();
+ FileSystem fs = FileSystem.getLocal(configuration);
+ log.info("fs working dir is {}", fs.getWorkingDirectory().toString());
+ SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration);
+
+ // Without accompany metainfo file, read metainfo from the zip file
+ Metainfo metainfo = AgentUtils.getApplicationMetainfo(
+ sliderFileSystem, zipFileName, false);
+ Assert.assertNotNull(metainfo.getApplication());
+ Assert.assertEquals("STORM", metainfo.getApplication().getName());
+
+ // With accompany metainfo file, read metainfo from the accompany file
+ String acompanyFileName = zipFileName + ".metainfo.xml";
+ File f = new File(acompanyFileName);
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(f))) {
+ writer.write(metainfo_str);
+ }
+ metainfo = AgentUtils.getApplicationMetainfo(
+ sliderFileSystem, zipFileName, false);
+ Assert.assertNotNull(metainfo.getApplication());
+ Assert.assertEquals("MYTESTAPPLICATION", metainfo.getApplication().getName());
+ }
+}
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 7237ff4..b5d6a94 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.tools.PortScanner;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
@@ -61,6 +62,7 @@
import java.net.URI;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
public class TestAMAgentWebServices {
@@ -84,6 +86,8 @@
public static final String AGENT_URL =
"https://localhost:${PORT}/ws/v1/slider/agents/";
+ public static final int MIN_PORT = 50000;
+ public static final int MAX_PORT = 50050;
static MockFactory factory = new MockFactory();
private static WebAppApi slider;
@@ -131,9 +135,14 @@
MapOperations compOperations = new MapOperations();
+ PortScanner portScanner = new PortScanner();
+ portScanner.setPortRange(Integer.toString(MIN_PORT) + "-" + Integer.toString(MAX_PORT));
+
webApp = AgentWebApp.$for(AgentWebApp.BASE_PATH, slider,
RestPaths.WS_AGENT_CONTEXT_ROOT)
.withComponentConfig(compOperations)
+ .withPort(portScanner.getAvailablePort())
+ .withSecuredPort(portScanner.getAvailablePort())
.start();
base_url = AGENT_URL.replace("${PORT}",
Integer.toString(webApp.getSecuredPort()));
@@ -184,6 +193,16 @@
assertEquals(200, response.getStatus());
}
+ @Test
+ public void testAllowedPortRange() throws Exception {
+ assertTrue(webApp.getPort() >= MIN_PORT && webApp.getPort() <= MAX_PORT);
+ }
+
+ @Test
+ public void testAllowedSecurePortRange() throws Exception {
+ assertTrue(webApp.getSecuredPort() >= MIN_PORT && webApp.getSecuredPort() <= MAX_PORT);
+ }
+
// @Test
// public void testSleepForAWhile() throws Throwable {
// log.info("Agent is running at {}", base_url);
