Merge branch 'develop' into feature/SLIDER-82-anti-affinity-attempt-2
diff --git a/NOTICE b/NOTICE
index 277b28a..aa37418 100644
--- a/NOTICE
+++ b/NOTICE
@@ -1,5 +1,5 @@
Apache Slider
-Copyright 2015-2016 The Apache Software Foundation
+Copyright 2014-2015 The Apache Software Foundation
This product includes software developed at The Apache Software
Foundation (http://www.apache.org/).
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 11d04b7..8744741 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/accumulo/src/license/THIRD-PARTY.properties b/app-packages/accumulo/src/license/THIRD-PARTY.properties
index 9322b8f..fb20f7e 100644
--- a/app-packages/accumulo/src/license/THIRD-PARTY.properties
+++ b/app-packages/accumulo/src/license/THIRD-PARTY.properties
@@ -2,10 +2,6 @@
#-------------------------------------------------------------------------------
# Already used licenses in project :
# - Apache License
-# - Apache License 2.0
-# - Apache License Version 2
-# - Apache License, Version 2.0
-# - Apache Software License - Version 2.0
# - BSD
# - CDDL + GPLv2 with classpath exception
# - CDDL 1.1
@@ -13,10 +9,10 @@
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - New BSD License
-# - New BSD license
# - Public Domain
# - Revised BSD
# - The Apache Software License, Version 2.0
@@ -26,8 +22,7 @@
# Please fill the missing licenses for dependencies :
#
#
-#Mon Jan 26 16:00:07 GMT 2015
-asm--asm--3.1=The BSD 3-Clause License
+#Thu Oct 15 16:45:03 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
diff --git a/app-packages/command-logger/application-pkg/pom.xml b/app-packages/command-logger/application-pkg/pom.xml
index 23d355c..82a4c81 100644
--- a/app-packages/command-logger/application-pkg/pom.xml
+++ b/app-packages/command-logger/application-pkg/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/command-logger/slider-pkg/pom.xml b/app-packages/command-logger/slider-pkg/pom.xml
index ea66881..0699a80 100644
--- a/app-packages/command-logger/slider-pkg/pom.xml
+++ b/app-packages/command-logger/slider-pkg/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/hbase-win/pom.xml b/app-packages/hbase-win/pom.xml
index e641006..424b2c2 100644
--- a/app-packages/hbase-win/pom.xml
+++ b/app-packages/hbase-win/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index 3188d60..4d1cea0 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/pom.xml b/app-packages/pom.xml
index 6f9d0d1..f32e71c 100644
--- a/app-packages/pom.xml
+++ b/app-packages/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/storm-win/pom.xml b/app-packages/storm-win/pom.xml
index b6e97c2..d8219db 100644
--- a/app-packages/storm-win/pom.xml
+++ b/app-packages/storm-win/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/app-packages/storm/pom.xml b/app-packages/storm/pom.xml
index 6bc95ac..75e918a 100644
--- a/app-packages/storm/pom.xml
+++ b/app-packages/storm/pom.xml
@@ -20,7 +20,7 @@
<parent>
<groupId>org.apache.slider.packages</groupId>
<artifactId>slider-app-packages</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index 3096527..2a7e41b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
<name>Slider</name>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<description>
@@ -524,7 +524,7 @@
<!-- hadoop-client includes the following jars, so they do not need to be
included separately:
- hadoop-common, hadoop-hdfs, hadoop-mapreduce-client-app,
+ hadoop-common, hadoop-hdfs (client?), hadoop-mapreduce-client-app,
hadoop-yarn-api, hadoop-mapreduce-client-core,
hadoop-mapreduce-client-jobclient, and hadoop-annotations
-->
@@ -552,6 +552,34 @@
</exclusions>
</dependency>
+ <!--
+ Hadoop HDFS is pulled in to guarantee that any split to hadoop-hdfs-client doesn't
+ break the build.
+ -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<!-- hadoop-minicluster includes the following test-jars, so they do not
need to be included separately:
hadoop-common, hadoop-hdfs, hadoop-yarn-server-tests,
@@ -1444,6 +1472,7 @@
<exclude>DISCLAIMER</exclude>
<exclude>app-packages/hbase/target/**</exclude>
<exclude>target/*</exclude>
+ <exclude>DEPENDENCIES</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index c494b77..009a864 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>slider-agent</artifactId>
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index e6a4790..bfd4a27 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -55,7 +55,7 @@
os._exit(0)
logger.info('signal received, exiting.')
global controller
- if controller is not None:
+ if controller is not None and hasattr(controller, 'actionQueue'):
tmpdir = controller.actionQueue.dockerManager.stop_container()
ProcessHelper.stopAgent()
diff --git a/slider-assembly/pom.xml b/slider-assembly/pom.xml
index acf2954..0f7340f 100644
--- a/slider-assembly/pom.xml
+++ b/slider-assembly/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
@@ -205,13 +205,53 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>${maven-assembly-plugin.version}</version>
- <configuration>
- <tarLongFileMode>gnu</tarLongFileMode>
- <descriptors>
- <descriptor>src/assembly/slider-bin.xml</descriptor>
- </descriptors>
- <finalName>slider-${project.version}</finalName>
- </configuration>
+ <executions>
+ <execution>
+ <id>make-dependency</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptors>
+ <descriptor>src/assembly/slider-dependency.xml</descriptor>
+ </descriptors>
+ <finalName>slider</finalName>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <version>${maven-resources-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy-dependency</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ <overwrite>true</overwrite>
+ <resources>
+ <resource>
+ <directory>${project.build.directory}</directory>
+ <includes>
+ <include>slider-dependency.tar.gz</include>
+ </includes>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven-assembly-plugin.version}</version>
<executions>
<execution>
<id>make-assembly</id>
@@ -219,6 +259,13 @@
<goals>
<goal>single</goal>
</goals>
+ <configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
+ <descriptors>
+ <descriptor>src/assembly/slider-bin.xml</descriptor>
+ </descriptors>
+ <finalName>slider-${project.version}</finalName>
+ </configuration>
</execution>
</executions>
</plugin>
diff --git a/slider-assembly/src/assembly/slider-bin.xml b/slider-assembly/src/assembly/slider-bin.xml
index 8887ec8..07066b5 100644
--- a/slider-assembly/src/assembly/slider-bin.xml
+++ b/slider-assembly/src/assembly/slider-bin.xml
@@ -86,6 +86,12 @@
</file>
<file>
+ <source>${project.build.directory}/lib/slider-dependency.tar.gz</source>
+ <outputDirectory>lib</outputDirectory>
+ <destName>slider.tar.gz</destName>
+ </file>
+
+ <file>
<source>${project.build.directory}/maven-shared-archive-resources/META-INF/DEPENDENCIES</source>
<outputDirectory>/</outputDirectory>
</file>
diff --git a/slider-assembly/src/assembly/slider-dependency.xml b/slider-assembly/src/assembly/slider-dependency.xml
new file mode 100644
index 0000000..b2bae61
--- /dev/null
+++ b/slider-assembly/src/assembly/slider-dependency.xml
@@ -0,0 +1,38 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+
+<assembly
+ xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
+ <id>dependency</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+
+ <dependencySets>
+ <dependencySet>
+ <scope>runtime</scope>
+ <outputDirectory></outputDirectory>
+ <useProjectArtifact>false</useProjectArtifact>
+ <unpack>false</unpack>
+ </dependencySet>
+ </dependencySets>
+</assembly>
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 7e13b92..704fa06 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -23,7 +23,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
<build>
@@ -258,6 +258,14 @@
<scope>compile</scope>
</dependency>
+<!-- if the move to hadoop-hdfs-client JAR causes problems insert this
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>compile</scope>
+ </dependency>
+-->
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
diff --git a/slider-core/src/license/THIRD-PARTY.properties b/slider-core/src/license/THIRD-PARTY.properties
index 662fde5..1abd56e 100644
--- a/slider-core/src/license/THIRD-PARTY.properties
+++ b/slider-core/src/license/THIRD-PARTY.properties
@@ -10,6 +10,7 @@
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - MPL 1.1
@@ -24,8 +25,7 @@
# Please fill the missing licenses for dependencies :
#
#
-#Mon Jan 26 15:27:00 GMT 2015
-asm--asm--3.1=The BSD 3-Clause License
+#Thu Oct 15 16:45:02 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 3404039..7a25629 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -34,7 +34,6 @@
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
@@ -2235,7 +2234,7 @@
// the relevant security settings go over
addConfOptionToCLI(commandLine,
config,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -2399,11 +2398,10 @@
*/
private void propagatePrincipals(Configuration config,
AggregateConf clusterSpec) {
- String dfsPrincipal = config.get(
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ String dfsPrincipal = config.get(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+ DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
clusterSpec.getAppConfOperations().getGlobalOptions().putIfUnset(
siteDfsPrincipal,
dfsPrincipal);
@@ -4399,8 +4397,7 @@
FilenameFilter jarFilter = new FilenameFilter() {
public boolean accept(File dir, String name) {
String lowercaseName = name.toLowerCase();
- if (lowercaseName.endsWith(".jar")
- && !lowercaseName.startsWith("slider-core")) {
+ if (lowercaseName.endsWith(".jar")) {
return true;
} else {
return false;
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
index 3d2b868..07214b2 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java
@@ -166,11 +166,39 @@
*/
String KEY_IPC_CLIENT_RETRY_POLICY_ENABLED =
"slider.ipc.client.retry.enabled";
- public static final boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
- public static final String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
+ boolean IPC_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = true;
+ String KEY_IPC_CLIENT_RETRY_POLICY_SPEC =
"slider.ipc.client.retry.policy.spec";
- public static final String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
+ String IPC_CLIENT_RETRY_POLICY_SPEC_DEFAULT =
"10000,6,60000,10"; //t1,n1,t2,n2,...
String KEY_AM_LAUNCH_ENV = "slider.am.launch.env";
+
+ /**
+ * From {@code DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY}
+ */
+ String DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY = "dfs.namenode.kerberos.principal";
+
+ String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY = "dfs.datanode.kerberos.principal";
+
+ //Delegation token related keys
+ String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY
+ = "dfs.namenode.delegation.key.update-interval";
+ long DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT = 24 * 60 * 60 *
+ 1000; // 1 day
+ String DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY
+ = "dfs.namenode.delegation.token.renew-interval";
+ long DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT = 24 * 60 * 60 *
+ 1000; // 1 day
+ String DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY
+ = "dfs.namenode.delegation.token.max-lifetime";
+ long DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT = 7 * 24 * 60 * 60 *
+ 1000; // 7 days
+ String DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY
+ = "dfs.namenode.delegation.token.always-use"; // for tests
+ boolean DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_DEFAULT = false;
+ String DFS_NAMENODE_KEYTAB_FILE_KEY = "dfs.namenode.keytab.file";
+ String DFS_NAMENODE_DU_RESERVED_KEY = "dfs.namenode.resource.du.reserved";
+
+
}
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 0349ebd..f380ec6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -35,7 +35,6 @@
import org.apache.hadoop.fs.GlobFilter;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
@@ -1289,8 +1288,7 @@
}
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
return true;
}
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 0a8dcdb..7756055 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -23,7 +23,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.StatusKeys;
@@ -203,11 +202,10 @@
* Propagate any critical principals from the current site config down to the HBase one.
*/
public void propagatePrincipals() {
- String dfsPrincipal = conf.get(
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ String dfsPrincipal = conf.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
if (dfsPrincipal != null) {
String siteDfsPrincipal = OptionKeys.SITE_XML_PREFIX +
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
+ SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY;
instanceDescription.getAppConfOperations().set(siteDfsPrincipal, dfsPrincipal);
}
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 5861256..df91d7f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -32,7 +32,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.http.HttpConfig;
@@ -473,8 +472,7 @@
UserGroupInformation.setConfiguration(conf);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
log.debug("Authenticating as {}", ugi);
- SliderUtils.verifyPrincipalSet(conf,
- DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
+ SliderUtils.verifyPrincipalSet(conf, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
} else {
log.info("Cluster is insecure");
}
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
index 27219e4..8948f0d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/monkey/ChaosMonkeyService.java
@@ -60,7 +60,8 @@
public synchronized void addTarget(String name,
ChaosTarget target, long probability) {
if (probability > 0) {
- log.info("Adding {} with probability {}", name, probability / InternalKeys.PROBABILITY_PERCENT_1);
+ log.info("Adding {} with probability {}", name,
+ ((double)probability) / InternalKeys.PROBABILITY_PERCENT_1);
chaosEntries.add(new ChaosEntry(name, target, probability, metrics));
} else {
log.debug("Action {} not enabled", name);
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
index 01cc5e5..63f66a3 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/FsDelegationTokenManager.java
@@ -17,14 +17,15 @@
package org.apache.slider.server.services.security;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
import org.apache.hadoop.util.Time;
+import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.server.appmaster.SliderAppMaster;
import org.apache.slider.server.appmaster.actions.AsyncAction;
@@ -63,15 +64,15 @@
private void createRemoteUser(Configuration configuration) throws IOException {
Configuration loginConfig = new Configuration(configuration);
- loginConfig.set(DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,
+ loginConfig.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
// using HDFS principal...
this.remoteUser = UserGroupInformation
.loginUserFromKeytabAndReturnUGI(
SecurityUtil.getServerPrincipal(
- loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
+ loginConfig.get(SliderXmlConfKeys.DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY),
InetAddress.getLocalHost().getCanonicalHostName()),
- loginConfig.get(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
+ loginConfig.get(SliderXmlConfKeys.DFS_NAMENODE_KEYTAB_FILE_KEY));
log.info("Created remote user {}. UGI reports current user is {}",
this.remoteUser, UserGroupInformation.getCurrentUser());
}
@@ -84,8 +85,8 @@
if (SliderUtils.isHadoopClusterSecure(configuration) &&
renewingAction == null) {
renewInterval = configuration.getLong(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+ SliderXmlConfKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ SliderXmlConfKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
// constructor of action will retrieve initial token. One may already be
// associated with user, but its lifecycle/management is not clear so let's
// create and manage a token explicitly
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
index a6d8538..72e7565 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestCommonArgParsing.groovy
@@ -23,7 +23,6 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.slider.api.ResourceKeys
import org.apache.slider.api.RoleKeys
@@ -132,7 +131,7 @@
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
SliderUtils.verifyPrincipalSet(
conf,
- DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
+ SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
}
@@ -150,7 +149,7 @@
assert ca.clusterName == CLUSTERNAME
assert conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH) == "/projects/slider/clusters"
SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL);
- SliderUtils.verifyPrincipalSet(conf, DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY);
+ SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY);
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
index bfb00b0..4237f1e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
@@ -21,6 +21,7 @@
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.ContainerState
import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.Resource
class MockContainerStatus extends ContainerStatus {
@@ -28,4 +29,6 @@
ContainerState state
String diagnostics
int exitStatus
+
+ Resource capability;
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy b/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
index 3de3c67..d82a79c 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/services/security/TestFsDelegationTokenManager.groovy
@@ -20,9 +20,9 @@
import groovy.util.logging.Slf4j
import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.RawLocalFileSystem
-import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.Credentials
@@ -32,6 +32,7 @@
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager
import org.apache.hadoop.service.ServiceOperations
import org.apache.hadoop.util.Time
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.tools.CoreFileSystem
import org.apache.slider.server.appmaster.actions.ActionStopQueue
import org.apache.slider.server.appmaster.actions.QueueExecutor
@@ -60,10 +61,10 @@
conf = new Configuration()
conf.set(
- DFSConfigKeys.HADOOP_SECURITY_AUTHENTICATION,
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"TOKEN")
conf.setLong(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ SliderXmlConfKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
1000)
queues.init(conf)
queues.start();
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 9fb75df..faaf571 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.FileUtil
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.DFSConfigKeys
import org.apache.hadoop.hdfs.MiniDFSCluster
import org.apache.hadoop.service.ServiceOperations
import org.apache.hadoop.util.Shell
@@ -303,7 +302,7 @@
conf.setFloat(
YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
99.0f)
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_DU_RESERVED_KEY, 2 * 1024 * 1024)
+ conf.setInt(SliderXmlConfKeys.DFS_NAMENODE_DU_RESERVED_KEY, 2 * 1024 * 1024)
}
/**
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index cfbc401..2d0ac58 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -25,7 +25,7 @@
<parent>
<groupId>org.apache.slider</groupId>
<artifactId>slider</artifactId>
- <version>0.81.0-incubating-SNAPSHOT</version>
+ <version>0.90.0-incubating-SNAPSHOT</version>
</parent>
<properties>
<work.dir>package-tmp</work.dir>
diff --git a/slider-funtest/src/license/THIRD-PARTY.properties b/slider-funtest/src/license/THIRD-PARTY.properties
index 2279ca6..4a2ba24 100644
--- a/slider-funtest/src/license/THIRD-PARTY.properties
+++ b/slider-funtest/src/license/THIRD-PARTY.properties
@@ -8,6 +8,7 @@
# - Common Public License Version 1.0
# - Eclipse Public License - Version 1.0
# - GNU Lesser General Public License (LGPL), Version 2.1
+# - GNU Lesser General Public License, Version 2.1
# - GPL2 w/ CPE
# - MIT License
# - New BSD License
@@ -20,8 +21,7 @@
# Please fill the missing licenses for dependencies :
#
#
-#Mon Jan 26 15:56:57 GMT 2015
-asm--asm--3.1=The BSD 3-Clause License
+#Thu Oct 15 16:45:03 EDT 2015
commons-beanutils--commons-beanutils--1.7.0=The Apache Software License, Version 2.0
javax.servlet--servlet-api--2.5=CDDL License
javax.servlet.jsp--jsp-api--2.1=CDDL License
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 4c02289..900b138 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -23,7 +23,6 @@
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.hdfs.HdfsConfiguration
import org.apache.hadoop.registry.client.api.RegistryConstants
import org.apache.hadoop.util.ExitUtil
import org.apache.hadoop.util.Shell
@@ -106,7 +105,6 @@
(which it may) the class will not be instantiable.
*/
static {
- new HdfsConfiguration()
ConfigHelper.injectSliderXMLResource()
ConfigHelper.registerDeprecatedConfigItems();
SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true);
diff --git a/src/test/clusters/remote/slider/slider-client.xml b/src/test/clusters/remote/slider/slider-client.xml
index 41dc32e..d77ccab 100644
--- a/src/test/clusters/remote/slider/slider-client.xml
+++ b/src/test/clusters/remote/slider/slider-client.xml
@@ -23,6 +23,11 @@
<configuration>
<property>
+ <name>HADOOP_CONF_DIR</name>
+ <value>/Users/jmaron/dev/workspaces/slider-asf/incubator-slider/src/test/clusters/remote/conf</value>
+ </property>
+<!--
+ <property>
<name>yarn.application.classpath</name>
<value>
/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
@@ -76,5 +81,5 @@
<name>slider.test.agent.tar</name>
<value>hdfs://c6401.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value>
</property>
-
+-->
</configuration>
