Merging r1565757 through r1565853 from trunk
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-5698@1565857 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 09ec986..47f931f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -348,6 +348,11 @@
HDFS-5807. TestBalancerWithNodeGroup.testBalancerWithNodeGroup fails
intermittently. (Chen He via kihwal)
+ HDFS-5882. TestAuditLogs is flaky (jxiang via cmccabe)
+
+ HDFS-5900. Cannot set cache pool limit of "unlimited" via CacheAdmin.
+ (wang)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
index 290e600..b674d09 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CacheAdmin.java
@@ -140,6 +140,18 @@
return maxTtl;
}
+ private static Long parseLimitString(String limitString) {
+ Long limit = null;
+ if (limitString != null) {
+ if (limitString.equalsIgnoreCase("unlimited")) {
+ limit = CachePoolInfo.LIMIT_UNLIMITED;
+ } else {
+ limit = Long.parseLong(limitString);
+ }
+ }
+ return limit;
+ }
+
private static Expiration parseExpirationString(String ttlString)
throws IOException {
Expiration ex = null;
@@ -650,8 +662,8 @@
info.setMode(new FsPermission(mode));
}
String limitString = StringUtils.popOptionWithArgument("-limit", args);
- if (limitString != null) {
- long limit = Long.parseLong(limitString);
+ Long limit = parseLimitString(limitString);
+ if (limit != null) {
info.setLimit(limit);
}
String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args);
@@ -726,8 +738,7 @@
Integer mode = (modeString == null) ?
null : Integer.parseInt(modeString, 8);
String limitString = StringUtils.popOptionWithArgument("-limit", args);
- Long limit = (limitString == null) ?
- null : Long.parseLong(limitString);
+ Long limit = parseLimitString(limitString);
String maxTtlString = StringUtils.popOptionWithArgument("-maxTtl", args);
Long maxTtl = null;
try {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
index 0757355..2591aee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAuditLogs.java
@@ -28,6 +28,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Enumeration;
import java.util.List;
import java.util.regex.Pattern;
@@ -301,11 +302,18 @@
// Turn off the logs
Logger logger = ((Log4JLogger) FSNamesystem.auditLog).getLogger();
logger.setLevel(Level.OFF);
-
+
+ // Close the appenders and force all logs to be flushed
+ Enumeration<?> appenders = logger.getAllAppenders();
+ while (appenders.hasMoreElements()) {
+ Appender appender = (Appender)appenders.nextElement();
+ appender.close();
+ }
+
BufferedReader reader = new BufferedReader(new FileReader(auditLogFile));
String line = null;
boolean ret = true;
-
+
try {
for (int i = 0; i < ndupe; i++) {
line = reader.readLine();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
index 64de6bf..13f1b9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCacheAdminConf.xml
@@ -469,6 +469,8 @@
</test-commands>
<cleanup-commands>
<cache-admin-command>-removePool pool1</cache-admin-command>
+ <cache-admin-command>-removePool pool2</cache-admin-command>
+ <cache-admin-command>-removePool pool3</cache-admin-command>
</cleanup-commands>
<comparators>
<comparator>
@@ -489,5 +491,33 @@
</comparator>
</comparators>
</test>
+
+ <test> <!--Tested -->
+ <description>Testing setting pool unlimited limits</description>
+ <test-commands>
+ <cache-admin-command>-addPool pool1 -limit unlimited -owner andrew -group andrew</cache-admin-command>
+ <cache-admin-command>-addPool pool2 -limit 10 -owner andrew -group andrew</cache-admin-command>
+ <cache-admin-command>-modifyPool pool2 -limit unlimited</cache-admin-command>
+ <cache-admin-command>-listPools</cache-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <cache-admin-command>-removePool pool1</cache-admin-command>
+ <cache-admin-command>-removePool pool2</cache-admin-command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Found 2 results</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>pool1 andrew andrew rwxr-xr-x unlimited never</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>pool2 andrew andrew rwxr-xr-x unlimited never</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8ac0af9..daa2f68 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -106,6 +106,16 @@
new APIs for retrieving and storing timeline information. (Zhijie Shen via
vinodkv)
+ YARN-1490. Introduced the ability to make ResourceManager optionally not kill
+ all containers when an ApplicationMaster exits. (Jian He via vinodkv)
+
+ YARN-1041. Added the ApplicationMasterProtocol API for applications to use the
+ ability in ResourceManager to optionally not kill containers when the
+ ApplicationMaster exits. (Jian He via vinodkv)
+
+ YARN-1566. Changed Distributed Shell to retain containers across application
+ attempts. (Jian He via vinodkv)
+
IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
@@ -150,6 +160,9 @@
yarn.resourcemanager.hostname.RMID and use the default ports for all service
addresses. (Xuan Gong via vinodkv)
+ YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
+ app-attempts separately from apps. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -205,6 +218,15 @@
YARN-1684. Fixed history server heap size in yarn script. (Billie Rinaldi
via zjshen)
+ YARN-1166. Fixed app-specific and attempt-specific QueueMetrics to be
+ triggered by accordingly app event and attempt event.
+
+ YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod
+ Kumar Vavilapalli via zjshen)
+
+ YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod
+ Kumar Vavilapalli via zjshen)
+
Release 2.3.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -248,18 +270,8 @@
YARN-1029. Added embedded leader election in the ResourceManager. (Karthik
Kambatla via vinodkv)
- YARN-1490. Introduced the ability to make ResourceManager optionally not kill
- all containers when an ApplicationMaster exits. (Jian He via vinodkv)
-
YARN-1033. Expose RM active/standby state to Web UI and REST API (kasha)
- YARN-1041. Added the ApplicationMasterProtocol API for applications to use the
- ability in ResourceManager to optionally not kill containers when the
- ApplicationMaster exits. (Jian He via vinodkv)
-
- YARN-1566. Changed Distributed Shell to retain containers across application
- attempts. (Jian He via vinodkv)
-
IMPROVEMENTS
YARN-305. Fair scheduler logs too many "Node offered to app" messages.
@@ -423,9 +435,6 @@
YARN-1541. Changed ResourceManager to invalidate ApplicationMaster host/port
information once an AM crashes. (Jian He via vinodkv)
- YARN-1493. Changed ResourceManager and Scheduler interfacing to recognize
- app-attempts separately from apps. (Jian He via vinodkv)
-
YARN-1482. Modified WebApplicationProxy to make it work across ResourceManager
fail-over. (Xuan Gong via vinodkv)
@@ -611,9 +620,6 @@
YARN-1574. RMDispatcher should be reset on transition to standby. (Xuan Gong
via kasha)
- YARN-1166. Fixed app-specific and attempt-specific QueueMetrics to be
- triggered by accordingly app event and attempt event.
-
YARN-1598. HA-related rmadmin commands don't work on a secure cluster (kasha)
YARN-1603. Remove two *.orig files which were unexpectedly committed.
@@ -641,12 +647,6 @@
YARN-1628. Fixed the test failure in TestContainerManagerSecurity. (Vinod
Kumar Vavilapalli via zjshen)
- YARN-1661. Fixed DS ApplicationMaster to write the correct exit log. (Vinod
- Kumar Vavilapalli via zjshen)
-
- YARN-1689. Made RMAppAttempt get killed when RMApp is at ACCEPTED. (Vinod
- Kumar Vavilapalli via zjshen)
-
Release 2.2.0 - 2013-10-13
INCOMPATIBLE CHANGES
