Hadoop 1.1.0 release candidate 1.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/tags/release-1.1.0-rc1@1365474 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES.txt b/CHANGES.txt
index 0c3442d..22e9dd8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -137,6 +137,11 @@
HDFS-2465. Add HDFS support for fadvise readahead and drop-behind.
(todd via suresh)
+ HADOOP-7154. Should set MALLOC_ARENA_MAX in hadoop-env.sh
+ (todd via mattf)
+
+ MAPREDUCE-4399. Change the Jetty response buffer size to improve
+ shuffle performance. (Luke Lu via suresh)
BUG FIXES
@@ -291,10 +296,10 @@
HDFS-3551. WebHDFS CREATE should use client location for HTTP redirection.
(szetszwo)
- HDFS-6527. Backport HADOOP-7389: Use of TestingGroups by tests causes
+ HADOOP-6527. Backport HADOOP-7389: Use of TestingGroups by tests causes
subsequent tests to fail. (Ivan Mitic via szetszwo)
- HDFS-8417. HADOOP-6963 didn't update hadoop-core-pom-template.xml
+ HADOOP-8417. HADOOP-6963 didn't update hadoop-core-pom-template.xml
(Zhihong Ted Yu via cos)
HADOOP-8399. Remove JDK5 dependency from Hadoop 1.0+ line
diff --git a/bin/hadoop-config.sh b/bin/hadoop-config.sh
index 9f93962..c23cbcf 100644
--- a/bin/hadoop-config.sh
+++ b/bin/hadoop-config.sh
@@ -75,6 +75,11 @@
echo 1>&2
fi
+# Newer versions of glibc use an arena memory allocator that causes virtual
+# memory usage to explode. This interacts badly with the many threads that
+# we use in Hadoop. Tune the variable down to prevent vmem explosion.
+export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4}
+
export HADOOP_HOME=${HADOOP_PREFIX}
export HADOOP_HOME_WARN_SUPPRESS=1
diff --git a/src/docs/releasenotes.html b/src/docs/releasenotes.html
index 6500675..5779f82 100644
--- a/src/docs/releasenotes.html
+++ b/src/docs/releasenotes.html
Binary files differ
diff --git a/src/mapred/org/apache/hadoop/mapred/TaskTracker.java b/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
index b06dae4..135f43c 100644
--- a/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
+++ b/src/mapred/org/apache/hadoop/mapred/TaskTracker.java
@@ -3867,7 +3867,8 @@
public static class MapOutputServlet extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final int MAX_BYTES_TO_READ = 64 * 1024;
-
+ // work around jetty internal buffering issues
+ private static final int RESPONSE_BUFFER_SIZE = MAX_BYTES_TO_READ + 16;
private static LRUCache<String, Path> fileCache = new LRUCache<String, Path>(FILE_CACHE_SIZE);
private static LRUCache<String, Path> fileIndexCache = new LRUCache<String, Path>(FILE_CACHE_SIZE);
@@ -3969,10 +3970,9 @@
//set the custom "for-reduce-task" http header to the reduce task number
//for which this map output is being transferred
response.setHeader(FOR_REDUCE_TASK, Integer.toString(reduce));
-
- //use the same buffersize as used for reading the data from disk
- response.setBufferSize(MAX_BYTES_TO_READ);
-
+
+ response.setBufferSize(RESPONSE_BUFFER_SIZE);
+
/**
* Read the data from the sigle map-output file and
* send it to the reducer.
