LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws a confusing OutOfMemoryError wrapped inside IOException if the FileChannel mapping failed because of lack of virtual address space. The IOException is rethrown with more useful information about the problem, omitting the incorrect OutOfMemoryError
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1595213 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6918b7e..702ddbe 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -188,6 +188,12 @@
* LUCENE-5668: Fix off-by-one in TieredMergePolicy (Mike McCandless)
+* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws
+ a confusing OutOfMemoryError wrapped inside IOException if the FileChannel
+ mapping failed because of lack of virtual address space. The IOException is
+ rethrown with more useful information about the problem, omitting the
+ incorrect OutOfMemoryError. (Robert Muir, Uwe Schindler)
+
Test Framework
* LUCENE-5622: Fail tests if they print over the given limit of bytes to
diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
index 42da940..f3d4c53 100644
--- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
@@ -28,6 +28,7 @@
import java.security.AccessController;
import java.security.PrivilegedExceptionAction;
import java.security.PrivilegedActionException;
+import java.util.Locale;
import java.lang.reflect.Method;
import org.apache.lucene.util.Constants;
@@ -75,6 +76,7 @@
* blocked on IO. The channel will remain closed and subsequent access
* to {@link MMapDirectory} will throw a {@link ClosedChannelException}.
* </p>
+ * @see <a href="http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html">Blog post about MMapDirectory</a>
*/
public class MMapDirectory extends FSDirectory {
private boolean useUnmapHack = UNMAP_SUPPORTED;
@@ -216,7 +218,7 @@
private final boolean useUnmapHack;
MMapIndexInput(String resourceDescription, FileChannel fc) throws IOException {
- super(resourceDescription, map(fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
+ super(resourceDescription, map(resourceDescription, fc, 0, fc.size()), fc.size(), chunkSizePower, getUseUnmap());
this.useUnmapHack = getUseUnmap();
}
@@ -244,18 +246,16 @@
}
});
} catch (PrivilegedActionException e) {
- final IOException ioe = new IOException("unable to unmap the mapped buffer");
- ioe.initCause(e.getCause());
- throw ioe;
+ throw new IOException("Unable to unmap the mapped buffer: " + toString(), e.getCause());
}
}
}
}
/** Maps a file into a set of buffers */
- ByteBuffer[] map(FileChannel fc, long offset, long length) throws IOException {
+ final ByteBuffer[] map(String resourceDescription, FileChannel fc, long offset, long length) throws IOException {
if ((length >>> chunkSizePower) >= Integer.MAX_VALUE)
- throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + fc.toString());
+ throw new IllegalArgumentException("RandomAccessFile too big for chunk size: " + resourceDescription);
final long chunkSize = 1L << chunkSizePower;
@@ -270,10 +270,45 @@
? chunkSize
: (length - bufferStart)
);
- buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
+ try {
+ buffers[bufNr] = fc.map(MapMode.READ_ONLY, offset + bufferStart, bufSize);
+ } catch (IOException ioe) {
+ throw convertMapFailedIOException(ioe, resourceDescription, bufSize);
+ }
bufferStart += bufSize;
}
return buffers;
}
+
+ private IOException convertMapFailedIOException(IOException ioe, String resourceDescription, int bufSize) {
+ final String originalMessage;
+ final Throwable originalCause;
+ if (ioe.getCause() instanceof OutOfMemoryError) {
+ // nested OOM confuses users, because its "incorrect", just print a plain message:
+ originalMessage = "Map failed";
+ originalCause = null;
+ } else {
+ originalMessage = ioe.getMessage();
+ originalCause = ioe.getCause();
+ }
+ final String moreInfo;
+ if (!Constants.JRE_IS_64BIT) {
+ moreInfo = "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. ";
+ } else if (Constants.WINDOWS) {
+ moreInfo = "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. ";
+ } else if (Constants.LINUX) {
+ moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. ";
+ } else {
+ moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). ";
+ }
+ final IOException newIoe = new IOException(String.format(Locale.ENGLISH,
+ "%s: %s [this may be caused by lack of enough unfragmented virtual address space "+
+ "or too restrictive virtual memory limits enforced by the operating system, "+
+ "preventing us to map a chunk of %d bytes. %sMore information: "+
+ "http://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]",
+ originalMessage, resourceDescription, bufSize, moreInfo), originalCause);
+ newIoe.setStackTrace(ioe.getStackTrace());
+ return newIoe;
+ }
}
