merge up to trunk
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4456@1393245 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
index 96aac28..b930b84 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesConsumer.java
@@ -198,6 +198,7 @@
IOUtils.close(output);
} else {
IOUtils.closeWhileHandlingException(output);
+ dir.deleteFile(fileName);
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
index 9c25290..772b9c0 100644
--- a/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
+++ b/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
@@ -137,6 +137,7 @@
* currently buffered docs. This resets our state,
* discarding any docs added since last flush. */
void abort() {
+ //System.out.println(Thread.currentThread().getName() + ": now abort seg=" + segmentInfo.name);
hasAborted = aborting = true;
try {
if (infoStream.isEnabled("DWPT")) {
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
index 175a295..73f9f36 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -333,12 +333,16 @@
segmentPrefix2 = null;
}
+ Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+
for(int i=0;i<files.length;i++) {
String fileName = files[i];
+ m.reset(fileName);
if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
!fileName.endsWith("write.lock") &&
!refCounts.containsKey(fileName) &&
- !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+ !fileName.equals(IndexFileNames.SEGMENTS_GEN) &&
+ (m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
// Unreferenced file, so remove it
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
@@ -554,7 +558,13 @@
void deleteNewFiles(Collection<String> files) throws IOException {
assert locked();
for (final String fileName: files) {
- if (!refCounts.containsKey(fileName)) {
+ // NOTE: it's very unusual yet possible for the
+ // refCount to be present and 0: it can happen if you
+ // open IW on a crashed index, and it removes a bunch
+ // of unref'd files, and then you add new docs / do
+ // merging, and it reuses that segment name.
+ // TestCrash.testCrashAfterReopen can hit this:
+ if (!refCounts.containsKey(fileName) || refCounts.get(fileName).count == 0) {
if (infoStream.isEnabled("IFD")) {
infoStream.message("IFD", "delete new file \"" + fileName + "\"");
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
index e4d74a3..dc0f66b 100644
--- a/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
@@ -2145,7 +2145,7 @@
// Now build compound file
Collection<String> oldFiles = createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, newSegment.info, context);
newSegment.info.setUseCompoundFile(true);
-
+
synchronized(this) {
deleter.deleteNewFiles(oldFiles);
}
@@ -2321,31 +2321,62 @@
flush(false, true);
List<SegmentInfoPerCommit> infos = new ArrayList<SegmentInfoPerCommit>();
- for (Directory dir : dirs) {
- if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "addIndexes: process directory " + dir);
- }
- SegmentInfos sis = new SegmentInfos(); // read infos from dir
- sis.read(dir);
- for (SegmentInfoPerCommit info : sis) {
- assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
-
- String newSegName = newSegmentName();
- String dsName = info.info.name;
-
+ boolean success = false;
+ try {
+ for (Directory dir : dirs) {
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
+ infoStream.message("IW", "addIndexes: process directory " + dir);
}
+ SegmentInfos sis = new SegmentInfos(); // read infos from dir
+ sis.read(dir);
- IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
+ for (SegmentInfoPerCommit info : sis) {
+ assert !infos.contains(info): "dup info dir=" + info.info.dir + " name=" + info.info.name;
+
+ String newSegName = newSegmentName();
+ String dsName = info.info.name;
+
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "addIndexes: process segment origName=" + info.info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
+ }
+
+ IOContext context = new IOContext(new MergeInfo(info.info.getDocCount(), info.info.sizeInBytes(), true, -1));
- infos.add(copySegmentAsIs(info, newSegName, context));
+ infos.add(copySegmentAsIs(info, newSegName, context));
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ for(SegmentInfoPerCommit sipc : infos) {
+ for(String file : sipc.files()) {
+ try {
+ directory.deleteFile(file);
+ } catch (Throwable t) {
+ }
+ }
+ }
}
}
synchronized (this) {
- ensureOpen();
+ success = false;
+ try {
+ ensureOpen();
+ success = true;
+ } finally {
+ if (!success) {
+ for(SegmentInfoPerCommit sipc : infos) {
+ for(String file : sipc.files()) {
+ try {
+ directory.deleteFile(file);
+ } catch (Throwable t) {
+ }
+ }
+ }
+ }
+ }
segmentInfos.addAll(infos);
checkpoint();
}
@@ -2420,7 +2451,18 @@
merger.add(reader);
}
- MergeState mergeState = merger.merge(); // merge 'em
+ MergeState mergeState;
+ boolean success = false;
+ try {
+ mergeState = merger.merge(); // merge 'em
+ success = true;
+ } finally {
+ if (!success) {
+ synchronized(this) {
+ deleter.refresh(info.name);
+ }
+ }
+ }
SegmentInfoPerCommit infoPerCommit = new SegmentInfoPerCommit(info, 0, -1L);
@@ -2442,12 +2484,14 @@
// Now create the compound file if needed
if (useCompoundFile) {
Collection<String> filesToDelete = infoPerCommit.files();
- createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context);
-
- // delete new non cfs files directly: they were never
- // registered with IFD
- synchronized(this) {
- deleter.deleteNewFiles(filesToDelete);
+ try{
+ createCompoundFile(infoStream, directory, MergeState.CheckAbort.NONE, info, context);
+ } finally {
+ // delete new non cfs files directly: they were never
+ // registered with IFD
+ synchronized(this) {
+ deleter.deleteNewFiles(filesToDelete);
+ }
}
info.setUseCompoundFile(true);
}
@@ -2456,7 +2500,18 @@
// creating CFS so that 1) .si isn't slurped into CFS,
// and 2) .si reflects useCompoundFile=true change
// above:
- codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context);
+ success = false;
+ try {
+ codec.segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, info, mergeState.fieldInfos, context);
+ success = true;
+ } finally {
+ if (!success) {
+ synchronized(this) {
+ deleter.refresh(info.name);
+ }
+ }
+ }
+
info.addFiles(trackingDir.getCreatedFiles());
// Register the new segment
@@ -2512,23 +2567,38 @@
// We must rewrite the SI file because it references segment name in its list of files, etc
TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(directory);
- newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
+ boolean success = false;
- final Collection<String> siFiles = trackingDir.getCreatedFiles();
+ try {
- // Copy the segment's files
- for (String file: info.files()) {
+ newInfo.getCodec().segmentInfoFormat().getSegmentInfoWriter().write(trackingDir, newInfo, fis, context);
- final String newFileName = segName + IndexFileNames.stripSegmentName(file);
+ final Collection<String> siFiles = trackingDir.getCreatedFiles();
- if (siFiles.contains(newFileName)) {
- // We already rewrote this above
- continue;
+ // Copy the segment's files
+ for (String file: info.files()) {
+
+ final String newFileName = segName + IndexFileNames.stripSegmentName(file);
+
+ if (siFiles.contains(newFileName)) {
+ // We already rewrote this above
+ continue;
+ }
+
+ assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
+
+ info.info.dir.copy(directory, file, newFileName, context);
}
-
- assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists; siFiles=" + siFiles;
-
- info.info.dir.copy(directory, file, newFileName, context);
+ success = true;
+ } finally {
+ if (!success) {
+ for(String file : newInfo.files()) {
+ try {
+ directory.deleteFile(file);
+ } catch (Throwable t) {
+ }
+ }
+ }
}
return newInfoPerCommit;
@@ -3091,6 +3161,7 @@
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commitMerge: skip: it was aborted");
}
+ deleter.deleteNewFiles(merge.info.files());
return false;
}
@@ -3136,6 +3207,11 @@
}
}
+ if (dropSegment) {
+ assert !segmentInfos.contains(merge.info);
+ deleter.deleteNewFiles(merge.info.files());
+ }
+
// Must close before checkpoint, otherwise IFD won't be
// able to delete the held-open files from the merge
// readers:
@@ -3407,7 +3483,7 @@
setDiagnostics(si, "merge", details);
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "merge seg=" + merge.info.info.name);
+ infoStream.message("IW", "merge seg=" + merge.info.info.name + " " + segString(merge.segments));
}
assert merge.estimatedMergeBytes == 0;
@@ -4116,7 +4192,22 @@
} catch(IOException ex) {
prior = ex;
} finally {
- IOUtils.closeWhileHandlingException(prior, cfsDir);
+ boolean success = false;
+ try {
+ IOUtils.closeWhileHandlingException(prior, cfsDir);
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ directory.deleteFile(fileName);
+ } catch (Throwable t) {
+ }
+ try {
+ directory.deleteFile(IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ } catch (Throwable t) {
+ }
+ }
+ }
}
// Replace all previous files with the CFS/CFE files:
diff --git a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
index 9f79c6d..7652fa2 100755
--- a/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
+++ b/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
@@ -547,7 +547,7 @@
sb.append("ramBufferSizeMB=").append(getRAMBufferSizeMB()).append("\n");
sb.append("maxBufferedDocs=").append(getMaxBufferedDocs()).append("\n");
sb.append("maxBufferedDeleteTerms=").append(getMaxBufferedDeleteTerms()).append("\n");
- sb.append("mergedSegmentWarmer=").append(getMergeScheduler()).append("\n");
+ sb.append("mergedSegmentWarmer=").append(getMergedSegmentWarmer()).append("\n");
sb.append("readerTermsIndexDivisor=").append(getReaderTermsIndexDivisor()).append("\n");
sb.append("termIndexInterval=").append(getTermIndexInterval()).append("\n"); // TODO: this should be private to the codec, not settable here
sb.append("delPolicy=").append(getIndexDeletionPolicy().getClass().getName()).append("\n");
diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java b/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
index 6f0bb3a..5459dc1 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java
@@ -79,6 +79,7 @@
/** Returns all files in use by this segment. */
public Collection<String> files() throws IOException {
+ // Start from the wrapped info's files:
Collection<String> files = new HashSet<String>(info.files());
// Must separately add any live docs files:
diff --git a/lucene/core/src/java/org/apache/lucene/store/Directory.java b/lucene/core/src/java/org/apache/lucene/store/Directory.java
index 8808eff..4be172e 100644
--- a/lucene/core/src/java/org/apache/lucene/store/Directory.java
+++ b/lucene/core/src/java/org/apache/lucene/store/Directory.java
@@ -201,7 +201,18 @@
} catch (IOException ioe) {
priorException = ioe;
} finally {
- IOUtils.closeWhileHandlingException(priorException, os, is);
+ boolean success = false;
+ try {
+ IOUtils.closeWhileHandlingException(priorException, os, is);
+ success = true;
+ } finally {
+ if (!success) {
+ try {
+ to.deleteFile(dest);
+ } catch (Throwable t) {
+ }
+ }
+ }
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java
index 01ae3c2..82c4352 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat.java
@@ -33,7 +33,7 @@
@Override
public void setUp() throws Exception {
super.setUp();
- codec = new RandomCodec(new Random(random().nextLong()), Collections.EMPTY_SET);
+ codec = new RandomCodec(new Random(random().nextLong()), Collections.<String>emptySet());
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCrash.java b/lucene/core/src/test/org/apache/lucene/index/TestCrash.java
index 64ad986..4f71335 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestCrash.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestCrash.java
@@ -20,12 +20,13 @@
import java.io.IOException;
import java.util.Random;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.NoLockFactory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
+import org.apache.lucene.store.NoLockFactory;
+import org.apache.lucene.util.LuceneTestCase;
public class TestCrash extends LuceneTestCase {
@@ -67,20 +68,42 @@
// before any documents were added.
IndexWriter writer = initIndex(random(), true);
MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
+
+ // We create leftover files because merging could be
+ // running when we crash:
+ dir.setAssertNoUnrefencedFilesOnClose(false);
+
crash(writer);
+
IndexReader reader = DirectoryReader.open(dir);
assertTrue(reader.numDocs() < 157);
reader.close();
+
+ // Make a new dir, copying from the crashed dir, and
+ // open IW on it, to confirm IW "recovers" after a
+ // crash:
+ Directory dir2 = newDirectory(dir);
dir.close();
+
+ new RandomIndexWriter(random(), dir2).close();
+ dir2.close();
}
public void testWriterAfterCrash() throws IOException {
// This test relies on being able to open a reader before any commit
// happened, so we must create an initial commit just to allow that, but
// before any documents were added.
+ System.out.println("TEST: initIndex");
IndexWriter writer = initIndex(random(), true);
+ System.out.println("TEST: done initIndex");
MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
+
+ // We create leftover files because merging could be
+ // running / store files could be open when we crash:
+ dir.setAssertNoUnrefencedFilesOnClose(false);
+
dir.setPreventDoubleWrite(false);
+ System.out.println("TEST: now crash");
crash(writer);
writer = initIndex(random(), dir, false);
writer.close();
@@ -88,12 +111,25 @@
IndexReader reader = DirectoryReader.open(dir);
assertTrue(reader.numDocs() < 314);
reader.close();
+
+ // Make a new dir, copying from the crashed dir, and
+ // open IW on it, to confirm IW "recovers" after a
+ // crash:
+ Directory dir2 = newDirectory(dir);
dir.close();
+
+ new RandomIndexWriter(random(), dir2).close();
+ dir2.close();
}
public void testCrashAfterReopen() throws IOException {
IndexWriter writer = initIndex(random(), false);
MockDirectoryWrapper dir = (MockDirectoryWrapper) writer.getDirectory();
+
+ // We create leftover files because merging could be
+ // running when we crash:
+ dir.setAssertNoUnrefencedFilesOnClose(false);
+
writer.close();
writer = initIndex(random(), dir, false);
assertEquals(314, writer.maxDoc());
@@ -111,7 +147,15 @@
IndexReader reader = DirectoryReader.open(dir);
assertTrue(reader.numDocs() >= 157);
reader.close();
+
+ // Make a new dir, copying from the crashed dir, and
+ // open IW on it, to confirm IW "recovers" after a
+ // crash:
+ Directory dir2 = newDirectory(dir);
dir.close();
+
+ new RandomIndexWriter(random(), dir2).close();
+ dir2.close();
}
public void testCrashAfterClose() throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
index f1dd366..3bc247d 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
@@ -39,6 +39,7 @@
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
index 6db930a..93aabba 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReaderReopen.java
@@ -38,6 +38,7 @@
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
index 772c0fb..db78ffd 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java
@@ -37,6 +37,7 @@
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.TrackingDirectoryWrapper;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.InfoStream;
@@ -112,6 +113,13 @@
PrintWriter out = new PrintWriter(sw, true);
Directory directory = newFSDirectory(indexDir, null);
+
+ if (directory instanceof MockDirectoryWrapper) {
+ // We create unreferenced files (we don't even write
+ // a segments file):
+ ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+ }
+
IndexWriter writer = new IndexWriter(
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
@@ -147,6 +155,13 @@
out = new PrintWriter(sw, true);
directory = newFSDirectory(indexDir, null);
+
+ if (directory instanceof MockDirectoryWrapper) {
+ // We create unreferenced files (we don't even write
+ // a segments file):
+ ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(false);
+ }
+
writer = new IndexWriter(
directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
index cfee991..4291f17 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
@@ -1070,6 +1070,9 @@
fail("segmentInfos failed to retry fallback to correct segments_N file");
}
reader.close();
+
+ // should remove the corrumpted segments_N
+ new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null)).close();
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
index 356fadc..770b69c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
@@ -284,6 +284,9 @@
failure.clearDoFail();
writer.close(false);
}
+ if (VERBOSE) {
+ System.out.println("TEST: success=" + success);
+ }
if (success) {
IndexReader reader = DirectoryReader.open(dir);
@@ -341,6 +344,12 @@
}
@Override
public void eval(MockDirectoryWrapper dir) throws IOException {
+
+ // Since we throw exc during abort, eg when IW is
+ // attempting to delete files, we will leave
+ // leftovers:
+ dir.setAssertNoUnrefencedFilesOnClose(false);
+
if (doFail) {
StackTraceElement[] trace = new Exception().getStackTrace();
boolean sawAbortOrFlushDoc = false;
@@ -405,6 +414,8 @@
if ("flush".equals(trace[i].getMethodName()) && "org.apache.lucene.index.DocFieldProcessor".equals(trace[i].getClassName())) {
if (onlyOnce)
doFail = false;
+ //System.out.println(Thread.currentThread().getName() + ": NOW FAIL: onlyOnce=" + onlyOnce);
+ //new Throwable().printStackTrace(System.out);
throw new IOException("now failing on purpose");
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java
index 3e032bb..66a02a1 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestNoDeletionPolicy.java
@@ -21,10 +21,12 @@
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.Arrays;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
index 27e99b2..5c10657 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestPersistentSnapshotDeletionPolicy.java
@@ -25,6 +25,7 @@
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
index 09157df..84aab03 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestRollingUpdates.java
@@ -39,7 +39,6 @@
public void testRollingUpdates() throws Exception {
Random random = new Random(random().nextLong());
final BaseDirectoryWrapper dir = newDirectory();
- dir.setCheckIndexOnClose(false); // we use a custom codec provider
final LineFileDocs docs = new LineFileDocs(random, true);
//provider.register(new MemoryCodec());
@@ -130,10 +129,11 @@
assertEquals(SIZE, w.numDocs());
w.close();
+
+ TestIndexWriter.assertNoUnreferencedFiles(dir, "leftover files after rolling updates");
+
docs.close();
- _TestUtil.checkIndex(dir);
-
// LUCENE-4455:
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java b/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
index bebf6c5..e31b71c 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestSnapshotDeletionPolicy.java
@@ -17,17 +17,18 @@
* limitations under the License.
*/
+import java.io.IOException;
import java.util.Collection;
import java.util.Map;
import java.util.Random;
-import java.io.IOException;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ThreadInterruptedException;
import org.junit.Test;
@@ -313,6 +314,7 @@
public void testRollbackToOldSnapshot() throws Exception {
int numSnapshots = 2;
Directory dir = newDirectory();
+
SnapshotDeletionPolicy sdp = getDeletionPolicy();
IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
@@ -325,10 +327,11 @@
writer.deleteUnusedFiles();
assertSnapshotExists(dir, sdp, numSnapshots - 1);
writer.close();
-
+
// but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
+
dir.close();
}
@@ -385,6 +388,7 @@
@Test
public void testSnapshotLastCommitTwice() throws Exception {
Directory dir = newDirectory();
+
SnapshotDeletionPolicy sdp = getDeletionPolicy();
IndexWriter writer = new IndexWriter(dir, getConfig(random(), sdp));
writer.addDocument(new Document());
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java b/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java
index abc155c..522abaf 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTransactionRollback.java
@@ -21,17 +21,18 @@
import java.io.IOException;
import java.util.BitSet;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.HashMap;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.LuceneTestCase;
/**
* Test class to illustrate using IndexDeletionPolicy to provide multi-level rollback capability.
@@ -55,13 +56,16 @@
for (Iterator<IndexCommit> iterator = commits.iterator(); iterator.hasNext();) {
IndexCommit commit = iterator.next();
Map<String,String> ud=commit.getUserData();
- if (ud.size() > 0)
- if (ud.get("index").endsWith(ids))
- last=commit;
+ if (ud.size() > 0) {
+ if (ud.get("index").endsWith(ids)) {
+ last = commit;
+ }
+ }
}
- if (last==null)
+ if (last==null) {
throw new RuntimeException("Couldn't find commit point "+id);
+ }
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random())).setIndexDeletionPolicy(
@@ -74,13 +78,13 @@
public void testRepeatedRollBacks() throws Exception {
- int expectedLastRecordId=100;
+ int expectedLastRecordId = 100;
while (expectedLastRecordId>10) {
- expectedLastRecordId -=10;
+ expectedLastRecordId -= 10;
rollBackLast(expectedLastRecordId);
BitSet expecteds = new BitSet(100);
- expecteds.set(1,(expectedLastRecordId+1),true);
+ expecteds.set(1, (expectedLastRecordId+1), true);
checkExpecteds(expecteds);
}
}
@@ -125,6 +129,7 @@
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
+
//Build index, of records 1 to 100, committing after each batch of 10
IndexDeletionPolicy sdp=new KeepAllDeletionPolicy();
IndexWriter w=new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setIndexDeletionPolicy(sdp));
@@ -199,6 +204,7 @@
}
public void testRollbackDeletionPolicy() throws Exception {
+
for(int i=0;i<2;i++) {
// Unless you specify a prior commit point, rollback
// should not work:
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java b/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
index b46bbf1..5635393 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestTransactions.java
@@ -215,6 +215,11 @@
dir1.setFailOnOpenInput(false);
dir2.setFailOnOpenInput(false);
+ // We throw exceptions in deleteFile, which creates
+ // leftover files:
+ dir1.setAssertNoUnrefencedFilesOnClose(false);
+ dir2.setAssertNoUnrefencedFilesOnClose(false);
+
initIndex(dir1);
initIndex(dir2);
diff --git a/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java
new file mode 100644
index 0000000..81a577c
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/util/junitcompat/TestFailIfUnreferencedFiles.java
@@ -0,0 +1,58 @@
+package org.apache.lucene.util.junitcompat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collections;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexOutput;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.JUnitCore;
+import org.junit.runner.Result;
+
+// LUCENE-4456: Test that we fail if there are unreferenced files
+public class TestFailIfUnreferencedFiles extends WithNestedTests {
+ public TestFailIfUnreferencedFiles() {
+ super(true);
+ }
+
+ public static class Nested1 extends WithNestedTests.AbstractNestedTest {
+ public void testDummy() throws Exception {
+ Directory dir = newMockDirectory();
+ IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, null));
+ iw.addDocument(new Document());
+ iw.close();
+ IndexOutput output = dir.createOutput("_hello.world", IOContext.DEFAULT);
+ output.writeString("i am unreferenced!");
+ output.close();
+ dir.sync(Collections.singleton("_hello.world"));
+ dir.close();
+ }
+ }
+
+ @Test
+ public void testFailIfUnreferencedFiles() {
+ Result r = JUnitCore.runClasses(Nested1.class);
+ Assert.assertEquals(1, r.getFailureCount());
+ }
+}
diff --git a/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java b/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
index 4ade463..1ed3bf1 100644
--- a/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
+++ b/lucene/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
@@ -22,6 +22,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -34,6 +35,11 @@
_TestUtil.rmDir(destDir);
destDir.mkdirs();
Directory fsDir = newFSDirectory(dir);
+ // IndexSplitter.split makes its own commit directly with SIPC/SegmentInfos,
+ // so the unreferenced files are expected.
+ if (fsDir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)fsDir).setAssertNoUnrefencedFilesOnClose(false);
+ }
LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setNoCFSRatio(1.0);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
index 97e4a31..cf6d3a8 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomIndexWriter.java
@@ -402,10 +402,16 @@
final int segCount = w.getSegmentCount();
if (r.nextBoolean() || segCount == 0) {
// full forceMerge
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("RIW: doRandomForceMerge(1)");
+ }
w.forceMerge(1);
} else {
// partial forceMerge
final int limit = _TestUtil.nextInt(r, 1, segCount);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("RIW: doRandomForceMerge(" + limit + ")");
+ }
w.forceMerge(limit);
assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount();
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
index c478731..25f0f0e 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java
@@ -28,14 +28,18 @@
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.NoDeletionPolicy;
+import org.apache.lucene.index.SegmentInfos;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -204,15 +208,17 @@
// first force-close all files, so we can corrupt on windows etc.
// clone the file map, as these guys want to remove themselves on close.
Map<Closeable,Exception> m = new IdentityHashMap<Closeable,Exception>(openFileHandles);
- for (Closeable f : m.keySet())
+ for (Closeable f : m.keySet()) {
try {
f.close();
} catch (Exception ignored) {}
+ }
while(it.hasNext()) {
String name = it.next();
int damage = randomState.nextInt(5);
String action = null;
+
if (damage == 0) {
action = "deleted";
deleteFile(name, true);
@@ -544,6 +550,9 @@
@Override
public synchronized void close() throws IOException {
+ // files that we tried to delete, but couldn't because readers were open.
+ // all that matters is that we tried! (they will eventually go away)
+ Set<String> pendingDeletions = new HashSet<String>(openFilesDeleted);
maybeYield();
if (openFiles == null) {
openFiles = new HashMap<String,Integer>();
@@ -574,17 +583,91 @@
}
_TestUtil.checkIndex(this, getCrossCheckTermVectorsOnClose());
+ // TODO: factor this out / share w/ TestIW.assertNoUnreferencedFiles
if (assertNoUnreferencedFilesOnClose) {
- // now look for unreferenced files:
- String[] startFiles = listAll();
- new IndexWriter(this, new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null)).rollback();
- String[] endFiles = listAll();
+ // now look for unreferenced files: discount ones that we tried to delete but could not
+ Set<String> allFiles = new HashSet<String>(Arrays.asList(listAll()));
+ allFiles.removeAll(pendingDeletions);
+ String[] startFiles = allFiles.toArray(new String[0]);
+ IndexWriterConfig iwc = new IndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, null);
+ iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
+ new IndexWriter(delegate, iwc).rollback();
+ String[] endFiles = delegate.listAll();
- Arrays.sort(startFiles);
- Arrays.sort(endFiles);
+ Set<String> startSet = new TreeSet<String>(Arrays.asList(startFiles));
+ Set<String> endSet = new TreeSet<String>(Arrays.asList(endFiles));
+
+ if (pendingDeletions.contains("segments.gen") && endSet.contains("segments.gen")) {
+ // this is possible if we hit an exception while writing segments.gen, we try to delete it
+ // and it ends out in pendingDeletions (but IFD wont remove this).
+ startSet.add("segments.gen");
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MDW: Unreferenced check: Ignoring segments.gen that we could not delete.");
+ }
+ }
+
+ // its possible we cannot delete the segments_N on windows if someone has it open and
+ // maybe other files too, depending on timing. normally someone on windows wouldnt have
+ // an issue (IFD would nuke this stuff eventually), but we pass NoDeletionPolicy...
+ for (String file : pendingDeletions) {
+ if (file.startsWith("segments") && !file.equals("segments.gen") && endSet.contains(file)) {
+ startSet.add(file);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MDW: Unreferenced check: Ignoring segments file: " + file + " that we could not delete.");
+ }
+ try {
+ SegmentInfos sis = new SegmentInfos();
+ sis.read(delegate, file);
+ Set<String> ghosts = new HashSet<String>(sis.files(delegate, false));
+ for (String s : ghosts) {
+ if (endSet.contains(s) && !startSet.contains(s)) {
+ assert pendingDeletions.contains(s);
+ if (LuceneTestCase.VERBOSE) {
+ System.out.println("MDW: Unreferenced check: Ignoring referenced file: " + s + " " +
+ "from " + file + " that we could not delete.");
+ }
+ startSet.add(s);
+ }
+ }
+ } catch (Throwable ignore) {}
+ }
+ }
+
+ startFiles = startSet.toArray(new String[0]);
+ endFiles = endSet.toArray(new String[0]);
if (!Arrays.equals(startFiles, endFiles)) {
- assert false : "unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles);
+ StringBuilder sb = new StringBuilder();
+ List<String> removed = new ArrayList<String>();
+ for(String fileName : startFiles) {
+ if (!endSet.contains(fileName)) {
+ removed.add(fileName);
+ }
+ }
+
+ List<String> added = new ArrayList<String>();
+ for(String fileName : endFiles) {
+ if (!startSet.contains(fileName)) {
+ added.add(fileName);
+ }
+ }
+
+ String extras;
+ if (removed.size() != 0) {
+ extras = "\n\nThese files were removed: " + removed;
+ } else {
+ extras = "";
+ }
+
+ if (added.size() != 0) {
+ extras += "\n\nThese files were added (waaaaaaaaaat!): " + added;
+ }
+
+ if (pendingDeletions.size() != 0) {
+ extras += "\n\nThese files we had previously tried to delete, but couldn't: " + pendingDeletions;
+ }
+
+ assert false : "unreferenced files: before delete:\n " + Arrays.toString(startFiles) + "\n after delete:\n " + Arrays.toString(endFiles) + extras;
}
DirectoryReader ir1 = DirectoryReader.open(this);
@@ -607,7 +690,6 @@
if (v != null) {
if (v.intValue() == 1) {
openFiles.remove(name);
- openFilesDeleted.remove(name);
} else {
v = Integer.valueOf(v.intValue()-1);
openFiles.put(name, v);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 97b4f60..eaefa92 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -948,6 +948,7 @@
if (rarely(random)) {
directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble());
}
+
if (bare) {
BaseDirectoryWrapper base = new BaseDirectoryWrapper(directory);
closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));
