LUCENE-3189: make sure we sync on IW when invoking IFD
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_3x@1166791 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
deleted file mode 100644
index 84af3a5..0000000
--- a/lucene/backwards/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ /dev/null
@@ -1,1088 +0,0 @@
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.Reader;
-import java.util.Arrays;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.MockTokenizer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Field.Index;
-import org.apache.lucene.document.Field.Store;
-import org.apache.lucene.document.Field.TermVector;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-
-public class TestIndexWriterDelete extends LuceneTestCase {
-
- // test the simple case
- public void testSimpleCase() throws IOException {
- String[] keywords = { "1", "2" };
- String[] unindexed = { "Netherlands", "Italy" };
- String[] unstored = { "Amsterdam has lots of bridges",
- "Venice has lots of canals" };
- String[] text = { "Amsterdam", "Venice" };
-
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDeleteTerms(1));
-
- for (int i = 0; i < keywords.length; i++) {
- Document doc = new Document();
- doc.add(newField("id", keywords[i], Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- doc.add(newField("country", unindexed[i], Field.Store.YES,
- Field.Index.NO));
- doc.add(newField("contents", unstored[i], Field.Store.NO,
- Field.Index.ANALYZED));
- doc
- .add(newField("city", text[i], Field.Store.YES,
- Field.Index.ANALYZED));
- modifier.addDocument(doc);
- }
- modifier.optimize();
- modifier.commit();
-
- Term term = new Term("city", "Amsterdam");
- int hitCount = getHitCount(dir, term);
- assertEquals(1, hitCount);
- modifier.deleteDocuments(term);
- modifier.commit();
- hitCount = getHitCount(dir, term);
- assertEquals(0, hitCount);
-
- modifier.close();
- dir.close();
- }
-
- // test when delete terms only apply to disk segments
- public void testNonRAMDelete() throws IOException {
-
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(2)
- .setMaxBufferedDeleteTerms(2));
- modifier.setInfoStream(VERBOSE ? System.out : null);
- int id = 0;
- int value = 100;
-
- for (int i = 0; i < 7; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.commit();
-
- assertEquals(0, modifier.getNumBufferedDocuments());
- assertTrue(0 < modifier.getSegmentCount());
-
- modifier.commit();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
-
- modifier.commit();
-
- reader = IndexReader.open(dir, true);
- assertEquals(0, reader.numDocs());
- reader.close();
- modifier.close();
- dir.close();
- }
-
- public void testMaxBufferedDeletes() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDeleteTerms(1));
-
- writer.setInfoStream(VERBOSE ? System.out : null);
- writer.addDocument(new Document());
- writer.deleteDocuments(new Term("foobar", "1"));
- writer.deleteDocuments(new Term("foobar", "1"));
- writer.deleteDocuments(new Term("foobar", "1"));
- assertEquals(3, writer.getFlushDeletesCount());
- writer.close();
- dir.close();
- }
-
- // test when delete terms only apply to ram segments
- public void testRAMDeletes() throws IOException {
- for(int t=0;t<2;t++) {
- if (VERBOSE) {
- System.out.println("TEST: t=" + t);
- }
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(4)
- .setMaxBufferedDeleteTerms(4));
- modifier.setInfoStream(VERBOSE ? System.out : null);
- int id = 0;
- int value = 100;
-
- addDoc(modifier, ++id, value);
- if (0 == t)
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
- else
- modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value))));
- addDoc(modifier, ++id, value);
- if (0 == t) {
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
- assertEquals(2, modifier.getNumBufferedDeleteTerms());
- assertEquals(1, modifier.getBufferedDeleteTermsSize());
- }
- else
- modifier.deleteDocuments(new TermQuery(new Term("value", String.valueOf(value))));
-
- addDoc(modifier, ++id, value);
- assertEquals(0, modifier.getSegmentCount());
- modifier.commit();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(1, reader.numDocs());
-
- int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
- assertEquals(1, hitCount);
- reader.close();
- modifier.close();
- dir.close();
- }
- }
-
- // test when delete terms apply to both disk and ram segments
- public void testBothDeletes() throws IOException {
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(100)
- .setMaxBufferedDeleteTerms(100));
-
- int id = 0;
- int value = 100;
-
- for (int i = 0; i < 5; i++) {
- addDoc(modifier, ++id, value);
- }
-
- value = 200;
- for (int i = 0; i < 5; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.commit();
-
- for (int i = 0; i < 5; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.deleteDocuments(new Term("value", String.valueOf(value)));
-
- modifier.commit();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(5, reader.numDocs());
- modifier.close();
- reader.close();
- dir.close();
- }
-
- // test that batched delete terms are flushed together
- public void testBatchDeletes() throws IOException {
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(2)
- .setMaxBufferedDeleteTerms(2));
-
- int id = 0;
- int value = 100;
-
- for (int i = 0; i < 7; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.commit();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- id = 0;
- modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
- modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
-
- modifier.commit();
-
- reader = IndexReader.open(dir, true);
- assertEquals(5, reader.numDocs());
- reader.close();
-
- Term[] terms = new Term[3];
- for (int i = 0; i < terms.length; i++) {
- terms[i] = new Term("id", String.valueOf(++id));
- }
- modifier.deleteDocuments(terms);
- modifier.commit();
- reader = IndexReader.open(dir, true);
- assertEquals(2, reader.numDocs());
- reader.close();
-
- modifier.close();
- dir.close();
- }
-
- // test deleteAll()
- public void testDeleteAll() throws IOException {
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(2)
- .setMaxBufferedDeleteTerms(2));
-
- int id = 0;
- int value = 100;
-
- for (int i = 0; i < 7; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.commit();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- // Add 1 doc (so we will have something buffered)
- addDoc(modifier, 99, value);
-
- // Delete all
- modifier.deleteAll();
-
- // Delete all shouldn't be on disk yet
- reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- // Add a doc and update a doc (after the deleteAll, before the commit)
- addDoc(modifier, 101, value);
- updateDoc(modifier, 102, value);
-
- // commit the delete all
- modifier.commit();
-
- // Validate there are no docs left
- reader = IndexReader.open(dir, true);
- assertEquals(2, reader.numDocs());
- reader.close();
-
- modifier.close();
- dir.close();
- }
-
- // test rollback of deleteAll()
- public void testDeleteAllRollback() throws IOException {
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(2)
- .setMaxBufferedDeleteTerms(2));
-
- int id = 0;
- int value = 100;
-
- for (int i = 0; i < 7; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.commit();
-
- addDoc(modifier, ++id, value);
-
- IndexReader reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- // Delete all
- modifier.deleteAll();
-
- // Roll it back
- modifier.rollback();
- modifier.close();
-
- // Validate that the docs are still there
- reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- dir.close();
- }
-
-
- // test deleteAll() w/ near real-time reader
- public void testDeleteAllNRT() throws IOException {
- Directory dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDocs(2)
- .setMaxBufferedDeleteTerms(2));
-
- int id = 0;
- int value = 100;
-
- for (int i = 0; i < 7; i++) {
- addDoc(modifier, ++id, value);
- }
- modifier.commit();
-
- IndexReader reader = modifier.getReader();
- assertEquals(7, reader.numDocs());
- reader.close();
-
- addDoc(modifier, ++id, value);
- addDoc(modifier, ++id, value);
-
- // Delete all
- modifier.deleteAll();
-
- reader = modifier.getReader();
- assertEquals(0, reader.numDocs());
- reader.close();
-
-
- // Roll it back
- modifier.rollback();
- modifier.close();
-
- // Validate that the docs are still there
- reader = IndexReader.open(dir, true);
- assertEquals(7, reader.numDocs());
- reader.close();
-
- dir.close();
- }
-
-
- private void updateDoc(IndexWriter modifier, int id, int value)
- throws IOException {
- Document doc = new Document();
- doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
- doc.add(newField("id", String.valueOf(id), Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- doc.add(newField("value", String.valueOf(value), Field.Store.NO,
- Field.Index.NOT_ANALYZED));
- modifier.updateDocument(new Term("id", String.valueOf(id)), doc);
- }
-
-
- private void addDoc(IndexWriter modifier, int id, int value)
- throws IOException {
- Document doc = new Document();
- doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
- doc.add(newField("id", String.valueOf(id), Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- doc.add(newField("value", String.valueOf(value), Field.Store.NO,
- Field.Index.NOT_ANALYZED));
- modifier.addDocument(doc);
- }
-
- private int getHitCount(Directory dir, Term term) throws IOException {
- IndexSearcher searcher = new IndexSearcher(dir, true);
- int hitCount = searcher.search(new TermQuery(term), null, 1000).totalHits;
- searcher.close();
- return hitCount;
- }
-
- public void testDeletesOnDiskFull() throws IOException {
- doTestOperationsOnDiskFull(false);
- }
-
- public void testUpdatesOnDiskFull() throws IOException {
- doTestOperationsOnDiskFull(true);
- }
-
- /**
- * Make sure if modifier tries to commit but hits disk full that modifier
- * remains consistent and usable. Similar to TestIndexReader.testDiskFull().
- */
- private void doTestOperationsOnDiskFull(boolean updates) throws IOException {
-
- Term searchTerm = new Term("content", "aaa");
- int START_COUNT = 157;
- int END_COUNT = 144;
-
- // First build up a starting index:
- MockDirectoryWrapper startDir = newDirectory();
- // TODO: find the resource leak that only occurs sometimes here.
- startDir.setNoDeleteOpenFile(false);
- IndexWriter writer = new IndexWriter(startDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
- for (int i = 0; i < 157; i++) {
- Document d = new Document();
- d.add(newField("id", Integer.toString(i), Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- d.add(newField("content", "aaa " + i, Field.Store.NO,
- Field.Index.ANALYZED));
- writer.addDocument(d);
- }
- writer.close();
-
- long diskUsage = startDir.sizeInBytes();
- long diskFree = diskUsage + 10;
-
- IOException err = null;
-
- boolean done = false;
-
- // Iterate w/ ever increasing free disk space:
- while (!done) {
- if (VERBOSE) {
- System.out.println("TEST: cycle");
- }
- MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir));
- dir.setPreventDoubleWrite(false);
- IndexWriter modifier = new IndexWriter(dir,
- newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))
- .setMaxBufferedDocs(1000)
- .setMaxBufferedDeleteTerms(1000)
- .setMergeScheduler(new ConcurrentMergeScheduler()));
- ((ConcurrentMergeScheduler) modifier.getConfig().getMergeScheduler()).setSuppressExceptions();
- modifier.setInfoStream(VERBOSE ? System.out : null);
-
- // For each disk size, first try to commit against
- // dir that will hit random IOExceptions & disk
- // full; after, give it infinite disk space & turn
- // off random IOExceptions & retry w/ same reader:
- boolean success = false;
-
- for (int x = 0; x < 2; x++) {
- if (VERBOSE) {
- System.out.println("TEST: x=" + x);
- }
-
- double rate = 0.1;
- double diskRatio = ((double)diskFree) / diskUsage;
- long thisDiskFree;
- String testName;
-
- if (0 == x) {
- thisDiskFree = diskFree;
- if (diskRatio >= 2.0) {
- rate /= 2;
- }
- if (diskRatio >= 4.0) {
- rate /= 2;
- }
- if (diskRatio >= 6.0) {
- rate = 0.0;
- }
- if (VERBOSE) {
- System.out.println("\ncycle: " + diskFree + " bytes");
- }
- testName = "disk full during reader.close() @ " + thisDiskFree
- + " bytes";
- } else {
- thisDiskFree = 0;
- rate = 0.0;
- if (VERBOSE) {
- System.out.println("\ncycle: same writer: unlimited disk space");
- }
- testName = "reader re-use after disk full";
- }
-
- dir.setMaxSizeInBytes(thisDiskFree);
- dir.setRandomIOExceptionRate(rate);
-
- try {
- if (0 == x) {
- int docId = 12;
- for (int i = 0; i < 13; i++) {
- if (updates) {
- Document d = new Document();
- d.add(newField("id", Integer.toString(i), Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- d.add(newField("content", "bbb " + i, Field.Store.NO,
- Field.Index.ANALYZED));
- modifier.updateDocument(new Term("id", Integer.toString(docId)), d);
- } else { // deletes
- modifier.deleteDocuments(new Term("id", Integer.toString(docId)));
- // modifier.setNorm(docId, "contents", (float)2.0);
- }
- docId += 12;
- }
- }
- modifier.close();
- success = true;
- if (0 == x) {
- done = true;
- }
- }
- catch (IOException e) {
- if (VERBOSE) {
- System.out.println(" hit IOException: " + e);
- e.printStackTrace(System.out);
- }
- err = e;
- if (1 == x) {
- e.printStackTrace();
- fail(testName + " hit IOException after disk space was freed up");
- }
- }
-
- if (!success) {
- // Must force the close else the writer can have
- // open files which cause exc in MockRAMDir.close
- modifier.rollback();
- }
-
- // If the close() succeeded, make sure there are
- // no unreferenced files.
- if (success) {
- _TestUtil.checkIndex(dir);
- TestIndexWriter.assertNoUnreferencedFiles(dir, "after writer.close");
- }
-
- // Finally, verify index is not corrupt, and, if
- // we succeeded, we see all docs changed, and if
- // we failed, we see either all docs or no docs
- // changed (transactional semantics):
- IndexReader newReader = null;
- try {
- newReader = IndexReader.open(dir, true);
- }
- catch (IOException e) {
- e.printStackTrace();
- fail(testName
- + ":exception when creating IndexReader after disk full during close: "
- + e);
- }
-
- IndexSearcher searcher = newSearcher(newReader);
- ScoreDoc[] hits = null;
- try {
- hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
- }
- catch (IOException e) {
- e.printStackTrace();
- fail(testName + ": exception when searching: " + e);
- }
- int result2 = hits.length;
- if (success) {
- if (x == 0 && result2 != END_COUNT) {
- fail(testName
- + ": method did not throw exception but hits.length for search on term 'aaa' is "
- + result2 + " instead of expected " + END_COUNT);
- } else if (x == 1 && result2 != START_COUNT && result2 != END_COUNT) {
- // It's possible that the first exception was
- // "recoverable" wrt pending deletes, in which
- // case the pending deletes are retained and
- // then re-flushing (with plenty of disk
- // space) will succeed in flushing the
- // deletes:
- fail(testName
- + ": method did not throw exception but hits.length for search on term 'aaa' is "
- + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
- }
- } else {
- // On hitting exception we still may have added
- // all docs:
- if (result2 != START_COUNT && result2 != END_COUNT) {
- err.printStackTrace();
- fail(testName
- + ": method did throw exception but hits.length for search on term 'aaa' is "
- + result2 + " instead of expected " + START_COUNT + " or " + END_COUNT);
- }
- }
-
- searcher.close();
- newReader.close();
- }
-
- modifier.close();
- dir.close();
-
- // Try again with 10 more bytes of free space:
- diskFree += 10;
- }
- startDir.close();
- }
-
- // This test tests that buffered deletes are cleared when
- // an Exception is hit during flush.
- public void testErrorAfterApplyDeletes() throws IOException {
-
- MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
- boolean sawMaybe = false;
- boolean failed = false;
- Thread thread;
- @Override
- public MockDirectoryWrapper.Failure reset() {
- thread = Thread.currentThread();
- sawMaybe = false;
- failed = false;
- return this;
- }
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- if (Thread.currentThread() != thread) {
- // don't fail during merging
- return;
- }
- if (sawMaybe && !failed) {
- boolean seen = false;
- StackTraceElement[] trace = new Exception().getStackTrace();
- for (int i = 0; i < trace.length; i++) {
- if ("applyDeletes".equals(trace[i].getMethodName())) {
- seen = true;
- break;
- }
- }
- if (!seen) {
- // Only fail once we are no longer in applyDeletes
- failed = true;
- if (VERBOSE) {
- System.out.println("TEST: mock failure: now fail");
- new Throwable().printStackTrace(System.out);
- }
- throw new IOException("fail after applyDeletes");
- }
- }
- if (!failed) {
- StackTraceElement[] trace = new Exception().getStackTrace();
- for (int i = 0; i < trace.length; i++) {
- if ("applyDeletes".equals(trace[i].getMethodName())) {
- if (VERBOSE) {
- System.out.println("TEST: mock failure: saw applyDeletes");
- new Throwable().printStackTrace(System.out);
- }
- sawMaybe = true;
- break;
- }
- }
- }
- }
- };
-
- // create a couple of files
-
- String[] keywords = { "1", "2" };
- String[] unindexed = { "Netherlands", "Italy" };
- String[] unstored = { "Amsterdam has lots of bridges",
- "Venice has lots of canals" };
- String[] text = { "Amsterdam", "Venice" };
-
- MockDirectoryWrapper dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setMaxBufferedDeleteTerms(2).setReaderPooling(false).setMergePolicy(newLogMergePolicy()));
- modifier.setInfoStream(VERBOSE ? System.out : null);
-
- LogMergePolicy lmp = (LogMergePolicy) modifier.getConfig().getMergePolicy();
- lmp.setUseCompoundFile(true);
-
- dir.failOn(failure.reset());
-
- for (int i = 0; i < keywords.length; i++) {
- Document doc = new Document();
- doc.add(newField("id", keywords[i], Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- doc.add(newField("country", unindexed[i], Field.Store.YES,
- Field.Index.NO));
- doc.add(newField("contents", unstored[i], Field.Store.NO,
- Field.Index.ANALYZED));
- doc.add(newField("city", text[i], Field.Store.YES,
- Field.Index.ANALYZED));
- modifier.addDocument(doc);
- }
- // flush (and commit if ac)
-
- if (VERBOSE) {
- System.out.println("TEST: now optimize");
- }
-
- modifier.optimize();
- if (VERBOSE) {
- System.out.println("TEST: now commit");
- }
- modifier.commit();
-
- // one of the two files hits
-
- Term term = new Term("city", "Amsterdam");
- int hitCount = getHitCount(dir, term);
- assertEquals(1, hitCount);
-
- // open the writer again (closed above)
-
- // delete the doc
- // max buf del terms is two, so this is buffered
-
- if (VERBOSE) {
- System.out.println("TEST: delete term=" + term);
- }
-
- modifier.deleteDocuments(term);
-
- // add a doc (needed for the !ac case; see below)
- // doc remains buffered
-
- if (VERBOSE) {
- System.out.println("TEST: add empty doc");
- }
- Document doc = new Document();
- modifier.addDocument(doc);
-
- // commit the changes, the buffered deletes, and the new doc
-
- // The failure object will fail on the first write after the del
- // file gets created when processing the buffered delete
-
- // in the ac case, this will be when writing the new segments
- // files so we really don't need the new doc, but it's harmless
-
- // a new segments file won't be created but in this
- // case, creation of the cfs file happens next so we
- // need the doc (to test that it's okay that we don't
- // lose deletes if failing while creating the cfs file)
- boolean failed = false;
- try {
- if (VERBOSE) {
- System.out.println("TEST: now commit for failure");
- }
- modifier.commit();
- } catch (IOException ioe) {
- // expected
- failed = true;
- }
-
- assertTrue(failed);
-
- // The commit above failed, so we need to retry it (which will
- // succeed, because the failure is a one-shot)
-
- modifier.commit();
-
- hitCount = getHitCount(dir, term);
-
- // Make sure the delete was successfully flushed:
- assertEquals(0, hitCount);
-
- modifier.close();
- dir.close();
- }
-
- // This test tests that the files created by the docs writer before
- // a segment is written are cleaned up if there's an i/o error
-
- public void testErrorInDocsWriterAdd() throws IOException {
-
- MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {
- boolean failed = false;
- @Override
- public MockDirectoryWrapper.Failure reset() {
- failed = false;
- return this;
- }
- @Override
- public void eval(MockDirectoryWrapper dir) throws IOException {
- if (!failed) {
- failed = true;
- throw new IOException("fail in add doc");
- }
- }
- };
-
- // create a couple of files
-
- String[] keywords = { "1", "2" };
- String[] unindexed = { "Netherlands", "Italy" };
- String[] unstored = { "Amsterdam has lots of bridges",
- "Venice has lots of canals" };
- String[] text = { "Amsterdam", "Venice" };
-
- MockDirectoryWrapper dir = newDirectory();
- IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
- modifier.commit();
- dir.failOn(failure.reset());
-
- for (int i = 0; i < keywords.length; i++) {
- Document doc = new Document();
- doc.add(newField("id", keywords[i], Field.Store.YES,
- Field.Index.NOT_ANALYZED));
- doc.add(newField("country", unindexed[i], Field.Store.YES,
- Field.Index.NO));
- doc.add(newField("contents", unstored[i], Field.Store.NO,
- Field.Index.ANALYZED));
- doc.add(newField("city", text[i], Field.Store.YES,
- Field.Index.ANALYZED));
- try {
- modifier.addDocument(doc);
- } catch (IOException io) {
- if (VERBOSE) {
- System.out.println("TEST: got expected exc:");
- io.printStackTrace(System.out);
- }
- break;
- }
- }
-
- String[] startFiles = dir.listAll();
- SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null);
- String[] endFiles = dir.listAll();
- modifier.close();
- dir.close();
-
- if (!Arrays.equals(startFiles, endFiles)) {
- fail("docswriter abort() failed to delete unreferenced files:\n before delete:\n "
- + arrayToString(startFiles) + "\n after delete:\n "
- + arrayToString(endFiles));
- }
-
- modifier.close();
-
- }
-
- private String arrayToString(String[] l) {
- String s = "";
- for (int i = 0; i < l.length; i++) {
- if (i > 0) {
- s += "\n ";
- }
- s += l[i];
- }
- return s;
- }
-
- public void testDeleteAllSlowly() throws Exception {
- final Directory dir = newDirectory();
- RandomIndexWriter w = new RandomIndexWriter(random, dir);
- final int NUM_DOCS = atLeast(1000);
- final List<Integer> ids = new ArrayList<Integer>(NUM_DOCS);
- for(int id=0;id<NUM_DOCS;id++) {
- ids.add(id);
- }
- Collections.shuffle(ids, random);
- for(int id : ids) {
- Document doc = new Document();
- doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
- w.addDocument(doc);
- }
- Collections.shuffle(ids, random);
- int upto = 0;
- while(upto < ids.size()) {
- final int left = ids.size() - upto;
- final int inc = Math.min(left, _TestUtil.nextInt(random, 1, 20));
- final int limit = upto + inc;
- while(upto < limit) {
- w.deleteDocuments(new Term("id", ""+ids.get(upto++)));
- }
- final IndexReader r = w.getReader();
- assertEquals(NUM_DOCS - upto, r.numDocs());
- r.close();
- }
-
- w.close();
- dir.close();
- }
-
- public void testIndexingThenDeleting() throws Exception {
- final Random r = random;
- Directory dir = newDirectory();
- // note this test explicitly disables payloads
- final Analyzer analyzer = new Analyzer() {
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
- }
- };
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
- w.setInfoStream(VERBOSE ? System.out : null);
- Document doc = new Document();
- doc.add(newField("field", "go 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20", Field.Store.NO, Field.Index.ANALYZED));
- int num = atLeast(3);
- for (int iter = 0; iter < num; iter++) {
- int count = 0;
-
- final boolean doIndexing = r.nextBoolean();
- if (VERBOSE) {
- System.out.println("TEST: iter doIndexing=" + doIndexing);
- }
- if (doIndexing) {
- // Add docs until a flush is triggered
- final int startFlushCount = w.getFlushCount();
- while(w.getFlushCount() == startFlushCount) {
- w.addDocument(doc);
- count++;
- }
- } else {
- // Delete docs until a flush is triggered
- final int startFlushCount = w.getFlushCount();
- while(w.getFlushCount() == startFlushCount) {
- w.deleteDocuments(new Term("foo", ""+count));
- count++;
- }
- }
- assertTrue("flush happened too quickly during " + (doIndexing ? "indexing" : "deleting") + " count=" + count, count > 3000);
- }
- w.close();
- dir.close();
- }
-
- // LUCENE-3340: make sure deletes that we don't apply
- // during flush (ie are just pushed into the stream) are
- // in fact later flushed due to their RAM usage:
- public void testFlushPushedDeletesByRAM() throws Exception {
- Directory dir = newDirectory();
- // Cannot use RandomIndexWriter because we don't want to
- // ever call commit() for this test:
- // note: tiny rambuffer used, as with a 1MB buffer the test is too slow (flush @ 128,999)
- IndexWriter w = new IndexWriter(dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setRAMBufferSizeMB(0.2f).setMaxBufferedDocs(1000).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false));
- w.setInfoStream(VERBOSE ? System.out : null);
- int count = 0;
- while(true) {
- Document doc = new Document();
- doc.add(new Field("id", count+"", Field.Store.NO, Field.Index.NOT_ANALYZED));
- final Term delTerm;
- if (count == 1010) {
- // This is the only delete that applies
- delTerm = new Term("id", ""+0);
- } else {
- // These get buffered, taking up RAM, but delete
- // nothing when applied:
- delTerm = new Term("id", "x" + count);
- }
- w.updateDocument(delTerm, doc);
- // Eventually segment 0 should get a del docs:
- if (dir.fileExists("_0_1.del")) {
- if (VERBOSE) {
- System.out.println("TEST: deletes created @ count=" + count);
- }
- break;
- }
- count++;
-
- // Today we applyDeletes @ count=21553; even if we make
- // sizable improvements to RAM efficiency of buffered
- // del term we're unlikely to go over 100K:
- if (count > 100000) {
- fail("delete's were not applied");
- }
- }
- w.close();
- dir.close();
- }
-
- // LUCENE-3340: make sure deletes that we don't apply
- // during flush (ie are just pushed into the stream) are
- // in fact later flushed due to their RAM usage:
- public void testFlushPushedDeletesByCount() throws Exception {
- Directory dir = newDirectory();
- // Cannot use RandomIndexWriter because we don't want to
- // ever call commit() for this test:
- final int flushAtDelCount = atLeast(1020);
- IndexWriter w = new IndexWriter(dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setMaxBufferedDeleteTerms(flushAtDelCount).setMaxBufferedDocs(1000).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false));
- w.setInfoStream(VERBOSE ? System.out : null);
- if (VERBOSE) {
- System.out.println("TEST: flush @ " + flushAtDelCount + " buffered delete terms");
- }
- int count = 0;
- while(true) {
- Document doc = new Document();
- doc.add(new Field("id", count+"", Field.Store.NO, Field.Index.NOT_ANALYZED));
- final Term delTerm;
- if (count == 1010) {
- // This is the only delete that applies
- delTerm = new Term("id", ""+0);
- } else {
- // These get buffered, taking up RAM, but delete
- // nothing when applied:
- delTerm = new Term("id", "x" + count);
- }
- w.updateDocument(delTerm, doc);
- // Eventually segment 0 should get a del docs:
- if (dir.fileExists("_0_1.del")) {
- break;
- }
- count++;
- if (count > flushAtDelCount) {
- fail("delete's were not applied at count=" + flushAtDelCount);
- }
- }
- w.close();
- dir.close();
- }
-
- // Make sure buffered (pushed) deletes don't use up so
- // much RAM that it forces long tail of tiny segments:
- public void testApplyDeletesOnFlush() throws Exception {
- Directory dir = newDirectory();
- // Cannot use RandomIndexWriter because we don't want to
- // ever call commit() for this test:
- final AtomicInteger docsInSegment = new AtomicInteger();
- final AtomicBoolean closing = new AtomicBoolean();
- final AtomicBoolean sawAfterFlush = new AtomicBoolean();
- IndexWriter w = new IndexWriter(dir,
- newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
- setRAMBufferSizeMB(0.5).setMaxBufferedDocs(-1).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false)) {
- @Override
- public void doAfterFlush() {
- assertTrue("only " + docsInSegment.get() + " in segment", closing.get() || docsInSegment.get() >= 7);
- docsInSegment.set(0);
- sawAfterFlush.set(true);
- }
- };
- w.setInfoStream(VERBOSE ? System.out : null);
- int id = 0;
- while(true) {
- StringBuilder sb = new StringBuilder();
- for(int termIDX=0;termIDX<100;termIDX++) {
- sb.append(' ').append(_TestUtil.randomRealisticUnicodeString(random));
- }
- if (id == 500) {
- w.deleteDocuments(new Term("id", "0"));
- }
- Document doc = new Document();
- doc.add(newField("id", ""+id, Field.Index.NOT_ANALYZED));
- doc.add(newField("body", sb.toString(), Field.Index.ANALYZED));
- w.updateDocument(new Term("id", ""+id), doc);
- docsInSegment.incrementAndGet();
- if (dir.fileExists("_0_1.del")) {
- if (VERBOSE) {
- System.out.println("TEST: deletes created @ id=" + id);
- }
- break;
- }
- id++;
- }
- closing.set(true);
- assertTrue(sawAfterFlush.get());
- w.close();
- dir.close();
- }
-}
diff --git a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
index 5c1a258..44b3903 100644
--- a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
+++ b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
@@ -795,7 +795,7 @@
// KeepOnlyLastCommitDeleter:
IndexFileDeleter deleter = new IndexFileDeleter(directory,
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
- segmentInfos, null);
+ segmentInfos, null, null);
segmentInfos.updateGeneration(deleter.getLastSegmentInfos());
segmentInfos.changed();
diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
index 938c46c..15acaa0 100644
--- a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
+++ b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
@@ -101,6 +101,9 @@
* infoStream != null */
public static boolean VERBOSE_REF_COUNTS = false;
+ // Used only for assert
+ private final IndexWriter writer;
+
void setInfoStream(PrintStream infoStream) {
this.infoStream = infoStream;
if (infoStream != null) {
@@ -112,6 +115,11 @@
infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message);
}
+ // called only from assert
+ private boolean locked() {
+ return writer == null || Thread.holdsLock(writer);
+ }
+
/**
* Initialize the deleter: find all previous commits in
* the Directory, incref the files they reference, call
@@ -120,10 +128,11 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream)
+ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, IndexWriter writer)
throws CorruptIndexException, IOException {
this.infoStream = infoStream;
+ this.writer = writer;
final String currentSegmentsFile = segmentInfos.getCurrentSegmentFileName();
@@ -316,6 +325,8 @@
* that segment.
*/
public void refresh(String segmentName) throws IOException {
+ assert locked();
+
String[] files = directory.listAll();
IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
String segmentPrefix1;
@@ -347,12 +358,14 @@
// Set to null so that we regenerate the list of pending
// files; else we can accumulate same file more than
// once
+ assert locked();
deletable = null;
refresh(null);
}
public void close() throws IOException {
// DecRef old files from the last checkpoint, if any:
+ assert locked();
int size = lastFiles.size();
if (size > 0) {
for(int i=0;i<size;i++) {
@@ -374,6 +387,7 @@
* unused commits again.
*/
void revisitPolicy() throws IOException {
+ assert locked();
if (infoStream != null) {
message("now revisitPolicy");
}
@@ -385,6 +399,7 @@
}
public void deletePendingFiles() throws IOException {
+ assert locked();
if (deletable != null) {
List<String> oldDeletable = deletable;
deletable = null;
@@ -419,6 +434,7 @@
* removed, we decref their files as well.
*/
public void checkpoint(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
+ assert locked();
if (infoStream != null) {
message("now checkpoint \"" + segmentInfos.getCurrentSegmentFileName() + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]");
@@ -453,6 +469,7 @@
}
void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
+ assert locked();
// If this is a commit point, also incRef the
// segments_N file:
for( final String fileName: segmentInfos.files(directory, isCommit) ) {
@@ -461,12 +478,14 @@
}
void incRef(Collection<String> files) throws IOException {
+ assert locked();
for(final String file : files) {
incRef(file);
}
}
void incRef(String fileName) throws IOException {
+ assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream != null && VERBOSE_REF_COUNTS) {
message(" IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
@@ -475,12 +494,14 @@
}
void decRef(Collection<String> files) throws IOException {
+ assert locked();
for(final String file : files) {
decRef(file);
}
}
void decRef(String fileName) throws IOException {
+ assert locked();
RefCount rc = getRefCount(fileName);
if (infoStream != null && VERBOSE_REF_COUNTS) {
message(" DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
@@ -494,12 +515,14 @@
}
void decRef(SegmentInfos segmentInfos) throws IOException {
+ assert locked();
for (final String file : segmentInfos.files(directory, false)) {
decRef(file);
}
}
public boolean exists(String fileName) {
+ assert locked();
if (!refCounts.containsKey(fileName)) {
return false;
} else {
@@ -508,6 +531,7 @@
}
private RefCount getRefCount(String fileName) {
+ assert locked();
RefCount rc;
if (!refCounts.containsKey(fileName)) {
rc = new RefCount(fileName);
@@ -519,6 +543,7 @@
}
void deleteFiles(List<String> files) throws IOException {
+ assert locked();
for(final String file: files) {
deleteFile(file);
}
@@ -527,6 +552,7 @@
/** Deletes the specified files, but only if they are new
* (have not yet been incref'd). */
void deleteNewFiles(Collection<String> files) throws IOException {
+ assert locked();
for (final String fileName: files) {
if (!refCounts.containsKey(fileName)) {
if (infoStream != null) {
@@ -539,6 +565,7 @@
void deleteFile(String fileName)
throws IOException {
+ assert locked();
try {
if (infoStream != null) {
message("delete \"" + fileName + "\"");
diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java
index 840ff71..84c8b75 100644
--- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java
+++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java
@@ -1178,9 +1178,12 @@
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
- deleter = new IndexFileDeleter(directory,
- conf.getIndexDeletionPolicy(),
- segmentInfos, infoStream);
+ synchronized(this) {
+ deleter = new IndexFileDeleter(directory,
+ conf.getIndexDeletionPolicy(),
+ segmentInfos, infoStream,
+ this);
+ }
if (deleter.startingCommitDeleted) {
// Deletion policy deleted the "head" commit point.
@@ -3109,7 +3112,9 @@
// delete new non cfs files directly: they were never
// registered with IFD
- deleter.deleteNewFiles(info.files());
+ synchronized(this) {
+ deleter.deleteNewFiles(info.files());
+ }
info.setUseCompoundFile(true);
}
@@ -3380,34 +3385,45 @@
message("prepareCommit: flush");
ensureOpen(false);
- final boolean anySegmentsFlushed;
- final SegmentInfos toCommit;
- synchronized (this) {
- anySegmentsFlushed = doFlush(true);
- readerPool.commit(segmentInfos);
- // Must clone the segmentInfos while we still
- // hold fullFlushLock and while sync'd so that
- // no partial changes (eg a delete w/o
- // corresponding add from an updateDocument) can
- // sneak into the commit point:
- toCommit = (SegmentInfos) segmentInfos.clone();
- pendingCommitChangeCount = changeCount;
- // This protects the segmentInfos we are now going
- // to commit. This is important in case, eg, while
- // we are trying to sync all referenced files, a
- // merge completes which would otherwise have
- // removed the files we are now syncing.
- deleter.incRef(toCommit, false);
- }
+ boolean anySegmentsFlushed = false;
+ SegmentInfos toCommit = null;
boolean success = false;
try {
+ try {
+ synchronized (this) {
+ anySegmentsFlushed = doFlush(true);
+ readerPool.commit(segmentInfos);
+ toCommit = (SegmentInfos) segmentInfos.clone();
+ pendingCommitChangeCount = changeCount;
+ // This protects the segmentInfos we are now going
+ // to commit. This is important in case, eg, while
+ // we are trying to sync all referenced files, a
+ // merge completes which would otherwise have
+ // removed the files we are now syncing.
+ deleter.incRef(toCommit, false);
+ }
+ success = true;
+ } finally {
+ if (!success && infoStream != null) {
+ message("hit exception during prepareCommit");
+ }
+ doAfterFlush();
+ }
+ } catch (OutOfMemoryError oom) {
+ handleOOM(oom, "prepareCommit");
+ }
+
+ success = false;
+ try {
if (anySegmentsFlushed) {
maybeMerge();
}
success = true;
} finally {
if (!success) {
- deleter.decRef(toCommit);
+ synchronized (this) {
+ deleter.decRef(toCommit);
+ }
}
}
diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
index 8bb051e..6138622 100644
--- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
+++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
@@ -1320,6 +1320,7 @@
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2);
w = new IndexWriter(dir, conf);
+ w.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(newField("field", "some text contents", Field.Store.YES, Field.Index.ANALYZED));
diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
index 84af3a5..edcdc59 100644
--- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
+++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
@@ -849,7 +849,7 @@
String[] startFiles = dir.listAll();
SegmentInfos infos = new SegmentInfos();
infos.read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null);
+ new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
String[] endFiles = dir.listAll();
modifier.close();
dir.close();