| package org.apache.lucene.index; |
| |
| /** |
| * Copyright 2004 The Apache Software Foundation |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import java.io.IOException; |
| import java.io.PrintStream; |
| import java.util.ArrayList; |
| import java.util.Collections; |
| import java.util.List; |
| import java.util.Random; |
| import java.util.concurrent.atomic.AtomicBoolean; |
| |
| import org.apache.lucene.analysis.MockAnalyzer; |
| import org.apache.lucene.analysis.WhitespaceAnalyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.document.Field.Index; |
| import org.apache.lucene.document.Field.Store; |
| import org.apache.lucene.document.Field.TermVector; |
| import org.apache.lucene.search.TermQuery; |
| import org.apache.lucene.search.IndexSearcher; |
| import org.apache.lucene.search.Query; |
| import org.apache.lucene.search.TopDocs; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.store.MockDirectoryWrapper; |
| import org.apache.lucene.store.AlreadyClosedException; |
| import org.apache.lucene.store.RAMDirectory; |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util._TestUtil; |
| import org.apache.lucene.util.ThreadInterruptedException; |
| import java.util.concurrent.atomic.AtomicInteger; |
| |
| public class TestIndexWriterReader extends LuceneTestCase { |
| static PrintStream infoStream = VERBOSE ? System.out : null; |
| |
| public static int count(Term t, IndexReader r) throws IOException { |
| int count = 0; |
| TermDocs td = r.termDocs(t); |
| while (td.next()) { |
| td.doc(); |
| count++; |
| } |
| td.close(); |
| return count; |
| } |
| |
| public void testAddCloseOpen() throws IOException { |
| Directory dir1 = newDirectory(); |
| IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); |
| |
| IndexWriter writer = new IndexWriter(dir1, iwc); |
| for (int i = 0; i < 97 ; i++) { |
| IndexReader reader = writer.getReader(); |
| if (i == 0) { |
| writer.addDocument(DocHelper.createDocument(i, "x", 1 + random.nextInt(5))); |
| } else { |
| int previous = random.nextInt(i); |
| // a check if the reader is current here could fail since there might be |
| // merges going on. |
| switch (random.nextInt(5)) { |
| case 0: |
| case 1: |
| case 2: |
| writer.addDocument(DocHelper.createDocument(i, "x", 1 + random.nextInt(5))); |
| break; |
| case 3: |
| writer.updateDocument(new Term("id", "" + previous), DocHelper.createDocument( |
| previous, "x", 1 + random.nextInt(5))); |
| break; |
| case 4: |
| writer.deleteDocuments(new Term("id", "" + previous)); |
| } |
| } |
| assertFalse(reader.isCurrent()); |
| reader.close(); |
| } |
| writer.optimize(); // make sure all merging is done etc. |
| IndexReader reader = writer.getReader(); |
| writer.commit(); // no changes that are not visible to the reader |
| assertTrue(reader.isCurrent()); |
| writer.close(); |
| assertTrue(reader.isCurrent()); // all changes are visible to the reader |
| iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); |
| writer = new IndexWriter(dir1, iwc); |
| assertTrue(reader.isCurrent()); |
| writer.addDocument(DocHelper.createDocument(1, "x", 1+random.nextInt(5))); |
| assertTrue(reader.isCurrent()); // segments in ram but IW is different to the readers one |
| writer.close(); |
| assertFalse(reader.isCurrent()); // segments written |
| reader.close(); |
| dir1.close(); |
| } |
| |
| public void testUpdateDocument() throws Exception { |
| boolean optimize = true; |
| |
| Directory dir1 = newDirectory(); |
| IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); |
| if (iwc.getMaxBufferedDocs() < 20) { |
| iwc.setMaxBufferedDocs(20); |
| } |
| // no merging |
| if (random.nextBoolean()) { |
| iwc.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES); |
| } else { |
| iwc.setMergePolicy(NoMergePolicy.COMPOUND_FILES); |
| } |
| IndexWriter writer = new IndexWriter(dir1, iwc); |
| |
| // create the index |
| createIndexNoClose(!optimize, "index1", writer); |
| |
| // writer.flush(false, true, true); |
| |
| // get a reader |
| IndexReader r1 = writer.getReader(); |
| assertTrue(r1.isCurrent()); |
| |
| String id10 = r1.document(10).getField("id").stringValue(); |
| |
| Document newDoc = r1.document(10); |
| newDoc.removeField("id"); |
| newDoc.add(newField("id", Integer.toString(8000), Store.YES, Index.NOT_ANALYZED)); |
| writer.updateDocument(new Term("id", id10), newDoc); |
| assertFalse(r1.isCurrent()); |
| |
| IndexReader r2 = writer.getReader(); |
| assertTrue(r2.isCurrent()); |
| assertEquals(0, count(new Term("id", id10), r2)); |
| assertEquals(1, count(new Term("id", Integer.toString(8000)), r2)); |
| |
| r1.close(); |
| writer.close(); |
| assertTrue(r2.isCurrent()); |
| |
| IndexReader r3 = IndexReader.open(dir1, true); |
| assertTrue(r3.isCurrent()); |
| assertTrue(r2.isCurrent()); |
| assertEquals(0, count(new Term("id", id10), r3)); |
| assertEquals(1, count(new Term("id", Integer.toString(8000)), r3)); |
| |
| writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| Document doc = new Document(); |
| doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); |
| writer.addDocument(doc); |
| assertTrue(r2.isCurrent()); |
| assertTrue(r3.isCurrent()); |
| |
| writer.close(); |
| |
| assertFalse(r2.isCurrent()); |
| assertTrue(!r3.isCurrent()); |
| |
| r2.close(); |
| r3.close(); |
| |
| dir1.close(); |
| } |
| |
| public void testIsCurrent() throws IOException { |
| Directory dir = newDirectory(); |
| IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); |
| |
| IndexWriter writer = new IndexWriter(dir, iwc); |
| Document doc = new Document(); |
| doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); |
| writer.addDocument(doc); |
| writer.close(); |
| |
| iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); |
| writer = new IndexWriter(dir, iwc); |
| doc = new Document(); |
| doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); |
| IndexReader nrtReader = writer.getReader(); |
| assertTrue(nrtReader.isCurrent()); |
| writer.addDocument(doc); |
| assertFalse(nrtReader.isCurrent()); // should see the changes |
| writer.optimize(); // make sure we don't have a merge going on |
| assertFalse(nrtReader.isCurrent()); |
| nrtReader.close(); |
| |
| IndexReader dirReader = IndexReader.open(dir); |
| nrtReader = writer.getReader(); |
| |
| assertTrue(dirReader.isCurrent()); |
| assertTrue(nrtReader.isCurrent()); // nothing was committed yet so we are still current |
| assertEquals(2, nrtReader.maxDoc()); // sees the actual document added |
| assertEquals(1, dirReader.maxDoc()); |
| writer.close(); // close is actually a commit both should see the changes |
| assertTrue(nrtReader.isCurrent()); |
| assertFalse(dirReader.isCurrent()); // this reader has been opened before the writer was closed / committed |
| |
| dirReader.close(); |
| nrtReader.close(); |
| dir.close(); |
| } |
| |
| /** |
| * Test using IW.addIndexes |
| * |
| * @throws Exception |
| */ |
| public void testAddIndexes() throws Exception { |
| boolean optimize = false; |
| |
| Directory dir1 = newDirectory(); |
| IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); |
| if (iwc.getMaxBufferedDocs() < 20) { |
| iwc.setMaxBufferedDocs(20); |
| } |
| // no merging |
| if (random.nextBoolean()) { |
| iwc.setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES); |
| } else { |
| iwc.setMergePolicy(NoMergePolicy.COMPOUND_FILES); |
| } |
| IndexWriter writer = new IndexWriter(dir1, iwc); |
| |
| writer.setInfoStream(infoStream); |
| // create the index |
| createIndexNoClose(!optimize, "index1", writer); |
| writer.flush(false, true); |
| |
| // create a 2nd index |
| Directory dir2 = newDirectory(); |
| IndexWriter writer2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| writer2.setInfoStream(infoStream); |
| createIndexNoClose(!optimize, "index2", writer2); |
| writer2.close(); |
| |
| IndexReader r0 = writer.getReader(); |
| assertTrue(r0.isCurrent()); |
| writer.addIndexes(new Directory[] { dir2 }); |
| assertFalse(r0.isCurrent()); |
| r0.close(); |
| |
| IndexReader r1 = writer.getReader(); |
| assertTrue(r1.isCurrent()); |
| |
| writer.commit(); |
| assertTrue(r1.isCurrent()); // we have seen all changes - no change after opening the NRT reader |
| |
| assertEquals(200, r1.maxDoc()); |
| |
| int index2df = r1.docFreq(new Term("indexname", "index2")); |
| |
| assertEquals(100, index2df); |
| |
| // verify the docs are from different indexes |
| Document doc5 = r1.document(5); |
| assertEquals("index1", doc5.get("indexname")); |
| Document doc150 = r1.document(150); |
| assertEquals("index2", doc150.get("indexname")); |
| r1.close(); |
| writer.close(); |
| dir1.close(); |
| dir2.close(); |
| } |
| |
| public void testAddIndexes2() throws Exception { |
| boolean optimize = false; |
| |
| Directory dir1 = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| writer.setInfoStream(infoStream); |
| |
| // create a 2nd index |
| Directory dir2 = newDirectory(); |
| IndexWriter writer2 = new IndexWriter(dir2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| writer2.setInfoStream(infoStream); |
| createIndexNoClose(!optimize, "index2", writer2); |
| writer2.close(); |
| |
| writer.addIndexes(new Directory[] { dir2 }); |
| writer.addIndexes(new Directory[] { dir2 }); |
| writer.addIndexes(new Directory[] { dir2 }); |
| writer.addIndexes(new Directory[] { dir2 }); |
| writer.addIndexes(new Directory[] { dir2 }); |
| |
| IndexReader r1 = writer.getReader(); |
| assertEquals(500, r1.maxDoc()); |
| |
| r1.close(); |
| writer.close(); |
| dir1.close(); |
| dir2.close(); |
| } |
| |
| /** |
| * Deletes using IW.deleteDocuments |
| * |
| * @throws Exception |
| */ |
| public void testDeleteFromIndexWriter() throws Exception { |
| boolean optimize = true; |
| |
| Directory dir1 = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderTermsIndexDivisor(2)); |
| writer.setInfoStream(infoStream); |
| // create the index |
| createIndexNoClose(!optimize, "index1", writer); |
| writer.flush(false, true); |
| // get a reader |
| IndexReader r1 = writer.getReader(); |
| |
| String id10 = r1.document(10).getField("id").stringValue(); |
| |
| // deleted IW docs should not show up in the next getReader |
| writer.deleteDocuments(new Term("id", id10)); |
| IndexReader r2 = writer.getReader(); |
| assertEquals(1, count(new Term("id", id10), r1)); |
| assertEquals(0, count(new Term("id", id10), r2)); |
| |
| String id50 = r1.document(50).getField("id").stringValue(); |
| assertEquals(1, count(new Term("id", id50), r1)); |
| |
| writer.deleteDocuments(new Term("id", id50)); |
| |
| IndexReader r3 = writer.getReader(); |
| assertEquals(0, count(new Term("id", id10), r3)); |
| assertEquals(0, count(new Term("id", id50), r3)); |
| |
| String id75 = r1.document(75).getField("id").stringValue(); |
| writer.deleteDocuments(new TermQuery(new Term("id", id75))); |
| IndexReader r4 = writer.getReader(); |
| assertEquals(1, count(new Term("id", id75), r3)); |
| assertEquals(0, count(new Term("id", id75), r4)); |
| |
| r1.close(); |
| r2.close(); |
| r3.close(); |
| r4.close(); |
| writer.close(); |
| |
| // reopen the writer to verify the delete made it to the directory |
| writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| writer.setInfoStream(infoStream); |
| IndexReader w2r1 = writer.getReader(); |
| assertEquals(0, count(new Term("id", id10), w2r1)); |
| w2r1.close(); |
| writer.close(); |
| dir1.close(); |
| } |
| |
| public void testAddIndexesAndDoDeletesThreads() throws Throwable { |
| final int numIter = 2; |
| int numDirs = 3; |
| |
| Directory mainDir = newDirectory(); |
| IndexWriter mainWriter = new IndexWriter(mainDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); |
| _TestUtil.reduceOpenFiles(mainWriter); |
| |
| mainWriter.setInfoStream(infoStream); |
| AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(numIter, mainWriter); |
| addDirThreads.launchThreads(numDirs); |
| addDirThreads.joinThreads(); |
| |
| //assertEquals(100 + numDirs * (3 * numIter / 4) * addDirThreads.NUM_THREADS |
| // * addDirThreads.NUM_INIT_DOCS, addDirThreads.mainWriter.numDocs()); |
| assertEquals(addDirThreads.count.intValue(), addDirThreads.mainWriter.numDocs()); |
| |
| addDirThreads.close(true); |
| |
| assertTrue(addDirThreads.failures.size() == 0); |
| |
| _TestUtil.checkIndex(mainDir); |
| |
| IndexReader reader = IndexReader.open(mainDir, true); |
| assertEquals(addDirThreads.count.intValue(), reader.numDocs()); |
| //assertEquals(100 + numDirs * (3 * numIter / 4) * addDirThreads.NUM_THREADS |
| // * addDirThreads.NUM_INIT_DOCS, reader.numDocs()); |
| reader.close(); |
| |
| addDirThreads.closeDir(); |
| mainDir.close(); |
| } |
| |
| private class AddDirectoriesThreads { |
| Directory addDir; |
| final static int NUM_THREADS = 5; |
| final static int NUM_INIT_DOCS = 100; |
| int numDirs; |
| final Thread[] threads = new Thread[NUM_THREADS]; |
| IndexWriter mainWriter; |
| final List<Throwable> failures = new ArrayList<Throwable>(); |
| IndexReader[] readers; |
| boolean didClose = false; |
| AtomicInteger count = new AtomicInteger(0); |
| AtomicInteger numaddIndexes = new AtomicInteger(0); |
| |
| public AddDirectoriesThreads(int numDirs, IndexWriter mainWriter) throws Throwable { |
| this.numDirs = numDirs; |
| this.mainWriter = mainWriter; |
| addDir = newDirectory(); |
| IndexWriter writer = new IndexWriter(addDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2)); |
| for (int i = 0; i < NUM_INIT_DOCS; i++) { |
| Document doc = DocHelper.createDocument(i, "addindex", 4); |
| writer.addDocument(doc); |
| } |
| |
| writer.close(); |
| |
| readers = new IndexReader[numDirs]; |
| for (int i = 0; i < numDirs; i++) |
| readers[i] = IndexReader.open(addDir, false); |
| } |
| |
| void joinThreads() { |
| for (int i = 0; i < NUM_THREADS; i++) |
| try { |
| threads[i].join(); |
| } catch (InterruptedException ie) { |
| throw new ThreadInterruptedException(ie); |
| } |
| } |
| |
| void close(boolean doWait) throws Throwable { |
| didClose = true; |
| if (doWait) { |
| mainWriter.waitForMerges(); |
| } |
| mainWriter.close(doWait); |
| } |
| |
| void closeDir() throws Throwable { |
| for (int i = 0; i < numDirs; i++) |
| readers[i].close(); |
| addDir.close(); |
| } |
| |
| void handle(Throwable t) { |
| t.printStackTrace(System.out); |
| synchronized (failures) { |
| failures.add(t); |
| } |
| } |
| |
| void launchThreads(final int numIter) { |
| for (int i = 0; i < NUM_THREADS; i++) { |
| threads[i] = new Thread() { |
| @Override |
| public void run() { |
| try { |
| final Directory[] dirs = new Directory[numDirs]; |
| for (int k = 0; k < numDirs; k++) |
| dirs[k] = new MockDirectoryWrapper(random, new RAMDirectory(addDir)); |
| //int j = 0; |
| //while (true) { |
| // System.out.println(Thread.currentThread().getName() + ": iter |
| // j=" + j); |
| for (int x=0; x < numIter; x++) { |
| // only do addIndexes |
| doBody(x, dirs); |
| } |
| //if (numIter > 0 && j == numIter) |
| // break; |
| //doBody(j++, dirs); |
| //doBody(5, dirs); |
| //} |
| } catch (Throwable t) { |
| handle(t); |
| } |
| } |
| }; |
| } |
| for (int i = 0; i < NUM_THREADS; i++) |
| threads[i].start(); |
| } |
| |
| void doBody(int j, Directory[] dirs) throws Throwable { |
| switch (j % 4) { |
| case 0: |
| mainWriter.addIndexes(dirs); |
| mainWriter.optimize(); |
| break; |
| case 1: |
| mainWriter.addIndexes(dirs); |
| numaddIndexes.incrementAndGet(); |
| break; |
| case 2: |
| mainWriter.addIndexes(readers); |
| break; |
| case 3: |
| mainWriter.commit(); |
| } |
| count.addAndGet(dirs.length*NUM_INIT_DOCS); |
| } |
| } |
| |
| public void testIndexWriterReopenSegmentOptimize() throws Exception { |
| doTestIndexWriterReopenSegment(true); |
| } |
| |
| public void testIndexWriterReopenSegment() throws Exception { |
| doTestIndexWriterReopenSegment(false); |
| } |
| |
| /** |
| * Tests creating a segment, then check to insure the segment can be seen via |
| * IW.getReader |
| */ |
| public void doTestIndexWriterReopenSegment(boolean optimize) throws Exception { |
| Directory dir1 = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| writer.setInfoStream(infoStream); |
| IndexReader r1 = writer.getReader(); |
| assertEquals(0, r1.maxDoc()); |
| createIndexNoClose(false, "index1", writer); |
| writer.flush(!optimize, true); |
| |
| IndexReader iwr1 = writer.getReader(); |
| assertEquals(100, iwr1.maxDoc()); |
| |
| IndexReader r2 = writer.getReader(); |
| assertEquals(r2.maxDoc(), 100); |
| // add 100 documents |
| for (int x = 10000; x < 10000 + 100; x++) { |
| Document d = DocHelper.createDocument(x, "index1", 5); |
| writer.addDocument(d); |
| } |
| writer.flush(false, true); |
| // verify the reader was reopened internally |
| IndexReader iwr2 = writer.getReader(); |
| assertTrue(iwr2 != r1); |
| assertEquals(200, iwr2.maxDoc()); |
| // should have flushed out a segment |
| IndexReader r3 = writer.getReader(); |
| assertTrue(r2 != r3); |
| assertEquals(200, r3.maxDoc()); |
| |
| // dec ref the readers rather than close them because |
| // closing flushes changes to the writer |
| r1.close(); |
| iwr1.close(); |
| r2.close(); |
| r3.close(); |
| iwr2.close(); |
| writer.close(); |
| |
| // test whether the changes made it to the directory |
| writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| IndexReader w2r1 = writer.getReader(); |
| // insure the deletes were actually flushed to the directory |
| assertEquals(200, w2r1.maxDoc()); |
| w2r1.close(); |
| writer.close(); |
| |
| dir1.close(); |
| } |
| |
| /* |
| * Delete a document by term and return the doc id |
| * |
| * public static int deleteDocument(Term term, IndexWriter writer) throws |
| * IOException { IndexReader reader = writer.getReader(); TermDocs td = |
| * reader.termDocs(term); int doc = -1; //if (td.next()) { // doc = td.doc(); |
| * //} //writer.deleteDocuments(term); td.close(); return doc; } |
| */ |
| |
| public static void createIndex(Random random, Directory dir1, String indexName, |
| boolean multiSegment) throws IOException { |
| IndexWriter w = new IndexWriter(dir1, LuceneTestCase.newIndexWriterConfig(random, |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setMergePolicy(new LogDocMergePolicy())); |
| for (int i = 0; i < 100; i++) { |
| w.addDocument(DocHelper.createDocument(i, indexName, 4)); |
| if (multiSegment && (i % 10) == 0) { |
| } |
| } |
| if (!multiSegment) { |
| w.optimize(); |
| } |
| w.close(); |
| } |
| |
| public static void createIndexNoClose(boolean multiSegment, String indexName, |
| IndexWriter w) throws IOException { |
| for (int i = 0; i < 100; i++) { |
| w.addDocument(DocHelper.createDocument(i, indexName, 4)); |
| } |
| if (!multiSegment) { |
| w.optimize(); |
| } |
| } |
| |
| private static class MyWarmer extends IndexWriter.IndexReaderWarmer { |
| int warmCount; |
| @Override |
| public void warm(IndexReader reader) throws IOException { |
| warmCount++; |
| } |
| } |
| |
| public void testMergeWarmer() throws Exception { |
| |
| Directory dir1 = newDirectory(); |
| // Enroll warmer |
| MyWarmer warmer = new MyWarmer(); |
| IndexWriter writer = new IndexWriter( |
| dir1, |
| newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). |
| setMaxBufferedDocs(2). |
| setMergedSegmentWarmer(warmer). |
| setMergeScheduler(new ConcurrentMergeScheduler()). |
| setMergePolicy(newLogMergePolicy()) |
| ); |
| writer.setInfoStream(infoStream); |
| |
| // create the index |
| createIndexNoClose(false, "test", writer); |
| |
| // get a reader to put writer into near real-time mode |
| IndexReader r1 = writer.getReader(); |
| |
| ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2); |
| |
| int num = atLeast(100); |
| for (int i = 0; i < num; i++) { |
| writer.addDocument(DocHelper.createDocument(i, "test", 4)); |
| } |
| ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync(); |
| |
| assertTrue(warmer.warmCount > 0); |
| final int count = warmer.warmCount; |
| |
| writer.addDocument(DocHelper.createDocument(17, "test", 4)); |
| writer.optimize(); |
| assertTrue(warmer.warmCount > count); |
| |
| writer.close(); |
| r1.close(); |
| dir1.close(); |
| } |
| |
| public void testAfterCommit() throws Exception { |
| Directory dir1 = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler())); |
| writer.commit(); |
| writer.setInfoStream(infoStream); |
| |
| // create the index |
| createIndexNoClose(false, "test", writer); |
| |
| // get a reader to put writer into near real-time mode |
| IndexReader r1 = writer.getReader(); |
| _TestUtil.checkIndex(dir1); |
| writer.commit(); |
| _TestUtil.checkIndex(dir1); |
| assertEquals(100, r1.numDocs()); |
| |
| for (int i = 0; i < 10; i++) { |
| writer.addDocument(DocHelper.createDocument(i, "test", 4)); |
| } |
| ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).sync(); |
| |
| IndexReader r2 = r1.reopen(); |
| if (r2 != r1) { |
| r1.close(); |
| r1 = r2; |
| } |
| assertEquals(110, r1.numDocs()); |
| writer.close(); |
| r1.close(); |
| dir1.close(); |
| } |
| |
| // Make sure reader remains usable even if IndexWriter closes |
| public void testAfterClose() throws Exception { |
| Directory dir1 = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| writer.setInfoStream(infoStream); |
| |
| // create the index |
| createIndexNoClose(false, "test", writer); |
| |
| IndexReader r = writer.getReader(); |
| writer.close(); |
| |
| _TestUtil.checkIndex(dir1); |
| |
| // reader should remain usable even after IndexWriter is closed: |
| assertEquals(100, r.numDocs()); |
| Query q = new TermQuery(new Term("indexname", "test")); |
| IndexSearcher searcher = newSearcher(r); |
| assertEquals(100, searcher.search(q, 10).totalHits); |
| searcher.close(); |
| try { |
| r.reopen(); |
| fail("failed to hit AlreadyClosedException"); |
| } catch (AlreadyClosedException ace) { |
| // expected |
| } |
| r.close(); |
| dir1.close(); |
| } |
| |
| // Stress test reopen during addIndexes |
| public void testDuringAddIndexes() throws Exception { |
| MockDirectoryWrapper dir1 = newDirectory(); |
| final IndexWriter writer = new IndexWriter( |
| dir1, |
| newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)). |
| setMergePolicy(newLogMergePolicy(2)) |
| ); |
| writer.setInfoStream(infoStream); |
| ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2); |
| |
| // create the index |
| createIndexNoClose(false, "test", writer); |
| writer.commit(); |
| |
| final Directory[] dirs = new Directory[10]; |
| for (int i=0;i<10;i++) { |
| dirs[i] = new MockDirectoryWrapper(random, new RAMDirectory(dir1)); |
| } |
| |
| IndexReader r = writer.getReader(); |
| |
| final int NUM_THREAD = 5; |
| final float SECONDS = 0.5f; |
| |
| final long endTime = (long) (System.currentTimeMillis() + 1000.*SECONDS); |
| final List<Throwable> excs = Collections.synchronizedList(new ArrayList<Throwable>()); |
| |
| final Thread[] threads = new Thread[NUM_THREAD]; |
| for(int i=0;i<NUM_THREAD;i++) { |
| threads[i] = new Thread() { |
| @Override |
| public void run() { |
| do { |
| try { |
| writer.addIndexes(dirs); |
| writer.maybeMerge(); |
| } catch (Throwable t) { |
| excs.add(t); |
| throw new RuntimeException(t); |
| } |
| } while(System.currentTimeMillis() < endTime); |
| } |
| }; |
| threads[i].setDaemon(true); |
| threads[i].start(); |
| } |
| |
| int lastCount = 0; |
| while(System.currentTimeMillis() < endTime) { |
| IndexReader r2 = r.reopen(); |
| if (r2 != r) { |
| r.close(); |
| r = r2; |
| } |
| Query q = new TermQuery(new Term("indexname", "test")); |
| IndexSearcher searcher = newSearcher(r); |
| final int count = searcher.search(q, 10).totalHits; |
| searcher.close(); |
| assertTrue(count >= lastCount); |
| lastCount = count; |
| } |
| |
| for(int i=0;i<NUM_THREAD;i++) { |
| threads[i].join(); |
| } |
| // final check |
| IndexReader r2 = r.reopen(); |
| if (r2 != r) { |
| r.close(); |
| r = r2; |
| } |
| Query q = new TermQuery(new Term("indexname", "test")); |
| IndexSearcher searcher = newSearcher(r); |
| final int count = searcher.search(q, 10).totalHits; |
| searcher.close(); |
| assertTrue(count >= lastCount); |
| |
| assertEquals(0, excs.size()); |
| r.close(); |
| assertEquals(0, dir1.getOpenDeletedFiles().size()); |
| |
| writer.close(); |
| |
| dir1.close(); |
| } |
| |
| // Stress test reopen during add/delete |
| public void testDuringAddDelete() throws Exception { |
| Directory dir1 = newDirectory(); |
| final IndexWriter writer = new IndexWriter( |
| dir1, |
| newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). |
| setMergePolicy(newLogMergePolicy(2)) |
| ); |
| writer.setInfoStream(infoStream); |
| ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(2); |
| |
| // create the index |
| createIndexNoClose(false, "test", writer); |
| writer.commit(); |
| |
| IndexReader r = writer.getReader(); |
| |
| final int NUM_THREAD = 5; |
| final float SECONDS = 0.5f; |
| |
| final long endTime = (long) (System.currentTimeMillis() + 1000.*SECONDS); |
| final List<Throwable> excs = Collections.synchronizedList(new ArrayList<Throwable>()); |
| |
| final Thread[] threads = new Thread[NUM_THREAD]; |
| for(int i=0;i<NUM_THREAD;i++) { |
| threads[i] = new Thread() { |
| final Random r = new Random(random.nextLong()); |
| |
| @Override |
| public void run() { |
| int count = 0; |
| do { |
| try { |
| for(int docUpto=0;docUpto<10;docUpto++) { |
| writer.addDocument(DocHelper.createDocument(10*count+docUpto, "test", 4)); |
| } |
| count++; |
| final int limit = count*10; |
| for(int delUpto=0;delUpto<5;delUpto++) { |
| int x = r.nextInt(limit); |
| writer.deleteDocuments(new Term("field3", "b"+x)); |
| } |
| } catch (Throwable t) { |
| excs.add(t); |
| throw new RuntimeException(t); |
| } |
| } while(System.currentTimeMillis() < endTime); |
| } |
| }; |
| threads[i].setDaemon(true); |
| threads[i].start(); |
| } |
| |
| int sum = 0; |
| while(System.currentTimeMillis() < endTime) { |
| IndexReader r2 = r.reopen(); |
| if (r2 != r) { |
| r.close(); |
| r = r2; |
| } |
| Query q = new TermQuery(new Term("indexname", "test")); |
| IndexSearcher searcher = newSearcher(r); |
| sum += searcher.search(q, 10).totalHits; |
| searcher.close(); |
| } |
| |
| for(int i=0;i<NUM_THREAD;i++) { |
| threads[i].join(); |
| } |
| // at least search once |
| IndexReader r2 = r.reopen(); |
| if (r2 != r) { |
| r.close(); |
| r = r2; |
| } |
| Query q = new TermQuery(new Term("indexname", "test")); |
| IndexSearcher searcher = newSearcher(r); |
| sum += searcher.search(q, 10).totalHits; |
| searcher.close(); |
| assertTrue("no documents found at all", sum > 0); |
| |
| assertEquals(0, excs.size()); |
| writer.close(); |
| |
| r.close(); |
| dir1.close(); |
| } |
| |
| public void testExpungeDeletes() throws Throwable { |
| Directory dir = newDirectory(); |
| final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); |
| Document doc = new Document(); |
| doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); |
| Field id = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED); |
| doc.add(id); |
| id.setValue("0"); |
| w.addDocument(doc); |
| id.setValue("1"); |
| w.addDocument(doc); |
| w.deleteDocuments(new Term("id", "0")); |
| |
| IndexReader r = w.getReader(); |
| w.expungeDeletes(); |
| w.close(); |
| r.close(); |
| r = IndexReader.open(dir, true); |
| assertEquals(1, r.numDocs()); |
| assertFalse(r.hasDeletions()); |
| r.close(); |
| dir.close(); |
| } |
| |
| public void testDeletesNumDocs() throws Throwable { |
| Directory dir = newDirectory(); |
| final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| Document doc = new Document(); |
| doc.add(newField("field", "a b c", Field.Store.NO, Field.Index.ANALYZED)); |
| Field id = newField("id", "", Field.Store.NO, Field.Index.NOT_ANALYZED); |
| doc.add(id); |
| id.setValue("0"); |
| w.addDocument(doc); |
| id.setValue("1"); |
| w.addDocument(doc); |
| IndexReader r = w.getReader(); |
| assertEquals(2, r.numDocs()); |
| r.close(); |
| |
| w.deleteDocuments(new Term("id", "0")); |
| r = w.getReader(); |
| assertEquals(1, r.numDocs()); |
| r.close(); |
| |
| w.deleteDocuments(new Term("id", "1")); |
| r = w.getReader(); |
| assertEquals(0, r.numDocs()); |
| r.close(); |
| |
| w.close(); |
| dir.close(); |
| } |
| |
| public void testEmptyIndex() throws Exception { |
| // Ensures that getReader works on an empty index, which hasn't been committed yet. |
| Directory dir = newDirectory(); |
| IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); |
| IndexReader r = w.getReader(); |
| assertEquals(0, r.numDocs()); |
| r.close(); |
| w.close(); |
| dir.close(); |
| } |
| |
| public void testSegmentWarmer() throws Exception { |
| Directory dir = newDirectory(); |
| final AtomicBoolean didWarm = new AtomicBoolean(); |
| IndexWriter w = new IndexWriter( |
| dir, |
| newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)). |
| setMaxBufferedDocs(2). |
| setReaderPooling(true). |
| setMergedSegmentWarmer(new IndexWriter.IndexReaderWarmer() { |
| @Override |
| public void warm(IndexReader r) throws IOException { |
| IndexSearcher s = newSearcher(r); |
| TopDocs hits = s.search(new TermQuery(new Term("foo", "bar")), 10); |
| assertEquals(20, hits.totalHits); |
| didWarm.set(true); |
| s.close(); |
| } |
| }). |
| setMergePolicy(newLogMergePolicy(10)) |
| ); |
| |
| Document doc = new Document(); |
| doc.add(newField("foo", "bar", Field.Store.YES, Field.Index.NOT_ANALYZED)); |
| for(int i=0;i<20;i++) { |
| w.addDocument(doc); |
| } |
| w.waitForMerges(); |
| w.close(); |
| dir.close(); |
| assertTrue(didWarm.get()); |
| } |
| |
| public void testNoTermsIndex() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) |
| .setReaderTermsIndexDivisor(-1)); |
| Document doc = new Document(); |
| doc.add(new Field("f", "val", Store.NO, Index.ANALYZED)); |
| w.addDocument(doc); |
| IndexReader r = IndexReader.open(w, true); |
| try { |
| r.termDocs(new Term("f", "val")); |
| fail("should have failed to seek since terms index was not loaded"); |
| } catch (IllegalStateException e) { |
| // expected - we didn't load the term index |
| } finally { |
| r.close(); |
| w.close(); |
| dir.close(); |
| } |
| } |
| |
| } |