| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.lucene.index; |
| |
| |
| import java.io.FileNotFoundException; |
| import java.io.IOException; |
| import java.nio.file.NoSuchFileException; |
| import java.nio.file.Path; |
| import java.util.Arrays; |
| import java.util.Collection; |
| import java.util.HashSet; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Random; |
| import java.util.Set; |
| import java.util.TreeSet; |
| |
| import org.apache.lucene.analysis.MockAnalyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.document.FieldType; |
| import org.apache.lucene.document.StoredField; |
| import org.apache.lucene.document.StringField; |
| import org.apache.lucene.document.TextField; |
| import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| import org.apache.lucene.search.DocIdSetIterator; |
| import org.apache.lucene.store.BaseDirectoryWrapper; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.util.Bits; |
| import org.apache.lucene.util.BytesRef; |
| import org.apache.lucene.util.IOUtils; |
| import org.apache.lucene.util.LuceneTestCase; |
| import org.apache.lucene.util.TestUtil; |
| import org.junit.Assume; |
| |
| import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; |
| |
| @LuceneTestCase.SuppressCodecs("SimpleText") |
| public class TestDirectoryReader extends LuceneTestCase { |
| |
| public void testDocument() throws IOException { |
| SegmentReader [] readers = new SegmentReader[2]; |
| Directory dir = newDirectory(); |
| Document doc1 = new Document(); |
| Document doc2 = new Document(); |
| DocHelper.setupDoc(doc1); |
| DocHelper.setupDoc(doc2); |
| DocHelper.writeDoc(random(), dir, doc1); |
| DocHelper.writeDoc(random(), dir, doc2); |
| DirectoryReader reader = DirectoryReader.open(dir); |
| assertTrue(reader != null); |
| assertTrue(reader instanceof StandardDirectoryReader); |
| |
| Document newDoc1 = reader.document(0); |
| assertTrue(newDoc1 != null); |
| assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size()); |
| Document newDoc2 = reader.document(1); |
| assertTrue(newDoc2 != null); |
| assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size()); |
| Terms vector = reader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY); |
| assertNotNull(vector); |
| |
| reader.close(); |
| if (readers[0] != null) readers[0].close(); |
| if (readers[1] != null) readers[1].close(); |
| dir.close(); |
| } |
| |
| public void testMultiTermDocs() throws IOException { |
| Directory ramDir1=newDirectory(); |
| addDoc(random(), ramDir1, "test foo", true); |
| Directory ramDir2=newDirectory(); |
| addDoc(random(), ramDir2, "test blah", true); |
| Directory ramDir3=newDirectory(); |
| addDoc(random(), ramDir3, "test wow", true); |
| |
| IndexReader[] readers1 = new IndexReader[]{DirectoryReader.open(ramDir1), DirectoryReader.open(ramDir3)}; |
| IndexReader[] readers2 = new IndexReader[]{DirectoryReader.open(ramDir1), DirectoryReader.open(ramDir2), DirectoryReader.open(ramDir3)}; |
| MultiReader mr2 = new MultiReader(readers1); |
| MultiReader mr3 = new MultiReader(readers2); |
| |
| // test mixing up TermDocs and TermEnums from different readers. |
| TermsEnum te2 = MultiTerms.getTerms(mr2, "body").iterator(); |
| te2.seekCeil(new BytesRef("wow")); |
| PostingsEnum td = TestUtil.docs(random(), mr2, |
| "body", |
| te2.term(), |
| null, |
| 0); |
| |
| TermsEnum te3 = MultiTerms.getTerms(mr3, "body").iterator(); |
| te3.seekCeil(new BytesRef("wow")); |
| td = TestUtil.docs(random(), te3, |
| td, |
| 0); |
| |
| int ret = 0; |
| |
| // This should blow up if we forget to check that the TermEnum is from the same |
| // reader as the TermDocs. |
| while (td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) ret += td.docID(); |
| |
| // really a dummy assert to ensure that we got some docs and to ensure that |
| // nothing is eliminated by hotspot |
| assertTrue(ret > 0); |
| readers1[0].close(); |
| readers1[1].close(); |
| readers2[0].close(); |
| readers2[1].close(); |
| readers2[2].close(); |
| ramDir1.close(); |
| ramDir2.close(); |
| ramDir3.close(); |
| } |
| |
| private void addDoc(Random random, Directory ramDir1, String s, boolean create) throws IOException { |
| IndexWriter iw = new IndexWriter(ramDir1, newIndexWriterConfig(new MockAnalyzer(random)) |
| .setOpenMode(create ? OpenMode.CREATE : OpenMode.APPEND)); |
| Document doc = new Document(); |
| doc.add(newTextField("body", s, Field.Store.NO)); |
| iw.addDocument(doc); |
| iw.close(); |
| } |
| |
| public void testIsCurrent() throws Exception { |
| Directory d = newDirectory(); |
| IndexWriter writer = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random()))); |
| addDocumentWithFields(writer); |
| writer.close(); |
| // set up reader: |
| DirectoryReader reader = DirectoryReader.open(d); |
| assertTrue(reader.isCurrent()); |
| // modify index by adding another document: |
| writer = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.APPEND)); |
| addDocumentWithFields(writer); |
| writer.close(); |
| assertFalse(reader.isCurrent()); |
| // re-create index: |
| writer = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.CREATE)); |
| addDocumentWithFields(writer); |
| writer.close(); |
| assertFalse(reader.isCurrent()); |
| reader.close(); |
| d.close(); |
| } |
| |
| /** |
| * Tests the IndexReader.getFieldNames implementation |
| * @throws Exception on error |
| */ |
| public void testGetFieldNames() throws Exception { |
| Directory d = newDirectory(); |
| // set up writer |
| IndexWriter writer = new IndexWriter( |
| d, |
| newIndexWriterConfig(new MockAnalyzer(random())) |
| ); |
| |
| Document doc = new Document(); |
| |
| FieldType customType3 = new FieldType(); |
| customType3.setStored(true); |
| |
| doc.add(new StringField("keyword", "test1", Field.Store.YES)); |
| doc.add(new TextField("text", "test1", Field.Store.YES)); |
| doc.add(new Field("unindexed", "test1", customType3)); |
| doc.add(new TextField("unstored","test1", Field.Store.NO)); |
| writer.addDocument(doc); |
| |
| writer.close(); |
| // set up reader |
| DirectoryReader reader = DirectoryReader.open(d); |
| FieldInfos fieldInfos = FieldInfos.getMergedFieldInfos(reader); |
| assertNotNull(fieldInfos.fieldInfo("keyword")); |
| assertNotNull(fieldInfos.fieldInfo("text")); |
| assertNotNull(fieldInfos.fieldInfo("unindexed")); |
| assertNotNull(fieldInfos.fieldInfo("unstored")); |
| reader.close(); |
| // add more documents |
| writer = new IndexWriter( |
| d, |
| newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.APPEND) |
| .setMergePolicy(newLogMergePolicy()) |
| ); |
| // want to get some more segments here |
| int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); |
| for (int i = 0; i < 5*mergeFactor; i++) { |
| doc = new Document(); |
| doc.add(new StringField("keyword", "test1", Field.Store.YES)); |
| doc.add(new TextField("text", "test1", Field.Store.YES)); |
| doc.add(new Field("unindexed", "test1", customType3)); |
| doc.add(new TextField("unstored","test1", Field.Store.NO)); |
| writer.addDocument(doc); |
| } |
| // new fields are in some different segments (we hope) |
| for (int i = 0; i < 5*mergeFactor; i++) { |
| doc = new Document(); |
| doc.add(new StringField("keyword2", "test1", Field.Store.YES)); |
| doc.add(new TextField("text2", "test1", Field.Store.YES)); |
| doc.add(new Field("unindexed2", "test1", customType3)); |
| doc.add(new TextField("unstored2","test1", Field.Store.NO)); |
| writer.addDocument(doc); |
| } |
| // new termvector fields |
| |
| FieldType customType5 = new FieldType(TextField.TYPE_STORED); |
| customType5.setStoreTermVectors(true); |
| FieldType customType6 = new FieldType(TextField.TYPE_STORED); |
| customType6.setStoreTermVectors(true); |
| customType6.setStoreTermVectorOffsets(true); |
| FieldType customType7 = new FieldType(TextField.TYPE_STORED); |
| customType7.setStoreTermVectors(true); |
| customType7.setStoreTermVectorPositions(true); |
| FieldType customType8 = new FieldType(TextField.TYPE_STORED); |
| customType8.setStoreTermVectors(true); |
| customType8.setStoreTermVectorOffsets(true); |
| customType8.setStoreTermVectorPositions(true); |
| |
| for (int i = 0; i < 5*mergeFactor; i++) { |
| doc = new Document(); |
| doc.add(new TextField("tvnot", "tvnot", Field.Store.YES)); |
| doc.add(new Field("termvector", "termvector", customType5)); |
| doc.add(new Field("tvoffset", "tvoffset", customType6)); |
| doc.add(new Field("tvposition", "tvposition", customType7)); |
| doc.add(new Field("tvpositionoffset", "tvpositionoffset", customType8)); |
| writer.addDocument(doc); |
| } |
| |
| writer.close(); |
| |
| // verify fields again |
| reader = DirectoryReader.open(d); |
| fieldInfos = FieldInfos.getMergedFieldInfos(reader); |
| |
| Collection<String> allFieldNames = new HashSet<>(); |
| Collection<String> indexedFieldNames = new HashSet<>(); |
| Collection<String> notIndexedFieldNames = new HashSet<>(); |
| Collection<String> tvFieldNames = new HashSet<>(); |
| |
| for(FieldInfo fieldInfo : fieldInfos) { |
| final String name = fieldInfo.name; |
| allFieldNames.add(name); |
| if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { |
| indexedFieldNames.add(name); |
| } else { |
| notIndexedFieldNames.add(name); |
| } |
| if (fieldInfo.hasVectors()) { |
| tvFieldNames.add(name); |
| } |
| } |
| |
| assertTrue(allFieldNames.contains("keyword")); |
| assertTrue(allFieldNames.contains("text")); |
| assertTrue(allFieldNames.contains("unindexed")); |
| assertTrue(allFieldNames.contains("unstored")); |
| assertTrue(allFieldNames.contains("keyword2")); |
| assertTrue(allFieldNames.contains("text2")); |
| assertTrue(allFieldNames.contains("unindexed2")); |
| assertTrue(allFieldNames.contains("unstored2")); |
| assertTrue(allFieldNames.contains("tvnot")); |
| assertTrue(allFieldNames.contains("termvector")); |
| assertTrue(allFieldNames.contains("tvposition")); |
| assertTrue(allFieldNames.contains("tvoffset")); |
| assertTrue(allFieldNames.contains("tvpositionoffset")); |
| |
| // verify that only indexed fields were returned |
| assertEquals(11, indexedFieldNames.size()); // 6 original + the 5 termvector fields |
| assertTrue(indexedFieldNames.contains("keyword")); |
| assertTrue(indexedFieldNames.contains("text")); |
| assertTrue(indexedFieldNames.contains("unstored")); |
| assertTrue(indexedFieldNames.contains("keyword2")); |
| assertTrue(indexedFieldNames.contains("text2")); |
| assertTrue(indexedFieldNames.contains("unstored2")); |
| assertTrue(indexedFieldNames.contains("tvnot")); |
| assertTrue(indexedFieldNames.contains("termvector")); |
| assertTrue(indexedFieldNames.contains("tvposition")); |
| assertTrue(indexedFieldNames.contains("tvoffset")); |
| assertTrue(indexedFieldNames.contains("tvpositionoffset")); |
| |
| // verify that only unindexed fields were returned |
| assertEquals(2, notIndexedFieldNames.size()); // the following fields |
| assertTrue(notIndexedFieldNames.contains("unindexed")); |
| assertTrue(notIndexedFieldNames.contains("unindexed2")); |
| |
| // verify index term vector fields |
| assertEquals(tvFieldNames.toString(), 4, tvFieldNames.size()); // 4 field has term vector only |
| assertTrue(tvFieldNames.contains("termvector")); |
| |
| reader.close(); |
| d.close(); |
| } |
| |
| public void testTermVectors() throws Exception { |
| Directory d = newDirectory(); |
| // set up writer |
| IndexWriter writer = new IndexWriter( |
| d, |
| newIndexWriterConfig(new MockAnalyzer(random())) |
| .setMergePolicy(newLogMergePolicy()) |
| ); |
| // want to get some more segments here |
| // new termvector fields |
| int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor(); |
| FieldType customType5 = new FieldType(TextField.TYPE_STORED); |
| customType5.setStoreTermVectors(true); |
| FieldType customType6 = new FieldType(TextField.TYPE_STORED); |
| customType6.setStoreTermVectors(true); |
| customType6.setStoreTermVectorOffsets(true); |
| FieldType customType7 = new FieldType(TextField.TYPE_STORED); |
| customType7.setStoreTermVectors(true); |
| customType7.setStoreTermVectorPositions(true); |
| FieldType customType8 = new FieldType(TextField.TYPE_STORED); |
| customType8.setStoreTermVectors(true); |
| customType8.setStoreTermVectorOffsets(true); |
| customType8.setStoreTermVectorPositions(true); |
| for (int i = 0; i < 5 * mergeFactor; i++) { |
| Document doc = new Document(); |
| doc.add(new TextField("tvnot", "one two two three three three", Field.Store.YES)); |
| doc.add(new Field("termvector", "one two two three three three", customType5)); |
| doc.add(new Field("tvoffset", "one two two three three three", customType6)); |
| doc.add(new Field("tvposition", "one two two three three three", customType7)); |
| doc.add(new Field("tvpositionoffset", "one two two three three three", customType8)); |
| |
| writer.addDocument(doc); |
| } |
| writer.close(); |
| d.close(); |
| } |
| |
| void assertTermDocsCount(String msg, |
| IndexReader reader, |
| Term term, |
| int expected) |
| throws IOException { |
| PostingsEnum tdocs = TestUtil.docs(random(), reader, |
| term.field(), |
| new BytesRef(term.text()), |
| null, |
| 0); |
| int count = 0; |
| if (tdocs != null) { |
| while(tdocs.nextDoc()!= DocIdSetIterator.NO_MORE_DOCS) { |
| count++; |
| } |
| } |
| assertEquals(msg + ", count mismatch", expected, count); |
| } |
| |
| public void testBinaryFields() throws IOException { |
| Directory dir = newDirectory(); |
| byte[] bin = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; |
| |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setMergePolicy(newLogMergePolicy())); |
| |
| for (int i = 0; i < 10; i++) { |
| addDoc(writer, "document number " + (i + 1)); |
| addDocumentWithFields(writer); |
| addDocumentWithDifferentFields(writer); |
| addDocumentWithTermVectorFields(writer); |
| } |
| writer.close(); |
| writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.APPEND) |
| .setMergePolicy(newLogMergePolicy())); |
| Document doc = new Document(); |
| doc.add(new StoredField("bin1", bin)); |
| doc.add(new TextField("junk", "junk text", Field.Store.NO)); |
| writer.addDocument(doc); |
| writer.close(); |
| DirectoryReader reader = DirectoryReader.open(dir); |
| Document doc2 = reader.document(reader.maxDoc() - 1); |
| IndexableField[] fields = doc2.getFields("bin1"); |
| assertNotNull(fields); |
| assertEquals(1, fields.length); |
| IndexableField b1 = fields[0]; |
| assertTrue(b1.binaryValue() != null); |
| BytesRef bytesRef = b1.binaryValue(); |
| assertEquals(bin.length, bytesRef.length); |
| for (int i = 0; i < bin.length; i++) { |
| assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]); |
| } |
| reader.close(); |
| // force merge |
| |
| |
| writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.APPEND) |
| .setMergePolicy(newLogMergePolicy())); |
| writer.forceMerge(1); |
| writer.close(); |
| reader = DirectoryReader.open(dir); |
| doc2 = reader.document(reader.maxDoc() - 1); |
| fields = doc2.getFields("bin1"); |
| assertNotNull(fields); |
| assertEquals(1, fields.length); |
| b1 = fields[0]; |
| assertTrue(b1.binaryValue() != null); |
| bytesRef = b1.binaryValue(); |
| assertEquals(bin.length, bytesRef.length); |
| for (int i = 0; i < bin.length; i++) { |
| assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]); |
| } |
| reader.close(); |
| dir.close(); |
| } |
| |
| /* ??? public void testOpenEmptyDirectory() throws IOException{ |
| String dirName = "test.empty"; |
| File fileDirName = new File(dirName); |
| if (!fileDirName.exists()) { |
| fileDirName.mkdir(); |
| } |
| try { |
| DirectoryReader.open(fileDirName); |
| fail("opening DirectoryReader on empty directory failed to produce FileNotFoundException/NoSuchFileException"); |
| } catch (FileNotFoundException | NoSuchFileException e) { |
| // GOOD |
| } |
| rmDir(fileDirName); |
| }*/ |
| |
| public void testFilesOpenClose() throws IOException { |
| // Create initial data set |
| Path dirFile = createTempDir("TestIndexReader.testFilesOpenClose"); |
| Directory dir = newFSDirectory(dirFile); |
| |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| addDoc(writer, "test"); |
| writer.close(); |
| dir.close(); |
| |
| // Try to erase the data - this ensures that the writer closed all files |
| IOUtils.rm(dirFile); |
| dir = newFSDirectory(dirFile); |
| |
| // Now create the data set again, just as before |
| writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.CREATE)); |
| addDoc(writer, "test"); |
| writer.close(); |
| dir.close(); |
| |
| // Now open existing directory and test that reader closes all files |
| dir = newFSDirectory(dirFile); |
| DirectoryReader reader1 = DirectoryReader.open(dir); |
| reader1.close(); |
| dir.close(); |
| |
| // The following will fail if reader did not close |
| // all files |
| IOUtils.rm(dirFile); |
| } |
| |
| public void testOpenReaderAfterDelete() throws IOException { |
| Path dirFile = createTempDir("deletetest"); |
| Directory dir = newFSDirectory(dirFile); |
| if (dir instanceof BaseDirectoryWrapper) { |
| ((BaseDirectoryWrapper)dir).setCheckIndexOnClose(false); // we will hit NoSuchFileException in MDW since we nuked it! |
| } |
| expectThrowsAnyOf(Arrays.asList(FileNotFoundException.class, NoSuchFileException.class), |
| () -> DirectoryReader.open(dir) |
| ); |
| |
| IOUtils.rm(dirFile); |
| |
| // Make sure we still get a CorruptIndexException (not NPE): |
| expectThrowsAnyOf(Arrays.asList(FileNotFoundException.class, NoSuchFileException.class), |
| () -> DirectoryReader.open(dir) |
| ); |
| |
| dir.close(); |
| } |
| |
| static void addDocumentWithFields(IndexWriter writer) throws IOException |
| { |
| Document doc = new Document(); |
| |
| FieldType customType3 = new FieldType(); |
| customType3.setStored(true); |
| doc.add(newStringField("keyword", "test1", Field.Store.YES)); |
| doc.add(newTextField("text", "test1", Field.Store.YES)); |
| doc.add(newField("unindexed", "test1", customType3)); |
| doc.add(new TextField("unstored","test1", Field.Store.NO)); |
| writer.addDocument(doc); |
| } |
| |
| static void addDocumentWithDifferentFields(IndexWriter writer) throws IOException |
| { |
| Document doc = new Document(); |
| |
| FieldType customType3 = new FieldType(); |
| customType3.setStored(true); |
| doc.add(newStringField("keyword2", "test1", Field.Store.YES)); |
| doc.add(newTextField("text2", "test1", Field.Store.YES)); |
| doc.add(newField("unindexed2", "test1", customType3)); |
| doc.add(new TextField("unstored2","test1", Field.Store.NO)); |
| writer.addDocument(doc); |
| } |
| |
| static void addDocumentWithTermVectorFields(IndexWriter writer) throws IOException |
| { |
| Document doc = new Document(); |
| FieldType customType5 = new FieldType(TextField.TYPE_STORED); |
| customType5.setStoreTermVectors(true); |
| FieldType customType6 = new FieldType(TextField.TYPE_STORED); |
| customType6.setStoreTermVectors(true); |
| customType6.setStoreTermVectorOffsets(true); |
| FieldType customType7 = new FieldType(TextField.TYPE_STORED); |
| customType7.setStoreTermVectors(true); |
| customType7.setStoreTermVectorPositions(true); |
| FieldType customType8 = new FieldType(TextField.TYPE_STORED); |
| customType8.setStoreTermVectors(true); |
| customType8.setStoreTermVectorOffsets(true); |
| customType8.setStoreTermVectorPositions(true); |
| doc.add(newTextField("tvnot", "tvnot", Field.Store.YES)); |
| doc.add(newField("termvector","termvector",customType5)); |
| doc.add(newField("tvoffset","tvoffset", customType6)); |
| doc.add(newField("tvposition","tvposition", customType7)); |
| doc.add(newField("tvpositionoffset","tvpositionoffset", customType8)); |
| |
| writer.addDocument(doc); |
| } |
| |
| static void addDoc(IndexWriter writer, String value) throws IOException { |
| Document doc = new Document(); |
| doc.add(newTextField("content", value, Field.Store.NO)); |
| writer.addDocument(doc); |
| } |
| |
| // TODO: maybe this can reuse the logic of test dueling codecs? |
| public static void assertIndexEquals(DirectoryReader index1, DirectoryReader index2) throws IOException { |
| assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs()); |
| assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc()); |
| assertEquals("Only one IndexReader has deletions.", index1.hasDeletions(), index2.hasDeletions()); |
| assertEquals("Single segment test differs.", index1.leaves().size() == 1, index2.leaves().size() == 1); |
| |
| // check field names |
| FieldInfos fieldInfos1 = FieldInfos.getMergedFieldInfos(index1); |
| FieldInfos fieldInfos2 = FieldInfos.getMergedFieldInfos(index2); |
| assertEquals("IndexReaders have different numbers of fields.", fieldInfos1.size(), fieldInfos2.size()); |
| final int numFields = fieldInfos1.size(); |
| for(int fieldID=0;fieldID<numFields;fieldID++) { |
| final FieldInfo fieldInfo1 = fieldInfos1.fieldInfo(fieldID); |
| final FieldInfo fieldInfo2 = fieldInfos2.fieldInfo(fieldID); |
| assertEquals("Different field names.", fieldInfo1.name, fieldInfo2.name); |
| } |
| |
| // check norms |
| for(FieldInfo fieldInfo : fieldInfos1) { |
| String curField = fieldInfo.name; |
| NumericDocValues norms1 = MultiDocValues.getNormValues(index1, curField); |
| NumericDocValues norms2 = MultiDocValues.getNormValues(index2, curField); |
| if (norms1 != null && norms2 != null) { |
| // todo: generalize this (like TestDuelingCodecs assert) |
| while (true) { |
| int docID = norms1.nextDoc(); |
| assertEquals(docID, norms2.nextDoc()); |
| if (docID == NO_MORE_DOCS) { |
| break; |
| } |
| assertEquals("Norm different for doc " + docID + " and field '" + curField + "'.", norms1.longValue(), norms2.longValue()); |
| } |
| } else { |
| assertNull(norms1); |
| assertNull(norms2); |
| } |
| } |
| |
| // check deletions |
| final Bits liveDocs1 = MultiBits.getLiveDocs(index1); |
| final Bits liveDocs2 = MultiBits.getLiveDocs(index2); |
| for (int i = 0; i < index1.maxDoc(); i++) { |
| assertEquals("Doc " + i + " only deleted in one index.", |
| liveDocs1 == null || !liveDocs1.get(i), |
| liveDocs2 == null || !liveDocs2.get(i)); |
| } |
| |
| // check stored fields |
| for (int i = 0; i < index1.maxDoc(); i++) { |
| if (liveDocs1 == null || liveDocs1.get(i)) { |
| Document doc1 = index1.document(i); |
| Document doc2 = index2.document(i); |
| List<IndexableField> field1 = doc1.getFields(); |
| List<IndexableField> field2 = doc2.getFields(); |
| assertEquals("Different numbers of fields for doc " + i + ".", field1.size(), field2.size()); |
| Iterator<IndexableField> itField1 = field1.iterator(); |
| Iterator<IndexableField> itField2 = field2.iterator(); |
| while (itField1.hasNext()) { |
| Field curField1 = (Field) itField1.next(); |
| Field curField2 = (Field) itField2.next(); |
| assertEquals("Different fields names for doc " + i + ".", curField1.name(), curField2.name()); |
| assertEquals("Different field values for doc " + i + ".", curField1.stringValue(), curField2.stringValue()); |
| } |
| } |
| } |
| |
| // check dictionary and posting lists |
| TreeSet<String> fields1 = new TreeSet<>(FieldInfos.getIndexedFields(index1)); |
| TreeSet<String> fields2 = new TreeSet<>(FieldInfos.getIndexedFields(index2)); |
| Iterator<String> fenum2 = fields2.iterator(); |
| for (String field1 : fields1) { |
| assertEquals("Different fields", field1, fenum2.next()); |
| Terms terms1 = MultiTerms.getTerms(index1, field1); |
| if (terms1 == null) { |
| assertNull(MultiTerms.getTerms(index2, field1)); |
| continue; |
| } |
| TermsEnum enum1 = terms1.iterator(); |
| |
| Terms terms2 = MultiTerms.getTerms(index2, field1); |
| assertNotNull(terms2); |
| TermsEnum enum2 = terms2.iterator(); |
| |
| while(enum1.next() != null) { |
| assertEquals("Different terms", enum1.term(), enum2.next()); |
| PostingsEnum tp1 = enum1.postings(null, PostingsEnum.ALL); |
| PostingsEnum tp2 = enum2.postings(null, PostingsEnum.ALL); |
| |
| while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { |
| assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); |
| assertEquals("Different doc id in postinglist of term " + enum1.term() + ".", tp1.docID(), tp2.docID()); |
| assertEquals("Different term frequence in postinglist of term " + enum1.term() + ".", tp1.freq(), tp2.freq()); |
| for (int i = 0; i < tp1.freq(); i++) { |
| assertEquals("Different positions in postinglist of term " + enum1.term() + ".", tp1.nextPosition(), tp2.nextPosition()); |
| } |
| } |
| } |
| } |
| assertFalse(fenum2.hasNext()); |
| } |
| |
| public void testGetIndexCommit() throws IOException { |
| |
| Directory d = newDirectory(); |
| |
| // set up writer |
| IndexWriter writer = new IndexWriter( |
| d, |
| newIndexWriterConfig(new MockAnalyzer(random())) |
| .setMaxBufferedDocs(2) |
| .setMergePolicy(newLogMergePolicy(10)) |
| ); |
| for(int i=0;i<27;i++) |
| addDocumentWithFields(writer); |
| writer.close(); |
| |
| SegmentInfos sis = SegmentInfos.readLatestCommit(d); |
| DirectoryReader r = DirectoryReader.open(d); |
| IndexCommit c = r.getIndexCommit(); |
| |
| assertEquals(sis.getSegmentsFileName(), c.getSegmentsFileName()); |
| |
| assertTrue(c.equals(r.getIndexCommit())); |
| |
| // Change the index |
| writer = new IndexWriter( |
| d, |
| newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.APPEND) |
| .setMaxBufferedDocs(2) |
| .setMergePolicy(newLogMergePolicy(10)) |
| ); |
| for(int i=0;i<7;i++) |
| addDocumentWithFields(writer); |
| writer.close(); |
| |
| DirectoryReader r2 = DirectoryReader.openIfChanged(r); |
| assertNotNull(r2); |
| assertFalse(c.equals(r2.getIndexCommit())); |
| assertFalse(r2.getIndexCommit().getSegmentCount() == 1); |
| r2.close(); |
| |
| writer = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setOpenMode(OpenMode.APPEND)); |
| writer.forceMerge(1); |
| writer.close(); |
| |
| r2 = DirectoryReader.openIfChanged(r); |
| assertNotNull(r2); |
| assertNull(DirectoryReader.openIfChanged(r2)); |
| assertEquals(1, r2.getIndexCommit().getSegmentCount()); |
| |
| r.close(); |
| r2.close(); |
| d.close(); |
| } |
| |
| static Document createDocument(String id) { |
| Document doc = new Document(); |
| FieldType customType = new FieldType(TextField.TYPE_STORED); |
| customType.setTokenized(false); |
| customType.setOmitNorms(true); |
| |
| doc.add(newField("id", id, customType)); |
| return doc; |
| } |
| |
| // LUCENE-1468 -- make sure on attempting to open an |
| // DirectoryReader on a non-existent directory, you get a |
| // good exception |
| public void testNoDir() throws Throwable { |
| Path tempDir = createTempDir("doesnotexist"); |
| Directory dir = newFSDirectory(tempDir); |
| expectThrows(IndexNotFoundException.class, () -> { |
| DirectoryReader.open(dir); |
| }); |
| dir.close(); |
| } |
| |
| // LUCENE-1509 |
| public void testNoDupCommitFileNames() throws Throwable { |
| |
| Directory dir = newDirectory(); |
| |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setMaxBufferedDocs(2)); |
| writer.addDocument(createDocument("a")); |
| writer.addDocument(createDocument("a")); |
| writer.addDocument(createDocument("a")); |
| writer.close(); |
| |
| Collection<IndexCommit> commits = DirectoryReader.listCommits(dir); |
| for (final IndexCommit commit : commits) { |
| Collection<String> files = commit.getFileNames(); |
| HashSet<String> seen = new HashSet<>(); |
| for (final String fileName : files) { |
| assertTrue("file " + fileName + " was duplicated", !seen.contains(fileName)); |
| seen.add(fileName); |
| } |
| } |
| |
| dir.close(); |
| } |
| |
| // LUCENE-1586: getUniqueTermCount |
| public void testUniqueTermCount() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| Document doc = new Document(); |
| doc.add(newTextField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", Field.Store.NO)); |
| doc.add(newTextField("number", "0 1 2 3 4 5 6 7 8 9", Field.Store.NO)); |
| writer.addDocument(doc); |
| writer.addDocument(doc); |
| writer.commit(); |
| |
| DirectoryReader r = DirectoryReader.open(dir); |
| LeafReader r1 = getOnlyLeafReader(r); |
| assertEquals(26, r1.terms("field").size()); |
| assertEquals(10, r1.terms("number").size()); |
| writer.addDocument(doc); |
| writer.commit(); |
| DirectoryReader r2 = DirectoryReader.openIfChanged(r); |
| assertNotNull(r2); |
| r.close(); |
| |
| for(LeafReaderContext s : r2.leaves()) { |
| assertEquals(26, s.reader().terms("field").size()); |
| assertEquals(10, s.reader().terms("number").size()); |
| } |
| r2.close(); |
| writer.close(); |
| dir.close(); |
| } |
| |
| // LUCENE-2046 |
| public void testPrepareCommitIsCurrent() throws Throwable { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| writer.commit(); |
| Document doc = new Document(); |
| writer.addDocument(doc); |
| DirectoryReader r = DirectoryReader.open(dir); |
| assertTrue(r.isCurrent()); |
| writer.addDocument(doc); |
| writer.prepareCommit(); |
| assertTrue(r.isCurrent()); |
| DirectoryReader r2 = DirectoryReader.openIfChanged(r); |
| assertNull(r2); |
| writer.commit(); |
| assertFalse(r.isCurrent()); |
| writer.close(); |
| r.close(); |
| dir.close(); |
| } |
| |
| // LUCENE-2753 |
| public void testListCommits() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(null) |
| .setIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()))); |
| SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy(); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| sdp.snapshot(); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| sdp.snapshot(); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| sdp.snapshot(); |
| writer.close(); |
| long currentGen = 0; |
| for (IndexCommit ic : DirectoryReader.listCommits(dir)) { |
| assertTrue("currentGen=" + currentGen + " commitGen=" + ic.getGeneration(), currentGen < ic.getGeneration()); |
| currentGen = ic.getGeneration(); |
| } |
| dir.close(); |
| } |
| |
| // Make sure totalTermFreq works correctly in the terms |
| // dict cache |
| public void testTotalTermFreqCached() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| Document d = new Document(); |
| d.add(newTextField("f", "a a b", Field.Store.NO)); |
| writer.addDocument(d); |
| DirectoryReader r = writer.getReader(); |
| writer.close(); |
| try { |
| // Make sure codec impls totalTermFreq (eg PreFlex doesn't) |
| Assume.assumeTrue(r.totalTermFreq(new Term("f", new BytesRef("b"))) != -1); |
| assertEquals(1, r.totalTermFreq(new Term("f", new BytesRef("b")))); |
| assertEquals(2, r.totalTermFreq(new Term("f", new BytesRef("a")))); |
| assertEquals(1, r.totalTermFreq(new Term("f", new BytesRef("b")))); |
| } finally { |
| r.close(); |
| dir.close(); |
| } |
| } |
| |
| public void testGetSumDocFreq() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| Document d = new Document(); |
| d.add(newTextField("f", "a", Field.Store.NO)); |
| writer.addDocument(d); |
| d = new Document(); |
| d.add(newTextField("f", "b", Field.Store.NO)); |
| writer.addDocument(d); |
| DirectoryReader r = writer.getReader(); |
| writer.close(); |
| try { |
| // Make sure codec impls getSumDocFreq (eg PreFlex doesn't) |
| Assume.assumeTrue(r.getSumDocFreq("f") != -1); |
| assertEquals(2, r.getSumDocFreq("f")); |
| } finally { |
| r.close(); |
| dir.close(); |
| } |
| } |
| |
| public void testGetDocCount() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| Document d = new Document(); |
| d.add(newTextField("f", "a", Field.Store.NO)); |
| writer.addDocument(d); |
| d = new Document(); |
| d.add(newTextField("f", "a", Field.Store.NO)); |
| writer.addDocument(d); |
| DirectoryReader r = writer.getReader(); |
| writer.close(); |
| try { |
| // Make sure codec impls getSumDocFreq (eg PreFlex doesn't) |
| Assume.assumeTrue(r.getDocCount("f") != -1); |
| assertEquals(2, r.getDocCount("f")); |
| } finally { |
| r.close(); |
| dir.close(); |
| } |
| } |
| |
| public void testGetSumTotalTermFreq() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| Document d = new Document(); |
| d.add(newTextField("f", "a b b", Field.Store.NO)); |
| writer.addDocument(d); |
| d = new Document(); |
| d.add(newTextField("f", "a a b", Field.Store.NO)); |
| writer.addDocument(d); |
| DirectoryReader r = writer.getReader(); |
| writer.close(); |
| try { |
| // Make sure codec impls getSumDocFreq (eg PreFlex doesn't) |
| Assume.assumeTrue(r.getSumTotalTermFreq("f") != -1); |
| assertEquals(6, r.getSumTotalTermFreq("f")); |
| } finally { |
| r.close(); |
| dir.close(); |
| } |
| } |
| |
| // LUCENE-2474 |
| public void testReaderFinishedListener() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())) |
| .setMergePolicy(newLogMergePolicy())); |
| ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(3); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| final DirectoryReader reader = writer.getReader(); |
| final int[] closeCount = new int[1]; |
| final IndexReader.ClosedListener listener = new IndexReader.ClosedListener() { |
| @Override |
| public void onClose(IndexReader.CacheKey key) { |
| closeCount[0]++; |
| } |
| }; |
| |
| reader.getReaderCacheHelper().addClosedListener(listener); |
| |
| reader.close(); |
| |
| // Close the top reader, it's the only one that should be closed |
| assertEquals(1, closeCount[0]); |
| writer.close(); |
| |
| DirectoryReader reader2 = DirectoryReader.open(dir); |
| reader2.getReaderCacheHelper().addClosedListener(listener); |
| |
| closeCount[0] = 0; |
| reader2.close(); |
| assertEquals(1, closeCount[0]); |
| dir.close(); |
| } |
| |
| public void testOOBDocID() throws Exception { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| writer.addDocument(new Document()); |
| DirectoryReader r = writer.getReader(); |
| writer.close(); |
| r.document(0); |
| expectThrows(IllegalArgumentException.class, () -> { |
| r.document(1); |
| }); |
| r.close(); |
| dir.close(); |
| } |
| |
| public void testTryIncRef() throws IOException { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| DirectoryReader r = DirectoryReader.open(dir); |
| assertTrue(r.tryIncRef()); |
| r.decRef(); |
| r.close(); |
| assertFalse(r.tryIncRef()); |
| writer.close(); |
| dir.close(); |
| } |
| |
| public void testStressTryIncRef() throws IOException, InterruptedException { |
| Directory dir = newDirectory(); |
| IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))); |
| writer.addDocument(new Document()); |
| writer.commit(); |
| DirectoryReader r = DirectoryReader.open(dir); |
| int numThreads = atLeast(2); |
| |
| IncThread[] threads = new IncThread[numThreads]; |
| for (int i = 0; i < threads.length; i++) { |
| threads[i] = new IncThread(r, random()); |
| threads[i].start(); |
| } |
| Thread.sleep(100); |
| |
| assertTrue(r.tryIncRef()); |
| r.decRef(); |
| r.close(); |
| |
| for (int i = 0; i < threads.length; i++) { |
| threads[i].join(); |
| assertNull(threads[i].failed); |
| } |
| assertFalse(r.tryIncRef()); |
| writer.close(); |
| dir.close(); |
| } |
| |
| static class IncThread extends Thread { |
| final IndexReader toInc; |
| final Random random; |
| Throwable failed; |
| |
| IncThread(IndexReader toInc, Random random) { |
| this.toInc = toInc; |
| this.random = random; |
| } |
| |
| @Override |
| public void run() { |
| try { |
| while (toInc.tryIncRef()) { |
| assertFalse(toInc.hasDeletions()); |
| toInc.decRef(); |
| } |
| assertFalse(toInc.tryIncRef()); |
| } catch (Throwable e) { |
| failed = e; |
| } |
| } |
| } |
| |
| public void testLoadCertainFields() throws Exception { |
| Directory dir = newDirectory(); |
| RandomIndexWriter writer = new RandomIndexWriter(random(), dir); |
| Document doc = new Document(); |
| doc.add(newStringField("field1", "foobar", Field.Store.YES)); |
| doc.add(newStringField("field2", "foobaz", Field.Store.YES)); |
| writer.addDocument(doc); |
| DirectoryReader r = writer.getReader(); |
| writer.close(); |
| Set<String> fieldsToLoad = new HashSet<>(); |
| assertEquals(0, r.document(0, fieldsToLoad).getFields().size()); |
| fieldsToLoad.add("field1"); |
| Document doc2 = r.document(0, fieldsToLoad); |
| assertEquals(1, doc2.getFields().size()); |
| assertEquals("foobar", doc2.get("field1")); |
| r.close(); |
| dir.close(); |
| } |
| |
| public void testIndexExistsOnNonExistentDirectory() throws Exception { |
| Path tempDir = createTempDir("testIndexExistsOnNonExistentDirectory"); |
| Directory dir = newFSDirectory(tempDir); |
| assertFalse(DirectoryReader.indexExists(dir)); |
| dir.close(); |
| } |
| } |
