| package org.apache.lucene.index; |
| |
| /** |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| import java.io.IOException; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Set; |
| import java.util.Collection; |
| |
| import org.apache.lucene.analysis.MockAnalyzer; |
| import org.apache.lucene.analysis.WhitespaceAnalyzer; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| import org.apache.lucene.search.IndexSearcher; |
| import org.apache.lucene.search.Query; |
| import org.apache.lucene.search.ScoreDoc; |
| import org.apache.lucene.search.TermQuery; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.util.LuceneTestCase; |
| |
| /* |
| Verify we can read the pre-2.1 file format, do searches |
| against it, and add documents to it. |
| */ |
| |
| public class TestDeletionPolicy extends LuceneTestCase { |
| |
| private void verifyCommitOrder(List<? extends IndexCommit> commits) throws IOException { |
| final IndexCommit firstCommit = commits.get(0); |
| long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName()); |
| assertEquals(last, firstCommit.getGeneration()); |
| long lastVersion = firstCommit.getVersion(); |
| long lastTimestamp = firstCommit.getTimestamp(); |
| for(int i=1;i<commits.size();i++) { |
| final IndexCommit commit = commits.get(i); |
| long now = SegmentInfos.generationFromSegmentsFileName(commit.getSegmentsFileName()); |
| long nowVersion = commit.getVersion(); |
| long nowTimestamp = commit.getTimestamp(); |
| assertTrue("SegmentInfos commits are out-of-order", now > last); |
| assertTrue("SegmentInfos versions are out-of-order", nowVersion > lastVersion); |
| assertTrue("SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp, nowTimestamp >= lastTimestamp); |
| assertEquals(now, commit.getGeneration()); |
| last = now; |
| lastVersion = nowVersion; |
| lastTimestamp = nowTimestamp; |
| } |
| } |
| |
| class KeepAllDeletionPolicy implements IndexDeletionPolicy { |
| int numOnInit; |
| int numOnCommit; |
| Directory dir; |
| public void onInit(List<? extends IndexCommit> commits) throws IOException { |
| verifyCommitOrder(commits); |
| numOnInit++; |
| } |
| public void onCommit(List<? extends IndexCommit> commits) throws IOException { |
| IndexCommit lastCommit = commits.get(commits.size()-1); |
| IndexReader r = IndexReader.open(dir, true); |
| assertEquals("lastCommit.isOptimized()=" + lastCommit.isOptimized() + " vs IndexReader.isOptimized=" + r.isOptimized(), r.isOptimized(), lastCommit.isOptimized()); |
| r.close(); |
| verifyCommitOrder(commits); |
| numOnCommit++; |
| } |
| } |
| |
| /** |
| * This is useful for adding to a big index when you know |
| * readers are not using it. |
| */ |
| class KeepNoneOnInitDeletionPolicy implements IndexDeletionPolicy { |
| int numOnInit; |
| int numOnCommit; |
| public void onInit(List<? extends IndexCommit> commits) throws IOException { |
| verifyCommitOrder(commits); |
| numOnInit++; |
| // On init, delete all commit points: |
| for (final IndexCommit commit : commits) { |
| commit.delete(); |
| assertTrue(commit.isDeleted()); |
| } |
| } |
| public void onCommit(List<? extends IndexCommit> commits) throws IOException { |
| verifyCommitOrder(commits); |
| int size = commits.size(); |
| // Delete all but last one: |
| for(int i=0;i<size-1;i++) { |
| ((IndexCommit) commits.get(i)).delete(); |
| } |
| numOnCommit++; |
| } |
| } |
| |
| class KeepLastNDeletionPolicy implements IndexDeletionPolicy { |
| int numOnInit; |
| int numOnCommit; |
| int numToKeep; |
| int numDelete; |
| Set<String> seen = new HashSet<String>(); |
| |
| public KeepLastNDeletionPolicy(int numToKeep) { |
| this.numToKeep = numToKeep; |
| } |
| |
| public void onInit(List<? extends IndexCommit> commits) throws IOException { |
| if (VERBOSE) { |
| System.out.println("TEST: onInit"); |
| } |
| verifyCommitOrder(commits); |
| numOnInit++; |
| // do no deletions on init |
| doDeletes(commits, false); |
| } |
| |
| public void onCommit(List<? extends IndexCommit> commits) throws IOException { |
| if (VERBOSE) { |
| System.out.println("TEST: onCommit"); |
| } |
| verifyCommitOrder(commits); |
| doDeletes(commits, true); |
| } |
| |
| private void doDeletes(List<? extends IndexCommit> commits, boolean isCommit) { |
| |
| // Assert that we really are only called for each new |
| // commit: |
| if (isCommit) { |
| String fileName = ((IndexCommit) commits.get(commits.size()-1)).getSegmentsFileName(); |
| if (seen.contains(fileName)) { |
| throw new RuntimeException("onCommit was called twice on the same commit point: " + fileName); |
| } |
| seen.add(fileName); |
| numOnCommit++; |
| } |
| int size = commits.size(); |
| for(int i=0;i<size-numToKeep;i++) { |
| ((IndexCommit) commits.get(i)).delete(); |
| numDelete++; |
| } |
| } |
| } |
| |
| /* |
| * Delete a commit only when it has been obsoleted by N |
| * seconds. |
| */ |
| class ExpirationTimeDeletionPolicy implements IndexDeletionPolicy { |
| |
| Directory dir; |
| double expirationTimeSeconds; |
| int numDelete; |
| |
| public ExpirationTimeDeletionPolicy(Directory dir, double seconds) { |
| this.dir = dir; |
| this.expirationTimeSeconds = seconds; |
| } |
| |
| public void onInit(List<? extends IndexCommit> commits) throws IOException { |
| verifyCommitOrder(commits); |
| onCommit(commits); |
| } |
| |
| public void onCommit(List<? extends IndexCommit> commits) throws IOException { |
| verifyCommitOrder(commits); |
| |
| IndexCommit lastCommit = commits.get(commits.size()-1); |
| |
| // Any commit older than expireTime should be deleted: |
| double expireTime = dir.fileModified(lastCommit.getSegmentsFileName())/1000.0 - expirationTimeSeconds; |
| |
| for (final IndexCommit commit : commits) { |
| double modTime = dir.fileModified(commit.getSegmentsFileName())/1000.0; |
| if (commit != lastCommit && modTime < expireTime) { |
| commit.delete(); |
| numDelete += 1; |
| } |
| } |
| } |
| } |
| |
| /* |
| * Test "by time expiration" deletion policy: |
| */ |
| public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException { |
| |
| final double SECONDS = 2.0; |
| |
| Directory dir = newDirectory(); |
| ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS); |
| IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, |
| new MockAnalyzer(random)) |
| .setIndexDeletionPolicy(policy); |
| MergePolicy mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, true); |
| } |
| IndexWriter writer = new IndexWriter(dir, conf); |
| writer.close(); |
| |
| final int ITER = 9; |
| |
| long lastDeleteTime = 0; |
| for(int i=0;i<ITER;i++) { |
| // Record last time when writer performed deletes of |
| // past commits |
| lastDeleteTime = System.currentTimeMillis(); |
| conf = newIndexWriterConfig(TEST_VERSION_CURRENT, |
| new MockAnalyzer(random)).setOpenMode( |
| OpenMode.APPEND).setIndexDeletionPolicy(policy); |
| mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, true); |
| } |
| writer = new IndexWriter(dir, conf); |
| for(int j=0;j<17;j++) { |
| addDoc(writer); |
| } |
| writer.close(); |
| |
| if (i < ITER-1) { |
| // Make sure to sleep long enough so that some commit |
| // points will be deleted: |
| Thread.sleep((int) (1000.0*(SECONDS/5.0))); |
| } |
| } |
| |
| // First, make sure the policy in fact deleted something: |
| assertTrue("no commits were deleted", policy.numDelete > 0); |
| |
| // Then simplistic check: just verify that the |
| // segments_N's that still exist are in fact within SECONDS |
| // seconds of the last one's mod time, and, that I can |
| // open a reader on each: |
| long gen = SegmentInfos.getCurrentSegmentGeneration(dir); |
| |
| String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, |
| "", |
| gen); |
| dir.deleteFile(IndexFileNames.SEGMENTS_GEN); |
| |
| boolean oneSecondResolution = true; |
| |
| while(gen > 0) { |
| try { |
| IndexReader reader = IndexReader.open(dir, true); |
| reader.close(); |
| fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, |
| "", |
| gen); |
| |
| // if we are on a filesystem that seems to have only |
| // 1 second resolution, allow +1 second in commit |
| // age tolerance: |
| long modTime = dir.fileModified(fileName); |
| oneSecondResolution &= (modTime % 1000) == 0; |
| final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000); |
| |
| assertTrue("commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ", lastDeleteTime - modTime <= leeway); |
| } catch (IOException e) { |
| // OK |
| break; |
| } |
| |
| dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); |
| gen--; |
| } |
| |
| dir.close(); |
| } |
| |
| /* |
| * Test a silly deletion policy that keeps all commits around. |
| */ |
| public void testKeepAllDeletionPolicy() throws IOException { |
| for(int pass=0;pass<2;pass++) { |
| |
| if (VERBOSE) { |
| System.out.println("TEST: cycle pass=" + pass); |
| } |
| |
| boolean useCompoundFile = (pass % 2) != 0; |
| |
| // Never deletes a commit |
| KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(); |
| |
| Directory dir = newDirectory(); |
| policy.dir = dir; |
| |
| IndexWriterConfig conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10) |
| .setMergeScheduler(new SerialMergeScheduler()); |
| MergePolicy mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| IndexWriter writer = new IndexWriter(dir, conf); |
| for(int i=0;i<107;i++) { |
| addDoc(writer); |
| } |
| writer.close(); |
| |
| final boolean isOptimized; |
| { |
| IndexReader r = IndexReader.open(dir); |
| isOptimized = r.isOptimized(); |
| r.close(); |
| } |
| if (!isOptimized) { |
| conf = newIndexWriterConfig(TEST_VERSION_CURRENT, |
| new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode( |
| OpenMode.APPEND).setIndexDeletionPolicy(policy); |
| mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, true); |
| } |
| if (VERBOSE) { |
| System.out.println("TEST: open writer for optimize"); |
| } |
| writer = new IndexWriter(dir, conf); |
| writer.setInfoStream(VERBOSE ? System.out : null); |
| writer.optimize(); |
| writer.close(); |
| } |
| assertEquals(isOptimized ? 0:1, policy.numOnInit); |
| |
| // If we are not auto committing then there should |
| // be exactly 2 commits (one per close above): |
| assertEquals(1 + (isOptimized ? 0:1), policy.numOnCommit); |
| |
| // Test listCommits |
| Collection<IndexCommit> commits = IndexReader.listCommits(dir); |
| // 2 from closing writer |
| assertEquals(1 + (isOptimized ? 0:1), commits.size()); |
| |
| // Make sure we can open a reader on each commit: |
| for (final IndexCommit commit : commits) { |
| IndexReader r = IndexReader.open(commit, null, false); |
| r.close(); |
| } |
| |
| // Simplistic check: just verify all segments_N's still |
| // exist, and, I can open a reader on each: |
| dir.deleteFile(IndexFileNames.SEGMENTS_GEN); |
| long gen = SegmentInfos.getCurrentSegmentGeneration(dir); |
| while(gen > 0) { |
| IndexReader reader = IndexReader.open(dir, true); |
| reader.close(); |
| dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); |
| gen--; |
| |
| if (gen > 0) { |
| // Now that we've removed a commit point, which |
| // should have orphan'd at least one index file. |
| // Open & close a writer and assert that it |
| // actually removed something: |
| int preCount = dir.listAll().length; |
| writer = new IndexWriter(dir, newIndexWriterConfig( |
| TEST_VERSION_CURRENT, |
| new MockAnalyzer(random)).setOpenMode( |
| OpenMode.APPEND).setIndexDeletionPolicy(policy)); |
| writer.close(); |
| int postCount = dir.listAll().length; |
| assertTrue(postCount < preCount); |
| } |
| } |
| |
| dir.close(); |
| } |
| } |
| |
| /* Uses KeepAllDeletionPolicy to keep all commits around, |
| * then, opens a new IndexWriter on a previous commit |
| * point. */ |
| public void testOpenPriorSnapshot() throws IOException { |
| // Never deletes a commit |
| KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy(); |
| |
| Directory dir = newDirectory(); |
| policy.dir = dir; |
| |
| IndexWriter writer = new IndexWriter( |
| dir, |
| newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). |
| setIndexDeletionPolicy(policy). |
| setMaxBufferedDocs(2). |
| setMergePolicy(newLogMergePolicy(10)) |
| ); |
| for(int i=0;i<10;i++) { |
| addDoc(writer); |
| if ((1+i)%2 == 0) |
| writer.commit(); |
| } |
| writer.close(); |
| |
| Collection<IndexCommit> commits = IndexReader.listCommits(dir); |
| assertEquals(5, commits.size()); |
| IndexCommit lastCommit = null; |
| for (final IndexCommit commit : commits) { |
| if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration()) |
| lastCommit = commit; |
| } |
| assertTrue(lastCommit != null); |
| |
| // Now add 1 doc and optimize |
| writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, |
| new MockAnalyzer(random)).setIndexDeletionPolicy(policy)); |
| addDoc(writer); |
| assertEquals(11, writer.numDocs()); |
| writer.optimize(); |
| writer.close(); |
| |
| assertEquals(6, IndexReader.listCommits(dir).size()); |
| |
| // Now open writer on the commit just before optimize: |
| writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit)); |
| assertEquals(10, writer.numDocs()); |
| |
| // Should undo our rollback: |
| writer.rollback(); |
| |
| IndexReader r = IndexReader.open(dir, true); |
| // Still optimized, still 11 docs |
| assertTrue(r.isOptimized()); |
| assertEquals(11, r.numDocs()); |
| r.close(); |
| |
| writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit)); |
| assertEquals(10, writer.numDocs()); |
| // Commits the rollback: |
| writer.close(); |
| |
| // Now 8 because we made another commit |
| assertEquals(7, IndexReader.listCommits(dir).size()); |
| |
| r = IndexReader.open(dir, true); |
| // Not optimized because we rolled it back, and now only |
| // 10 docs |
| assertTrue(!r.isOptimized()); |
| assertEquals(10, r.numDocs()); |
| r.close(); |
| |
| // Reoptimize |
| writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(policy)); |
| writer.optimize(); |
| writer.close(); |
| |
| r = IndexReader.open(dir, true); |
| assertTrue(r.isOptimized()); |
| assertEquals(10, r.numDocs()); |
| r.close(); |
| |
| // Now open writer on the commit just before optimize, |
| // but this time keeping only the last commit: |
| writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexCommit(lastCommit)); |
| assertEquals(10, writer.numDocs()); |
| |
| // Reader still sees optimized index, because writer |
| // opened on the prior commit has not yet committed: |
| r = IndexReader.open(dir, true); |
| assertTrue(r.isOptimized()); |
| assertEquals(10, r.numDocs()); |
| r.close(); |
| |
| writer.close(); |
| |
| // Now reader sees unoptimized index: |
| r = IndexReader.open(dir, true); |
| assertTrue(!r.isOptimized()); |
| assertEquals(10, r.numDocs()); |
| r.close(); |
| |
| dir.close(); |
| } |
| |
| |
| /* Test keeping NO commit points. This is a viable and |
| * useful case eg where you want to build a big index and |
| * you know there are no readers. |
| */ |
| public void testKeepNoneOnInitDeletionPolicy() throws IOException { |
| for(int pass=0;pass<2;pass++) { |
| |
| boolean useCompoundFile = (pass % 2) != 0; |
| |
| KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy(); |
| |
| Directory dir = newDirectory(); |
| |
| IndexWriterConfig conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy) |
| .setMaxBufferedDocs(10); |
| MergePolicy mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| IndexWriter writer = new IndexWriter(dir, conf); |
| for(int i=0;i<107;i++) { |
| addDoc(writer); |
| } |
| writer.close(); |
| |
| conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); |
| mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, true); |
| } |
| writer = new IndexWriter(dir, conf); |
| writer.optimize(); |
| writer.close(); |
| |
| assertEquals(1, policy.numOnInit); |
| // If we are not auto committing then there should |
| // be exactly 2 commits (one per close above): |
| assertEquals(2, policy.numOnCommit); |
| |
| // Simplistic check: just verify the index is in fact |
| // readable: |
| IndexReader reader = IndexReader.open(dir, true); |
| reader.close(); |
| |
| dir.close(); |
| } |
| } |
| |
| /* |
| * Test a deletion policy that keeps last N commits. |
| */ |
| public void testKeepLastNDeletionPolicy() throws IOException { |
| final int N = 5; |
| |
| for(int pass=0;pass<2;pass++) { |
| |
| boolean useCompoundFile = (pass % 2) != 0; |
| |
| Directory dir = newDirectory(); |
| |
| KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N); |
| |
| for(int j=0;j<N+1;j++) { |
| IndexWriterConfig conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy) |
| .setMaxBufferedDocs(10); |
| MergePolicy mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| IndexWriter writer = new IndexWriter(dir, conf); |
| for(int i=0;i<17;i++) { |
| addDoc(writer); |
| } |
| writer.optimize(); |
| writer.close(); |
| } |
| |
| assertTrue(policy.numDelete > 0); |
| assertEquals(N, policy.numOnInit); |
| assertEquals(N+1, policy.numOnCommit); |
| |
| // Simplistic check: just verify only the past N segments_N's still |
| // exist, and, I can open a reader on each: |
| dir.deleteFile(IndexFileNames.SEGMENTS_GEN); |
| long gen = SegmentInfos.getCurrentSegmentGeneration(dir); |
| for(int i=0;i<N+1;i++) { |
| try { |
| IndexReader reader = IndexReader.open(dir, true); |
| reader.close(); |
| if (i == N) { |
| fail("should have failed on commits prior to last " + N); |
| } |
| } catch (IOException e) { |
| if (i != N) { |
| throw e; |
| } |
| } |
| if (i < N) { |
| dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); |
| } |
| gen--; |
| } |
| |
| dir.close(); |
| } |
| } |
| |
| /* |
| * Test a deletion policy that keeps last N commits |
| * around, with reader doing deletes. |
| */ |
| public void testKeepLastNDeletionPolicyWithReader() throws IOException { |
| final int N = 10; |
| |
| for(int pass=0;pass<2;pass++) { |
| |
| boolean useCompoundFile = (pass % 2) != 0; |
| |
| KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N); |
| |
| Directory dir = newDirectory(); |
| IndexWriterConfig conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy()); |
| MergePolicy mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| IndexWriter writer = new IndexWriter(dir, conf); |
| writer.close(); |
| Term searchTerm = new Term("content", "aaa"); |
| Query query = new TermQuery(searchTerm); |
| |
| for(int i=0;i<N+1;i++) { |
| if (VERBOSE) { |
| System.out.println("\nTEST: cycle i=" + i); |
| } |
| conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy()); |
| mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| writer = new IndexWriter(dir, conf); |
| writer.setInfoStream(VERBOSE ? System.out : null); |
| for(int j=0;j<17;j++) { |
| addDoc(writer); |
| } |
| // this is a commit |
| if (VERBOSE) { |
| System.out.println("TEST: close writer"); |
| } |
| writer.close(); |
| IndexReader reader = IndexReader.open(dir, policy, false); |
| reader.deleteDocument(3*i+1); |
| reader.setNorm(4*i+1, "content", 2.0F); |
| IndexSearcher searcher = newSearcher(reader); |
| ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; |
| assertEquals(16*(1+i), hits.length); |
| // this is a commit |
| if (VERBOSE) { |
| System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit); |
| } |
| reader.close(); |
| searcher.close(); |
| } |
| conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy); |
| mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| IndexReader r = IndexReader.open(dir); |
| final boolean wasOptimized = r.isOptimized(); |
| r.close(); |
| writer = new IndexWriter(dir, conf); |
| writer.optimize(); |
| // this is a commit |
| writer.close(); |
| |
| assertEquals(2*(N+1)+1, policy.numOnInit); |
| assertEquals(2*(N+2) - (wasOptimized ? 1:0), policy.numOnCommit); |
| |
| IndexSearcher searcher = new IndexSearcher(dir, false); |
| ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; |
| assertEquals(176, hits.length); |
| |
| // Simplistic check: just verify only the past N segments_N's still |
| // exist, and, I can open a reader on each: |
| long gen = SegmentInfos.getCurrentSegmentGeneration(dir); |
| |
| dir.deleteFile(IndexFileNames.SEGMENTS_GEN); |
| int expectedCount = 176; |
| searcher.close(); |
| for(int i=0;i<N+1;i++) { |
| try { |
| IndexReader reader = IndexReader.open(dir, true); |
| |
| // Work backwards in commits on what the expected |
| // count should be. |
| searcher = newSearcher(reader); |
| hits = searcher.search(query, null, 1000).scoreDocs; |
| if (i > 1) { |
| if (i % 2 == 0) { |
| expectedCount += 1; |
| } else { |
| expectedCount -= 17; |
| } |
| } |
| assertEquals(expectedCount, hits.length); |
| searcher.close(); |
| reader.close(); |
| if (i == N) { |
| fail("should have failed on commits before last 5"); |
| } |
| } catch (IOException e) { |
| if (i != N) { |
| throw e; |
| } |
| } |
| if (i < N) { |
| dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); |
| } |
| gen--; |
| } |
| dir.close(); |
| } |
| } |
| |
| /* |
| * Test a deletion policy that keeps last N commits |
| * around, through creates. |
| */ |
| public void testKeepLastNDeletionPolicyWithCreates() throws IOException { |
| |
| final int N = 10; |
| |
| for(int pass=0;pass<2;pass++) { |
| |
| boolean useCompoundFile = (pass % 2) != 0; |
| |
| KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N); |
| |
| Directory dir = newDirectory(); |
| IndexWriterConfig conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy) |
| .setMaxBufferedDocs(10); |
| MergePolicy mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| IndexWriter writer = new IndexWriter(dir, conf); |
| writer.close(); |
| Term searchTerm = new Term("content", "aaa"); |
| Query query = new TermQuery(searchTerm); |
| |
| for(int i=0;i<N+1;i++) { |
| |
| conf = newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy) |
| .setMaxBufferedDocs(10); |
| mp = conf.getMergePolicy(); |
| if (mp instanceof LogMergePolicy) { |
| setUseCompoundFile(mp, useCompoundFile); |
| } |
| writer = new IndexWriter(dir, conf); |
| for(int j=0;j<17;j++) { |
| addDoc(writer); |
| } |
| // this is a commit |
| writer.close(); |
| IndexReader reader = IndexReader.open(dir, policy, false); |
| reader.deleteDocument(3); |
| reader.setNorm(5, "content", 2.0F); |
| IndexSearcher searcher = newSearcher(reader); |
| ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; |
| assertEquals(16, hits.length); |
| // this is a commit |
| reader.close(); |
| searcher.close(); |
| |
| writer = new IndexWriter(dir, newIndexWriterConfig( |
| TEST_VERSION_CURRENT, new MockAnalyzer(random)) |
| .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)); |
| // This will not commit: there are no changes |
| // pending because we opened for "create": |
| writer.close(); |
| } |
| |
| assertEquals(3*(N+1), policy.numOnInit); |
| assertEquals(3*(N+1)+1, policy.numOnCommit); |
| |
| IndexSearcher searcher = new IndexSearcher(dir, false); |
| ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; |
| assertEquals(0, hits.length); |
| |
| // Simplistic check: just verify only the past N segments_N's still |
| // exist, and, I can open a reader on each: |
| long gen = SegmentInfos.getCurrentSegmentGeneration(dir); |
| |
| dir.deleteFile(IndexFileNames.SEGMENTS_GEN); |
| int expectedCount = 0; |
| |
| for(int i=0;i<N+1;i++) { |
| try { |
| IndexReader reader = IndexReader.open(dir, true); |
| |
| // Work backwards in commits on what the expected |
| // count should be. |
| searcher = newSearcher(reader); |
| hits = searcher.search(query, null, 1000).scoreDocs; |
| assertEquals(expectedCount, hits.length); |
| searcher.close(); |
| if (expectedCount == 0) { |
| expectedCount = 16; |
| } else if (expectedCount == 16) { |
| expectedCount = 17; |
| } else if (expectedCount == 17) { |
| expectedCount = 0; |
| } |
| reader.close(); |
| if (i == N) { |
| fail("should have failed on commits before last " + N); |
| } |
| } catch (IOException e) { |
| if (i != N) { |
| throw e; |
| } |
| } |
| if (i < N) { |
| dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen)); |
| } |
| gen--; |
| } |
| |
| dir.close(); |
| } |
| } |
| |
| private void addDoc(IndexWriter writer) throws IOException |
| { |
| Document doc = new Document(); |
| doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED)); |
| writer.addDocument(doc); |
| } |
| } |
