| Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java |
| =================================================================== |
| --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (revision 1655332) |
| +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextCompoundFormat.java (working copy) |
| @@ -209,11 +209,6 @@ |
| SimpleTextUtil.writeNewline(out); |
| } |
| } |
| - |
| - @Override |
| - public String[] files(SegmentInfo si) { |
| - return new String[] { IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION) }; |
| - } |
| |
| // helper method to strip strip away 'prefix' from 'scratch' and return as String |
| private String stripPrefix(BytesRefBuilder scratch, BytesRef prefix) throws IOException { |
| Index: lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java |
| =================================================================== |
| --- lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java (revision 1655332) |
| +++ lucene/core/src/java/org/apache/lucene/codecs/CompoundFormat.java (working copy) |
| @@ -47,11 +47,4 @@ |
| * Packs the provided files into a compound format. |
| */ |
| public abstract void write(Directory dir, SegmentInfo si, Collection<String> files, IOContext context) throws IOException; |
| - |
| - /** |
| - * Returns the compound file names used by this segment. |
| - */ |
| - // TODO: get this out of here, and use trackingdirwrapper. but this is really scary in IW right now... |
| - // NOTE: generally si.useCompoundFile is not even yet 'set' when this is called. |
| - public abstract String[] files(SegmentInfo si); |
| } |
| Index: lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundFormat.java |
| =================================================================== |
| --- lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundFormat.java (revision 1655332) |
| +++ lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50CompoundFormat.java (working copy) |
| @@ -105,14 +105,6 @@ |
| } |
| } |
| |
| - @Override |
| - public String[] files(SegmentInfo si) { |
| - return new String[] { |
| - IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION), |
| - IndexFileNames.segmentFileName(si.name, "", ENTRIES_EXTENSION) |
| - }; |
| - } |
| - |
| /** Extension of compound file */ |
| static final String DATA_EXTENSION = "cfs"; |
| /** Extension of compound file entries */ |
| Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java |
| =================================================================== |
| --- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (revision 1655332) |
| +++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (working copy) |
| @@ -485,7 +485,10 @@ |
| try { |
| |
| if (indexWriterConfig.getUseCompoundFile()) { |
| - filesToDelete.addAll(IndexWriter.createCompoundFile(infoStream, directory, newSegment.info, context)); |
| + Set<String> originalFiles = newSegment.info.files(); |
| + // TODO: like addIndexes, we are relying on createCompoundFile to successfully cleanup... |
| + IndexWriter.createCompoundFile(infoStream, new TrackingDirectoryWrapper(directory), newSegment.info, context); |
| + filesToDelete.addAll(originalFiles); |
| newSegment.info.setUseCompoundFile(true); |
| } |
| |
| Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java |
| =================================================================== |
| --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1655332) |
| +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy) |
| @@ -2580,8 +2580,11 @@ |
| // Now create the compound file if needed |
| if (useCompoundFile) { |
| Collection<String> filesToDelete = infoPerCommit.files(); |
| + TrackingDirectoryWrapper trackingCFSDir = new TrackingDirectoryWrapper(mergeDirectory); |
| + // TODO: unlike merge, on exception we arent sniping any trash cfs files here? |
| + // createCompoundFile tries to cleanup, but it might not always be able to... |
| try { |
| - createCompoundFile(infoStream, mergeDirectory, info, context); |
| + createCompoundFile(infoStream, trackingCFSDir, info, context); |
| } finally { |
| // delete new non cfs files directly: they were never |
| // registered with IFD |
| @@ -4023,11 +4026,10 @@ |
| if (useCompoundFile) { |
| success = false; |
| |
| - String cfsFiles[] = merge.info.info.getCodec().compoundFormat().files(merge.info.info); |
| Collection<String> filesToRemove = merge.info.files(); |
| - |
| + TrackingDirectoryWrapper trackingCFSDir = new TrackingDirectoryWrapper(mergeDirectory); |
| try { |
| - filesToRemove = createCompoundFile(infoStream, mergeDirectory, merge.info.info, context); |
| + createCompoundFile(infoStream, trackingCFSDir, merge.info.info, context); |
| success = true; |
| } catch (IOException ioe) { |
| synchronized(this) { |
| @@ -4048,6 +4050,7 @@ |
| } |
| |
| synchronized(this) { |
| + Set<String> cfsFiles = new HashSet<>(trackingCFSDir.getCreatedFiles()); |
| for (String cfsFile : cfsFiles) { |
| deleter.deleteFile(cfsFile); |
| } |
| @@ -4071,6 +4074,7 @@ |
| if (infoStream.isEnabled("IW")) { |
| infoStream.message("IW", "abort merge after building CFS"); |
| } |
| + Set<String> cfsFiles = new HashSet<>(trackingCFSDir.getCreatedFiles()); |
| for (String cfsFile : cfsFiles) { |
| deleter.deleteFile(cfsFile); |
| } |
| @@ -4521,11 +4525,13 @@ |
| * deletion files, this SegmentInfo must not reference such files when this |
| * method is called, because they are not allowed within a compound file. |
| */ |
| - static final Collection<String> createCompoundFile(InfoStream infoStream, Directory directory, final SegmentInfo info, IOContext context) |
| + static final void createCompoundFile(InfoStream infoStream, TrackingDirectoryWrapper directory, final SegmentInfo info, IOContext context) |
| throws IOException { |
| |
| - // TODO: use trackingdirectorywrapper instead of files() to know which files to delete when things fail: |
| - String cfsFiles[] = info.getCodec().compoundFormat().files(info); |
| + // maybe this check is not needed, but why take the risk? |
| + if (!directory.getCreatedFiles().isEmpty()) { |
| + throw new IllegalStateException("pass a clean trackingdir for CFS creation"); |
| + } |
| |
| if (infoStream.isEnabled("IW")) { |
| infoStream.message("IW", "create compound file"); |
| @@ -4539,18 +4545,16 @@ |
| success = true; |
| } finally { |
| if (!success) { |
| - IOUtils.deleteFilesIgnoringExceptions(directory, cfsFiles); |
| + Set<String> cfsFiles = new HashSet<>(directory.getCreatedFiles()); |
| + for (String file : cfsFiles) { |
| + IOUtils.deleteFilesIgnoringExceptions(directory, file); |
| + } |
| } |
| } |
| |
| // Replace all previous files with the CFS/CFE files: |
| - Set<String> siFiles = new HashSet<>(); |
| - for (String cfsFile : cfsFiles) { |
| - siFiles.add(cfsFile); |
| - } |
| + Set<String> siFiles = new HashSet<>(directory.getCreatedFiles()); |
| info.setFiles(siFiles); |
| - |
| - return files; |
| } |
| |
| /** |
| Index: lucene/core/src/test/org/apache/lucene/index/TestDoc.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/index/TestDoc.java (revision 1655332) |
| +++ lucene/core/src/test/org/apache/lucene/index/TestDoc.java (working copy) |
| @@ -232,7 +232,8 @@ |
| si.setFiles(new HashSet<>(trackingDir.getCreatedFiles())); |
| |
| if (useCompoundFile) { |
| - Collection<String> filesToDelete = IndexWriter.createCompoundFile(InfoStream.getDefault(), dir, si, newIOContext(random())); |
| + Collection<String> filesToDelete = si.files(); |
| + IndexWriter.createCompoundFile(InfoStream.getDefault(), new TrackingDirectoryWrapper(dir), si, newIOContext(random())); |
| si.setUseCompoundFile(true); |
| for (final String fileToDelete : filesToDelete) { |
| si1.info.dir.deleteFile(fileToDelete); |
| Index: lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 1655332) |
| +++ lucene/core/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) |
| @@ -22,6 +22,7 @@ |
| import java.util.concurrent.atomic.AtomicBoolean; |
| |
| import org.apache.lucene.analysis.MockAnalyzer; |
| +import org.apache.lucene.codecs.simpletext.SimpleTextCodec; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.index.IndexWriterConfig.OpenMode; |
| @@ -115,7 +116,7 @@ |
| // non-existent segment: |
| copyFile(dir, "_0_1" + ext, "_188_1" + ext); |
| |
| - String cfsFiles0[] = si0.getCodec().compoundFormat().files(si0); |
| + String cfsFiles0[] = si0.getCodec() instanceof SimpleTextCodec ? new String[] { "_0.scf" } : new String[] { "_0.cfs", "_0.cfe" }; |
| |
| // Create a bogus segment file: |
| copyFile(dir, cfsFiles0[0], "_188.cfs"); |
| @@ -128,12 +129,12 @@ |
| // TODO: assert is bogus (relies upon codec-specific filenames) |
| assertTrue(slowFileExists(dir, "_3.fdt") || slowFileExists(dir, "_3.fld")); |
| |
| - String cfsFiles3[] = si3.getCodec().compoundFormat().files(si3); |
| + String cfsFiles3[] = si3.getCodec() instanceof SimpleTextCodec ? new String[] { "_3.scf" } : new String[] { "_3.cfs", "_3.cfe" }; |
| for (String f : cfsFiles3) { |
| assertTrue(!slowFileExists(dir, f)); |
| } |
| |
| - String cfsFiles1[] = si1.getCodec().compoundFormat().files(si1); |
| + String cfsFiles1[] = si1.getCodec() instanceof SimpleTextCodec ? new String[] { "_1.scf" } : new String[] { "_1.cfs", "_1.cfe" }; |
| copyFile(dir, cfsFiles1[0], "_3.cfs"); |
| |
| String[] filesPre = dir.listAll(); |
| Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java |
| =================================================================== |
| --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1655332) |
| +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) |
| @@ -23,6 +23,7 @@ |
| import java.nio.file.NoSuchFileException; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| +import java.util.Collections; |
| import java.util.Iterator; |
| import java.util.List; |
| import java.util.Random; |
| @@ -1199,7 +1200,7 @@ |
| dir.close(); |
| } |
| |
| - // Simulate a corrupt index by removing one of the cfs |
| + // Simulate a corrupt index by removing one of the |
| // files and make sure we get an IOException trying to |
| // open the index: |
| public void testSimulatedCorruptIndex2() throws IOException { |
| @@ -1237,8 +1238,9 @@ |
| SegmentInfos sis = SegmentInfos.readLatestCommit(dir); |
| for (SegmentCommitInfo si : sis) { |
| assertTrue(si.info.getUseCompoundFile()); |
| - String cfsFiles[] = si.info.getCodec().compoundFormat().files(si.info); |
| - dir.deleteFile(cfsFiles[0]); |
| + List<String> victims = new ArrayList<String>(si.info.files()); |
| + Collections.shuffle(victims, random()); |
| + dir.deleteFile(victims.get(0)); |
| corrupted = true; |
| break; |
| } |
| Index: lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCompoundFormat.java |
| =================================================================== |
| --- lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCompoundFormat.java (revision 1655332) |
| +++ lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyCompoundFormat.java (working copy) |
| @@ -47,9 +47,4 @@ |
| } |
| delegate.write(dir, si, files, context); |
| } |
| - |
| - @Override |
| - public String[] files(SegmentInfo si) { |
| - return delegate.files(si); |
| - } |
| } |
