/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;


import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.List;

import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.asserting.AssertingCodec;
import org.apache.lucene.codecs.memory.DirectPostingsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;

public class TestAddIndexes extends LuceneTestCase {
  
  public void testSimpleCase() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // two auxiliary directories
    Directory aux = newDirectory();
    Directory aux2 = newDirectory();

    IndexWriter writer = null;

    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
        .setOpenMode(OpenMode.CREATE));
    // add 100 documents
    addDocs(writer, 100);
    assertEquals(100, writer.getDocStats().maxDoc);
    writer.close();
    TestUtil.checkIndex(dir);

    writer = newWriter(
        aux,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setMergePolicy(newLogMergePolicy(false))
    );
    // add 40 documents in separate files
    addDocs(writer, 40);
    assertEquals(40, writer.getDocStats().maxDoc);
    writer.close();

    writer = newWriter(aux2, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
    // add 50 documents in compound files
    addDocs2(writer, 50);
    assertEquals(50, writer.getDocStats().maxDoc);
    writer.close();

    // test doc count before segments are merged
    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    assertEquals(100, writer.getDocStats().maxDoc);
    writer.addIndexes(aux, aux2);
    assertEquals(190, writer.getDocStats().maxDoc);
    writer.close();
    TestUtil.checkIndex(dir);

    // make sure the old index is correct
    verifyNumDocs(aux, 40);

    // make sure the new index is correct
    verifyNumDocs(dir, 190);

    // now add another set in.
    Directory aux3 = newDirectory();
    writer = newWriter(aux3, newIndexWriterConfig(new MockAnalyzer(random())));
    // add 40 documents
    addDocs(writer, 40);
    assertEquals(40, writer.getDocStats().maxDoc);
    writer.close();

    // test doc count before segments are merged
    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    assertEquals(190, writer.getDocStats().maxDoc);
    writer.addIndexes(aux3);
    assertEquals(230, writer.getDocStats().maxDoc);
    writer.close();

    // make sure the new index is correct
    verifyNumDocs(dir, 230);

    verifyTermDocs(dir, new Term("content", "aaa"), 180);

    verifyTermDocs(dir, new Term("content", "bbb"), 50);

    // now fully merge it.
    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    writer.forceMerge(1);
    writer.close();

    // make sure the new index is correct
    verifyNumDocs(dir, 230);

    verifyTermDocs(dir, new Term("content", "aaa"), 180);

    verifyTermDocs(dir, new Term("content", "bbb"), 50);

    // now add a single document
    Directory aux4 = newDirectory();
    writer = newWriter(aux4, newIndexWriterConfig(new MockAnalyzer(random())));
    addDocs2(writer, 1);
    writer.close();

    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    assertEquals(230, writer.getDocStats().maxDoc);
    writer.addIndexes(aux4);
    assertEquals(231, writer.getDocStats().maxDoc);
    writer.close();

    verifyNumDocs(dir, 231);

    verifyTermDocs(dir, new Term("content", "bbb"), 51);
    dir.close();
    aux.close();
    aux2.close();
    aux3.close();
    aux4.close();
  }

  public void testWithPendingDeletes() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux);
    IndexWriter writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    writer.addIndexes(aux);

    // Adds 10 docs, then replaces them with another 10
    // docs, so 10 pending deletes:
    for (int i = 0; i < 20; i++) {
      Document doc = new Document();
      doc.add(newStringField("id", "" + (i % 10), Field.Store.NO));
      doc.add(newTextField("content", "bbb " + i, Field.Store.NO));
      doc.add(new IntPoint("doc", i));
      doc.add(new IntPoint("doc2d", i, i));
      doc.add(new NumericDocValuesField("dv", i));
      writer.updateDocument(new Term("id", "" + (i%10)), doc);
    }
    // Deletes one of the 10 added docs, leaving 9:
    PhraseQuery q = new PhraseQuery("content", "bbb", "14");
    writer.deleteDocuments(q);

    writer.forceMerge(1);
    writer.commit();

    verifyNumDocs(dir, 1039);
    verifyTermDocs(dir, new Term("content", "aaa"), 1030);
    verifyTermDocs(dir, new Term("content", "bbb"), 9);

    writer.close();
    dir.close();
    aux.close();
  }

  public void testWithPendingDeletes2() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux);
    IndexWriter writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));

    // Adds 10 docs, then replaces them with another 10
    // docs, so 10 pending deletes:
    for (int i = 0; i < 20; i++) {
      Document doc = new Document();
      doc.add(newStringField("id", "" + (i % 10), Field.Store.NO));
      doc.add(newTextField("content", "bbb " + i, Field.Store.NO));
      doc.add(new IntPoint("doc", i));
      doc.add(new IntPoint("doc2d", i, i));
      doc.add(new NumericDocValuesField("dv", i));
      writer.updateDocument(new Term("id", "" + (i%10)), doc);
    }
    
    writer.addIndexes(aux);
    
    // Deletes one of the 10 added docs, leaving 9:
    PhraseQuery q = new PhraseQuery("content", "bbb", "14");
    writer.deleteDocuments(q);

    writer.forceMerge(1);
    writer.commit();

    verifyNumDocs(dir, 1039);
    verifyTermDocs(dir, new Term("content", "aaa"), 1030);
    verifyTermDocs(dir, new Term("content", "bbb"), 9);

    writer.close();
    dir.close();
    aux.close();
  }

  public void testWithPendingDeletes3() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux);
    IndexWriter writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));

    // Adds 10 docs, then replaces them with another 10
    // docs, so 10 pending deletes:
    for (int i = 0; i < 20; i++) {
      Document doc = new Document();
      doc.add(newStringField("id", "" + (i % 10), Field.Store.NO));
      doc.add(newTextField("content", "bbb " + i, Field.Store.NO));
      doc.add(new IntPoint("doc", i));
      doc.add(new IntPoint("doc2d", i, i));
      doc.add(new NumericDocValuesField("dv", i));
      writer.updateDocument(new Term("id", "" + (i%10)), doc);
    }

    // Deletes one of the 10 added docs, leaving 9:
    PhraseQuery q = new PhraseQuery("content", "bbb", "14");
    writer.deleteDocuments(q);

    writer.addIndexes(aux);

    writer.forceMerge(1);
    writer.commit();

    verifyNumDocs(dir, 1039);
    verifyTermDocs(dir, new Term("content", "aaa"), 1030);
    verifyTermDocs(dir, new Term("content", "bbb"), 9);

    writer.close();
    dir.close();
    aux.close();
  }

  // case 0: add self or exceed maxMergeDocs, expect exception
  public void testAddSelf() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    IndexWriter writer = null;

    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    // add 100 documents
    addDocs(writer, 100);
    assertEquals(100, writer.getDocStats().maxDoc);
    writer.close();

    writer = newWriter(
        aux,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setMaxBufferedDocs(1000).
            setMergePolicy(newLogMergePolicy(false))
    );
    // add 140 documents in separate files
    addDocs(writer, 40);
    writer.close();
    writer = newWriter(
        aux,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setMaxBufferedDocs(1000).
            setMergePolicy(newLogMergePolicy(false))
    );
    addDocs(writer, 100);
    writer.close();

    // cannot add self
    IndexWriter writer2 = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
    expectThrows(IllegalArgumentException.class, () -> {
      writer2.addIndexes(aux, dir);
    });
    assertEquals(100, writer2.getDocStats().maxDoc);
    writer2.close();

    // make sure the index is correct
    verifyNumDocs(dir, 100);
    dir.close();
    aux.close();
  }

  // in all the remaining tests, make the doc count of the oldest segment
  // in dir large so that it is never merged in addIndexes()
  // case 1: no tail segments
  public void testNoTailSegments() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux);

    IndexWriter writer = newWriter(
        dir,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.APPEND).
            setMaxBufferedDocs(10).
            setMergePolicy(newLogMergePolicy(4))
    );
    addDocs(writer, 10);

    writer.addIndexes(aux);
    assertEquals(1040, writer.getDocStats().maxDoc);
    assertEquals(1000, writer.maxDoc(0));
    writer.close();

    // make sure the index is correct
    verifyNumDocs(dir, 1040);
    dir.close();
    aux.close();
  }

  // case 2: tail segments, invariants hold, no copy
  public void testNoCopySegments() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux);

    IndexWriter writer = newWriter(
        dir,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.APPEND).
            setMaxBufferedDocs(9).
            setMergePolicy(newLogMergePolicy(4))
    );
    addDocs(writer, 2);

    writer.addIndexes(aux);
    assertEquals(1032, writer.getDocStats().maxDoc);
    assertEquals(1000, writer.maxDoc(0));
    writer.close();

    // make sure the index is correct
    verifyNumDocs(dir, 1032);
    dir.close();
    aux.close();
  }

  // case 3: tail segments, invariants hold, copy, invariants hold
  public void testNoMergeAfterCopy() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux);

    IndexWriter writer = newWriter(
        dir,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.APPEND).
            setMaxBufferedDocs(10).
            setMergePolicy(newLogMergePolicy(4))
    );

    writer.addIndexes(aux, new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(aux)));
    assertEquals(1060, writer.getDocStats().maxDoc);
    assertEquals(1000, writer.maxDoc(0));
    writer.close();

    // make sure the index is correct
    verifyNumDocs(dir, 1060);
    dir.close();
    aux.close();
  }

  // case 4: tail segments, invariants hold, copy, invariants not hold
  public void testMergeAfterCopy() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();

    setUpDirs(dir, aux, true);

    IndexWriterConfig dontMergeConfig = new IndexWriterConfig(new MockAnalyzer(random()))
      .setMergePolicy(NoMergePolicy.INSTANCE);
    IndexWriter writer = new IndexWriter(aux, dontMergeConfig);
    for (int i = 0; i < 20; i++) {
      writer.deleteDocuments(new Term("id", "" + i));
    }
    writer.close();
    IndexReader reader = DirectoryReader.open(aux);
    assertEquals(10, reader.numDocs());
    reader.close();

    writer = newWriter(
        dir,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.APPEND).
            setMaxBufferedDocs(4).
            setMergePolicy(newLogMergePolicy(4))
    );

    if (VERBOSE) {
      System.out.println("\nTEST: now addIndexes");
    }
    writer.addIndexes(aux, new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(aux)));
    assertEquals(1020, writer.getDocStats().maxDoc);
    assertEquals(1000, writer.maxDoc(0));
    writer.close();
    dir.close();
    aux.close();
  }

  // case 5: tail segments, invariants not hold
  public void testMoreMerges() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // auxiliary directory
    Directory aux = newDirectory();
    Directory aux2 = newDirectory();

    setUpDirs(dir, aux, true);

    IndexWriter writer = newWriter(
        aux2,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setMaxBufferedDocs(100).
            setMergePolicy(newLogMergePolicy(10))
    );
    writer.addIndexes(aux);
    assertEquals(30, writer.getDocStats().maxDoc);
    assertEquals(3, writer.getSegmentCount());
    writer.close();

    IndexWriterConfig dontMergeConfig = new IndexWriterConfig(new MockAnalyzer(random()))
      .setMergePolicy(NoMergePolicy.INSTANCE);
    writer = new IndexWriter(aux, dontMergeConfig);
    for (int i = 0; i < 27; i++) {
      writer.deleteDocuments(new Term("id", "" + i));
    }
    writer.close();
    IndexReader reader = DirectoryReader.open(aux);
    assertEquals(3, reader.numDocs());
    reader.close();

    dontMergeConfig = new IndexWriterConfig(new MockAnalyzer(random()))
    .setMergePolicy(NoMergePolicy.INSTANCE);
    writer = new IndexWriter(aux2, dontMergeConfig);
    for (int i = 0; i < 8; i++) {
      writer.deleteDocuments(new Term("id", "" + i));
    }
    writer.close();
    reader = DirectoryReader.open(aux2);
    assertEquals(22, reader.numDocs());
    reader.close();

    writer = newWriter(
        dir,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.APPEND).
            setMaxBufferedDocs(6).
            setMergePolicy(newLogMergePolicy(4))
    );

    writer.addIndexes(aux, aux2);
    assertEquals(1040, writer.getDocStats().maxDoc);
    assertEquals(1000, writer.maxDoc(0));
    writer.close();
    dir.close();
    aux.close();
    aux2.close();
  }

  private IndexWriter newWriter(Directory dir, IndexWriterConfig conf)
      throws IOException {
    conf.setMergePolicy(new LogDocMergePolicy());
    final IndexWriter writer = new IndexWriter(dir, conf);
    return writer;
  }

  private void addDocs(IndexWriter writer, int numDocs) throws IOException {
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      doc.add(newTextField("content", "aaa", Field.Store.NO));
      doc.add(new IntPoint("doc", i));
      doc.add(new IntPoint("doc2d", i, i));
      doc.add(new NumericDocValuesField("dv", i));
      writer.addDocument(doc);
    }
  }

  private void addDocs2(IndexWriter writer, int numDocs) throws IOException {
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      doc.add(newTextField("content", "bbb", Field.Store.NO));
      doc.add(new IntPoint("doc", i));
      doc.add(new IntPoint("doc2d", i, i));
      doc.add(new NumericDocValuesField("dv", i));
      writer.addDocument(doc);
    }
  }

  private void verifyNumDocs(Directory dir, int numDocs) throws IOException {
    IndexReader reader = DirectoryReader.open(dir);
    assertEquals(numDocs, reader.maxDoc());
    assertEquals(numDocs, reader.numDocs());
    reader.close();
  }

  private void verifyTermDocs(Directory dir, Term term, int numDocs)
      throws IOException {
    IndexReader reader = DirectoryReader.open(dir);
    PostingsEnum postingsEnum = TestUtil.docs(random(), reader, term.field, term.bytes, null, PostingsEnum.NONE);
    int count = 0;
    while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)
      count++;
    assertEquals(numDocs, count);
    reader.close();
  }

  private void setUpDirs(Directory dir, Directory aux) throws IOException {
    setUpDirs(dir, aux, false);
  }
  
  private void setUpDirs(Directory dir, Directory aux, boolean withID) throws IOException {
    IndexWriter writer = null;

    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000));
    // add 1000 documents in 1 segment
    if (withID) {
      addDocsWithID(writer, 1000, 0);
    } else {
      addDocs(writer, 1000);
    }
    assertEquals(1000, writer.getDocStats().maxDoc);
    assertEquals(1, writer.getSegmentCount());
    writer.close();

    writer = newWriter(
        aux,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setMaxBufferedDocs(1000).
            setMergePolicy(newLogMergePolicy(false, 10))
    );
    // add 30 documents in 3 segments
    for (int i = 0; i < 3; i++) {
      if (withID) {
        addDocsWithID(writer, 10, 10*i);
      } else {
        addDocs(writer, 10);
      }
      writer.close();
      writer = newWriter(
          aux,
          newIndexWriterConfig(new MockAnalyzer(random())).
              setOpenMode(OpenMode.APPEND).
              setMaxBufferedDocs(1000).
              setMergePolicy(newLogMergePolicy(false, 10))
      );
    }
    assertEquals(30, writer.getDocStats().maxDoc);
    assertEquals(3, writer.getSegmentCount());
    writer.close();
  }

  // LUCENE-1270
  public void testHangOnClose() throws IOException {

    Directory dir = newDirectory();
    LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
    lmp.setNoCFSRatio(0.0);
    lmp.setMergeFactor(100);
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
        new MockAnalyzer(random()))
        .setMaxBufferedDocs(5).setMergePolicy(lmp));

    Document doc = new Document();
    FieldType customType = new FieldType(TextField.TYPE_STORED);
    customType.setStoreTermVectors(true);
    customType.setStoreTermVectorPositions(true);
    customType.setStoreTermVectorOffsets(true);
    doc.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType));
    for(int i=0;i<60;i++)
      writer.addDocument(doc);

    Document doc2 = new Document();
    FieldType customType2 = new FieldType();
    customType2.setStored(true);
    doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
    doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
    doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
    doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType2));
    for(int i=0;i<10;i++)
      writer.addDocument(doc2);
    writer.close();

    Directory dir2 = newDirectory();
    lmp = new LogByteSizeMergePolicy();
    lmp.setMinMergeMB(0.0001);
    lmp.setNoCFSRatio(0.0);
    lmp.setMergeFactor(4);
    writer = new IndexWriter(dir2, newIndexWriterConfig(new MockAnalyzer(random()))
        .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp));
    writer.addIndexes(dir);
    writer.close();
    dir.close();
    dir2.close();
  }

  // TODO: these are also in TestIndexWriter... add a simple doc-writing method
  // like this to LuceneTestCase?
  private void addDoc(IndexWriter writer) throws IOException
  {
      Document doc = new Document();
      doc.add(newTextField("content", "aaa", Field.Store.NO));
      writer.addDocument(doc);
  }
  
  private abstract class RunAddIndexesThreads {

    Directory dir, dir2;
    final static int NUM_INIT_DOCS = 17;
    IndexWriter writer2;
    final List<Throwable> failures = new ArrayList<>();
    volatile boolean didClose;
    final DirectoryReader[] readers;
    final int NUM_COPY;
    final Thread[] threads;

    public RunAddIndexesThreads(int numCopy) throws Throwable {
      NUM_COPY = numCopy;
      dir = new MockDirectoryWrapper(random(), new RAMDirectory());
      IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))
          .setMaxBufferedDocs(2));
      for (int i = 0; i < NUM_INIT_DOCS; i++)
        addDoc(writer);
      writer.close();

      dir2 = newDirectory();
      writer2 = new IndexWriter(dir2, new IndexWriterConfig(new MockAnalyzer(random())));
      writer2.commit();
      

      readers = new DirectoryReader[NUM_COPY];
      for(int i=0;i<NUM_COPY;i++)
        readers[i] = DirectoryReader.open(dir);
      int numThreads = TEST_NIGHTLY ? 5 : 2;
      threads = new Thread[numThreads];
    }

    void launchThreads(final int numIter) {

      for(int i=0;i<threads.length;i++) {
        threads[i] = new Thread() {
            @Override
            public void run() {
              try {

                final Directory[] dirs = new Directory[NUM_COPY];
                for(int k=0;k<NUM_COPY;k++)
                  dirs[k] = new MockDirectoryWrapper(random(), TestUtil.ramCopyOf(dir));

                int j=0;

                while(true) {
                  // System.out.println(Thread.currentThread().getName() + ": iter j=" + j);
                  if (numIter > 0 && j == numIter)
                    break;
                  doBody(j++, dirs);
                }
              } catch (Throwable t) {
                handle(t);
              }
            }
          };
      }

      for (Thread thread : threads) {
        thread.start();
      }
    }

    void joinThreads() throws Exception {
      for (Thread thread : threads) {
        thread.join();
      }
    }

    void close(boolean doWait) throws Throwable {
      didClose = true;
      if (doWait == false) {
        writer2.rollback();
      } else {
        writer2.close();
      }
    }

    void closeDir() throws Throwable {
      for(int i=0;i<NUM_COPY;i++)
        readers[i].close();
      dir2.close();
    }

    abstract void doBody(int j, Directory[] dirs) throws Throwable;
    abstract void handle(Throwable t);
  }

  private class CommitAndAddIndexes extends RunAddIndexesThreads {
    public CommitAndAddIndexes(int numCopy) throws Throwable {
      super(numCopy);
    }

    @Override
    void handle(Throwable t) {
      t.printStackTrace(System.out);
      synchronized(failures) {
        failures.add(t);
      }
    }

    @Override
    void doBody(int j, Directory[] dirs) throws Throwable {
      switch(j%5) {
      case 0:
        if (VERBOSE) {
          System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(Dir[]) then full merge");
        }
        writer2.addIndexes(dirs);
        try {
          writer2.forceMerge(1);
        } catch (IOException ioe) {
          if (ioe.getCause() instanceof MergePolicy.MergeAbortedException) {
            // OK
          } else {
            throw ioe;
          }
        }
        break;
      case 1:
        if (VERBOSE) {
          System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(Dir[])");
        }
        writer2.addIndexes(dirs);
        break;
      case 2:
        if (VERBOSE) {
          System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(LeafReader[])");
        }
        TestUtil.addIndexesSlowly(writer2, readers);
        break;
      case 3:
        if (VERBOSE) {
          System.out.println(Thread.currentThread().getName() + ": TEST: addIndexes(Dir[]) then maybeMerge");
        }
        writer2.addIndexes(dirs);
        writer2.maybeMerge();
        break;
      case 4:
        if (VERBOSE) {
          System.out.println(Thread.currentThread().getName() + ": TEST: commit");
        }
        writer2.commit();
      }
    }
  }
  
  // LUCENE-1335: test simultaneous addIndexes & commits
  // from multiple threads
  public void testAddIndexesWithThreads() throws Throwable {

    final int NUM_ITER = TEST_NIGHTLY ? 15 : 5;
    final int NUM_COPY = 3;
    CommitAndAddIndexes c = new CommitAndAddIndexes(NUM_COPY);
    c.launchThreads(NUM_ITER);

    for(int i=0;i<100;i++)
      addDoc(c.writer2);

    c.joinThreads();

    int expectedNumDocs = 100+NUM_COPY*(4*NUM_ITER/5)*c.threads.length*RunAddIndexesThreads.NUM_INIT_DOCS;
    assertEquals("expected num docs don't match - failures: " + c.failures, expectedNumDocs, c.writer2.getDocStats().numDocs);

    c.close(true);

    assertTrue("found unexpected failures: " + c.failures, c.failures.isEmpty());

    IndexReader reader = DirectoryReader.open(c.dir2);
    assertEquals(expectedNumDocs, reader.numDocs());
    reader.close();

    c.closeDir();
  }

  private class CommitAndAddIndexes2 extends CommitAndAddIndexes {
    public CommitAndAddIndexes2(int numCopy) throws Throwable {
      super(numCopy);
    }

    @Override
    void handle(Throwable t) {
      if (!(t instanceof AlreadyClosedException) && !(t instanceof NullPointerException)) {
        t.printStackTrace(System.out);
        synchronized(failures) {
          failures.add(t);
        }
      }
    }
  }

  // LUCENE-1335: test simultaneous addIndexes & close
  public void testAddIndexesWithClose() throws Throwable {
    final int NUM_COPY = 3;
    CommitAndAddIndexes2 c = new CommitAndAddIndexes2(NUM_COPY);
    //c.writer2.setInfoStream(System.out);
    c.launchThreads(-1);

    // Close w/o first stopping/joining the threads
    c.close(true);
    //c.writer2.close();

    c.joinThreads();

    c.closeDir();

    assertTrue(c.failures.size() == 0);
  }

  private class CommitAndAddIndexes3 extends RunAddIndexesThreads {
    public CommitAndAddIndexes3(int numCopy) throws Throwable {
      super(numCopy);
    }

    @Override
    void doBody(int j, Directory[] dirs) throws Throwable {
      switch(j%5) {
      case 0:
        if (VERBOSE) {
          System.out.println("TEST: " + Thread.currentThread().getName() + ": addIndexes + full merge");
        }
        writer2.addIndexes(dirs);
        writer2.forceMerge(1);
        break;
      case 1:
        if (VERBOSE) {
          System.out.println("TEST: " + Thread.currentThread().getName() + ": addIndexes");
        }
        writer2.addIndexes(dirs);
        break;
      case 2:
        if (VERBOSE) {
          System.out.println("TEST: " + Thread.currentThread().getName() + ": addIndexes(LR[])");
        }
        TestUtil.addIndexesSlowly(writer2, readers);
        break;
      case 3:
        if (VERBOSE) {
          System.out.println("TEST: " + Thread.currentThread().getName() + ": full merge");
        }
        writer2.forceMerge(1);
        break;
      case 4:
        if (VERBOSE) {
          System.out.println("TEST: " + Thread.currentThread().getName() + ": commit");
        }
        writer2.commit();
      }
    }

    @Override
    void handle(Throwable t) {
      boolean report = true;

      if (t instanceof AlreadyClosedException || t instanceof MergePolicy.MergeAbortedException || t instanceof NullPointerException) {
        report = !didClose;
      } else if (t instanceof FileNotFoundException || t instanceof NoSuchFileException)  {
        report = !didClose;
      } else if (t instanceof IOException)  {
        Throwable t2 = t.getCause();
        if (t2 instanceof MergePolicy.MergeAbortedException) {
          report = !didClose;
        }
      }
      if (report) {
        t.printStackTrace(System.out);
        synchronized(failures) {
          failures.add(t);
        }
      }
    }
  }

  // LUCENE-1335: test simultaneous addIndexes & close
  public void testAddIndexesWithCloseNoWait() throws Throwable {

    final int NUM_COPY = 50;
    CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
    c.launchThreads(-1);

    Thread.sleep(TestUtil.nextInt(random(), 10, 500));

    // Close w/o first stopping/joining the threads
    if (VERBOSE) {
      System.out.println("TEST: now close(false)");
    }
    c.close(false);

    c.joinThreads();

    if (VERBOSE) {
      System.out.println("TEST: done join threads");
    }
    c.closeDir();

    assertTrue(c.failures.size() == 0);
  }

  // LUCENE-1335: test simultaneous addIndexes & close
  public void testAddIndexesWithRollback() throws Throwable {

    final int NUM_COPY = TEST_NIGHTLY ? 50 : 5;
    CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
    c.launchThreads(-1);

    Thread.sleep(TestUtil.nextInt(random(), 10, 500));

    // Close w/o first stopping/joining the threads
    if (VERBOSE) {
      System.out.println("TEST: now force rollback");
    }
    c.didClose = true;
    MergeScheduler ms = c.writer2.getConfig().getMergeScheduler();

    c.writer2.rollback();

    c.joinThreads();

    if (ms instanceof ConcurrentMergeScheduler) {
      assertEquals(0, ((ConcurrentMergeScheduler) ms).mergeThreadCount());
    }

    c.closeDir();

    assertTrue(c.failures.size() == 0);
  }
 
  // LUCENE-2996: tests that addIndexes(IndexReader) applies existing deletes correctly.
  public void testExistingDeletes() throws Exception {
    Directory[] dirs = new Directory[2];
    for (int i = 0; i < dirs.length; i++) {
      dirs[i] = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
      IndexWriter writer = new IndexWriter(dirs[i], conf);
      Document doc = new Document();
      doc.add(new StringField("id", "myid", Field.Store.NO));
      writer.addDocument(doc);
      writer.close();
    }

    IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter writer = new IndexWriter(dirs[0], conf);

    // Now delete the document
    writer.deleteDocuments(new Term("id", "myid"));
    try (DirectoryReader r = DirectoryReader.open(dirs[1])) {
      TestUtil.addIndexesSlowly(writer, r);
    }
    writer.commit();
    assertEquals("Documents from the incoming index should not have been deleted", 1, writer.getDocStats().numDocs);
    writer.close();

    for (Directory dir : dirs) {
      dir.close();
    }

  }
  
  // just like addDocs but with ID, starting from docStart
  private void addDocsWithID(IndexWriter writer, int numDocs, int docStart) throws IOException {
    for (int i = 0; i < numDocs; i++) {
      Document doc = new Document();
      doc.add(newTextField("content", "aaa", Field.Store.NO));
      doc.add(newTextField("id", "" + (docStart + i), Field.Store.YES));
      doc.add(new IntPoint("doc", i));
      doc.add(new IntPoint("doc2d", i, i));
      doc.add(new NumericDocValuesField("dv", i));
      writer.addDocument(doc);
    }
  }

  public void testSimpleCaseCustomCodec() throws IOException {
    // main directory
    Directory dir = newDirectory();
    // two auxiliary directories
    Directory aux = newDirectory();
    Directory aux2 = newDirectory();
    Codec codec = new CustomPerFieldCodec();
    IndexWriter writer = null;

    writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))
                              .setOpenMode(OpenMode.CREATE).setCodec(codec));
    // add 100 documents
    addDocsWithID(writer, 100, 0);
    assertEquals(100, writer.getDocStats().maxDoc);
    writer.commit();
    writer.close();
    TestUtil.checkIndex(dir);

    writer = newWriter(
        aux,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setCodec(codec).
            setMaxBufferedDocs(10).
            setMergePolicy(newLogMergePolicy(false))
    );
    // add 40 documents in separate files
    addDocs(writer, 40);
    assertEquals(40, writer.getDocStats().maxDoc);
    writer.commit();
    writer.close();

    writer = newWriter(
        aux2,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.CREATE).
            setCodec(codec)
    );
    // add 40 documents in compound files
    addDocs2(writer, 50);
    assertEquals(50, writer.getDocStats().maxDoc);
    writer.commit();
    writer.close();

    // test doc count before segments are merged
    writer = newWriter(
        dir,
        newIndexWriterConfig(new MockAnalyzer(random())).
            setOpenMode(OpenMode.APPEND).
            setCodec(codec)
    );
    assertEquals(100, writer.getDocStats().maxDoc);
    writer.addIndexes(aux, aux2);
    assertEquals(190, writer.getDocStats().maxDoc);
    writer.close();

    dir.close();
    aux.close();
    aux2.close();
  }

  private static final class CustomPerFieldCodec extends AssertingCodec {
    private final PostingsFormat directFormat = PostingsFormat.forName("Direct");
    private final PostingsFormat defaultFormat = TestUtil.getDefaultPostingsFormat();

    @Override
    public PostingsFormat getPostingsFormatForField(String field) {
      if (field.equals("id")) {
        return directFormat;
      } else {
        return defaultFormat;
      }
    }
  }

  // LUCENE-2790: tests that the non CFS files were deleted by addIndexes
  public void testNonCFSLeftovers() throws Exception {
    Directory[] dirs = new Directory[2];
    for (int i = 0; i < dirs.length; i++) {
      dirs[i] = new RAMDirectory();
      IndexWriter w = new IndexWriter(dirs[i], new IndexWriterConfig(new MockAnalyzer(random())));
      Document d = new Document();
      FieldType customType = new FieldType(TextField.TYPE_STORED);
      customType.setStoreTermVectors(true);
      d.add(new Field("c", "v", customType));
      w.addDocument(d);
      w.close();
    }
    
    DirectoryReader[] readers = new DirectoryReader[] { DirectoryReader.open(dirs[0]), DirectoryReader.open(dirs[1]) };
    
    MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory());
    IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy(true));
    MergePolicy lmp = conf.getMergePolicy();
    // Force creation of CFS:
    lmp.setNoCFSRatio(1.0);
    lmp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
    IndexWriter w3 = new IndexWriter(dir, conf);
    TestUtil.addIndexesSlowly(w3, readers);
    w3.close();
    // we should now see segments_X,
    // _Y.cfs,_Y.cfe, _Z.si
    SegmentInfos sis = SegmentInfos.readLatestCommit(dir);
    assertEquals("Only one compound segment should exist", 1, sis.size());
    assertTrue(sis.info(0).info.getUseCompoundFile());
    dir.close();
  }
  
  private static final class UnRegisteredCodec extends FilterCodec {
    public UnRegisteredCodec() {
      super("NotRegistered", TestUtil.getDefaultCodec());
    }
  }
  
  /*
   * simple test that ensures we getting expected exceptions 
   */
  public void testAddIndexMissingCodec() throws IOException {
    BaseDirectoryWrapper toAdd = newDirectory();
    // Disable checkIndex, else we get an exception because
    // of the unregistered codec:
    toAdd.setCheckIndexOnClose(false);
    {
      IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
      conf.setCodec(new UnRegisteredCodec());
      IndexWriter w = new IndexWriter(toAdd, conf);
      Document doc = new Document();
      FieldType customType = new FieldType();
      customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS); 
      doc.add(newField("foo", "bar", customType));
      w.addDocument(doc);
      w.close();
    }

    {
      Directory dir = newDirectory();
      IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
      conf.setCodec(TestUtil.alwaysPostingsFormat(new DirectPostingsFormat()));
      IndexWriter w = new IndexWriter(dir, conf);
      expectThrows(IllegalArgumentException.class, () -> {
        w.addIndexes(toAdd);
      });
      w.close();
      IndexReader open = DirectoryReader.open(dir);
      assertEquals(0, open.numDocs());
      open.close();
      dir.close();
    }

    expectThrows(IllegalArgumentException.class, () -> {
      DirectoryReader.open(toAdd);
    });
    toAdd.close();
  }

  // LUCENE-3575
  public void testFieldNamesChanged() throws IOException {
    Directory d1 = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), d1);
    Document doc = new Document();
    doc.add(newStringField("f1", "doc1 field1", Field.Store.YES));
    doc.add(newStringField("id", "1", Field.Store.YES));
    w.addDocument(doc);
    DirectoryReader r1 = w.getReader();
    w.close();

    Directory d2 = newDirectory();
    w = new RandomIndexWriter(random(), d2);
    doc = new Document();
    doc.add(newStringField("f2", "doc2 field2", Field.Store.YES));
    doc.add(newStringField("id", "2", Field.Store.YES));
    w.addDocument(doc);
    DirectoryReader r2 = w.getReader();
    w.close();

    Directory d3 = newDirectory();
    w = new RandomIndexWriter(random(), d3);
    TestUtil.addIndexesSlowly(w.w, r1, r2);
    r1.close();
    d1.close();
    r2.close();
    d2.close();

    IndexReader r3 = w.getReader();
    w.close();
    assertEquals(2, r3.numDocs());
    for(int docID=0;docID<2;docID++) {
      Document d = r3.document(docID);
      if (d.get("id").equals("1")) {
        assertEquals("doc1 field1", d.get("f1"));
      } else {
        assertEquals("doc2 field2", d.get("f2"));
      }
    }
    r3.close();
    d3.close();
  }
  
  public void testAddEmpty() throws Exception {
    Directory d1 = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), d1);
    w.addIndexes(new CodecReader[0]);
    w.close();
    DirectoryReader dr = DirectoryReader.open(d1);
    for (LeafReaderContext ctx : dr.leaves()) {
      assertTrue("empty segments should be dropped by addIndexes", ctx.reader().maxDoc() > 0);
    }
    dr.close();
    d1.close();
  }

  // Currently it's impossible to end up with a segment with all documents
  // deleted, as such segments are dropped. Still, to validate that addIndexes
  // works with such segments, or readers that end up in such state, we fake an
  // all deleted segment.
  public void testFakeAllDeleted() throws Exception {
    Directory src = newDirectory(), dest = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), src);
    w.addDocument(new Document());
    LeafReader allDeletedReader = new AllDeletedFilterReader(w.getReader().leaves().get(0).reader());
    w.close();
    
    w = new RandomIndexWriter(random(), dest);
    w.addIndexes(SlowCodecReaderWrapper.wrap(allDeletedReader));
    w.close();
    DirectoryReader dr = DirectoryReader.open(src);
    for (LeafReaderContext ctx : dr.leaves()) {
      assertTrue("empty segments should be dropped by addIndexes", ctx.reader().maxDoc() > 0);
    }
    dr.close();
    allDeletedReader.close();
    src.close();
    dest.close();
  }

  /** Make sure an open IndexWriter on an incoming Directory
   *  causes a LockObtainFailedException */
  public void testLocksBlock() throws Exception {
    Directory src = newDirectory();
    RandomIndexWriter w1 = new RandomIndexWriter(random(), src);
    w1.addDocument(new Document());
    w1.commit();

    Directory dest = newDirectory();

    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    RandomIndexWriter w2 = new RandomIndexWriter(random(), dest, iwc);

    expectThrows(LockObtainFailedException.class, () -> {
      w2.addIndexes(src);
    });

    w1.close();
    w2.close();
    IOUtils.close(src, dest);
  }

  public void testIllegalIndexSortChange1() throws Exception {
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc1.setIndexSort(new Sort(new SortField("foo", SortField.Type.INT)));
    RandomIndexWriter w1 = new RandomIndexWriter(random(), dir1, iwc1);
    w1.addDocument(new Document());
    w1.commit();
    w1.addDocument(new Document());
    w1.commit();
    // so the index sort is in fact burned into the index:
    w1.forceMerge(1);
    w1.close();

    Directory dir2 = newDirectory();
    IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc2.setIndexSort(new Sort(new SortField("foo", SortField.Type.STRING)));
    RandomIndexWriter w2 = new RandomIndexWriter(random(), dir2, iwc2);
    String message = expectThrows(IllegalArgumentException.class, () -> {
        w2.addIndexes(dir1);
      }).getMessage();
    assertEquals("cannot change index sort from <int: \"foo\"> to <string: \"foo\">", message);
    IOUtils.close(dir1, w2, dir2);
  }

  public void testIllegalIndexSortChange2() throws Exception {
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc1.setIndexSort(new Sort(new SortField("foo", SortField.Type.INT)));
    RandomIndexWriter w1 = new RandomIndexWriter(random(), dir1, iwc1);
    w1.addDocument(new Document());
    w1.commit();
    w1.addDocument(new Document());
    w1.commit();
    // so the index sort is in fact burned into the index:
    w1.forceMerge(1);
    w1.close();

    Directory dir2 = newDirectory();
    IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc2.setIndexSort(new Sort(new SortField("foo", SortField.Type.STRING)));
    RandomIndexWriter w2 = new RandomIndexWriter(random(), dir2, iwc2);
    IndexReader r1 = DirectoryReader.open(dir1);
    String message = expectThrows(IllegalArgumentException.class, () -> {
        w2.addIndexes((SegmentReader) getOnlyLeafReader(r1));
      }).getMessage();
    assertEquals("cannot change index sort from <int: \"foo\"> to <string: \"foo\">", message);
    IOUtils.close(r1, dir1, w2, dir2);
  }

  public void testAddIndexesDVUpdateSameSegmentName() throws Exception {
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w1 = new IndexWriter(dir1, iwc1);
    Document doc = new Document();
    doc.add(new StringField("id", "1", Field.Store.YES));
    doc.add(new StringField("version", "1", Field.Store.YES));
    doc.add(new NumericDocValuesField("soft_delete", 1));
    w1.addDocument(doc);
    w1.flush();

    w1.updateDocValues(new Term("id", "1"), new NumericDocValuesField("soft_delete", 1));
    w1.commit();
    w1.close();

    IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
    Directory dir2 = newDirectory();
    IndexWriter w2 = new IndexWriter(dir2, iwc2);
    w2.addIndexes(dir1);
    w2.commit();
    w2.close();

    if (VERBOSE) {
      System.out.println("\nTEST: now open w3");
    }
    IndexWriterConfig iwc3 = newIndexWriterConfig(new MockAnalyzer(random()));
    if (VERBOSE) {
      iwc3.setInfoStream(System.out);
    }        
    IndexWriter w3 = new IndexWriter(dir2, iwc3);
    w3.close();

    iwc3 = newIndexWriterConfig(new MockAnalyzer(random()));
    w3 = new IndexWriter(dir2, iwc3);
    w3.close();
    dir1.close();
    dir2.close();
  }

  public void testAddIndexesDVUpdateNewSegmentName() throws Exception {
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w1 = new IndexWriter(dir1, iwc1);
    Document doc = new Document();
    doc.add(new StringField("id", "1", Field.Store.YES));
    doc.add(new StringField("version", "1", Field.Store.YES));
    doc.add(new NumericDocValuesField("soft_delete", 1));
    w1.addDocument(doc);
    w1.flush();

    w1.updateDocValues(new Term("id", "1"), new NumericDocValuesField("soft_delete", 1));
    w1.commit();
    w1.close();

    IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
    Directory dir2 = newDirectory();
    IndexWriter w2 = new IndexWriter(dir2, iwc2);
    w2.addDocument(new Document());
    w2.commit();
    
    w2.addIndexes(dir1);
    w2.commit();
    w2.close();

    if (VERBOSE) {
      System.out.println("\nTEST: now open w3");
    }
    IndexWriterConfig iwc3 = newIndexWriterConfig(new MockAnalyzer(random()));
    if (VERBOSE) {
      iwc3.setInfoStream(System.out);
    }        
    IndexWriter w3 = new IndexWriter(dir2, iwc3);
    w3.close();

    iwc3 = newIndexWriterConfig(new MockAnalyzer(random()));
    w3 = new IndexWriter(dir2, iwc3);
    w3.close();
    dir1.close();
    dir2.close();
  }

  public void testAddIndicesWithSoftDeletes() throws IOException {
    Directory dir1 = newDirectory();
    IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("soft_delete");
    IndexWriter writer = new IndexWriter(dir1, iwc1);
    for (int i = 0; i < 30; i++) {
      Document doc = new Document();
      int docID = random().nextInt(5);
      doc.add(new StringField("id", "" + docID, Field.Store.YES));
      writer.softUpdateDocument(new Term("id", "" + docID), doc, new NumericDocValuesField("soft_delete", 1));
      if (random().nextBoolean()) {
        writer.flush();
      }
    }
    writer.commit();
    writer.close();
    DirectoryReader reader = DirectoryReader.open(dir1);
    DirectoryReader wrappedReader = new SoftDeletesDirectoryReaderWrapper(reader, "soft_delete");
    Directory dir2 = newDirectory();
    int numDocs = reader.numDocs();
    int maxDoc = reader.maxDoc();
    assertEquals(numDocs, maxDoc);
    iwc1 = newIndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("soft_delete");
    writer = new IndexWriter(dir2, iwc1);
    CodecReader[] readers = new CodecReader[reader.leaves().size()];
    for (int i = 0; i < readers.length; i++) {
      readers[i] = (CodecReader)reader.leaves().get(i).reader();
    }
    writer.addIndexes(readers);
    assertEquals(wrappedReader.numDocs(), writer.getDocStats().numDocs);
    assertEquals(maxDoc, writer.getDocStats().maxDoc);
    writer.commit();
    SegmentCommitInfo commitInfo = writer.cloneSegmentInfos().info(0);
    assertEquals(maxDoc-wrappedReader.numDocs(), commitInfo.getSoftDelCount());
    writer.close();
    Directory dir3 = newDirectory();
    iwc1 = newIndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("soft_delete");
    writer = new IndexWriter(dir3, iwc1);
    // Resize as some fully deleted sub-readers might be dropped in the wrappedReader
    readers = new CodecReader[(wrappedReader.leaves().size())];
    for (int i = 0; i < readers.length; i++) {
      readers[i] = (CodecReader)wrappedReader.leaves().get(i).reader();
    }
    writer.addIndexes(readers);
    assertEquals(wrappedReader.numDocs(), writer.getDocStats().numDocs);
    assertEquals(wrappedReader.numDocs(), writer.getDocStats().maxDoc);
    IOUtils.close(reader, writer, dir3, dir2, dir1);
  }
}