/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Function;
import java.util.stream.Collectors;

import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.store.MockDirectoryWrapper.Throttling;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;

import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import com.carrotsearch.randomizedtesting.generators.RandomStrings;

/**
 * Base class aiming at testing {@link StoredFieldsFormat stored fields formats}.
 * To test a new format, all you need is to register a new {@link Codec} which
 * uses it and extend this class and override {@link #getCodec()}.
 * @lucene.experimental
 */
public abstract class BaseStoredFieldsFormatTestCase extends BaseIndexFileFormatTestCase {

  @Override
  protected void addRandomFields(Document d) {
    final int numValues = random().nextInt(3);
    for (int i = 0; i < numValues; ++i) {
      d.add(new StoredField("f", TestUtil.randomSimpleString(random(), 100)));
    }
  }

  public void testRandomStoredFields() throws IOException {
    Directory dir = newDirectory();
    Random rand = random();
    RandomIndexWriter w = new RandomIndexWriter(rand, dir, newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(TestUtil.nextInt(rand, 5, 20)));
    //w.w.setNoCFSRatio(0.0);
    final int docCount = atLeast(200);
    final int fieldCount = TestUtil.nextInt(rand, 1, 5);

    final List<Integer> fieldIDs = new ArrayList<>();

    FieldType customType = new FieldType(TextField.TYPE_STORED);
    customType.setTokenized(false);
    Field idField = newField("id", "", customType);

    for(int i=0;i<fieldCount;i++) {
      fieldIDs.add(i);
    }

    final Map<String,Document> docs = new HashMap<>();

    if (VERBOSE) {
      System.out.println("TEST: build index docCount=" + docCount);
    }

    FieldType customType2 = new FieldType();
    customType2.setStored(true);
    for(int i=0;i<docCount;i++) {
      Document doc = new Document();
      doc.add(idField);
      final String id = ""+i;
      idField.setStringValue(id);
      docs.put(id, doc);
      if (VERBOSE) {
        System.out.println("TEST: add doc id=" + id);
      }

      for(int field: fieldIDs) {
        final String s;
        if (rand.nextInt(4) != 3) {
          s = TestUtil.randomUnicodeString(rand, 1000);
          doc.add(newField("f"+field, s, customType2));
        } else {
          s = null;
        }
      }
      w.addDocument(doc);
      if (rand.nextInt(50) == 17) {
        // mixup binding of field name -> Number every so often
        Collections.shuffle(fieldIDs, random());
      }
      if (rand.nextInt(5) == 3 && i > 0) {
        final String delID = ""+rand.nextInt(i);
        if (VERBOSE) {
          System.out.println("TEST: delete doc id=" + delID);
        }
        w.deleteDocuments(new Term("id", delID));
        docs.remove(delID);
      }
    }

    if (VERBOSE) {
      System.out.println("TEST: " + docs.size() + " docs in index; now load fields");
    }
    if (docs.size() > 0) {
      String[] idsList = docs.keySet().toArray(new String[docs.size()]);

      for(int x=0;x<2;x++) {
        DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
        IndexSearcher s = newSearcher(r);

        if (VERBOSE) {
          System.out.println("TEST: cycle x=" + x + " r=" + r);
        }

        int num = atLeast(100);
        for(int iter=0;iter<num;iter++) {
          String testID = idsList[rand.nextInt(idsList.length)];
          if (VERBOSE) {
            System.out.println("TEST: test id=" + testID);
          }
          TopDocs hits = s.search(new TermQuery(new Term("id", testID)), 1);
          assertEquals(1, hits.totalHits.value);
          Document doc = r.document(hits.scoreDocs[0].doc);
          Document docExp = docs.get(testID);
          for(int i=0;i<fieldCount;i++) {
            assertEquals("doc " + testID + ", field f" + fieldCount + " is wrong", docExp.get("f"+i),  doc.get("f"+i));
          }
        }
        r.close();
        w.forceMerge(1);
      }
    }
    w.close();
    dir.close();
  }
  
  // LUCENE-1727: make sure doc fields are stored in order
  public void testStoredFieldsOrder() throws Throwable {
    Directory d = newDirectory();
    IndexWriter w = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();

    FieldType customType = new FieldType();
    customType.setStored(true);
    doc.add(newField("zzz", "a b c", customType));
    doc.add(newField("aaa", "a b c", customType));
    doc.add(newField("zzz", "1 2 3", customType));
    w.addDocument(doc);
    IndexReader r = maybeWrapWithMergingReader(w.getReader());
    Document doc2 = r.document(0);
    Iterator<IndexableField> it = doc2.getFields().iterator();
    assertTrue(it.hasNext());
    Field f = (Field) it.next();
    assertEquals(f.name(), "zzz");
    assertEquals(f.stringValue(), "a b c");

    assertTrue(it.hasNext());
    f = (Field) it.next();
    assertEquals(f.name(), "aaa");
    assertEquals(f.stringValue(), "a b c");

    assertTrue(it.hasNext());
    f = (Field) it.next();
    assertEquals(f.name(), "zzz");
    assertEquals(f.stringValue(), "1 2 3");
    assertFalse(it.hasNext());
    r.close();
    w.close();
    d.close();
  }
  
  // LUCENE-1219
  public void testBinaryFieldOffsetLength() throws IOException {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    byte[] b = new byte[50];
    for(int i=0;i<50;i++)
      b[i] = (byte) (i+77);

    Document doc = new Document();
    Field f = new StoredField("binary", b, 10, 17);
    byte[] bx = f.binaryValue().bytes;
    assertTrue(bx != null);
    assertEquals(50, bx.length);
    assertEquals(10, f.binaryValue().offset);
    assertEquals(17, f.binaryValue().length);
    doc.add(f);
    w.addDocument(doc);
    w.close();

    IndexReader ir = DirectoryReader.open(dir);
    Document doc2 = ir.document(0);
    IndexableField f2 = doc2.getField("binary");
    b = f2.binaryValue().bytes;
    assertTrue(b != null);
    assertEquals(17, b.length, 17);
    assertEquals(87, b[0]);
    ir.close();
    dir.close();
  }
  
  public void testNumericField() throws Exception {
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(500);
    final Number[] answers = new Number[numDocs];
    final Class<?>[] typeAnswers = new Class<?>[numDocs];
    for(int id=0;id<numDocs;id++) {
      Document doc = new Document();
      final Field nf;
      final Number answer;
      final Class<?> typeAnswer;
      if (random().nextBoolean()) {
        // float/double
        if (random().nextBoolean()) {
          final float f = random().nextFloat();
          answer = Float.valueOf(f);
          nf = new StoredField("nf", f);
          typeAnswer = Float.class;
        } else {
          final double d = random().nextDouble();
          answer = Double.valueOf(d);
          nf = new StoredField("nf", d);
          typeAnswer = Double.class;
        }
      } else {
        // int/long
        if (random().nextBoolean()) {
          final int i = random().nextInt();
          answer = Integer.valueOf(i);
          nf = new StoredField("nf", i);
          typeAnswer = Integer.class;
        } else {
          final long l = random().nextLong();
          answer = Long.valueOf(l);
          nf = new StoredField("nf", l);
          typeAnswer = Long.class;
        }
      }
      doc.add(nf);
      answers[id] = answer;
      typeAnswers[id] = typeAnswer;
      doc.add(new StoredField("id", id));
      doc.add(new IntPoint("id", id));
      doc.add(new NumericDocValuesField("id", id));
      w.addDocument(doc);
    }
    final DirectoryReader r = maybeWrapWithMergingReader(w.getReader());
    w.close();
    
    assertEquals(numDocs, r.numDocs());

    for(LeafReaderContext ctx : r.leaves()) {
      final LeafReader sub = ctx.reader();
      final NumericDocValues ids = DocValues.getNumeric(sub, "id");
      for(int docID=0;docID<sub.numDocs();docID++) {
        final Document doc = sub.document(docID);
        final Field f = (Field) doc.getField("nf");
        assertTrue("got f=" + f, f instanceof StoredField);
        assertEquals(docID, ids.nextDoc());
        assertEquals(answers[(int) ids.longValue()], f.numericValue());
      }
    }
    r.close();
    dir.close();
  }

  public void testIndexedBit() throws Exception {
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    Document doc = new Document();
    FieldType onlyStored = new FieldType();
    onlyStored.setStored(true);
    doc.add(new Field("field", "value", onlyStored));
    doc.add(new StringField("field2", "value", Field.Store.YES));
    w.addDocument(doc);
    IndexReader r = maybeWrapWithMergingReader(w.getReader());
    w.close();
    assertEquals(IndexOptions.NONE, r.document(0).getField("field").fieldType().indexOptions());
    assertNotNull(r.document(0).getField("field2").fieldType().indexOptions());
    r.close();
    dir.close();
  }
  
  public void testReadSkip() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
    
    FieldType ft = new FieldType();
    ft.setStored(true);
    ft.freeze();

    final String string = TestUtil.randomSimpleString(random(), 50);
    final byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
    final long l = random().nextBoolean() ? random().nextInt(42) : random().nextLong();
    final int i = random().nextBoolean() ? random().nextInt(42) : random().nextInt();
    final float f = random().nextFloat();
    final double d = random().nextDouble();

    List<Field> fields = Arrays.asList(
        new Field("bytes", bytes, ft),
        new Field("string", string, ft),
        new StoredField("long", l),
        new StoredField("int", i),
        new StoredField("float", f),
        new StoredField("double", d)
    );

    for (int k = 0; k < 100; ++k) {
      Document doc = new Document();
      for (Field fld : fields) {
        doc.add(fld);
      }
      iw.w.addDocument(doc);
    }
    iw.commit();

    final DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    final int docID = random().nextInt(100);
    for (Field fld : fields) {
      String fldName = fld.name();
      final Document sDoc = reader.document(docID, Collections.singleton(fldName));
      final IndexableField sField = sDoc.getField(fldName);
      if (Field.class.equals(fld.getClass())) {
        assertEquals(fld.binaryValue(), sField.binaryValue());
        assertEquals(fld.stringValue(), sField.stringValue());
      } else {
        assertEquals(fld.numericValue(), sField.numericValue());
      }
    }
    reader.close();
    iw.close();
    dir.close();
  }
  
  public void testEmptyDocs() throws IOException {
    Directory dir = newDirectory();
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
    
    // make sure that the fact that documents might be empty is not a problem
    final Document emptyDoc = new Document();
    final int numDocs = random().nextBoolean() ? 1 : atLeast(1000);
    for (int i = 0; i < numDocs; ++i) {
      iw.addDocument(emptyDoc);
    }
    iw.commit();
    final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    for (int i = 0; i < numDocs; ++i) {
      final Document doc = rd.document(i);
      assertNotNull(doc);
      assertTrue(doc.getFields().isEmpty());
    }
    rd.close();
    
    iw.close();
    dir.close();
  }
  
  public void testConcurrentReads() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);
    
    // make sure the readers are properly cloned
    final Document doc = new Document();
    final Field field = new StringField("fld", "", Store.YES);
    doc.add(field);
    final int numDocs = atLeast(1000);
    for (int i = 0; i < numDocs; ++i) {
      field.setStringValue("" + i);
      iw.addDocument(doc);
    }
    iw.commit();

    final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    final IndexSearcher searcher = new IndexSearcher(rd);
    final int concurrentReads = atLeast(5);
    final int readsPerThread = atLeast(50);
    final List<Thread> readThreads = new ArrayList<>();
    final AtomicReference<Exception> ex = new AtomicReference<>();
    for (int i = 0; i < concurrentReads; ++i) {
      readThreads.add(new Thread() {

        int[] queries;

        {
          queries = new int[readsPerThread];
          for (int i = 0; i < queries.length; ++i) {
            queries[i] = random().nextInt(numDocs);
          }
        }

        @Override
        public void run() {
          for (int q : queries) {
            final Query query = new TermQuery(new Term("fld", "" + q));
            try {
              final TopDocs topDocs = searcher.search(query, 1);
              if (topDocs.totalHits.value != 1) {
                throw new IllegalStateException("Expected 1 hit, got " + topDocs.totalHits.value);
              }
              final Document sdoc = rd.document(topDocs.scoreDocs[0].doc);
              if (sdoc == null || sdoc.get("fld") == null) {
                throw new IllegalStateException("Could not find document " + q);
              }
              if (!Integer.toString(q).equals(sdoc.get("fld"))) {
                throw new IllegalStateException("Expected " + q + ", but got " + sdoc.get("fld"));
              }
            } catch (Exception e) {
              ex.compareAndSet(null, e);
            }
          }
        }
      });
    }
    for (Thread thread : readThreads) {
      thread.start();
    }
    for (Thread thread : readThreads) {
      thread.join();
    }
    rd.close();
    if (ex.get() != null) {
      throw ex.get();
    }
    
    iw.close();
    dir.close();
  }
  
  private byte[] randomByteArray(int length, int max) {
    final byte[] result = new byte[length];
    for (int i = 0; i < length; ++i) {
      result[i] = (byte) random().nextInt(max);
    }
    return result;
  }
  
  public void testWriteReadMerge() throws IOException {
    // get another codec, other than the default: so we are merging segments across different codecs
    final Codec otherCodec;
    if ("SimpleText".equals(Codec.getDefault().getName())) {
      otherCodec = TestUtil.getDefaultCodec();
    } else {
      otherCodec = new SimpleTextCodec();
    }
    Directory dir = newDirectory();
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);

    final int docCount = atLeast(200);
    final byte[][][] data = new byte [docCount][][];
    for (int i = 0; i < docCount; ++i) {
      final int fieldCount = rarely()
          ? RandomNumbers.randomIntBetween(random(), 1, 500)
          : RandomNumbers.randomIntBetween(random(), 1, 5);
      data[i] = new byte[fieldCount][];
      for (int j = 0; j < fieldCount; ++j) {
        final int length = rarely()
            ? random().nextInt(1000)
            : random().nextInt(10);
        final int max = rarely() ? 256 : 2;
        data[i][j] = randomByteArray(length, max);
      }
    }

    final FieldType type = new FieldType(StringField.TYPE_STORED);
    type.setIndexOptions(IndexOptions.NONE);
    type.freeze();
    IntPoint id = new IntPoint("id", 0);
    StoredField idStored = new StoredField("id", 0);
    for (int i = 0; i < data.length; ++i) {
      Document doc = new Document();
      doc.add(id);
      doc.add(idStored);
      id.setIntValue(i);
      idStored.setIntValue(i);
      for (int j = 0; j < data[i].length; ++j) {
        Field f = new Field("bytes" + j, data[i][j], type);
        doc.add(f);
      }
      iw.w.addDocument(doc);
      if (random().nextBoolean() && (i % (data.length / 10) == 0)) {
        iw.w.close();
        IndexWriterConfig iwConfNew = newIndexWriterConfig(new MockAnalyzer(random()));
        // test merging against a non-compressing codec
        if (iwConf.getCodec() == otherCodec) {
          iwConfNew.setCodec(Codec.getDefault());
        } else {
          iwConfNew.setCodec(otherCodec);
        }
        iwConf = iwConfNew;
        iw = new RandomIndexWriter(random(), dir, iwConf);
      }
    }

    for (int i = 0; i < 10; ++i) {
      final int min = random().nextInt(data.length);
      final int max = min + random().nextInt(20);
      iw.deleteDocuments(IntPoint.newRangeQuery("id", min, max-1));
    }

    iw.forceMerge(2); // force merges with deletions

    iw.commit();

    final DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    assertTrue(ir.numDocs() > 0);
    int numDocs = 0;
    for (int i = 0; i < ir.maxDoc(); ++i) {
      final Document doc = ir.document(i);
      if (doc == null) {
        continue;
      }
      ++ numDocs;
      final int docId = doc.getField("id").numericValue().intValue();
      assertEquals(data[docId].length + 1, doc.getFields().size());
      for (int j = 0; j < data[docId].length; ++j) {
        final byte[] arr = data[docId][j];
        final BytesRef arr2Ref = doc.getBinaryValue("bytes" + j);
        final byte[] arr2 = BytesRef.deepCopyOf(arr2Ref).bytes;
        assertArrayEquals(arr, arr2);
      }
    }
    assertTrue(ir.numDocs() <= numDocs);
    ir.close();

    iw.deleteAll();
    iw.commit();
    iw.forceMerge(1);
    
    iw.close();
    dir.close();
  }

  /** A dummy filter reader that reverse the order of documents in stored fields. */
  private static class DummyFilterLeafReader extends FilterLeafReader {

    public DummyFilterLeafReader(LeafReader in) {
      super(in);
    }

    @Override
    public void document(int docID, StoredFieldVisitor visitor) throws IOException {
      super.document(maxDoc() - 1 - docID, visitor);
    }

    @Override
    public CacheHelper getCoreCacheHelper() {
      return null;
    }

    @Override
    public CacheHelper getReaderCacheHelper() {
      return null;
    }

  }

  private static class DummyFilterDirectoryReader extends FilterDirectoryReader {

    public DummyFilterDirectoryReader(DirectoryReader in) throws IOException {
      super(in, new SubReaderWrapper() {
        @Override
        public LeafReader wrap(LeafReader reader) {
          return new DummyFilterLeafReader(reader);
        }
      });
    }

    @Override
    protected DirectoryReader doWrapDirectoryReader(DirectoryReader in) throws IOException {
      return new DummyFilterDirectoryReader(in);
    }

    @Override
    public CacheHelper getReaderCacheHelper() {
      return null;
    }
    
  }

  public void testMergeFilterReader() throws IOException {
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(200);
    final String[] stringValues = new String[10];
    for (int i = 0; i < stringValues.length; ++i) {
      stringValues[i] = RandomStrings.randomRealisticUnicodeOfLength(random(), 10);
    }
    Document[] docs = new Document[numDocs];
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      doc.add(new StringField("to_delete", random().nextBoolean() ? "yes" : "no", Store.NO));
      doc.add(new StoredField("id", i));
      doc.add(new StoredField("i", random().nextInt(50)));
      doc.add(new StoredField("l", random().nextLong()));
      doc.add(new StoredField("d", random().nextDouble()));
      doc.add(new StoredField("f", random().nextFloat()));
      doc.add(new StoredField("s", RandomPicks.randomFrom(random(), stringValues)));
      doc.add(new StoredField("b", new BytesRef(RandomPicks.randomFrom(random(), stringValues))));
      docs[i] = doc;
      w.addDocument(doc);
    }
    if (random().nextBoolean()) {
      w.deleteDocuments(new Term("to_delete", "yes"));
    }
    w.commit();
    w.close();
    
    DirectoryReader reader = new DummyFilterDirectoryReader(maybeWrapWithMergingReader(DirectoryReader.open(dir)));
    
    Directory dir2 = newDirectory();
    w = new RandomIndexWriter(random(), dir2);
    TestUtil.addIndexesSlowly(w.w, reader);
    reader.close();
    dir.close();

    reader = maybeWrapWithMergingReader(w.getReader());
    for (int i = 0; i < reader.maxDoc(); ++i) {
      final Document doc = reader.document(i);
      final int id = doc.getField("id").numericValue().intValue();
      final Document expected = docs[id];
      assertEquals(expected.get("s"), doc.get("s"));
      assertEquals(expected.getField("i").numericValue(), doc.getField("i").numericValue());
      assertEquals(expected.getField("l").numericValue(), doc.getField("l").numericValue());
      assertEquals(expected.getField("d").numericValue(), doc.getField("d").numericValue());
      assertEquals(expected.getField("f").numericValue(), doc.getField("f").numericValue());
      assertEquals(expected.getField("b").binaryValue(), doc.getField("b").binaryValue());
    }

    reader.close();
    w.close();
    TestUtil.checkIndex(dir2);
    dir2.close();
  }

  @Nightly
  public void testBigDocuments() throws IOException {
    assumeWorkingMMapOnWindows();
    
    // "big" as "much bigger than the chunk size"
    // for this test we force a FS dir
    // we can't just use newFSDirectory, because this test doesn't really index anything.
    // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
    Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(createTempDir("testBigDocuments")));
    IndexWriterConfig iwConf = newIndexWriterConfig(new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomNumbers.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);

    if (dir instanceof MockDirectoryWrapper) {
      ((MockDirectoryWrapper) dir).setThrottling(Throttling.NEVER);
    }

    final Document emptyDoc = new Document(); // emptyDoc
    final Document bigDoc1 = new Document(); // lot of small fields
    final Document bigDoc2 = new Document(); // 1 very big field

    final Field idField = new StringField("id", "", Store.NO);
    emptyDoc.add(idField);
    bigDoc1.add(idField);
    bigDoc2.add(idField);

    final FieldType onlyStored = new FieldType(StringField.TYPE_STORED);
    onlyStored.setIndexOptions(IndexOptions.NONE);

    final Field smallField = new Field("fld", randomByteArray(random().nextInt(10), 256), onlyStored);
    final int numFields = RandomNumbers.randomIntBetween(random(), 500000, 1000000);
    for (int i = 0; i < numFields; ++i) {
      bigDoc1.add(smallField);
    }

    final Field bigField = new Field("fld", randomByteArray(RandomNumbers.randomIntBetween(random(), 1000000, 5000000), 2), onlyStored);
    bigDoc2.add(bigField);

    final int numDocs = atLeast(5);
    final Document[] docs = new Document[numDocs];
    for (int i = 0; i < numDocs; ++i) {
      docs[i] = RandomPicks.randomFrom(random(), Arrays.asList(emptyDoc, bigDoc1, bigDoc2));
    }
    for (int i = 0; i < numDocs; ++i) {
      idField.setStringValue("" + i);
      iw.addDocument(docs[i]);
      if (random().nextInt(numDocs) == 0) {
        iw.commit();
      }
    }
    iw.commit();
    iw.forceMerge(1); // look at what happens when big docs are merged
    final DirectoryReader rd = maybeWrapWithMergingReader(DirectoryReader.open(dir));
    final IndexSearcher searcher = new IndexSearcher(rd);
    for (int i = 0; i < numDocs; ++i) {
      final Query query = new TermQuery(new Term("id", "" + i));
      final TopDocs topDocs = searcher.search(query, 1);
      assertEquals("" + i, 1, topDocs.totalHits.value);
      final Document doc = rd.document(topDocs.scoreDocs[0].doc);
      assertNotNull(doc);
      final IndexableField[] fieldValues = doc.getFields("fld");
      assertEquals(docs[i].getFields("fld").length, fieldValues.length);
      if (fieldValues.length > 0) {
        assertEquals(docs[i].getFields("fld")[0].binaryValue(), fieldValues[0].binaryValue());
      }
    }
    rd.close();
    iw.close();
    dir.close();
  }

  public void testBulkMergeWithDeletes() throws IOException {
    final int numDocs = atLeast(200);
    Directory dir = newDirectory();
    RandomIndexWriter w = new RandomIndexWriter(random(), dir, newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(NoMergePolicy.INSTANCE));
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      doc.add(new StringField("id", Integer.toString(i), Store.YES));
      doc.add(new StoredField("f", TestUtil.randomSimpleString(random())));
      w.addDocument(doc);
    }
    final int deleteCount = TestUtil.nextInt(random(), 5, numDocs);
    for (int i = 0; i < deleteCount; ++i) {
      final int id = random().nextInt(numDocs);
      w.deleteDocuments(new Term("id", Integer.toString(id)));
    }
    w.commit();
    w.close();
    w = new RandomIndexWriter(random(), dir);
    w.forceMerge(TestUtil.nextInt(random(), 1, 3));
    w.commit();
    w.close();
    TestUtil.checkIndex(dir);
    dir.close();
  }

  /** mix up field numbers, merge, and check that data is correct */
  public void testMismatchedFields() throws Exception {
    Directory dirs[] = new Directory[10];
    for (int i = 0; i < dirs.length; i++) {
      Directory dir = newDirectory();
      IndexWriterConfig iwc = new IndexWriterConfig(null);
      IndexWriter iw = new IndexWriter(dir, iwc);
      Document doc = new Document();
      for (int j = 0; j < 10; j++) {
        // add fields where name=value (e.g. 3=3) so we can detect if stuff gets screwed up.
        doc.add(new StringField(Integer.toString(j), Integer.toString(j), Field.Store.YES));
      }
      for (int j = 0; j < 10; j++) {
        iw.addDocument(doc);
      }
      
      DirectoryReader reader = maybeWrapWithMergingReader(DirectoryReader.open(iw));
      // mix up fields explicitly
      if (random().nextBoolean()) {
        reader = new MismatchedDirectoryReader(reader, random());
      }
      dirs[i] = newDirectory();
      IndexWriter adder = new IndexWriter(dirs[i], new IndexWriterConfig(null));
      TestUtil.addIndexesSlowly(adder, reader);
      adder.commit();
      adder.close();
      
      IOUtils.close(reader, iw, dir);
    }
    
    Directory everything = newDirectory();
    IndexWriter iw = new IndexWriter(everything, new IndexWriterConfig(null));
    iw.addIndexes(dirs);
    iw.forceMerge(1);
    
    LeafReader ir = getOnlyLeafReader(DirectoryReader.open(iw));
    for (int i = 0; i < ir.maxDoc(); i++) {
      Document doc = ir.document(i);
      assertEquals(10, doc.getFields().size());
      for (int j = 0; j < 10; j++) {
        assertEquals(Integer.toString(j), doc.get(Integer.toString(j)));
      }
    }

    IOUtils.close(iw, ir, everything);
    IOUtils.close(dirs);
  }

  public void testRandomStoredFieldsWithIndexSort() throws Exception {
    final SortField[] sortFields;
    if (random().nextBoolean()) {
      sortFields =
          new SortField[]{
              new SortField("sort-1", SortField.Type.LONG),
              new SortField("sort-2", SortField.Type.INT)
          };
    } else {
      sortFields = new SortField[]{new SortField("sort-1", SortField.Type.LONG)};
    }
    List<String> storedFields = new ArrayList<>();
    int numFields = TestUtil.nextInt(random(), 1, 10);
    for (int i = 0; i < numFields; i++) {
      storedFields.add("f-" + i);
    }
    FieldType storeType = new FieldType(TextField.TYPE_STORED);
    storeType.setStored(true);
    Function<String, Document> documentFactory =
        id -> {
          Document doc = new Document();
          doc.add(new StringField("id", id, random().nextBoolean() ? Store.YES : Store.NO));
          if (random().nextInt(100) <= 5) {
            Collections.shuffle(storedFields, random());
          }
          for (String fieldName : storedFields) {
            if (random().nextBoolean()) {
              String s = TestUtil.randomUnicodeString(random(), 100);
              doc.add(newField(fieldName, s, storeType));
            }
          }
          for (SortField sortField : sortFields) {
            doc.add(
                new NumericDocValuesField(
                    sortField.getField(), TestUtil.nextInt(random(), 0, 10000)));
          }
          return doc;
        };

    Map<String, Document> docs = new HashMap<>();
    int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; i++) {
      String id = Integer.toString(i);
      docs.put(id, documentFactory.apply(id));
    }

    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 5, 20));
    iwc.setIndexSort(new Sort(sortFields));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
    List<String> addedIds = new ArrayList<>();
    Runnable verifyStoreFields =
        () -> {
          if (addedIds.isEmpty()) {
            return;
          }
          try (DirectoryReader reader = maybeWrapWithMergingReader(iw.getReader())) {
            IndexSearcher searcher = new IndexSearcher(reader);
            int iters = TestUtil.nextInt(random(), 1, 10);
            for (int i = 0; i < iters; i++) {
              String testID = addedIds.get(random().nextInt(addedIds.size()));
              if (VERBOSE) {
                System.out.println("TEST: test id=" + testID);
              }
              TopDocs hits = searcher.search(new TermQuery(new Term("id", testID)), 1);
              assertEquals(1, hits.totalHits.value);
              List<IndexableField> expectedFields =
                  docs.get(testID).getFields().stream()
                      .filter(f -> f.fieldType().stored())
                      .collect(Collectors.toList());
              Document actualDoc = reader.document(hits.scoreDocs[0].doc);
              assertEquals(expectedFields.size(), actualDoc.getFields().size());
              for (IndexableField expectedField : expectedFields) {
                IndexableField[] actualFields = actualDoc.getFields(expectedField.name());
                assertEquals(1, actualFields.length);
                assertEquals(expectedField.stringValue(), actualFields[0].stringValue());
              }
            }
          } catch (IOException e) {
            throw new UncheckedIOException(e);
          }
        };
    final List<String> ids = new ArrayList<>(docs.keySet());
    Collections.shuffle(ids, random());
    for (String id : ids) {
      if (random().nextInt(100) < 5) {
        // add via foreign reader
        IndexWriterConfig otherIwc = newIndexWriterConfig();
        otherIwc.setIndexSort(new Sort(sortFields));
        try (Directory otherDir = newDirectory();
             RandomIndexWriter otherIw = new RandomIndexWriter(random(), otherDir, otherIwc)) {
          otherIw.addDocument(docs.get(id));
          try (DirectoryReader otherReader = otherIw.getReader()) {
            TestUtil.addIndexesSlowly(iw.w, otherReader);
          }
        }
      } else {
        // add normally
        iw.addDocument(docs.get(id));
      }
      addedIds.add(id);
      if (random().nextInt(100) < 5) {
        String deletingId = addedIds.remove(random().nextInt(addedIds.size()));
        if (random().nextBoolean()) {
          iw.deleteDocuments(new TermQuery(new Term("id", deletingId)));
          addedIds.remove(deletingId);
        } else {
          final Document newDoc = documentFactory.apply(deletingId);
          docs.put(deletingId, newDoc);
          iw.updateDocument(new Term("id", deletingId), newDoc);
        }
      }
      if (random().nextInt(100) < 5) {
        verifyStoreFields.run();
      }
      if (random().nextInt(100) < 2) {
        iw.forceMerge(TestUtil.nextInt(random(), 1, 3));
      }
    }
    verifyStoreFields.run();
    iw.forceMerge(TestUtil.nextInt(random(), 1, 3));
    verifyStoreFields.run();
    IOUtils.close(iw, dir);
  }

}