lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.lucene.search.suggest;

 import java.io.IOException;
 import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.Set;

 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.NumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.spell.Dictionary;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 import org.junit.Test;

 // See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
 public class DocumentDictionaryTest extends LuceneTestCase {

   static final String FIELD_NAME = "f1";
   static final String WEIGHT_FIELD_NAME = "w1";
   static final String PAYLOAD_FIELD_NAME = "p1";
   static final String CONTEXT_FIELD_NAME = "c1";

   @Test
   public void testEmptyReader() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     // Make sure the index is created?
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
     writer.commit();
     writer.close();
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
     InputIterator inputIterator = dictionary.getEntryIterator();

     assertNull(inputIterator.next());
     assertEquals(inputIterator.weight(), 0);
     assertNull(inputIterator.payload());

     IOUtils.close(ir, analyzer, dir);
   }

   @Test
   public void testBasic() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
     Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), false);
     Map<String, Document> docs = res.getValue();
     List<String> invalidDocTerms = res.getKey();
     for(Document doc: docs.values()) {
       writer.addDocument(doc);
     }
     writer.commit();
     writer.close();
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
     InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
       assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
       IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
       assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
       IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
       if (payloadField == null) assertTrue(inputIterator.payload().length == 0);
       else assertEquals(inputIterator.payload(), payloadField.binaryValue());
     }

     for (String invalidTerm : invalidDocTerms) {
       assertNotNull(docs.remove(invalidTerm));
     }
     assertTrue(docs.isEmpty());

     IOUtils.close(ir, analyzer, dir);
   }

   @Test
   public void testWithOptionalPayload() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);

     // Create a document that is missing the payload field
     Document doc = new Document();
     Field field = new TextField(FIELD_NAME, "some field", Field.Store.YES);
     doc.add(field);
     // do not store the payload or the contexts
     Field weight = new NumericDocValuesField(WEIGHT_FIELD_NAME, 100);
     doc.add(weight);
     writer.addDocument(doc);
     writer.commit();
     writer.close();
     IndexReader ir = DirectoryReader.open(dir);

     // Even though the payload field is missing, the dictionary iterator should not skip the document
     // because the payload field is optional.
     Dictionary dictionaryOptionalPayload =
         new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
     InputIterator inputIterator = dictionaryOptionalPayload.getEntryIterator();
     BytesRef f = inputIterator.next();
     assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
     IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
     assertEquals(inputIterator.weight(), weightField.numericValue().longValue());
     IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
     assertNull(payloadField);
     assertTrue(inputIterator.payload().length == 0);
     IOUtils.close(ir, analyzer, dir);
   }

   @Test
   public void testWithoutPayload() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
     Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), false);
     Map<String, Document> docs = res.getValue();
     List<String> invalidDocTerms = res.getKey();
     for(Document doc: docs.values()) {
       writer.addDocument(doc);
     }
     writer.commit();
     writer.close();
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
     InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
       assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
       IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
       assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
       assertNull(inputIterator.payload());
     }

     for (String invalidTerm : invalidDocTerms) {
       assertNotNull(docs.remove(invalidTerm));
     }

     assertTrue(docs.isEmpty());

     IOUtils.close(ir, analyzer, dir);
   }

   @Test
   public void testWithContexts() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
     Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), true);
     Map<String, Document> docs = res.getValue();
     List<String> invalidDocTerms = res.getKey();
     for(Document doc: docs.values()) {
       writer.addDocument(doc);
     }
     writer.commit();
     writer.close();
     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME, CONTEXT_FIELD_NAME);
     InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
       assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
       IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
       assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
       IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
       if (payloadField == null) assertTrue(inputIterator.payload().length == 0);
       else assertEquals(inputIterator.payload(), payloadField.binaryValue());
       Set<BytesRef> oriCtxs = new HashSet<>();
       Set<BytesRef> contextSet = inputIterator.contexts();
       for (IndexableField ctxf : doc.getFields(CONTEXT_FIELD_NAME)) {
         oriCtxs.add(ctxf.binaryValue());
       }
       assertEquals(oriCtxs.size(), contextSet.size());
     }

     for (String invalidTerm : invalidDocTerms) {
       assertNotNull(docs.remove(invalidTerm));
     }
     assertTrue(docs.isEmpty());

     IOUtils.close(ir, analyzer, dir);
   }

   @Test
   public void testWithDeletions() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
     Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), false);
     Map<String, Document> docs = res.getValue();
     List<String> invalidDocTerms = res.getKey();
     Random rand = random();
     List<String> termsToDel = new ArrayList<>();
     for(Document doc : docs.values()) {
       IndexableField f = doc.getField(FIELD_NAME);
       if(rand.nextBoolean() && f != null && !invalidDocTerms.contains(f.stringValue())) {
         termsToDel.add(doc.get(FIELD_NAME));
       }
       writer.addDocument(doc);
     }
     writer.commit();

     Term[] delTerms = new Term[termsToDel.size()];
     for(int i=0; i < termsToDel.size() ; i++) {
       delTerms[i] = new Term(FIELD_NAME, termsToDel.get(i));
     }

     for(Term delTerm: delTerms) {
       writer.deleteDocuments(delTerm);
     }
     writer.commit();
     writer.close();

     for(String termToDel: termsToDel) {
       assertTrue(null!=docs.remove(termToDel));
     }

     IndexReader ir = DirectoryReader.open(dir);
     assertEquals(ir.numDocs(), docs.size());
     Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
     InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     while((f = inputIterator.next())!=null) {
       Document doc = docs.remove(f.utf8ToString());
       assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
       IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
       assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
       assertNull(inputIterator.payload());
     }

     for (String invalidTerm : invalidDocTerms) {
       assertNotNull(docs.remove(invalidTerm));
     }
     assertTrue(docs.isEmpty());

     IOUtils.close(ir, analyzer, dir);
   }

   @Test
   public void testMultiValuedField() throws IOException {
     Directory dir = newDirectory();
     Analyzer analyzer = new MockAnalyzer(random());
     IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
     iwc.setMergePolicy(newLogMergePolicy());
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);

     List<Suggestion> suggestions = indexMultiValuedDocuments(atLeast(1000), writer);
     writer.commit();
     writer.close();

     IndexReader ir = DirectoryReader.open(dir);
     Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME, CONTEXT_FIELD_NAME);
     InputIterator inputIterator = dictionary.getEntryIterator();
     BytesRef f;
     Iterator<Suggestion> suggestionsIter = suggestions.iterator();
     while((f = inputIterator.next())!=null) {
       Suggestion nextSuggestion = suggestionsIter.next();
       assertTrue(f.equals(nextSuggestion.term));
       long weight = nextSuggestion.weight;
       assertEquals(inputIterator.weight(), (weight != -1) ? weight : 0);
       assertEquals(inputIterator.payload(), nextSuggestion.payload);
       assertTrue(inputIterator.contexts().equals(nextSuggestion.contexts));
     }
     assertFalse(suggestionsIter.hasNext());
     IOUtils.close(ir, analyzer, dir);
   }

   /** Returns Pair(list of invalid document terms, Map of document term -&gt; document) */
   private Map.Entry<List<String>, Map<String, Document>> generateIndexDocuments(int ndocs, boolean requiresContexts) {
     Map<String, Document> docs = new HashMap<>();
     List<String> invalidDocTerms = new ArrayList<>();
     for(int i = 0; i < ndocs ; i++) {
       Document doc = new Document();
       boolean invalidDoc = false;
       Field field = null;
       // usually have valid term field in document
       if (usually()) {
         field = new TextField(FIELD_NAME, "field_" + i, Field.Store.YES);
         doc.add(field);
       } else {
         invalidDoc = true;
       }

       // even if payload is not required usually have it
       if (usually()) {
         Field payload = new StoredField(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
         doc.add(payload);
       }

       if (requiresContexts || usually()) {
         if (usually()) {
           for (int j = 0; j < atLeast(2); j++) {
             doc.add(new StoredField(CONTEXT_FIELD_NAME, new BytesRef("context_" + i + "_"+ j)));
           }
         }
         // we should allow entries without context
       }

       // usually have valid weight field in document
       if (usually()) {
         Field weight = (rarely()) ?
                 new StoredField(WEIGHT_FIELD_NAME, 100d + i) :
                 new NumericDocValuesField(WEIGHT_FIELD_NAME, 100 + i);
         doc.add(weight);
       }

       String term = null;
       if (invalidDoc) {
         term = (field!=null) ? field.stringValue() : "invalid_" + i;
         invalidDocTerms.add(term);
       } else {
         term = field.stringValue();
       }

       docs.put(term, doc);
     }
     return new SimpleEntry<>(invalidDocTerms, docs);
   }

   private List<Suggestion> indexMultiValuedDocuments(int numDocs, RandomIndexWriter writer) throws IOException {
     List<Suggestion> suggestionList = new ArrayList<>(numDocs);

     for(int i=0; i<numDocs; i++) {
       Document doc = new Document();
       Field field;
       BytesRef payloadValue;
       Set<BytesRef> contextValues = new HashSet<>();
       long numericValue = -1; //-1 for missing weight
       BytesRef term;

       payloadValue = new BytesRef("payload_" + i);
       field = new StoredField(PAYLOAD_FIELD_NAME, payloadValue);
       doc.add(field);

       if (usually()) {
         numericValue = 100 + i;
         field = new NumericDocValuesField(WEIGHT_FIELD_NAME, numericValue);
         doc.add(field);
       }

       int numContexts = atLeast(1);
       for (int j=0; j<numContexts; j++) {
         BytesRef contextValue = new BytesRef("context_" + i + "_" + j);
         field = new StoredField(CONTEXT_FIELD_NAME, contextValue);
         doc.add(field);
         contextValues.add(contextValue);
       }

       int numSuggestions = atLeast(2);
       for (int j=0; j<numSuggestions; j++) {
         term = new BytesRef("field_" + i + "_" + j);
         field = new StoredField(FIELD_NAME, term);
         doc.add(field);

         Suggestion suggestionValue = new Suggestion();
         suggestionValue.payload = payloadValue;
         suggestionValue.contexts = contextValues;
         suggestionValue.weight = numericValue;
         suggestionValue.term = term;
         suggestionList.add(suggestionValue);
       }
       writer.addDocument(doc);
     }
     return suggestionList;
   }

   private static class Suggestion {
     private long weight;
     private BytesRef payload;
     private Set<BytesRef> contexts;
     private BytesRef term;
   }


 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.lucene.search.suggest;

	import java.io.IOException;
	import java.util.AbstractMap.SimpleEntry;
	import java.util.ArrayList;
	import java.util.HashMap;
	import java.util.HashSet;
	import java.util.Iterator;
	import java.util.List;
	import java.util.Map;
	import java.util.Random;
	import java.util.Set;

	import org.apache.lucene.analysis.Analyzer;
	import org.apache.lucene.analysis.MockAnalyzer;
	import org.apache.lucene.document.Document;
	import org.apache.lucene.document.Field;
	import org.apache.lucene.document.NumericDocValuesField;
	import org.apache.lucene.document.StoredField;
	import org.apache.lucene.document.TextField;
	import org.apache.lucene.index.DirectoryReader;
	import org.apache.lucene.index.IndexReader;
	import org.apache.lucene.index.IndexWriterConfig;
	import org.apache.lucene.index.IndexableField;
	import org.apache.lucene.index.RandomIndexWriter;
	import org.apache.lucene.index.Term;
	import org.apache.lucene.search.spell.Dictionary;
	import org.apache.lucene.store.Directory;
	import org.apache.lucene.util.BytesRef;
	import org.apache.lucene.util.IOUtils;
	import org.apache.lucene.util.LuceneTestCase;
	import org.junit.Test;

	// See: https://issues.apache.org/jira/browse/SOLR-12028 Tests cannot remove files on Windows machines occasionally
	public class DocumentDictionaryTest extends LuceneTestCase {

	static final String FIELD_NAME = "f1";
	static final String WEIGHT_FIELD_NAME = "w1";
	static final String PAYLOAD_FIELD_NAME = "p1";
	static final String CONTEXT_FIELD_NAME = "c1";

	@Test
	public void testEmptyReader() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	// Make sure the index is created?
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
	writer.commit();
	writer.close();
	IndexReader ir = DirectoryReader.open(dir);
	Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
	InputIterator inputIterator = dictionary.getEntryIterator();

	assertNull(inputIterator.next());
	assertEquals(inputIterator.weight(), 0);
	assertNull(inputIterator.payload());

	IOUtils.close(ir, analyzer, dir);
	}

	@Test
	public void testBasic() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
	Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), false);
	Map<String, Document> docs = res.getValue();
	List<String> invalidDocTerms = res.getKey();
	for(Document doc: docs.values()) {
	writer.addDocument(doc);
	}
	writer.commit();
	writer.close();
	IndexReader ir = DirectoryReader.open(dir);
	Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
	InputIterator inputIterator = dictionary.getEntryIterator();
	BytesRef f;
	while((f = inputIterator.next())!=null) {
	Document doc = docs.remove(f.utf8ToString());
	assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
	IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
	assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
	IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
	if (payloadField == null) assertTrue(inputIterator.payload().length == 0);
	else assertEquals(inputIterator.payload(), payloadField.binaryValue());
	}

	for (String invalidTerm : invalidDocTerms) {
	assertNotNull(docs.remove(invalidTerm));
	}
	assertTrue(docs.isEmpty());

	IOUtils.close(ir, analyzer, dir);
	}

	@Test
	public void testWithOptionalPayload() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);

	// Create a document that is missing the payload field
	Document doc = new Document();
	Field field = new TextField(FIELD_NAME, "some field", Field.Store.YES);
	doc.add(field);
	// do not store the payload or the contexts
	Field weight = new NumericDocValuesField(WEIGHT_FIELD_NAME, 100);
	doc.add(weight);
	writer.addDocument(doc);
	writer.commit();
	writer.close();
	IndexReader ir = DirectoryReader.open(dir);

	// Even though the payload field is missing, the dictionary iterator should not skip the document
	// because the payload field is optional.
	Dictionary dictionaryOptionalPayload =
	new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
	InputIterator inputIterator = dictionaryOptionalPayload.getEntryIterator();
	BytesRef f = inputIterator.next();
	assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
	IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
	assertEquals(inputIterator.weight(), weightField.numericValue().longValue());
	IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
	assertNull(payloadField);
	assertTrue(inputIterator.payload().length == 0);
	IOUtils.close(ir, analyzer, dir);
	}

	@Test
	public void testWithoutPayload() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
	Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), false);
	Map<String, Document> docs = res.getValue();
	List<String> invalidDocTerms = res.getKey();
	for(Document doc: docs.values()) {
	writer.addDocument(doc);
	}
	writer.commit();
	writer.close();
	IndexReader ir = DirectoryReader.open(dir);
	Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
	InputIterator inputIterator = dictionary.getEntryIterator();
	BytesRef f;
	while((f = inputIterator.next())!=null) {
	Document doc = docs.remove(f.utf8ToString());
	assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
	IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
	assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
	assertNull(inputIterator.payload());
	}

	for (String invalidTerm : invalidDocTerms) {
	assertNotNull(docs.remove(invalidTerm));
	}

	assertTrue(docs.isEmpty());

	IOUtils.close(ir, analyzer, dir);
	}

	@Test
	public void testWithContexts() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
	Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), true);
	Map<String, Document> docs = res.getValue();
	List<String> invalidDocTerms = res.getKey();
	for(Document doc: docs.values()) {
	writer.addDocument(doc);
	}
	writer.commit();
	writer.close();
	IndexReader ir = DirectoryReader.open(dir);
	Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME, CONTEXT_FIELD_NAME);
	InputIterator inputIterator = dictionary.getEntryIterator();
	BytesRef f;
	while((f = inputIterator.next())!=null) {
	Document doc = docs.remove(f.utf8ToString());
	assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
	IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
	assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
	IndexableField payloadField = doc.getField(PAYLOAD_FIELD_NAME);
	if (payloadField == null) assertTrue(inputIterator.payload().length == 0);
	else assertEquals(inputIterator.payload(), payloadField.binaryValue());
	Set<BytesRef> oriCtxs = new HashSet<>();
	Set<BytesRef> contextSet = inputIterator.contexts();
	for (IndexableField ctxf : doc.getFields(CONTEXT_FIELD_NAME)) {
	oriCtxs.add(ctxf.binaryValue());
	}
	assertEquals(oriCtxs.size(), contextSet.size());
	}

	for (String invalidTerm : invalidDocTerms) {
	assertNotNull(docs.remove(invalidTerm));
	}
	assertTrue(docs.isEmpty());

	IOUtils.close(ir, analyzer, dir);
	}

	@Test
	public void testWithDeletions() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
	Map.Entry<List<String>, Map<String, Document>> res = generateIndexDocuments(atLeast(1000), false);
	Map<String, Document> docs = res.getValue();
	List<String> invalidDocTerms = res.getKey();
	Random rand = random();
	List<String> termsToDel = new ArrayList<>();
	for(Document doc : docs.values()) {
	IndexableField f = doc.getField(FIELD_NAME);
	if(rand.nextBoolean() && f != null && !invalidDocTerms.contains(f.stringValue())) {
	termsToDel.add(doc.get(FIELD_NAME));
	}
	writer.addDocument(doc);
	}
	writer.commit();

	Term[] delTerms = new Term[termsToDel.size()];
	for(int i=0; i < termsToDel.size() ; i++) {
	delTerms[i] = new Term(FIELD_NAME, termsToDel.get(i));
	}

	for(Term delTerm: delTerms) {
	writer.deleteDocuments(delTerm);
	}
	writer.commit();
	writer.close();

	for(String termToDel: termsToDel) {
	assertTrue(null!=docs.remove(termToDel));
	}

	IndexReader ir = DirectoryReader.open(dir);
	assertEquals(ir.numDocs(), docs.size());
	Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
	InputIterator inputIterator = dictionary.getEntryIterator();
	BytesRef f;
	while((f = inputIterator.next())!=null) {
	Document doc = docs.remove(f.utf8ToString());
	assertTrue(f.equals(new BytesRef(doc.get(FIELD_NAME))));
	IndexableField weightField = doc.getField(WEIGHT_FIELD_NAME);
	assertEquals(inputIterator.weight(), (weightField != null) ? weightField.numericValue().longValue() : 0);
	assertNull(inputIterator.payload());
	}

	for (String invalidTerm : invalidDocTerms) {
	assertNotNull(docs.remove(invalidTerm));
	}
	assertTrue(docs.isEmpty());

	IOUtils.close(ir, analyzer, dir);
	}

	@Test
	public void testMultiValuedField() throws IOException {
	Directory dir = newDirectory();
	Analyzer analyzer = new MockAnalyzer(random());
	IndexWriterConfig iwc = newIndexWriterConfig(random(), analyzer);
	iwc.setMergePolicy(newLogMergePolicy());
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);

	List<Suggestion> suggestions = indexMultiValuedDocuments(atLeast(1000), writer);
	writer.commit();
	writer.close();

	IndexReader ir = DirectoryReader.open(dir);
	Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME, CONTEXT_FIELD_NAME);
	InputIterator inputIterator = dictionary.getEntryIterator();
	BytesRef f;
	Iterator<Suggestion> suggestionsIter = suggestions.iterator();
	while((f = inputIterator.next())!=null) {
	Suggestion nextSuggestion = suggestionsIter.next();
	assertTrue(f.equals(nextSuggestion.term));
	long weight = nextSuggestion.weight;
	assertEquals(inputIterator.weight(), (weight != -1) ? weight : 0);
	assertEquals(inputIterator.payload(), nextSuggestion.payload);
	assertTrue(inputIterator.contexts().equals(nextSuggestion.contexts));
	}
	assertFalse(suggestionsIter.hasNext());
	IOUtils.close(ir, analyzer, dir);
	}

	/** Returns Pair(list of invalid document terms, Map of document term -> document) */
	private Map.Entry<List<String>, Map<String, Document>> generateIndexDocuments(int ndocs, boolean requiresContexts) {
	Map<String, Document> docs = new HashMap<>();
	List<String> invalidDocTerms = new ArrayList<>();
	for(int i = 0; i < ndocs ; i++) {
	Document doc = new Document();
	boolean invalidDoc = false;
	Field field = null;
	// usually have valid term field in document
	if (usually()) {
	field = new TextField(FIELD_NAME, "field_" + i, Field.Store.YES);
	doc.add(field);
	} else {
	invalidDoc = true;
	}

	// even if payload is not required usually have it
	if (usually()) {
	Field payload = new StoredField(PAYLOAD_FIELD_NAME, new BytesRef("payload_" + i));
	doc.add(payload);
	}

	if (requiresContexts \|\| usually()) {
	if (usually()) {
	for (int j = 0; j < atLeast(2); j++) {
	doc.add(new StoredField(CONTEXT_FIELD_NAME, new BytesRef("context_" + i + "_"+ j)));
	}
	}
	// we should allow entries without context
	}

	// usually have valid weight field in document
	if (usually()) {
	Field weight = (rarely()) ?
	new StoredField(WEIGHT_FIELD_NAME, 100d + i) :
	new NumericDocValuesField(WEIGHT_FIELD_NAME, 100 + i);
	doc.add(weight);
	}

	String term = null;
	if (invalidDoc) {
	term = (field!=null) ? field.stringValue() : "invalid_" + i;
	invalidDocTerms.add(term);
	} else {
	term = field.stringValue();
	}

	docs.put(term, doc);
	}
	return new SimpleEntry<>(invalidDocTerms, docs);
	}

	private List<Suggestion> indexMultiValuedDocuments(int numDocs, RandomIndexWriter writer) throws IOException {
	List<Suggestion> suggestionList = new ArrayList<>(numDocs);

	for(int i=0; i<numDocs; i++) {
	Document doc = new Document();
	Field field;
	BytesRef payloadValue;
	Set<BytesRef> contextValues = new HashSet<>();
	long numericValue = -1; //-1 for missing weight
	BytesRef term;

	payloadValue = new BytesRef("payload_" + i);
	field = new StoredField(PAYLOAD_FIELD_NAME, payloadValue);
	doc.add(field);

	if (usually()) {
	numericValue = 100 + i;
	field = new NumericDocValuesField(WEIGHT_FIELD_NAME, numericValue);
	doc.add(field);
	}

	int numContexts = atLeast(1);
	for (int j=0; j<numContexts; j++) {
	BytesRef contextValue = new BytesRef("context_" + i + "_" + j);
	field = new StoredField(CONTEXT_FIELD_NAME, contextValue);
	doc.add(field);
	contextValues.add(contextValue);
	}

	int numSuggestions = atLeast(2);
	for (int j=0; j<numSuggestions; j++) {
	term = new BytesRef("field_" + i + "_" + j);
	field = new StoredField(FIELD_NAME, term);
	doc.add(field);

	Suggestion suggestionValue = new Suggestion();
	suggestionValue.payload = payloadValue;
	suggestionValue.contexts = contextValues;
	suggestionValue.weight = numericValue;
	suggestionValue.term = term;
	suggestionList.add(suggestionValue);
	}
	writer.addDocument(doc);
	}
	return suggestionList;
	}

	private static class Suggestion {
	private long weight;
	private BytesRef payload;
	private Set<BytesRef> contexts;
	private BytesRef term;
	}


	}