lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java - lucene-solr - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.lucene.index;

 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;

 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.TestUtil;

 public class TestDocsAndPositions extends LuceneTestCase {
   private String fieldName;

   @Override
   public void setUp() throws Exception {
     super.setUp();
     fieldName = "field" + random().nextInt();
   }

   /**
    * Simple testcase for {@link PostingsEnum}
    */
   public void testPositionsSimple() throws IOException {
     Directory directory = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
         newIndexWriterConfig(new MockAnalyzer(random())));
     for (int i = 0; i < 39; i++) {
       Document doc = new Document();
       FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
       customType.setOmitNorms(true);
       doc.add(newField(fieldName, "1 2 3 4 5 6 7 8 9 10 "
           + "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 "
           + "1 2 3 4 5 6 7 8 9 10", customType));
       writer.addDocument(doc);
     }
     IndexReader reader = writer.getReader();
     writer.close();

     int num = atLeast(13);
     for (int i = 0; i < num; i++) {
       BytesRef bytes = new BytesRef("1");
       IndexReaderContext topReaderContext = reader.getContext();
       for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
         PostingsEnum docsAndPosEnum = getDocsAndPositions(
             leafReaderContext.reader(), bytes);
         assertNotNull(docsAndPosEnum);
         if (leafReaderContext.reader().maxDoc() == 0) {
           continue;
         }
         final int advance = docsAndPosEnum.advance(random().nextInt(leafReaderContext.reader().maxDoc()));
         do {
           String msg = "Advanced to: " + advance + " current doc: "
               + docsAndPosEnum.docID(); // TODO: + " usePayloads: " + usePayload;
           assertEquals(msg, 4, docsAndPosEnum.freq());
           assertEquals(msg, 0, docsAndPosEnum.nextPosition());
           assertEquals(msg, 4, docsAndPosEnum.freq());
           assertEquals(msg, 10, docsAndPosEnum.nextPosition());
           assertEquals(msg, 4, docsAndPosEnum.freq());
           assertEquals(msg, 20, docsAndPosEnum.nextPosition());
           assertEquals(msg, 4, docsAndPosEnum.freq());
           assertEquals(msg, 30, docsAndPosEnum.nextPosition());
         } while (docsAndPosEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
       }
     }
     reader.close();
     directory.close();
   }

   public PostingsEnum getDocsAndPositions(LeafReader reader,
       BytesRef bytes) throws IOException {
     Terms terms = reader.terms(fieldName);
     if (terms != null) {
       TermsEnum te = terms.iterator();
       if (te.seekExact(bytes)) {
         return te.postings(null, PostingsEnum.ALL);
       }
     }
     return null;
   }

   /**
    * this test indexes random numbers within a range into a field and checks
    * their occurrences by searching for a number from that range selected at
    * random. All positions for that number are saved up front and compared to
    * the enums positions.
    */
   public void testRandomPositions() throws IOException {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
         newIndexWriterConfig(new MockAnalyzer(random()))
           .setMergePolicy(newLogMergePolicy()));
     int numDocs = atLeast(47);
     int max = 1051;
     int term = random().nextInt(max);
     Integer[][] positionsInDoc = new Integer[numDocs][];
     FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
     customType.setOmitNorms(true);
     for (int i = 0; i < numDocs; i++) {
       Document doc = new Document();
       ArrayList<Integer> positions = new ArrayList<>();
       StringBuilder builder = new StringBuilder();
       int num = atLeast(131);
       for (int j = 0; j < num; j++) {
         int nextInt = random().nextInt(max);
         builder.append(nextInt).append(" ");
         if (nextInt == term) {
           positions.add(Integer.valueOf(j));
         }
       }
       if (positions.size() == 0) {
         builder.append(term);
         positions.add(num);
       }
       doc.add(newField(fieldName, builder.toString(), customType));
       positionsInDoc[i] = positions.toArray(new Integer[0]);
       writer.addDocument(doc);
     }

     IndexReader reader = writer.getReader();
     writer.close();

     int num = atLeast(13);
     for (int i = 0; i < num; i++) {
       BytesRef bytes = new BytesRef("" + term);
       IndexReaderContext topReaderContext = reader.getContext();
       for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
         PostingsEnum docsAndPosEnum = getDocsAndPositions(
             leafReaderContext.reader(), bytes);
         assertNotNull(docsAndPosEnum);
         int initDoc = 0;
         int maxDoc = leafReaderContext.reader().maxDoc();
         // initially advance or do next doc
         if (random().nextBoolean()) {
           initDoc = docsAndPosEnum.nextDoc();
         } else {
           initDoc = docsAndPosEnum.advance(random().nextInt(maxDoc));
         }
         // now run through the scorer and check if all positions are there...
         do {
           int docID = docsAndPosEnum.docID();
           if (docID == DocIdSetIterator.NO_MORE_DOCS) {
             break;
           }
           Integer[] pos = positionsInDoc[leafReaderContext.docBase + docID];
           assertEquals(pos.length, docsAndPosEnum.freq());
           // number of positions read should be random - don't read all of them
           // allways
           final int howMany = random().nextInt(20) == 0 ? pos.length
               - random().nextInt(pos.length) : pos.length;
           for (int j = 0; j < howMany; j++) {
             assertEquals("iteration: " + i + " initDoc: " + initDoc + " doc: "
                 + docID + " base: " + leafReaderContext.docBase
                 + " positions: " + Arrays.toString(pos) /* TODO: + " usePayloads: "
                 + usePayload*/, pos[j].intValue(), docsAndPosEnum.nextPosition());
           }

           if (random().nextInt(10) == 0) { // once is a while advance
             if (docsAndPosEnum.advance(docID + 1 + random().nextInt((maxDoc - docID))) == DocIdSetIterator.NO_MORE_DOCS) {
               break;
             }
           }

         } while (docsAndPosEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
       }

     }
     reader.close();
     dir.close();
   }

   public void testRandomDocs() throws IOException {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
                                                      newIndexWriterConfig(new MockAnalyzer(random()))
                                                        .setMergePolicy(newLogMergePolicy()));
     int numDocs = atLeast(49);
     int max = 15678;
     int term = random().nextInt(max);
     int[] freqInDoc = new int[numDocs];
     FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
     customType.setOmitNorms(true);
     for (int i = 0; i < numDocs; i++) {
       Document doc = new Document();
       StringBuilder builder = new StringBuilder();
       for (int j = 0; j < 199; j++) {
         int nextInt = random().nextInt(max);
         builder.append(nextInt).append(' ');
         if (nextInt == term) {
           freqInDoc[i]++;
         }
       }
       doc.add(newField(fieldName, builder.toString(), customType));
       writer.addDocument(doc);
     }

     IndexReader reader = writer.getReader();
     writer.close();

     int num = atLeast(13);
     for (int i = 0; i < num; i++) {
       BytesRef bytes = new BytesRef("" + term);
       IndexReaderContext topReaderContext = reader.getContext();
       for (LeafReaderContext context : topReaderContext.leaves()) {
         int maxDoc = context.reader().maxDoc();
         PostingsEnum postingsEnum = TestUtil.docs(random(), context.reader(), fieldName, bytes, null, PostingsEnum.FREQS);
         if (findNext(freqInDoc, context.docBase, context.docBase + maxDoc) == Integer.MAX_VALUE) {
           assertNull(postingsEnum);
           continue;
         }
         assertNotNull(postingsEnum);
         postingsEnum.nextDoc();
         for (int j = 0; j < maxDoc; j++) {
           if (freqInDoc[context.docBase + j] != 0) {
             assertEquals(j, postingsEnum.docID());
             assertEquals(postingsEnum.freq(), freqInDoc[context.docBase +j]);
             if (i % 2 == 0 && random().nextInt(10) == 0) {
               int next = findNext(freqInDoc, context.docBase+j+1, context.docBase + maxDoc) - context.docBase;
               int advancedTo = postingsEnum.advance(next);
               if (next >= maxDoc) {
                 assertEquals(DocIdSetIterator.NO_MORE_DOCS, advancedTo);
               } else {
                 assertTrue("advanced to: " +advancedTo + " but should be <= " + next, next >= advancedTo);
               }
             } else {
               postingsEnum.nextDoc();
             }
           }
         }
         assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + postingsEnum.getClass(), DocIdSetIterator.NO_MORE_DOCS, postingsEnum.docID());
       }

     }

     reader.close();
     dir.close();
   }

   private static int findNext(int[] docs, int pos, int max) {
     for (int i = pos; i < max; i++) {
       if( docs[i] != 0) {
         return i;
       }
     }
     return Integer.MAX_VALUE;
   }

   /**
    * tests retrieval of positions for terms that have a large number of
    * occurrences to force test of buffer refill during positions iteration.
    */
   public void testLargeNumberOfPositions() throws IOException {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
         newIndexWriterConfig(new MockAnalyzer(random())));
     int howMany = 1000;
     FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
     customType.setOmitNorms(true);
     for (int i = 0; i < 39; i++) {
       Document doc = new Document();
       StringBuilder builder = new StringBuilder();
       for (int j = 0; j < howMany; j++) {
         if (j % 2 == 0) {
           builder.append("even ");
         } else {
           builder.append("odd ");
         }
       }
       doc.add(newField(fieldName, builder.toString(), customType));
       writer.addDocument(doc);
     }

     // now do searches
     IndexReader reader = writer.getReader();
     writer.close();

     int num = atLeast(13);
     for (int i = 0; i < num; i++) {
       BytesRef bytes = new BytesRef("even");

       IndexReaderContext topReaderContext = reader.getContext();
       for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
         PostingsEnum docsAndPosEnum = getDocsAndPositions(
             leafReaderContext.reader(), bytes);
         assertNotNull(docsAndPosEnum);

         int initDoc = 0;
         int maxDoc = leafReaderContext.reader().maxDoc();
         // initially advance or do next doc
         if (random().nextBoolean()) {
           initDoc = docsAndPosEnum.nextDoc();
         } else {
           initDoc = docsAndPosEnum.advance(random().nextInt(maxDoc));
         }
         String msg = "Iteration: " + i + " initDoc: " + initDoc; // TODO: + " payloads: " + usePayload;
         assertEquals(howMany / 2, docsAndPosEnum.freq());
         for (int j = 0; j < howMany; j += 2) {
           assertEquals("position missmatch index: " + j + " with freq: "
               + docsAndPosEnum.freq() + " -- " + msg, j,
               docsAndPosEnum.nextPosition());
         }
       }
     }
     reader.close();
     dir.close();
   }

   public void testDocsEnumStart() throws Exception {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
     Document doc = new Document();
     doc.add(newStringField("foo", "bar", Field.Store.NO));
     writer.addDocument(doc);
     DirectoryReader reader = writer.getReader();
     LeafReader r = getOnlyLeafReader(reader);
     PostingsEnum disi = TestUtil.docs(random(), r, "foo", new BytesRef("bar"), null, PostingsEnum.NONE);
     int docid = disi.docID();
     assertEquals(-1, docid);
     assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

     // now reuse and check again
     TermsEnum te = r.terms("foo").iterator();
     assertTrue(te.seekExact(new BytesRef("bar")));
     disi = TestUtil.docs(random(), te, disi, PostingsEnum.NONE);
     docid = disi.docID();
     assertEquals(-1, docid);
     assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     writer.close();
     r.close();
     dir.close();
   }

   public void testDocsAndPositionsEnumStart() throws Exception {
     Directory dir = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
     Document doc = new Document();
     doc.add(newTextField("foo", "bar", Field.Store.NO));
     writer.addDocument(doc);
     DirectoryReader reader = writer.getReader();
     LeafReader r = getOnlyLeafReader(reader);
     PostingsEnum disi = r.postings(new Term("foo", "bar"), PostingsEnum.ALL);
     int docid = disi.docID();
     assertEquals(-1, docid);
     assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

     // now reuse and check again
     TermsEnum te = r.terms("foo").iterator();
     assertTrue(te.seekExact(new BytesRef("bar")));
     disi = te.postings(disi, PostingsEnum.ALL);
     docid = disi.docID();
     assertEquals(-1, docid);
     assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
     writer.close();
     r.close();
     dir.close();
   }
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.lucene.index;

	import java.io.IOException;
	import java.util.ArrayList;
	import java.util.Arrays;

	import org.apache.lucene.analysis.MockAnalyzer;
	import org.apache.lucene.document.Document;
	import org.apache.lucene.document.Field;
	import org.apache.lucene.document.FieldType;
	import org.apache.lucene.document.TextField;
	import org.apache.lucene.search.DocIdSetIterator;
	import org.apache.lucene.store.Directory;
	import org.apache.lucene.util.BytesRef;
	import org.apache.lucene.util.LuceneTestCase;
	import org.apache.lucene.util.TestUtil;

	public class TestDocsAndPositions extends LuceneTestCase {
	private String fieldName;

	@Override
	public void setUp() throws Exception {
	super.setUp();
	fieldName = "field" + random().nextInt();
	}

	/**
	* Simple testcase for {@link PostingsEnum}
	*/
	public void testPositionsSimple() throws IOException {
	Directory directory = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
	newIndexWriterConfig(new MockAnalyzer(random())));
	for (int i = 0; i < 39; i++) {
	Document doc = new Document();
	FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
	customType.setOmitNorms(true);
	doc.add(newField(fieldName, "1 2 3 4 5 6 7 8 9 10 "
	+ "1 2 3 4 5 6 7 8 9 10 " + "1 2 3 4 5 6 7 8 9 10 "
	+ "1 2 3 4 5 6 7 8 9 10", customType));
	writer.addDocument(doc);
	}
	IndexReader reader = writer.getReader();
	writer.close();

	int num = atLeast(13);
	for (int i = 0; i < num; i++) {
	BytesRef bytes = new BytesRef("1");
	IndexReaderContext topReaderContext = reader.getContext();
	for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
	PostingsEnum docsAndPosEnum = getDocsAndPositions(
	leafReaderContext.reader(), bytes);
	assertNotNull(docsAndPosEnum);
	if (leafReaderContext.reader().maxDoc() == 0) {
	continue;
	}
	final int advance = docsAndPosEnum.advance(random().nextInt(leafReaderContext.reader().maxDoc()));
	do {
	String msg = "Advanced to: " + advance + " current doc: "
	+ docsAndPosEnum.docID(); // TODO: + " usePayloads: " + usePayload;
	assertEquals(msg, 4, docsAndPosEnum.freq());
	assertEquals(msg, 0, docsAndPosEnum.nextPosition());
	assertEquals(msg, 4, docsAndPosEnum.freq());
	assertEquals(msg, 10, docsAndPosEnum.nextPosition());
	assertEquals(msg, 4, docsAndPosEnum.freq());
	assertEquals(msg, 20, docsAndPosEnum.nextPosition());
	assertEquals(msg, 4, docsAndPosEnum.freq());
	assertEquals(msg, 30, docsAndPosEnum.nextPosition());
	} while (docsAndPosEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
	}
	}
	reader.close();
	directory.close();
	}

	public PostingsEnum getDocsAndPositions(LeafReader reader,
	BytesRef bytes) throws IOException {
	Terms terms = reader.terms(fieldName);
	if (terms != null) {
	TermsEnum te = terms.iterator();
	if (te.seekExact(bytes)) {
	return te.postings(null, PostingsEnum.ALL);
	}
	}
	return null;
	}

	/**
	* this test indexes random numbers within a range into a field and checks
	* their occurrences by searching for a number from that range selected at
	* random. All positions for that number are saved up front and compared to
	* the enums positions.
	*/
	public void testRandomPositions() throws IOException {
	Directory dir = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
	newIndexWriterConfig(new MockAnalyzer(random()))
	.setMergePolicy(newLogMergePolicy()));
	int numDocs = atLeast(47);
	int max = 1051;
	int term = random().nextInt(max);
	Integer[][] positionsInDoc = new Integer[numDocs][];
	FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
	customType.setOmitNorms(true);
	for (int i = 0; i < numDocs; i++) {
	Document doc = new Document();
	ArrayList<Integer> positions = new ArrayList<>();
	StringBuilder builder = new StringBuilder();
	int num = atLeast(131);
	for (int j = 0; j < num; j++) {
	int nextInt = random().nextInt(max);
	builder.append(nextInt).append(" ");
	if (nextInt == term) {
	positions.add(Integer.valueOf(j));
	}
	}
	if (positions.size() == 0) {
	builder.append(term);
	positions.add(num);
	}
	doc.add(newField(fieldName, builder.toString(), customType));
	positionsInDoc[i] = positions.toArray(new Integer[0]);
	writer.addDocument(doc);
	}

	IndexReader reader = writer.getReader();
	writer.close();

	int num = atLeast(13);
	for (int i = 0; i < num; i++) {
	BytesRef bytes = new BytesRef("" + term);
	IndexReaderContext topReaderContext = reader.getContext();
	for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
	PostingsEnum docsAndPosEnum = getDocsAndPositions(
	leafReaderContext.reader(), bytes);
	assertNotNull(docsAndPosEnum);
	int initDoc = 0;
	int maxDoc = leafReaderContext.reader().maxDoc();
	// initially advance or do next doc
	if (random().nextBoolean()) {
	initDoc = docsAndPosEnum.nextDoc();
	} else {
	initDoc = docsAndPosEnum.advance(random().nextInt(maxDoc));
	}
	// now run through the scorer and check if all positions are there...
	do {
	int docID = docsAndPosEnum.docID();
	if (docID == DocIdSetIterator.NO_MORE_DOCS) {
	break;
	}
	Integer[] pos = positionsInDoc[leafReaderContext.docBase + docID];
	assertEquals(pos.length, docsAndPosEnum.freq());
	// number of positions read should be random - don't read all of them
	// allways
	final int howMany = random().nextInt(20) == 0 ? pos.length
	- random().nextInt(pos.length) : pos.length;
	for (int j = 0; j < howMany; j++) {
	assertEquals("iteration: " + i + " initDoc: " + initDoc + " doc: "
	+ docID + " base: " + leafReaderContext.docBase
	+ " positions: " + Arrays.toString(pos) /* TODO: + " usePayloads: "
	+ usePayload*/, pos[j].intValue(), docsAndPosEnum.nextPosition());
	}

	if (random().nextInt(10) == 0) { // once is a while advance
	if (docsAndPosEnum.advance(docID + 1 + random().nextInt((maxDoc - docID))) == DocIdSetIterator.NO_MORE_DOCS) {
	break;
	}
	}

	} while (docsAndPosEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
	}

	}
	reader.close();
	dir.close();
	}

	public void testRandomDocs() throws IOException {
	Directory dir = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
	newIndexWriterConfig(new MockAnalyzer(random()))
	.setMergePolicy(newLogMergePolicy()));
	int numDocs = atLeast(49);
	int max = 15678;
	int term = random().nextInt(max);
	int[] freqInDoc = new int[numDocs];
	FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
	customType.setOmitNorms(true);
	for (int i = 0; i < numDocs; i++) {
	Document doc = new Document();
	StringBuilder builder = new StringBuilder();
	for (int j = 0; j < 199; j++) {
	int nextInt = random().nextInt(max);
	builder.append(nextInt).append(' ');
	if (nextInt == term) {
	freqInDoc[i]++;
	}
	}
	doc.add(newField(fieldName, builder.toString(), customType));
	writer.addDocument(doc);
	}

	IndexReader reader = writer.getReader();
	writer.close();

	int num = atLeast(13);
	for (int i = 0; i < num; i++) {
	BytesRef bytes = new BytesRef("" + term);
	IndexReaderContext topReaderContext = reader.getContext();
	for (LeafReaderContext context : topReaderContext.leaves()) {
	int maxDoc = context.reader().maxDoc();
	PostingsEnum postingsEnum = TestUtil.docs(random(), context.reader(), fieldName, bytes, null, PostingsEnum.FREQS);
	if (findNext(freqInDoc, context.docBase, context.docBase + maxDoc) == Integer.MAX_VALUE) {
	assertNull(postingsEnum);
	continue;
	}
	assertNotNull(postingsEnum);
	postingsEnum.nextDoc();
	for (int j = 0; j < maxDoc; j++) {
	if (freqInDoc[context.docBase + j] != 0) {
	assertEquals(j, postingsEnum.docID());
	assertEquals(postingsEnum.freq(), freqInDoc[context.docBase +j]);
	if (i % 2 == 0 && random().nextInt(10) == 0) {
	int next = findNext(freqInDoc, context.docBase+j+1, context.docBase + maxDoc) - context.docBase;
	int advancedTo = postingsEnum.advance(next);
	if (next >= maxDoc) {
	assertEquals(DocIdSetIterator.NO_MORE_DOCS, advancedTo);
	} else {
	assertTrue("advanced to: " +advancedTo + " but should be <= " + next, next >= advancedTo);
	}
	} else {
	postingsEnum.nextDoc();
	}
	}
	}
	assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + postingsEnum.getClass(), DocIdSetIterator.NO_MORE_DOCS, postingsEnum.docID());
	}

	}

	reader.close();
	dir.close();
	}

	private static int findNext(int[] docs, int pos, int max) {
	for (int i = pos; i < max; i++) {
	if( docs[i] != 0) {
	return i;
	}
	}
	return Integer.MAX_VALUE;
	}

	/**
	* tests retrieval of positions for terms that have a large number of
	* occurrences to force test of buffer refill during positions iteration.
	*/
	public void testLargeNumberOfPositions() throws IOException {
	Directory dir = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir,
	newIndexWriterConfig(new MockAnalyzer(random())));
	int howMany = 1000;
	FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
	customType.setOmitNorms(true);
	for (int i = 0; i < 39; i++) {
	Document doc = new Document();
	StringBuilder builder = new StringBuilder();
	for (int j = 0; j < howMany; j++) {
	if (j % 2 == 0) {
	builder.append("even ");
	} else {
	builder.append("odd ");
	}
	}
	doc.add(newField(fieldName, builder.toString(), customType));
	writer.addDocument(doc);
	}

	// now do searches
	IndexReader reader = writer.getReader();
	writer.close();

	int num = atLeast(13);
	for (int i = 0; i < num; i++) {
	BytesRef bytes = new BytesRef("even");

	IndexReaderContext topReaderContext = reader.getContext();
	for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) {
	PostingsEnum docsAndPosEnum = getDocsAndPositions(
	leafReaderContext.reader(), bytes);
	assertNotNull(docsAndPosEnum);

	int initDoc = 0;
	int maxDoc = leafReaderContext.reader().maxDoc();
	// initially advance or do next doc
	if (random().nextBoolean()) {
	initDoc = docsAndPosEnum.nextDoc();
	} else {
	initDoc = docsAndPosEnum.advance(random().nextInt(maxDoc));
	}
	String msg = "Iteration: " + i + " initDoc: " + initDoc; // TODO: + " payloads: " + usePayload;
	assertEquals(howMany / 2, docsAndPosEnum.freq());
	for (int j = 0; j < howMany; j += 2) {
	assertEquals("position missmatch index: " + j + " with freq: "
	+ docsAndPosEnum.freq() + " -- " + msg, j,
	docsAndPosEnum.nextPosition());
	}
	}
	}
	reader.close();
	dir.close();
	}

	public void testDocsEnumStart() throws Exception {
	Directory dir = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
	Document doc = new Document();
	doc.add(newStringField("foo", "bar", Field.Store.NO));
	writer.addDocument(doc);
	DirectoryReader reader = writer.getReader();
	LeafReader r = getOnlyLeafReader(reader);
	PostingsEnum disi = TestUtil.docs(random(), r, "foo", new BytesRef("bar"), null, PostingsEnum.NONE);
	int docid = disi.docID();
	assertEquals(-1, docid);
	assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

	// now reuse and check again
	TermsEnum te = r.terms("foo").iterator();
	assertTrue(te.seekExact(new BytesRef("bar")));
	disi = TestUtil.docs(random(), te, disi, PostingsEnum.NONE);
	docid = disi.docID();
	assertEquals(-1, docid);
	assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
	writer.close();
	r.close();
	dir.close();
	}

	public void testDocsAndPositionsEnumStart() throws Exception {
	Directory dir = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
	Document doc = new Document();
	doc.add(newTextField("foo", "bar", Field.Store.NO));
	writer.addDocument(doc);
	DirectoryReader reader = writer.getReader();
	LeafReader r = getOnlyLeafReader(reader);
	PostingsEnum disi = r.postings(new Term("foo", "bar"), PostingsEnum.ALL);
	int docid = disi.docID();
	assertEquals(-1, docid);
	assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

	// now reuse and check again
	TermsEnum te = r.terms("foo").iterator();
	assertTrue(te.seekExact(new BytesRef("bar")));
	disi = te.postings(disi, PostingsEnum.ALL);
	docid = disi.docID();
	assertEquals(-1, docid);
	assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
	writer.close();
	r.close();
	dir.close();
	}
	}