lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java - manifoldcf-integration-solr-3.x - Git at Google

 package org.apache.lucene.search;

 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.TermEnum;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.queryParser.ParseException;
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.search.Explanation.IDFExplanation;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.SimpleAnalyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.RAMDirectory;
 import org.apache.lucene.util.LuceneTestCase;

 import java.io.IOException;
 import java.util.Collection;
 import java.util.LinkedList;
 import java.io.Reader;

 /**
  * This class tests the MultiPhraseQuery class.
  *
  *
  */
 public class TestMultiPhraseQuery extends LuceneTestCase {

   public void testPhrasePrefix() throws IOException {
     Directory indexStore = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
     add("blueberry pie", writer);
     add("blueberry strudel", writer);
     add("blueberry pizza", writer);
     add("blueberry chewing gum", writer);
     add("bluebird pizza", writer);
     add("bluebird foobar pizza", writer);
     add("piccadilly circus", writer);

     IndexReader reader = writer.getReader();
     IndexSearcher searcher = newSearcher(reader);

     // search for "blueberry pi*":
     MultiPhraseQuery query1 = new MultiPhraseQuery();
     // search for "strawberry pi*":
     MultiPhraseQuery query2 = new MultiPhraseQuery();
     query1.add(new Term("body", "blueberry"));
     query2.add(new Term("body", "strawberry"));

     LinkedList<Term> termsWithPrefix = new LinkedList<Term>();
     IndexReader ir = reader;

     // this TermEnum gives "piccadilly", "pie" and "pizza".
     String prefix = "pi";
     TermEnum te = ir.terms(new Term("body", prefix));
     do {
         if (te.term().text().startsWith(prefix))
         {
             termsWithPrefix.add(te.term());
         }
     } while (te.next());

     query1.add(termsWithPrefix.toArray(new Term[0]));
     assertEquals("body:\"blueberry (piccadilly pie pizza)\"", query1.toString());
     query2.add(termsWithPrefix.toArray(new Term[0]));
     assertEquals("body:\"strawberry (piccadilly pie pizza)\"", query2.toString());

     ScoreDoc[] result;
     result = searcher.search(query1, null, 1000).scoreDocs;
     assertEquals(2, result.length);
     result = searcher.search(query2, null, 1000).scoreDocs;
     assertEquals(0, result.length);

     // search for "blue* pizza":
     MultiPhraseQuery query3 = new MultiPhraseQuery();
     termsWithPrefix.clear();
     prefix = "blue";
     te = ir.terms(new Term("body", prefix));
     do {
         if (te.term().text().startsWith(prefix))
         {
             termsWithPrefix.add(te.term());
         }
     } while (te.next());
     query3.add(termsWithPrefix.toArray(new Term[0]));
     query3.add(new Term("body", "pizza"));

     result = searcher.search(query3, null, 1000).scoreDocs;
     assertEquals(2, result.length); // blueberry pizza, bluebird pizza
     assertEquals("body:\"(blueberry bluebird) pizza\"", query3.toString());

     // test slop:
     query3.setSlop(1);
     result = searcher.search(query3, null, 1000).scoreDocs;

     // just make sure no exc:
     searcher.explain(query3, 0);

     assertEquals(3, result.length); // blueberry pizza, bluebird pizza, bluebird foobar pizza

     MultiPhraseQuery query4 = new MultiPhraseQuery();
     try {
       query4.add(new Term("field1", "foo"));
       query4.add(new Term("field2", "foobar"));
       fail();
     } catch(IllegalArgumentException e) {
       // okay, all terms must belong to the same field
     }

     writer.close();
     searcher.close();
     reader.close();
     indexStore.close();
   }

   // LUCENE-2580
   public void testTall() throws IOException {
     Directory indexStore = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
     add("blueberry chocolate pie", writer);
     add("blueberry chocolate tart", writer);
     IndexReader r = writer.getReader();
     writer.close();

     IndexSearcher searcher = newSearcher(r);
     MultiPhraseQuery q = new MultiPhraseQuery();
     q.add(new Term("body", "blueberry"));
     q.add(new Term("body", "chocolate"));
     q.add(new Term[] {new Term("body", "pie"), new Term("body", "tart")});
     assertEquals(2, searcher.search(q, 1).totalHits);
     searcher.close();
     r.close();
     indexStore.close();
   }

   private void add(String s, RandomIndexWriter writer) throws IOException {
     Document doc = new Document();
     doc.add(newField("body", s, Field.Store.YES, Field.Index.ANALYZED));
     writer.addDocument(doc);
   }

   public void testBooleanQueryContainingSingleTermPrefixQuery()
       throws IOException {
     // this tests against bug 33161 (now fixed)
     // In order to cause the bug, the outer query must have more than one term
     // and all terms required.
     // The contained PhraseMultiQuery must contain exactly one term array.
     Directory indexStore = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
     add("blueberry pie", writer);
     add("blueberry chewing gum", writer);
     add("blue raspberry pie", writer);

     IndexReader reader = writer.getReader();
     IndexSearcher searcher = newSearcher(reader);
     // This query will be equivalent to +body:pie +body:"blue*"
     BooleanQuery q = new BooleanQuery();
     q.add(new TermQuery(new Term("body", "pie")), BooleanClause.Occur.MUST);

     MultiPhraseQuery trouble = new MultiPhraseQuery();
     trouble.add(new Term[] {new Term("body", "blueberry"),
         new Term("body", "blue")});
     q.add(trouble, BooleanClause.Occur.MUST);

     // exception will be thrown here without fix
     ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;

     assertEquals("Wrong number of hits", 2, hits.length);

     // just make sure no exc:
     searcher.explain(q, 0);

     writer.close();
     searcher.close();
     reader.close();
     indexStore.close();
   }

   public void testPhrasePrefixWithBooleanQuery() throws IOException {
     Directory indexStore = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
     add("This is a test", "object", writer);
     add("a note", "note", writer);

     IndexReader reader = writer.getReader();
     IndexSearcher searcher = newSearcher(reader);

     // This query will be equivalent to +type:note +body:"a t*"
     BooleanQuery q = new BooleanQuery();
     q.add(new TermQuery(new Term("type", "note")), BooleanClause.Occur.MUST);

     MultiPhraseQuery trouble = new MultiPhraseQuery();
     trouble.add(new Term("body", "a"));
     trouble
         .add(new Term[] {new Term("body", "test"), new Term("body", "this")});
     q.add(trouble, BooleanClause.Occur.MUST);

     // exception will be thrown here without fix for #35626:
     ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;
     assertEquals("Wrong number of hits", 0, hits.length);
     writer.close();
     searcher.close();
     reader.close();
     indexStore.close();
   }

   public void testNoDocs() throws Exception {
     Directory indexStore = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
     add("a note", "note", writer);

     IndexReader reader = writer.getReader();
     IndexSearcher searcher = newSearcher(reader);

     MultiPhraseQuery q = new MultiPhraseQuery();
     q.add(new Term("body", "a"));
     q.add(new Term[] {new Term("body", "nope"), new Term("body", "nope")});
     assertEquals("Wrong number of hits", 0,
         searcher.search(q, null, 1).totalHits);

     // just make sure no exc:
     searcher.explain(q, 0);

     writer.close();
     searcher.close();
     reader.close();
     indexStore.close();
   }

   public void testHashCodeAndEquals() {
     MultiPhraseQuery query1 = new MultiPhraseQuery();
     MultiPhraseQuery query2 = new MultiPhraseQuery();

     assertEquals(query1.hashCode(), query2.hashCode());
     assertEquals(query1, query2);

     Term term1 = new Term("someField", "someText");

     query1.add(term1);
     query2.add(term1);

     assertEquals(query1.hashCode(), query2.hashCode());
     assertEquals(query1, query2);

     Term term2 = new Term("someField", "someMoreText");

     query1.add(term2);

     assertFalse(query1.hashCode() == query2.hashCode());
     assertFalse(query1.equals(query2));

     query2.add(term2);

     assertEquals(query1.hashCode(), query2.hashCode());
     assertEquals(query1, query2);
   }

   private void add(String s, String type, RandomIndexWriter writer)
       throws IOException {
     Document doc = new Document();
     doc.add(newField("body", s, Field.Store.YES, Field.Index.ANALYZED));
     doc.add(newField("type", type, Field.Store.YES, Field.Index.NOT_ANALYZED));
     writer.addDocument(doc);
   }

   // LUCENE-2526
   public void testEmptyToString() {
     new MultiPhraseQuery().toString();
   }

   public void testCustomIDF() throws Exception {
     Directory indexStore = newDirectory();
     RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
     add("This is a test", "object", writer);
     add("a note", "note", writer);

     IndexReader reader = writer.getReader();
     IndexSearcher searcher = newSearcher(reader);
     searcher.setSimilarity(new DefaultSimilarity() {

       @Override
       public IDFExplanation idfExplain(Collection<Term> terms,
           Searcher searcher) throws IOException {
         return new IDFExplanation() {

           @Override
           public float getIdf() {
             return 10f;
           }

           @Override
           public String explain() {
             return "just a test";
           }

         };
       }
     });

     MultiPhraseQuery query = new MultiPhraseQuery();
     query.add(new Term[] { new Term("body", "this"), new Term("body", "that") });
     query.add(new Term("body", "is"));
     Weight weight = query.createWeight(searcher);
     assertEquals(10f * 10f, weight.sumOfSquaredWeights(), 0.001f);

     writer.close();
     searcher.close();
     reader.close();
     indexStore.close();
   }

   private static class TokenAndPos {
     public final String token;
     public final int pos;
     public TokenAndPos(String token, int pos) {
       this.token = token;
       this.pos = pos;
     }
   }

   private static class CannedAnalyzer extends Analyzer {
     private final TokenAndPos[] tokens;

     public CannedAnalyzer(TokenAndPos[] tokens) {
       this.tokens = tokens;
     }

     @Override
     public TokenStream tokenStream(String fieldName, Reader reader) {
       return new CannedTokenizer(tokens);
     }
   }

   private static class CannedTokenizer extends Tokenizer {
     private final TokenAndPos[] tokens;
     private int upto = 0;
     private int lastPos = 0;
     private final TermAttribute termAtt = addAttribute(TermAttribute.class);
     private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);

     public CannedTokenizer(TokenAndPos[] tokens) {
       this.tokens = tokens;
     }

     @Override
     public final boolean incrementToken() throws IOException {
       clearAttributes();
       if (upto < tokens.length) {
         final TokenAndPos token = tokens[upto++];
         termAtt.setTermBuffer(token.token);
         posIncrAtt.setPositionIncrement(token.pos - lastPos);
         lastPos = token.pos;
         return true;
       } else {
         return false;
       }
     }

     @Override
     public void reset() throws IOException {
       super.reset();
       this.upto = 0;
       this.lastPos = 0;
     }
   }

   public void testZeroPosIncr() throws IOException {
     Directory dir = new RAMDirectory();
     final TokenAndPos[] tokens = new TokenAndPos[3];
     tokens[0] = new TokenAndPos("a", 0);
     tokens[1] = new TokenAndPos("b", 0);
     tokens[2] = new TokenAndPos("c", 0);

     IndexWriter writer = new IndexWriter(dir, new CannedAnalyzer(tokens), true, IndexWriter.MaxFieldLength.LIMITED);
     Document doc = new Document();
     doc.add(new Field("field", "", Field.Store.NO, Field.Index.ANALYZED));
     writer.addDocument(doc);
     writer.addDocument(doc);
     IndexReader r = writer.getReader();
     writer.close();
     IndexSearcher s = new IndexSearcher(r);
     MultiPhraseQuery mpq = new MultiPhraseQuery();
     //mpq.setSlop(1);

     // NOTE: not great that if we do the else clause here we
     // get different scores!  MultiPhraseQuery counts that
     // phrase as occurring twice per doc (it should be 1, I
     // think?).  This is because MultipleTermPositions is able to
     // return the same position more than once (0, in this
     // case):
     if (true) {
       mpq.add(new Term[] {new Term("field", "b"), new Term("field", "c")}, 0);
       mpq.add(new Term[] {new Term("field", "a")}, 0);
     } else {
       mpq.add(new Term[] {new Term("field", "a")}, 0);
       mpq.add(new Term[] {new Term("field", "b"), new Term("field", "c")}, 0);
     }
     TopDocs hits = s.search(mpq, 2);
     assertEquals(2, hits.totalHits);
     assertEquals(hits.scoreDocs[0].score, hits.scoreDocs[1].score, 1e-5);
     /*
     for(int hit=0;hit<hits.totalHits;hit++) {
       ScoreDoc sd = hits.scoreDocs[hit];
       System.out.println("  hit doc=" + sd.doc + " score=" + sd.score);
     }
     */
     r.close();
     dir.close();
   }

   private final static TokenAndPos[] INCR_0_DOC_TOKENS = new TokenAndPos[] {
       new TokenAndPos("x", 0),
       new TokenAndPos("a", 1),
       new TokenAndPos("1", 1),
       new TokenAndPos("m", 2), // not existing, relying on slop=2
       new TokenAndPos("b", 3),
       new TokenAndPos("1", 3),
       new TokenAndPos("n", 4), // not existing, relying on slop=2
       new TokenAndPos("c", 5),
       new TokenAndPos("y", 6)
   };

   private final static TokenAndPos[] INCR_0_QUERY_TOKENS_AND = new TokenAndPos[] {
       new TokenAndPos("a", 0),
       new TokenAndPos("1", 0),
       new TokenAndPos("b", 1),
       new TokenAndPos("1", 1),
       new TokenAndPos("c", 2)
   };

   private final static TokenAndPos[][] INCR_0_QUERY_TOKENS_AND_OR_MATCH = new TokenAndPos[][] {
       { new TokenAndPos("a", 0) },
       { new TokenAndPos("x", 0), new TokenAndPos("1", 0) },
       { new TokenAndPos("b", 1) },
       { new TokenAndPos("x", 1), new TokenAndPos("1", 1) },
       { new TokenAndPos("c", 2) }
   };

   private final static TokenAndPos[][] INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN = new TokenAndPos[][] {
       { new TokenAndPos("x", 0) },
       { new TokenAndPos("a", 0), new TokenAndPos("1", 0) },
       { new TokenAndPos("x", 1) },
       { new TokenAndPos("b", 1), new TokenAndPos("1", 1) },
       { new TokenAndPos("c", 2) }
   };

   /**
    * using query parser, MPQ will be created, and will not be strict about having all query terms
    * in each position - one of each position is sufficient (OR logic)
    */
   public void testZeroPosIncrSloppyParsedAnd() throws IOException, ParseException {
     QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new CannedAnalyzer(INCR_0_QUERY_TOKENS_AND));
     final Query q = qp.parse("\"this text is acually ignored\"");
     assertTrue("wrong query type!", q instanceof MultiPhraseQuery);
     doTestZeroPosIncrSloppy(q, 0);
     ((MultiPhraseQuery) q).setSlop(1);
     doTestZeroPosIncrSloppy(q, 0);
     ((MultiPhraseQuery) q).setSlop(2);
     doTestZeroPosIncrSloppy(q, 1);
   }

   private void doTestZeroPosIncrSloppy(Query q, int nExpected) throws IOException {
     Directory dir = newDirectory(); // random dir
     IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new CannedAnalyzer(INCR_0_DOC_TOKENS));
     IndexWriter writer = new IndexWriter(dir, cfg);
     Document doc = new Document();
     doc.add(new Field("field", "", Field.Store.NO, Field.Index.ANALYZED));
     writer.addDocument(doc);
     IndexReader r = IndexReader.open(writer,false);
     writer.close();
     IndexSearcher s = new IndexSearcher(r);

     if (VERBOSE) {
       System.out.println("QUERY=" + q);
     }

     TopDocs hits = s.search(q, 1);
     assertEquals("wrong number of results", nExpected, hits.totalHits);

     if (VERBOSE) {
       for(int hit=0;hit<hits.totalHits;hit++) {
         ScoreDoc sd = hits.scoreDocs[hit];
         System.out.println("  hit doc=" + sd.doc + " score=" + sd.score);
       }
     }

     r.close();
     dir.close();
   }

   /**
    * PQ AND Mode - Manually creating a phrase query
    */
   public void testZeroPosIncrSloppyPqAnd() throws IOException, ParseException {
     final PhraseQuery pq = new PhraseQuery();
     for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
       pq.add(new Term("field",tap.token), tap.pos);
     }
     doTestZeroPosIncrSloppy(pq, 0);
     pq.setSlop(1);
     doTestZeroPosIncrSloppy(pq, 0);
     pq.setSlop(2);
     doTestZeroPosIncrSloppy(pq, 1);
   }

   /**
    * MPQ AND Mode - Manually creating a multiple phrase query
    */
   public void testZeroPosIncrSloppyMpqAnd() throws IOException, ParseException {
     final MultiPhraseQuery mpq = new MultiPhraseQuery();
     for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
       mpq.add(new Term[]{new Term("field",tap.token)}, tap.pos); //AND logic
     }
     doTestZeroPosIncrSloppy(mpq, 0);
     mpq.setSlop(1);
     doTestZeroPosIncrSloppy(mpq, 0);
     mpq.setSlop(2);
     doTestZeroPosIncrSloppy(mpq, 1);
   }

   /**
    * MPQ Combined AND OR Mode - Manually creating a multiple phrase query
    */
   public void testZeroPosIncrSloppyMpqAndOrMatch() throws IOException, ParseException {
     final MultiPhraseQuery mpq = new MultiPhraseQuery();
     for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_MATCH) {
       Term[] terms = tapTerms(tap);
       final int pos = tap[0].pos;
       mpq.add(terms, pos); //AND logic in pos, OR across lines
     }
     doTestZeroPosIncrSloppy(mpq, 0);
     mpq.setSlop(1);
     doTestZeroPosIncrSloppy(mpq, 0);
     mpq.setSlop(2);
     doTestZeroPosIncrSloppy(mpq, 1);
   }

   /**
    * MPQ Combined AND OR Mode - Manually creating a multiple phrase query - with no match
    */
   public void testZeroPosIncrSloppyMpqAndOrNoMatch() throws IOException, ParseException {
     final MultiPhraseQuery mpq = new MultiPhraseQuery();
     for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN) {
       Term[] terms = tapTerms(tap);
       final int pos = tap[0].pos;
       mpq.add(terms, pos); //AND logic in pos, OR across lines
     }
     doTestZeroPosIncrSloppy(mpq, 0);
     mpq.setSlop(2);
     doTestZeroPosIncrSloppy(mpq, 0);
   }

   private Term[] tapTerms(TokenAndPos[] tap) {
     Term[] terms = new Term[tap.length];
     for (int i=0; i<terms.length; i++) {
       terms[i] = new Term("field",tap[i].token);
     }
     return terms;
   }

 }
	package org.apache.lucene.search;

	/**
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	import org.apache.lucene.index.IndexWriterConfig;
	import org.apache.lucene.index.RandomIndexWriter;
	import org.apache.lucene.index.Term;
	import org.apache.lucene.index.TermEnum;
	import org.apache.lucene.index.IndexReader;
	import org.apache.lucene.queryParser.ParseException;
	import org.apache.lucene.queryParser.QueryParser;
	import org.apache.lucene.search.Explanation.IDFExplanation;
	import org.apache.lucene.store.Directory;
	import org.apache.lucene.analysis.Analyzer;
	import org.apache.lucene.analysis.SimpleAnalyzer;
	import org.apache.lucene.analysis.TokenStream;
	import org.apache.lucene.analysis.Tokenizer;
	import org.apache.lucene.analysis.standard.StandardAnalyzer;
	import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
	import org.apache.lucene.analysis.tokenattributes.TermAttribute;
	import org.apache.lucene.document.Document;
	import org.apache.lucene.document.Field;
	import org.apache.lucene.index.IndexWriter;
	import org.apache.lucene.search.IndexSearcher;
	import org.apache.lucene.store.RAMDirectory;
	import org.apache.lucene.util.LuceneTestCase;

	import java.io.IOException;
	import java.util.Collection;
	import java.util.LinkedList;
	import java.io.Reader;

	/**
	* This class tests the MultiPhraseQuery class.
	*
	*
	*/
	public class TestMultiPhraseQuery extends LuceneTestCase {

	public void testPhrasePrefix() throws IOException {
	Directory indexStore = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
	add("blueberry pie", writer);
	add("blueberry strudel", writer);
	add("blueberry pizza", writer);
	add("blueberry chewing gum", writer);
	add("bluebird pizza", writer);
	add("bluebird foobar pizza", writer);
	add("piccadilly circus", writer);

	IndexReader reader = writer.getReader();
	IndexSearcher searcher = newSearcher(reader);

	// search for "blueberry pi*":
	MultiPhraseQuery query1 = new MultiPhraseQuery();
	// search for "strawberry pi*":
	MultiPhraseQuery query2 = new MultiPhraseQuery();
	query1.add(new Term("body", "blueberry"));
	query2.add(new Term("body", "strawberry"));

	LinkedList<Term> termsWithPrefix = new LinkedList<Term>();
	IndexReader ir = reader;

	// this TermEnum gives "piccadilly", "pie" and "pizza".
	String prefix = "pi";
	TermEnum te = ir.terms(new Term("body", prefix));
	do {
	if (te.term().text().startsWith(prefix))
	{
	termsWithPrefix.add(te.term());
	}
	} while (te.next());

	query1.add(termsWithPrefix.toArray(new Term[0]));
	assertEquals("body:\"blueberry (piccadilly pie pizza)\"", query1.toString());
	query2.add(termsWithPrefix.toArray(new Term[0]));
	assertEquals("body:\"strawberry (piccadilly pie pizza)\"", query2.toString());

	ScoreDoc[] result;
	result = searcher.search(query1, null, 1000).scoreDocs;
	assertEquals(2, result.length);
	result = searcher.search(query2, null, 1000).scoreDocs;
	assertEquals(0, result.length);

	// search for "blue* pizza":
	MultiPhraseQuery query3 = new MultiPhraseQuery();
	termsWithPrefix.clear();
	prefix = "blue";
	te = ir.terms(new Term("body", prefix));
	do {
	if (te.term().text().startsWith(prefix))
	{
	termsWithPrefix.add(te.term());
	}
	} while (te.next());
	query3.add(termsWithPrefix.toArray(new Term[0]));
	query3.add(new Term("body", "pizza"));

	result = searcher.search(query3, null, 1000).scoreDocs;
	assertEquals(2, result.length); // blueberry pizza, bluebird pizza
	assertEquals("body:\"(blueberry bluebird) pizza\"", query3.toString());

	// test slop:
	query3.setSlop(1);
	result = searcher.search(query3, null, 1000).scoreDocs;

	// just make sure no exc:
	searcher.explain(query3, 0);

	assertEquals(3, result.length); // blueberry pizza, bluebird pizza, bluebird foobar pizza

	MultiPhraseQuery query4 = new MultiPhraseQuery();
	try {
	query4.add(new Term("field1", "foo"));
	query4.add(new Term("field2", "foobar"));
	fail();
	} catch(IllegalArgumentException e) {
	// okay, all terms must belong to the same field
	}

	writer.close();
	searcher.close();
	reader.close();
	indexStore.close();
	}

	// LUCENE-2580
	public void testTall() throws IOException {
	Directory indexStore = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
	add("blueberry chocolate pie", writer);
	add("blueberry chocolate tart", writer);
	IndexReader r = writer.getReader();
	writer.close();

	IndexSearcher searcher = newSearcher(r);
	MultiPhraseQuery q = new MultiPhraseQuery();
	q.add(new Term("body", "blueberry"));
	q.add(new Term("body", "chocolate"));
	q.add(new Term[] {new Term("body", "pie"), new Term("body", "tart")});
	assertEquals(2, searcher.search(q, 1).totalHits);
	searcher.close();
	r.close();
	indexStore.close();
	}

	private void add(String s, RandomIndexWriter writer) throws IOException {
	Document doc = new Document();
	doc.add(newField("body", s, Field.Store.YES, Field.Index.ANALYZED));
	writer.addDocument(doc);
	}

	public void testBooleanQueryContainingSingleTermPrefixQuery()
	throws IOException {
	// this tests against bug 33161 (now fixed)
	// In order to cause the bug, the outer query must have more than one term
	// and all terms required.
	// The contained PhraseMultiQuery must contain exactly one term array.
	Directory indexStore = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
	add("blueberry pie", writer);
	add("blueberry chewing gum", writer);
	add("blue raspberry pie", writer);

	IndexReader reader = writer.getReader();
	IndexSearcher searcher = newSearcher(reader);
	// This query will be equivalent to +body:pie +body:"blue*"
	BooleanQuery q = new BooleanQuery();
	q.add(new TermQuery(new Term("body", "pie")), BooleanClause.Occur.MUST);

	MultiPhraseQuery trouble = new MultiPhraseQuery();
	trouble.add(new Term[] {new Term("body", "blueberry"),
	new Term("body", "blue")});
	q.add(trouble, BooleanClause.Occur.MUST);

	// exception will be thrown here without fix
	ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;

	assertEquals("Wrong number of hits", 2, hits.length);

	// just make sure no exc:
	searcher.explain(q, 0);

	writer.close();
	searcher.close();
	reader.close();
	indexStore.close();
	}

	public void testPhrasePrefixWithBooleanQuery() throws IOException {
	Directory indexStore = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
	add("This is a test", "object", writer);
	add("a note", "note", writer);

	IndexReader reader = writer.getReader();
	IndexSearcher searcher = newSearcher(reader);

	// This query will be equivalent to +type:note +body:"a t*"
	BooleanQuery q = new BooleanQuery();
	q.add(new TermQuery(new Term("type", "note")), BooleanClause.Occur.MUST);

	MultiPhraseQuery trouble = new MultiPhraseQuery();
	trouble.add(new Term("body", "a"));
	trouble
	.add(new Term[] {new Term("body", "test"), new Term("body", "this")});
	q.add(trouble, BooleanClause.Occur.MUST);

	// exception will be thrown here without fix for #35626:
	ScoreDoc[] hits = searcher.search(q, null, 1000).scoreDocs;
	assertEquals("Wrong number of hits", 0, hits.length);
	writer.close();
	searcher.close();
	reader.close();
	indexStore.close();
	}

	public void testNoDocs() throws Exception {
	Directory indexStore = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
	add("a note", "note", writer);

	IndexReader reader = writer.getReader();
	IndexSearcher searcher = newSearcher(reader);

	MultiPhraseQuery q = new MultiPhraseQuery();
	q.add(new Term("body", "a"));
	q.add(new Term[] {new Term("body", "nope"), new Term("body", "nope")});
	assertEquals("Wrong number of hits", 0,
	searcher.search(q, null, 1).totalHits);

	// just make sure no exc:
	searcher.explain(q, 0);

	writer.close();
	searcher.close();
	reader.close();
	indexStore.close();
	}

	public void testHashCodeAndEquals() {
	MultiPhraseQuery query1 = new MultiPhraseQuery();
	MultiPhraseQuery query2 = new MultiPhraseQuery();

	assertEquals(query1.hashCode(), query2.hashCode());
	assertEquals(query1, query2);

	Term term1 = new Term("someField", "someText");

	query1.add(term1);
	query2.add(term1);

	assertEquals(query1.hashCode(), query2.hashCode());
	assertEquals(query1, query2);

	Term term2 = new Term("someField", "someMoreText");

	query1.add(term2);

	assertFalse(query1.hashCode() == query2.hashCode());
	assertFalse(query1.equals(query2));

	query2.add(term2);

	assertEquals(query1.hashCode(), query2.hashCode());
	assertEquals(query1, query2);
	}

	private void add(String s, String type, RandomIndexWriter writer)
	throws IOException {
	Document doc = new Document();
	doc.add(newField("body", s, Field.Store.YES, Field.Index.ANALYZED));
	doc.add(newField("type", type, Field.Store.YES, Field.Index.NOT_ANALYZED));
	writer.addDocument(doc);
	}

	// LUCENE-2526
	public void testEmptyToString() {
	new MultiPhraseQuery().toString();
	}

	public void testCustomIDF() throws Exception {
	Directory indexStore = newDirectory();
	RandomIndexWriter writer = new RandomIndexWriter(random, indexStore);
	add("This is a test", "object", writer);
	add("a note", "note", writer);

	IndexReader reader = writer.getReader();
	IndexSearcher searcher = newSearcher(reader);
	searcher.setSimilarity(new DefaultSimilarity() {

	@Override
	public IDFExplanation idfExplain(Collection<Term> terms,
	Searcher searcher) throws IOException {
	return new IDFExplanation() {

	@Override
	public float getIdf() {
	return 10f;
	}

	@Override
	public String explain() {
	return "just a test";
	}

	};
	}
	});

	MultiPhraseQuery query = new MultiPhraseQuery();
	query.add(new Term[] { new Term("body", "this"), new Term("body", "that") });
	query.add(new Term("body", "is"));
	Weight weight = query.createWeight(searcher);
	assertEquals(10f * 10f, weight.sumOfSquaredWeights(), 0.001f);

	writer.close();
	searcher.close();
	reader.close();
	indexStore.close();
	}

	private static class TokenAndPos {
	public final String token;
	public final int pos;
	public TokenAndPos(String token, int pos) {
	this.token = token;
	this.pos = pos;
	}
	}

	private static class CannedAnalyzer extends Analyzer {
	private final TokenAndPos[] tokens;

	public CannedAnalyzer(TokenAndPos[] tokens) {
	this.tokens = tokens;
	}

	@Override
	public TokenStream tokenStream(String fieldName, Reader reader) {
	return new CannedTokenizer(tokens);
	}
	}

	private static class CannedTokenizer extends Tokenizer {
	private final TokenAndPos[] tokens;
	private int upto = 0;
	private int lastPos = 0;
	private final TermAttribute termAtt = addAttribute(TermAttribute.class);
	private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);

	public CannedTokenizer(TokenAndPos[] tokens) {
	this.tokens = tokens;
	}

	@Override
	public final boolean incrementToken() throws IOException {
	clearAttributes();
	if (upto < tokens.length) {
	final TokenAndPos token = tokens[upto++];
	termAtt.setTermBuffer(token.token);
	posIncrAtt.setPositionIncrement(token.pos - lastPos);
	lastPos = token.pos;
	return true;
	} else {
	return false;
	}
	}

	@Override
	public void reset() throws IOException {
	super.reset();
	this.upto = 0;
	this.lastPos = 0;
	}
	}

	public void testZeroPosIncr() throws IOException {
	Directory dir = new RAMDirectory();
	final TokenAndPos[] tokens = new TokenAndPos[3];
	tokens[0] = new TokenAndPos("a", 0);
	tokens[1] = new TokenAndPos("b", 0);
	tokens[2] = new TokenAndPos("c", 0);

	IndexWriter writer = new IndexWriter(dir, new CannedAnalyzer(tokens), true, IndexWriter.MaxFieldLength.LIMITED);
	Document doc = new Document();
	doc.add(new Field("field", "", Field.Store.NO, Field.Index.ANALYZED));
	writer.addDocument(doc);
	writer.addDocument(doc);
	IndexReader r = writer.getReader();
	writer.close();
	IndexSearcher s = new IndexSearcher(r);
	MultiPhraseQuery mpq = new MultiPhraseQuery();
	//mpq.setSlop(1);

	// NOTE: not great that if we do the else clause here we
	// get different scores! MultiPhraseQuery counts that
	// phrase as occurring twice per doc (it should be 1, I
	// think?). This is because MultipleTermPositions is able to
	// return the same position more than once (0, in this
	// case):
	if (true) {
	mpq.add(new Term[] {new Term("field", "b"), new Term("field", "c")}, 0);
	mpq.add(new Term[] {new Term("field", "a")}, 0);
	} else {
	mpq.add(new Term[] {new Term("field", "a")}, 0);
	mpq.add(new Term[] {new Term("field", "b"), new Term("field", "c")}, 0);
	}
	TopDocs hits = s.search(mpq, 2);
	assertEquals(2, hits.totalHits);
	assertEquals(hits.scoreDocs[0].score, hits.scoreDocs[1].score, 1e-5);
	/*
	for(int hit=0;hit<hits.totalHits;hit++) {
	ScoreDoc sd = hits.scoreDocs[hit];
	System.out.println(" hit doc=" + sd.doc + " score=" + sd.score);
	}
	*/
	r.close();
	dir.close();
	}

	private final static TokenAndPos[] INCR_0_DOC_TOKENS = new TokenAndPos[] {
	new TokenAndPos("x", 0),
	new TokenAndPos("a", 1),
	new TokenAndPos("1", 1),
	new TokenAndPos("m", 2), // not existing, relying on slop=2
	new TokenAndPos("b", 3),
	new TokenAndPos("1", 3),
	new TokenAndPos("n", 4), // not existing, relying on slop=2
	new TokenAndPos("c", 5),
	new TokenAndPos("y", 6)
	};

	private final static TokenAndPos[] INCR_0_QUERY_TOKENS_AND = new TokenAndPos[] {
	new TokenAndPos("a", 0),
	new TokenAndPos("1", 0),
	new TokenAndPos("b", 1),
	new TokenAndPos("1", 1),
	new TokenAndPos("c", 2)
	};

	private final static TokenAndPos[][] INCR_0_QUERY_TOKENS_AND_OR_MATCH = new TokenAndPos[][] {
	{ new TokenAndPos("a", 0) },
	{ new TokenAndPos("x", 0), new TokenAndPos("1", 0) },
	{ new TokenAndPos("b", 1) },
	{ new TokenAndPos("x", 1), new TokenAndPos("1", 1) },
	{ new TokenAndPos("c", 2) }
	};

	private final static TokenAndPos[][] INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN = new TokenAndPos[][] {
	{ new TokenAndPos("x", 0) },
	{ new TokenAndPos("a", 0), new TokenAndPos("1", 0) },
	{ new TokenAndPos("x", 1) },
	{ new TokenAndPos("b", 1), new TokenAndPos("1", 1) },
	{ new TokenAndPos("c", 2) }
	};

	/**
	* using query parser, MPQ will be created, and will not be strict about having all query terms
	* in each position - one of each position is sufficient (OR logic)
	*/
	public void testZeroPosIncrSloppyParsedAnd() throws IOException, ParseException {
	QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new CannedAnalyzer(INCR_0_QUERY_TOKENS_AND));
	final Query q = qp.parse("\"this text is acually ignored\"");
	assertTrue("wrong query type!", q instanceof MultiPhraseQuery);
	doTestZeroPosIncrSloppy(q, 0);
	((MultiPhraseQuery) q).setSlop(1);
	doTestZeroPosIncrSloppy(q, 0);
	((MultiPhraseQuery) q).setSlop(2);
	doTestZeroPosIncrSloppy(q, 1);
	}

	private void doTestZeroPosIncrSloppy(Query q, int nExpected) throws IOException {
	Directory dir = newDirectory(); // random dir
	IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new CannedAnalyzer(INCR_0_DOC_TOKENS));
	IndexWriter writer = new IndexWriter(dir, cfg);
	Document doc = new Document();
	doc.add(new Field("field", "", Field.Store.NO, Field.Index.ANALYZED));
	writer.addDocument(doc);
	IndexReader r = IndexReader.open(writer,false);
	writer.close();
	IndexSearcher s = new IndexSearcher(r);

	if (VERBOSE) {
	System.out.println("QUERY=" + q);
	}

	TopDocs hits = s.search(q, 1);
	assertEquals("wrong number of results", nExpected, hits.totalHits);

	if (VERBOSE) {
	for(int hit=0;hit<hits.totalHits;hit++) {
	ScoreDoc sd = hits.scoreDocs[hit];
	System.out.println(" hit doc=" + sd.doc + " score=" + sd.score);
	}
	}

	r.close();
	dir.close();
	}

	/**
	* PQ AND Mode - Manually creating a phrase query
	*/
	public void testZeroPosIncrSloppyPqAnd() throws IOException, ParseException {
	final PhraseQuery pq = new PhraseQuery();
	for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
	pq.add(new Term("field",tap.token), tap.pos);
	}
	doTestZeroPosIncrSloppy(pq, 0);
	pq.setSlop(1);
	doTestZeroPosIncrSloppy(pq, 0);
	pq.setSlop(2);
	doTestZeroPosIncrSloppy(pq, 1);
	}

	/**
	* MPQ AND Mode - Manually creating a multiple phrase query
	*/
	public void testZeroPosIncrSloppyMpqAnd() throws IOException, ParseException {
	final MultiPhraseQuery mpq = new MultiPhraseQuery();
	for (TokenAndPos tap : INCR_0_QUERY_TOKENS_AND) {
	mpq.add(new Term[]{new Term("field",tap.token)}, tap.pos); //AND logic
	}
	doTestZeroPosIncrSloppy(mpq, 0);
	mpq.setSlop(1);
	doTestZeroPosIncrSloppy(mpq, 0);
	mpq.setSlop(2);
	doTestZeroPosIncrSloppy(mpq, 1);
	}

	/**
	* MPQ Combined AND OR Mode - Manually creating a multiple phrase query
	*/
	public void testZeroPosIncrSloppyMpqAndOrMatch() throws IOException, ParseException {
	final MultiPhraseQuery mpq = new MultiPhraseQuery();
	for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_MATCH) {
	Term[] terms = tapTerms(tap);
	final int pos = tap[0].pos;
	mpq.add(terms, pos); //AND logic in pos, OR across lines
	}
	doTestZeroPosIncrSloppy(mpq, 0);
	mpq.setSlop(1);
	doTestZeroPosIncrSloppy(mpq, 0);
	mpq.setSlop(2);
	doTestZeroPosIncrSloppy(mpq, 1);
	}

	/**
	* MPQ Combined AND OR Mode - Manually creating a multiple phrase query - with no match
	*/
	public void testZeroPosIncrSloppyMpqAndOrNoMatch() throws IOException, ParseException {
	final MultiPhraseQuery mpq = new MultiPhraseQuery();
	for (TokenAndPos tap[] : INCR_0_QUERY_TOKENS_AND_OR_NO_MATCHN) {
	Term[] terms = tapTerms(tap);
	final int pos = tap[0].pos;
	mpq.add(terms, pos); //AND logic in pos, OR across lines
	}
	doTestZeroPosIncrSloppy(mpq, 0);
	mpq.setSlop(2);
	doTestZeroPosIncrSloppy(mpq, 0);
	}

	private Term[] tapTerms(TokenAndPos[] tap) {
	Term[] terms = new Term[tap.length];
	for (int i=0; i<terms.length; i++) {
	terms[i] = new Term("field",tap[i].token);
	}
	return terms;
	}

	}