/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.solr.search;

import java.util.HashSet;
import java.util.Random;
import java.util.Set;

import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.AbstractSolrTestCase;
import org.apache.solr.util.SolrPluginUtils;
import org.junit.BeforeClass;
import org.junit.Test;

public class TestExtendedDismaxParser extends SolrTestCaseJ4 {

  @BeforeClass
  public static void beforeClass() throws Exception {
    System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_
    initCore("solrconfig.xml", "schema12.xml");
    index();
  }
  
   public static void index() throws Exception {
    assertU(adoc("id", "42", "trait_ss", "Tool", "trait_ss", "Obnoxious",
            "name", "Zapp Brannigan"));
    assertU(adoc("id", "43" ,
            "title", "Democratic Order op Planets"));
    assertU(adoc("id", "44", "trait_ss", "Tool",
            "name", "The Zapper"));
    assertU(adoc("id", "45", "trait_ss", "Chauvinist",
            "title", "25 star General"));
    assertU(adoc("id", "46", 
                 "trait_ss", "Obnoxious",
                 "subject", "Defeated the pacifists op the Gandhi nebula",
                 "t_special", "literal:colon value",
                 "movies_t", "first is Mission: Impossible, second is Terminator 2: Judgement Day.  Terminator:3 ok...",
                 "foo_i", "8"
    ));
    assertU(adoc("id", "47", "trait_ss", "Pig",
            "text", "line up and fly directly at the enemy death cannons, clogging them with wreckage!"));
    assertU(adoc("id", "48", "text_sw", "this has gigabyte potential", "foo_i","100"));
    assertU(adoc("id", "49", "text_sw", "start the big apple end", "foo_i","-100"));
    assertU(adoc("id", "50", "text_sw", "start new big city end"));
    assertU(adoc("id", "51", "store",   "12.34,-56.78"));
    assertU(adoc("id", "52", "text_sw", "tekna theou klethomen"));
    assertU(adoc("id", "53", "text_sw", "nun tekna theou esmen"));
    assertU(adoc("id", "54", "text_sw", "phanera estin ta tekna tou theou"));
    assertU(adoc("id", "55", "standardtok", "大"));
    assertU(adoc("id", "56", "standardtok", "大亚"));
    assertU(adoc("id", "57", "standardtok", "大亚湾"));
    assertU(adoc("id", "58", "HTMLstandardtok", "大"));
    assertU(adoc("id", "59", "HTMLstandardtok", "大亚"));
    assertU(adoc("id", "60", "HTMLstandardtok", "大亚湾"));
    assertU(adoc("id", "61", "text_sw", "bazaaa")); // synonyms in an expansion group
    assertU(commit());
  }

  @Test
  public void testSyntax() throws Exception {
    // a bare * should be treated as *:*
    assertJQ(req("defType","edismax", "q","*", "df","doesnotexist_s")
        ,"/response/docs/[0]=="   // make sure we get something...
    );
    assertJQ(req("defType","edismax", "q","doesnotexist_s:*")
        ,"/response/numFound==0"   // nothing should be found
    );
    assertJQ(req("defType","edismax","q","doesnotexist_s:*")
        ,"/response/numFound==0"   // nothing should be found
    );
    assertJQ(req("defType","edismax","q","doesnotexist_s:( * * * )")
        ,"/response/numFound==0"   // nothing should be found
    );
  }


  public void testTrailingOperators() throws Exception {
    // really just test that exceptions aren't thrown by
    // single + -

    assertJQ(req("defType","edismax", "q","-")
        ,"/response==");

    assertJQ(req("defType","edismax", "q","+")
        ,"/response==");

    assertJQ(req("defType","edismax", "q","+ - +")
        ,"/response==");

    assertJQ(req("defType","edismax", "q","- + -")
        ,"/response==");

    assertJQ(req("defType","edismax", "q","id:47 +")
        ,"/response/numFound==1");

    assertJQ(req("defType","edismax", "q","id:47 -")
        ,"/response/numFound==1");

    Random r = random();
    for (int i=0; i<100; i++) {
      StringBuilder sb = new StringBuilder();
      for (int j=0; j<r.nextInt(10); j++) {
        switch (r.nextInt(3)) {
          case 0: sb.append(' '); break;
          case 1: sb.append('+'); break;
          case 2: sb.append('-'); break;
          case 3: sb.append((char)r.nextInt(127)); break;
        }
      }

      String q = sb.toString();
      assertJQ(req("defType","edismax", "q",q)
          ,"/response==");
    }
  }


  public void testLowercaseOperators() {
    assertQ("Upper case operator",
        req("q","Zapp AND Brannigan",
            "qf", "name",
            "lowercaseOperators", "false",
            "defType","edismax")
        ,"*[count(//doc)=1]");
    
    assertQ("Upper case operator, allow lowercase",
        req("q","Zapp AND Brannigan",
            "qf", "name",
            "lowercaseOperators", "true",
            "defType","edismax")
        ,"*[count(//doc)=1]");
    
    assertQ("Lower case operator, don't allow lowercase operators",
        req("q","Zapp and Brannigan",
            "qf", "name",
            "q.op", "AND", 
            "lowercaseOperators", "false",
            "defType","edismax")
        ,"*[count(//doc)=0]");
    
    assertQ("Lower case operator, allow lower case operators",
        req("q","Zapp and Brannigan",
            "qf", "name",
            "q.op", "AND", 
            "lowercaseOperators", "true",
            "defType","edismax")
        ,"*[count(//doc)=1]");
  }
    
  // test the edismax query parser based on the dismax parser
  public void testFocusQueryParser() {
    String allq = "id:[42 TO 51]";
    String allr = "*[count(//doc)=10]";
    String oner = "*[count(//doc)=1]";
    String twor = "*[count(//doc)=2]";
    String nor = "*[count(//doc)=0]";
    
    assertQ("blank q",
        req("q"," ",
            "q.alt",allq,
            "defType","edismax")
        ,allr);
    
    assertQ("expected doc is missing (using un-escaped edismax w/qf)",
          req("q", "literal:colon", 
              "qf", "t_special",
              "defType", "edismax"),
          "//doc[1]/str[@name='id'][.='46']"); 

    assertQ("standard request handler returns all matches",
            req(allq),
            allr
    );

   assertQ("edismax query parser returns all matches",
            req("q", allq,
                "defType", "edismax"
            ),
            allr
    );

   assertQ(req("defType", "edismax", "qf", "trait_ss",
               "q","Tool"), twor
    );

   // test that field types that aren't applicable don't cause an exception to be thrown
   assertQ(req("defType", "edismax", "qf", "trait_ss foo_i foo_f foo_dt foo_l foo_d foo_b",
               "q","Tool"), twor
    );

   // test that numeric field types can be queried
   assertQ(req("defType", "edismax", "qf", "text_sw",
               "q","foo_i:100"), oner
    );

   // test that numeric field types can be queried
   assertQ(req("defType", "edismax", "qf", "text_sw",
               "q","foo_i:-100"), oner
    );

   // test that numeric field types can be queried  via qf
   assertQ(req("defType", "edismax", "qf", "text_sw foo_i",
               "q","100"), oner
    );

    assertQ("qf defaults to df",
        req("defType", "edismax", "df", "trait_ss",
        "q","Tool"), twor
    );

   assertQ("qf defaults to defaultSearchField"
           , req( "defType", "edismax"
                 ,"q","op")
           , twor
           );
   
   assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","op"), twor
    );
   assertQ(req("defType", "edismax", 
               "qf", "name title subject text",
               "q.op", "AND",
               "q","Order op"), oner
    );
   assertQ(req("defType", "edismax", 
               "qf", "name title subject text",
               "q.op", "OR",
               "q","Order op"), twor
    );
   assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","Order AND op"), oner
    );
   assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","Order and op"), oner
    );
    assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","+Order op"), oner
    );
    assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","Order OR op"), twor
    );
    assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","Order or op"), twor
    );
    assertQ(req("defType", "edismax", "qf", "name title subject text",
               "q","*:*"), allr
    );

    assertQ(req("defType", "edismax", "qf", "name title subject text",
           "q","star OR (-star)"), allr
    );
    assertQ(req("defType", "edismax", "qf", "name title subject text",
           "q","id:42 OR (-id:42)"), allr
    );

    // test that basic synonyms work
    assertQ(req("defType", "edismax", "qf", "text_sw",
           "q","GB"), oner
    );

    // test for stopword removal in main query part
    assertQ(req("defType", "edismax", "qf", "text_sw",
           "q","the big"), twor
    );

    // test for stopwords not removed   
    assertQ(req("defType", "edismax", 
                "qf", "text_sw", 
                "stopwords","false",
                "q.op","AND",
                "q","the big"), oner
    );

    // searching for a literal colon value when clearly not used for a field
    assertQ("expected doc is missing (using standard)",
            req("q", "t_special:literal\\:colon"),
            "//doc[1]/str[@name='id'][.='46']"); 
    assertQ("expected doc is missing (using escaped edismax w/field)",
            req("q", "t_special:literal\\:colon", 
                "defType", "edismax"),
            "//doc[1]/str[@name='id'][.='46']"); 
    assertQ("expected doc is missing (using un-escaped edismax w/field)",
            req("q", "t_special:literal:colon", 
                "defType", "edismax"),
            "//doc[1]/str[@name='id'][.='46']"); 
    assertQ("expected doc is missing (using escaped edismax w/qf)",
            req("q", "literal\\:colon", 
                "qf", "t_special",
                "defType", "edismax"),
            "//doc[1]/str[@name='id'][.='46']"); 
    assertQ("expected doc is missing (using un-escaped edismax w/qf)",
            req("q", "literal:colon", 
                "qf", "t_special",
                "defType", "edismax"),
            "//doc[1]/str[@name='id'][.='46']");

    assertQ(req("defType","edismax", "mm","100%", "q","terminator:3", "qf","movies_t"),
            oner);
    assertQ(req("defType","edismax", "mm","100%", "q","Mission:Impossible", "qf","movies_t"),
            oner);
    assertQ(req("defType","edismax", "mm","100%", "q","Mission : Impossible", "qf","movies_t"),
            oner);
    assertQ(req("defType","edismax", "mm","100%", "q","Mission: Impossible", "qf","movies_t"),
            oner);
    assertQ(req("defType","edismax", "mm","100%", "q","Terminator 2: Judgement Day", "qf","movies_t"),
            oner);

    // make sure the clause wasn't eliminated
    assertQ(req("defType","edismax", "mm","100%", "q","Terminator 10: Judgement Day", "qf","movies_t"),
            nor);

    // throw in a numeric field
    assertQ(req("defType","edismax", "mm","0", "q","Terminator: 100", "qf","movies_t foo_i"),
            twor);

    assertQ(req("defType","edismax", "mm","100%", "q","Terminator: 100", "qf","movies_t foo_i"),
            nor);

    assertQ(req("defType","edismax", "mm","100%", "q","Terminator: 8", "qf","movies_t foo_i"),
            oner);

    assertQ(req("defType","edismax", "mm","0", "q","movies_t:Terminator 100", "qf","movies_t foo_i"),
            twor);
    
    // special psuedo-fields like _query_ and _val_

    // special fields (and real field id) should be included by default
    assertQ(req("defType", "edismax", 
                "mm", "100%",
                "fq", "id:51",
                "q", "_query_:\"{!geofilt d=20 sfield=store pt=12.34,-56.78}\""),
            oner);
    // should also work when explicitly allowed
    assertQ(req("defType", "edismax", 
                "mm", "100%",
                "fq", "id:51",
                "uf", "id _query_",
                "q", "_query_:\"{!geofilt d=20 sfield=store pt=12.34,-56.78}\""),
            oner);
    assertQ(req("defType", "edismax", 
                "mm", "100%",
                "fq", "id:51",
                "uf", "id",
                "uf", "_query_",
                "q", "_query_:\"{!geofilt d=20 sfield=store pt=12.34,-56.78}\""),
            oner);

    // should fail when prohibited
    assertQ(req("defType", "edismax", 
                "mm", "100%",
                "fq", "id:51",
                "uf", "* -_query_", // explicitly excluded
                "q", "_query_:\"{!geofilt d=20 sfield=store pt=12.34,-56.78}\""),
            nor);
    assertQ(req("defType", "edismax", 
                "mm", "100%",
                "fq", "id:51",
                "uf", "id", // excluded by ommision
                "q", "_query_:\"{!geofilt d=20 sfield=store pt=12.34,-56.78}\""),
            nor);


    /** stopword removal in conjunction with multi-word synonyms at query time
     * break this test.
     // multi-word synonyms
     // remove id:50 which contans the false match      
    assertQ(req("defType", "edismax", "qf", "text_t", "indent","true", "debugQuery","true",
           "q","-id:50 nyc"), oner
    );
    **/

    /*** these fail because multi-word synonyms are being used at query time
    // this will incorrectly match "new big city"
    assertQ(req("defType", "edismax", "qf", "id title",
           "q","nyc"), oner
    );

    // this will incorrectly match "new big city"
    assertQ(req("defType", "edismax", "qf", "title",
           "q","the big apple"), nor
    );
    ***/

  }
  
  public void testBoostQuery() {
    assertQ(
        req("q", "tekna", "qf", "text_sw", "defType", "edismax", "bq", "id:54^100", "bq", "id:53^10", "fq", "id:[52 TO 54]", "fl", "id,score"), 
        "//doc[1]/str[@name='id'][.='54']",
        "//doc[2]/str[@name='id'][.='53']",
        "//doc[3]/str[@name='id'][.='52']"
     );
    
    // non-trivial bqs
    assertQ(req("q", "tekna", 
                "qf", "text_sw", 
                "defType", "edismax", 
                "bq", "(text_sw:blasdfadsf id:54)^100", 
                "bq", "id:[53 TO 53]^10", 
                "fq", "id:[52 TO 54]", 
                "fl", "id,score"), 
            "//doc[1]/str[@name='id'][.='54']",
            "//doc[2]/str[@name='id'][.='53']",
            "//doc[3]/str[@name='id'][.='52']"
            );

    // genuine negative boosts are not legal
    // see SOLR-3823, SOLR-3278, LUCENE-4378 and 
    // https://wiki.apache.org/solr/SolrRelevancyFAQ#How_do_I_give_a_negative_.28or_very_low.29_boost_to_documents_that_match_a_query.3F
    assertQ(
        req("q", "tekna", "qf", "text_sw", "defType", "edismax", "bq", "(*:* -id:54)^100", "bq", "id:53^10", "bq", "id:52", "fq", "id:[52 TO 54]", "fl", "id,score"), 
        "//doc[1]/str[@name='id'][.='53']",
        "//doc[2]/str[@name='id'][.='52']",
        "//doc[3]/str[@name='id'][.='54']"
     );
  }

  public void testUserFields() {
    String allr = "*[count(//doc)=10]";
    String oner = "*[count(//doc)=1]";
    String nor = "*[count(//doc)=0]";
    
    // User fields
    // Default is allow all "*"
    // If a list of fields are given, only those are allowed "foo bar"
    // Possible to invert with "-" syntax:
    //   Disallow all: "-*"
    //   Allow all but id: "* -id"
    // Also supports "dynamic" field name wildcarding
    assertQ(req("defType","edismax", "q","id:42"),
        oner);
    
    // SOLR-3377 - parens should be allowed immediately before field name
    assertQ(req("defType","edismax", "q","( id:42 )"),
        oner);
    assertQ(req("defType","edismax", "q","(id:42)"),
        oner);
    assertQ(req("defType","edismax", "q","(+id:42)"),
        oner);
    assertQ(req("defType","edismax", "q","+(+id:42)"),
        oner);
    assertQ(req("defType","edismax", "q","+(+((id:42)))"),
        oner);
    assertQ(req("defType","edismax", "q","+(+((+id:42)))"),
        oner);
    assertQ(req("defType","edismax", "q"," +( +( ( +id:42) ) ) "),
        oner);
    assertQ(req("defType","edismax", "q","(id:(*:*)^200)"),
        allr);

    assertQ(req("defType","edismax", "uf","id", "q","id:42"),
        oner);
    
    assertQ(req("defType","edismax", "uf","-*", "q","id:42"),
        nor);
    
    assertQ(req("defType","edismax", "uf","loremipsum", "q","id:42"),
        nor);
    
    assertQ(req("defType","edismax", "uf","* -id", "q","id:42"),
        nor);
    
    assertQ(req("defType","edismax", "uf","* -loremipsum", "q","id:42"),
        oner);
    
    assertQ(req("defType","edismax", "uf","id^5.0", "q","id:42"),
        oner);
    
    assertQ(req("defType","edismax", "uf","*^5.0", "q","id:42"),
        oner);
    
    assertQ(req("defType","edismax", "uf","id^5.0", "q","id:42^10.0"),
        oner);
    
    assertQ(req("defType","edismax", "uf","na*", "q","name:Zapp"),
        oner);
    
    assertQ(req("defType","edismax", "uf","*me", "q","name:Zapp"),
        oner);
    
    assertQ(req("defType","edismax", "uf","* -na*", "q","name:Zapp"),
        nor);
    
    assertQ(req("defType","edismax", "uf","*me -name", "q","name:Zapp"),
        nor);
    
    assertQ(req("defType","edismax", "uf","*ame -*e", "q","name:Zapp"),
        nor);
    
    // Boosts from user fields
    assertQ(req("defType","edismax", "debugQuery","true", "rows","0", "q","id:42"),
        "//str[@name='parsedquery_toString'][.='+id:42']");
    
    assertQ(req("defType","edismax", "debugQuery","true", "rows","0", "uf","*^5.0", "q","id:42"),
        "//str[@name='parsedquery_toString'][.='+id:42^5.0']");
    
    assertQ(req("defType","edismax", "debugQuery","true", "rows","0", "uf","*^2.0 id^5.0 -xyz", "q","name:foo"),
        "//str[@name='parsedquery_toString'][.='+name:foo^2.0']");
    
    assertQ(req("defType","edismax", "debugQuery","true", "rows","0", "uf","i*^5.0", "q","id:42"),
        "//str[@name='parsedquery_toString'][.='+id:42^5.0']");
    
    
    assertQ(req("defType","edismax", "uf","-*", "q","cannons"),
        oner);
    
    assertQ(req("defType","edismax", "uf","* -id", "q","42", "qf", "id"), oner);
    
  }
  
  public void testAliasing() throws Exception {
    String oner = "*[count(//doc)=1]";
    String twor = "*[count(//doc)=2]";
    String nor = "*[count(//doc)=0]";
    
 // Aliasing
    // Single field
    assertQ(req("defType","edismax", "q","myalias:Zapp"),
        nor);
    
    assertQ(req("defType","edismax", "q","myalias:Zapp", "f.myalias.qf","name"),
        oner);
    
    // Multi field
    assertQ(req("defType","edismax", "uf", "myalias", "q","myalias:(Zapp Obnoxious)", "f.myalias.qf","name^2.0 mytrait_ss^5.0", "mm", "50%"),
        oner);
    
    // Multi field
    assertQ(req("defType","edismax", "q","Zapp Obnoxious", "f.myalias.qf","name^2.0 mytrait_ss^5.0"),
        nor);
    
    assertQ(req("defType","edismax", "q","Zapp Obnoxious", "qf","myalias^10.0", "f.myalias.qf","name^2.0 mytrait_ss^5.0"), oner);
    assertQ(req("defType","edismax", "q","Zapp Obnoxious", "qf","myalias^10.0", "f.myalias.qf","name^2.0 trait_ss^5.0"), twor);
    assertQ(req("defType","edismax", "q","Zapp Obnoxious", "qf","myalias^10.0", "f.myalias.qf","name^2.0 trait_ss^5.0", "mm", "100%"), oner);
    assertQ(req("defType","edismax", "q","Zapp Obnoxious", "qf","who^10.0 where^3.0", "f.who.qf","name^2.0", "f.where.qf", "mytrait_ss^5.0"), oner);
    
    assertQ(req("defType","edismax", "q","Zapp Obnoxious", "qf","myalias", "f.myalias.qf","name mytrait_ss", "uf", "myalias"), oner);
    
    assertQ(req("defType","edismax", "uf","who", "q","who:(Zapp Obnoxious)", "f.who.qf", "name^2.0 trait_ss^5.0", "qf", "id"), twor);
    assertQ(req("defType","edismax", "uf","* -name", "q","who:(Zapp Obnoxious)", "f.who.qf", "name^2.0 trait_ss^5.0"), twor);
    
  }
  
  public void testAliasingBoost() throws Exception {
    assertQ(req("defType","edismax", "q","Zapp Pig", "qf","myalias", "f.myalias.qf","name trait_ss^0.5"), "//result/doc[1]/str[@name='id']=42", "//result/doc[2]/str[@name='id']=47");//doc 42 should score higher than 46
    assertQ(req("defType","edismax", "q","Zapp Pig", "qf","myalias^100 name", "f.myalias.qf","trait_ss^0.5"), "//result/doc[1]/str[@name='id']=47", "//result/doc[2]/str[@name='id']=42");//Now the order should be inverse
  }
  
  public void testCyclicAliasing() throws Exception {
    try {
      ignoreException(".*Field aliases lead to a cycle.*");
      try {
        h.query(req("defType","edismax", "q","blarg", "qf","who", "f.who.qf","name","f.name.qf","who"));
        fail("Simple cyclic alising not detected");
      } catch (SolrException e) {
        assertTrue(e.getCause().getMessage().contains("Field aliases lead to a cycle"));
      }
      
      try {
        h.query(req("defType","edismax", "q","blarg", "qf","who", "f.who.qf","name","f.name.qf","myalias", "f.myalias.qf","who"));
        fail("Cyclic alising not detected");
      } catch (SolrException e) {
        assertTrue(e.getCause().getMessage().contains("Field aliases lead to a cycle"));
      }
      
      try {
        h.query(req("defType","edismax", "q","blarg", "qf","field1", "f.field1.qf","field2 field3","f.field2.qf","field4 field5", "f.field4.qf","field5", "f.field5.qf","field6", "f.field3.qf","field6"));
      } catch (SolrException e) {
        fail("This is not cyclic alising");
      }
      
      try {
        h.query(req("defType","edismax", "q","blarg", "qf","field1", "f.field1.qf","field2 field3", "f.field2.qf","field4 field5", "f.field4.qf","field5", "f.field5.qf","field4"));
        fail("Cyclic alising not detected");
      } catch (SolrException e) {
        assertTrue(e.getCause().getMessage().contains("Field aliases lead to a cycle"));
      }
      
      try {
        h.query(req("defType","edismax", "q","who:(Zapp Pig)", "qf","field1", "f.who.qf","name","f.name.qf","myalias", "f.myalias.qf","who"));
        fail("Cyclic alising not detected");
      } catch (SolrException e) {
        assertTrue(e.getCause().getMessage().contains("Field aliases lead to a cycle"));
      }
    } finally {
      resetExceptionIgnores();
    }
  }

  public void testOperatorsWithLiteralColons() {
    assertU(adoc("id", "142", "a_s", "bogus:xxx", "text_s", "yak"));
    assertU(adoc("id", "143", "a_s", "bogus:xxx"));
    assertU(adoc("id", "144", "text_s", "yak"));
    assertU(adoc("id", "145", "a_s", "a_s:xxx", "text_s", "yak"));
    assertU(adoc("id", "146", "a_s", "a_s:xxx"));
    assertU(adoc("id", "147", "a_s", "AND", "a_s", "NOT"));
    assertU(commit());

    assertQ(req("q", "bogus:xxx AND text_s:yak",
                "fl", "id",
                "qf", "a_s b_s",
                "defType", "edismax",
                "mm", "0"),
            "//*[@numFound='1']",
            "//str[@name='id'][.='142']");
    
    assertQ(req("q", "a_s:xxx AND text_s:yak",
                "fl", "id",
                "qf", "a_s b_s",
                "defType", "edismax",
                "mm", "0",
                "uf", "text_s"),
            "//*[@numFound='1']",
            "//str[@name='id'][.='145']");

    assertQ(req("q", "NOT bogus:xxx +text_s:yak",
                "fl", "id",
                "qf", "a_s b_s",
                "defType", "edismax",
                "mm", "0",
                "debugQuery", "true"),
            "//*[@numFound='2']",
            "//str[@name='id'][.='144']",
            "//str[@name='id'][.='145']");
    
    assertQ(req("q", "NOT a_s:xxx +text_s:yak",
                "fl", "id",
                "qf", "a_s b_s",
                "defType", "edismax",
                "mm", "0",
                "uf", "text_s"),
            "//*[@numFound='2']",
            "//str[@name='id'][.='142']",
            "//str[@name='id'][.='144']");
    
    assertQ(req("q", "+bogus:xxx yak",
                "fl", "id",
                "qf", "a_s b_s text_s",
                "defType", "edismax",
                "mm", "0"),
            "//*[@numFound='2']",
            "//str[@name='id'][.='142']",
            "//str[@name='id'][.='143']");

    assertQ(req("q", "+a_s:xxx yak",
                "fl", "id",
                "qf", "a_s b_s text_s",
                "defType", "edismax",
                "mm", "0",
                "uf", "b_s"),
            "//*[@numFound='2']",
            "//str[@name='id'][.='145']",
            "//str[@name='id'][.='146']");
  }
  
  // test phrase fields including pf2 pf3 and phrase slop
  public void testPfPs() {
    assertU(adoc("id", "s0", "phrase_sw", "foo bar a b c", "boost_d", "1.0"));    
    assertU(adoc("id", "s1", "phrase_sw", "foo a bar b c", "boost_d", "2.0"));    
    assertU(adoc("id", "s2", "phrase_sw", "foo a b bar c", "boost_d", "3.0"));    
    assertU(adoc("id", "s3", "phrase_sw", "foo a b c bar", "boost_d", "4.0"));    
    assertU(commit());

    assertQ("default order assumption wrong",
        req("q", "foo bar",
            "qf", "phrase_sw",
            "bf", "boost_d",
            "fl", "score,*",
            "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s3']",
        "//doc[2]/str[@name='id'][.='s2']",
        "//doc[3]/str[@name='id'][.='s1']",
        "//doc[4]/str[@name='id'][.='s0']"); 

    assertQ("pf not working",
        req("q", "foo bar",
            "qf", "phrase_sw",
            "pf", "phrase_sw^10",
            "bf", "boost_d",
            "fl", "score,*",
            "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s0']");
    
    assertQ("pf2 not working",
        req("q",   "foo bar", 
            "qf",  "phrase_sw",
            "pf2", "phrase_sw^10",
            "bf",  "boost_d",
            "fl",  "score,*",
            "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s0']"); 

    assertQ("pf3 not working",
        req("q",   "a b bar", 
            "qf",  "phrase_sw",
            "pf3", "phrase_sw^10",
            "bf",  "boost_d",
            "fl",  "score,*",
            "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s2']"); 

    assertQ("ps not working for pf2",
        req("q",   "bar foo", 
            "qf",  "phrase_sw",
            "pf2", "phrase_sw^10",
            "ps",  "2",
            "bf",  "boost_d",
            "fl",  "score,*",
            "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s0']"); 

    assertQ("ps not working for pf3",
        req("q",   "a bar foo", 
            "qf",  "phrase_sw",
            "pf3", "phrase_sw^10",
            "ps",  "3",
            "bf",  "boost_d",
            "fl",  "score,*",
            "debugQuery",  "true",
            "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s1']"); 
    
    assertQ("ps/ps2/ps3 with default slop overrides not working",
        req("q", "zzzz xxxx cccc vvvv",
            "qf", "phrase_sw",
            "pf", "phrase_sw~1^10 phrase_sw~2^20 phrase_sw^30",
            "pf2", "phrase_sw~2^22 phrase_sw^33",
            "pf3", "phrase_sw~2^222 phrase_sw^333",
            "ps", "3",
            "defType", "edismax",
            "debugQuery", "true"),
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx cccc vvvv\"~1^10.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx cccc vvvv\"~2^20.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx cccc vvvv\"~3^30.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx\"~2^22.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"xxxx cccc\"~2^22.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"cccc vvvv\"~2^22.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx\"~3^33.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"xxxx cccc\"~3^33.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"cccc vvvv\"~3^33.0')]",        
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx cccc\"~2^222.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"xxxx cccc vvvv\"~2^222.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx cccc\"~3^333.0')]",
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"xxxx cccc vvvv\"~3^333.0')]"
     );

    assertQ(
        "ps2 not working",
        req("q", "bar foo", "qf", "phrase_sw", "pf2", "phrase_sw^10", "ps2",
            "2", "bf", "boost_d", "fl", "score,*", "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s0']");
    
    assertQ(
        "Specifying slop in pf2 param not working",
        req("q", "bar foo", "qf", "phrase_sw", "pf2", "phrase_sw~2^10", "bf",
            "boost_d", "fl", "score,*", "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s0']");
    
    assertQ(
        "Slop in ps2 parameter should override ps",
        req("q", "bar foo", "qf", "phrase_sw", "pf2", "phrase_sw^10", "ps",
            "0", "ps2", "2", "bf", "boost_d", "fl", "score,*", "defType",
            "edismax"), "//doc[1]/str[@name='id'][.='s0']");

    assertQ(
        "ps3 not working",
        req("q", "a bar foo", "qf", "phrase_sw", "pf3", "phrase_sw^10", "ps3",
            "3", "bf", "boost_d", "fl", "score,*", "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s1']");
    
    assertQ(
        "Specifying slop in pf3 param not working",
        req("q", "a bar foo", "qf", "phrase_sw", "pf3", "phrase_sw~3^10", "bf",
            "boost_d", "fl", "score,*", "defType", "edismax"),
        "//doc[1]/str[@name='id'][.='s1']");
   
    assertQ("ps2 should not override slop specified inline in pf2",
        req("q", "zzzz xxxx cccc vvvv",
            "qf", "phrase_sw",
            "pf2", "phrase_sw~2^22",
            "ps2", "4",
            "defType", "edismax",
            "debugQuery", "true"),
        "//str[@name='parsedquery'][contains(.,'phrase_sw:\"zzzz xxxx\"~2^22.0')]"
     );

  }

  /**
   * verify that all reserved characters are properly escaped when being set in
   * {@link org.apache.solr.search.ExtendedDismaxQParser.Clause#val}.
   *
   * @see ExtendedDismaxQParser#splitIntoClauses(String, boolean)
   */
  @Test
  public void testEscapingOfReservedCharacters() throws Exception {
    // create a document that contains all reserved characters
    String allReservedCharacters = "!():^[]{}~*?\"+-\\|&/";

    assertU(adoc("id", "reservedChars",
                 "name", allReservedCharacters,
                 "cat_s", "foo/"));
    assertU(commit());

    // the backslash needs to be manually escaped (the query parser sees the raw backslash as an escape the subsequent
    // character)
    String query = allReservedCharacters.replace("\\", "\\\\");

    // query for all those reserved characters. This will fail to parse in the initial parse, meaning that the escaped
    // query will then be used
    assertQ("Escaping reserved characters",
        req("q", query,
            "qf", "name",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=1]");
    
    // Query string field 'cat_s' for special char / - causes SyntaxError without patch SOLR-3467
    assertQ("Escaping string with reserved / character",
        req("q", "foo/",
            "qf", "cat_s",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=1]");
    
    assertQ(
        "Might be double-escaping a client-escaped colon", 
        req("q", "text_sw:(theos OR thistokenhasa\\:preescapedcolon OR theou)", "defType", "edismax", "qf", "id"),
        "*[count(//doc)=3]");
    assertQ(
        "Might be double-escaping a client-escaped colon", 
        req("q", "text_sw:(theos OR thistokenhasa\\:preescapedcolon OR theou)", "defType", "edismax", "qf", "text"),
        "*[count(//doc)=3]");    
    
  }
  
  /**
   * SOLR-3589: Edismax parser does not honor mm parameter if analyzer splits a token
   */
  public void testCJK() throws Exception {
    assertQ("test cjk (disjunction)",
        req("q", "大亚湾",
            "qf", "standardtok",
            "mm", "0%",
            "defType", "edismax")
        , "*[count(//doc)=3]");
    assertQ("test cjk (minShouldMatch)",
        req("q", "大亚湾",
            "qf", "standardtok",
            "mm", "67%",
            "defType", "edismax")
        , "*[count(//doc)=2]");
    assertQ("test cjk (conjunction)",
        req("q", "大亚湾",
            "qf", "standardtok",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=1]");
  }
  
  /** 
   * test that minShouldMatch works with aliasing
   * for implicit boolean queries
   */
  public void testCJKAliasing() throws Exception {
    // single field
    assertQ("test cjk (aliasing+disjunction)",
        req("q", "myalias:大亚湾",
            "f.myalias.qf", "standardtok",
            "mm", "0%",
            "defType", "edismax")
        , "*[count(//doc)=3]");
    assertQ("test cjk (aliasing+minShouldMatch)",
        req("q", "myalias:大亚湾",
            "f.myalias.qf", "standardtok",
            "mm", "67%",
            "defType", "edismax")
        , "*[count(//doc)=2]");
    assertQ("test cjk (aliasing+conjunction)",
        req("q", "myalias:大亚湾",
            "f.myalias.qf", "standardtok",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=1]");
    // multifield
    assertQ("test cjk (aliasing+disjunction)",
        req("q", "myalias:大亚湾",
            "f.myalias.qf", "standardtok HTMLstandardtok",
            "mm", "0%",
            "defType", "edismax")
        , "*[count(//doc)=6]");
    assertQ("test cjk (aliasing+minShouldMatch)",
        req("q", "myalias:大亚湾",
            "f.myalias.qf", "standardtok HTMLstandardtok",
            "mm", "67%",
            "defType", "edismax")
        , "*[count(//doc)=4]");
    assertQ("test cjk (aliasing+conjunction)",
        req("q", "myalias:大亚湾",
            "f.myalias.qf", "standardtok HTMLstandardtok",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=2]");
  }
  
  /** Test that we apply boosts correctly */
  public void testCJKBoosts() throws Exception {
    assertQ("test cjk (disjunction)",
        req("q", "大亚湾",
            "qf", "standardtok^2 HTMLstandardtok",
            "mm", "0%",
            "defType", "edismax")
        , "*[count(//doc)=6]", "//result/doc[1]/str[@name='id'][.='57']");
    assertQ("test cjk (minShouldMatch)",
        req("q", "大亚湾",
            "qf", "standardtok^2 HTMLstandardtok",
            "mm", "67%",
            "defType", "edismax")
        , "*[count(//doc)=4]", "//result/doc[1]/str[@name='id'][.='57']");
    assertQ("test cjk (conjunction)",
        req("q", "大亚湾",
            "qf", "standardtok^2 HTMLstandardtok",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=2]", "//result/doc[1]/str[@name='id'][.='57']");
    
    // now boost the other field
    assertQ("test cjk (disjunction)",
        req("q", "大亚湾",
            "qf", "standardtok HTMLstandardtok^2",
            "mm", "0%",
            "defType", "edismax")
        , "*[count(//doc)=6]", "//result/doc[1]/str[@name='id'][.='60']");
    assertQ("test cjk (minShouldMatch)",
        req("q", "大亚湾",
            "qf", "standardtok HTMLstandardtok^2",
            "mm", "67%",
            "defType", "edismax")
        , "*[count(//doc)=4]", "//result/doc[1]/str[@name='id'][.='60']");
    assertQ("test cjk (conjunction)",
        req("q", "大亚湾",
            "qf", "standardtok HTMLstandardtok^2",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=2]", "//result/doc[1]/str[@name='id'][.='60']");
  }
  
  /** always apply minShouldMatch to the inner booleanqueries
   *  created from whitespace, as these are never structured lucene queries
   *  but only come from unstructured text */
  public void testCJKStructured() throws Exception {
    assertQ("test cjk (disjunction)",
        req("q", "大亚湾 OR bogus",
            "qf", "standardtok",
            "mm", "0%",
            "defType", "edismax")
        , "*[count(//doc)=3]");
    assertQ("test cjk (minShouldMatch)",
        req("q", "大亚湾 OR bogus",
            "qf", "standardtok",
            "mm", "67%",
            "defType", "edismax")
        , "*[count(//doc)=2]");
    assertQ("test cjk (conjunction)",
        req("q", "大亚湾 OR bogus",
            "qf", "standardtok",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=1]");
  }
  
  /**
   * Test that we don't apply minShouldMatch to the inner boolean queries
   * when there are synonyms (these are indicated by coordination factor)
   */
  public void testSynonyms() throws Exception {
    // document only contains baraaa, but should still match.
    assertQ("test synonyms",
        req("q", "fooaaa",
            "qf", "text_sw",
            "mm", "100%",
            "defType", "edismax")
        , "*[count(//doc)=1]");
  }
  
  public void testEdismaxSimpleExtension() throws SyntaxError {
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("q", "foo bar");
    params.set("qf", "subject title^5");
    params.set("qf_fr", "subject_fr title_fr^5");
    params.set("qf_en", "subject_en title_en^5");
    params.set("qf_es", "subject_es title_es^5");
    
    MultilanguageQueryParser parser = new MultilanguageQueryParser("foo bar", new ModifiableSolrParams(), params, req(params));
    Query query = parser.parse();
    assertNotNull(query);
    assertTrue(containsClause(query, "title", "foo", 5, false));
    assertTrue(containsClause(query, "title", "bar", 5, false));
    assertTrue(containsClause(query, "subject", "foo", 1, false));
    assertTrue(containsClause(query, "subject", "bar", 1, false));
    
    params.set("language", "es");
    parser = new MultilanguageQueryParser("foo bar", new ModifiableSolrParams(), params, req(params));
    query = parser.parse();
    assertNotNull(query);
    assertTrue(containsClause(query, "title_es", "foo", 5, false));
    assertTrue(containsClause(query, "title_es", "bar", 5, false));
    assertTrue(containsClause(query, "subject_es", "foo", 1, false));
    assertTrue(containsClause(query, "subject_es", "bar", 1, false));
    
    FuzzyDismaxQParser parser2 = new FuzzyDismaxQParser("foo bar absence", new ModifiableSolrParams(), params, req(params));
    query = parser2.parse();
    assertNotNull(query);
    assertTrue(containsClause(query, "title", "foo", 5, false));
    assertTrue(containsClause(query, "title", "bar", 5, false));
    assertTrue(containsClause(query, "title", "absence", 5, true));
    
  }

  private boolean containsClause(Query query, String field, String value,
      int boost, boolean fuzzy) {
    
    if(query instanceof BooleanQuery) {
      return containsClause((BooleanQuery)query, field, value, boost, fuzzy);
    }
    if(query instanceof DisjunctionMaxQuery) {
      return containsClause((DisjunctionMaxQuery)query, field, value, boost, fuzzy);
    }
    if(query instanceof TermQuery && !fuzzy) {
      return containsClause((TermQuery)query, field, value, boost);
    }
    if(query instanceof FuzzyQuery && fuzzy) {
      return containsClause((FuzzyQuery)query, field, value, boost);
    }
    return false;
  }

  private boolean containsClause(FuzzyQuery query, String field, String value,
      int boost) {
    if(query.getTerm().field().equals(field) && 
       query.getTerm().bytes().utf8ToString().equals(value) && 
       query.getBoost() == boost) {
      return true;
    }
    return false;
  }

  private boolean containsClause(BooleanQuery query, String field, String value, int boost, boolean fuzzy) {
    for(BooleanClause clause:query.getClauses()) {
      if(containsClause(clause.getQuery(), field, value, boost, fuzzy)) {
        return true;
      }
    }
    return false;
  }
  
  private boolean containsClause(TermQuery query, String field, String value, int boost) {
    if(query.getTerm().field().equals(field) && 
       query.getTerm().bytes().utf8ToString().equals(value) && 
       query.getBoost() == boost) {
      return true;
    }
    return false;
  }
  
  private boolean containsClause(DisjunctionMaxQuery query, String field, String value, int boost, boolean fuzzy) {
    for(Query disjunct:query.getDisjuncts()) {
      if(containsClause(disjunct, field, value, boost, fuzzy)) {
        return true;
      }
    }
    return false;
  }

  class MultilanguageQueryParser extends ExtendedDismaxQParser {

    public MultilanguageQueryParser(String qstr, SolrParams localParams,
        SolrParams params, SolrQueryRequest req) {
      super(qstr, localParams, params, req);
    }
    
    @Override
    protected ExtendedDismaxConfiguration createConfiguration(String qstr,
        SolrParams localParams, SolrParams params, SolrQueryRequest req) {
      return new MultilanguageDismaxConfiguration(localParams, params, req);
    }
    
    class MultilanguageDismaxConfiguration extends ExtendedDismaxConfiguration {

      public MultilanguageDismaxConfiguration(SolrParams localParams,
          SolrParams params, SolrQueryRequest req) {
        super(localParams, params, req);
        String language = params.get("language");
        if(language != null) {
          super.queryFields = SolrPluginUtils.parseFieldBoosts(solrParams.getParams("qf_" + language)); 
        }
      }
      
    }
    
  }
  
  
  class FuzzyDismaxQParser extends ExtendedDismaxQParser {

    public FuzzyDismaxQParser(String qstr, SolrParams localParams,
        SolrParams params, SolrQueryRequest req) {
      super(qstr, localParams, params, req);
    }
    
    @Override
    protected ExtendedSolrQueryParser createEdismaxQueryParser(QParser qParser,
        String field) {
      return new FuzzyQueryParser(qParser, field);
    }
    
    class FuzzyQueryParser extends ExtendedSolrQueryParser{
      
      private Set<String> frequentlyMisspelledWords;

      public FuzzyQueryParser(QParser parser, String defaultField) {
        super(parser, defaultField);
        frequentlyMisspelledWords = new HashSet<String>();
        frequentlyMisspelledWords.add("absence");
      }
      
      @Override
      protected Query getFieldQuery(String field,
          String val, boolean quoted) throws SyntaxError {
        if(frequentlyMisspelledWords.contains(val)) {
          return getFuzzyQuery(field, val, 0.75F);
        }
        return super.getFieldQuery(field, val, quoted);
      }
    }
  }
}