/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.search.join;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;

import javax.xml.xpath.XPathConstants;

import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.metrics.MetricsMap;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.util.BaseTestHarness;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;

public class BJQParserTest extends SolrTestCaseJ4 {
  
  private static final String[] klm = new String[] {"k", "l", "m"};
  private static final List<String> xyz = Arrays.asList("x", "y", "z");
  private static final String[] abcdef = new String[] {"a", "b", "c", "d", "e", "f"};
  
  @BeforeClass
  public static void beforeClass() throws Exception {
    initCore("solrconfig.xml", "schema15.xml");
    createIndex();
  }
  
  public static void createIndex() throws IOException, Exception {
    int i = 0;
    List<List<String[]>> blocks = createBlocks();
    for (List<String[]> block : blocks) {
      List<XmlDoc> updBlock = new ArrayList<>();
      
      for (String[] doc : block) {
        String[] idDoc = Arrays.copyOf(doc,doc.length+2);
        idDoc[doc.length]="id";
        idDoc[doc.length+1]=Integer.toString(i);
        updBlock.add(doc(idDoc));
        i++;
      }
      //got xmls for every doc. now nest all into the last one
      XmlDoc parentDoc = updBlock.get(updBlock.size()-1);
      parentDoc.xml = parentDoc.xml.replace("</doc>", 
          updBlock.subList(0, updBlock.size()-1).toString().replaceAll("[\\[\\]]","")+"</doc>");
      assertU(add(parentDoc));
      
      if (random().nextBoolean()) {
        assertU(commit());
        // force empty segment (actually, this will no longer create an empty segment, only a new segments_n)
        if (random().nextBoolean()) {
          assertU(commit());
        }
      }
    }
    assertU(commit());
    assertQ(req("q", "*:*"), "//*[@numFound='" + i + "']");
  }

  private static List<List<String[]>> createBlocks() {
    List<List<String[]>> blocks = new ArrayList<>();
    for (String parent : abcdef) {
      List<String[]> block = createChildrenBlock(parent);
      block.add(new String[] {"parent_s", parent});
      blocks.add(block);
    }
    Collections.shuffle(blocks, random());
    return blocks;
  }

  private static List<String[]> createChildrenBlock(String parent) {
    List<String[]> block = new ArrayList<>();
    for (String child : klm) {
      block
          .add(new String[] {"child_s", child, "parentchild_s", parent + child, "childparent_s", parent});
    }
    Collections.shuffle(block, random());
    addGrandChildren(block);
    return block;
  }
  
  private static void addGrandChildren(List<String[]> block) {
    List<String> grandChildren = new ArrayList<>(xyz);
    // add grandchildren after children
    for (ListIterator<String[]> iter = block.listIterator(); iter.hasNext();) {
      String[] child = iter.next();
      assert child[0]=="child_s" && child[2]=="parentchild_s": Arrays.toString(child);
      String child_s = child[1];
      String parentchild_s = child[3];
      int grandChildPos = 0;
      boolean lastLoopButStillHasGrCh = !iter.hasNext()
          && !grandChildren.isEmpty();
      while (!grandChildren.isEmpty()
          && ((grandChildPos = random().nextInt(grandChildren.size() * 2)) < grandChildren
              .size() || lastLoopButStillHasGrCh)) {
        grandChildPos = grandChildPos >= grandChildren.size() ? 0
            : grandChildPos;
        iter.add(new String[] {"grand_s", grandChildren.remove(grandChildPos),
            "grand_child_s", child_s, "grand_parentchild_s", parentchild_s});
      }
    }
    // and reverse after that
    Collections.reverse(block);
  }
  
  @Test
  public void testFull() throws IOException, Exception {
    String childb = "{!parent which=\"parent_s:[* TO *]\"}child_s:l";
    assertQ(req("q", childb), sixParents);
  }
  
  private static final String sixParents[] = new String[] {
      "//*[@numFound='6']", "//doc/arr[@name=\"parent_s\"]/str='a'",
      "//doc/arr[@name=\"parent_s\"]/str='b'",
      "//doc/arr[@name=\"parent_s\"]/str='c'",
      "//doc/arr[@name=\"parent_s\"]/str='d'",
      "//doc/arr[@name=\"parent_s\"]/str='e'",
      "//doc/arr[@name=\"parent_s\"]/str='f'"};
  
  @Test
  public void testJustParentsFilter() throws IOException {
    assertQ(req("q", "{!parent which=\"parent_s:[* TO *]\"}"), sixParents);
  }
  
  @Test
  public void testJustParentsFilterInChild() throws IOException {
    assertQ(req("q", "{!child of=\"parent_s:[* TO *]\"}",
          "fq", "childparent_s:"+abcdef[random().nextInt(abcdef.length)],
        "indent","on"), 
        "//*[@numFound='"+klm.length+"']", //for any parent we have all three children
        "//doc/arr[@name='child_s']/str='"+klm[0]+"'",
        "//doc/arr[@name='child_s']/str='"+klm[1]+"'",
        "//doc/arr[@name='child_s']/str='"+klm[2]+"'"
        );
    assert klm.length==3 : "change asserts pls "+klm;
  }
  
  private final static String beParents[] = new String[] {"//*[@numFound='2']",
      "//doc/arr[@name=\"parent_s\"]/str='b'",
      "//doc/arr[@name=\"parent_s\"]/str='e'"};
  
  @Test
  public void testIntersectBqBjq() {
    
    assertQ(
        req("q", "+parent_s:(e b) +_query_:\"{!parent which=$pq v=$chq}\"",
            "chq", "child_s:l", "pq", "parent_s:[* TO *]"), beParents);
    assertQ(
        req("fq", "{!parent which=$pq v=$chq}\"", "q", "parent_s:(e b)", "chq",
            "child_s:l", "pq", "parent_s:[* TO *]"), beParents);
    
    assertQ(
        req("q", "*:*", "fq", "{!parent which=$pq v=$chq}\"", "fq",
            "parent_s:(e b)", "chq", "child_s:l", "pq", "parent_s:[* TO *]"),
        beParents);
  }

  public void testScoreNoneScoringForParent() throws Exception {
    assertQ("score=none yields 0.0 score",
        req("q", "{!parent which=\"parent_s:[* TO *]\" "+(
            rarely()? "":(rarely()? "score=None":"score=none")
            )+"}child_s:l","fl","score"),
        "//*[@numFound='6']",
        "(//float[@name='score'])["+(random().nextInt(6)+1)+"]=0.0");
  }

  public void testWrongScoreExceptionForParent() throws Exception {
    final String aMode = ScoreMode.values()[random().nextInt(ScoreMode.values().length)].name();
    final String wrongMode = rarely()? "":(rarely()? " ":
      rarely()? aMode.substring(1):aMode.toUpperCase(Locale.ROOT));
    assertQEx("wrong score mode", 
        req("q", "{!parent which=\"parent_s:[* TO *]\" score="+wrongMode+"}child_s:l","fl","score")
        , SolrException.ErrorCode.BAD_REQUEST.code);
  }

  public void testScoresForParent() throws Exception{
    final ArrayList<ScoreMode> noNone = new ArrayList<>(Arrays.asList(ScoreMode.values()));
    noNone.remove(ScoreMode.None);
    final String notNoneMode = (noNone.get(random().nextInt(noNone.size()))).name();
    
    String leastScore = getLeastScore("child_s:l");
    assertTrue(leastScore+" > 0.0", Float.parseFloat(leastScore)>0.0);
    final String notNoneLower = usually() ? notNoneMode: notNoneMode.toLowerCase(Locale.ROOT);
    
    assertQ(req("q", "{!parent which=\"parent_s:[* TO *]\" score="+notNoneLower+"}child_s:l","fl","score"),
        "//*[@numFound='6']","(//float[@name='score'])["+(random().nextInt(6)+1)+"]>='"+leastScore+"'");
  }
  
  public void testScoresForChild() throws Exception{ 
    String leastScore = getLeastScore("parent_s:a");
      assertTrue(leastScore+" > 0.0", Float.parseFloat(leastScore)>0.0);
      assertQ(
          req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:a","fl","score"), 
          "//*[@numFound='6']","(//float[@name='score'])["+(random().nextInt(6)+1)+"]>='"+leastScore+"'");
  }
  
  private String getLeastScore(String query) throws Exception {
    final String resp = h.query(req("q",query, "sort","score asc", "fl","score"));
    return (String) BaseTestHarness.
        evaluateXPath(resp,"(//float[@name='score'])[1]/text()", 
            XPathConstants.STRING);
  }

  @Test
  public void testFq() {
    assertQ(
        req("q", "{!parent which=$pq v=$chq}", "fq", "parent_s:(e b)", "chq",
            "child_s:l", "pq", "parent_s:[* TO *]"// ,"debugQuery","on"
        ), beParents);
    
    boolean qfq = random().nextBoolean();
    assertQ(
        req(qfq ? "q" : "fq", "parent_s:(a e b)", (!qfq) ? "q" : "fq",
            "{!parent which=$pq v=$chq}", "chq", "parentchild_s:(bm ek cl)",
            "pq", "parent_s:[* TO *]"), beParents);
    
  }
  
  @Test
  public void testIntersectParentBqChildBq() throws IOException {
    
    assertQ(
        req("q", "+parent_s:(a e b) +_query_:\"{!parent which=$pq v=$chq}\"",
            "chq", "parentchild_s:(bm ek cl)", "pq", "parent_s:[* TO *]"),
        beParents);
  }
  
  @Test
  public void testGrandChildren() throws IOException {
    assertQ(
        req("q", "{!parent which=$parentfilter v=$children}", "children",
            "{!parent which=$childrenfilter v=$grandchildren}",
            "grandchildren", "grand_s:" + "x", "parentfilter",
            "parent_s:[* TO *]", "childrenfilter", "child_s:[* TO *]"),
        sixParents);
    // int loops = atLeast(1);
    String grandChildren = xyz.get(random().nextInt(xyz.size()));
    assertQ(
        req("q", "+parent_s:(a e b) +_query_:\"{!parent which=$pq v=$chq}\"",
            "chq", "{!parent which=$childfilter v=$grandchq}", "grandchq",
            "+grand_s:" + grandChildren + " +grand_parentchild_s:(b* e* c*)",
            "pq", "parent_s:[* TO *]", "childfilter", "child_s:[* TO *]"),
        beParents);
  }
  
  @Test
  public void testChildrenParser() {
    assertQ(
        req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:a", "fq",
            "NOT grand_s:[* TO *]"), "//*[@numFound='3']",
        "//doc/arr[@name=\"child_s\"]/str='k'",
        "//doc/arr[@name=\"child_s\"]/str='l'",
        "//doc/arr[@name=\"child_s\"]/str='m'");
    assertQ(
        req("q", "{!child of=\"parent_s:[* TO *]\"}parent_s:b", "fq",
            "-parentchild_s:bm", "fq", "-grand_s:*"), "//*[@numFound='2']",
        "//doc/arr[@name=\"child_s\"]/str='k'",
        "//doc/arr[@name=\"child_s\"]/str='l'");
  }

  @Test
  public void testCacheHit() throws IOException {

    MetricsMap parentFilterCache = (MetricsMap)((SolrMetricManager.GaugeWrapper<?>)h.getCore().getCoreMetricManager().getRegistry()
        .getMetrics().get("CACHE.searcher.perSegFilter")).getGauge();
    MetricsMap filterCache = (MetricsMap)((SolrMetricManager.GaugeWrapper<?>)h.getCore().getCoreMetricManager().getRegistry()
        .getMetrics().get("CACHE.searcher.filterCache")).getGauge();

    Map<String,Object> parentsBefore = parentFilterCache.getValue();

    Map<String,Object> filtersBefore = filterCache.getValue();

    // it should be weird enough to be uniq
    String parentFilter = "parent_s:([a TO c] [d TO f])";

    assertQ("search by parent filter",
        req("q", "{!parent which=\"" + parentFilter + "\"}"),
        "//*[@numFound='6']");

    assertQ("filter by parent filter",
        req("q", "*:*", "fq", "{!parent which=\"" + parentFilter + "\"}"),
        "//*[@numFound='6']");

    assertEquals("didn't hit fqCache yet ", 0L,
        delta("hits", filterCache.getValue(), filtersBefore));

    assertQ(
        "filter by join",
        req("q", "*:*", "fq", "{!parent which=\"" + parentFilter
            + "\"}child_s:l"), "//*[@numFound='6']");

    assertEquals("in cache mode every request lookups", 3,
        delta("lookups", parentFilterCache.getValue(), parentsBefore));
    assertEquals("last two lookups causes hits", 2,
        delta("hits", parentFilterCache.getValue(), parentsBefore));
    assertEquals("the first lookup gets insert", 1,
        delta("inserts", parentFilterCache.getValue(), parentsBefore));


    assertEquals("true join query was not in fqCache", 0L,
        delta("hits", filterCache.getValue(), filtersBefore));
    assertEquals("true join query is cached in fqCache", 1L,
        delta("inserts", filterCache.getValue(), filtersBefore));
  }
  
  private long delta(String key, Map<String,Object> a, Map<String,Object> b) {
    return (Long) a.get(key) - (Long) b.get(key);
  }

  
  @Test
  public void nullInit() {
    new BlockJoinParentQParserPlugin().init(null);
  }

  private final static String eParent[] = new String[]{"//*[@numFound='1']",
      "//doc/arr[@name=\"parent_s\"]/str='e'"};

  @Test
  public void testToParentFilters() {
    assertQ(
        req("fq", "{!parent filters=$child.fq which=$pq v=$chq}\"",
            "q", "parent_s:(e b)",
            "child.fq", "+childparent_s:e +child_s:l",
            "chq", "child_s:[* TO *]",
            "pq", "parent_s:[* TO *]"), eParent);

    assertQ(
        req("fq", "{!parent filters=$child.fq which=$pq v=$chq}\"",
            "q", "parent_s:(e b)",
            "child.fq", "childparent_s:e",
            "child.fq", "child_s:l",
            "chq", "child_s:[* TO *]",
            "pq", "parent_s:[* TO *]"), eParent);
  }

  @Test
  public void testToChildFilters() {
    assertQ(
        req("fq", "{!child of=$pq filters=$parent.fq  v=$pq}\"",
            "q", "child_s:(l m)",
            "parent.fq", "+parent_s:(d c)",
            "pq", "parent_s:[* TO *]"),
        "//*[@numFound='4']",
        "//doc/arr[@name=\"parentchild_s\"]/str='dl'",
        "//doc/arr[@name=\"parentchild_s\"]/str='dm'",
        "//doc/arr[@name=\"parentchild_s\"]/str='cl'",
        "//doc/arr[@name=\"parentchild_s\"]/str='cm'"
    );

    assertQ(
        req("fq", "{!child of=$pq filters=$parent.fq  v=$pq}\"",
            "q", "child_s:(l m)",
            "parent.fq", "+parent_s:(d c)",
            "parent.fq", "+parent_s:(c a)",
            "pq", "parent_s:[* TO *]"),
        "//*[@numFound='2']",
        "//doc/arr[@name=\"parentchild_s\"]/str='cl'",
        "//doc/arr[@name=\"parentchild_s\"]/str='cm'"
    );
  }

  private final static String elChild[] = new String[]{"//*[@numFound='1']",
      "//doc[" + 
          "arr[@name=\"child_s\"]/str='l' and arr[@name=\"childparent_s\"]/str='e']"};


  @Test
  public void testFilters() {
    assertQ(
        req("q", "{!filters param=$child.fq v=$gchq}",
            "child.fq", "childparent_s:e",
            "child.fq", "child_s:l",
            "gchq", "child_s:[* TO *]"), elChild);

    assertQ(
        req("q", "{!filters param=$child.fq excludeTags=firstTag v=$gchq}",
            "child.fq", "{!tag=zeroTag,firstTag}childparent_s:e",
            "child.fq", "{!tag=secondTag}child_s:l",
            "gchq", "child_s:[* TO *]"), "//*[@numFound='6']");

    assertQ(
        req("q", "{!filters param=$child.fq excludeTags=secondTag v=$gchq}",
            "child.fq", "{!tag=firstTag}childparent_s:e",
            "child.fq", "{!tag=secondTag}child_s:l",
            "gchq", "child_s:[* TO *]"), "//*[@numFound='3']");

    assertQ(req("q",
             random().nextBoolean() ? "{!filters param=$child.fq excludeTags=firstTag,secondTag v=$gchq}" :
               random().nextBoolean() ? "{!filters param=$thereAreNoLikeThese v=$gchq}" :
                 "{!filters v=$gchq}" ,
            "child.fq", "{!tag=firstTag}childparent_s:e",
            "child.fq", "{!tag=secondTag}child_s:l",
            "gchq", "child_s:[* TO *]"), "//*[@numFound='18']");
    
    assertQEx("expecting exception on weird param",
        req("q", "{!filters v=$gchq param=}\"" ,
            "gchq", "child_s:[* TO *]"
       ),ErrorCode.BAD_REQUEST);
    
    assertQ( // omit main query
        req("q", "{!filters param=$child.fq}",
            "child.fq", "{!tag=firstTag}childparent_s:(e f)",
            "child.fq", "{!tag=secondTag}child_s:l"), "//*[@numFound='2']");
    
    assertQ( // all excluded, matching all 
        req("q", "{!filters param=$child.fq excludeTags=firstTag,secondTag}",
            "child.fq", "{!tag=firstTag}childparent_s:(e f)",
            "child.fq", "{!tag=secondTag}child_s:l"), "//*[@numFound='42']");
    
    assertQ(req("q", // excluding top level
            "{!filters param=$child.fq excludeTags=bot,top v=$gchq}" ,
       "child.fq", "{!tag=secondTag}child_s:l", // 6 ls remains
       "gchq", "{!tag=top}childparent_s:e"), "//*[@numFound='6']");
    
    assertQ(req("q", // top and filter are excluded, got all results
        "{!filters excludeTags=bot,secondTag,top v=$gchq}" ,
         "child.fq", "{!tag=secondTag}child_s:l",
         "gchq", "{!tag=top}childparent_s:e"), "//*[@numFound='42']");
  }
  
  @Test
  public void testFiltersCache() throws SyntaxError, IOException {
    final String [] elFilterQuery = new String[] {"q", "{!filters param=$child.fq v=$gchq}",
        "child.fq", "childparent_s:e",
        "child.fq", "child_s:l",
        "gchq", "child_s:[* TO *]"};
    assertQ("precondition: single doc match", 
         req(elFilterQuery), elChild);
    final Query query;
    try(final SolrQueryRequest req = req(elFilterQuery)) {
      QParser parser = QParser.getParser(req.getParams().get("q"), null, req);
      query = parser.getQuery();
      final TopDocs topDocs = req.getSearcher().search(query, 10);
      assertEquals(1, topDocs.totalHits.value);
    }
    assertU(adoc("id", "12275", 
        "child_s", "l", "childparent_s", "e"));
    assertU(commit());

    assertQ("here we rely on autowarming for cathing cache leak",  //cache=false
          req(elFilterQuery), "//*[@numFound='2']");

    try(final SolrQueryRequest req = req()) {
        final int count = req.getSearcher().count(query);
        assertEquals("expecting new doc is visible to old query", 2, count);
    }
  }

  @After
  public void cleanAfterTestFiltersCache(){
    assertU("should be noop", delI("12275"));
    assertU("most of the time", commit());
  }
}