/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.analysis.opennlp;

import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.custom.CustomAnalyzer;
import org.apache.lucene.analysis.miscellaneous.KeywordRepeatFilterFactory;
import org.apache.lucene.analysis.miscellaneous.RemoveDuplicatesTokenFilterFactory;
import org.apache.lucene.util.ClasspathResourceLoader;

public class TestOpenNLPLemmatizerFilterFactory extends BaseTokenStreamTestCase {

  private static final String SENTENCE = "They sent him running in the evening.";
  private static final String[] SENTENCE_dict_punc = {
    "they", "send", "he", "run", "in", "the", "evening", "."
  };
  private static final String[] SENTENCE_maxent_punc = {
    "they", "send", "he", "runn", "in", "the", "evening", "."
  };
  private static final String[] SENTENCE_posTags = {
    "NNP", "VBD", "PRP", "VBG", "IN", "DT", "NN", "."
  };

  private static final String SENTENCES =
      "They sent him running in the evening. He did not come back.";
  private static final String[] SENTENCES_dict_punc = {
    "they", "send", "he", "run", "in", "the", "evening", ".", "he", "do", "not", "come", "back", "."
  };
  private static final String[] SENTENCES_maxent_punc = {
    "they", "send", "he", "runn", "in", "the", "evening", ".", "he", "do", "not", "come", "back",
    "."
  };
  private static final String[] SENTENCES_posTags = {
    "NNP", "VBD", "PRP", "VBG", "IN", "DT", "NN", ".", "PRP", "VBD", "RB", "VB", "RB", "."
  };

  private static final String SENTENCE_both = "Konstantin Kalashnitsov constantly caliphed.";
  private static final String[] SENTENCE_both_punc = {
    "konstantin", "kalashnitsov", "constantly", "caliph", "."
  };
  private static final String[] SENTENCE_both_posTags = {"IN", "JJ", "NN", "VBN", "."};

  private static final String SENTENCES_both =
      "Konstantin Kalashnitsov constantly caliphed. Coreena could care, completely.";
  private static final String[] SENTENCES_both_punc = {
    "konstantin",
    "kalashnitsov",
    "constantly",
    "caliph",
    ".",
    "coreena",
    "could",
    "care",
    ",",
    "completely",
    "."
  };
  private static final String[] SENTENCES_both_posTags = {
    "IN", "JJ", "NN", "VBN", ".", "NNP", "VBN", "NN", ",", "NN", "."
  };

  private static final String[] SENTENCES_dict_keep_orig_punc = {
    "They", "they", "sent", "send", "him", "he", "running", "run", "in", "the", "evening", ".",
    "He", "he", "did", "do", "not", "come", "back", "."
  };
  private static final String[] SENTENCES_max_ent_keep_orig_punc = {
    "They", "they", "sent", "send", "him", "he", "running", "runn", "in", "the", "evening", ".",
    "He", "he", "did", "do", "not", "come", "back", "."
  };
  private static final String[] SENTENCES_keep_orig_posTags = {
    "NNP", "NNP", "VBD", "VBD", "PRP", "PRP", "VBG", "VBG", "IN", "DT", "NN", ".", "PRP", "PRP",
    "VBD", "VBD", "RB", "VB", "RB", "."
  };

  private static final String[] SENTENCES_both_keep_orig_punc = {
    "Konstantin",
    "konstantin",
    "Kalashnitsov",
    "kalashnitsov",
    "constantly",
    "caliphed",
    "caliph",
    ".",
    "Coreena",
    "coreena",
    "could",
    "care",
    ",",
    "completely",
    "."
  };
  private static final String[] SENTENCES_both_keep_orig_posTags = {
    "IN", "IN", "JJ", "JJ", "NN", "VBN", "VBN", ".", "NNP", "NNP", "VBN", "NN", ",", "NN", "."
  };

  private static final String tokenizerModelFile = "en-test-tokenizer.bin";
  private static final String sentenceModelFile = "en-test-sent.bin";
  private static final String posTaggerModelFile = "en-test-pos-maxent.bin";
  private static final String lemmatizerModelFile = "en-test-lemmatizer.bin";
  private static final String lemmatizerDictFile = "en-test-lemmas.dict";

  public void test1SentenceDictionaryOnly() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", "en-test-pos-maxent.bin")
            .addTokenFilter("opennlplemmatizer", "dictionary", "en-test-lemmas.dict")
            .build();
    assertAnalyzesTo(
        analyzer, SENTENCE, SENTENCE_dict_punc, null, null, SENTENCE_posTags, null, null, true);
  }

  public void test2SentencesDictionaryOnly() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter("opennlplemmatizer", "dictionary", lemmatizerDictFile)
            .build();
    assertAnalyzesTo(
        analyzer, SENTENCES, SENTENCES_dict_punc, null, null, SENTENCES_posTags, null, null, true);
  }

  public void test1SentenceMaxEntOnly() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter("opennlplemmatizer", "lemmatizerModel", lemmatizerModelFile)
            .build();
    assertAnalyzesTo(
        analyzer, SENTENCE, SENTENCE_maxent_punc, null, null, SENTENCE_posTags, null, null, true);
  }

  public void test2SentencesMaxEntOnly() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter("OpenNLPLemmatizer", "lemmatizerModel", lemmatizerModelFile)
            .build();
    assertAnalyzesTo(
        analyzer,
        SENTENCES,
        SENTENCES_maxent_punc,
        null,
        null,
        SENTENCES_posTags,
        null,
        null,
        true);
  }

  public void test1SentenceDictionaryAndMaxEnt() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", "en-test-pos-maxent.bin")
            .addTokenFilter(
                "opennlplemmatizer",
                "dictionary",
                "en-test-lemmas.dict",
                "lemmatizerModel",
                lemmatizerModelFile)
            .build();
    assertAnalyzesTo(
        analyzer,
        SENTENCE_both,
        SENTENCE_both_punc,
        null,
        null,
        SENTENCE_both_posTags,
        null,
        null,
        true);
  }

  public void test2SentencesDictionaryAndMaxEnt() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter(
                "opennlplemmatizer",
                "dictionary",
                lemmatizerDictFile,
                "lemmatizerModel",
                lemmatizerModelFile)
            .build();
    assertAnalyzesTo(
        analyzer,
        SENTENCES_both,
        SENTENCES_both_punc,
        null,
        null,
        SENTENCES_both_posTags,
        null,
        null,
        true);
  }

  public void testKeywordAttributeAwarenessDictionaryOnly() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter(KeywordRepeatFilterFactory.class)
            .addTokenFilter("opennlplemmatizer", "dictionary", lemmatizerDictFile)
            .addTokenFilter(RemoveDuplicatesTokenFilterFactory.class)
            .build();
    assertAnalyzesTo(
        analyzer,
        SENTENCES,
        SENTENCES_dict_keep_orig_punc,
        null,
        null,
        SENTENCES_keep_orig_posTags,
        null,
        null,
        true);
  }

  public void testKeywordAttributeAwarenessMaxEntOnly() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter(KeywordRepeatFilterFactory.class)
            .addTokenFilter("opennlplemmatizer", "lemmatizerModel", lemmatizerModelFile)
            .addTokenFilter(RemoveDuplicatesTokenFilterFactory.class)
            .build();
    assertAnalyzesTo(
        analyzer,
        SENTENCES,
        SENTENCES_max_ent_keep_orig_punc,
        null,
        null,
        SENTENCES_keep_orig_posTags,
        null,
        null,
        true);
  }

  public void testKeywordAttributeAwarenessDictionaryAndMaxEnt() throws Exception {
    CustomAnalyzer analyzer =
        CustomAnalyzer.builder(new ClasspathResourceLoader(getClass()))
            .withTokenizer(
                "opennlp", "tokenizerModel", tokenizerModelFile, "sentenceModel", sentenceModelFile)
            .addTokenFilter("opennlpPOS", "posTaggerModel", posTaggerModelFile)
            .addTokenFilter(KeywordRepeatFilterFactory.class)
            .addTokenFilter(
                "opennlplemmatizer",
                "dictionary",
                lemmatizerDictFile,
                "lemmatizerModel",
                lemmatizerModelFile)
            .addTokenFilter(RemoveDuplicatesTokenFilterFactory.class)
            .build();
    assertAnalyzesTo(
        analyzer,
        SENTENCES_both,
        SENTENCES_both_keep_orig_punc,
        null,
        null,
        SENTENCES_both_keep_orig_posTags,
        null,
        null,
        true);
  }
}