lucene/monitor/src/test/org/apache/lucene/monitor/TestSuffixingNGramTokenizer.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.lucene.monitor;

import java.io.IOException;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;

public class TestSuffixingNGramTokenizer extends BaseTokenStreamTestCase {

  private Analyzer analyzer =
      new Analyzer() {
        @Override
        protected TokenStreamComponents createComponents(String fieldName) {
          Tokenizer source = new WhitespaceTokenizer();
          TokenStream sink = new SuffixingNGramTokenFilter(source, "XX", "ANY", 10);
          return new TokenStreamComponents(source, sink);
        }
      };

  public void testTokensAreSuffixed() throws IOException {
    assertAnalyzesTo(
        analyzer,
        "term",
        new String[] {
          "term", "termXX", "terXX", "teXX", "tXX", "ermXX", "erXX", "eXX", "rmXX", "rXX", "mXX",
          "XX"
        });
  }

  public void testRepeatedSuffixesAreNotEmitted() throws IOException {
    assertAnalyzesTo(
        analyzer,
        "arm harm term",
        new String[] {
          "arm", "armXX", "arXX", "aXX", "rmXX", "rXX", "mXX", "XX", "harm", "harmXX", "harXX",
          "haXX", "hXX", "term", "termXX", "terXX", "teXX", "tXX", "ermXX", "erXX", "eXX"
        });
  }

  public void testRepeatedInfixesAreNotEmitted() throws IOException {
    assertAnalyzesTo(
        analyzer,
        "alarm alas harm",
        new String[] {
          "alarm", "alarmXX", "alarXX", "alaXX", "alXX", "aXX", "larmXX", "larXX", "laXX", "lXX",
          "armXX", "arXX", "rmXX", "rXX", "mXX", "XX", "alas", "alasXX", "lasXX", "asXX", "sXX",
          "harm", "harmXX", "harXX", "haXX", "hXX"
        });
  }

  public void testLengthyTokensAreNotNgrammed() throws IOException {
    assertAnalyzesTo(
        analyzer,
        "alongtermthatshouldntbengrammed",
        new String[] {"alongtermthatshouldntbengrammed", "ANY"});
  }
}