lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.analysis.ja;

import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;

/** Simple tests for {@link JapaneseTokenizerFactory} */
public class TestJapaneseTokenizerFactory extends BaseTokenStreamTestCase {
  public void testSimple() throws IOException {
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap<String, String>());
    factory.inform(new StringMockResourceLoader(""));
    TokenStream ts = factory.create(newAttributeFactory());
    ((Tokenizer) ts).setReader(new StringReader("これは本ではない"));
    assertTokenStreamContents(
        ts,
        new String[] {"これ", "は", "本", "で", "は", "ない"},
        new int[] {0, 2, 3, 4, 5, 6},
        new int[] {2, 3, 4, 5, 6, 8});
  }

  /** Test that search mode is enabled and working by default */
  public void testDefaults() throws IOException {
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap<String, String>());
    factory.inform(new StringMockResourceLoader(""));
    TokenStream ts = factory.create(newAttributeFactory());
    ((Tokenizer) ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
    assertTokenStreamContents(ts, new String[] {"シニア", "ソフトウェア", "エンジニア"});
  }

  /** Test mode parameter: specifying normal mode */
  public void testMode() throws IOException {
    Map<String, String> args = new HashMap<>();
    args.put("mode", "normal");
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
    factory.inform(new StringMockResourceLoader(""));
    TokenStream ts = factory.create(newAttributeFactory());
    ((Tokenizer) ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
    assertTokenStreamContents(ts, new String[] {"シニアソフトウェアエンジニア"});
  }

  /** Test user dictionary */
  public void testUserDict() throws IOException {
    String userDict =
        "# Custom segmentation for long entries\n"
            + "日本経済新聞,日本 経済 新聞,ニホン ケイザイ シンブン,カスタム名詞\n"
            + "関西国際空港,関西 国際 空港,カンサイ コクサイ クウコウ,テスト名詞\n"
            + "# Custom reading for sumo wrestler\n"
            + "朝青龍,朝青龍,アサショウリュウ,カスタム人名\n";
    Map<String, String> args = new HashMap<>();
    args.put("userDictionary", "userdict.txt");
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
    factory.inform(new StringMockResourceLoader(userDict));
    TokenStream ts = factory.create(newAttributeFactory());
    ((Tokenizer) ts).setReader(new StringReader("関西国際空港に行った"));
    assertTokenStreamContents(ts, new String[] {"関西", "国際", "空港", "に", "行っ", "た"});
  }

  /** Test preserving punctuation */
  public void testPreservePunctuation() throws IOException {
    Map<String, String> args = new HashMap<>();
    args.put("discardPunctuation", "false");
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
    factory.inform(new StringMockResourceLoader(""));
    TokenStream ts = factory.create(newAttributeFactory());
    ((Tokenizer) ts).setReader(new StringReader("今ノルウェーにいますが、来週の頭日本に戻ります。楽しみにしています！お寿司が食べたいな。。。"));
    assertTokenStreamContents(
        ts,
        new String[] {
          "今", "ノルウェー", "に", "い", "ます", "が", "、", "来週", "の", "頭", "日本", "に", "戻り", "ます", "。", "楽しみ",
          "に", "し", "て", "い", "ます", "！", "お", "寿司", "が", "食べ", "たい", "な", "。", "。", "。"
        });
  }

  /** Test discarding compound (original) token */
  public void testPreserveCompoundToken() throws IOException {
    Map<String, String> args = new HashMap<>();
    args.put("discardCompoundToken", "false");
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
    factory.inform(new StringMockResourceLoader(""));
    TokenStream ts = factory.create(newAttributeFactory());
    ((Tokenizer) ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
    assertTokenStreamContents(ts, new String[] {"シニア", "シニアソフトウェアエンジニア", "ソフトウェア", "エンジニア"});
  }

  /** Test that bogus arguments result in exception */
  public void testBogusArguments() throws Exception {
    IllegalArgumentException expected =
        expectThrows(
            IllegalArgumentException.class,
            () -> {
              new JapaneseTokenizerFactory(
                  new HashMap<String, String>() {
                    {
                      put("bogusArg", "bogusValue");
                    }
                  });
            });
    assertTrue(expected.getMessage().contains("Unknown parameters"));
  }

  private TokenStream makeTokenStream(HashMap<String, String> args, String in) throws IOException {
    JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
    factory.inform(new StringMockResourceLoader(""));
    Tokenizer t = factory.create(newAttributeFactory());
    t.setReader(new StringReader(in));
    return t;
  }

  /** Test nbestCost parameter */
  public void testNbestCost() throws IOException {
    assertTokenStreamContents(
        makeTokenStream(
            new HashMap<String, String>() {
              {
                put("nBestCost", "2000");
              }
            },
            "鳩山積み"),
        new String[] {"鳩", "鳩山", "山積み", "積み"});
  }

  /** Test nbestExamples parameter */
  public void testNbestExample() throws IOException {
    assertTokenStreamContents(
        makeTokenStream(
            new HashMap<String, String>() {
              {
                put("nBestExamples", "/鳩山積み-鳩山/鳩山積み-鳩/");
              }
            },
            "鳩山積み"),
        new String[] {"鳩", "鳩山", "山積み", "積み"});
  }
}