lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanAnalyzer.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.analysis.ko;

import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;

/**
 * Test Korean morphological analyzer
 */
public class TestKoreanAnalyzer extends BaseTokenStreamTestCase {
  public void testSentence() throws IOException {
    Analyzer a = new KoreanAnalyzer();
    assertAnalyzesTo(a, "한국은 대단한 나라입니다.",
        new String[]{"한국", "대단", "나라", "이"},
        new int[]{ 0, 4, 8, 10 },
        new int[]{ 2, 6, 10, 13 },
        new int[]{ 1, 2, 3, 1 }
    );
    a.close();
  }

  public void testStopTags() throws IOException {
    Set<POS.Tag> stopTags = new HashSet<>(Arrays.asList(POS.Tag.NNP, POS.Tag.NNG));
    Analyzer a = new KoreanAnalyzer(null, KoreanTokenizer.DecompoundMode.DISCARD, stopTags, false);
    assertAnalyzesTo(a, "한국은 대단한 나라입니다.",
        new String[]{"은", "대단", "하", "ᆫ", "이", "ᄇ니다"},
        new int[]{ 2, 4, 6, 6, 10, 10 },
        new int[]{ 3, 6, 7, 7, 13, 13 },
        new int[]{ 2, 1, 1, 1, 2, 1 }
    );
    a.close();
  }

  public void testUnknownWord() throws IOException {
    Analyzer a = new KoreanAnalyzer(null, KoreanTokenizer.DecompoundMode.DISCARD,
        KoreanPartOfSpeechStopFilter.DEFAULT_STOP_TAGS, true);

    assertAnalyzesTo(a,"2018 평창 동계올림픽대회",
        new String[]{"2", "0", "1", "8", "평창", "동계", "올림픽", "대회"},
        new int[]{0, 1, 2, 3, 5, 8, 10, 13},
        new int[]{1, 2, 3, 4, 7, 10, 13, 15},
        new int[]{1, 1, 1, 1, 1, 1, 1, 1});
    a.close();

    a = new KoreanAnalyzer(null, KoreanTokenizer.DecompoundMode.DISCARD,
        KoreanPartOfSpeechStopFilter.DEFAULT_STOP_TAGS, false);

    assertAnalyzesTo(a,"2018 평창 동계올림픽대회",
        new String[]{"2018", "평창", "동계", "올림픽", "대회"},
        new int[]{0, 5, 8, 10, 13},
        new int[]{4, 7, 10, 13, 15},
        new int[]{1, 1, 1, 1, 1});
    a.close();
  }

  /**
   * blast random strings against the analyzer
   */
  public void testRandom() throws IOException {
    Random random = random();
    final Analyzer a = new KoreanAnalyzer();
    checkRandomData(random, a, atLeast(200));
    a.close();
  }

  /**
   * blast some random large strings through the analyzer
   */
  public void testRandomHugeStrings() throws Exception {
    Random random = random();
    final Analyzer a = new KoreanAnalyzer();
    checkRandomData(random, a, RANDOM_MULTIPLIER, 4096);
    a.close();
  }
  
  @Nightly
  public void testRandomHugeStringsAtNight() throws Exception {
    Random random = random();
    final Analyzer a = new KoreanAnalyzer();
    checkRandomData(random, a, 3 * RANDOM_MULTIPLIER, 8192);
    a.close();
  }

  // Copied from TestKoreanTokenizer, to make sure passing
  // user dict to analyzer works:
  public void testUserDict() throws IOException {
    final Analyzer analyzer = new KoreanAnalyzer(TestKoreanTokenizer.readDict(),
        KoreanTokenizer.DEFAULT_DECOMPOUND, KoreanPartOfSpeechStopFilter.DEFAULT_STOP_TAGS, false);
    assertAnalyzesTo(analyzer, "c++ 프로그래밍 언어",
        new String[]{"c++", "프로그래밍", "언어"},
        new int[]{0, 4, 10},
        new int[]{3, 9, 12},
        new int[]{1, 1, 1}
    );
  }
}