| using Lucene.Net.Analysis.Core; |
| using Lucene.Net.Analysis.Miscellaneous; |
| using Lucene.Net.Util; |
| using NUnit.Framework; |
| using System; |
| using System.Text; |
| |
| namespace Lucene.Net.Analysis.Cn.Smart |
| { |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| public class TestSmartChineseAnalyzer : BaseTokenStreamTestCase |
| { |
| [Test] |
| public void TestChineseStopWordsDefault() |
| { |
| #pragma warning disable 612, 618 |
| Analyzer ca = new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT); /* will load stopwords */ |
| #pragma warning restore 612, 618 |
| String sentence = "我购买了道具和服装。"; |
| String[] result = { "我", "购买", "了", "道具", "和", "服装" }; |
| AssertAnalyzesTo(ca, sentence, result); |
| // set stop-words from the outer world - must yield same behavior |
| ca = new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT, |
| #pragma warning restore 612, 618 |
| SmartChineseAnalyzer.GetDefaultStopSet()); |
| AssertAnalyzesTo(ca, sentence, result); |
| } |
| |
| /* |
| * This test is the same as the above, except with two phrases. |
| * This tests to ensure the SentenceTokenizer->WordTokenFilter chain works correctly. |
| */ |
| [Test] |
| public void TestChineseStopWordsDefaultTwoPhrases() |
| { |
| #pragma warning disable 612, 618 |
| Analyzer ca = new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT); /* will load stopwords */ |
| #pragma warning restore 612, 618 |
| String sentence = "我购买了道具和服装。 我购买了道具和服装。"; |
| String[] result = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" }; |
| AssertAnalyzesTo(ca, sentence, result); |
| } |
| |
| /* |
| * This test is the same as the above, except using an ideographic space as a separator. |
| * This tests to ensure the stopwords are working correctly. |
| */ |
| [Test] |
| public void TestChineseStopWordsDefaultTwoPhrasesIdeoSpace() |
| { |
| #pragma warning disable 612, 618 |
| Analyzer ca = new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT); /* will load stopwords */ |
| #pragma warning restore 612, 618 |
| String sentence = "我购买了道具和服装 我购买了道具和服装。"; |
| String[] result = { "我", "购买", "了", "道具", "和", "服装", "我", "购买", "了", "道具", "和", "服装" }; |
| AssertAnalyzesTo(ca, sentence, result); |
| } |
| |
| /* |
| * Punctuation is handled in a strange way if you disable stopwords |
| * In this example the IDEOGRAPHIC FULL STOP is converted into a comma. |
| * if you don't supply (true) to the constructor, or use a different stopwords list, |
| * then punctuation is indexed. |
| */ |
| [Test] |
| public void TestChineseStopWordsOff() |
| { |
| Analyzer[] |
| analyzers = new Analyzer[] { |
| #pragma warning disable 612, 618 |
| new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT, false),/* doesn't load stopwords */ |
| new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT, null) /* sets stopwords to empty set */}; |
| #pragma warning restore 612, 618 |
| String sentence = "我购买了道具和服装。"; |
| String[] result = { "我", "购买", "了", "道具", "和", "服装", "," }; |
| foreach (Analyzer analyzer in analyzers) |
| { |
| AssertAnalyzesTo(analyzer, sentence, result); |
| AssertAnalyzesTo(analyzer, sentence, result); |
| } |
| } |
| |
| /* |
| * Check that position increments after stopwords are correct, |
| * when stopfilter is configured with enablePositionIncrements |
| */ |
| [Test] |
| public void TestChineseStopWords2() |
| { |
| #pragma warning disable 612, 618 |
| Analyzer ca = new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT); /* will load stopwords */ |
| #pragma warning restore 612, 618 |
| String sentence = "Title:San"; // : is a stopword |
| String[] result = { "titl", "san" }; |
| int[] startOffsets = { 0, 6 }; |
| int[] endOffsets = { 5, 9 }; |
| int[] posIncr = { 1, 2 }; |
| AssertAnalyzesTo(ca, sentence, result, startOffsets, endOffsets, posIncr); |
| } |
| |
| [Test] |
| public void TestChineseAnalyzer() |
| { |
| #pragma warning disable 612, 618 |
| Analyzer ca = new SmartChineseAnalyzer(LuceneVersion.LUCENE_CURRENT, true); |
| #pragma warning restore 612, 618 |
| String sentence = "我购买了道具和服装。"; |
| String[] result = { "我", "购买", "了", "道具", "和", "服装" }; |
| AssertAnalyzesTo(ca, sentence, result); |
| } |
| |
| /* |
| * English words are lowercased and porter-stemmed. |
| */ |
| [Test] |
| public void TestMixedLatinChinese() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "我购买 Tests 了道具和服装", |
| new String[] { "我", "购买", "test", "了", "道具", "和", "服装" }); |
| } |
| |
| /* |
| * Numerics are parsed as their own tokens |
| */ |
| [Test] |
| public void TestNumerics() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "我购买 Tests 了道具和服装1234", |
| new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234" }); |
| } |
| |
| /* |
| * Full width alphas and numerics are folded to half-width |
| */ |
| [Test] |
| public void TestFullWidth() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "我购买 Tests 了道具和服装1234", |
| new String[] { "我", "购买", "test", "了", "道具", "和", "服装", "1234" }); |
| } |
| |
| /* |
| * Presentation form delimiters are removed |
| */ |
| [Test] |
| public void TestDelimiters() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "我购买︱ Tests 了道具和服装", |
| new String[] { "我", "购买", "test", "了", "道具", "和", "服装" }); |
| } |
| |
| /* |
| * Text from writing systems other than Chinese and Latin are parsed as individual characters. |
| * (regardless of Unicode category) |
| */ |
| [Test] |
| public void TestNonChinese() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "我购买 روبرتTests 了道具和服装", |
| new String[] { "我", "购买", "ر", "و", "ب", "ر", "ت", "test", "了", "道具", "和", "服装" }); |
| } |
| |
| /* |
| * Test what the analyzer does with out-of-vocabulary words. |
| * In this case the name is Yousaf Raza Gillani. |
| * Currently it is being analyzed into single characters... |
| */ |
| [Test] |
| public void TestOOV() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "优素福·拉扎·吉拉尼", |
| new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" }); |
| |
| |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "优素福拉扎吉拉尼", |
| new String[] { "优", "素", "福", "拉", "扎", "吉", "拉", "尼" }); |
| } |
| |
| [Test] |
| public void TestOffsets() |
| { |
| AssertAnalyzesTo(new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT |
| #pragma warning restore 612, 618 |
| , true), "我购买了道具和服装", |
| new String[] { "我", "购买", "了", "道具", "和", "服装" }, |
| new int[] { 0, 1, 3, 4, 6, 7 }, |
| new int[] { 1, 3, 4, 6, 7, 9 }); |
| } |
| |
| [Test] |
| public void TestReusableTokenStream() |
| { |
| Analyzer a = new SmartChineseAnalyzer( |
| #pragma warning disable 612, 618 |
| LuceneVersion.LUCENE_CURRENT); |
| #pragma warning restore 612, 618 |
| AssertAnalyzesTo(a, "我购买 Tests 了道具和服装", |
| new String[] { "我", "购买", "test", "了", "道具", "和", "服装" }, |
| new int[] { 0, 1, 4, 10, 11, 13, 14 }, |
| new int[] { 1, 3, 9, 11, 13, 14, 16 }); |
| AssertAnalyzesTo(a, "我购买了道具和服装。", |
| new String[] { "我", "购买", "了", "道具", "和", "服装" }, |
| new int[] { 0, 1, 3, 4, 6, 7 }, |
| new int[] { 1, 3, 4, 6, 7, 9 }); |
| } |
| |
| // LUCENE-3026 |
| [Test] |
| public void TestLargeDocument() |
| { |
| StringBuilder sb = new StringBuilder(); |
| for (int i = 0; i < 5000; i++) |
| { |
| sb.append("我购买了道具和服装。"); |
| } |
| Analyzer analyzer = new SmartChineseAnalyzer(TEST_VERSION_CURRENT); |
| TokenStream stream = analyzer.GetTokenStream("", sb.toString()); |
| try |
| { |
| stream.Reset(); |
| while (stream.IncrementToken()) |
| { |
| } |
| stream.End(); |
| } |
| finally |
| { |
| IOUtils.DisposeWhileHandlingException(stream); |
| } |
| } |
| |
| // LUCENE-3026 |
| [Test] |
| public void TestLargeSentence() |
| { |
| StringBuilder sb = new StringBuilder(); |
| for (int i = 0; i < 5000; i++) |
| { |
| sb.append("我购买了道具和服装"); |
| } |
| Analyzer analyzer = new SmartChineseAnalyzer(TEST_VERSION_CURRENT); |
| TokenStream stream = analyzer.GetTokenStream("", sb.toString()); |
| try |
| { |
| stream.Reset(); |
| while (stream.IncrementToken()) |
| { |
| } |
| stream.End(); |
| } |
| finally |
| { |
| IOUtils.DisposeWhileHandlingException(stream); |
| } |
| } |
| |
| // LUCENE-3642 |
| [Test] |
| public void TestInvalidOffset() |
| { |
| Analyzer analyzer = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => |
| { |
| Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); |
| TokenFilter filters = new ASCIIFoldingFilter(tokenizer); |
| #pragma warning disable 612, 618 |
| filters = new WordTokenFilter(filters); |
| #pragma warning restore 612, 618 |
| return new TokenStreamComponents(tokenizer, filters); |
| }); |
| |
| AssertAnalyzesTo(analyzer, "mosfellsbær", |
| new string[] { "mosfellsbaer" }, |
| new int[] { 0 }, |
| new int[] { 11 }); |
| } |
| |
| /** blast some random strings through the analyzer */ |
| [Test] |
| public void TestRandomStrings() |
| { |
| CheckRandomData(Random, new SmartChineseAnalyzer(TEST_VERSION_CURRENT), 1000 * RandomMultiplier); |
| } |
| |
| /** blast some random large strings through the analyzer */ |
| [Test] |
| [Slow] |
| public void TestRandomHugeStrings() |
| { |
| Random random = Random; |
| CheckRandomData(random, new SmartChineseAnalyzer(TEST_VERSION_CURRENT), 100 * RandomMultiplier, 8192); |
| } |
| |
| [Test] |
| public void TestEmptyTerm() |
| { |
| Random random = Random; |
| Analyzer a = Analyzer.NewAnonymous(createComponents: (fieldName, reader) => |
| { |
| Tokenizer tokenizer = new KeywordTokenizer(reader); |
| #pragma warning disable 612, 618 |
| return new TokenStreamComponents(tokenizer, new WordTokenFilter(tokenizer)); |
| #pragma warning restore 612, 618 |
| }); |
| CheckAnalysisConsistency(random, a, random.nextBoolean(), ""); |
| } |
| } |
| } |