| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.lucene.analysis.synonym; |
| |
| import java.io.IOException; |
| import java.io.StringReader; |
| import java.text.ParseException; |
| import java.util.ArrayList; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Locale; |
| import java.util.Set; |
| |
| import org.apache.lucene.analysis.Analyzer; |
| import org.apache.lucene.analysis.BaseTokenStreamTestCase; |
| import org.apache.lucene.analysis.MockAnalyzer; |
| import org.apache.lucene.analysis.MockGraphTokenFilter; |
| import org.apache.lucene.analysis.MockTokenizer; |
| import org.apache.lucene.analysis.TokenStream; |
| import org.apache.lucene.analysis.Tokenizer; |
| import org.apache.lucene.analysis.core.FlattenGraphFilter; |
| import org.apache.lucene.analysis.tokenattributes.*; |
| import org.apache.lucene.document.Document; |
| import org.apache.lucene.document.Field; |
| import org.apache.lucene.index.IndexReader; |
| import org.apache.lucene.index.RandomIndexWriter; |
| import org.apache.lucene.search.IndexSearcher; |
| import org.apache.lucene.search.PhraseQuery; |
| import org.apache.lucene.store.ByteArrayDataInput; |
| import org.apache.lucene.store.Directory; |
| import org.apache.lucene.util.BytesRef; |
| import org.apache.lucene.util.CharsRef; |
| import org.apache.lucene.util.CharsRefBuilder; |
| import org.apache.lucene.util.IOUtils; |
| import org.apache.lucene.util.IntsRef; |
| import org.apache.lucene.util.IntsRefBuilder; |
| import org.apache.lucene.util.TestUtil; |
| import org.apache.lucene.util.automaton.Automaton; |
| import org.apache.lucene.util.automaton.AutomatonTestUtil; |
| import org.apache.lucene.util.automaton.Operations; |
| import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; |
| import org.apache.lucene.util.automaton.Transition; |
| import org.apache.lucene.util.fst.Util; |
| |
| public class TestSynonymGraphFilter extends BaseTokenStreamTestCase { |
| |
| /** Set as a side effect by {@link #getAnalyzer} and {@link #getFlattenAnalyzer}. */ |
| private SynonymGraphFilter synFilter; |
| private FlattenGraphFilter flattenFilter; |
| |
| public void testBasicKeepOrigOneOutput() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b", |
| new String[] {"c", "x", "a", "b"}, |
| new int[] { 0, 2, 2, 4}, |
| new int[] { 1, 5, 3, 5}, |
| new String[] {"word", "SYNONYM", "word", "word"}, |
| new int[] { 1, 1, 0, 1}, |
| new int[] { 1, 2, 1, 1}); |
| a.close(); |
| } |
| |
| public void testMixedKeepOrig() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", true); |
| add(b, "e f", "y", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b c e f g", |
| new String[] {"c", "x", "a", "b", "c", "y", "g"}, |
| new int[] { 0, 2, 2, 4, 6, 8, 12}, |
| new int[] { 1, 5, 3, 5, 7, 11, 13}, |
| new String[] {"word", "SYNONYM", "word", "word", "word", "SYNONYM", "word"}, |
| new int[] { 1, 1, 0, 1, 1, 1, 1}, |
| new int[] { 1, 2, 1, 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testNoParseAfterBuffer() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "b a", "x", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "b b b", |
| new String[] {"b", "b", "b"}, |
| new int[] { 0, 2, 4}, |
| new int[] { 1, 3, 5}, |
| new String[] {"word", "word", "word"}, |
| new int[] { 1, 1, 1}, |
| new int[] { 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testOneInputMultipleOutputKeepOrig() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", true); |
| add(b, "a b", "y", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b c", |
| new String[] {"c", "x", "y", "a", "b", "c"}, |
| new int[] { 0, 2, 2, 2, 4, 6}, |
| new int[] { 1, 5, 5, 3, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "word", "word", "word"}, |
| new int[] { 1, 1, 0, 0, 1, 1, 1, 1}, |
| new int[] { 1, 2, 2, 1, 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| /** |
| * Verify type of token and positionLength after analyzer. |
| */ |
| public void testPositionLengthAndTypeSimple() throws Exception { |
| String testFile = |
| "spider man, spiderman"; |
| |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| assertAnalyzesToPositions(analyzer, "spider man", |
| new String[]{"spiderman", "spider", "man"}, |
| new String[]{"SYNONYM", "word", "word"}, |
| new int[]{1, 0, 1}, |
| new int[]{2, 1, 1}); |
| } |
| |
| /** |
| * parse a syn file with some escaped syntax chars |
| */ |
| public void testEscapedStuff() throws Exception { |
| String testFile = |
| "a\\=>a => b\\=>b\n" + |
| "a\\,a => b\\,b"; |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| assertAnalyzesTo(analyzer, "ball", |
| new String[]{"ball"}, |
| new int[]{1}); |
| |
| assertAnalyzesTo(analyzer, "a=>a", |
| new String[]{"b=>b"}, |
| new int[]{1}); |
| |
| assertAnalyzesTo(analyzer, "a,a", |
| new String[]{"b,b"}, |
| new int[]{1}); |
| analyzer.close(); |
| } |
| |
| /** |
| * parse a syn file with bad syntax |
| */ |
| public void testInvalidAnalyzesToNothingOutput() throws Exception { |
| String testFile = "a => 1"; |
| Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, false); |
| SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); |
| expectThrows(ParseException.class, () -> parser.parse(new StringReader(testFile))); |
| analyzer.close(); |
| } |
| |
| /** |
| * parse a syn file with bad syntax |
| */ |
| public void testInvalidDoubleMap() throws Exception { |
| String testFile = "a => b => c"; |
| Analyzer analyzer = new MockAnalyzer(random()); |
| SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); |
| expectThrows(ParseException.class, () -> parser.parse(new StringReader(testFile))); |
| analyzer.close(); |
| } |
| |
| /** |
| * Tests some simple examples from the solr wiki |
| */ |
| public void testSimple() throws Exception { |
| String testFile = |
| "i-pod, ipod, ipoooood\n" + |
| "foo => foo bar\n" + |
| "foo => baz\n" + |
| "this test, that testing"; |
| |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| assertAnalyzesTo(analyzer, "ball", |
| new String[]{"ball"}, |
| new int[]{1}); |
| |
| assertAnalyzesTo(analyzer, "i-pod", |
| new String[]{"ipod", "ipoooood", "i-pod"}, |
| new int[]{1, 0, 0}); |
| |
| assertAnalyzesTo(analyzer, "foo", |
| new String[]{"foo", "baz", "bar"}, |
| new int[]{1, 0, 1}); |
| |
| assertAnalyzesTo(analyzer, "this test", |
| new String[]{"that", "this", "testing", "test"}, |
| new int[]{1, 0, 1, 0}); |
| analyzer.close(); |
| } |
| |
| public void testBufferLength() throws Exception { |
| String testFile = |
| "c => 8 2 5 6 7\n" + |
| "f c e d f, 1\n" + |
| "c g a f d, 6 5 5\n" + |
| "e c => 4\n" + |
| "g => 5\n" + |
| "a g b f e => 5 0 7 7\n" + |
| "b => 1"; |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| String doc = "b c g a f b d"; |
| String[] expected = new String[]{"1", "8", "2", "5", "6", "7", "5", "a", "f", "1", "d"}; |
| assertAnalyzesTo(analyzer, doc, expected); |
| } |
| |
| private Analyzer solrSynsToAnalyzer(String syns) throws IOException, ParseException { |
| Analyzer analyzer = new MockAnalyzer(random()); |
| SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); |
| parser.parse(new StringReader(syns)); |
| analyzer.close(); |
| return getFlattenAnalyzer(parser, true); |
| } |
| |
| public void testMoreThanOneLookAhead() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b c d", "x", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "a b c e", |
| new String[] {"a", "b", "c", "e"}, |
| new int[] { 0, 2, 4, 6}, |
| new int[] { 1, 3, 5, 7}, |
| new String[] {"word", "word", "word", "word"}, |
| new int[] { 1, 1, 1, 1}, |
| new int[] { 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testLookaheadAfterParse() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "b b", "x", true); |
| add(b, "b", "y", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "b a b b", |
| new String[] {"y", "b", "a", "x", "b", "b"}, |
| new int[] {0, 0, 2, 4, 4, 6}, |
| new int[] {1, 1, 3, 7, 5, 7}, |
| null, |
| new int[] {1, 0, 1, 1, 0, 1}, |
| new int[] {1, 1, 1, 2, 1, 1}, |
| true); |
| } |
| |
| public void testLookaheadSecondParse() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "b b b", "x", true); |
| add(b, "b", "y", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "b b", |
| new String[] {"y", "b", "y", "b"}, |
| new int[] { 0, 0, 2, 2}, |
| new int[] { 1, 1, 3, 3}, |
| null, |
| new int[] { 1, 0, 1, 0}, |
| new int[] { 1, 1, 1, 1}, |
| true); |
| } |
| |
| public void testOneInputMultipleOutputNoKeepOrig() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", false); |
| add(b, "a b", "y", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b c", |
| new String[] {"c", "x", "y", "c"}, |
| new int[] { 0, 2, 2, 6}, |
| new int[] { 1, 5, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "word"}, |
| new int[] { 1, 1, 0, 1}, |
| new int[] { 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testOneInputMultipleOutputMixedKeepOrig() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", true); |
| add(b, "a b", "y", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b c", |
| new String[] {"c", "x", "y", "a", "b", "c"}, |
| new int[] { 0, 2, 2, 2, 4, 6}, |
| new int[] { 1, 5, 5, 3, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "word", "word", "word"}, |
| new int[] { 1, 1, 0, 0, 1, 1, 1, 1}, |
| new int[] { 1, 2, 2, 1, 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testSynAtEnd() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c d e a b", |
| new String[] {"c", "d", "e", "x", "a", "b"}, |
| new int[] { 0, 2, 4, 6, 6, 8}, |
| new int[] { 1, 3, 5, 9, 7, 9}, |
| new String[] {"word", "word", "word", "SYNONYM", "word", "word"}, |
| new int[] { 1, 1, 1, 1, 0, 1}, |
| new int[] { 1, 1, 1, 2, 1, 1}); |
| a.close(); |
| } |
| |
| public void testTwoSynsInARow() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a", "x", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a a b", |
| new String[] {"c", "x", "x", "b"}, |
| new int[] { 0, 2, 4, 6}, |
| new int[] { 1, 3, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "word"}, |
| new int[] { 1, 1, 1, 1}, |
| new int[] { 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testBasicKeepOrigTwoOutputs() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x y", true); |
| add(b, "a b", "m n o", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b d", |
| new String[] {"c", "x", "m", "a", "y", "n", "o", "b", "d"}, |
| new int[] { 0, 2, 2, 2, 2, 2, 2, 4, 6}, |
| new int[] { 1, 5, 5, 3, 5, 5, 5, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "word"}, |
| new int[] { 1, 1, 0, 0, 1, 1, 1, 1, 1}, |
| new int[] { 1, 1, 2, 4, 4, 1, 2, 1, 1}); |
| a.close(); |
| } |
| |
| public void testNoCaptureIfNoMatch() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x y", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, |
| "c d d", |
| new String[] {"c", "d", "d"}, |
| new int[] { 0, 2, 4}, |
| new int[] { 1, 3, 5}, |
| new String[] {"word", "word", "word"}, |
| new int[] { 1, 1, 1}, |
| new int[] { 1, 1, 1}); |
| assertEquals(0, synFilter.getCaptureCount()); |
| a.close(); |
| } |
| |
| public void testBasicNotKeepOrigOneOutput() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b", |
| new String[] {"c", "x"}, |
| new int[] {0, 2}, |
| new int[] {1, 5}, |
| new String[] {"word", "SYNONYM"}, |
| new int[] {1, 1}, |
| new int[] {1, 1}); |
| a.close(); |
| } |
| |
| public void testBasicNoKeepOrigTwoOutputs() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x y", false); |
| add(b, "a b", "m n o", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b d", |
| new String[] {"c", "x", "m", "y", "n", "o", "d"}, |
| new int[] { 0, 2, 2, 2, 2, 2, 6}, |
| new int[] { 1, 5, 5, 5, 5, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "word"}, |
| new int[] { 1, 1, 0, 1, 1, 1, 1}, |
| new int[] { 1, 1, 2, 3, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testIgnoreCase() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x y", false); |
| add(b, "a b", "m n o", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c A B D", |
| new String[] {"c", "x", "m", "y", "n", "o", "D"}, |
| new int[] { 0, 2, 2, 2, 2, 2, 6}, |
| new int[] { 1, 5, 5, 5, 5, 5, 7}, |
| new String[] {"word", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "word"}, |
| new int[] { 1, 1, 0, 1, 1, 1, 1}, |
| new int[] { 1, 1, 2, 3, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testDoNotIgnoreCase() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x y", false); |
| add(b, "a b", "m n o", false); |
| |
| Analyzer a = getAnalyzer(b, false); |
| assertAnalyzesTo(a, |
| "c A B D", |
| new String[] {"c", "A", "B", "D"}, |
| new int[] { 0, 2, 4, 6}, |
| new int[] { 1, 3, 5, 7}, |
| new String[] {"word", "word", "word", "word"}, |
| new int[] { 1, 1, 1, 1}, |
| new int[] { 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testBufferedFinish1() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b c", "m n o", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a b", |
| new String[] {"c", "a", "b"}, |
| new int[] { 0, 2, 4}, |
| new int[] { 1, 3, 5}, |
| new String[] {"word", "word", "word"}, |
| new int[] { 1, 1, 1}, |
| new int[] { 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testBufferedFinish2() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "m n o", false); |
| add(b, "d e", "m n o", false); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "c a d", |
| new String[] {"c", "a", "d"}, |
| new int[] { 0, 2, 4}, |
| new int[] { 1, 3, 5}, |
| new String[] {"word", "word", "word"}, |
| new int[] { 1, 1, 1}, |
| new int[] { 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testCanReuse() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b", "x", true); |
| Analyzer a = getAnalyzer(b, true); |
| for(int i=0;i<10;i++) { |
| assertAnalyzesTo(a, |
| "c a b", |
| new String[] {"c", "x", "a", "b"}, |
| new int[] { 0, 2, 2, 4}, |
| new int[] { 1, 5, 3, 5}, |
| new String[] {"word", "SYNONYM", "word", "word"}, |
| new int[] { 1, 1, 0, 1}, |
| new int[] { 1, 2, 1, 1}); |
| } |
| a.close(); |
| } |
| |
| /** Multiple input tokens map to a single output token */ |
| public void testManyToOne() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b c", "z", true); |
| |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "a b c d", |
| new String[] {"z", "a", "b", "c", "d"}, |
| new int[] { 0, 0, 2, 4, 6}, |
| new int[] { 5, 1, 3, 5, 7}, |
| new String[] {"SYNONYM", "word", "word", "word", "word"}, |
| new int[] { 1, 0, 1, 1, 1}, |
| new int[] { 3, 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testBufferAfterMatch() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "a b c d", "x", true); |
| add(b, "a b", "y", false); |
| |
| // The 'c' token has to be buffered because SynGraphFilter |
| // needs to know whether a b c d -> x matches: |
| Analyzer a = getAnalyzer(b, true); |
| assertAnalyzesTo(a, |
| "f a b c e", |
| new String[] {"f", "y", "c", "e"}, |
| new int[] { 0, 2, 6, 8}, |
| new int[] { 1, 5, 7, 9}, |
| new String[] {"word", "SYNONYM", "word", "word"}, |
| new int[] { 1, 1, 1, 1}, |
| new int[] { 1, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testZeroSyns() throws Exception { |
| Tokenizer tokenizer = new MockTokenizer(); |
| tokenizer.setReader(new StringReader("aa bb")); |
| |
| IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> |
| new SynonymGraphFilter(tokenizer, new SynonymMap.Builder(true).build(), true)); |
| assertEquals("fst must be non-null", ex.getMessage()); |
| } |
| |
| public void testOutputHangsOffEnd() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = false; |
| // b hangs off the end (no input token under it): |
| add(b, "a", "a b", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| assertAnalyzesTo(a, "a", |
| new String[] {"a", "b"}, |
| new int[] { 0, 0}, |
| new int[] { 1, 1}, |
| null, |
| new int[] { 1, 1}, |
| new int[] { 1, 1}, |
| true); |
| a.close(); |
| } |
| |
| public void testDedup() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = false; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "a b", |
| new String[]{"ab"}, |
| new int[]{1}); |
| a.close(); |
| } |
| |
| public void testNoDedup() throws Exception { |
| // dedup is false: |
| SynonymMap.Builder b = new SynonymMap.Builder(false); |
| final boolean keepOrig = false; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "a b", |
| new String[]{"ab", "ab", "ab"}, |
| new int[]{1, 0, 0}); |
| a.close(); |
| } |
| |
| public void testMatching() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = false; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a c", "ac", keepOrig); |
| add(b, "a", "aa", keepOrig); |
| add(b, "b", "bb", keepOrig); |
| add(b, "z x c v", "zxcv", keepOrig); |
| add(b, "x c", "xc", keepOrig); |
| |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| checkOneTerm(a, "$", "$"); |
| checkOneTerm(a, "a", "aa"); |
| checkOneTerm(a, "b", "bb"); |
| |
| assertAnalyzesTo(a, "a $", |
| new String[]{"aa", "$"}, |
| new int[]{1, 1}); |
| |
| assertAnalyzesTo(a, "$ a", |
| new String[]{"$", "aa"}, |
| new int[]{1, 1}); |
| |
| assertAnalyzesTo(a, "a a", |
| new String[]{"aa", "aa"}, |
| new int[]{1, 1}); |
| |
| assertAnalyzesTo(a, "z x c v", |
| new String[]{"zxcv"}, |
| new int[]{1}); |
| |
| assertAnalyzesTo(a, "z x c $", |
| new String[]{"z", "xc", "$"}, |
| new int[]{1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testBasic1() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| add(b, "a", "foo", true); |
| add(b, "a b", "bar fee", true); |
| add(b, "b c", "dog collar", true); |
| add(b, "c d", "dog harness holder extras", true); |
| add(b, "m c e", "dog barks loudly", false); |
| add(b, "i j k", "feep", true); |
| |
| add(b, "e f", "foo bar", false); |
| add(b, "e f", "baz bee", false); |
| |
| add(b, "z", "boo", false); |
| add(b, "y", "bee", true); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "a b c", |
| new String[] {"bar", "a", "fee", "b", "c"}, |
| new int[] {1, 0, 1, 0, 1}); |
| |
| assertAnalyzesTo(a, "x a b c d", |
| new String[] {"x", "bar", "a", "fee", "b", "dog", "c", "harness", "d", "holder", "extras"}, |
| new int[] {1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1}); |
| |
| assertAnalyzesTo(a, "a b a", |
| new String[] {"bar", "a", "fee", "b", "foo", "a"}, |
| new int[] {1, 0, 1, 0, 1, 0}); |
| |
| // outputs no longer add to one another: |
| assertAnalyzesTo(a, "c d c d", |
| new String[] {"dog", "c", "harness", "d", "holder", "extras", "dog", "c", "harness", "d", "holder", "extras"}, |
| new int[] {1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 1}); |
| |
| // two outputs for same input |
| assertAnalyzesTo(a, "e f", |
| new String[] {"foo", "baz", "bar", "bee"}, |
| new int[] {1, 0, 1, 0}); |
| |
| // verify multi-word / single-output offsets: |
| assertAnalyzesTo(a, "g i j k g", |
| new String[] {"g", "feep", "i", "j", "k", "g"}, |
| new int[] {1, 1, 0, 1, 1, 1}); |
| |
| // mixed keepOrig true/false: |
| assertAnalyzesTo(a, "a m c e x", |
| new String[] {"foo", "a", "dog", "barks", "loudly", "x"}, |
| new int[] {1, 0, 1, 1, 1, 1}); |
| assertAnalyzesTo(a, "c d m c e x", |
| new String[] {"dog", "c", "harness", "d", "holder", "extras", "dog", "barks", "loudly","x"}, |
| new int[] {1, 0, 1, 0, 1, 1, 1, 1, 1, 1}); |
| assertTrue(synFilter.getCaptureCount() > 0); |
| |
| // no captureStates when no syns matched |
| assertAnalyzesTo(a, "p q r s t", |
| new String[] {"p", "q", "r", "s", "t"}, |
| new int[] {1, 1, 1, 1, 1}); |
| assertEquals(0, synFilter.getCaptureCount()); |
| |
| // captureStates are necessary for the single-token syn case: |
| assertAnalyzesTo(a, "p q z y t", |
| new String[] {"p", "q", "boo", "bee", "y", "t"}, |
| new int[] {1, 1, 1, 1, 0, 1}); |
| assertTrue(synFilter.getCaptureCount() > 0); |
| } |
| |
| public void testBasic2() throws Exception { |
| boolean keepOrig = true; |
| do { |
| keepOrig = !keepOrig; |
| |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| add(b,"aaa", "aaaa1 aaaa2 aaaa3", keepOrig); |
| add(b, "bbb", "bbbb1 bbbb2", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| if (keepOrig) { |
| assertAnalyzesTo(a, "xyzzy bbb pot of gold", |
| new String[] {"xyzzy", "bbbb1", "bbb", "bbbb2", "pot", "of", "gold"}, |
| new int[] {1, 1, 0, 1, 1, 1, 1}); |
| assertAnalyzesTo(a, "xyzzy aaa pot of gold", |
| new String[] {"xyzzy", "aaaa1", "aaa", "aaaa2", "aaaa2", "pot", "of", "gold"}, |
| new int[] {1, 1, 0, 1, 1, 1, 1, 1}); |
| } else { |
| assertAnalyzesTo(a, "xyzzy bbb pot of gold", |
| new String[] {"xyzzy", "bbbb1", "bbbb2", "pot", "of", "gold"}, |
| new int[] {1, 1, 1, 1, 1, 1}); |
| assertAnalyzesTo(a, "xyzzy aaa pot of gold", |
| new String[] {"xyzzy", "aaaa1", "aaaa2", "aaaa3", "pot", "of", "gold"}, |
| new int[] {1, 1, 1, 1, 1, 1, 1}); |
| } |
| } while (keepOrig); |
| } |
| |
| /** If we expand synonyms during indexing, it's a bit better than |
| * SynonymFilter is today, but still necessarily has false |
| * positive and negative PhraseQuery matches because we do not |
| * index posLength, so we lose information. */ |
| public void testFlattenedGraph() throws Exception { |
| |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "wtf", "what the fudge", true); |
| |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "wtf happened", |
| new String[] {"what", "wtf", "the", "fudge", "happened"}, |
| new int[] { 0, 0, 0, 0, 4}, |
| new int[] { 3, 3, 3, 3, 12}, |
| null, |
| new int[] { 1, 0, 1, 1, 1}, |
| new int[] { 1, 3, 1, 1, 1}, |
| true); |
| |
| Directory dir = newDirectory(); |
| RandomIndexWriter w = new RandomIndexWriter(random(), dir, a); |
| Document doc = new Document(); |
| doc.add(newTextField("field", "wtf happened", Field.Store.NO)); |
| w.addDocument(doc); |
| IndexReader r = w.getReader(); |
| w.close(); |
| |
| IndexSearcher s = newSearcher(r); |
| |
| // Good (this should not match, and doesn't): |
| assertEquals(0, s.count(new PhraseQuery("field", "what", "happened"))); |
| |
| // Bad (this should match, but doesn't): |
| assertEquals(0, s.count(new PhraseQuery("field", "wtf", "happened"))); |
| |
| // Good (this should match, and does): |
| assertEquals(1, s.count(new PhraseQuery("field", "what", "the", "fudge", "happened"))); |
| |
| // Bad (this should not match, but does): |
| assertEquals(1, s.count(new PhraseQuery("field", "wtf", "the"))); |
| |
| IOUtils.close(r, dir); |
| } |
| |
| // Needs TermAutomatonQuery, which is in sandbox still: |
| /* |
| public void testAccurateGraphQuery1() throws Exception { |
| Directory dir = newDirectory(); |
| RandomIndexWriter w = new RandomIndexWriter(random(), dir); |
| Document doc = new Document(); |
| doc.add(newTextField("field", "wtf happened", Field.Store.NO)); |
| w.addDocument(doc); |
| IndexReader r = w.getReader(); |
| w.close(); |
| |
| IndexSearcher s = newSearcher(r); |
| |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "what the fudge", "wtf", true); |
| |
| SynonymMap map = b.build(); |
| |
| TokenStreamToTermAutomatonQuery ts2q = new TokenStreamToTermAutomatonQuery(); |
| |
| TokenStream in = new CannedTokenStream(0, 23, new Token[] { |
| token("what", 1, 1, 0, 4), |
| token("the", 1, 1, 5, 8), |
| token("fudge", 1, 1, 9, 14), |
| token("happened", 1, 1, 15, 23), |
| }); |
| |
| assertEquals(1, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| in = new CannedTokenStream(0, 12, new Token[] { |
| token("wtf", 1, 1, 0, 3), |
| token("happened", 1, 1, 4, 12), |
| }); |
| |
| assertEquals(1, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| // "what happened" should NOT match: |
| in = new CannedTokenStream(0, 13, new Token[] { |
| token("what", 1, 1, 0, 4), |
| token("happened", 1, 1, 5, 13), |
| }); |
| assertEquals(0, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| IOUtils.close(r, dir); |
| } |
| */ |
| |
| /** If we expand synonyms at search time, the results are correct. */ |
| // Needs TermAutomatonQuery, which is in sandbox still: |
| /* |
| public void testAccurateGraphQuery2() throws Exception { |
| Directory dir = newDirectory(); |
| RandomIndexWriter w = new RandomIndexWriter(random(), dir); |
| Document doc = new Document(); |
| doc.add(newTextField("field", "say wtf happened", Field.Store.NO)); |
| w.addDocument(doc); |
| IndexReader r = w.getReader(); |
| w.close(); |
| |
| IndexSearcher s = newSearcher(r); |
| |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "what the fudge", "wtf", true); |
| |
| SynonymMap map = b.build(); |
| |
| TokenStream in = new CannedTokenStream(0, 26, new Token[] { |
| token("say", 1, 1, 0, 3), |
| token("what", 1, 1, 3, 7), |
| token("the", 1, 1, 8, 11), |
| token("fudge", 1, 1, 12, 17), |
| token("happened", 1, 1, 18, 26), |
| }); |
| |
| TokenStreamToTermAutomatonQuery ts2q = new TokenStreamToTermAutomatonQuery(); |
| |
| assertEquals(1, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| // "what happened" should NOT match: |
| in = new CannedTokenStream(0, 13, new Token[] { |
| token("what", 1, 1, 0, 4), |
| token("happened", 1, 1, 5, 13), |
| }); |
| assertEquals(0, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| IOUtils.close(r, dir); |
| } |
| */ |
| |
| // Needs TermAutomatonQuery, which is in sandbox still: |
| /* |
| public void testAccurateGraphQuery3() throws Exception { |
| Directory dir = newDirectory(); |
| RandomIndexWriter w = new RandomIndexWriter(random(), dir); |
| Document doc = new Document(); |
| doc.add(newTextField("field", "say what the fudge happened", Field.Store.NO)); |
| w.addDocument(doc); |
| IndexReader r = w.getReader(); |
| w.close(); |
| |
| IndexSearcher s = newSearcher(r); |
| |
| SynonymMap.Builder b = new SynonymMap.Builder(); |
| add(b, "wtf", "what the fudge", true); |
| |
| SynonymMap map = b.build(); |
| |
| TokenStream in = new CannedTokenStream(0, 15, new Token[] { |
| token("say", 1, 1, 0, 3), |
| token("wtf", 1, 1, 3, 6), |
| token("happened", 1, 1, 7, 15), |
| }); |
| |
| TokenStreamToTermAutomatonQuery ts2q = new TokenStreamToTermAutomatonQuery(); |
| |
| assertEquals(1, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| // "what happened" should NOT match: |
| in = new CannedTokenStream(0, 13, new Token[] { |
| token("what", 1, 1, 0, 4), |
| token("happened", 1, 1, 5, 13), |
| }); |
| assertEquals(0, s.count(ts2q.toQuery("field", new SynonymGraphFilter(in, map, true)))); |
| |
| IOUtils.close(r, dir); |
| } |
| |
| private static Token token(String term, int posInc, int posLength, int startOffset, int endOffset) { |
| final Token t = new Token(term, startOffset, endOffset); |
| t.setPositionIncrement(posInc); |
| t.setPositionLength(posLength); |
| return t; |
| } |
| */ |
| |
| private String randomNonEmptyString() { |
| while(true) { |
| String s = TestUtil.randomUnicodeString(random()).trim(); |
| //String s = TestUtil.randomSimpleString(random()).trim(); |
| if (s.length() != 0 && s.indexOf('\u0000') == -1) { |
| return s; |
| } |
| } |
| } |
| |
| // Adds MockGraphTokenFilter after SynFilter: |
| public void testRandomGraphAfter() throws Exception { |
| final int numIters = atLeast(3); |
| for (int i = 0; i < numIters; i++) { |
| SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean()); |
| final int numEntries = atLeast(10); |
| for (int j = 0; j < numEntries; j++) { |
| add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean()); |
| } |
| final SynonymMap map = b.build(); |
| final boolean ignoreCase = random().nextBoolean(); |
| final boolean doFlatten = random().nextBoolean(); |
| |
| final Analyzer analyzer = new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true); |
| TokenStream syns = new SynonymGraphFilter(tokenizer, map, ignoreCase); |
| TokenStream graph = new MockGraphTokenFilter(random(), syns); |
| if (doFlatten) { |
| graph = new FlattenGraphFilter(graph); |
| } |
| return new TokenStreamComponents(tokenizer, graph); |
| } |
| }; |
| |
| checkRandomData(random(), analyzer, 100); |
| analyzer.close(); |
| } |
| } |
| |
| public void testEmptyStringInput() throws IOException { |
| final int numIters = atLeast(10); |
| for (int i = 0; i < numIters; i++) { |
| SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean()); |
| final int numEntries = atLeast(10); |
| for (int j = 0; j < numEntries; j++) { |
| add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean()); |
| } |
| final boolean ignoreCase = random().nextBoolean(); |
| |
| Analyzer analyzer = getAnalyzer(b, ignoreCase); |
| |
| checkAnalysisConsistency(random(), analyzer, random().nextBoolean(), ""); |
| analyzer.close(); |
| } |
| } |
| |
| /** simple random test, doesn't verify correctness. |
| * does verify it doesnt throw exceptions, or that the stream doesn't misbehave |
| */ |
| public void testRandom2() throws Exception { |
| final int numIters = atLeast(3); |
| for (int i = 0; i < numIters; i++) { |
| SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean()); |
| final int numEntries = atLeast(10); |
| for (int j = 0; j < numEntries; j++) { |
| add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean()); |
| } |
| final boolean ignoreCase = random().nextBoolean(); |
| final boolean doFlatten = random().nextBoolean(); |
| |
| Analyzer analyzer; |
| if (doFlatten) { |
| analyzer = getFlattenAnalyzer(b, ignoreCase); |
| } else { |
| analyzer = getAnalyzer(b, ignoreCase); |
| } |
| |
| checkRandomData(random(), analyzer, 100); |
| analyzer.close(); |
| } |
| } |
| |
| /** simple random test like testRandom2, but for larger docs |
| */ |
| public void testRandomHuge() throws Exception { |
| final int numIters = atLeast(1); |
| for (int i = 0; i < numIters; i++) { |
| SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean()); |
| final int numEntries = atLeast(10); |
| if (VERBOSE) { |
| System.out.println("TEST: iter=" + i + " numEntries=" + numEntries); |
| } |
| for (int j = 0; j < numEntries; j++) { |
| add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean()); |
| } |
| final boolean ignoreCase = random().nextBoolean(); |
| final boolean doFlatten = random().nextBoolean(); |
| |
| Analyzer analyzer; |
| if (doFlatten) { |
| analyzer = getFlattenAnalyzer(b, ignoreCase); |
| } else { |
| analyzer = getAnalyzer(b, ignoreCase); |
| } |
| |
| checkRandomData(random(), analyzer, 100, 1024); |
| analyzer.close(); |
| } |
| } |
| |
| public void testEmptyTerm() throws IOException { |
| final int numIters = atLeast(10); |
| for (int i = 0; i < numIters; i++) { |
| SynonymMap.Builder b = new SynonymMap.Builder(random().nextBoolean()); |
| final int numEntries = atLeast(10); |
| for (int j = 0; j < numEntries; j++) { |
| add(b, randomNonEmptyString(), randomNonEmptyString(), random().nextBoolean()); |
| } |
| final boolean ignoreCase = random().nextBoolean(); |
| |
| final Analyzer analyzer = getAnalyzer(b, ignoreCase); |
| |
| checkAnalysisConsistency(random(), analyzer, random().nextBoolean(), ""); |
| analyzer.close(); |
| } |
| } |
| |
| // LUCENE-3375 |
| public void testVanishingTermsNoFlatten() throws Exception { |
| String testFile = |
| "aaa => aaaa1 aaaa2 aaaa3\n" + |
| "bbb => bbbb1 bbbb2\n"; |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| assertAnalyzesTo(analyzer, "xyzzy bbb pot of gold", |
| new String[] { "xyzzy", "bbbb1", "bbbb2", "pot", "of", "gold" }); |
| |
| // xyzzy aaa pot of gold -> xyzzy aaaa1 aaaa2 aaaa3 gold |
| assertAnalyzesTo(analyzer, "xyzzy aaa pot of gold", |
| new String[] { "xyzzy", "aaaa1", "aaaa2", "aaaa3", "pot", "of", "gold" }); |
| analyzer.close(); |
| } |
| |
| // LUCENE-3375 |
| public void testVanishingTermsWithFlatten() throws Exception { |
| String testFile = |
| "aaa => aaaa1 aaaa2 aaaa3\n" + |
| "bbb => bbbb1 bbbb2\n"; |
| |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| assertAnalyzesTo(analyzer, "xyzzy bbb pot of gold", |
| new String[] { "xyzzy", "bbbb1", "bbbb2", "pot", "of", "gold" }); |
| |
| // xyzzy aaa pot of gold -> xyzzy aaaa1 aaaa2 aaaa3 gold |
| assertAnalyzesTo(analyzer, "xyzzy aaa pot of gold", |
| new String[] { "xyzzy", "aaaa1", "aaaa2", "aaaa3", "pot", "of", "gold" }); |
| analyzer.close(); |
| } |
| |
| public void testBuilderDedup() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = false; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "a b", |
| new String[] { "ab" }, |
| new int[] { 1 }); |
| a.close(); |
| } |
| |
| public void testBuilderNoDedup() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(false); |
| final boolean keepOrig = false; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a b", "ab", keepOrig); |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "a b", |
| new String[] { "ab", "ab", "ab" }, |
| new int[] { 1, 0, 0 }); |
| a.close(); |
| } |
| |
| public void testRecursion1() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = false; |
| add(b, "zoo", "zoo", keepOrig); |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "zoo zoo $ zoo", |
| new String[] { "zoo", "zoo", "$", "zoo" }, |
| new int[] { 1, 1, 1, 1 }); |
| a.close(); |
| } |
| |
| public void testRecursion2() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = false; |
| add(b, "zoo", "zoo", keepOrig); |
| add(b, "zoo", "zoo zoo", keepOrig); |
| Analyzer a = getAnalyzer(b, true); |
| |
| // verify("zoo zoo $ zoo", "zoo/zoo zoo/zoo/zoo $/zoo zoo/zoo zoo"); |
| assertAnalyzesTo(a, "zoo zoo $ zoo", |
| new String[] { "zoo", "zoo", "zoo", "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" }, |
| new int[] { 1, 0, 1, 1, 0, 1, 1, 1, 0, 1 }); |
| a.close(); |
| } |
| |
| public void testRecursion3() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = true; |
| add(b, "zoo zoo", "zoo", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "zoo zoo $ zoo", |
| new String[]{"zoo", "zoo", "zoo", "$", "zoo"}, |
| new int[]{1, 0, 1, 1, 1}); |
| a.close(); |
| } |
| |
| public void testRecursion4() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = true; |
| add(b, "zoo zoo", "zoo", keepOrig); |
| add(b, "zoo", "zoo zoo", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| assertAnalyzesTo(a, "zoo zoo $ zoo", |
| new String[]{"zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo"}, |
| new int[]{1, 0, 1, 1, 1, 0, 1}); |
| a.close(); |
| } |
| |
| public void testKeepOrig() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = true; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a c", "ac", keepOrig); |
| add(b, "a", "aa", keepOrig); |
| add(b, "b", "bb", keepOrig); |
| add(b, "z x c v", "zxcv", keepOrig); |
| add(b, "x c", "xc", keepOrig); |
| Analyzer a = getAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "$", |
| new String[] { "$" }, |
| new int[] { 1 }); |
| assertAnalyzesTo(a, "a", |
| new String[] { "aa", "a" }, |
| new int[] { 1, 0 }); |
| assertAnalyzesTo(a, "a", |
| new String[] { "aa", "a" }, |
| new int[] { 1, 0 }); |
| assertAnalyzesTo(a, "$ a", |
| new String[] { "$", "aa", "a" }, |
| new int[] { 1, 1, 0 }); |
| assertAnalyzesTo(a, "a $", |
| new String[] { "aa", "a", "$" }, |
| new int[] { 1, 0, 1 }); |
| assertAnalyzesTo(a, "$ a !", |
| new String[] { "$", "aa", "a", "!" }, |
| new int[] { 1, 1, 0, 1 }); |
| assertAnalyzesTo(a, "a a", |
| new String[] { "aa", "a", "aa", "a" }, |
| new int[] { 1, 0, 1, 0 }); |
| assertAnalyzesTo(a, "b", |
| new String[] { "bb", "b" }, |
| new int[] { 1, 0 }); |
| assertAnalyzesTo(a, "z x c v", |
| new String[] { "zxcv", "z", "x", "c", "v" }, |
| new int[] { 1, 0, 1, 1, 1 }); |
| assertAnalyzesTo(a, "z x c $", |
| new String[] { "z", "xc", "x", "c", "$" }, |
| new int[] { 1, 1, 0, 1, 1 }); |
| a.close(); |
| } |
| |
| /** |
| * verify type of token and positionLengths on synonyms of different word counts, with non preserving, explicit rules. |
| */ |
| public void testNonPreservingMultiwordSynonyms() throws Exception { |
| String testFile = |
| "aaa => two words\n" + |
| "bbb => one two, very many multiple words\n" + |
| "ee ff, gg, h i j k, h i => one\n" + |
| "cc dd => usa,united states,u s a,united states of america"; |
| |
| Analyzer analyzer = solrSynsToAnalyzer(testFile); |
| |
| assertAnalyzesTo(analyzer, "aaa", |
| new String[]{"two", "words"}, |
| new int[]{0, 0}, |
| new int[]{3, 3}, |
| new String[]{"SYNONYM", "SYNONYM"}, |
| new int[]{1, 1}, |
| new int[]{1, 1}); |
| |
| assertAnalyzesToPositions(analyzer, "amazing aaa", |
| new String[]{"amazing", "two", "words"}, |
| new String[]{"word", "SYNONYM", "SYNONYM"}, |
| new int[]{1, 1, 1}, |
| new int[]{1, 1, 1}); |
| |
| assertAnalyzesTo(analyzer, "p bbb s", |
| new String[]{"p", "one", "very", "two", "many", "multiple", "words", "s"}, |
| new int[]{0, 2, 2, 2, 2, 2, 2, 6}, |
| new int[]{1, 5, 5, 5, 5, 5, 5, 7}, |
| new String[]{"word", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "word"}, |
| new int[]{1, 1, 0, 1, 0, 1, 1, 1}, |
| new int[]{1, 1, 1, 3, 1, 1, 1, 1}); |
| |
| assertAnalyzesTo(analyzer, "p ee ff s", |
| new String[]{"p", "one", "s"}, |
| new int[]{0, 2, 8}, |
| new int[]{1, 7, 9}, |
| new String[]{"word", "SYNONYM", "word"}, |
| new int[]{1, 1, 1}, |
| new int[]{1, 1, 1}); |
| |
| assertAnalyzesTo(analyzer, "p h i j s", |
| new String[]{"p", "one", "j", "s"}, |
| new int[]{0, 2, 6, 8}, |
| new int[]{1, 5, 7, 9}, |
| new String[]{"word", "SYNONYM", "word", "word"}, |
| new int[]{1, 1, 1, 1}, |
| new int[]{1, 1, 1, 1}); |
| |
| analyzer.close(); |
| } |
| |
| private Analyzer getAnalyzer(SynonymMap.Builder b, final boolean ignoreCase) throws IOException { |
| final SynonymMap map = b.build(); |
| return new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| // Make a local variable so testRandomHuge doesn't share it across threads! |
| SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase); |
| TestSynonymGraphFilter.this.flattenFilter = null; |
| TestSynonymGraphFilter.this.synFilter = synFilter; |
| return new TokenStreamComponents(tokenizer, synFilter); |
| } |
| }; |
| } |
| |
| /** Appends FlattenGraphFilter too */ |
| private Analyzer getFlattenAnalyzer(SynonymMap.Builder b, boolean ignoreCase) throws IOException { |
| final SynonymMap map = b.build(); |
| return new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true); |
| // Make a local variable so testRandomHuge doesn't share it across threads! |
| SynonymGraphFilter synFilter = new SynonymGraphFilter(tokenizer, map, ignoreCase); |
| FlattenGraphFilter flattenFilter = new FlattenGraphFilter(synFilter); |
| TestSynonymGraphFilter.this.synFilter = synFilter; |
| TestSynonymGraphFilter.this.flattenFilter = flattenFilter; |
| return new TokenStreamComponents(tokenizer, flattenFilter); |
| } |
| }; |
| } |
| |
| private void add(SynonymMap.Builder b, String input, String output, boolean keepOrig) { |
| if (VERBOSE) { |
| //System.out.println(" add input=" + input + " output=" + output + " keepOrig=" + keepOrig); |
| } |
| CharsRefBuilder inputCharsRef = new CharsRefBuilder(); |
| SynonymMap.Builder.join(input.split(" +"), inputCharsRef); |
| |
| CharsRefBuilder outputCharsRef = new CharsRefBuilder(); |
| SynonymMap.Builder.join(output.split(" +"), outputCharsRef); |
| |
| b.add(inputCharsRef.get(), outputCharsRef.get(), keepOrig); |
| } |
| |
| private char[] randomBinaryChars(int minLen, int maxLen, double bias, char base) { |
| int len = TestUtil.nextInt(random(), minLen, maxLen); |
| char[] chars = new char[len]; |
| for(int i=0;i<len;i++) { |
| char ch; |
| if (random().nextDouble() < bias) { |
| ch = base; |
| } else { |
| ch = (char) (base+1); |
| } |
| chars[i] = ch; |
| } |
| |
| return chars; |
| } |
| |
| private static String toTokenString(char[] chars) { |
| StringBuilder b = new StringBuilder(); |
| for(char c : chars) { |
| if (b.length() > 0) { |
| b.append(' '); |
| } |
| b.append(c); |
| } |
| return b.toString(); |
| } |
| |
| private static class OneSyn { |
| char[] in; |
| char[] out; |
| boolean keepOrig; |
| |
| @Override |
| public String toString() { |
| return toTokenString(in) + " --> " + toTokenString(out) + " (keepOrig=" + keepOrig + ")"; |
| } |
| } |
| |
| public void testRandomSyns() throws Exception { |
| int synCount = atLeast(10); |
| double bias = random().nextDouble(); |
| boolean dedup = random().nextBoolean(); |
| |
| boolean flatten = random().nextBoolean(); |
| |
| SynonymMap.Builder b = new SynonymMap.Builder(dedup); |
| List<OneSyn> syns = new ArrayList<>(); |
| // Makes random syns from random a / b tokens, mapping to random x / y tokens |
| if (VERBOSE) { |
| System.out.println("TEST: make " + synCount + " syns"); |
| System.out.println(" bias for a over b=" + bias); |
| System.out.println(" dedup=" + dedup); |
| System.out.println(" flatten=" + flatten); |
| } |
| |
| int maxSynLength = 0; |
| |
| for(int i=0;i<synCount;i++) { |
| OneSyn syn = new OneSyn(); |
| syn.in = randomBinaryChars(1, 5, bias, 'a'); |
| syn.out = randomBinaryChars(1, 5, 0.5, 'x'); |
| syn.keepOrig = random().nextBoolean(); |
| syns.add(syn); |
| |
| maxSynLength = Math.max(maxSynLength, syn.in.length); |
| |
| if (VERBOSE) { |
| System.out.println(" " + syn); |
| } |
| add(b, toTokenString(syn.in), toTokenString(syn.out), syn.keepOrig); |
| } |
| |
| // Compute max allowed lookahead for flatten filter: |
| int maxFlattenLookahead = 0; |
| if (flatten) { |
| for(int i=0;i<synCount;i++) { |
| OneSyn syn1 = syns.get(i); |
| int count = syn1.out.length; |
| boolean keepOrig = syn1.keepOrig; |
| for(int j=0;j<synCount;j++) { |
| OneSyn syn2 = syns.get(i); |
| keepOrig |= syn2.keepOrig; |
| if (syn1.in.equals(syn2.in)) { |
| count += syn2.out.length; |
| } |
| } |
| |
| if (keepOrig) { |
| count += syn1.in.length; |
| } |
| |
| maxFlattenLookahead = Math.max(maxFlattenLookahead, count); |
| } |
| } |
| |
| // Only used w/ VERBOSE: |
| Analyzer aNoFlattened; |
| if (VERBOSE) { |
| aNoFlattened = getAnalyzer(b, true); |
| } else { |
| aNoFlattened = null; |
| } |
| |
| Analyzer a; |
| if (flatten) { |
| a = getFlattenAnalyzer(b, true); |
| } else { |
| a = getAnalyzer(b, true); |
| } |
| |
| int iters = atLeast(1); |
| for(int iter=0;iter<iters;iter++) { |
| |
| String doc = toTokenString(randomBinaryChars(50, 100, bias, 'a')); |
| //String doc = toTokenString(randomBinaryChars(10, 50, bias, 'a')); |
| |
| if (VERBOSE) { |
| System.out.println("TEST: iter="+ iter + " doc=" + doc); |
| } |
| Automaton expected = slowSynFilter(doc, syns, flatten); |
| if (VERBOSE) { |
| System.out.println(" expected:\n" + expected.toDot()); |
| if (flatten) { |
| Automaton unflattened = toAutomaton(aNoFlattened.tokenStream("field", new StringReader(doc))); |
| System.out.println(" actual unflattened:\n" + unflattened.toDot()); |
| } |
| } |
| Automaton actual = toAutomaton(a.tokenStream("field", new StringReader(doc))); |
| if (VERBOSE) { |
| System.out.println(" actual:\n" + actual.toDot()); |
| } |
| |
| assertTrue("maxLookaheadUsed=" + synFilter.getMaxLookaheadUsed() + " maxSynLength=" + maxSynLength, |
| synFilter.getMaxLookaheadUsed() <= maxSynLength); |
| if (flatten) { |
| assertTrue("flatten maxLookaheadUsed=" + flattenFilter.getMaxLookaheadUsed() + " maxFlattenLookahead=" + maxFlattenLookahead, |
| flattenFilter.getMaxLookaheadUsed() <= maxFlattenLookahead); |
| } |
| |
| checkAnalysisConsistency(random(), a, random().nextBoolean(), doc); |
| // We can easily have a non-deterministic automaton at this point, e.g. if |
| // more than one syn matched at given point, or if the syn mapped to an |
| // output token that also happens to be in the input: |
| try { |
| actual = Operations.determinize(actual, 50000); |
| } catch (TooComplexToDeterminizeException tctde) { |
| // Unfortunately the syns can easily create difficult-to-determinize graphs: |
| assertTrue(approxEquals(actual, expected)); |
| continue; |
| } |
| |
| try { |
| expected = Operations.determinize(expected, 50000); |
| } catch (TooComplexToDeterminizeException tctde) { |
| // Unfortunately the syns can easily create difficult-to-determinize graphs: |
| assertTrue(approxEquals(actual, expected)); |
| continue; |
| } |
| |
| assertTrue(approxEquals(actual, expected)); |
| assertTrue(Operations.sameLanguage(actual, expected)); |
| } |
| |
| a.close(); |
| } |
| |
| /** Only used when true equality is too costly to check! */ |
| private boolean approxEquals(Automaton actual, Automaton expected) { |
| // Don't collapse these into one line else the thread stack won't say which direction failed!: |
| boolean b1 = approxSubsetOf(actual, expected); |
| boolean b2 = approxSubsetOf(expected, actual); |
| return b1 && b2; |
| } |
| |
| private boolean approxSubsetOf(Automaton a1, Automaton a2) { |
| AutomatonTestUtil.RandomAcceptedStrings ras = new AutomatonTestUtil.RandomAcceptedStrings(a1); |
| for(int i=0;i<2000;i++) { |
| int[] ints = ras.getRandomAcceptedString(random()); |
| IntsRef path = new IntsRef(ints, 0, ints.length); |
| if (accepts(a2, path) == false) { |
| throw new RuntimeException("a2 does not accept " + path); |
| } |
| } |
| |
| // Presumed true |
| return true; |
| } |
| |
| /** Like {@link Operations#run} except the incoming automaton is allowed to be non-deterministic. */ |
| private static boolean accepts(Automaton a, IntsRef path) { |
| Set<Integer> states = new HashSet<>(); |
| states.add(0); |
| Transition t = new Transition(); |
| for(int i=0;i<path.length;i++) { |
| int digit = path.ints[path.offset+i]; |
| Set<Integer> nextStates = new HashSet<>(); |
| for(int state : states) { |
| int count = a.initTransition(state, t); |
| for(int j=0;j<count;j++) { |
| a.getNextTransition(t); |
| if (digit >= t.min && digit <= t.max) { |
| nextStates.add(t.dest); |
| } |
| } |
| } |
| states = nextStates; |
| if (states.isEmpty()) { |
| return false; |
| } |
| } |
| |
| for(int state : states) { |
| if (a.isAccept(state)) { |
| return true; |
| } |
| } |
| |
| return false; |
| } |
| |
| /** Stupid, slow brute-force, yet hopefully bug-free, synonym filter. */ |
| private Automaton slowSynFilter(String doc, List<OneSyn> syns, boolean flatten) { |
| String[] tokens = doc.split(" +"); |
| if (VERBOSE) { |
| System.out.println(" doc has " + tokens.length + " tokens"); |
| } |
| int i=0; |
| Automaton.Builder a = new Automaton.Builder(); |
| int lastState = a.createState(); |
| while (i<tokens.length) { |
| // Consider all possible syn matches starting at this point: |
| assert tokens[i].length() == 1; |
| if (VERBOSE) { |
| System.out.println(" i=" + i); |
| } |
| |
| List<OneSyn> matches = new ArrayList<>(); |
| for(OneSyn syn : syns) { |
| if (i + syn.in.length <= tokens.length) { |
| boolean match = true; |
| for(int j=0;j<syn.in.length;j++) { |
| if (tokens[i+j].charAt(0) != syn.in[j]) { |
| match = false; |
| break; |
| } |
| } |
| |
| if (match) { |
| if (matches.isEmpty() == false) { |
| if (syn.in.length < matches.get(0).in.length) { |
| // Greedy matching: we already found longer syns matching here |
| continue; |
| } else if (syn.in.length > matches.get(0).in.length) { |
| // Greedy matching: all previous matches were shorter, so we drop them |
| matches.clear(); |
| } else { |
| // Keep the current matches: we allow multiple synonyms matching the same input string |
| } |
| } |
| |
| matches.add(syn); |
| } |
| } |
| } |
| |
| int nextState = a.createState(); |
| |
| if (matches.isEmpty() == false) { |
| // We have match(es) starting at this token |
| if (VERBOSE) { |
| System.out.println(" matches @ i=" + i + ": " + matches); |
| } |
| // We keepOrig if any of the matches said to: |
| boolean keepOrig = false; |
| for(OneSyn syn : matches) { |
| keepOrig |= syn.keepOrig; |
| } |
| |
| List<Integer> flatStates; |
| if (flatten) { |
| flatStates = new ArrayList<>(); |
| } else { |
| flatStates = null; |
| } |
| |
| if (keepOrig) { |
| // Add path for the original tokens |
| addSidePath(a, lastState, nextState, matches.get(0).in, flatStates); |
| } |
| |
| for(OneSyn syn : matches) { |
| addSidePath(a, lastState, nextState, syn.out, flatStates); |
| } |
| |
| i += matches.get(0).in.length; |
| } else { |
| a.addTransition(lastState, nextState, tokens[i].charAt(0)); |
| i++; |
| } |
| |
| lastState = nextState; |
| } |
| |
| a.setAccept(lastState, true); |
| |
| return topoSort(a.finish()); |
| } |
| |
| /** Just creates a side path from startState to endState with the provided tokens. */ |
| private static void addSidePath(Automaton.Builder a, int startState, int endState, char[] tokens, List<Integer> flatStates) { |
| int lastState = startState; |
| for(int i=0;i<tokens.length;i++) { |
| int nextState; |
| if (i == tokens.length-1) { |
| nextState = endState; |
| } else if (flatStates == null || i >= flatStates.size()) { |
| nextState = a.createState(); |
| if (flatStates != null) { |
| assert i == flatStates.size(); |
| flatStates.add(nextState); |
| } |
| } else { |
| nextState = flatStates.get(i); |
| } |
| a.addTransition(lastState, nextState, tokens[i]); |
| |
| lastState = nextState; |
| } |
| } |
| |
| private Automaton toAutomaton(TokenStream ts) throws IOException { |
| PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); |
| PositionLengthAttribute posLenAtt = ts.addAttribute(PositionLengthAttribute.class); |
| CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); |
| ts.reset(); |
| Automaton a = new Automaton(); |
| int srcNode = -1; |
| int destNode = -1; |
| int state = a.createState(); |
| while (ts.incrementToken()) { |
| assert termAtt.length() == 1; |
| char c = termAtt.charAt(0); |
| int posInc = posIncAtt.getPositionIncrement(); |
| if (posInc != 0) { |
| srcNode += posInc; |
| while (state < srcNode) { |
| state = a.createState(); |
| } |
| } |
| destNode = srcNode + posLenAtt.getPositionLength(); |
| while (state < destNode) { |
| state = a.createState(); |
| } |
| a.addTransition(srcNode, destNode, c); |
| } |
| ts.end(); |
| ts.close(); |
| a.finishState(); |
| a.setAccept(destNode, true); |
| return a; |
| } |
| |
| /* |
| private String toDot(TokenStream ts) throws IOException { |
| PositionIncrementAttribute posIncAtt = ts.addAttribute(PositionIncrementAttribute.class); |
| PositionLengthAttribute posLenAtt = ts.addAttribute(PositionLengthAttribute.class); |
| CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); |
| TypeAttribute typeAtt = ts.addAttribute(TypeAttribute.class); |
| ts.reset(); |
| int srcNode = -1; |
| int destNode = -1; |
| |
| StringBuilder b = new StringBuilder(); |
| b.append("digraph Automaton {\n"); |
| b.append(" rankdir = LR\n"); |
| b.append(" node [width=0.2, height=0.2, fontsize=8]\n"); |
| b.append(" initial [shape=plaintext,label=\"\"]\n"); |
| b.append(" initial -> 0\n"); |
| |
| while (ts.incrementToken()) { |
| int posInc = posIncAtt.getPositionIncrement(); |
| if (posInc != 0) { |
| srcNode += posInc; |
| b.append(" "); |
| b.append(srcNode); |
| b.append(" [shape=circle,label=\"" + srcNode + "\"]\n"); |
| } |
| destNode = srcNode + posLenAtt.getPositionLength(); |
| b.append(" "); |
| b.append(srcNode); |
| b.append(" -> "); |
| b.append(destNode); |
| b.append(" [label=\""); |
| b.append(termAtt); |
| b.append("\""); |
| if (typeAtt.type().equals("word") == false) { |
| b.append(" color=red"); |
| } |
| b.append("]\n"); |
| } |
| ts.end(); |
| ts.close(); |
| |
| b.append('}'); |
| return b.toString(); |
| } |
| */ |
| |
| /** Renumbers nodes according to their topo sort */ |
| private Automaton topoSort(Automaton in) { |
| int[] newToOld = Operations.topoSortStates(in); |
| int[] oldToNew = new int[newToOld.length]; |
| |
| Automaton.Builder a = new Automaton.Builder(); |
| //System.out.println("remap:"); |
| for(int i=0;i<newToOld.length;i++) { |
| a.createState(); |
| oldToNew[newToOld[i]] = i; |
| //System.out.println(" " + newToOld[i] + " -> " + i); |
| if (in.isAccept(newToOld[i])) { |
| a.setAccept(i, true); |
| //System.out.println(" **"); |
| } |
| } |
| |
| Transition t = new Transition(); |
| for(int i=0;i<newToOld.length;i++) { |
| int count = in.initTransition(newToOld[i], t); |
| for(int j=0;j<count;j++) { |
| in.getNextTransition(t); |
| a.addTransition(i, oldToNew[t.dest], t.min, t.max); |
| } |
| } |
| |
| return a.finish(); |
| } |
| |
| /** |
| * verify type of token and positionLengths on synonyms of different word counts. |
| */ |
| public void testPositionLengthAndType() throws Exception { |
| String testFile = |
| "spider man, spiderman\n" + |
| "usa,united states,u s a,united states of america"; |
| Analyzer analyzer = new MockAnalyzer(random()); |
| SolrSynonymParser parser = new SolrSynonymParser(true, true, analyzer); |
| |
| parser.parse(new StringReader(testFile)); |
| analyzer.close(); |
| |
| SynonymMap map = parser.build(); |
| analyzer = getFlattenAnalyzer(parser, true); |
| |
| BytesRef value = Util.get(map.fst, Util.toUTF32(new CharsRef("usa"), new IntsRefBuilder())); |
| ByteArrayDataInput bytesReader = new ByteArrayDataInput(value.bytes, value.offset, value.length); |
| final int code = bytesReader.readVInt(); |
| final int count = code >>> 1; |
| |
| final int[] synonymsIdxs = new int[count]; |
| for (int i = 0; i < count; i++) { |
| synonymsIdxs[i] = bytesReader.readVInt(); |
| } |
| |
| BytesRef scratchBytes = new BytesRef(); |
| map.words.get(synonymsIdxs[2], scratchBytes); |
| |
| int synonymLength = 1; |
| for (int i = scratchBytes.offset; i < scratchBytes.offset + scratchBytes.length; i++) { |
| if (scratchBytes.bytes[i] == SynonymMap.WORD_SEPARATOR) { |
| synonymLength++; |
| } |
| } |
| |
| assertEquals(count, 3); |
| assertEquals(synonymLength, 4); |
| |
| assertAnalyzesTo(analyzer, "spider man", |
| new String[]{"spiderman", "spider", "man"}, |
| new int[]{0, 0, 7}, |
| new int[]{10, 6, 10}, |
| new String[]{"SYNONYM", "word", "word"}, |
| new int[]{1, 0, 1}, |
| new int[]{2, 1, 1}); |
| |
| assertAnalyzesToPositions(analyzer, "amazing spider man", |
| new String[]{"amazing", "spiderman", "spider", "man"}, |
| new String[]{"word", "SYNONYM", "word", "word"}, |
| new int[]{1, 1, 0, 1}, |
| new int[]{1, 2, 1, 1}); |
| |
| // System.out.println(toDot(getAnalyzer(parser, true).tokenStream("field", new StringReader("the usa is wealthy")))); |
| |
| assertAnalyzesTo(analyzer, "the united states of america is wealthy", |
| new String[]{"the", "usa", "united", "u", "united", "states", "s", "states", "a", "of", "america", "is", "wealthy"}, |
| new int[] {0, 4, 4, 4, 4, 11, 11, 11, 18, 18, 21, 29, 32}, |
| new int[] {3, 28, 10, 10, 10, 28, 17, 17, 28, 20, 28, 31, 39}, |
| new String[]{"word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "word", "SYNONYM", "word", "word", "word", "word"}, |
| new int[] {1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1}, |
| new int[] {1, 4, 1, 1, 1, 3, 1, 1, 2, 1, 1, 1, 1}); |
| |
| assertAnalyzesToPositions(analyzer, "spiderman", |
| new String[]{"spider", "spiderman", "man"}, |
| new String[]{"SYNONYM", "word", "SYNONYM"}, |
| new int[]{1, 0, 1}, |
| new int[]{1, 2, 1}); |
| |
| assertAnalyzesTo(analyzer, "spiderman enemies", |
| new String[]{"spider", "spiderman", "man", "enemies"}, |
| new int[]{0, 0, 0, 10}, |
| new int[]{9, 9, 9, 17}, |
| new String[]{"SYNONYM", "word", "SYNONYM", "word"}, |
| new int[]{1, 0, 1, 1}, |
| new int[]{1, 2, 1, 1}); |
| |
| assertAnalyzesTo(analyzer, "the usa is wealthy", |
| new String[]{"the", "united", "u", "united", "usa", "states", "s", "states", "a", "of", "america", "is", "wealthy"}, |
| new int[] {0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 8, 11}, |
| new int[] {3, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 10, 18}, |
| new String[]{"word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "SYNONYM", "word", "word"}, |
| new int[] {1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1}, |
| new int[] {1, 1, 1, 1, 4, 3, 1, 1, 2, 1, 1, 1, 1}); |
| |
| assertGraphStrings(analyzer, "the usa is wealthy", new String[] { |
| "the usa is wealthy", |
| "the united states is wealthy", |
| "the u s a is wealthy", |
| "the united states of america is wealthy", |
| // Wrong. Here only due to "sausagization" of the multi word synonyms. |
| "the u states is wealthy", |
| "the u states a is wealthy", |
| "the u s of america is wealthy", |
| "the u states of america is wealthy", |
| "the united s a is wealthy", |
| "the united states a is wealthy", |
| "the united s of america is wealthy"}); |
| |
| assertAnalyzesTo(analyzer, "the united states is wealthy", |
| new String[]{"the", "usa", "u", "united", "united", "s", "states", "states", "a", "of", "america", "is", "wealthy"}, |
| new int[] {0, 4, 4, 4, 4, 11, 11, 11, 11, 11, 11, 18, 21}, |
| new int[] {3, 17, 10, 10, 10, 17, 17, 17, 17, 17, 17, 20, 28}, |
| new String[]{"word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "word"}, |
| new int[] {1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1}, |
| new int[] {1, 4, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1, 1}, |
| false); |
| |
| assertAnalyzesTo(analyzer, "the united states of balance", |
| new String[]{"the", "usa", "u", "united", "united", "s", "states", "states", "a", "of", "america", "of", "balance"}, |
| new int[] {0, 4, 4, 4, 4, 11, 11, 11, 11, 11, 11, 18, 21}, |
| new int[] {3, 17, 10, 10, 10, 17, 17, 17, 17, 17, 17, 20, 28}, |
| new String[]{"word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "word", "SYNONYM", "SYNONYM", "SYNONYM", "word", "word"}, |
| new int[] {1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1}, |
| new int[] {1, 4, 1, 1, 1, 1, 1, 3, 2, 1, 1, 1, 1}); |
| |
| analyzer.close(); |
| } |
| |
| public void testMultiwordOffsets() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = true; |
| add(b, "national hockey league", "nhl", keepOrig); |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "national hockey league", |
| new String[]{"nhl", "national", "hockey", "league"}, |
| new int[]{0, 0, 9, 16}, |
| new int[]{22, 8, 15, 22}, |
| new int[]{1, 0, 1, 1}); |
| a.close(); |
| } |
| |
| public void testIncludeOrig() throws Exception { |
| SynonymMap.Builder b = new SynonymMap.Builder(true); |
| final boolean keepOrig = true; |
| add(b, "a b", "ab", keepOrig); |
| add(b, "a c", "ac", keepOrig); |
| add(b, "a", "aa", keepOrig); |
| add(b, "b", "bb", keepOrig); |
| add(b, "z x c v", "zxcv", keepOrig); |
| add(b, "x c", "xc", keepOrig); |
| |
| Analyzer a = getFlattenAnalyzer(b, true); |
| |
| assertAnalyzesTo(a, "$", |
| new String[]{"$"}, |
| new int[]{1}); |
| assertAnalyzesTo(a, "a", |
| new String[]{"aa", "a"}, |
| new int[]{1, 0}); |
| assertAnalyzesTo(a, "a", |
| new String[]{"aa", "a"}, |
| new int[]{1, 0}); |
| assertAnalyzesTo(a, "$ a", |
| new String[]{"$", "aa", "a"}, |
| new int[]{1, 1, 0}); |
| assertAnalyzesTo(a, "a $", |
| new String[]{"aa", "a", "$"}, |
| new int[]{1, 0, 1}); |
| assertAnalyzesTo(a, "$ a !", |
| new String[]{"$", "aa", "a", "!"}, |
| new int[]{1, 1, 0, 1}); |
| assertAnalyzesTo(a, "a a", |
| new String[]{"aa", "a", "aa", "a"}, |
| new int[]{1, 0, 1, 0}); |
| assertAnalyzesTo(a, "b", |
| new String[]{"bb", "b"}, |
| new int[]{1, 0}); |
| assertAnalyzesTo(a, "z x c v", |
| new String[]{"zxcv", "z", "x", "c", "v"}, |
| new int[]{1, 0, 1, 1, 1}); |
| assertAnalyzesTo(a, "z x c $", |
| new String[]{"z", "xc", "x", "c", "$"}, |
| new int[]{1, 1, 0, 1, 1}); |
| a.close(); |
| } |
| |
| public void testUpperCase() throws IOException { |
| assertMapping("word", "synonym"); |
| assertMapping("word".toUpperCase(Locale.ROOT), "synonym"); |
| } |
| |
| private void assertMapping(String inputString, String outputString) throws IOException { |
| SynonymMap.Builder builder = new SynonymMap.Builder(false); |
| // the rules must be lowercased up front, but the incoming tokens will be case insensitive: |
| CharsRef input = SynonymMap.Builder.join(inputString.toLowerCase(Locale.ROOT).split(" "), new CharsRefBuilder()); |
| CharsRef output = SynonymMap.Builder.join(outputString.split(" "), new CharsRefBuilder()); |
| builder.add(input, output, true); |
| Analyzer analyzer = new CustomAnalyzer(builder.build()); |
| TokenStream tokenStream = analyzer.tokenStream("field", inputString); |
| assertTokenStreamContents(tokenStream, new String[]{ |
| outputString, inputString |
| }); |
| } |
| |
| static class CustomAnalyzer extends Analyzer { |
| private SynonymMap synonymMap; |
| |
| CustomAnalyzer(SynonymMap synonymMap) { |
| this.synonymMap = synonymMap; |
| } |
| |
| @Override |
| protected TokenStreamComponents createComponents(String s) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| TokenStream tokenStream = new SynonymGraphFilter(tokenizer, synonymMap, true); // Ignore case True |
| return new TokenStreamComponents(tokenizer, tokenStream); |
| } |
| } |
| } |
