| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| package org.apache.lucene.analysis.shingle; |
| |
| import java.io.IOException; |
| import java.io.StringReader; |
| import java.util.Random; |
| import org.apache.lucene.analysis.Analyzer; |
| import org.apache.lucene.analysis.BaseTokenStreamTestCase; |
| import org.apache.lucene.analysis.CannedTokenStream; |
| import org.apache.lucene.analysis.MockTokenizer; |
| import org.apache.lucene.analysis.Token; |
| import org.apache.lucene.analysis.TokenStream; |
| import org.apache.lucene.analysis.Tokenizer; |
| import org.apache.lucene.analysis.core.KeywordTokenizer; |
| import org.apache.lucene.analysis.core.WhitespaceTokenizer; |
| import org.apache.lucene.analysis.tokenattributes.TypeAttribute; |
| |
| public class TestShingleFilter extends BaseTokenStreamTestCase { |
| |
| public static final Token[] TEST_TOKEN = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("divide", 7, 13), |
| createToken("this", 14, 18), |
| createToken("sentence", 19, 27), |
| createToken("into", 28, 32), |
| createToken("shingles", 33, 39), |
| }; |
| |
| public static final int[] UNIGRAM_ONLY_POSITION_INCREMENTS = new int[] {1, 1, 1, 1, 1, 1}; |
| |
| public static final String[] UNIGRAM_ONLY_TYPES = |
| new String[] {"word", "word", "word", "word", "word", "word"}; |
| |
| public static Token[] testTokenWithHoles; |
| |
| public static final Token[] BI_GRAM_TOKENS = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide", 0, 13), |
| createToken("divide", 7, 13), |
| createToken("divide this", 7, 18), |
| createToken("this", 14, 18), |
| createToken("this sentence", 14, 27), |
| createToken("sentence", 19, 27), |
| createToken("sentence into", 19, 32), |
| createToken("into", 28, 32), |
| createToken("into shingles", 28, 39), |
| createToken("shingles", 33, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] BI_GRAM_TYPES = |
| new String[] { |
| "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", |
| "shingle", "word" |
| }; |
| |
| public static final Token[] BI_GRAM_TOKENS_WITH_HOLES = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide", 0, 13), |
| createToken("divide", 7, 13), |
| createToken("divide _", 7, 19), |
| createToken("_ sentence", 19, 27), |
| createToken("sentence", 19, 27), |
| createToken("sentence _", 19, 33), |
| createToken("_ shingles", 33, 39), |
| createToken("shingles", 33, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES = |
| new int[] {1, 0, 1, 0, 1, 1, 0, 1, 1}; |
| |
| private static final String[] BI_GRAM_TYPES_WITH_HOLES = { |
| "word", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", "word" |
| }; |
| |
| public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS = |
| new Token[] { |
| createToken("please divide", 0, 13), |
| createToken("divide this", 7, 18), |
| createToken("this sentence", 14, 27), |
| createToken("sentence into", 19, 32), |
| createToken("into shingles", 28, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = |
| new int[] {1, 1, 1, 1, 1}; |
| |
| public static final String[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS = |
| new String[] {"shingle", "shingle", "shingle", "shingle", "shingle"}; |
| |
| public static final Token[] BI_GRAM_TOKENS_WITH_HOLES_WITHOUT_UNIGRAMS = |
| new Token[] { |
| createToken("please divide", 0, 13), |
| createToken("divide _", 7, 19), |
| createToken("_ sentence", 19, 27), |
| createToken("sentence _", 19, 33), |
| createToken("_ shingles", 33, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES_WITHOUT_UNIGRAMS = |
| new int[] {1, 1, 1, 1, 1, 1}; |
| |
| public static final Token[] TEST_SINGLE_TOKEN = new Token[] {createToken("please", 0, 6)}; |
| |
| public static final Token[] SINGLE_TOKEN = new Token[] {createToken("please", 0, 6)}; |
| |
| public static final int[] SINGLE_TOKEN_INCREMENTS = new int[] {1}; |
| |
| public static final String[] SINGLE_TOKEN_TYPES = new String[] {"word"}; |
| |
| public static final Token[] EMPTY_TOKEN_ARRAY = new Token[] {}; |
| |
| public static final int[] EMPTY_TOKEN_INCREMENTS_ARRAY = new int[] {}; |
| |
| public static final String[] EMPTY_TOKEN_TYPES_ARRAY = new String[] {}; |
| |
| public static final Token[] TRI_GRAM_TOKENS = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide", 0, 13), |
| createToken("please divide this", 0, 18), |
| createToken("divide", 7, 13), |
| createToken("divide this", 7, 18), |
| createToken("divide this sentence", 7, 27), |
| createToken("this", 14, 18), |
| createToken("this sentence", 14, 27), |
| createToken("this sentence into", 14, 32), |
| createToken("sentence", 19, 27), |
| createToken("sentence into", 19, 32), |
| createToken("sentence into shingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("into shingles", 28, 39), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES = |
| new String[] { |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", |
| "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS = |
| new Token[] { |
| createToken("please divide", 0, 13), |
| createToken("please divide this", 0, 18), |
| createToken("divide this", 7, 18), |
| createToken("divide this sentence", 7, 27), |
| createToken("this sentence", 14, 27), |
| createToken("this sentence into", 14, 32), |
| createToken("sentence into", 19, 32), |
| createToken("sentence into shingles", 19, 39), |
| createToken("into shingles", 28, 39), |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS = |
| new String[] { |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", |
| }; |
| |
| public static final Token[] FOUR_GRAM_TOKENS = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide", 0, 13), |
| createToken("please divide this", 0, 18), |
| createToken("please divide this sentence", 0, 27), |
| createToken("divide", 7, 13), |
| createToken("divide this", 7, 18), |
| createToken("divide this sentence", 7, 27), |
| createToken("divide this sentence into", 7, 32), |
| createToken("this", 14, 18), |
| createToken("this sentence", 14, 27), |
| createToken("this sentence into", 14, 32), |
| createToken("this sentence into shingles", 14, 39), |
| createToken("sentence", 19, 27), |
| createToken("sentence into", 19, 32), |
| createToken("sentence into shingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("into shingles", 28, 39), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] FOUR_GRAM_POSITION_INCREMENTS = |
| new int[] {1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] FOUR_GRAM_TYPES = |
| new String[] { |
| "word", "shingle", "shingle", "shingle", "word", "shingle", "shingle", "shingle", "word", |
| "shingle", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS = |
| new Token[] { |
| createToken("please divide", 0, 13), |
| createToken("please divide this", 0, 18), |
| createToken("please divide this sentence", 0, 27), |
| createToken("divide this", 7, 18), |
| createToken("divide this sentence", 7, 27), |
| createToken("divide this sentence into", 7, 32), |
| createToken("this sentence", 14, 27), |
| createToken("this sentence into", 14, 32), |
| createToken("this sentence into shingles", 14, 39), |
| createToken("sentence into", 19, 32), |
| createToken("sentence into shingles", 19, 39), |
| createToken("into shingles", 28, 39), |
| }; |
| |
| public static final int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS = |
| new String[] { |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_MIN_TRI_GRAM = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide this", 0, 18), |
| createToken("divide", 7, 13), |
| createToken("divide this sentence", 7, 27), |
| createToken("this", 14, 18), |
| createToken("this sentence into", 14, 32), |
| createToken("sentence", 19, 27), |
| createToken("sentence into shingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_MIN_TRI_GRAM = |
| new String[] { |
| "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", "word" |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = |
| new Token[] { |
| createToken("please divide this", 0, 18), |
| createToken("divide this sentence", 7, 27), |
| createToken("this sentence into", 14, 32), |
| createToken("sentence into shingles", 19, 39) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = |
| new int[] {1, 1, 1, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = |
| new String[] {"shingle", "shingle", "shingle", "shingle"}; |
| |
| public static final Token[] FOUR_GRAM_TOKENS_MIN_TRI_GRAM = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide this", 0, 18), |
| createToken("please divide this sentence", 0, 27), |
| createToken("divide", 7, 13), |
| createToken("divide this sentence", 7, 27), |
| createToken("divide this sentence into", 7, 32), |
| createToken("this", 14, 18), |
| createToken("this sentence into", 14, 32), |
| createToken("this sentence into shingles", 14, 39), |
| createToken("sentence", 19, 27), |
| createToken("sentence into shingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] FOUR_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1}; |
| |
| public static final String[] FOUR_GRAM_TYPES_MIN_TRI_GRAM = |
| new String[] { |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", |
| "word", "shingle", "word", "word" |
| }; |
| |
| public static final Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = |
| new Token[] { |
| createToken("please divide this", 0, 18), |
| createToken("please divide this sentence", 0, 27), |
| createToken("divide this sentence", 7, 27), |
| createToken("divide this sentence into", 7, 32), |
| createToken("this sentence into", 14, 32), |
| createToken("this sentence into shingles", 14, 39), |
| createToken("sentence into shingles", 19, 39), |
| }; |
| |
| public static final int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = |
| new int[] {1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = |
| new String[] { |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle" |
| }; |
| |
| public static final Token[] FOUR_GRAM_TOKENS_MIN_FOUR_GRAM = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide this sentence", 0, 27), |
| createToken("divide", 7, 13), |
| createToken("divide this sentence into", 7, 32), |
| createToken("this", 14, 18), |
| createToken("this sentence into shingles", 14, 39), |
| createToken("sentence", 19, 27), |
| createToken("into", 28, 32), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] FOUR_GRAM_POSITION_INCREMENTS_MIN_FOUR_GRAM = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 1, 1}; |
| |
| public static final String[] FOUR_GRAM_TYPES_MIN_FOUR_GRAM = |
| new String[] { |
| "word", "shingle", "word", "shingle", "word", "shingle", "word", "word", "word" |
| }; |
| |
| public static final Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = |
| new Token[] { |
| createToken("please divide this sentence", 0, 27), |
| createToken("divide this sentence into", 7, 32), |
| createToken("this sentence into shingles", 14, 39), |
| }; |
| |
| public static final int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = |
| new int[] {1, 1, 1}; |
| |
| public static final String[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = |
| new String[] {"shingle", "shingle", "shingle"}; |
| |
| public static final Token[] BI_GRAM_TOKENS_NO_SEPARATOR = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("pleasedivide", 0, 13), |
| createToken("divide", 7, 13), |
| createToken("dividethis", 7, 18), |
| createToken("this", 14, 18), |
| createToken("thissentence", 14, 27), |
| createToken("sentence", 19, 27), |
| createToken("sentenceinto", 19, 32), |
| createToken("into", 28, 32), |
| createToken("intoshingles", 28, 39), |
| createToken("shingles", 33, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] BI_GRAM_TYPES_NO_SEPARATOR = |
| new String[] { |
| "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", |
| "shingle", "word" |
| }; |
| |
| public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR = |
| new Token[] { |
| createToken("pleasedivide", 0, 13), |
| createToken("dividethis", 7, 18), |
| createToken("thissentence", 14, 27), |
| createToken("sentenceinto", 19, 32), |
| createToken("intoshingles", 28, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR = |
| new int[] {1, 1, 1, 1, 1}; |
| |
| public static final String[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR = |
| new String[] {"shingle", "shingle", "shingle", "shingle", "shingle"}; |
| |
| public static final Token[] TRI_GRAM_TOKENS_NO_SEPARATOR = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("pleasedivide", 0, 13), |
| createToken("pleasedividethis", 0, 18), |
| createToken("divide", 7, 13), |
| createToken("dividethis", 7, 18), |
| createToken("dividethissentence", 7, 27), |
| createToken("this", 14, 18), |
| createToken("thissentence", 14, 27), |
| createToken("thissentenceinto", 14, 32), |
| createToken("sentence", 19, 27), |
| createToken("sentenceinto", 19, 32), |
| createToken("sentenceintoshingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("intoshingles", 28, 39), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_NO_SEPARATOR = |
| new String[] { |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", |
| "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR = |
| new Token[] { |
| createToken("pleasedivide", 0, 13), |
| createToken("pleasedividethis", 0, 18), |
| createToken("dividethis", 7, 18), |
| createToken("dividethissentence", 7, 27), |
| createToken("thissentence", 14, 27), |
| createToken("thissentenceinto", 14, 32), |
| createToken("sentenceinto", 19, 32), |
| createToken("sentenceintoshingles", 19, 39), |
| createToken("intoshingles", 28, 39), |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR = |
| new String[] { |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", |
| }; |
| |
| public static final Token[] BI_GRAM_TOKENS_ALT_SEPARATOR = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please<SEP>divide", 0, 13), |
| createToken("divide", 7, 13), |
| createToken("divide<SEP>this", 7, 18), |
| createToken("this", 14, 18), |
| createToken("this<SEP>sentence", 14, 27), |
| createToken("sentence", 19, 27), |
| createToken("sentence<SEP>into", 19, 32), |
| createToken("into", 28, 32), |
| createToken("into<SEP>shingles", 28, 39), |
| createToken("shingles", 33, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] BI_GRAM_TYPES_ALT_SEPARATOR = |
| new String[] { |
| "word", "shingle", "word", "shingle", "word", "shingle", "word", "shingle", "word", |
| "shingle", "word" |
| }; |
| |
| public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = |
| new Token[] { |
| createToken("please<SEP>divide", 0, 13), |
| createToken("divide<SEP>this", 7, 18), |
| createToken("this<SEP>sentence", 14, 27), |
| createToken("sentence<SEP>into", 19, 32), |
| createToken("into<SEP>shingles", 28, 39), |
| }; |
| |
| public static final int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = |
| new int[] {1, 1, 1, 1, 1}; |
| |
| public static final String[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR = |
| new String[] {"shingle", "shingle", "shingle", "shingle", "shingle"}; |
| |
| public static final Token[] TRI_GRAM_TOKENS_ALT_SEPARATOR = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please<SEP>divide", 0, 13), |
| createToken("please<SEP>divide<SEP>this", 0, 18), |
| createToken("divide", 7, 13), |
| createToken("divide<SEP>this", 7, 18), |
| createToken("divide<SEP>this<SEP>sentence", 7, 27), |
| createToken("this", 14, 18), |
| createToken("this<SEP>sentence", 14, 27), |
| createToken("this<SEP>sentence<SEP>into", 14, 32), |
| createToken("sentence", 19, 27), |
| createToken("sentence<SEP>into", 19, 32), |
| createToken("sentence<SEP>into<SEP>shingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("into<SEP>shingles", 28, 39), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_ALT_SEPARATOR = |
| new String[] { |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", |
| "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = |
| new Token[] { |
| createToken("please<SEP>divide", 0, 13), |
| createToken("please<SEP>divide<SEP>this", 0, 18), |
| createToken("divide<SEP>this", 7, 18), |
| createToken("divide<SEP>this<SEP>sentence", 7, 27), |
| createToken("this<SEP>sentence", 14, 27), |
| createToken("this<SEP>sentence<SEP>into", 14, 32), |
| createToken("sentence<SEP>into", 19, 32), |
| createToken("sentence<SEP>into<SEP>shingles", 19, 39), |
| createToken("into<SEP>shingles", 28, 39), |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR = |
| new String[] { |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", "shingle", |
| "shingle", |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_NULL_SEPARATOR = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("pleasedivide", 0, 13), |
| createToken("pleasedividethis", 0, 18), |
| createToken("divide", 7, 13), |
| createToken("dividethis", 7, 18), |
| createToken("dividethissentence", 7, 27), |
| createToken("this", 14, 18), |
| createToken("thissentence", 14, 27), |
| createToken("thissentenceinto", 14, 32), |
| createToken("sentence", 19, 27), |
| createToken("sentenceinto", 19, 32), |
| createToken("sentenceintoshingles", 19, 39), |
| createToken("into", 28, 32), |
| createToken("intoshingles", 28, 39), |
| createToken("shingles", 33, 39) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_NULL_SEPARATOR = |
| new String[] { |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", |
| "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] TEST_TOKEN_POS_INCR_EQUAL_TO_N = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("divide", 7, 13), |
| createToken("this", 14, 18), |
| createToken("sentence", 29, 37, 3), |
| createToken("into", 38, 42), |
| createToken("shingles", 43, 49), |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please divide", 0, 13), |
| createToken("please divide this", 0, 18), |
| createToken("divide", 7, 13), |
| createToken("divide this", 7, 18), |
| createToken("divide this _", 7, 29), |
| createToken("this", 14, 18), |
| createToken("this _", 14, 29), |
| createToken("this _ _", 14, 29), |
| createToken("_ _ sentence", 29, 37), |
| createToken("_ sentence", 29, 37), |
| createToken("_ sentence into", 29, 42), |
| createToken("sentence", 29, 37), |
| createToken("sentence into", 29, 42), |
| createToken("sentence into shingles", 29, 49), |
| createToken("into", 38, 42), |
| createToken("into shingles", 38, 49), |
| createToken("shingles", 43, 49) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N = |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N = |
| new String[] { |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "shingle", |
| "shingle", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = |
| new Token[] { |
| createToken("please divide", 0, 13), |
| createToken("please divide this", 0, 18), |
| createToken("divide this", 7, 18), |
| createToken("divide this _", 7, 29), |
| createToken("this _", 14, 29), |
| createToken("this _ _", 14, 29), |
| createToken("_ _ sentence", 29, 37), |
| createToken("_ sentence", 29, 37), |
| createToken("_ sentence into", 29, 42), |
| createToken("sentence into", 29, 42), |
| createToken("sentence into shingles", 29, 49), |
| createToken("into shingles", 38, 49), |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = |
| new int[] {1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = |
| new String[] { |
| "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", |
| "shingle", "shingle", "shingle", "shingle", |
| }; |
| |
| public static final Token[] TEST_TOKEN_POS_INCR_GREATER_THAN_N = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("divide", 57, 63, 8), |
| createToken("this", 64, 68), |
| createToken("sentence", 69, 77), |
| createToken("into", 78, 82), |
| createToken("shingles", 83, 89), |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("please _", 0, 57), |
| createToken("please _ _", 0, 57), |
| createToken("_ _ divide", 57, 63), |
| createToken("_ divide", 57, 63), |
| createToken("_ divide this", 57, 68), |
| createToken("divide", 57, 63), |
| createToken("divide this", 57, 68), |
| createToken("divide this sentence", 57, 77), |
| createToken("this", 64, 68), |
| createToken("this sentence", 64, 77), |
| createToken("this sentence into", 64, 82), |
| createToken("sentence", 69, 77), |
| createToken("sentence into", 69, 82), |
| createToken("sentence into shingles", 69, 89), |
| createToken("into", 78, 82), |
| createToken("into shingles", 78, 89), |
| createToken("shingles", 83, 89) |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N = |
| new int[] {1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}; |
| public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N = |
| new String[] { |
| "word", "shingle", "shingle", "shingle", "shingle", "shingle", "word", "shingle", "shingle", |
| "word", "shingle", "shingle", "word", "shingle", "shingle", "word", "shingle", "word" |
| }; |
| |
| public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = |
| new Token[] { |
| createToken("please _", 0, 57), |
| createToken("please _ _", 0, 57), |
| createToken("_ _ divide", 57, 63), |
| createToken("_ divide", 57, 63), |
| createToken("_ divide this", 57, 68), |
| createToken("divide this", 57, 68), |
| createToken("divide this sentence", 57, 77), |
| createToken("this sentence", 64, 77), |
| createToken("this sentence into", 64, 82), |
| createToken("sentence into", 69, 82), |
| createToken("sentence into shingles", 69, 89), |
| createToken("into shingles", 78, 89), |
| }; |
| |
| public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = |
| new int[] {1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1}; |
| |
| public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = |
| new String[] { |
| "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", "shingle", |
| "shingle", "shingle", "shingle", "shingle", |
| }; |
| |
| @Override |
| public void setUp() throws Exception { |
| super.setUp(); |
| testTokenWithHoles = |
| new Token[] { |
| createToken("please", 0, 6), |
| createToken("divide", 7, 13), |
| createToken("sentence", 19, 27, 2), |
| createToken("shingles", 33, 39, 2), |
| }; |
| } |
| |
| /* |
| * Class under test for void ShingleFilter(TokenStream, int) |
| */ |
| public void testBiGramFilter() throws IOException { |
| this.shingleFilterTest( |
| 2, TEST_TOKEN, BI_GRAM_TOKENS, BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES, true); |
| } |
| |
| public void testBiGramFilterWithHoles() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| testTokenWithHoles, |
| BI_GRAM_TOKENS_WITH_HOLES, |
| BI_GRAM_POSITION_INCREMENTS_WITH_HOLES, |
| BI_GRAM_TYPES_WITH_HOLES, |
| true); |
| } |
| |
| public void testBiGramFilterWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| TEST_TOKEN, |
| BI_GRAM_TOKENS_WITHOUT_UNIGRAMS, |
| BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, |
| BI_GRAM_TYPES_WITHOUT_UNIGRAMS, |
| false); |
| } |
| |
| public void testBiGramFilterWithHolesWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| testTokenWithHoles, |
| BI_GRAM_TOKENS_WITH_HOLES_WITHOUT_UNIGRAMS, |
| BI_GRAM_POSITION_INCREMENTS_WITH_HOLES_WITHOUT_UNIGRAMS, |
| BI_GRAM_TYPES_WITHOUT_UNIGRAMS, |
| false); |
| } |
| |
| public void testBiGramFilterWithSingleToken() throws IOException { |
| this.shingleFilterTest( |
| 2, TEST_SINGLE_TOKEN, SINGLE_TOKEN, SINGLE_TOKEN_INCREMENTS, SINGLE_TOKEN_TYPES, true); |
| } |
| |
| public void testBiGramFilterWithSingleTokenWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| TEST_SINGLE_TOKEN, |
| EMPTY_TOKEN_ARRAY, |
| EMPTY_TOKEN_INCREMENTS_ARRAY, |
| EMPTY_TOKEN_TYPES_ARRAY, |
| false); |
| } |
| |
| public void testBiGramFilterWithEmptyTokenStream() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| EMPTY_TOKEN_ARRAY, |
| EMPTY_TOKEN_ARRAY, |
| EMPTY_TOKEN_INCREMENTS_ARRAY, |
| EMPTY_TOKEN_TYPES_ARRAY, |
| true); |
| } |
| |
| public void testBiGramFilterWithEmptyTokenStreamWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| EMPTY_TOKEN_ARRAY, |
| EMPTY_TOKEN_ARRAY, |
| EMPTY_TOKEN_INCREMENTS_ARRAY, |
| EMPTY_TOKEN_TYPES_ARRAY, |
| false); |
| } |
| |
| public void testTriGramFilter() throws IOException { |
| this.shingleFilterTest( |
| 3, TEST_TOKEN, TRI_GRAM_TOKENS, TRI_GRAM_POSITION_INCREMENTS, TRI_GRAM_TYPES, true); |
| } |
| |
| public void testTriGramFilterWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS, |
| TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, |
| TRI_GRAM_TYPES_WITHOUT_UNIGRAMS, |
| false); |
| } |
| |
| public void testFourGramFilter() throws IOException { |
| this.shingleFilterTest( |
| 4, TEST_TOKEN, FOUR_GRAM_TOKENS, FOUR_GRAM_POSITION_INCREMENTS, FOUR_GRAM_TYPES, true); |
| } |
| |
| public void testFourGramFilterWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 4, |
| TEST_TOKEN, |
| FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS, |
| FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, |
| FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS, |
| false); |
| } |
| |
| public void testTriGramFilterMinTriGram() throws IOException { |
| this.shingleFilterTest( |
| 3, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_MIN_TRI_GRAM, |
| TRI_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM, |
| TRI_GRAM_TYPES_MIN_TRI_GRAM, |
| true); |
| } |
| |
| public void testTriGramFilterWithoutUnigramsMinTriGram() throws IOException { |
| this.shingleFilterTest( |
| 3, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, |
| TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, |
| TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, |
| false); |
| } |
| |
| public void testFourGramFilterMinTriGram() throws IOException { |
| this.shingleFilterTest( |
| 3, |
| 4, |
| TEST_TOKEN, |
| FOUR_GRAM_TOKENS_MIN_TRI_GRAM, |
| FOUR_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM, |
| FOUR_GRAM_TYPES_MIN_TRI_GRAM, |
| true); |
| } |
| |
| public void testFourGramFilterWithoutUnigramsMinTriGram() throws IOException { |
| this.shingleFilterTest( |
| 3, |
| 4, |
| TEST_TOKEN, |
| FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, |
| FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, |
| FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, |
| false); |
| } |
| |
| public void testFourGramFilterMinFourGram() throws IOException { |
| this.shingleFilterTest( |
| 4, |
| 4, |
| TEST_TOKEN, |
| FOUR_GRAM_TOKENS_MIN_FOUR_GRAM, |
| FOUR_GRAM_POSITION_INCREMENTS_MIN_FOUR_GRAM, |
| FOUR_GRAM_TYPES_MIN_FOUR_GRAM, |
| true); |
| } |
| |
| public void testFourGramFilterWithoutUnigramsMinFourGram() throws IOException { |
| this.shingleFilterTest( |
| 4, |
| 4, |
| TEST_TOKEN, |
| FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, |
| FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, |
| FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, |
| false); |
| } |
| |
| public void testBiGramFilterNoSeparator() throws IOException { |
| this.shingleFilterTest( |
| "", |
| 2, |
| 2, |
| TEST_TOKEN, |
| BI_GRAM_TOKENS_NO_SEPARATOR, |
| BI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR, |
| BI_GRAM_TYPES_NO_SEPARATOR, |
| true); |
| } |
| |
| public void testBiGramFilterWithoutUnigramsNoSeparator() throws IOException { |
| this.shingleFilterTest( |
| "", |
| 2, |
| 2, |
| TEST_TOKEN, |
| BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR, |
| BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR, |
| BI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR, |
| false); |
| } |
| |
| public void testTriGramFilterNoSeparator() throws IOException { |
| this.shingleFilterTest( |
| "", |
| 2, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_NO_SEPARATOR, |
| TRI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR, |
| TRI_GRAM_TYPES_NO_SEPARATOR, |
| true); |
| } |
| |
| public void testTriGramFilterWithoutUnigramsNoSeparator() throws IOException { |
| this.shingleFilterTest( |
| "", |
| 2, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR, |
| TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR, |
| TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR, |
| false); |
| } |
| |
| public void testBiGramFilterAltSeparator() throws IOException { |
| this.shingleFilterTest( |
| "<SEP>", |
| 2, |
| 2, |
| TEST_TOKEN, |
| BI_GRAM_TOKENS_ALT_SEPARATOR, |
| BI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR, |
| BI_GRAM_TYPES_ALT_SEPARATOR, |
| true); |
| } |
| |
| public void testBiGramFilterWithoutUnigramsAltSeparator() throws IOException { |
| this.shingleFilterTest( |
| "<SEP>", |
| 2, |
| 2, |
| TEST_TOKEN, |
| BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, |
| BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, |
| BI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR, |
| false); |
| } |
| |
| public void testTriGramFilterAltSeparator() throws IOException { |
| this.shingleFilterTest( |
| "<SEP>", |
| 2, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_ALT_SEPARATOR, |
| TRI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR, |
| TRI_GRAM_TYPES_ALT_SEPARATOR, |
| true); |
| } |
| |
| public void testTriGramFilterWithoutUnigramsAltSeparator() throws IOException { |
| this.shingleFilterTest( |
| "<SEP>", |
| 2, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, |
| TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR, |
| TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR, |
| false); |
| } |
| |
| public void testTriGramFilterNullSeparator() throws IOException { |
| this.shingleFilterTest( |
| null, |
| 2, |
| 3, |
| TEST_TOKEN, |
| TRI_GRAM_TOKENS_NULL_SEPARATOR, |
| TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR, |
| TRI_GRAM_TYPES_NULL_SEPARATOR, |
| true); |
| } |
| |
| public void testPositionIncrementEqualToN() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| 3, |
| TEST_TOKEN_POS_INCR_EQUAL_TO_N, |
| TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N, |
| TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N, |
| TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N, |
| true); |
| } |
| |
| public void testPositionIncrementEqualToNWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| 3, |
| TEST_TOKEN_POS_INCR_EQUAL_TO_N, |
| TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, |
| TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, |
| TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, |
| false); |
| } |
| |
| public void testPositionIncrementGreaterThanN() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| 3, |
| TEST_TOKEN_POS_INCR_GREATER_THAN_N, |
| TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N, |
| TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N, |
| TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N, |
| true); |
| } |
| |
| public void testPositionIncrementGreaterThanNWithoutUnigrams() throws IOException { |
| this.shingleFilterTest( |
| 2, |
| 3, |
| TEST_TOKEN_POS_INCR_GREATER_THAN_N, |
| TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, |
| TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, |
| TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, |
| false); |
| } |
| |
| public void testReset() throws Exception { |
| Tokenizer wsTokenizer = new WhitespaceTokenizer(); |
| wsTokenizer.setReader(new StringReader("please divide this sentence")); |
| TokenStream filter = new ShingleFilter(wsTokenizer, 2); |
| assertTokenStreamContents( |
| filter, |
| new String[] { |
| "please", "please divide", "divide", "divide this", "this", "this sentence", "sentence" |
| }, |
| new int[] {0, 0, 7, 7, 14, 14, 19}, |
| new int[] {6, 13, 13, 18, 18, 27, 27}, |
| new String[] { |
| TypeAttribute.DEFAULT_TYPE, |
| "shingle", |
| TypeAttribute.DEFAULT_TYPE, |
| "shingle", |
| TypeAttribute.DEFAULT_TYPE, |
| "shingle", |
| TypeAttribute.DEFAULT_TYPE |
| }, |
| new int[] {1, 0, 1, 0, 1, 0, 1}); |
| wsTokenizer.setReader(new StringReader("please divide this sentence")); |
| assertTokenStreamContents( |
| filter, |
| new String[] { |
| "please", "please divide", "divide", "divide this", "this", "this sentence", "sentence" |
| }, |
| new int[] {0, 0, 7, 7, 14, 14, 19}, |
| new int[] {6, 13, 13, 18, 18, 27, 27}, |
| new String[] { |
| TypeAttribute.DEFAULT_TYPE, |
| "shingle", |
| TypeAttribute.DEFAULT_TYPE, |
| "shingle", |
| TypeAttribute.DEFAULT_TYPE, |
| "shingle", |
| TypeAttribute.DEFAULT_TYPE |
| }, |
| new int[] {1, 0, 1, 0, 1, 0, 1}); |
| } |
| |
| public void testOutputUnigramsIfNoShinglesSingleTokenCase() throws IOException { |
| // Single token input with outputUnigrams==false is the primary case where |
| // enabling this option should alter program behavior. |
| this.shingleFilterTest( |
| 2, |
| 2, |
| TEST_SINGLE_TOKEN, |
| SINGLE_TOKEN, |
| SINGLE_TOKEN_INCREMENTS, |
| SINGLE_TOKEN_TYPES, |
| false, |
| true); |
| } |
| |
| public void testOutputUnigramsIfNoShinglesWithSimpleBigram() throws IOException { |
| // Here we expect the same result as with testBiGramFilter(). |
| this.shingleFilterTest( |
| 2, 2, TEST_TOKEN, BI_GRAM_TOKENS, BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES, true, true); |
| } |
| |
| public void testOutputUnigramsIfNoShinglesWithSimpleUnigramlessBigram() throws IOException { |
| // Here we expect the same result as with testBiGramFilterWithoutUnigrams(). |
| this.shingleFilterTest( |
| 2, |
| 2, |
| TEST_TOKEN, |
| BI_GRAM_TOKENS_WITHOUT_UNIGRAMS, |
| BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, |
| BI_GRAM_TYPES_WITHOUT_UNIGRAMS, |
| false, |
| true); |
| } |
| |
| public void testOutputUnigramsIfNoShinglesWithMultipleInputTokens() throws IOException { |
| // Test when the minimum shingle size is greater than the number of input tokens |
| this.shingleFilterTest( |
| 7, |
| 7, |
| TEST_TOKEN, |
| TEST_TOKEN, |
| UNIGRAM_ONLY_POSITION_INCREMENTS, |
| UNIGRAM_ONLY_TYPES, |
| false, |
| true); |
| } |
| |
| protected void shingleFilterTest( |
| int maxSize, |
| Token[] tokensToShingle, |
| Token[] tokensToCompare, |
| int[] positionIncrements, |
| String[] types, |
| boolean outputUnigrams) |
| throws IOException { |
| |
| ShingleFilter filter = new ShingleFilter(new CannedTokenStream(tokensToShingle), maxSize); |
| filter.setOutputUnigrams(outputUnigrams); |
| shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types); |
| } |
| |
| protected void shingleFilterTest( |
| int minSize, |
| int maxSize, |
| Token[] tokensToShingle, |
| Token[] tokensToCompare, |
| int[] positionIncrements, |
| String[] types, |
| boolean outputUnigrams) |
| throws IOException { |
| ShingleFilter filter = |
| new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize); |
| filter.setOutputUnigrams(outputUnigrams); |
| shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types); |
| } |
| |
| protected void shingleFilterTest( |
| int minSize, |
| int maxSize, |
| Token[] tokensToShingle, |
| Token[] tokensToCompare, |
| int[] positionIncrements, |
| String[] types, |
| boolean outputUnigrams, |
| boolean outputUnigramsIfNoShingles) |
| throws IOException { |
| ShingleFilter filter = |
| new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize); |
| filter.setOutputUnigrams(outputUnigrams); |
| filter.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles); |
| shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types); |
| } |
| |
| protected void shingleFilterTest( |
| String tokenSeparator, |
| int minSize, |
| int maxSize, |
| Token[] tokensToShingle, |
| Token[] tokensToCompare, |
| int[] positionIncrements, |
| String[] types, |
| boolean outputUnigrams) |
| throws IOException { |
| ShingleFilter filter = |
| new ShingleFilter(new CannedTokenStream(tokensToShingle), minSize, maxSize); |
| filter.setTokenSeparator(tokenSeparator); |
| filter.setOutputUnigrams(outputUnigrams); |
| shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types); |
| } |
| |
| protected void shingleFilterTestCommon( |
| ShingleFilter filter, Token[] tokensToCompare, int[] positionIncrements, String[] types) |
| throws IOException { |
| String text[] = new String[tokensToCompare.length]; |
| int startOffsets[] = new int[tokensToCompare.length]; |
| int endOffsets[] = new int[tokensToCompare.length]; |
| |
| for (int i = 0; i < tokensToCompare.length; i++) { |
| text[i] = new String(tokensToCompare[i].buffer(), 0, tokensToCompare[i].length()); |
| startOffsets[i] = tokensToCompare[i].startOffset(); |
| endOffsets[i] = tokensToCompare[i].endOffset(); |
| } |
| |
| assertTokenStreamContents(filter, text, startOffsets, endOffsets, types, positionIncrements); |
| } |
| |
| private static Token createToken(String term, int start, int offset) { |
| return createToken(term, start, offset, 1); |
| } |
| |
| private static Token createToken(String term, int start, int offset, int positionIncrement) { |
| Token token = new Token(); |
| token.setOffset(start, offset); |
| token.copyBuffer(term.toCharArray(), 0, term.length()); |
| token.setPositionIncrement(positionIncrement); |
| return token; |
| } |
| |
| /** blast some random strings through the analyzer */ |
| public void testRandomStrings() throws Exception { |
| Analyzer a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer)); |
| } |
| }; |
| checkRandomData(random(), a, 200 * RANDOM_MULTIPLIER); |
| a.close(); |
| } |
| |
| /** blast some random large strings through the analyzer */ |
| public void testRandomHugeStrings() throws Exception { |
| Random random = random(); |
| Analyzer a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer)); |
| } |
| }; |
| checkRandomData(random, a, 3 * RANDOM_MULTIPLIER, 8192); |
| a.close(); |
| } |
| |
| public void testEmptyTerm() throws IOException { |
| Analyzer a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new KeywordTokenizer(); |
| return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer)); |
| } |
| }; |
| checkOneTerm(a, "", ""); |
| a.close(); |
| } |
| |
| public void testTrailingHole1() throws IOException { |
| // Analyzing "wizard of", where of is removed as a |
| // stopword leaving a trailing hole: |
| Token[] inputTokens = new Token[] {createToken("wizard", 0, 6)}; |
| ShingleFilter filter = new ShingleFilter(new CannedTokenStream(1, 9, inputTokens), 2, 2); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] {"wizard", "wizard _"}, |
| new int[] {0, 0}, |
| new int[] {6, 9}, |
| new int[] {1, 0}, |
| 9); |
| } |
| |
| public void testTrailingHole2() throws IOException { |
| // Analyzing "purple wizard of", where of is removed as a |
| // stopword leaving a trailing hole: |
| Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)}; |
| ShingleFilter filter = new ShingleFilter(new CannedTokenStream(1, 16, inputTokens), 2, 2); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] {"purple", "purple wizard", "wizard", "wizard _"}, |
| new int[] {0, 0, 7, 7}, |
| new int[] {6, 13, 13, 16}, |
| new int[] {1, 0, 1, 0}, |
| 16); |
| } |
| |
| public void testTwoTrailingHoles() throws IOException { |
| // Analyzing "purple wizard of the", where of and the are removed as a |
| // stopwords, leaving two trailing holes: |
| Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)}; |
| ShingleFilter filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 2); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] {"purple", "purple wizard", "wizard", "wizard _"}, |
| new int[] {0, 0, 7, 7}, |
| new int[] {6, 13, 13, 20}, |
| new int[] {1, 0, 1, 0}, |
| 20); |
| } |
| |
| public void testTwoTrailingHolesTriShingle() throws IOException { |
| // Analyzing "purple wizard of the", where of and the are removed as a |
| // stopwords, leaving two trailing holes: |
| Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)}; |
| ShingleFilter filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] { |
| "purple", "purple wizard", "purple wizard _", "wizard", "wizard _", "wizard _ _" |
| }, |
| new int[] {0, 0, 0, 7, 7, 7}, |
| new int[] {6, 13, 20, 13, 20, 20}, |
| new int[] {1, 0, 0, 1, 0, 0}, |
| 20); |
| } |
| |
| public void testTwoTrailingHolesTriShingleWithTokenFiller() throws IOException { |
| // Analyzing "purple wizard of the", where of and the are removed as a |
| // stopwords, leaving two trailing holes: |
| Token[] inputTokens = new Token[] {createToken("purple", 0, 6), createToken("wizard", 7, 13)}; |
| ShingleFilter filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3); |
| filter.setFillerToken("--"); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] { |
| "purple", "purple wizard", "purple wizard --", "wizard", "wizard --", "wizard -- --" |
| }, |
| new int[] {0, 0, 0, 7, 7, 7}, |
| new int[] {6, 13, 20, 13, 20, 20}, |
| new int[] {1, 0, 0, 1, 0, 0}, |
| 20); |
| |
| filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3); |
| filter.setFillerToken(""); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] {"purple", "purple wizard", "purple wizard ", "wizard", "wizard ", "wizard "}, |
| new int[] {0, 0, 0, 7, 7, 7}, |
| new int[] {6, 13, 20, 13, 20, 20}, |
| new int[] {1, 0, 0, 1, 0, 0}, |
| 20); |
| |
| filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3); |
| filter.setFillerToken(null); |
| |
| assertTokenStreamContents( |
| filter, |
| new String[] {"purple", "purple wizard", "purple wizard ", "wizard", "wizard ", "wizard "}, |
| new int[] {0, 0, 0, 7, 7, 7}, |
| new int[] {6, 13, 20, 13, 20, 20}, |
| new int[] {1, 0, 0, 1, 0, 0}, |
| 20); |
| |
| filter = new ShingleFilter(new CannedTokenStream(2, 20, inputTokens), 2, 3); |
| filter.setFillerToken(null); |
| filter.setTokenSeparator(null); |
| assertTokenStreamContents( |
| filter, |
| new String[] {"purple", "purplewizard", "purplewizard", "wizard", "wizard", "wizard"}, |
| new int[] {0, 0, 0, 7, 7, 7}, |
| new int[] {6, 13, 20, 13, 20, 20}, |
| new int[] {1, 0, 0, 1, 0, 0}, |
| 20); |
| } |
| |
| public void testPositionLength() throws Exception { |
| Analyzer a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| ShingleFilter filter = new ShingleFilter(tokenizer, 4, 4); |
| filter.setOutputUnigrams(false); |
| return new TokenStreamComponents(tokenizer, filter); |
| } |
| }; |
| assertTokenStreamContents( |
| a.tokenStream("", "to be or not to be"), |
| new String[] {"to be or not", "be or not to", "or not to be"}, |
| new int[] {0, 3, 6}, |
| new int[] {12, 15, 18}, |
| null, |
| new int[] {1, 1, 1}, |
| new int[] {1, 1, 1}, |
| 18, |
| // offsets are correct but assertTokenStreamContents does not handle multiple terms with |
| // different offsets |
| // finishing at the same position |
| false); |
| |
| a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| ShingleFilter filter = new ShingleFilter(tokenizer, 2, 4); |
| filter.setOutputUnigrams(false); |
| return new TokenStreamComponents(tokenizer, filter); |
| } |
| }; |
| assertTokenStreamContents( |
| a.tokenStream("", "to be or not to be"), |
| new String[] { |
| "to be", |
| "to be or", |
| "to be or not", |
| "be or", |
| "be or not", |
| "be or not to", |
| "or not", |
| "or not to", |
| "or not to be", |
| "not to", |
| "not to be", |
| "to be" |
| }, |
| new int[] {0, 0, 0, 3, 3, 3, 6, 6, 6, 9, 9, 13}, |
| new int[] {5, 8, 12, 8, 12, 15, 12, 15, 18, 15, 18, 18}, |
| null, |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1}, |
| new int[] {1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 1}, |
| 18, |
| // offsets are correct but assertTokenStreamContents does not handle multiple terms with |
| // different offsets |
| // finishing at the same position |
| false); |
| |
| a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| ShingleFilter filter = new ShingleFilter(tokenizer, 3, 4); |
| filter.setOutputUnigrams(false); |
| return new TokenStreamComponents(tokenizer, filter); |
| } |
| }; |
| |
| assertTokenStreamContents( |
| a.tokenStream("", "to be or not to be"), |
| new String[] { |
| "to be or", |
| "to be or not", |
| "be or not", |
| "be or not to", |
| "or not to", |
| "or not to be", |
| "not to be" |
| }, |
| new int[] {0, 0, 3, 3, 6, 6, 9}, |
| new int[] {8, 12, 12, 15, 15, 18, 18}, |
| null, |
| new int[] {1, 0, 1, 0, 1, 0, 1, 0}, |
| new int[] {1, 2, 1, 2, 1, 2, 1, 2}, |
| 18, |
| // offsets are correct but assertTokenStreamContents does not handle multiple terms with |
| // different offsets |
| // finishing at the same position |
| false); |
| |
| a = |
| new Analyzer() { |
| @Override |
| protected TokenStreamComponents createComponents(String fieldName) { |
| Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false); |
| ShingleFilter filter = new ShingleFilter(tokenizer, 3, 5); |
| filter.setOutputUnigrams(false); |
| return new TokenStreamComponents(tokenizer, filter); |
| } |
| }; |
| assertTokenStreamContents( |
| a.tokenStream("", "to be or not to be"), |
| new String[] { |
| "to be or", |
| "to be or not", |
| "to be or not to", |
| "be or not", |
| "be or not to", |
| "be or not to be", |
| "or not to", |
| "or not to be", |
| "not to be" |
| }, |
| new int[] {0, 0, 0, 3, 3, 3, 6, 6, 9, 9}, |
| new int[] {8, 12, 15, 12, 15, 18, 15, 18, 18}, |
| null, |
| new int[] {1, 0, 0, 1, 0, 0, 1, 0, 1, 0}, |
| new int[] {1, 2, 3, 1, 2, 3, 1, 2, 1}, |
| 18, |
| // offsets are correct but assertTokenStreamContents does not handle multiple terms with |
| // different offsets |
| // finishing at the same position |
| false); |
| } |
| } |
