LUCENE-3456: use MockTokenizer in analysis tests
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/branch_3x@1175653 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java
index 6da1129..3757663 100644
--- a/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/analysis/CommonGramsFilterFactoryTest.java
@@ -17,9 +17,9 @@
* limitations under the License.
*/
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -77,7 +77,7 @@
Set<?> words = factory.getCommonWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue(words.contains("the"));
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("testing the factory"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader("testing the factory"), MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "testing", "testing_the", "the", "the_factory", "factory" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java
index bf09300..711752b 100644
--- a/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/analysis/CommonGramsQueryFilterFactoryTest.java
@@ -16,9 +16,9 @@
*/
package org.apache.solr.analysis;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -76,7 +76,7 @@
Set<?> words = factory.getCommonWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue(words.contains("the"));
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("testing the factory"));
+ Tokenizer tokenizer = new MockTokenizer(new StringReader("testing the factory"), MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "testing_the", "the_factory" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java
index 45f8c1f..3edf0de 100644
--- a/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/analysis/DoubleMetaphoneFilterFactoryTest.java
@@ -20,8 +20,8 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public class DoubleMetaphoneFilterFactoryTest extends BaseTokenTestCase {
@@ -29,7 +29,7 @@
public void testDefaults() throws Exception {
DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory();
factory.init(new HashMap<String, String>());
- TokenStream inputStream = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("international"));
+ TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
TokenStream filteredStream = factory.create(inputStream);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
@@ -43,7 +43,7 @@
parameters.put("maxCodeLength", "8");
factory.init(parameters);
- TokenStream inputStream = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("international"));
+ TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
TokenStream filteredStream = factory.create(inputStream);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
@@ -56,12 +56,13 @@
public void testReset() throws Exception {
DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory();
factory.init(new HashMap<String, String>());
- TokenStream inputStream = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("international"));
+ TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
TokenStream filteredStream = factory.create(inputStream);
CharTermAttribute termAtt = filteredStream.addAttribute(CharTermAttribute.class);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
+ filteredStream.reset();
assertTrue(filteredStream.incrementToken());
assertEquals(13, termAtt.length());
assertEquals("international", termAtt.toString());
diff --git a/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java b/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java
index ec1959f..676d98c 100644
--- a/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java
+++ b/solr/core/src/test/org/apache/solr/analysis/LengthFilterTest.java
@@ -21,8 +21,8 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
public class LengthFilterTest extends BaseTokenTestCase {
@@ -34,7 +34,7 @@
// default: args.put("enablePositionIncrements", "false");
factory.init(args);
String test = "foo foobar super-duper-trooper";
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(test)));
+ TokenStream stream = factory.create(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "foobar" }, new int[] { 1 });
factory = new LengthFilterFactory();
@@ -43,7 +43,7 @@
args.put(LengthFilterFactory.MAX_KEY, String.valueOf(10));
args.put("enablePositionIncrements", "true");
factory.init(args);
- stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(test)));
+ stream = factory.create(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "foobar" }, new int[] { 2 });
}
}
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java b/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java
index e786c00..75751d9 100644
--- a/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java
+++ b/solr/core/src/test/org/apache/solr/analysis/SnowballPorterFilterFactoryTest.java
@@ -16,9 +16,9 @@
* limitations under the License.
*/
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrResourceLoader;
@@ -53,8 +53,8 @@
factory.init(args);
factory.inform(new LinesMockSolrResourceLoader(new ArrayList<String>()));
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION,
- new StringReader(StrUtils.join(Arrays.asList(test), ' ')));
+ Tokenizer tokenizer = new MockTokenizer(
+ new StringReader(StrUtils.join(Arrays.asList(test), ' ')), MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, gold);
}
@@ -84,8 +84,8 @@
List<String> lines = new ArrayList<String>();
Collections.addAll(lines, "banks", "fledgling");
factory.inform(new LinesMockSolrResourceLoader(lines));
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION,
- new StringReader(StrUtils.join(Arrays.asList(test), ' ')));
+ Tokenizer tokenizer = new MockTokenizer(
+ new StringReader(StrUtils.join(Arrays.asList(test), ' ')), MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, gold);
}
@@ -122,7 +122,7 @@
factory.init(args);
factory.inform(loader);
Reader reader = new StringReader("ridding of some stemming");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "ridding", "of", "some", "stem" });
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java
index bba8893..520fdf6 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestBrazilianStemFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Brazilian stem filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("Brasília");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
BrazilianStemFilterFactory factory = new BrazilianStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "brasil" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java
index 1a9a558..65ed14e 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestBulgarianStemFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Bulgarian stem filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("компютри");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
BulgarianStemFilterFactory factory = new BulgarianStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "компютр" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java
index 8c0d9e3..96240f7 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestChineseFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Chinese filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testFiltering() throws Exception {
Reader reader = new StringReader("this 1234 Is such a silly filter");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ChineseFilterFactory factory = new ChineseFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "Is", "silly", "filter" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java
index c508a65..d6b6299 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestCollationKeyFilterFactory.java
@@ -28,7 +28,7 @@
import java.util.Locale;
import java.util.Map;
-import org.apache.lucene.analysis.KeywordTokenizer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.solr.common.ResourceLoader;
@@ -51,9 +51,9 @@
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsUpper = factory.create(
- new KeywordTokenizer(new StringReader(turkishUpperCase)));
+ new MockTokenizer(new StringReader(turkishUpperCase), MockTokenizer.KEYWORD, false));
TokenStream tsLower = factory.create(
- new KeywordTokenizer(new StringReader(turkishLowerCase)));
+ new MockTokenizer(new StringReader(turkishLowerCase), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUpper, tsLower);
}
@@ -71,9 +71,9 @@
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsUpper = factory.create(
- new KeywordTokenizer(new StringReader(turkishUpperCase)));
+ new MockTokenizer(new StringReader(turkishUpperCase), MockTokenizer.KEYWORD, false));
TokenStream tsLower = factory.create(
- new KeywordTokenizer(new StringReader(turkishLowerCase)));
+ new MockTokenizer(new StringReader(turkishLowerCase), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUpper, tsLower);
}
@@ -92,9 +92,9 @@
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsFull = factory.create(
- new KeywordTokenizer(new StringReader(fullWidth)));
+ new MockTokenizer(new StringReader(fullWidth), MockTokenizer.KEYWORD, false));
TokenStream tsHalf = factory.create(
- new KeywordTokenizer(new StringReader(halfWidth)));
+ new MockTokenizer(new StringReader(halfWidth), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsFull, tsHalf);
}
@@ -112,9 +112,9 @@
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(""));
TokenStream tsUpper = factory.create(
- new KeywordTokenizer(new StringReader(upperCase)));
+ new MockTokenizer(new StringReader(upperCase), MockTokenizer.KEYWORD, false));
TokenStream tsLower = factory.create(
- new KeywordTokenizer(new StringReader(lowerCase)));
+ new MockTokenizer(new StringReader(lowerCase), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUpper, tsLower);
}
@@ -148,9 +148,9 @@
factory.init(args);
factory.inform(new StringMockSolrResourceLoader(tailoredRules));
TokenStream tsUmlaut = factory.create(
- new KeywordTokenizer(new StringReader(germanUmlaut)));
+ new MockTokenizer(new StringReader(germanUmlaut), MockTokenizer.KEYWORD, false));
TokenStream tsOE = factory.create(
- new KeywordTokenizer(new StringReader(germanOE)));
+ new MockTokenizer(new StringReader(germanOE), MockTokenizer.KEYWORD, false));
assertCollatesToSame(tsUmlaut, tsOE);
}
@@ -177,6 +177,8 @@
private void assertCollatesToSame(TokenStream stream1, TokenStream stream2)
throws IOException {
+ stream1.reset();
+ stream2.reset();
CharTermAttribute term1 = stream1
.addAttribute(CharTermAttribute.class);
CharTermAttribute term2 = stream2
@@ -186,5 +188,9 @@
assertEquals(term1.toString(), term2.toString());
assertFalse(stream1.incrementToken());
assertFalse(stream2.incrementToken());
+ stream1.end();
+ stream2.end();
+ stream1.close();
+ stream2.close();
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java
index 5f94194..f040ffc 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestCzechStemFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Czech stem filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("angličtí");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
CzechStemFilterFactory factory = new CzechStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "anglick" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java
index bc7f4b9..a48a00e 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestDelimitedPayloadTokenFilterFactory.java
@@ -21,8 +21,8 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.payloads.DelimitedPayloadTokenFilter;
import org.apache.lucene.analysis.payloads.FloatEncoder;
import org.apache.lucene.analysis.payloads.PayloadHelper;
@@ -40,8 +40,9 @@
ResourceLoader loader = new SolrResourceLoader(null, null);
factory.inform(loader);
- TokenStream input = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("the|0.1 quick|0.1 red|0.1"));
+ TokenStream input = new MockTokenizer(new StringReader("the|0.1 quick|0.1 red|0.1"), MockTokenizer.WHITESPACE, false);
DelimitedPayloadTokenFilter tf = factory.create(input);
+ tf.reset();
while (tf.incrementToken()){
PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
assertTrue("payAttr is null and it shouldn't be", payAttr != null);
@@ -62,8 +63,9 @@
ResourceLoader loader = new SolrResourceLoader(null, null);
factory.inform(loader);
- TokenStream input = new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("the*0.1 quick*0.1 red*0.1"));
+ TokenStream input = new MockTokenizer(new StringReader("the*0.1 quick*0.1 red*0.1"), MockTokenizer.WHITESPACE, false);
DelimitedPayloadTokenFilter tf = factory.create(input);
+ tf.reset();
while (tf.incrementToken()){
PayloadAttribute payAttr = tf.getAttribute(PayloadAttribute.class);
assertTrue("payAttr is null and it shouldn't be", payAttr != null);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java
index 85fbc5c..8a22e33 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestDictionaryCompoundWordTokenFilterFactory.java
@@ -22,9 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@
*/
public void testDecompounding() throws Exception {
Reader reader = new StringReader("I like to play softball");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
DictionaryCompoundWordTokenFilterFactory factory = new DictionaryCompoundWordTokenFilterFactory();
ResourceLoader loader = new SolrResourceLoader(null, null);
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java
index 8964a8a..ebac678 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestElisionFilterFactory.java
@@ -22,9 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@
*/
public void testElision() throws Exception {
Reader reader = new StringReader("l'avion");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ElisionFilterFactory factory = new ElisionFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
@@ -54,7 +54,7 @@
*/
public void testDefaultArticles() throws Exception {
Reader reader = new StringReader("l'avion");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ElisionFilterFactory factory = new ElisionFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java
index 541efe0..65ddf2f 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestEnglishMinimalStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the English minimal stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
EnglishMinimalStemFilterFactory factory = new EnglishMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "brick" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java
index 14c2d5b..7defdf5 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestFinnishLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Finnish light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("aseistettujen");
FinnishLightStemFilterFactory factory = new FinnishLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "aseistet" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java
index 0833056..79aa41a 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestFrenchLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the French light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("administrativement");
FrenchLightStemFilterFactory factory = new FrenchLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "administratif" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java
index 474f30c..ad5f917 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestFrenchMinimalStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the French minimal stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("chevaux");
FrenchMinimalStemFilterFactory factory = new FrenchMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "cheval" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java
index 290812f..8b69e45 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestGalicianStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Galician stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("cariñosa");
GalicianStemFilterFactory factory = new GalicianStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "cariñ" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java
index 7e4182b..6281174 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the German light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("häuser");
GermanLightStemFilterFactory factory = new GermanLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "haus" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java
index b096285..32fe243 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanMinimalStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the German minimal stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("bilder");
GermanMinimalStemFilterFactory factory = new GermanMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "bild" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java
index 618ab52..80730e5 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestGermanStemFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the German stem filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("Tischen");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
GermanStemFilterFactory factory = new GermanStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "tisch" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java
index 929204e..8ed5075 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestGreekLowerCaseFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Greek lowercase filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("Μάϊος ΜΆΪΟΣ");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
GreekLowerCaseFilterFactory factory = new GreekLowerCaseFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java
index c2ede0d..3a48e78 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestGreekStemFilterFactory.java
@@ -3,9 +3,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.el.GreekLowerCaseFilter;
/**
@@ -31,7 +31,7 @@
public class TestGreekStemFilterFactory extends BaseTokenTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("άνθρωπος");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TokenStream normalized = new GreekLowerCaseFilter(DEFAULT_VERSION, tokenizer);
GreekStemFilterFactory factory = new GreekStemFilterFactory();
TokenStream stream = factory.create(normalized);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java
index c1c0912..a8fb9ce 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestHungarianLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Hungarian light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("házakat");
HungarianLightStemFilterFactory factory = new HungarianLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "haz" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java
index da4a193..1512462 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestHunspellStemFilterFactory.java
@@ -22,8 +22,8 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.schema.IndexSchema;
@@ -41,7 +41,7 @@
factory.inform(new SolrResourceLoader("solr"));
Reader reader = new StringReader("abc");
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "ab" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java
index 076e1d5..1d9f0f9 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestHyphenationCompoundWordTokenFilterFactory.java
@@ -22,9 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@
*/
public void testHyphenationWithDictionary() throws Exception {
Reader reader = new StringReader("min veninde som er lidt af en læsehest");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
HyphenationCompoundWordTokenFilterFactory factory = new HyphenationCompoundWordTokenFilterFactory();
ResourceLoader loader = new SolrResourceLoader(null, null);
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
@@ -60,7 +60,7 @@
*/
public void testHyphenationOnly() throws Exception {
Reader reader = new StringReader("basketballkurv");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
HyphenationCompoundWordTokenFilterFactory factory = new HyphenationCompoundWordTokenFilterFactory();
ResourceLoader loader = new SolrResourceLoader(null, null);
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java
index 37a35af..77b1d54 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestIndonesianStemFilterFactory.java
@@ -22,9 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Indonesian stem filter factory is working.
@@ -35,7 +35,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dibukukannya");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
IndonesianStemFilterFactory factory = new IndonesianStemFilterFactory();
Map<String,String> args = new HashMap<String,String>();
factory.init(args);
@@ -48,7 +48,7 @@
*/
public void testStemmingInflectional() throws Exception {
Reader reader = new StringReader("dibukukannya");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
IndonesianStemFilterFactory factory = new IndonesianStemFilterFactory();
Map<String,String> args = new HashMap<String,String>();
args.put("stemDerivational", "false");
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java
index 416cb72..34a307f 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestItalianLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Italian light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("ragazzo ragazzi");
ItalianLightStemFilterFactory factory = new ItalianLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "ragazz", "ragazz" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java
index f837d4b..b6b25fd 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestKStemFilterFactory.java
@@ -3,8 +3,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
KStemFilterFactory factory = new KStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "brick" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java
index cec702c..4764732 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestKeywordMarkerFilterFactory.java
@@ -24,9 +24,9 @@
import java.util.Map;
import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -36,7 +36,7 @@
public class TestKeywordMarkerFilterFactory extends BaseTokenTestCase {
public void testKeywords() throws IOException {
Reader reader = new StringReader("dogs cats");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory();
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
@@ -50,7 +50,7 @@
public void testKeywordsCaseInsensitive() throws IOException {
Reader reader = new StringReader("dogs cats Cats");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
KeywordMarkerFilterFactory factory = new KeywordMarkerFilterFactory();
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java
index 7bf066c..027168e 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestLatvianStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Latvian stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("tirgiem tirgus");
LatvianStemFilterFactory factory = new LatvianStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "tirg", "tirg" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java b/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java
index 4e6ecb2..3846ef2 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestMultiWordSynonyms.java
@@ -19,7 +19,6 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import java.io.ByteArrayInputStream;
@@ -46,7 +45,7 @@
SlowSynonymMap synMap = new SlowSynonymMap(true);
SlowSynonymFilterFactory.parseRules(rules, synMap, "=>", ",", true, null);
- SlowSynonymFilter ts = new SlowSynonymFilter(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader("a e")), synMap);
+ SlowSynonymFilter ts = new SlowSynonymFilter(new MockTokenizer(new StringReader("a e"), MockTokenizer.WHITESPACE, false), synMap);
// This fails because ["e","e"] is the value of the token stream
assertTokenStreamContents(ts, new String[] { "a", "e" });
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java b/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java
index 0b31ee2..7c3eb17 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestNGramFilters.java
@@ -22,9 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the NGram filter factories are working.
@@ -64,7 +64,7 @@
Map<String,String> args = new HashMap<String,String>();
NGramFilterFactory factory = new NGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "t", "e", "s", "t", "te", "es", "st" });
}
@@ -78,7 +78,7 @@
args.put("maxGramSize", "3");
NGramFilterFactory factory = new NGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "te", "es", "st", "tes", "est" });
}
@@ -129,7 +129,7 @@
Map<String,String> args = new HashMap<String,String>();
EdgeNGramFilterFactory factory = new EdgeNGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "t" });
}
@@ -143,7 +143,7 @@
args.put("maxGramSize", "2");
EdgeNGramFilterFactory factory = new EdgeNGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "t", "te" });
}
@@ -156,7 +156,7 @@
args.put("side", "back");
EdgeNGramFilterFactory factory = new EdgeNGramFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "y" });
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java
index 8831f36..372609b 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestPersianNormalizationFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Persian normalization factory is working.
@@ -33,7 +33,7 @@
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("های");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
PersianNormalizationFilterFactory factory = new PersianNormalizationFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "هاي" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java
index acfc2a7..d3eebaa 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestPorterStemFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Porter stem filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dogs");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
PorterStemFilterFactory factory = new PorterStemFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "dog" });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java
index b80d3df..d66c5da 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Portuguese Light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("evidentemente");
PortugueseLightStemFilterFactory factory = new PortugueseLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "evident" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java
index 434e2de..9b1249b 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseMinimalStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Portuguese Minimal stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("questões");
PortugueseMinimalStemFilterFactory factory = new PortugueseMinimalStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "questão" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java
index c6b09ca..900c00d 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestPortugueseStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Portuguese stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("maluquice");
PortugueseStemFilterFactory factory = new PortugueseStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "maluc" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java
index 9672693..0a1bf58 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestReverseStringFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Reverse string filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testReversing() throws Exception {
Reader reader = new StringReader("simple test");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ReverseStringFilterFactory factory = new ReverseStringFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
index 0b8c54b..ee0a7d6 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
@@ -24,8 +24,8 @@
import java.util.Map;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.search.Query;
import org.apache.solr.SolrTestCaseJ4;
@@ -59,7 +59,7 @@
String text = "simple text";
args.put("withOriginal", "true");
factory.init(args);
- TokenStream input = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(text)));
+ TokenStream input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "simple", "\u0001txet", "text" },
new int[] { 1, 0, 1, 0 });
@@ -67,7 +67,7 @@
// now without original tokens
args.put("withOriginal", "false");
factory.init(args);
- input = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, new StringReader(text)));
+ input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "\u0001txet" },
new int[] { 1, 1 });
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java
index 68a697d..11cfdaf 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestRussianLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Russian light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("журналы");
RussianLightStemFilterFactory factory = new RussianLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "журнал" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java
index a1378b3..25c9873 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestShingleFilterFactory.java
@@ -22,8 +22,8 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Shingle filter factory works.
@@ -37,7 +37,7 @@
Map<String,String> args = new HashMap<String,String>();
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] {"this", "this is", "is",
"is a", "a", "a test", "test"});
}
@@ -51,7 +51,7 @@
args.put("outputUnigrams", "false");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] {"this is", "is a", "a test"});
}
@@ -65,7 +65,7 @@
args.put("maxShingleSize", "3");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] {"this", "this is", "this is a", "is",
"is a", "is a test", "a", "a test", "test"});
@@ -81,7 +81,7 @@
args.put("maxShingleSize", "4");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this", "this is a", "this is a test",
"is", "is a test", "a", "test" });
@@ -98,7 +98,7 @@
args.put("outputUnigrams", "false");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this is a", "this is a test", "is a test" });
}
@@ -113,7 +113,7 @@
args.put("maxShingleSize", "3");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this", "this is a", "is", "is a test", "a", "test" });
}
@@ -129,7 +129,7 @@
args.put("outputUnigrams", "false");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this is a", "is a test" });
}
@@ -143,7 +143,7 @@
args.put("tokenSeparator", "=BLAH=");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this", "this=BLAH=is", "is", "is=BLAH=a",
"a", "a=BLAH=test", "test" });
@@ -159,7 +159,7 @@
args.put("outputUnigrams", "false");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this=BLAH=is", "is=BLAH=a", "a=BLAH=test" });
}
@@ -173,7 +173,7 @@
args.put("tokenSeparator", "");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this", "thisis", "is", "isa", "a", "atest", "test" });
}
@@ -190,7 +190,7 @@
args.put("tokenSeparator", "=BLAH=");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this", "this=BLAH=is=BLAH=a",
"this=BLAH=is=BLAH=a=BLAH=test", "is",
@@ -211,7 +211,7 @@
args.put("outputUnigrams", "false");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream,
new String[] { "this=BLAH=is=BLAH=a", "this=BLAH=is=BLAH=a=BLAH=test",
"is=BLAH=a=BLAH=test", });
@@ -232,7 +232,7 @@
args.put("outputUnigramsIfNoShingles", "true");
ShingleFilterFactory factory = new ShingleFilterFactory();
factory.init(args);
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "test" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java
index 8117b08..c4ef814 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestSpanishLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Spanish Light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("sociedades");
SpanishLightStemFilterFactory factory = new SpanishLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "sociedad" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java b/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java
index 8c9605c..04854f4 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestStandardFactories.java
@@ -22,9 +22,9 @@
import java.util.HashMap;
import java.util.Map;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the standard lucene factories are working.
@@ -158,7 +158,7 @@
*/
public void testASCIIFolding() throws Exception {
Reader reader = new StringReader("Česká");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ASCIIFoldingFilterFactory factory = new ASCIIFoldingFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
@@ -171,7 +171,7 @@
*/
public void testISOLatin1Folding() throws Exception {
Reader reader = new StringReader("Česká");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ISOLatin1AccentFilterFactory factory = new ISOLatin1AccentFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java
index 6c17358..b1659b6 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestStemmerOverrideFilterFactory.java
@@ -24,9 +24,9 @@
import java.util.Map;
import org.apache.lucene.analysis.PorterStemFilter;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.core.SolrResourceLoader;
@@ -37,7 +37,7 @@
public void testKeywords() throws IOException {
// our stemdict stems dogs to 'cat'
Reader reader = new StringReader("testing dogs");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
StemmerOverrideFilterFactory factory = new StemmerOverrideFilterFactory();
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
@@ -51,7 +51,7 @@
public void testKeywordsCaseInsensitive() throws IOException {
Reader reader = new StringReader("testing DoGs");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
StemmerOverrideFilterFactory factory = new StemmerOverrideFilterFactory();
Map<String,String> args = new HashMap<String,String>(DEFAULT_VERSION_PARAM);
ResourceLoader loader = new SolrResourceLoader(null, null);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java
index 91d6b60..5882369 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestSwedishLightStemFilterFactory.java
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Swedish Light stem factory is working.
@@ -30,7 +30,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("äpplen äpple");
SwedishLightStemFilterFactory factory = new SwedishLightStemFilterFactory();
- TokenStream stream = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, reader));
+ TokenStream stream = factory.create(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
assertTokenStreamContents(stream, new String[] { "äppl", "äppl" });
}
}
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java
index adc65c6..62ce5e0 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.th.ThaiWordFilter;
/**
@@ -35,7 +35,7 @@
public void testWordBreak() throws Exception {
assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
ThaiWordFilterFactory factory = new ThaiWordFilterFactory();
factory.init(DEFAULT_VERSION_PARAM);
TokenStream stream = factory.create(tokenizer);
diff --git a/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java b/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java
index c533a53..d6899cf 100644
--- a/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/analysis/TestTurkishLowerCaseFilterFactory.java
@@ -20,9 +20,9 @@
import java.io.Reader;
import java.io.StringReader;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
/**
* Simple tests to ensure the Turkish lowercase filter factory is working.
@@ -33,7 +33,7 @@
*/
public void testCasing() throws Exception {
Reader reader = new StringReader("AĞACI");
- Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
TurkishLowerCaseFilterFactory factory = new TurkishLowerCaseFilterFactory();
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream, new String[] { "ağacı" });
