src/Test/Analysis/TeeSinkTokenTest.cs - lucenenet - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 using System;

 using NUnit.Framework;

 using StandardFilter = Lucene.Net.Analysis.Standard.StandardFilter;
 using StandardTokenizer = Lucene.Net.Analysis.Standard.StandardTokenizer;
 using English = Lucene.Net.Util.English;

 namespace Lucene.Net.Analysis
 {

 	/// <summary> tests for the TeeTokenFilter and SinkTokenizer</summary>
 	[TestFixture]
 	public class TeeSinkTokenTest
 	{
 		private class AnonymousClassSinkTokenizer : SinkTokenizer
 		{
 			private void  InitBlock(TeeSinkTokenTest enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
 			}
 			private TeeSinkTokenTest enclosingInstance;
 			public TeeSinkTokenTest Enclosing_Instance
 			{
 				get
 				{
 					return enclosingInstance;
 				}

 			}
 			internal AnonymousClassSinkTokenizer(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
 			{
 				InitBlock(enclosingInstance);
 			}
 			public override void  Add(Token t)
 			{
 				if (t != null && t.TermText().ToUpper().Equals("The".ToUpper()))
 				{
 					base.Add(t);
 				}
 			}
 		}

 		private class AnonymousClassSinkTokenizer1 : SinkTokenizer
 		{
 			private void  InitBlock(TeeSinkTokenTest enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
 			}
 			private TeeSinkTokenTest enclosingInstance;
 			public TeeSinkTokenTest Enclosing_Instance
 			{
 				get
 				{
 					return enclosingInstance;
 				}

 			}
 			internal AnonymousClassSinkTokenizer1(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
 			{
 				InitBlock(enclosingInstance);
 			}
 			public override void  Add(Token t)
 			{
 				if (t != null && t.TermText().ToUpper().Equals("The".ToUpper()))
 				{
 					base.Add(t);
 				}
 			}
 		}

 		private class AnonymousClassSinkTokenizer2 : SinkTokenizer
 		{
 			private void  InitBlock(TeeSinkTokenTest enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
 			}
 			private TeeSinkTokenTest enclosingInstance;
 			public TeeSinkTokenTest Enclosing_Instance
 			{
 				get
 				{
 					return enclosingInstance;
 				}

 			}
 			internal AnonymousClassSinkTokenizer2(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
 			{
 				InitBlock(enclosingInstance);
 			}
 			public override void  Add(Token t)
 			{
 				if (t != null && t.TermText().ToUpper().Equals("Dogs".ToUpper()))
 				{
 					base.Add(t);
 				}
 			}
 		}
 		protected internal System.Text.StringBuilder buffer1;
 		protected internal System.Text.StringBuilder buffer2;
 		protected internal System.String[] tokens1;
 		protected internal System.String[] tokens2;

 		[SetUp]
 		public virtual void  SetUp()
 		{
 			tokens1 = new System.String[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
 			tokens2 = new System.String[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
 			buffer1 = new System.Text.StringBuilder();

 			for (int i = 0; i < tokens1.Length; i++)
 			{
 				buffer1.Append(tokens1[i]).Append(' ');
 			}
 			buffer2 = new System.Text.StringBuilder();
 			for (int i = 0; i < tokens2.Length; i++)
 			{
 				buffer2.Append(tokens2[i]).Append(' ');
 			}
 		}

 		[TearDown]
 		public virtual void  TearDown()
 		{

 		}

 		[Test]
 		public virtual void  Test()
 		{

 			SinkTokenizer sink1 = new AnonymousClassSinkTokenizer(this, null);
 			TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer1.ToString())), sink1);
 			Token token = null;
 			int i = 0;
 			while ((token = source.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().Equals(tokens1[i]) == true, token.TermText() + " is not equal to " + tokens1[i]);
 				i++;
 			}
 			Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
 			Assert.IsTrue(sink1.GetTokens().Count == 2, "sink1 Size: " + sink1.GetTokens().Count + " is not: " + 2);
 			i = 0;
 			while ((token = sink1.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().ToUpper().Equals("The".ToUpper()) == true, token.TermText() + " is not equal to " + "The");
 				i++;
 			}
 			Assert.IsTrue(i == sink1.GetTokens().Count, i + " does not equal: " + sink1.GetTokens().Count);
 		}

 		[Test]
 		public virtual void  TestMultipleSources()
 		{
 			SinkTokenizer theDetector = new AnonymousClassSinkTokenizer1(this, null);
 			SinkTokenizer dogDetector = new AnonymousClassSinkTokenizer2(this, null);
 			TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer1.ToString())), theDetector), dogDetector));
 			TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer2.ToString())), theDetector), dogDetector);
 			Token token = null;
 			int i = 0;
 			while ((token = source1.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().Equals(tokens1[i]) == true, token.TermText() + " is not equal to " + tokens1[i]);
 				i++;
 			}
 			Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
 			Assert.IsTrue(theDetector.GetTokens().Count == 2, "theDetector Size: " + theDetector.GetTokens().Count + " is not: " + 2);
 			Assert.IsTrue(dogDetector.GetTokens().Count == 1, "dogDetector Size: " + dogDetector.GetTokens().Count + " is not: " + 1);
 			i = 0;
 			while ((token = source2.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().Equals(tokens2[i]) == true, token.TermText() + " is not equal to " + tokens2[i]);
 				i++;
 			}
 			Assert.IsTrue(i == tokens2.Length, i + " does not equal: " + tokens2.Length);
 			Assert.IsTrue(theDetector.GetTokens().Count == 4, "theDetector Size: " + theDetector.GetTokens().Count + " is not: " + 4);
 			Assert.IsTrue(dogDetector.GetTokens().Count == 2, "dogDetector Size: " + dogDetector.GetTokens().Count + " is not: " + 2);
 			i = 0;
 			while ((token = theDetector.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().ToUpper().Equals("The".ToUpper()) == true, token.TermText() + " is not equal to " + "The");
 				i++;
 			}
 			Assert.IsTrue(i == theDetector.GetTokens().Count, i + " does not equal: " + theDetector.GetTokens().Count);
 			i = 0;
 			while ((token = dogDetector.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().ToUpper().Equals("Dogs".ToUpper()) == true, token.TermText() + " is not equal to " + "Dogs");
 				i++;
 			}
 			Assert.IsTrue(i == dogDetector.GetTokens().Count, i + " does not equal: " + dogDetector.GetTokens().Count);
 			source1.Reset();
 			TokenStream lowerCasing = new LowerCaseFilter(source1);
 			i = 0;
 			while ((token = lowerCasing.Next()) != null)
 			{
 				Assert.IsTrue(token.TermText().Equals(tokens1[i].ToLower()) == true, token.TermText() + " is not equal to " + tokens1[i].ToLower());
 				i++;
 			}
 			Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
 		}

 		/// <summary> Not an explicit test, just useful to print out some info on performance
 		///
 		/// </summary>
 		/// <throws>  Exception </throws>
 		[Test]
 		public virtual void  TestPerformance()
 		{
 			int[] tokCount = new int[]{100, 500, 1000, 2000, 5000, 10000};
 			int[] modCounts = new int[]{1, 2, 5, 10, 20, 50, 100, 200, 500};
 			for (int k = 0; k < tokCount.Length; k++)
 			{
 				System.Text.StringBuilder buffer = new System.Text.StringBuilder();
 				System.Console.Out.WriteLine("-----Tokens: " + tokCount[k] + "-----");
 				for (int i = 0; i < tokCount[k]; i++)
 				{
 					buffer.Append(English.IntToEnglish(i).ToUpper()).Append(' ');
 				}
 				//make sure we produce the same tokens
 				ModuloSinkTokenizer sink = new ModuloSinkTokenizer(this, tokCount[k], 100);
 				Token next = new Token();
 				TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), sink);
 				while ((next = result.Next(next)) != null)
 				{
 				}
 				result = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), 100);
 				next = new Token();
 				System.Collections.IList tmp = new System.Collections.ArrayList();
 				while ((next = result.Next(next)) != null)
 				{
 					tmp.Add(next.Clone());
 				}
 				System.Collections.IList sinkList = sink.GetTokens();
 				Assert.IsTrue(tmp.Count == sinkList.Count, "tmp Size: " + tmp.Count + " is not: " + sinkList.Count);
 				for (int i = 0; i < tmp.Count; i++)
 				{
 					Token tfTok = (Token) tmp[i];
 					Token sinkTok = (Token) sinkList[i];
 					Assert.IsTrue(tfTok.TermText().Equals(sinkTok.TermText()) == true, tfTok.TermText() + " is not equal to " + sinkTok.TermText() + " at token: " + i);
 				}
 				//simulate two fields, each being analyzed once, for 20 documents

 				for (int j = 0; j < modCounts.Length; j++)
 				{
 					int tfPos = 0;
 					long start = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
 					for (int i = 0; i < 20; i++)
 					{
 						next = new Token();
 						result = new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString())));
 						while ((next = result.Next(next)) != null)
 						{
 							tfPos += next.GetPositionIncrement();
 						}
 						next = new Token();
 						result = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), modCounts[j]);
 						while ((next = result.Next(next)) != null)
 						{
 							tfPos += next.GetPositionIncrement();
 						}
 					}
 					long finish = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
 					System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
 					int sinkPos = 0;
 					//simulate one field with one sink
 					start = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
 					for (int i = 0; i < 20; i++)
 					{
 						sink = new ModuloSinkTokenizer(this, tokCount[k], modCounts[j]);
 						next = new Token();
 						result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), sink);
 						while ((next = result.Next(next)) != null)
 						{
 							sinkPos += next.GetPositionIncrement();
 						}
 						//System.out.println("Modulo--------");
 						result = sink;
 						while ((next = result.Next(next)) != null)
 						{
 							sinkPos += next.GetPositionIncrement();
 						}
 					}
 					finish = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
 					System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
 					Assert.IsTrue(sinkPos == tfPos, sinkPos + " does not equal: " + tfPos);
 				}
 				System.Console.Out.WriteLine("- End Tokens: " + tokCount[k] + "-----");
 			}
 		}


 		internal class ModuloTokenFilter : TokenFilter
 		{
 			private void  InitBlock(TeeSinkTokenTest enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
 			}
 			private TeeSinkTokenTest enclosingInstance;
 			public TeeSinkTokenTest Enclosing_Instance
 			{
 				get
 				{
 					return enclosingInstance;
 				}

 			}

 			internal int modCount;

 			internal ModuloTokenFilter(TeeSinkTokenTest enclosingInstance, TokenStream input, int mc):base(input)
 			{
 				InitBlock(enclosingInstance);
 				modCount = mc;
 			}

 			internal int count = 0;

 			//return every 100 tokens
 			public override Token Next(Token result)
 			{

 				while ((result = input.Next(result)) != null && count % modCount != 0)
 				{
 					count++;
 				}
 				count++;
 				return result;
 			}
 		}

 		internal class ModuloSinkTokenizer : SinkTokenizer
 		{
 			private void  InitBlock(TeeSinkTokenTest enclosingInstance)
 			{
 				this.enclosingInstance = enclosingInstance;
 			}
 			private TeeSinkTokenTest enclosingInstance;
 			public TeeSinkTokenTest Enclosing_Instance
 			{
 				get
 				{
 					return enclosingInstance;
 				}

 			}
 			internal int count = 0;
 			internal int modCount;


 			internal ModuloSinkTokenizer(TeeSinkTokenTest enclosingInstance, int numToks, int mc)
 			{
 				InitBlock(enclosingInstance);
 				modCount = mc;
 				lst = new System.Collections.ArrayList(numToks % mc);
 			}

 			public override void  Add(Token t)
 			{
 				if (t != null && count % modCount == 0)
 				{
 					lst.Add(t.Clone());
 				}
 				count++;
 			}
 		}
 	}
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	using System;

	using NUnit.Framework;

	using StandardFilter = Lucene.Net.Analysis.Standard.StandardFilter;
	using StandardTokenizer = Lucene.Net.Analysis.Standard.StandardTokenizer;
	using English = Lucene.Net.Util.English;

	namespace Lucene.Net.Analysis
	{

	/// <summary> tests for the TeeTokenFilter and SinkTokenizer</summary>
	[TestFixture]
	public class TeeSinkTokenTest
	{
	private class AnonymousClassSinkTokenizer : SinkTokenizer
	{
	private void InitBlock(TeeSinkTokenTest enclosingInstance)
	{
	this.enclosingInstance = enclosingInstance;
	}
	private TeeSinkTokenTest enclosingInstance;
	public TeeSinkTokenTest Enclosing_Instance
	{
	get
	{
	return enclosingInstance;
	}

	}
	internal AnonymousClassSinkTokenizer(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
	{
	InitBlock(enclosingInstance);
	}
	public override void Add(Token t)
	{
	if (t != null && t.TermText().ToUpper().Equals("The".ToUpper()))
	{
	base.Add(t);
	}
	}
	}

	private class AnonymousClassSinkTokenizer1 : SinkTokenizer
	{
	private void InitBlock(TeeSinkTokenTest enclosingInstance)
	{
	this.enclosingInstance = enclosingInstance;
	}
	private TeeSinkTokenTest enclosingInstance;
	public TeeSinkTokenTest Enclosing_Instance
	{
	get
	{
	return enclosingInstance;
	}

	}
	internal AnonymousClassSinkTokenizer1(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
	{
	InitBlock(enclosingInstance);
	}
	public override void Add(Token t)
	{
	if (t != null && t.TermText().ToUpper().Equals("The".ToUpper()))
	{
	base.Add(t);
	}
	}
	}

	private class AnonymousClassSinkTokenizer2 : SinkTokenizer
	{
	private void InitBlock(TeeSinkTokenTest enclosingInstance)
	{
	this.enclosingInstance = enclosingInstance;
	}
	private TeeSinkTokenTest enclosingInstance;
	public TeeSinkTokenTest Enclosing_Instance
	{
	get
	{
	return enclosingInstance;
	}

	}
	internal AnonymousClassSinkTokenizer2(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
	{
	InitBlock(enclosingInstance);
	}
	public override void Add(Token t)
	{
	if (t != null && t.TermText().ToUpper().Equals("Dogs".ToUpper()))
	{
	base.Add(t);
	}
	}
	}
	protected internal System.Text.StringBuilder buffer1;
	protected internal System.Text.StringBuilder buffer2;
	protected internal System.String[] tokens1;
	protected internal System.String[] tokens2;

	[SetUp]
	public virtual void SetUp()
	{
	tokens1 = new System.String[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
	tokens2 = new System.String[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
	buffer1 = new System.Text.StringBuilder();

	for (int i = 0; i < tokens1.Length; i++)
	{
	buffer1.Append(tokens1[i]).Append(' ');
	}
	buffer2 = new System.Text.StringBuilder();
	for (int i = 0; i < tokens2.Length; i++)
	{
	buffer2.Append(tokens2[i]).Append(' ');
	}
	}

	[TearDown]
	public virtual void TearDown()
	{

	}

	[Test]
	public virtual void Test()
	{

	SinkTokenizer sink1 = new AnonymousClassSinkTokenizer(this, null);
	TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer1.ToString())), sink1);
	Token token = null;
	int i = 0;
	while ((token = source.Next()) != null)
	{
	Assert.IsTrue(token.TermText().Equals(tokens1[i]) == true, token.TermText() + " is not equal to " + tokens1[i]);
	i++;
	}
	Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
	Assert.IsTrue(sink1.GetTokens().Count == 2, "sink1 Size: " + sink1.GetTokens().Count + " is not: " + 2);
	i = 0;
	while ((token = sink1.Next()) != null)
	{
	Assert.IsTrue(token.TermText().ToUpper().Equals("The".ToUpper()) == true, token.TermText() + " is not equal to " + "The");
	i++;
	}
	Assert.IsTrue(i == sink1.GetTokens().Count, i + " does not equal: " + sink1.GetTokens().Count);
	}

	[Test]
	public virtual void TestMultipleSources()
	{
	SinkTokenizer theDetector = new AnonymousClassSinkTokenizer1(this, null);
	SinkTokenizer dogDetector = new AnonymousClassSinkTokenizer2(this, null);
	TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer1.ToString())), theDetector), dogDetector));
	TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer2.ToString())), theDetector), dogDetector);
	Token token = null;
	int i = 0;
	while ((token = source1.Next()) != null)
	{
	Assert.IsTrue(token.TermText().Equals(tokens1[i]) == true, token.TermText() + " is not equal to " + tokens1[i]);
	i++;
	}
	Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
	Assert.IsTrue(theDetector.GetTokens().Count == 2, "theDetector Size: " + theDetector.GetTokens().Count + " is not: " + 2);
	Assert.IsTrue(dogDetector.GetTokens().Count == 1, "dogDetector Size: " + dogDetector.GetTokens().Count + " is not: " + 1);
	i = 0;
	while ((token = source2.Next()) != null)
	{
	Assert.IsTrue(token.TermText().Equals(tokens2[i]) == true, token.TermText() + " is not equal to " + tokens2[i]);
	i++;
	}
	Assert.IsTrue(i == tokens2.Length, i + " does not equal: " + tokens2.Length);
	Assert.IsTrue(theDetector.GetTokens().Count == 4, "theDetector Size: " + theDetector.GetTokens().Count + " is not: " + 4);
	Assert.IsTrue(dogDetector.GetTokens().Count == 2, "dogDetector Size: " + dogDetector.GetTokens().Count + " is not: " + 2);
	i = 0;
	while ((token = theDetector.Next()) != null)
	{
	Assert.IsTrue(token.TermText().ToUpper().Equals("The".ToUpper()) == true, token.TermText() + " is not equal to " + "The");
	i++;
	}
	Assert.IsTrue(i == theDetector.GetTokens().Count, i + " does not equal: " + theDetector.GetTokens().Count);
	i = 0;
	while ((token = dogDetector.Next()) != null)
	{
	Assert.IsTrue(token.TermText().ToUpper().Equals("Dogs".ToUpper()) == true, token.TermText() + " is not equal to " + "Dogs");
	i++;
	}
	Assert.IsTrue(i == dogDetector.GetTokens().Count, i + " does not equal: " + dogDetector.GetTokens().Count);
	source1.Reset();
	TokenStream lowerCasing = new LowerCaseFilter(source1);
	i = 0;
	while ((token = lowerCasing.Next()) != null)
	{
	Assert.IsTrue(token.TermText().Equals(tokens1[i].ToLower()) == true, token.TermText() + " is not equal to " + tokens1[i].ToLower());
	i++;
	}
	Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
	}

	/// <summary> Not an explicit test, just useful to print out some info on performance
	///
	/// </summary>
	/// <throws> Exception </throws>
	[Test]
	public virtual void TestPerformance()
	{
	int[] tokCount = new int[]{100, 500, 1000, 2000, 5000, 10000};
	int[] modCounts = new int[]{1, 2, 5, 10, 20, 50, 100, 200, 500};
	for (int k = 0; k < tokCount.Length; k++)
	{
	System.Text.StringBuilder buffer = new System.Text.StringBuilder();
	System.Console.Out.WriteLine("-----Tokens: " + tokCount[k] + "-----");
	for (int i = 0; i < tokCount[k]; i++)
	{
	buffer.Append(English.IntToEnglish(i).ToUpper()).Append(' ');
	}
	//make sure we produce the same tokens
	ModuloSinkTokenizer sink = new ModuloSinkTokenizer(this, tokCount[k], 100);
	Token next = new Token();
	TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), sink);
	while ((next = result.Next(next)) != null)
	{
	}
	result = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), 100);
	next = new Token();
	System.Collections.IList tmp = new System.Collections.ArrayList();
	while ((next = result.Next(next)) != null)
	{
	tmp.Add(next.Clone());
	}
	System.Collections.IList sinkList = sink.GetTokens();
	Assert.IsTrue(tmp.Count == sinkList.Count, "tmp Size: " + tmp.Count + " is not: " + sinkList.Count);
	for (int i = 0; i < tmp.Count; i++)
	{
	Token tfTok = (Token) tmp[i];
	Token sinkTok = (Token) sinkList[i];
	Assert.IsTrue(tfTok.TermText().Equals(sinkTok.TermText()) == true, tfTok.TermText() + " is not equal to " + sinkTok.TermText() + " at token: " + i);
	}
	//simulate two fields, each being analyzed once, for 20 documents

	for (int j = 0; j < modCounts.Length; j++)
	{
	int tfPos = 0;
	long start = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
	for (int i = 0; i < 20; i++)
	{
	next = new Token();
	result = new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString())));
	while ((next = result.Next(next)) != null)
	{
	tfPos += next.GetPositionIncrement();
	}
	next = new Token();
	result = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), modCounts[j]);
	while ((next = result.Next(next)) != null)
	{
	tfPos += next.GetPositionIncrement();
	}
	}
	long finish = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
	System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
	int sinkPos = 0;
	//simulate one field with one sink
	start = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
	for (int i = 0; i < 20; i++)
	{
	sink = new ModuloSinkTokenizer(this, tokCount[k], modCounts[j]);
	next = new Token();
	result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), sink);
	while ((next = result.Next(next)) != null)
	{
	sinkPos += next.GetPositionIncrement();
	}
	//System.out.println("Modulo--------");
	result = sink;
	while ((next = result.Next(next)) != null)
	{
	sinkPos += next.GetPositionIncrement();
	}
	}
	finish = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
	System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
	Assert.IsTrue(sinkPos == tfPos, sinkPos + " does not equal: " + tfPos);
	}
	System.Console.Out.WriteLine("- End Tokens: " + tokCount[k] + "-----");
	}
	}


	internal class ModuloTokenFilter : TokenFilter
	{
	private void InitBlock(TeeSinkTokenTest enclosingInstance)
	{
	this.enclosingInstance = enclosingInstance;
	}
	private TeeSinkTokenTest enclosingInstance;
	public TeeSinkTokenTest Enclosing_Instance
	{
	get
	{
	return enclosingInstance;
	}

	}

	internal int modCount;

	internal ModuloTokenFilter(TeeSinkTokenTest enclosingInstance, TokenStream input, int mc):base(input)
	{
	InitBlock(enclosingInstance);
	modCount = mc;
	}

	internal int count = 0;

	//return every 100 tokens
	public override Token Next(Token result)
	{

	while ((result = input.Next(result)) != null && count % modCount != 0)
	{
	count++;
	}
	count++;
	return result;
	}
	}

	internal class ModuloSinkTokenizer : SinkTokenizer
	{
	private void InitBlock(TeeSinkTokenTest enclosingInstance)
	{
	this.enclosingInstance = enclosingInstance;
	}
	private TeeSinkTokenTest enclosingInstance;
	public TeeSinkTokenTest Enclosing_Instance
	{
	get
	{
	return enclosingInstance;
	}

	}
	internal int count = 0;
	internal int modCount;


	internal ModuloSinkTokenizer(TeeSinkTokenTest enclosingInstance, int numToks, int mc)
	{
	InitBlock(enclosingInstance);
	modCount = mc;
	lst = new System.Collections.ArrayList(numToks % mc);
	}

	public override void Add(Token t)
	{
	if (t != null && count % modCount == 0)
	{
	lst.Add(t.Clone());
	}
	count++;
	}
	}
	}
	}