src/Test/Index/TestStressIndexing2.cs - lucenenet - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 using System;

 using Lucene.Net.Documents;
 using Lucene.Net.Store;
 using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
 using Lucene.Net.Analysis;

 using NUnit.Framework;

 namespace Lucene.Net.Index
 {

 	[TestFixture]
 	public class TestStressIndexing2 : LuceneTestCase
 	{
 		internal class AnonymousClassComparator : System.Collections.IComparer
 		{
             Fieldable f1, f2;
 			public virtual int Compare(System.Object o1, System.Object o2)
 			{
                 if (o1 == o2) return 0;
                 f1 = (Fieldable)o1;
                 f2 = (Fieldable)o2;
                 return String.CompareOrdinal(f1.Name() + f1.StringValue(), f2.Name() + f2.StringValue());
 			}
 		}
 		internal static int maxFields = 4;
 		internal static int bigFieldSize = 10;
 		internal static bool sameFieldOrder = false;
 		internal static bool autoCommit = false;
 		internal static int mergeFactor = 3;
 		internal static int maxBufferedDocs = 3;
 		internal static int seed = 0;

 		internal static System.Random r = new System.Random((System.Int32) 0);

 		[Test]
 		public virtual void  TestRandom()
 		{
 			Directory dir1 = new MockRAMDirectory();
 			Directory dir2 = new MockRAMDirectory();
             System.Collections.IDictionary docs = IndexRandom(10, 100, 100, dir1);
 			IndexSerial(docs, dir2);
 			VerifyEquals(dir1, dir2, "id");
 		}

 		[Test]
 		public virtual void  TestMultiConfig()
 		{
 			// test lots of smaller different params together
 			for (int i = 0; i < 100; i++)
 			{
 				// increase iterations for better testing
 				sameFieldOrder = r.NextDouble() > 0.5;
 				autoCommit = r.NextDouble() > 0.5;
 				mergeFactor = r.Next(3) + 2;
 				maxBufferedDocs = r.Next(3) + 2;
 				seed++;

 				int nThreads = r.Next(5) + 1;
 				int iter = r.Next(10) + 1;
 				int range = r.Next(20) + 1;

 				Directory dir1 = new MockRAMDirectory();
 				Directory dir2 = new MockRAMDirectory();
                 System.Collections.IDictionary docs = IndexRandom(nThreads, iter, range, dir1);
 				IndexSerial(docs, dir2);
 				VerifyEquals(dir1, dir2, "id");
 			}
 		}


 		internal static Term idTerm = new Term("id", "");
 		internal IndexingThread[] threads;
 		internal static System.Collections.IComparer fieldNameComparator;

 		// This test avoids using any extra synchronization in the multiple
 		// indexing threads to test that IndexWriter does correctly synchronize
 		// everything.

 		public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir)
 		{
 			IndexWriter w = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
 			w.SetUseCompoundFile(false);

 			// force many merges
 			w.SetMergeFactor(mergeFactor);
 			w.SetRAMBufferSizeMB(.1);
 			w.SetMaxBufferedDocs(maxBufferedDocs);

 			threads = new IndexingThread[nThreads];
 			for (int i = 0; i < threads.Length; i++)
 			{
 				IndexingThread th = new IndexingThread();
 				th.w = w;
 				th.base_Renamed = 1000000 * i;
 				th.range = range;
 				th.iterations = iterations;
 				threads[i] = th;
 			}

 			for (int i = 0; i < threads.Length; i++)
 			{
 				threads[i].Start();
 			}
 			for (int i = 0; i < threads.Length; i++)
 			{
 				threads[i].Join();
 			}

 			// w.optimize();
 			w.Close();

 			System.Collections.IDictionary docs = new System.Collections.Hashtable();
 			for (int i = 0; i < threads.Length; i++)
 			{
 				IndexingThread th = threads[i];
 				lock (th)
 				{
 					System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator();
 					while (e.MoveNext())
 					{
 						docs[e.Current] = th.docs[e.Current];
 					}
 				}
 			}

 			return docs;
 		}


 		public static void  IndexSerial(System.Collections.IDictionary docs, Directory dir)
 		{
 			IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());

 			// index all docs in a single thread
 			System.Collections.IEnumerator iter = docs.Values.GetEnumerator();
 			while (iter.MoveNext())
 			{
 				Document d = (Document) iter.Current;
 				System.Collections.ArrayList fields = new System.Collections.ArrayList();
 				fields.AddRange(d.GetFields());

                 // nonono - can't do this (below)
                 //
                 // if multiple fields w/ same name, each instance must be
                 // added in the same order as orginal doc, as the fields
                 // are effectively concatendated
                 //
                 // term position/offset information must be maintained

                 // put fields in same order each time
                 //fields.Sort(fieldNameComparator);

 				Document d1 = new Document();
 				d1.SetBoost(d.GetBoost());
 				for (int i = 0; i < fields.Count; i++)
 				{
 					d1.Add((Fieldable) fields[i]);
 				}
 				w.AddDocument(d1);
 				// System.out.println("indexing "+d1);
 			}

 			w.Close();
 		}

 		public static void  VerifyEquals(Directory dir1, Directory dir2, System.String idField)
 		{
 			IndexReader r1 = IndexReader.Open(dir1);
 			IndexReader r2 = IndexReader.Open(dir2);
 			VerifyEquals(r1, r2, idField);
 			r1.Close();
 			r2.Close();
 		}


 		public static void  VerifyEquals(IndexReader r1, IndexReader r2, System.String idField)
 		{
 			Assert.AreEqual(r1.NumDocs(), r2.NumDocs());
 			bool hasDeletes = !(r1.MaxDoc() == r2.MaxDoc() && r1.NumDocs() == r1.MaxDoc());

 			int[] r2r1 = new int[r2.MaxDoc()]; // r2 id to r1 id mapping

 			TermDocs termDocs1 = r1.TermDocs();
 			TermDocs termDocs2 = r2.TermDocs();

 			// create mapping from id2 space to id2 based on idField
 			idField = String.Intern(idField);
 			TermEnum termEnum = r1.Terms(new Term(idField, ""));
 			do
 			{
 				Term term = termEnum.Term();
 				if (term == null || (System.Object) term.Field() != (System.Object) idField)
 					break;

 				termDocs1.Seek(termEnum);
 				Assert.IsTrue(termDocs1.Next());
 				int id1 = termDocs1.Doc();
 				Assert.IsFalse(termDocs1.Next());

 				termDocs2.Seek(termEnum);
 				Assert.IsTrue(termDocs2.Next());
 				int id2 = termDocs2.Doc();
 				Assert.IsFalse(termDocs2.Next());

 				r2r1[id2] = id1;

 				// verify stored fields are equivalent
 				VerifyEquals(r1.Document(id1), r2.Document(id2));

 				try
 				{
 					// verify term vectors are equivalent
 					VerifyEquals(r1.GetTermFreqVectors(id1), r2.GetTermFreqVectors(id2));
 				}
 				catch (System.ApplicationException e)
 				{
 					System.Console.Out.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
 					TermFreqVector[] tv1 = r1.GetTermFreqVectors(id1);
 					System.Console.Out.WriteLine("  d1=" + tv1);
 					if (tv1 != null)
 						for (int i = 0; i < tv1.Length; i++)
 						{
 							System.Console.Out.WriteLine("    " + i + ": " + tv1[i]);
 						}

 					TermFreqVector[] tv2 = r2.GetTermFreqVectors(id2);
 					System.Console.Out.WriteLine("  d2=" + tv2);
 					if (tv2 != null)
 						for (int i = 0; i < tv2.Length; i++)
 						{
 							System.Console.Out.WriteLine("    " + i + ": " + tv2[i]);
 						}

 					throw e;
 				}
 			}
 			while (termEnum.Next());

 			termEnum.Close();

 			// Verify postings
 			TermEnum termEnum1 = r1.Terms(new Term("", ""));
 			TermEnum termEnum2 = r2.Terms(new Term("", ""));

 			// pack both doc and freq into single element for easy sorting
 			long[] info1 = new long[r1.NumDocs()];
 			long[] info2 = new long[r2.NumDocs()];

 			for (; ; )
 			{
 				Term term1, term2;

 				// iterate until we get some docs
 				int len1;
 				for (; ; )
 				{
 					len1 = 0;
 					term1 = termEnum1.Term();
 					if (term1 == null)
 						break;
 					termDocs1.Seek(termEnum1);
 					while (termDocs1.Next())
 					{
 						int d1 = termDocs1.Doc();
 						int f1 = termDocs1.Freq();
 						info1[len1] = (((long) d1) << 32) | f1;
 						len1++;
 					}
 					if (len1 > 0)
 						break;
 					if (!termEnum1.Next())
 						break;
 				}

 				// iterate until we get some docs
 				int len2;
 				for (; ; )
 				{
 					len2 = 0;
 					term2 = termEnum2.Term();
 					if (term2 == null)
 						break;
 					termDocs2.Seek(termEnum2);
 					while (termDocs2.Next())
 					{
 						int d2 = termDocs2.Doc();
 						int f2 = termDocs2.Freq();
 						info2[len2] = (((long) r2r1[d2]) << 32) | f2;
 						len2++;
 					}
 					if (len2 > 0)
 						break;
 					if (!termEnum2.Next())
 						break;
 				}

 				if (!hasDeletes)
 					Assert.AreEqual(termEnum1.DocFreq(), termEnum2.DocFreq());

 				Assert.AreEqual(len1, len2);
 				if (len1 == 0)
 					break; // no more terms

 				Assert.AreEqual(term1, term2);

 				// sort info2 to get it into ascending docid
 				System.Array.Sort(info2, 0, len2 - 0);

 				// now compare
 				for (int i = 0; i < len1; i++)
 				{
 					Assert.AreEqual(info1[i], info2[i]);
 				}

 				termEnum1.Next();
 				termEnum2.Next();
 			}
 		}

 		public static void  VerifyEquals(Document d1, Document d2)
 		{
 			System.Collections.ArrayList ff1 = new System.Collections.ArrayList(d1.GetFields());
 			System.Collections.ArrayList ff2 = new System.Collections.ArrayList(d2.GetFields());

 			ff1.Sort(fieldNameComparator);
 			ff2.Sort(fieldNameComparator);

 			if (ff1.Count != ff2.Count)
 			{
                 // print out whole doc on error
                 System.Console.Write("Doc 1:");
                 for (int j = 0; j < ff1.Count; j++)
                 {
                     Fieldable field = (Fieldable)ff1[j];
                     System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
                 }
                 System.Console.WriteLine();
                 System.Console.Write("Doc 2:");
                 for (int j = 0; j < ff2.Count; j++)
                 {
                     Fieldable field = (Fieldable)ff2[j];
                     System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
                 }
                 System.Console.WriteLine(); Assert.AreEqual(ff1.Count, ff2.Count);
 			}

 			for (int i = 0; i < ff1.Count; i++)
 			{
 				Fieldable f1 = (Fieldable) ff1[i];
 				Fieldable f2 = (Fieldable) ff2[i];
 				if (f1.IsBinary())
 				{
 					System.Diagnostics.Debug.Assert(f2.IsBinary());
 					//TODO
 				}
 				else
 				{
 					System.String s1 = f1.StringValue();
 					System.String s2 = f2.StringValue();
 					if (!s1.Equals(s2))
 					{
 						// print out whole doc on error
                         System.Console.Write("Doc 1:");
                         for (int j = 0; j < ff1.Count; j++)
                         {
                             Fieldable field = (Fieldable)ff1[j];
                             System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
                         }
                         System.Console.WriteLine();
                         System.Console.Write("Doc 2:");
                         for (int j = 0; j < ff2.Count; j++)
                         {
                             Fieldable field = (Fieldable)ff2[j];
                             System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
                         }
                         System.Console.WriteLine();
                         Assert.AreEqual(s1, s2);
 					}
 				}
 			}
 		}

 		public static void  VerifyEquals(TermFreqVector[] d1, TermFreqVector[] d2)
 		{
 			if (d1 == null)
 			{
 				Assert.IsTrue(d2 == null);
 				return ;
 			}
 			Assert.IsTrue(d2 != null);

 			Assert.AreEqual(d1.Length, d2.Length);
 			for (int i = 0; i < d1.Length; i++)
 			{
 				TermFreqVector v1 = d1[i];
 				TermFreqVector v2 = d2[i];
 				Assert.AreEqual(v1.Size(), v2.Size());
 				int numTerms = v1.Size();
 				System.String[] terms1 = v1.GetTerms();
 				System.String[] terms2 = v2.GetTerms();
 				int[] freq1 = v1.GetTermFrequencies();
 				int[] freq2 = v2.GetTermFrequencies();
 				for (int j = 0; j < numTerms; j++)
 				{
 					if (!terms1[j].Equals(terms2[j]))
 						Assert.AreEqual(terms1[j], terms2[j]);
 					Assert.AreEqual(freq1[j], freq2[j]);
 				}
 				if (v1 is TermPositionVector)
 				{
 					Assert.IsTrue(v2 is TermPositionVector);
 					TermPositionVector tpv1 = (TermPositionVector) v1;
 					TermPositionVector tpv2 = (TermPositionVector) v2;
 					for (int j = 0; j < numTerms; j++)
 					{
 						int[] pos1 = tpv1.GetTermPositions(j);
 						int[] pos2 = tpv2.GetTermPositions(j);
 						Assert.AreEqual(pos1.Length, pos2.Length);
 						TermVectorOffsetInfo[] offsets1 = tpv1.GetOffsets(j);
 						TermVectorOffsetInfo[] offsets2 = tpv2.GetOffsets(j);
 						if (offsets1 == null)
 							Assert.IsTrue(offsets2 == null);
 						else
 							Assert.IsTrue(offsets2 != null);
 						for (int k = 0; k < pos1.Length; k++)
 						{
                             Assert.AreEqual(pos1[k], pos2[k]);
 							if (offsets1 != null)
 							{
 								Assert.AreEqual(offsets1[k].GetStartOffset(), offsets2[k].GetStartOffset());
 								Assert.AreEqual(offsets1[k].GetEndOffset(), offsets2[k].GetEndOffset());
 							}
 						}
 					}
 				}
 			}
 		}

 		internal class IndexingThread : SupportClass.ThreadClass
 		{
 			internal IndexWriter w;
 			internal int base_Renamed;
 			internal int range;
 			internal int iterations;
 			internal System.Collections.IDictionary docs = new System.Collections.Hashtable(); // Map<String,Document>
 			internal System.Random r;

 			public virtual int NextInt(int lim)
 			{
 				return r.Next(lim);
 			}

 			public virtual System.String GetString(int nTokens)
 			{
 				nTokens = nTokens != 0?nTokens:r.Next(4) + 1;
 				// avoid StringBuffer because it adds extra synchronization.
 				char[] arr = new char[nTokens * 2];
 				for (int i = 0; i < nTokens; i++)
 				{
 					arr[i * 2] = (char) ('A' + r.Next(10));
 					arr[i * 2 + 1] = ' ';
 				}
 				return new System.String(arr);
 			}


 			public virtual void  IndexDoc()
 			{
 				Document d = new Document();

 				System.Collections.ArrayList fields = new System.Collections.ArrayList();
 				int id = base_Renamed + NextInt(range);
 				System.String idString = "" + id;
 				Field idField = new Field("id", idString, Field.Store.YES, Field.Index.NO_NORMS);
 				fields.Add(idField);

 				int nFields = NextInt(Lucene.Net.Index.TestStressIndexing2.maxFields);
 				for (int i = 0; i < nFields; i++)
 				{

 					Field.TermVector tvVal = Field.TermVector.NO;
                     switch (NextInt(4))
                     {

                         case 0:
                             tvVal = Field.TermVector.NO;
                             break;

                         case 1:
                             tvVal = Field.TermVector.YES;
                             break;

                         case 2:
                             tvVal = Field.TermVector.WITH_POSITIONS;
                             break;

                         case 3:
                             tvVal = Field.TermVector.WITH_POSITIONS_OFFSETS;
                             break;
                     }

                     switch (NextInt(4))
                     {

                         case 0:
                             fields.Add(new Field("f0", GetString(1), Field.Store.YES, Field.Index.NO_NORMS, tvVal));
                             break;

                         case 1:
                             fields.Add(new Field("f1", GetString(0), Field.Store.NO, Field.Index.TOKENIZED, tvVal));
                             break;

                         case 2:
                             fields.Add(new Field("f2", GetString(0), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
                             break;

                         case 3:
                             fields.Add(new Field("f3", GetString(Lucene.Net.Index.TestStressIndexing2.bigFieldSize), Field.Store.YES, Field.Index.TOKENIZED, tvVal));
                             break;
                     }
                 }

 				if (Lucene.Net.Index.TestStressIndexing2.sameFieldOrder)
 				{
 					fields.Sort(Lucene.Net.Index.TestStressIndexing2.fieldNameComparator);
 				}
 				else
 				{
 					// random placement of id field also
                     int index = NextInt(fields.Count);
                     fields[0] = fields[index];
                     fields[index] = idField;
 				}

 				for (int i = 0; i < fields.Count; i++)
 				{
 					d.Add((Fieldable) fields[i]);
 				}
 				w.UpdateDocument(Lucene.Net.Index.TestStressIndexing2.idTerm.CreateTerm(idString), d);
 				// System.out.println("indexing "+d);
 				docs[idString] = d;
 			}

 			override public void  Run()
 			{
 				try
 				{
 					r = new System.Random((System.Int32) (base_Renamed + range + Lucene.Net.Index.TestStressIndexing2.seed));
 					for (int i = 0; i < iterations; i++)
 					{
 						IndexDoc();
 					}
 				}
 				catch (System.Exception e)
 				{
 					System.Console.Error.WriteLine(e.StackTrace);
 					Assert.Fail(e.ToString());
 				}

 				lock (this)
 				{
 					int generatedAux = docs.Count;
 				}
 			}
 		}
 		static TestStressIndexing2()
 		{
 			fieldNameComparator = new AnonymousClassComparator();
 		}
 	}
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	using System;

	using Lucene.Net.Documents;
	using Lucene.Net.Store;
	using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
	using Lucene.Net.Analysis;

	using NUnit.Framework;

	namespace Lucene.Net.Index
	{

	[TestFixture]
	public class TestStressIndexing2 : LuceneTestCase
	{
	internal class AnonymousClassComparator : System.Collections.IComparer
	{
	Fieldable f1, f2;
	public virtual int Compare(System.Object o1, System.Object o2)
	{
	if (o1 == o2) return 0;
	f1 = (Fieldable)o1;
	f2 = (Fieldable)o2;
	return String.CompareOrdinal(f1.Name() + f1.StringValue(), f2.Name() + f2.StringValue());
	}
	}
	internal static int maxFields = 4;
	internal static int bigFieldSize = 10;
	internal static bool sameFieldOrder = false;
	internal static bool autoCommit = false;
	internal static int mergeFactor = 3;
	internal static int maxBufferedDocs = 3;
	internal static int seed = 0;

	internal static System.Random r = new System.Random((System.Int32) 0);

	[Test]
	public virtual void TestRandom()
	{
	Directory dir1 = new MockRAMDirectory();
	Directory dir2 = new MockRAMDirectory();
	System.Collections.IDictionary docs = IndexRandom(10, 100, 100, dir1);
	IndexSerial(docs, dir2);
	VerifyEquals(dir1, dir2, "id");
	}

	[Test]
	public virtual void TestMultiConfig()
	{
	// test lots of smaller different params together
	for (int i = 0; i < 100; i++)
	{
	// increase iterations for better testing
	sameFieldOrder = r.NextDouble() > 0.5;
	autoCommit = r.NextDouble() > 0.5;
	mergeFactor = r.Next(3) + 2;
	maxBufferedDocs = r.Next(3) + 2;
	seed++;

	int nThreads = r.Next(5) + 1;
	int iter = r.Next(10) + 1;
	int range = r.Next(20) + 1;

	Directory dir1 = new MockRAMDirectory();
	Directory dir2 = new MockRAMDirectory();
	System.Collections.IDictionary docs = IndexRandom(nThreads, iter, range, dir1);
	IndexSerial(docs, dir2);
	VerifyEquals(dir1, dir2, "id");
	}
	}


	internal static Term idTerm = new Term("id", "");
	internal IndexingThread[] threads;
	internal static System.Collections.IComparer fieldNameComparator;

	// This test avoids using any extra synchronization in the multiple
	// indexing threads to test that IndexWriter does correctly synchronize
	// everything.

	public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir)
	{
	IndexWriter w = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
	w.SetUseCompoundFile(false);

	// force many merges
	w.SetMergeFactor(mergeFactor);
	w.SetRAMBufferSizeMB(.1);
	w.SetMaxBufferedDocs(maxBufferedDocs);

	threads = new IndexingThread[nThreads];
	for (int i = 0; i < threads.Length; i++)
	{
	IndexingThread th = new IndexingThread();
	th.w = w;
	th.base_Renamed = 1000000 * i;
	th.range = range;
	th.iterations = iterations;
	threads[i] = th;
	}

	for (int i = 0; i < threads.Length; i++)
	{
	threads[i].Start();
	}
	for (int i = 0; i < threads.Length; i++)
	{
	threads[i].Join();
	}

	// w.optimize();
	w.Close();

	System.Collections.IDictionary docs = new System.Collections.Hashtable();
	for (int i = 0; i < threads.Length; i++)
	{
	IndexingThread th = threads[i];
	lock (th)
	{
	System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator();
	while (e.MoveNext())
	{
	docs[e.Current] = th.docs[e.Current];
	}
	}
	}

	return docs;
	}


	public static void IndexSerial(System.Collections.IDictionary docs, Directory dir)
	{
	IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());

	// index all docs in a single thread
	System.Collections.IEnumerator iter = docs.Values.GetEnumerator();
	while (iter.MoveNext())
	{
	Document d = (Document) iter.Current;
	System.Collections.ArrayList fields = new System.Collections.ArrayList();
	fields.AddRange(d.GetFields());

	// nonono - can't do this (below)
	//
	// if multiple fields w/ same name, each instance must be
	// added in the same order as orginal doc, as the fields
	// are effectively concatendated
	//
	// term position/offset information must be maintained

	// put fields in same order each time
	//fields.Sort(fieldNameComparator);

	Document d1 = new Document();
	d1.SetBoost(d.GetBoost());
	for (int i = 0; i < fields.Count; i++)
	{
	d1.Add((Fieldable) fields[i]);
	}
	w.AddDocument(d1);
	// System.out.println("indexing "+d1);
	}

	w.Close();
	}

	public static void VerifyEquals(Directory dir1, Directory dir2, System.String idField)
	{
	IndexReader r1 = IndexReader.Open(dir1);
	IndexReader r2 = IndexReader.Open(dir2);
	VerifyEquals(r1, r2, idField);
	r1.Close();
	r2.Close();
	}


	public static void VerifyEquals(IndexReader r1, IndexReader r2, System.String idField)
	{
	Assert.AreEqual(r1.NumDocs(), r2.NumDocs());
	bool hasDeletes = !(r1.MaxDoc() == r2.MaxDoc() && r1.NumDocs() == r1.MaxDoc());

	int[] r2r1 = new int[r2.MaxDoc()]; // r2 id to r1 id mapping

	TermDocs termDocs1 = r1.TermDocs();
	TermDocs termDocs2 = r2.TermDocs();

	// create mapping from id2 space to id2 based on idField
	idField = String.Intern(idField);
	TermEnum termEnum = r1.Terms(new Term(idField, ""));
	do
	{
	Term term = termEnum.Term();
	if (term == null \|\| (System.Object) term.Field() != (System.Object) idField)
	break;

	termDocs1.Seek(termEnum);
	Assert.IsTrue(termDocs1.Next());
	int id1 = termDocs1.Doc();
	Assert.IsFalse(termDocs1.Next());

	termDocs2.Seek(termEnum);
	Assert.IsTrue(termDocs2.Next());
	int id2 = termDocs2.Doc();
	Assert.IsFalse(termDocs2.Next());

	r2r1[id2] = id1;

	// verify stored fields are equivalent
	VerifyEquals(r1.Document(id1), r2.Document(id2));

	try
	{
	// verify term vectors are equivalent
	VerifyEquals(r1.GetTermFreqVectors(id1), r2.GetTermFreqVectors(id2));
	}
	catch (System.ApplicationException e)
	{
	System.Console.Out.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
	TermFreqVector[] tv1 = r1.GetTermFreqVectors(id1);
	System.Console.Out.WriteLine(" d1=" + tv1);
	if (tv1 != null)
	for (int i = 0; i < tv1.Length; i++)
	{
	System.Console.Out.WriteLine(" " + i + ": " + tv1[i]);
	}

	TermFreqVector[] tv2 = r2.GetTermFreqVectors(id2);
	System.Console.Out.WriteLine(" d2=" + tv2);
	if (tv2 != null)
	for (int i = 0; i < tv2.Length; i++)
	{
	System.Console.Out.WriteLine(" " + i + ": " + tv2[i]);
	}

	throw e;
	}
	}
	while (termEnum.Next());

	termEnum.Close();

	// Verify postings
	TermEnum termEnum1 = r1.Terms(new Term("", ""));
	TermEnum termEnum2 = r2.Terms(new Term("", ""));

	// pack both doc and freq into single element for easy sorting
	long[] info1 = new long[r1.NumDocs()];
	long[] info2 = new long[r2.NumDocs()];

	for (; ; )
	{
	Term term1, term2;

	// iterate until we get some docs
	int len1;
	for (; ; )
	{
	len1 = 0;
	term1 = termEnum1.Term();
	if (term1 == null)
	break;
	termDocs1.Seek(termEnum1);
	while (termDocs1.Next())
	{
	int d1 = termDocs1.Doc();
	int f1 = termDocs1.Freq();
	info1[len1] = (((long) d1) << 32) \| f1;
	len1++;
	}
	if (len1 > 0)
	break;
	if (!termEnum1.Next())
	break;
	}

	// iterate until we get some docs
	int len2;
	for (; ; )
	{
	len2 = 0;
	term2 = termEnum2.Term();
	if (term2 == null)
	break;
	termDocs2.Seek(termEnum2);
	while (termDocs2.Next())
	{
	int d2 = termDocs2.Doc();
	int f2 = termDocs2.Freq();
	info2[len2] = (((long) r2r1[d2]) << 32) \| f2;
	len2++;
	}
	if (len2 > 0)
	break;
	if (!termEnum2.Next())
	break;
	}

	if (!hasDeletes)
	Assert.AreEqual(termEnum1.DocFreq(), termEnum2.DocFreq());

	Assert.AreEqual(len1, len2);
	if (len1 == 0)
	break; // no more terms

	Assert.AreEqual(term1, term2);

	// sort info2 to get it into ascending docid
	System.Array.Sort(info2, 0, len2 - 0);

	// now compare
	for (int i = 0; i < len1; i++)
	{
	Assert.AreEqual(info1[i], info2[i]);
	}

	termEnum1.Next();
	termEnum2.Next();
	}
	}

	public static void VerifyEquals(Document d1, Document d2)
	{
	System.Collections.ArrayList ff1 = new System.Collections.ArrayList(d1.GetFields());
	System.Collections.ArrayList ff2 = new System.Collections.ArrayList(d2.GetFields());

	ff1.Sort(fieldNameComparator);
	ff2.Sort(fieldNameComparator);

	if (ff1.Count != ff2.Count)
	{
	// print out whole doc on error
	System.Console.Write("Doc 1:");
	for (int j = 0; j < ff1.Count; j++)
	{
	Fieldable field = (Fieldable)ff1[j];
	System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
	}
	System.Console.WriteLine();
	System.Console.Write("Doc 2:");
	for (int j = 0; j < ff2.Count; j++)
	{
	Fieldable field = (Fieldable)ff2[j];
	System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
	}
	System.Console.WriteLine(); Assert.AreEqual(ff1.Count, ff2.Count);
	}

	for (int i = 0; i < ff1.Count; i++)
	{
	Fieldable f1 = (Fieldable) ff1[i];
	Fieldable f2 = (Fieldable) ff2[i];
	if (f1.IsBinary())
	{
	System.Diagnostics.Debug.Assert(f2.IsBinary());
	//TODO
	}
	else
	{
	System.String s1 = f1.StringValue();
	System.String s2 = f2.StringValue();
	if (!s1.Equals(s2))
	{
	// print out whole doc on error
	System.Console.Write("Doc 1:");
	for (int j = 0; j < ff1.Count; j++)
	{
	Fieldable field = (Fieldable)ff1[j];
	System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
	}
	System.Console.WriteLine();
	System.Console.Write("Doc 2:");
	for (int j = 0; j < ff2.Count; j++)
	{
	Fieldable field = (Fieldable)ff2[j];
	System.Console.Write(" {0}={1};", field.Name(), field.StringValue());
	}
	System.Console.WriteLine();
	Assert.AreEqual(s1, s2);
	}
	}
	}
	}

	public static void VerifyEquals(TermFreqVector[] d1, TermFreqVector[] d2)
	{
	if (d1 == null)
	{
	Assert.IsTrue(d2 == null);
	return ;
	}
	Assert.IsTrue(d2 != null);

	Assert.AreEqual(d1.Length, d2.Length);
	for (int i = 0; i < d1.Length; i++)
	{
	TermFreqVector v1 = d1[i];
	TermFreqVector v2 = d2[i];
	Assert.AreEqual(v1.Size(), v2.Size());
	int numTerms = v1.Size();
	System.String[] terms1 = v1.GetTerms();
	System.String[] terms2 = v2.GetTerms();
	int[] freq1 = v1.GetTermFrequencies();
	int[] freq2 = v2.GetTermFrequencies();
	for (int j = 0; j < numTerms; j++)
	{
	if (!terms1[j].Equals(terms2[j]))
	Assert.AreEqual(terms1[j], terms2[j]);
	Assert.AreEqual(freq1[j], freq2[j]);
	}
	if (v1 is TermPositionVector)
	{
	Assert.IsTrue(v2 is TermPositionVector);
	TermPositionVector tpv1 = (TermPositionVector) v1;
	TermPositionVector tpv2 = (TermPositionVector) v2;
	for (int j = 0; j < numTerms; j++)
	{
	int[] pos1 = tpv1.GetTermPositions(j);
	int[] pos2 = tpv2.GetTermPositions(j);
	Assert.AreEqual(pos1.Length, pos2.Length);
	TermVectorOffsetInfo[] offsets1 = tpv1.GetOffsets(j);
	TermVectorOffsetInfo[] offsets2 = tpv2.GetOffsets(j);
	if (offsets1 == null)
	Assert.IsTrue(offsets2 == null);
	else
	Assert.IsTrue(offsets2 != null);
	for (int k = 0; k < pos1.Length; k++)
	{
	Assert.AreEqual(pos1[k], pos2[k]);
	if (offsets1 != null)
	{
	Assert.AreEqual(offsets1[k].GetStartOffset(), offsets2[k].GetStartOffset());
	Assert.AreEqual(offsets1[k].GetEndOffset(), offsets2[k].GetEndOffset());
	}
	}
	}
	}
	}
	}

	internal class IndexingThread : SupportClass.ThreadClass
	{
	internal IndexWriter w;
	internal int base_Renamed;
	internal int range;
	internal int iterations;
	internal System.Collections.IDictionary docs = new System.Collections.Hashtable(); // Map<String,Document>
	internal System.Random r;

	public virtual int NextInt(int lim)
	{
	return r.Next(lim);
	}

	public virtual System.String GetString(int nTokens)
	{
	nTokens = nTokens != 0?nTokens:r.Next(4) + 1;
	// avoid StringBuffer because it adds extra synchronization.
	char[] arr = new char[nTokens * 2];
	for (int i = 0; i < nTokens; i++)
	{
	arr[i * 2] = (char) ('A' + r.Next(10));
	arr[i * 2 + 1] = ' ';
	}
	return new System.String(arr);
	}


	public virtual void IndexDoc()
	{
	Document d = new Document();

	System.Collections.ArrayList fields = new System.Collections.ArrayList();
	int id = base_Renamed + NextInt(range);
	System.String idString = "" + id;
	Field idField = new Field("id", idString, Field.Store.YES, Field.Index.NO_NORMS);
	fields.Add(idField);

	int nFields = NextInt(Lucene.Net.Index.TestStressIndexing2.maxFields);
	for (int i = 0; i < nFields; i++)
	{

	Field.TermVector tvVal = Field.TermVector.NO;
	switch (NextInt(4))
	{

	case 0:
	tvVal = Field.TermVector.NO;
	break;

	case 1:
	tvVal = Field.TermVector.YES;
	break;

	case 2:
	tvVal = Field.TermVector.WITH_POSITIONS;
	break;

	case 3:
	tvVal = Field.TermVector.WITH_POSITIONS_OFFSETS;
	break;
	}

	switch (NextInt(4))
	{

	case 0:
	fields.Add(new Field("f0", GetString(1), Field.Store.YES, Field.Index.NO_NORMS, tvVal));
	break;

	case 1:
	fields.Add(new Field("f1", GetString(0), Field.Store.NO, Field.Index.TOKENIZED, tvVal));
	break;

	case 2:
	fields.Add(new Field("f2", GetString(0), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
	break;

	case 3:
	fields.Add(new Field("f3", GetString(Lucene.Net.Index.TestStressIndexing2.bigFieldSize), Field.Store.YES, Field.Index.TOKENIZED, tvVal));
	break;
	}
	}

	if (Lucene.Net.Index.TestStressIndexing2.sameFieldOrder)
	{
	fields.Sort(Lucene.Net.Index.TestStressIndexing2.fieldNameComparator);
	}
	else
	{
	// random placement of id field also
	int index = NextInt(fields.Count);
	fields[0] = fields[index];
	fields[index] = idField;
	}

	for (int i = 0; i < fields.Count; i++)
	{
	d.Add((Fieldable) fields[i]);
	}
	w.UpdateDocument(Lucene.Net.Index.TestStressIndexing2.idTerm.CreateTerm(idString), d);
	// System.out.println("indexing "+d);
	docs[idString] = d;
	}

	override public void Run()
	{
	try
	{
	r = new System.Random((System.Int32) (base_Renamed + range + Lucene.Net.Index.TestStressIndexing2.seed));
	for (int i = 0; i < iterations; i++)
	{
	IndexDoc();
	}
	}
	catch (System.Exception e)
	{
	System.Console.Error.WriteLine(e.StackTrace);
	Assert.Fail(e.ToString());
	}

	lock (this)
	{
	int generatedAux = docs.Count;
	}
	}
	}
	static TestStressIndexing2()
	{
	fieldNameComparator = new AnonymousClassComparator();
	}
	}
	}