using Lucene.Net.Analysis.Tokenattributes;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using Lucene.Net.Attributes;
using Lucene.Net.Codecs;
using Lucene.Net.Documents;

namespace Lucene.Net.Index
{
    using Lucene.Net.Randomized.Generators;
    using Lucene.Net.Support;
    using NUnit.Framework;
    using Attribute = Lucene.Net.Util.Attribute;
    using BytesRef = Lucene.Net.Util.BytesRef;
    using CharTermAttribute = Lucene.Net.Analysis.Tokenattributes.CharTermAttribute;
    using Codec = Lucene.Net.Codecs.Codec;
    using Directory = Lucene.Net.Store.Directory;
    using Document = Documents.Document;
    using Field = Field;
    using FieldType = FieldType;
    using FixedBitSet = Lucene.Net.Util.FixedBitSet;
    using IndexSearcher = Lucene.Net.Search.IndexSearcher;
    using OffsetAttribute = Lucene.Net.Analysis.Tokenattributes.OffsetAttribute;
    using PayloadAttribute = Lucene.Net.Analysis.Tokenattributes.PayloadAttribute;
    using PositionIncrementAttribute = Lucene.Net.Analysis.Tokenattributes.PositionIncrementAttribute;
    using SeekStatus = Lucene.Net.Index.TermsEnum.SeekStatus;
    using StringField = StringField;
    using TermQuery = Lucene.Net.Search.TermQuery;
    using TestUtil = Lucene.Net.Util.TestUtil;
    using TextField = TextField;

    /*
         * Licensed to the Apache Software Foundation (ASF) under one or more
         * contributor license agreements.  See the NOTICE file distributed with
         * this work for additional information regarding copyright ownership.
         * The ASF licenses this file to You under the Apache License, Version 2.0
         * (the "License"); you may not use this file except in compliance with
         * the License.  You may obtain a copy of the License at
         *
         *     http://www.apache.org/licenses/LICENSE-2.0
         *
         * Unless required by applicable law or agreed to in writing, software
         * distributed under the License is distributed on an "AS IS" BASIS,
         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
         * See the License for the specific language governing permissions and
         * limitations under the License.
         */

    using TokenStream = Lucene.Net.Analysis.TokenStream;

    /// <summary>
    /// Base class aiming at testing <seealso cref="TermVectorsFormat"/>.
    /// To test a new format, all you need is to register a new <seealso cref="Codec"/> which
    /// uses it and extend this class and override <seealso cref="#getCodec()"/>.
    /// @lucene.experimental
    /// </summary>
    public abstract class BaseTermVectorsFormatTestCase : BaseIndexFileFormatTestCase
    {
        /// <summary>
        /// A combination of term vectors options.
        /// </summary>
        protected internal enum Options
        {
            NONE,//(false, false, false),
            POSITIONS,//(true, false, false),
            OFFSETS,//(false, true, false),
            POSITIONS_AND_OFFSETS,//(true, true, false),
            POSITIONS_AND_PAYLOADS,//(true, false, true),
            POSITIONS_AND_OFFSETS_AND_PAYLOADS//(true, true, true);
            //		final boolean positions, offsets, payloads;
            //		private Options(boolean positions, boolean offsets, boolean payloads)
            //	{
            //	  this.positions = positions;
            //	  this.offsets = offsets;
            //	  this.payloads = payloads;
            //	}
        }

        private class OptionsWrapper
        {
            public bool positions, offsets, payloads;

            private void SetOptionsWrapper(bool positions, bool offsets, bool payloads)
            {
                this.positions = positions;
                this.offsets = offsets;
                this.payloads = payloads;
            }

            public OptionsWrapper(Options opt)
            {
                switch (opt)
                {
                    case Options.NONE:
                        SetOptionsWrapper(false, false, false);
                        break;

                    case Options.POSITIONS:
                        SetOptionsWrapper(true, false, false);
                        break;

                    case Options.OFFSETS:
                        SetOptionsWrapper(false, true, false);
                        break;

                    case Options.POSITIONS_AND_OFFSETS:
                        SetOptionsWrapper(true, true, false);
                        break;

                    case Options.POSITIONS_AND_PAYLOADS:
                        SetOptionsWrapper(true, false, true);
                        break;

                    case Options.POSITIONS_AND_OFFSETS_AND_PAYLOADS:
                        SetOptionsWrapper(true, true, true);
                        break;

                    default:
                        throw new InvalidOperationException("Invalid Options enum type");
                }
            }

            public static IEnumerable<Options> GetAsEnumer()
            {
                return Enum.GetValues(typeof(Options)).Cast<Options>();
            }

            public static IEnumerable<Options> GetAsEnumer(Options startInc, Options endInc)
            {
                foreach (Options opt in Enum.GetValues(typeof(Options)))
                {
                    if (opt >= startInc && opt <= endInc)
                        yield return opt;
                }
            }
        }

        protected virtual IEnumerable<Options> ValidOptions()
        {
            return OptionsWrapper.GetAsEnumer();
        }

        protected virtual IEnumerable<Options> ValidOptions(Options startInc, Options endInc)
        {
            return OptionsWrapper.GetAsEnumer(startInc, endInc);
        }

        protected internal virtual Options RandomOptions()
        {
            return RandomInts.RandomFrom(Random(), new List<Options>(ValidOptions()));
        }

        protected internal virtual FieldType FieldType(Options options)
        {
            var ft = new FieldType(TextField.TYPE_NOT_STORED)
            {
                StoreTermVectors = true,
                StoreTermVectorPositions = (new OptionsWrapper(options)).positions,
                StoreTermVectorOffsets = (new OptionsWrapper(options)).offsets,
                StoreTermVectorPayloads = (new OptionsWrapper(options)).payloads
            };
            ft.Freeze();
            return ft;
        }

        protected internal virtual BytesRef RandomPayload()
        {
            int len = Random().Next(5);
            if (len == 0)
            {
                return null;
            }
            BytesRef payload = new BytesRef(len);
            Random().NextBytes((byte[])(Array)payload.Bytes);
            payload.Length = len;
            return payload;
        }

        protected internal override void AddRandomFields(Document doc)
        {
            foreach (Options opts in ValidOptions())
            {
                FieldType ft = FieldType(opts);
                int numFields = Random().Next(5);
                for (int j = 0; j < numFields; ++j)
                {
                    doc.Add(new Field("f_" + opts, TestUtil.RandomSimpleString(Random(), 2), ft));
                }
            }
        }

        // custom impl to test cases that are forbidden by the default OffsetAttribute impl
        private class PermissiveOffsetAttributeImpl : Attribute, IOffsetAttribute
        {
            internal int Start, End;

            public int StartOffset()
            {
                return Start;
            }

            public int EndOffset()
            {
                return End;
            }

            public void SetOffset(int startOffset, int endOffset)
            {
                // no check!
                Start = startOffset;
                End = endOffset;
            }

            public override void Clear()
            {
                Start = End = 0;
            }

            public override bool Equals(object other)
            {
                if (other == this)
                {
                    return true;
                }

                if (other is PermissiveOffsetAttributeImpl)
                {
                    PermissiveOffsetAttributeImpl o = (PermissiveOffsetAttributeImpl)other;
                    return o.Start == Start && o.End == End;
                }

                return false;
            }

            public override int GetHashCode()
            {
                return Start + 31 * End;
            }

            public override void CopyTo(Attribute target)
            {
                OffsetAttribute t = (OffsetAttribute)target;
                t.SetOffset(Start, End);
            }
        }

        // TODO: use CannedTokenStream?
        protected internal class RandomTokenStream : TokenStream
        {
            private readonly BaseTermVectorsFormatTestCase OuterInstance;

            internal readonly string[] Terms;
            internal readonly BytesRef[] TermBytes;
            internal readonly int[] PositionsIncrements;
            internal readonly int[] Positions;
            internal readonly int[] StartOffsets, EndOffsets;
            internal readonly BytesRef[] Payloads;

            internal readonly IDictionary<string, int?> Freqs;
            internal readonly IDictionary<int?, ISet<int?>> PositionToTerms;
            internal readonly IDictionary<int?, ISet<int?>> StartOffsetToTerms;

            internal readonly ICharTermAttribute TermAtt;
            internal readonly IPositionIncrementAttribute PiAtt;
            internal readonly IOffsetAttribute OAtt;
            internal readonly IPayloadAttribute PAtt;
            internal int i = 0;

            protected internal RandomTokenStream(BaseTermVectorsFormatTestCase outerInstance, int len, string[] sampleTerms, BytesRef[] sampleTermBytes)
                : this(outerInstance, len, sampleTerms, sampleTermBytes, Rarely())
            {
                this.OuterInstance = outerInstance;
            }

            protected internal RandomTokenStream(BaseTermVectorsFormatTestCase outerInstance, int len, string[] sampleTerms, BytesRef[] sampleTermBytes, bool offsetsGoBackwards)
            {
                this.OuterInstance = outerInstance;
                Terms = new string[len];
                TermBytes = new BytesRef[len];
                PositionsIncrements = new int[len];
                Positions = new int[len];
                StartOffsets = new int[len];
                EndOffsets = new int[len];
                Payloads = new BytesRef[len];
                for (int i = 0; i < len; ++i)
                {
                    int o = Random().Next(sampleTerms.Length);
                    Terms[i] = sampleTerms[o];
                    TermBytes[i] = sampleTermBytes[o];
                    PositionsIncrements[i] = TestUtil.NextInt(Random(), i == 0 ? 1 : 0, 10);
                    if (offsetsGoBackwards)
                    {
                        StartOffsets[i] = Random().Next();
                        EndOffsets[i] = Random().Next();
                    }
                    else
                    {
                        if (i == 0)
                        {
                            StartOffsets[i] = TestUtil.NextInt(Random(), 0, 1 << 16);
                        }
                        else
                        {
                            StartOffsets[i] = StartOffsets[i - 1] + TestUtil.NextInt(Random(), 0, Rarely() ? 1 << 16 : 20);
                        }
                        EndOffsets[i] = StartOffsets[i] + TestUtil.NextInt(Random(), 0, Rarely() ? 1 << 10 : 20);
                    }
                }

                for (int i = 0; i < len; ++i)
                {
                    if (i == 0)
                    {
                        Positions[i] = PositionsIncrements[i] - 1;
                    }
                    else
                    {
                        Positions[i] = Positions[i - 1] + PositionsIncrements[i];
                    }
                }
                if (Rarely())
                {
                    Arrays.Fill(Payloads, outerInstance.RandomPayload());
                }
                else
                {
                    for (int i = 0; i < len; ++i)
                    {
                        Payloads[i] = outerInstance.RandomPayload();
                    }
                }

                PositionToTerms = new Dictionary<int?, ISet<int?>>(len);
                StartOffsetToTerms = new Dictionary<int?, ISet<int?>>(len);
                for (int i = 0; i < len; ++i)
                {
                    if (!PositionToTerms.ContainsKey(Positions[i]))
                    {
                        PositionToTerms[Positions[i]] = new HashSet<int?>();//size1
                    }
                    PositionToTerms[Positions[i]].Add(i);
                    if (!StartOffsetToTerms.ContainsKey(StartOffsets[i]))
                    {
                        StartOffsetToTerms[StartOffsets[i]] = new HashSet<int?>();//size1
                    }
                    StartOffsetToTerms[StartOffsets[i]].Add(i);
                }

                Freqs = new Dictionary<string, int?>();
                foreach (string term in Terms)
                {
                    if (Freqs.ContainsKey(term))
                    {
                        Freqs[term] = Freqs[term] + 1;
                    }
                    else
                    {
                        Freqs[term] = 1;
                    }
                }

                AddAttributeImpl(new PermissiveOffsetAttributeImpl());

                TermAtt = AddAttribute<ICharTermAttribute>();
                PiAtt = AddAttribute<IPositionIncrementAttribute>();
                OAtt = AddAttribute<IOffsetAttribute>();
                PAtt = AddAttribute<IPayloadAttribute>();
            }

            public virtual bool HasPayloads()
            {
                foreach (BytesRef payload in Payloads)
                {
                    if (payload != null && payload.Length > 0)
                    {
                        return true;
                    }
                }
                return false;
            }

            public sealed override bool IncrementToken()
            {
                if (i < Terms.Length)
                {
                    TermAtt.SetLength(0).Append(Terms[i]);
                    PiAtt.PositionIncrement = PositionsIncrements[i];
                    OAtt.SetOffset(StartOffsets[i], EndOffsets[i]);
                    PAtt.Payload = Payloads[i];
                    ++i;
                    return true;
                }
                else
                {
                    return false;
                }
            }
        }

        protected internal class RandomDocument
        {
            private readonly BaseTermVectorsFormatTestCase OuterInstance;

            internal readonly string[] FieldNames;
            internal readonly FieldType[] FieldTypes;
            internal readonly RandomTokenStream[] TokenStreams;

            protected internal RandomDocument(BaseTermVectorsFormatTestCase outerInstance, int fieldCount, int maxTermCount, Options options, string[] fieldNames, string[] sampleTerms, BytesRef[] sampleTermBytes)
            {
                this.OuterInstance = outerInstance;
                if (fieldCount > fieldNames.Length)
                {
                    throw new System.ArgumentException();
                }
                this.FieldNames = new string[fieldCount];
                FieldTypes = new FieldType[fieldCount];
                TokenStreams = new RandomTokenStream[fieldCount];
                Arrays.Fill(FieldTypes, outerInstance.FieldType(options));
                HashSet<string> usedFileNames = new HashSet<string>();
                for (int i = 0; i < fieldCount; ++i)
                {
                    do
                    {
                        this.FieldNames[i] = RandomInts.RandomFrom(Random(), fieldNames);
                    } while (usedFileNames.Contains(this.FieldNames[i]));
                    usedFileNames.Add(this.FieldNames[i]);
                    TokenStreams[i] = new RandomTokenStream(outerInstance, TestUtil.NextInt(Random(), 1, maxTermCount), sampleTerms, sampleTermBytes);
                }
            }

            public virtual Document ToDocument()
            {
                Document doc = new Document();
                for (int i = 0; i < FieldNames.Length; ++i)
                {
                    doc.Add(new Field(FieldNames[i], TokenStreams[i], FieldTypes[i]));
                }
                return doc;
            }
        }

        protected internal class RandomDocumentFactory
        {
            private readonly BaseTermVectorsFormatTestCase OuterInstance;

            internal readonly string[] FieldNames;
            internal readonly string[] Terms;
            internal readonly BytesRef[] TermBytes;

            protected internal RandomDocumentFactory(BaseTermVectorsFormatTestCase outerInstance, int distinctFieldNames, int disctinctTerms)
            {
                this.OuterInstance = outerInstance;
                HashSet<string> fieldNames = new HashSet<string>();
                while (fieldNames.Count < distinctFieldNames)
                {
                    fieldNames.Add(TestUtil.RandomSimpleString(Random()));
                    fieldNames.Remove("id");
                }
                this.FieldNames = fieldNames.ToArray(/*new string[0]*/);
                Terms = new string[disctinctTerms];
                TermBytes = new BytesRef[disctinctTerms];
                for (int i = 0; i < disctinctTerms; ++i)
                {
                    Terms[i] = TestUtil.RandomRealisticUnicodeString(Random());
                    TermBytes[i] = new BytesRef(Terms[i]);
                }
            }

            public virtual RandomDocument NewDocument(int fieldCount, int maxTermCount, Options options)
            {
                return new RandomDocument(OuterInstance, fieldCount, maxTermCount, options, FieldNames, Terms, TermBytes);
            }
        }

        protected internal virtual void AssertEquals(RandomDocument doc, Fields fields)
        {
            // compare field names
            Assert.AreEqual(doc == null, fields == null);
            Assert.AreEqual(doc.FieldNames.Length, fields.Size);
            HashSet<string> fields1 = new HashSet<string>();
            HashSet<string> fields2 = new HashSet<string>();
            for (int i = 0; i < doc.FieldNames.Length; ++i)
            {
                fields1.Add(doc.FieldNames[i]);
            }
            foreach (string field in fields)
            {
                fields2.Add(field);
            }
            Assert.IsTrue(fields1.SetEquals(fields2));

            for (int i = 0; i < doc.FieldNames.Length; ++i)
            {
                AssertEquals(doc.TokenStreams[i], doc.FieldTypes[i], fields.Terms(doc.FieldNames[i]));
            }
        }

        protected internal static bool Equals(object o1, object o2)
        {
            if (o1 == null)
            {
                return o2 == null;
            }
            else
            {
                return o1.Equals(o2);
            }
        }

        // to test reuse
        private readonly ThreadLocal<TermsEnum> termsEnum = new ThreadLocal<TermsEnum>();

        private readonly ThreadLocal<DocsEnum> docsEnum = new ThreadLocal<DocsEnum>();
        private readonly ThreadLocal<DocsAndPositionsEnum> docsAndPositionsEnum = new ThreadLocal<DocsAndPositionsEnum>();

        protected internal virtual void AssertEquals(RandomTokenStream tk, FieldType ft, Terms terms)
        {
            Assert.AreEqual(1, terms.DocCount);
            int termCount = (new HashSet<string>(Arrays.AsList(tk.Terms))).Count;
            Assert.AreEqual(termCount, terms.Size());
            Assert.AreEqual(termCount, terms.SumDocFreq);
            Assert.AreEqual(ft.StoreTermVectorPositions, terms.HasPositions());
            Assert.AreEqual(ft.StoreTermVectorOffsets, terms.HasOffsets());
            Assert.AreEqual(ft.StoreTermVectorPayloads && tk.HasPayloads(), terms.HasPayloads());
            HashSet<BytesRef> uniqueTerms = new HashSet<BytesRef>();
            foreach (string term in tk.Freqs.Keys)
            {
                uniqueTerms.Add(new BytesRef(term));
            }
            BytesRef[] sortedTerms = uniqueTerms.ToArray(/*new BytesRef[0]*/);
            Array.Sort(sortedTerms, terms.Comparator);
            TermsEnum termsEnum = terms.Iterator(Random().NextBoolean() ? null : this.termsEnum.Value);
            this.termsEnum.Value = termsEnum;
            for (int i = 0; i < sortedTerms.Length; ++i)
            {
                BytesRef nextTerm = termsEnum.Next();
                Assert.AreEqual(sortedTerms[i], nextTerm);
                Assert.AreEqual(sortedTerms[i], termsEnum.Term());
                Assert.AreEqual(1, termsEnum.DocFreq());

                FixedBitSet bits = new FixedBitSet(1);
                DocsEnum docsEnum = termsEnum.Docs(bits, Random().NextBoolean() ? null : this.docsEnum.Value);
                Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc());
                bits.Set(0);

                docsEnum = termsEnum.Docs(Random().NextBoolean() ? bits : null, Random().NextBoolean() ? null : docsEnum);
                Assert.IsNotNull(docsEnum);
                Assert.AreEqual(0, docsEnum.NextDoc());
                Assert.AreEqual(0, docsEnum.DocID());
                Assert.AreEqual(tk.Freqs[termsEnum.Term().Utf8ToString()], (int?)docsEnum.Freq());
                Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc());
                this.docsEnum.Value = docsEnum;

                bits.Clear(0);
                DocsAndPositionsEnum docsAndPositionsEnum = termsEnum.DocsAndPositions(bits, Random().NextBoolean() ? null : this.docsAndPositionsEnum.Value);
                Assert.AreEqual(ft.StoreTermVectorOffsets || ft.StoreTermVectorPositions, docsAndPositionsEnum != null);
                if (docsAndPositionsEnum != null)
                {
                    Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.NextDoc());
                }
                bits.Set(0);

                docsAndPositionsEnum = termsEnum.DocsAndPositions(Random().NextBoolean() ? bits : null, Random().NextBoolean() ? null : docsAndPositionsEnum);
                Assert.AreEqual(ft.StoreTermVectorOffsets || ft.StoreTermVectorPositions, docsAndPositionsEnum != null);
                if (terms.HasPositions() || terms.HasOffsets())
                {
                    Assert.AreEqual(0, docsAndPositionsEnum.NextDoc());
                    int freq = docsAndPositionsEnum.Freq();
                    Assert.AreEqual(tk.Freqs[termsEnum.Term().Utf8ToString()], (int?)freq);
                    if (docsAndPositionsEnum != null)
                    {
                        for (int k = 0; k < freq; ++k)
                        {
                            int position = docsAndPositionsEnum.NextPosition();
                            ISet<int?> indexes;
                            if (terms.HasPositions())
                            {
                                indexes = tk.PositionToTerms[position];
                                Assert.IsNotNull(indexes);
                            }
                            else
                            {
                                indexes = tk.StartOffsetToTerms[docsAndPositionsEnum.StartOffset()];
                                Assert.IsNotNull(indexes);
                            }
                            if (terms.HasPositions())
                            {
                                bool foundPosition = false;
                                foreach (int index in indexes)
                                {
                                    if (tk.TermBytes[index].Equals(termsEnum.Term()) && tk.Positions[index] == position)
                                    {
                                        foundPosition = true;
                                        break;
                                    }
                                }
                                Assert.IsTrue(foundPosition);
                            }
                            if (terms.HasOffsets())
                            {
                                bool foundOffset = false;
                                foreach (int index in indexes)
                                {
                                    if (tk.TermBytes[index].Equals(termsEnum.Term()) && tk.StartOffsets[index] == docsAndPositionsEnum.StartOffset() && tk.EndOffsets[index] == docsAndPositionsEnum.EndOffset())
                                    {
                                        foundOffset = true;
                                        break;
                                    }
                                }
                                Assert.IsTrue(foundOffset);
                            }
                            if (terms.HasPayloads())
                            {
                                bool foundPayload = false;
                                foreach (int index in indexes)
                                {
                                    if (tk.TermBytes[index].Equals(termsEnum.Term()) && Equals(tk.Payloads[index], docsAndPositionsEnum.Payload))
                                    {
                                        foundPayload = true;
                                        break;
                                    }
                                }
                                Assert.IsTrue(foundPayload);
                            }
                        }
                        try
                        {
                            docsAndPositionsEnum.NextPosition();
                            Assert.Fail();
                        }
                        catch (Exception e)
                        {
                            // ok
                        }
                    }
                    Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.NextDoc());
                }
                this.docsAndPositionsEnum.Value = docsAndPositionsEnum;
            }
            Assert.IsNull(termsEnum.Next());
            for (int i = 0; i < 5; ++i)
            {
                if (Random().NextBoolean())
                {
                    Assert.IsTrue(termsEnum.SeekExact(RandomInts.RandomFrom(Random(), tk.TermBytes)));
                }
                else
                {
                    Assert.AreEqual(SeekStatus.FOUND, termsEnum.SeekCeil(RandomInts.RandomFrom(Random(), tk.TermBytes)));
                }
            }
        }

        protected internal virtual Document AddId(Document doc, string id)
        {
            doc.Add(new StringField("id", id, Field.Store.NO));
            return doc;
        }

        protected internal virtual int DocID(IndexReader reader, string id)
        {
            return (new IndexSearcher(reader)).Search(new TermQuery(new Term("id", id)), 1).ScoreDocs[0].Doc;
        }

        [Test]
        // only one doc with vectors
        public virtual void TestRareVectors()
        {
            RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 10, 20);
            foreach (Options options in ValidOptions())
            {
                int numDocs = AtLeast(200);
                int docWithVectors = Random().Next(numDocs);
                Document emptyDoc = new Document();
                Directory dir = NewDirectory();
                RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
                RandomDocument doc = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 3), 20, options);
                for (int i = 0; i < numDocs; ++i)
                {
                    if (i == docWithVectors)
                    {
                        writer.AddDocument(AddId(doc.ToDocument(), "42"));
                    }
                    else
                    {
                        writer.AddDocument(emptyDoc);
                    }
                }
                IndexReader reader = writer.Reader;
                int docWithVectorsID = DocID(reader, "42");
                for (int i = 0; i < 10; ++i)
                {
                    int docID = Random().Next(numDocs);
                    Fields fields = reader.GetTermVectors(docID);
                    if (docID == docWithVectorsID)
                    {
                        AssertEquals(doc, fields);
                    }
                    else
                    {
                        Assert.IsNull(fields);
                    }
                }
                Fields fields_ = reader.GetTermVectors(docWithVectorsID);
                AssertEquals(doc, fields_);
                reader.Dispose();
                writer.Dispose();
                dir.Dispose();
            }
        }

        [Test]
        public virtual void TestHighFreqs()
        {
            RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 3, 5);
            foreach (Options options in ValidOptions())
            {
                if (options == Options.NONE)
                {
                    continue;
                }
                using (Directory dir = NewDirectory())
                using (RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone))
                {
                    RandomDocument doc = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 2), AtLeast(20000),
                        options);
                    writer.AddDocument(doc.ToDocument());
                    using (IndexReader reader = writer.Reader)
                        AssertEquals(doc, reader.GetTermVectors(0));
                }
            }
        }

        [Test, LongRunningTest, Timeout(int.MaxValue)]
        public virtual void TestLotsOfFields()
        {
            RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 500, 10);
            foreach (Options options in ValidOptions())
            {
                Directory dir = NewDirectory();
                RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
                RandomDocument doc = docFactory.NewDocument(AtLeast(100), 5, options);
                writer.AddDocument(doc.ToDocument());
                IndexReader reader = writer.Reader;
                AssertEquals(doc, reader.GetTermVectors(0));
                reader.Dispose();
                writer.Dispose();
                dir.Dispose();
            }
        }

        [Test, Timeout(300000)]
        // different options for the same field
        public virtual void TestMixedOptions()
        {
            int numFields = TestUtil.NextInt(Random(), 1, 3);
            var docFactory = new RandomDocumentFactory(this, numFields, 10);
            foreach (var options1 in ValidOptions())
            {
                foreach (var options2 in ValidOptions())
                {
                    if (options1 == options2)
                    {
                        continue;
                    }
                    using (Directory dir = NewDirectory())
                    {
                        using (var writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone))
                        {
                            RandomDocument doc1 = docFactory.NewDocument(numFields, 20, options1);
                            RandomDocument doc2 = docFactory.NewDocument(numFields, 20, options2);
                            writer.AddDocument(AddId(doc1.ToDocument(), "1"));
                            writer.AddDocument(AddId(doc2.ToDocument(), "2"));
                            using (IndexReader reader = writer.Reader)
                            {
                                int doc1ID = DocID(reader, "1");
                                AssertEquals(doc1, reader.GetTermVectors(doc1ID));
                                int doc2ID = DocID(reader, "2");
                                AssertEquals(doc2, reader.GetTermVectors(doc2ID));
                            }
                        }
                    }
                }
            }
        }

        [Test]
        public virtual void TestRandom()
        {
            RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 5, 20);
            int numDocs = AtLeast(100);
            RandomDocument[] docs = new RandomDocument[numDocs];
            for (int i = 0; i < numDocs; ++i)
            {
                docs[i] = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 3), TestUtil.NextInt(Random(), 10, 50), RandomOptions());
            }
            Directory dir = NewDirectory();
            RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
            for (int i = 0; i < numDocs; ++i)
            {
                writer.AddDocument(AddId(docs[i].ToDocument(), "" + i));
            }
            IndexReader reader = writer.Reader;
            for (int i = 0; i < numDocs; ++i)
            {
                int docID = DocID(reader, "" + i);
                AssertEquals(docs[i], reader.GetTermVectors(docID));
            }
            reader.Dispose();
            writer.Dispose();
            dir.Dispose();
        }

        [Test]
        public virtual void TestMerge()
        {
            RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 5, 20);
            int numDocs = AtLeast(100);
            int numDeletes = Random().Next(numDocs);
            HashSet<int?> deletes = new HashSet<int?>();
            while (deletes.Count < numDeletes)
            {
                deletes.Add(Random().Next(numDocs));
            }
            foreach (Options options in ValidOptions())
            {
                RandomDocument[] docs = new RandomDocument[numDocs];
                for (int i = 0; i < numDocs; ++i)
                {
                    docs[i] = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 3), AtLeast(10), options);
                }
                Directory dir = NewDirectory();
                RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
                for (int i = 0; i < numDocs; ++i)
                {
                    writer.AddDocument(AddId(docs[i].ToDocument(), "" + i));
                    if (Rarely())
                    {
                        writer.Commit();
                    }
                }
                foreach (int delete in deletes)
                {
                    writer.DeleteDocuments(new Term("id", "" + delete));
                }
                // merge with deletes
                writer.ForceMerge(1);
                IndexReader reader = writer.Reader;
                for (int i = 0; i < numDocs; ++i)
                {
                    if (!deletes.Contains(i))
                    {
                        int docID = DocID(reader, "" + i);
                        AssertEquals(docs[i], reader.GetTermVectors(docID));
                    }
                }
                reader.Dispose();
                writer.Dispose();
                dir.Dispose();
            }
        }

        [Test]
        // run random tests from different threads to make sure the per-thread clones
        // don't share mutable data
        public virtual void TestClone()
        {
            RandomDocumentFactory docFactory = new RandomDocumentFactory(this, 5, 20);
            int numDocs = AtLeast(100);
            foreach (Options options in ValidOptions())
            {
                RandomDocument[] docs = new RandomDocument[numDocs];
                for (int i = 0; i < numDocs; ++i)
                {
                    docs[i] = docFactory.NewDocument(TestUtil.NextInt(Random(), 1, 3), AtLeast(10), options);
                }
                Directory dir = NewDirectory();
                RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, ClassEnvRule.Similarity, ClassEnvRule.TimeZone);
                for (int i = 0; i < numDocs; ++i)
                {
                    writer.AddDocument(AddId(docs[i].ToDocument(), "" + i));
                }
                IndexReader reader = writer.Reader;
                for (int i = 0; i < numDocs; ++i)
                {
                    int docID = DocID(reader, "" + i);
                    AssertEquals(docs[i], reader.GetTermVectors(docID));
                }

                AtomicObject<Exception> exception = new AtomicObject<Exception>();
                ThreadClass[] threads = new ThreadClass[2];
                for (int i = 0; i < threads.Length; ++i)
                {
                    threads[i] = new ThreadAnonymousInnerClassHelper(this, numDocs, docs, reader, exception, i);
                }
                foreach (ThreadClass thread in threads)
                {
                    thread.Start();
                }
                foreach (ThreadClass thread in threads)
                {
                    thread.Join();
                }
                reader.Dispose();
                writer.Dispose();
                dir.Dispose();
                Assert.IsNull(exception.Value, "One thread threw an exception");
            }
        }

        private class ThreadAnonymousInnerClassHelper : ThreadClass
        {
            private readonly BaseTermVectorsFormatTestCase OuterInstance;

            private int NumDocs;
            private Lucene.Net.Index.BaseTermVectorsFormatTestCase.RandomDocument[] Docs;
            private IndexReader Reader;
            private AtomicObject<Exception> ARException;
            private int i;

            public ThreadAnonymousInnerClassHelper(BaseTermVectorsFormatTestCase outerInstance, int numDocs, Lucene.Net.Index.BaseTermVectorsFormatTestCase.RandomDocument[] docs, IndexReader reader, AtomicObject<Exception> exception, int i)
            {
                this.OuterInstance = outerInstance;
                this.NumDocs = numDocs;
                this.Docs = docs;
                this.Reader = reader;
                this.ARException = exception;
                this.i = i;
            }

            public override void Run()
            {
                try
                {
                    for (int i = 0; i < AtLeast(100); ++i)
                    {
                        int idx = Random().Next(NumDocs);
                        int docID = OuterInstance.DocID(Reader, "" + idx);
                        OuterInstance.AssertEquals(Docs[idx], Reader.GetTermVectors(docID));
                    }
                }
                catch (Exception t)
                {
                    this.ARException.Value = t;
                }
            }
        }
    }
}