using Lucene.Net.Documents;
using Lucene.Net.Randomized.Generators;
using Lucene.Net.Support;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;

namespace Lucene.Net.Index
{
    using NUnit.Framework;
    using Bits = Lucene.Net.Util.Bits;
    using BytesRef = Lucene.Net.Util.BytesRef;

    /*
         * Licensed to the Apache Software Foundation (ASF) under one or more
         * contributor license agreements.  See the NOTICE file distributed with
         * this work for additional information regarding copyright ownership.
         * The ASF licenses this file to You under the Apache License, Version 2.0
         * (the "License"); you may not use this file except in compliance with
         * the License.  You may obtain a copy of the License at
         *
         *     http://www.apache.org/licenses/LICENSE-2.0
         *
         * Unless required by applicable law or agreed to in writing, software
         * distributed under the License is distributed on an "AS IS" BASIS,
         * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
         * See the License for the specific language governing permissions and
         * limitations under the License.
         */

    using Codec = Lucene.Net.Codecs.Codec;
    using Constants = Lucene.Net.Util.Constants;
    using Directory = Lucene.Net.Store.Directory;
    using Document = Documents.Document;
    using DocValuesType = Lucene.Net.Index.FieldInfo.DocValuesType_e;
    using Field = Field;
    using FieldsConsumer = Lucene.Net.Codecs.FieldsConsumer;
    using FieldsProducer = Lucene.Net.Codecs.FieldsProducer;
    using FieldType = FieldType;
    using FixedBitSet = Lucene.Net.Util.FixedBitSet;
    using FlushInfo = Lucene.Net.Store.FlushInfo;
    using IOContext = Lucene.Net.Store.IOContext;
    using PostingsConsumer = Lucene.Net.Codecs.PostingsConsumer;
    using TermsConsumer = Lucene.Net.Codecs.TermsConsumer;
    using TermStats = Lucene.Net.Codecs.TermStats;
    using TestUtil = Lucene.Net.Util.TestUtil;

    /// <summary>
    /// Abstract class to do basic tests for a postings format.
    /// NOTE: this test focuses on the postings
    /// (docs/freqs/positions/payloads/offsets) impl, not the
    /// terms dict.  The [stretch] goal is for this test to be
    /// so thorough in testing a new PostingsFormat that if this
    /// test passes, then all Lucene/Solr tests should also pass.  Ie,
    /// if there is some bug in a given PostingsFormat that this
    /// test fails to catch then this test needs to be improved!
    /// </summary>

    // TODO can we make it easy for testing to pair up a "random terms dict impl" with your postings base format...

    // TODO test when you reuse after skipping a term or two, eg the block reuse case

    /* TODO
      - threads
      - assert doc=-1 before any nextDoc
      - if a PF passes this test but fails other tests then this
        test has a bug!!
      - test tricky reuse cases, eg across fields
      - verify you get null if you pass needFreq/needOffset but
        they weren't indexed
    */

    [TestFixture]
    public abstract class BasePostingsFormatTestCase : BaseIndexFileFormatTestCase
    {
        private enum Option
        {
            // Sometimes use .Advance():
            SKIPPING,

            // Sometimes reuse the Docs/AndPositionsEnum across terms:
            REUSE_ENUMS,

            // Sometimes pass non-null live docs:
            LIVE_DOCS,

            // Sometimes seek to term using previously saved TermState:
            TERM_STATE,

            // Sometimes don't fully consume docs from the enum
            PARTIAL_DOC_CONSUME,

            // Sometimes don't fully consume positions at each doc
            PARTIAL_POS_CONSUME,

            // Sometimes check payloads
            PAYLOADS,

            // Test w/ multiple threads
            THREADS
        }

        /// <summary>
        /// Given the same random seed this always enumerates the
        ///  same random postings
        /// </summary>
        private class SeedPostings : DocsAndPositionsEnum
        {
            // Used only to generate docIDs; this way if you pull w/
            // or w/o positions you get the same docID sequence:
            internal readonly Random DocRandom;

            internal readonly Random Random;
            public int DocFreq;
            internal readonly int MaxDocSpacing;
            internal readonly int PayloadSize;
            internal readonly bool FixedPayloads;
            internal readonly Bits LiveDocs;
            internal readonly BytesRef Payload_Renamed;
            internal readonly FieldInfo.IndexOptions Options;
            internal readonly bool DoPositions;

            internal int DocID_Renamed;
            internal int Freq_Renamed;
            public int Upto;

            internal int Pos;
            internal int Offset;
            internal int StartOffset_Renamed;
            internal int EndOffset_Renamed;
            internal int PosSpacing;
            internal int PosUpto;

            public SeedPostings(long seed, int minDocFreq, int maxDocFreq, Bits liveDocs, FieldInfo.IndexOptions options)
            {
                Random = new Random((int)seed);
                DocRandom = new Random(Random.Next());
                DocFreq = TestUtil.NextInt(Random, minDocFreq, maxDocFreq);
                this.LiveDocs = liveDocs;

                // TODO: more realistic to inversely tie this to numDocs:
                MaxDocSpacing = TestUtil.NextInt(Random, 1, 100);

                if (Random.Next(10) == 7)
                {
                    // 10% of the time create big payloads:
                    PayloadSize = 1 + Random.Next(3);
                }
                else
                {
                    PayloadSize = 1 + Random.Next(1);
                }

                FixedPayloads = Random.NextBoolean();
                var payloadBytes = new byte[PayloadSize];
                Payload_Renamed = new BytesRef(payloadBytes);
                this.Options = options;
                DoPositions = FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS.CompareTo(options) <= 0;
            }

            public override int NextDoc()
            {
                while (true)
                {
                    _nextDoc();
                    if (LiveDocs == null || DocID_Renamed == NO_MORE_DOCS || LiveDocs.Get(DocID_Renamed))
                    {
                        return DocID_Renamed;
                    }
                }
            }

            internal virtual int _nextDoc()
            {
                // Must consume random:
                while (PosUpto < Freq_Renamed)
                {
                    NextPosition();
                }

                if (Upto < DocFreq)
                {
                    if (Upto == 0 && DocRandom.NextBoolean())
                    {
                        // Sometimes index docID = 0
                    }
                    else if (MaxDocSpacing == 1)
                    {
                        DocID_Renamed++;
                    }
                    else
                    {
                        // TODO: sometimes have a biggish gap here!
                        DocID_Renamed += TestUtil.NextInt(DocRandom, 1, MaxDocSpacing);
                    }

                    if (Random.Next(200) == 17)
                    {
                        Freq_Renamed = TestUtil.NextInt(Random, 1, 1000);
                    }
                    else if (Random.Next(10) == 17)
                    {
                        Freq_Renamed = TestUtil.NextInt(Random, 1, 20);
                    }
                    else
                    {
                        Freq_Renamed = TestUtil.NextInt(Random, 1, 4);
                    }

                    Pos = 0;
                    Offset = 0;
                    PosUpto = 0;
                    PosSpacing = TestUtil.NextInt(Random, 1, 100);

                    Upto++;
                    return DocID_Renamed;
                }
                else
                {
                    return DocID_Renamed = NO_MORE_DOCS;
                }
            }

            public override int DocID()
            {
                return DocID_Renamed;
            }

            public override int Freq()
            {
                return Freq_Renamed;
            }

            public override int NextPosition()
            {
                if (!DoPositions)
                {
                    PosUpto = Freq_Renamed;
                    return 0;
                }
                Debug.Assert(PosUpto < Freq_Renamed);

                if (PosUpto == 0 && Random.NextBoolean())
                {
                    // Sometimes index pos = 0
                }
                else if (PosSpacing == 1)
                {
                    Pos++;
                }
                else
                {
                    Pos += TestUtil.NextInt(Random, 1, PosSpacing);
                }

                if (PayloadSize != 0)
                {
                    if (FixedPayloads)
                    {
                        Payload_Renamed.Length = PayloadSize;
                        Random.NextBytes((byte[])(Array)Payload_Renamed.Bytes);
                    }
                    else
                    {
                        int thisPayloadSize = Random.Next(PayloadSize);
                        if (thisPayloadSize != 0)
                        {
                            Payload_Renamed.Length = PayloadSize;
                            Random.NextBytes((byte[])(Array)Payload_Renamed.Bytes);
                        }
                        else
                        {
                            Payload_Renamed.Length = 0;
                        }
                    }
                }
                else
                {
                    Payload_Renamed.Length = 0;
                }

                StartOffset_Renamed = Offset + Random.Next(5);
                EndOffset_Renamed = StartOffset_Renamed + Random.Next(10);
                Offset = EndOffset_Renamed;

                PosUpto++;
                return Pos;
            }

            public override int StartOffset()
            {
                return StartOffset_Renamed;
            }

            public override int EndOffset()
            {
                return EndOffset_Renamed;
            }

            public override BytesRef Payload
            {
                get
                {
                    return Payload_Renamed.Length == 0 ? null : Payload_Renamed;
                }
            }

            public override int Advance(int target)
            {
                return SlowAdvance(target);
            }

            public override long Cost()
            {
                return DocFreq;
            }
        }

        private class FieldAndTerm
        {
            internal string Field;
            internal BytesRef Term;

            public FieldAndTerm(string field, BytesRef term)
            {
                this.Field = field;
                this.Term = BytesRef.DeepCopyOf(term);
            }
        }

        // Holds all postings:
        private static SortedDictionary<string, SortedDictionary<BytesRef, long>> Fields;

        private static FieldInfos FieldInfos;

        private static FixedBitSet GlobalLiveDocs;

        private static IList<FieldAndTerm> AllTerms;
        private static int MaxDoc;

        private static long TotalPostings;
        private static long TotalPayloadBytes;

        private static SeedPostings GetSeedPostings(string term, long seed, bool withLiveDocs, FieldInfo.IndexOptions options)
        {
            int minDocFreq, maxDocFreq;
            if (term.StartsWith("big_"))
            {
                minDocFreq = RANDOM_MULTIPLIER * 50000;
                maxDocFreq = RANDOM_MULTIPLIER * 70000;
            }
            else if (term.StartsWith("medium_"))
            {
                minDocFreq = RANDOM_MULTIPLIER * 3000;
                maxDocFreq = RANDOM_MULTIPLIER * 6000;
            }
            else if (term.StartsWith("low_"))
            {
                minDocFreq = RANDOM_MULTIPLIER;
                maxDocFreq = RANDOM_MULTIPLIER * 40;
            }
            else
            {
                minDocFreq = 1;
                maxDocFreq = 3;
            }

            return new SeedPostings(seed, minDocFreq, maxDocFreq, withLiveDocs ? GlobalLiveDocs : null, options);
        }

        [TestFixtureSetUp]
        public static void CreatePostings()
        {
            TotalPostings = 0;
            TotalPayloadBytes = 0;
            Fields = new SortedDictionary<string, SortedDictionary<BytesRef, long>>();

            int numFields = TestUtil.NextInt(Random(), 1, 5);
            if (VERBOSE)
            {
                Console.WriteLine("TEST: " + numFields + " fields");
            }
            MaxDoc = 0;

            FieldInfo[] fieldInfoArray = new FieldInfo[numFields];
            int fieldUpto = 0;
            while (fieldUpto < numFields)
            {
                string field = TestUtil.RandomSimpleString(Random());
                if (Fields.ContainsKey(field))
                {
                    continue;
                }

                fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true, FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, null, DocValuesType.NUMERIC, null);
                fieldUpto++;

                SortedDictionary<BytesRef, long> postings = new SortedDictionary<BytesRef, long>();
                Fields[field] = postings;
                HashSet<string> seenTerms = new HashSet<string>();

                int numTerms;
                if (Random().Next(10) == 7)
                {
                    numTerms = AtLeast(50);
                }
                else
                {
                    numTerms = TestUtil.NextInt(Random(), 2, 20);
                }

                for (int termUpto = 0; termUpto < numTerms; termUpto++)
                {
                    string term = TestUtil.RandomSimpleString(Random());
                    if (seenTerms.Contains(term))
                    {
                        continue;
                    }
                    seenTerms.Add(term);

                    if (TEST_NIGHTLY && termUpto == 0 && fieldUpto == 1)
                    {
                        // Make 1 big term:
                        term = "big_" + term;
                    }
                    else if (termUpto == 1 && fieldUpto == 1)
                    {
                        // Make 1 medium term:
                        term = "medium_" + term;
                    }
                    else if (Random().NextBoolean())
                    {
                        // Low freq term:
                        term = "low_" + term;
                    }
                    else
                    {
                        // Very low freq term (don't multiply by RANDOM_MULTIPLIER):
                        term = "verylow_" + term;
                    }

                    long termSeed = Random().NextLong();
                    postings[new BytesRef(term)] = termSeed;

                    // NOTE: sort of silly: we enum all the docs just to
                    // get the maxDoc
                    DocsEnum docsEnum = GetSeedPostings(term, termSeed, false, FieldInfo.IndexOptions.DOCS_ONLY);
                    int doc;
                    int lastDoc = 0;
                    while ((doc = docsEnum.NextDoc()) != DocsEnum.NO_MORE_DOCS)
                    {
                        lastDoc = doc;
                    }
                    MaxDoc = Math.Max(lastDoc, MaxDoc);
                }
            }

            FieldInfos = new FieldInfos(fieldInfoArray);

            // It's the count, not the last docID:
            MaxDoc++;

            GlobalLiveDocs = new FixedBitSet(MaxDoc);
            double liveRatio = Random().NextDouble();
            for (int i = 0; i < MaxDoc; i++)
            {
                if (Random().NextDouble() <= liveRatio)
                {
                    GlobalLiveDocs.Set(i);
                }
            }

            AllTerms = new List<FieldAndTerm>();
            foreach (KeyValuePair<string, SortedDictionary<BytesRef, long>> fieldEnt in Fields)
            {
                string field = fieldEnt.Key;
                foreach (KeyValuePair<BytesRef, long> termEnt in fieldEnt.Value.EntrySet())
                {
                    AllTerms.Add(new FieldAndTerm(field, termEnt.Key));
                }
            }

            if (VERBOSE)
            {
                Console.WriteLine("TEST: done init postings; " + AllTerms.Count + " total terms, across " + FieldInfos.Size() + " fields");
            }
        }

        [TestFixtureTearDown]
        public static void AfterClass()
        {
            AllTerms = null;
            FieldInfos = null;
            Fields = null;
            GlobalLiveDocs = null;
        }

        // TODO maybe instead of @BeforeClass just make a single test run: build postings & index & test it?

        private FieldInfos CurrentFieldInfos;

        // maxAllowed = the "highest" we can index, but we will still
        // randomly index at lower IndexOption
        private FieldsProducer BuildIndex(Directory dir, FieldInfo.IndexOptions maxAllowed, bool allowPayloads, bool alwaysTestMax)
        {
            Codec codec = Codec;
            SegmentInfo segmentInfo = new SegmentInfo(dir, Constants.LUCENE_MAIN_VERSION, "_0", MaxDoc, false, codec, null);

            int maxIndexOption = Enum.GetValues(typeof(FieldInfo.IndexOptions)).Cast<FieldInfo.IndexOptions>().ToList().IndexOf(maxAllowed);
            if (VERBOSE)
            {
                Console.WriteLine("\nTEST: now build index");
            }

            int maxIndexOptionNoOffsets = Enum.GetValues(typeof(FieldInfo.IndexOptions)).Cast<FieldInfo.IndexOptions>().ToList().IndexOf(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);

            // TODO use allowPayloads

            var newFieldInfoArray = new FieldInfo[Fields.Count];
            for (int fieldUpto = 0; fieldUpto < Fields.Count; fieldUpto++)
            {
                FieldInfo oldFieldInfo = FieldInfos.FieldInfo(fieldUpto);

                string pf = TestUtil.GetPostingsFormat(codec, oldFieldInfo.Name);
                int fieldMaxIndexOption;
                if (DoesntSupportOffsets.Contains(pf))
                {
                    fieldMaxIndexOption = Math.Min(maxIndexOptionNoOffsets, maxIndexOption);
                }
                else
                {
                    fieldMaxIndexOption = maxIndexOption;
                }

                // Randomly picked the IndexOptions to index this
                // field with:
                FieldInfo.IndexOptions indexOptions = Enum.GetValues(typeof(FieldInfo.IndexOptions)).Cast<FieldInfo.IndexOptions>().ToArray()[alwaysTestMax ? fieldMaxIndexOption : Random().Next(1 + fieldMaxIndexOption)];
                bool doPayloads = indexOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 && allowPayloads;

                newFieldInfoArray[fieldUpto] = new FieldInfo(oldFieldInfo.Name, true, fieldUpto, false, false, doPayloads, indexOptions, null, DocValuesType.NUMERIC, null);
            }

            FieldInfos newFieldInfos = new FieldInfos(newFieldInfoArray);

            // Estimate that flushed segment size will be 25% of
            // what we use in RAM:
            long bytes = TotalPostings * 8 + TotalPayloadBytes;

            SegmentWriteState writeState = new SegmentWriteState(null, dir, segmentInfo, newFieldInfos, 32, null, new IOContext(new FlushInfo(MaxDoc, bytes)));
            FieldsConsumer fieldsConsumer = codec.PostingsFormat().FieldsConsumer(writeState);

            foreach (KeyValuePair<string, SortedDictionary<BytesRef, long>> fieldEnt in Fields)
            {
                string field = fieldEnt.Key;
                IDictionary<BytesRef, long> terms = fieldEnt.Value;

                FieldInfo fieldInfo = newFieldInfos.FieldInfo(field);

                FieldInfo.IndexOptions? indexOptions = fieldInfo.FieldIndexOptions;

                if (VERBOSE)
                {
                    Console.WriteLine("field=" + field + " indexOtions=" + indexOptions);
                }

                bool doFreq = indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS;
                bool doPos = indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
                bool doPayloads = indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && allowPayloads;
                bool doOffsets = indexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;

                TermsConsumer termsConsumer = fieldsConsumer.AddField(fieldInfo);
                long sumTotalTF = 0;
                long sumDF = 0;
                FixedBitSet seenDocs = new FixedBitSet(MaxDoc);
                foreach (KeyValuePair<BytesRef, long> termEnt in terms)
                {
                    BytesRef term = termEnt.Key;
                    SeedPostings postings = GetSeedPostings(term.Utf8ToString(), termEnt.Value, false, maxAllowed);
                    if (VERBOSE)
                    {
                        Console.WriteLine("  term=" + field + ":" + term.Utf8ToString() + " docFreq=" + postings.DocFreq + " seed=" + termEnt.Value);
                    }

                    PostingsConsumer postingsConsumer = termsConsumer.StartTerm(term);
                    long totalTF = 0;
                    int docID = 0;
                    while ((docID = postings.NextDoc()) != DocsEnum.NO_MORE_DOCS)
                    {
                        int freq = postings.Freq();
                        if (VERBOSE)
                        {
                            Console.WriteLine("    " + postings.Upto + ": docID=" + docID + " freq=" + postings.Freq_Renamed);
                        }
                        postingsConsumer.StartDoc(docID, doFreq ? postings.Freq_Renamed : -1);
                        seenDocs.Set(docID);
                        if (doPos)
                        {
                            totalTF += postings.Freq_Renamed;
                            for (int posUpto = 0; posUpto < freq; posUpto++)
                            {
                                int pos = postings.NextPosition();
                                BytesRef payload = postings.Payload;

                                if (VERBOSE)
                                {
                                    if (doPayloads)
                                    {
                                        Console.WriteLine("      pos=" + pos + " payload=" + (payload == null ? "null" : payload.Length + " bytes"));
                                    }
                                    else
                                    {
                                        Console.WriteLine("      pos=" + pos);
                                    }
                                }
                                postingsConsumer.AddPosition(pos, doPayloads ? payload : null, doOffsets ? postings.StartOffset() : -1, doOffsets ? postings.EndOffset() : -1);
                            }
                        }
                        else if (doFreq)
                        {
                            totalTF += freq;
                        }
                        else
                        {
                            totalTF++;
                        }
                        postingsConsumer.FinishDoc();
                    }
                    termsConsumer.FinishTerm(term, new TermStats(postings.DocFreq, doFreq ? totalTF : -1));
                    sumTotalTF += totalTF;
                    sumDF += postings.DocFreq;
                }

                termsConsumer.Finish(doFreq ? sumTotalTF : -1, sumDF, seenDocs.Cardinality());
            }

            fieldsConsumer.Dispose();

            if (VERBOSE)
            {
                Console.WriteLine("TEST: after indexing: files=");
                foreach (string file in dir.ListAll())
                {
                    Console.WriteLine("  " + file + ": " + dir.FileLength(file) + " bytes");
                }
            }

            CurrentFieldInfos = newFieldInfos;

            SegmentReadState readState = new SegmentReadState(dir, segmentInfo, newFieldInfos, IOContext.READ, 1);

            return codec.PostingsFormat().FieldsProducer(readState);
        }

        private class ThreadState
        {
            // Only used with REUSE option:
            public DocsEnum ReuseDocsEnum;

            public DocsAndPositionsEnum ReuseDocsAndPositionsEnum;
        }

        private void VerifyEnum(ThreadState threadState, string field, BytesRef term, TermsEnum termsEnum, FieldInfo.IndexOptions maxTestOptions, FieldInfo.IndexOptions maxIndexOptions, ISet<Option> options, bool alwaysTestMax)
        // Maximum options (docs/freqs/positions/offsets) to test:
        {
            if (VERBOSE)
            {
                Console.WriteLine("  verifyEnum: options=" + options + " maxTestOptions=" + maxTestOptions);
            }

            // Make sure TermsEnum really is positioned on the
            // expected term:
            Assert.AreEqual(term, termsEnum.Term());

            // 50% of the time time pass liveDocs:
            bool useLiveDocs = options.Contains(Option.LIVE_DOCS) && Random().NextBoolean();
            Bits liveDocs;
            if (useLiveDocs)
            {
                liveDocs = GlobalLiveDocs;
                if (VERBOSE)
                {
                    Console.WriteLine("  use liveDocs");
                }
            }
            else
            {
                liveDocs = null;
                if (VERBOSE)
                {
                    Console.WriteLine("  no liveDocs");
                }
            }

            FieldInfo fieldInfo = CurrentFieldInfos.FieldInfo(field);

            // NOTE: can be empty list if we are using liveDocs:
            SeedPostings expected = GetSeedPostings(term.Utf8ToString(), Fields[field][term], useLiveDocs, maxIndexOptions);
            Assert.AreEqual(expected.DocFreq, termsEnum.DocFreq());

            bool allowFreqs = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS && maxTestOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS) >= 0;
            bool doCheckFreqs = allowFreqs && (alwaysTestMax || Random().Next(3) <= 2);

            bool allowPositions = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS && maxTestOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
            bool doCheckPositions = allowPositions && (alwaysTestMax || Random().Next(3) <= 2);

            bool allowOffsets = fieldInfo.FieldIndexOptions >= FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS && maxTestOptions.CompareTo(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
            bool doCheckOffsets = allowOffsets && (alwaysTestMax || Random().Next(3) <= 2);

            bool doCheckPayloads = options.Contains(Option.PAYLOADS) && allowPositions && fieldInfo.HasPayloads() && (alwaysTestMax || Random().Next(3) <= 2);

            DocsEnum prevDocsEnum = null;

            DocsEnum docsEnum;
            DocsAndPositionsEnum docsAndPositionsEnum;

            if (!doCheckPositions)
            {
                if (allowPositions && Random().Next(10) == 7)
                {
                    // 10% of the time, even though we will not check positions, pull a DocsAndPositions enum

                    if (options.Contains(Option.REUSE_ENUMS) && Random().Next(10) < 9)
                    {
                        prevDocsEnum = threadState.ReuseDocsAndPositionsEnum;
                    }

                    int flags = 0;
                    if (alwaysTestMax || Random().NextBoolean())
                    {
                        flags |= DocsAndPositionsEnum.FLAG_OFFSETS;
                    }
                    if (alwaysTestMax || Random().NextBoolean())
                    {
                        flags |= DocsAndPositionsEnum.FLAG_PAYLOADS;
                    }

                    if (VERBOSE)
                    {
                        Console.WriteLine("  get DocsAndPositionsEnum (but we won't check positions) flags=" + flags);
                    }

                    threadState.ReuseDocsAndPositionsEnum = termsEnum.DocsAndPositions(liveDocs, (DocsAndPositionsEnum)prevDocsEnum, flags);
                    docsEnum = threadState.ReuseDocsAndPositionsEnum;
                    docsAndPositionsEnum = threadState.ReuseDocsAndPositionsEnum;
                }
                else
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("  get DocsEnum");
                    }
                    if (options.Contains(Option.REUSE_ENUMS) && Random().Next(10) < 9)
                    {
                        prevDocsEnum = threadState.ReuseDocsEnum;
                    }
                    threadState.ReuseDocsEnum = termsEnum.Docs(liveDocs, prevDocsEnum, doCheckFreqs ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE);
                    docsEnum = threadState.ReuseDocsEnum;
                    docsAndPositionsEnum = null;
                }
            }
            else
            {
                if (options.Contains(Option.REUSE_ENUMS) && Random().Next(10) < 9)
                {
                    prevDocsEnum = threadState.ReuseDocsAndPositionsEnum;
                }

                int flags = 0;
                if (alwaysTestMax || doCheckOffsets || Random().Next(3) == 1)
                {
                    flags |= DocsAndPositionsEnum.FLAG_OFFSETS;
                }
                if (alwaysTestMax || doCheckPayloads || Random().Next(3) == 1)
                {
                    flags |= DocsAndPositionsEnum.FLAG_PAYLOADS;
                }

                if (VERBOSE)
                {
                    Console.WriteLine("  get DocsAndPositionsEnum flags=" + flags);
                }

                threadState.ReuseDocsAndPositionsEnum = termsEnum.DocsAndPositions(liveDocs, (DocsAndPositionsEnum)prevDocsEnum, flags);
                docsEnum = threadState.ReuseDocsAndPositionsEnum;
                docsAndPositionsEnum = threadState.ReuseDocsAndPositionsEnum;
            }

            Assert.IsNotNull(docsEnum, "null DocsEnum");
            int initialDocID = docsEnum.DocID();
            Assert.AreEqual(-1, initialDocID, "inital docID should be -1" + docsEnum);

            if (VERBOSE)
            {
                if (prevDocsEnum == null)
                {
                    Console.WriteLine("  got enum=" + docsEnum);
                }
                else if (prevDocsEnum == docsEnum)
                {
                    Console.WriteLine("  got reuse enum=" + docsEnum);
                }
                else
                {
                    Console.WriteLine("  got enum=" + docsEnum + " (reuse of " + prevDocsEnum + " failed)");
                }
            }

            // 10% of the time don't consume all docs:
            int stopAt;
            if (!alwaysTestMax && options.Contains(Option.PARTIAL_DOC_CONSUME) && expected.DocFreq > 1 && Random().Next(10) == 7)
            {
                stopAt = Random().Next(expected.DocFreq - 1);
                if (VERBOSE)
                {
                    Console.WriteLine("  will not consume all docs (" + stopAt + " vs " + expected.DocFreq + ")");
                }
            }
            else
            {
                stopAt = expected.DocFreq;
                if (VERBOSE)
                {
                    Console.WriteLine("  consume all docs");
                }
            }

            double skipChance = alwaysTestMax ? 0.5 : Random().NextDouble();
            int numSkips = expected.DocFreq < 3 ? 1 : TestUtil.NextInt(Random(), 1, Math.Min(20, expected.DocFreq / 3));
            int skipInc = expected.DocFreq / numSkips;
            int skipDocInc = MaxDoc / numSkips;

            // Sometimes do 100% skipping:
            bool doAllSkipping = options.Contains(Option.SKIPPING) && Random().Next(7) == 1;

            double freqAskChance = alwaysTestMax ? 1.0 : Random().NextDouble();
            double payloadCheckChance = alwaysTestMax ? 1.0 : Random().NextDouble();
            double offsetCheckChance = alwaysTestMax ? 1.0 : Random().NextDouble();

            if (VERBOSE)
            {
                if (options.Contains(Option.SKIPPING))
                {
                    Console.WriteLine("  skipChance=" + skipChance + " numSkips=" + numSkips);
                }
                else
                {
                    Console.WriteLine("  no skipping");
                }
                if (doCheckFreqs)
                {
                    Console.WriteLine("  freqAskChance=" + freqAskChance);
                }
                if (doCheckPayloads)
                {
                    Console.WriteLine("  payloadCheckChance=" + payloadCheckChance);
                }
                if (doCheckOffsets)
                {
                    Console.WriteLine("  offsetCheckChance=" + offsetCheckChance);
                }
            }

            while (expected.Upto <= stopAt)
            {
                if (expected.Upto == stopAt)
                {
                    if (stopAt == expected.DocFreq)
                    {
                        Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.NextDoc(), "DocsEnum should have ended but didn't");

                        // Common bug is to forget to set this.Doc=NO_MORE_DOCS in the enum!:
                        Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.DocID(), "DocsEnum should have ended but didn't");
                    }
                    break;
                }

                if (options.Contains(Option.SKIPPING) && (doAllSkipping || Random().NextDouble() <= skipChance))
                {
                    int targetDocID = -1;
                    if (expected.Upto < stopAt && Random().NextBoolean())
                    {
                        // Pick target we know exists:
                        int skipCount = TestUtil.NextInt(Random(), 1, skipInc);
                        for (int skip = 0; skip < skipCount; skip++)
                        {
                            if (expected.NextDoc() == DocsEnum.NO_MORE_DOCS)
                            {
                                break;
                            }
                        }
                    }
                    else
                    {
                        // Pick random target (might not exist):
                        int skipDocIDs = TestUtil.NextInt(Random(), 1, skipDocInc);
                        if (skipDocIDs > 0)
                        {
                            targetDocID = expected.DocID() + skipDocIDs;
                            expected.Advance(targetDocID);
                        }
                    }

                    if (expected.Upto >= stopAt)
                    {
                        int target = Random().NextBoolean() ? MaxDoc : DocsEnum.NO_MORE_DOCS;
                        if (VERBOSE)
                        {
                            Console.WriteLine("  now advance to end (target=" + target + ")");
                        }
                        Assert.AreEqual(DocsEnum.NO_MORE_DOCS, docsEnum.Advance(target), "DocsEnum should have ended but didn't");
                        break;
                    }
                    else
                    {
                        if (VERBOSE)
                        {
                            if (targetDocID != -1)
                            {
                                Console.WriteLine("  now advance to random target=" + targetDocID + " (" + expected.Upto + " of " + stopAt + ") current=" + docsEnum.DocID());
                            }
                            else
                            {
                                Console.WriteLine("  now advance to known-exists target=" + expected.DocID() + " (" + expected.Upto + " of " + stopAt + ") current=" + docsEnum.DocID());
                            }
                        }
                        int docID = docsEnum.Advance(targetDocID != -1 ? targetDocID : expected.DocID());
                        Assert.AreEqual(expected.DocID(), docID, "docID is wrong");
                    }
                }
                else
                {
                    expected.NextDoc();
                    if (VERBOSE)
                    {
                        Console.WriteLine("  now nextDoc to " + expected.DocID() + " (" + expected.Upto + " of " + stopAt + ")");
                    }
                    int docID = docsEnum.NextDoc();
                    Assert.AreEqual(expected.DocID(), docID, "docID is wrong");
                    if (docID == DocsEnum.NO_MORE_DOCS)
                    {
                        break;
                    }
                }

                if (doCheckFreqs && Random().NextDouble() <= freqAskChance)
                {
                    if (VERBOSE)
                    {
                        Console.WriteLine("    now freq()=" + expected.Freq());
                    }
                    int freq = docsEnum.Freq();
                    Assert.AreEqual(expected.Freq(), freq, "freq is wrong");
                }

                if (doCheckPositions)
                {
                    int freq = docsEnum.Freq();
                    int numPosToConsume;
                    if (!alwaysTestMax && options.Contains(Option.PARTIAL_POS_CONSUME) && Random().Next(5) == 1)
                    {
                        numPosToConsume = Random().Next(freq);
                    }
                    else
                    {
                        numPosToConsume = freq;
                    }

                    for (int i = 0; i < numPosToConsume; i++)
                    {
                        int pos = expected.NextPosition();
                        if (VERBOSE)
                        {
                            Console.WriteLine("    now nextPosition to " + pos);
                        }
                        Assert.AreEqual(pos, docsAndPositionsEnum.NextPosition(), "position is wrong");

                        if (doCheckPayloads)
                        {
                            BytesRef expectedPayload = expected.Payload;
                            if (Random().NextDouble() <= payloadCheckChance)
                            {
                                if (VERBOSE)
                                {
                                    Console.WriteLine("      now check expectedPayload length=" + (expectedPayload == null ? 0 : expectedPayload.Length));
                                }
                                if (expectedPayload == null || expectedPayload.Length == 0)
                                {
                                    Assert.IsNull(docsAndPositionsEnum.Payload, "should not have payload");
                                }
                                else
                                {
                                    BytesRef payload = docsAndPositionsEnum.Payload;
                                    Assert.IsNotNull(payload, "should have payload but doesn't");

                                    Assert.AreEqual(expectedPayload.Length, payload.Length, "payload length is wrong");
                                    for (int byteUpto = 0; byteUpto < expectedPayload.Length; byteUpto++)
                                    {
                                        Assert.AreEqual(expectedPayload.Bytes[expectedPayload.Offset + byteUpto], payload.Bytes[payload.Offset + byteUpto], "payload bytes are wrong");
                                    }

                                    // make a deep copy
                                    payload = BytesRef.DeepCopyOf(payload);
                                    Assert.AreEqual(payload, docsAndPositionsEnum.Payload, "2nd call to getPayload returns something different!");
                                }
                            }
                            else
                            {
                                if (VERBOSE)
                                {
                                    Console.WriteLine("      skip check payload length=" + (expectedPayload == null ? 0 : expectedPayload.Length));
                                }
                            }
                        }

                        if (doCheckOffsets)
                        {
                            if (Random().NextDouble() <= offsetCheckChance)
                            {
                                if (VERBOSE)
                                {
                                    Console.WriteLine("      now check offsets: startOff=" + expected.StartOffset() + " endOffset=" + expected.EndOffset());
                                }
                                Assert.AreEqual(expected.StartOffset(), docsAndPositionsEnum.StartOffset(), "startOffset is wrong");
                                Assert.AreEqual(expected.EndOffset(), docsAndPositionsEnum.EndOffset(), "endOffset is wrong");
                            }
                            else
                            {
                                if (VERBOSE)
                                {
                                    Console.WriteLine("      skip check offsets");
                                }
                            }
                        }
                        else if (fieldInfo.FieldIndexOptions < FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
                        {
                            if (VERBOSE)
                            {
                                Console.WriteLine("      now check offsets are -1");
                            }
                            Assert.AreEqual(-1, docsAndPositionsEnum.StartOffset(), "startOffset isn't -1");
                            Assert.AreEqual(-1, docsAndPositionsEnum.EndOffset(), "endOffset isn't -1");
                        }
                    }
                }
            }
        }

        private class TestThread : ThreadClass
        {
            internal Fields FieldsSource;
            internal ISet<Option> Options;
            internal FieldInfo.IndexOptions MaxIndexOptions;
            internal FieldInfo.IndexOptions MaxTestOptions;
            internal bool AlwaysTestMax;
            internal BasePostingsFormatTestCase TestCase;

            public TestThread(BasePostingsFormatTestCase testCase, Fields fieldsSource, ISet<Option> options, FieldInfo.IndexOptions maxTestOptions, FieldInfo.IndexOptions maxIndexOptions, bool alwaysTestMax)
            {
                this.FieldsSource = fieldsSource;
                this.Options = options;
                this.MaxTestOptions = maxTestOptions;
                this.MaxIndexOptions = maxIndexOptions;
                this.AlwaysTestMax = alwaysTestMax;
                this.TestCase = testCase;
            }

            public override void Run()
            {
                try
                {
                    try
                    {
                        TestCase.TestTermsOneThread(FieldsSource, Options, MaxTestOptions, MaxIndexOptions, AlwaysTestMax);
                    }
                    catch (Exception t)
                    {
                        throw new Exception(t.Message, t);
                    }
                }
                finally
                {
                    FieldsSource = null;
                    TestCase = null;
                }
            }
        }

        private void TestTerms(Fields fieldsSource, ISet<Option> options, FieldInfo.IndexOptions maxTestOptions, FieldInfo.IndexOptions maxIndexOptions, bool alwaysTestMax)
        {
            if (options.Contains(Option.THREADS))
            {
                int numThreads = TestUtil.NextInt(Random(), 2, 5);
                ThreadClass[] threads = new ThreadClass[numThreads];
                for (int threadUpto = 0; threadUpto < numThreads; threadUpto++)
                {
                    threads[threadUpto] = new TestThread(this, fieldsSource, options, maxTestOptions, maxIndexOptions, alwaysTestMax);
                    threads[threadUpto].Start();
                }
                for (int threadUpto = 0; threadUpto < numThreads; threadUpto++)
                {
                    threads[threadUpto].Join();
                }
            }
            else
            {
                TestTermsOneThread(fieldsSource, options, maxTestOptions, maxIndexOptions, alwaysTestMax);
            }
        }

        private void TestTermsOneThread(Fields fieldsSource, ISet<Option> options, FieldInfo.IndexOptions maxTestOptions, FieldInfo.IndexOptions maxIndexOptions, bool alwaysTestMax)
        {
            ThreadState threadState = new ThreadState();

            // Test random terms/fields:
            IList<TermState> termStates = new List<TermState>();
            IList<FieldAndTerm> termStateTerms = new List<FieldAndTerm>();

            AllTerms = CollectionsHelper.Shuffle(AllTerms);
            int upto = 0;
            while (upto < AllTerms.Count)
            {
                bool useTermState = termStates.Count != 0 && Random().Next(5) == 1;
                FieldAndTerm fieldAndTerm;
                TermsEnum termsEnum;

                TermState termState = null;

                if (!useTermState)
                {
                    // Seek by random field+term:
                    fieldAndTerm = AllTerms[upto++];
                    if (VERBOSE)
                    {
                        Console.WriteLine("\nTEST: seek to term=" + fieldAndTerm.Field + ":" + fieldAndTerm.Term.Utf8ToString());
                    }
                }
                else
                {
                    // Seek by previous saved TermState
                    int idx = Random().Next(termStates.Count);
                    fieldAndTerm = termStateTerms[idx];
                    if (VERBOSE)
                    {
                        Console.WriteLine("\nTEST: seek using TermState to term=" + fieldAndTerm.Field + ":" + fieldAndTerm.Term.Utf8ToString());
                    }
                    termState = termStates[idx];
                }

                Terms terms = fieldsSource.Terms(fieldAndTerm.Field);
                Assert.IsNotNull(terms);
                termsEnum = terms.Iterator(null);

                if (!useTermState)
                {
                    Assert.IsTrue(termsEnum.SeekExact(fieldAndTerm.Term));
                }
                else
                {
                    termsEnum.SeekExact(fieldAndTerm.Term, termState);
                }

                bool savedTermState = false;

                if (options.Contains(Option.TERM_STATE) && !useTermState && Random().Next(5) == 1)
                {
                    // Save away this TermState:
                    termStates.Add(termsEnum.TermState());
                    termStateTerms.Add(fieldAndTerm);
                    savedTermState = true;
                }

                VerifyEnum(threadState, fieldAndTerm.Field, fieldAndTerm.Term, termsEnum, maxTestOptions, maxIndexOptions, options, alwaysTestMax);

                // Sometimes save term state after pulling the enum:
                if (options.Contains(Option.TERM_STATE) && !useTermState && !savedTermState && Random().Next(5) == 1)
                {
                    // Save away this TermState:
                    termStates.Add(termsEnum.TermState());
                    termStateTerms.Add(fieldAndTerm);
                    useTermState = true;
                }

                // 10% of the time make sure you can pull another enum
                // from the same term:
                if (alwaysTestMax || Random().Next(10) == 7)
                {
                    // Try same term again
                    if (VERBOSE)
                    {
                        Console.WriteLine("TEST: try enum again on same term");
                    }

                    VerifyEnum(threadState, fieldAndTerm.Field, fieldAndTerm.Term, termsEnum, maxTestOptions, maxIndexOptions, options, alwaysTestMax);
                }
            }
        }

        private void TestFields(Fields fields)
        {
            IEnumerator<string> iterator = fields.GetEnumerator();
            while (iterator.MoveNext())
            {
                var dummy = iterator.Current;
                // .NET: Testing for iterator.Remove() isn't applicable
            }
            Assert.IsFalse(iterator.MoveNext());

            // .NET: Testing for NoSuchElementException with .NET iterators isn't applicable
        }

        /// <summary>
        /// Indexes all fields/terms at the specified
        ///  IndexOptions, and fully tests at that IndexOptions.
        /// </summary>
        private void TestFull(FieldInfo.IndexOptions options, bool withPayloads)
        {
            DirectoryInfo path = CreateTempDir("testPostingsFormat.testExact");
            using (Directory dir = NewFSDirectory(path))
            {
                // TODO test thread safety of buildIndex too
                var fieldsProducer = BuildIndex(dir, options, withPayloads, true);

                TestFields(fieldsProducer);

                var allOptions = (FieldInfo.IndexOptions[]) Enum.GetValues(typeof (FieldInfo.IndexOptions));
                    //IndexOptions_e.values();
                int maxIndexOption = Arrays.AsList(allOptions).IndexOf(options);

                for (int i = 0; i <= maxIndexOption; i++)
                {
                    ISet<Option> allOptionsHashSet = new HashSet<Option>(Enum.GetValues(typeof (Option)).Cast<Option>());
                    TestTerms(fieldsProducer, allOptionsHashSet, allOptions[i], options, true);
                    if (withPayloads)
                    {
                        // If we indexed w/ payloads, also test enums w/o accessing payloads:
                        ISet<Option> payloadsHashSet = new HashSet<Option>() {Option.PAYLOADS};
                        var complementHashSet = new HashSet<Option>(allOptionsHashSet.Except(payloadsHashSet));
                        TestTerms(fieldsProducer, complementHashSet, allOptions[i], options, true);
                    }
                }

                fieldsProducer.Dispose();
            }
        }

        [Test, Timeout(300000)]
        public virtual void TestDocsOnly()
        {
            TestFull(FieldInfo.IndexOptions.DOCS_ONLY, false);
        }

        [Test, Timeout(300000)]
        public virtual void TestDocsAndFreqs()
        {
            TestFull(FieldInfo.IndexOptions.DOCS_AND_FREQS, false);
        }

        [Test, Timeout(300000)]
        public virtual void TestDocsAndFreqsAndPositions()
        {
            TestFull(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, false);
        }

        [Test, Timeout(300000)]
        public virtual void TestDocsAndFreqsAndPositionsAndPayloads()
        {
            TestFull(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, true);
        }

        [Test, Timeout(300000)]
        public virtual void TestDocsAndFreqsAndPositionsAndOffsets()
        {
            TestFull(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, false);
        }

        [Test, Timeout(300000)]
        public virtual void TestDocsAndFreqsAndPositionsAndOffsetsAndPayloads()
        {
            TestFull(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, true);
        }

        [Test, Timeout(300000)]
        public virtual void TestRandom()
        {
            int iters = 5;

            for (int iter = 0; iter < iters; iter++)
            {
                DirectoryInfo path = CreateTempDir("testPostingsFormat");
                Directory dir = NewFSDirectory(path);

                bool indexPayloads = Random().NextBoolean();
                // TODO test thread safety of buildIndex too
                FieldsProducer fieldsProducer = BuildIndex(dir, FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, indexPayloads, false);

                TestFields(fieldsProducer);

                // NOTE: you can also test "weaker" index options than
                // you indexed with:
                TestTerms(fieldsProducer, new HashSet<Option>(Enum.GetValues(typeof(Option)).Cast<Option>()), FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, false);

                fieldsProducer.Dispose();
                fieldsProducer = null;

                dir.Dispose();
                System.IO.Directory.Delete(path.FullName, true);
            }
        }

        protected internal override void AddRandomFields(Document doc)
        {
            foreach (FieldInfo.IndexOptions opts in Enum.GetValues(typeof(FieldInfo.IndexOptions)))
            {
                string field = "f_" + opts;
                string pf = TestUtil.GetPostingsFormat(Codec.Default, field);
                if (opts == FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS && DoesntSupportOffsets.Contains(pf))
                {
                    continue;
                }
                var ft = new FieldType {IndexOptions = opts, Indexed = true, OmitNorms = true};
                ft.Freeze();
                int numFields = Random().Next(5);
                for (int j = 0; j < numFields; ++j)
                {
                    doc.Add(new Field("f_" + opts, TestUtil.RandomSimpleString(Random(), 2), ft));
                }
            }
        }
    }
}