| using Lucene.Net.Index; |
| using Lucene.Net.Store; |
| using Lucene.Net.Support; |
| using Lucene.Net.Util; |
| using Lucene.Net.Util.Fst; |
| using System; |
| using System.Collections.Generic; |
| using System.Diagnostics; |
| |
| namespace Lucene.Net.Codecs.BlockTerms |
| { |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| /// <summary> |
| /// See <see cref="VariableGapTermsIndexWriter"/>. |
| /// <para/> |
| /// @lucene.experimental |
| /// </summary> |
| public class VariableGapTermsIndexReader : TermsIndexReaderBase |
| { |
| private readonly PositiveInt32Outputs fstOutputs = PositiveInt32Outputs.Singleton; |
| private int indexDivisor; |
| |
| // Closed if indexLoaded is true: |
| private IndexInput input; |
| private volatile bool indexLoaded; |
| |
| private readonly IDictionary<FieldInfo, FieldIndexData> fields = new Dictionary<FieldInfo, FieldIndexData>(); |
| |
| // start of the field info data |
| private long dirOffset; |
| |
| private readonly int version; |
| |
| private readonly string segment; |
| |
| public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, string segment, int indexDivisor, |
| string segmentSuffix, IOContext context) |
| { |
| input = dir.OpenInput(IndexFileNames.SegmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); |
| this.segment = segment; |
| bool success = false; |
| Debug.Assert(indexDivisor == -1 || indexDivisor > 0); |
| |
| try |
| { |
| version = ReadHeader(input); |
| this.indexDivisor = indexDivisor; |
| |
| if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) |
| { |
| CodecUtil.ChecksumEntireFile(input); |
| } |
| |
| SeekDir(input, dirOffset); |
| |
| // Read directory |
| int numFields = input.ReadVInt32(); |
| if (numFields < 0) |
| { |
| throw new CorruptIndexException("invalid numFields: " + numFields + " (resource=" + input + ")"); |
| } |
| |
| for (int i = 0; i < numFields; i++) |
| { |
| int field = input.ReadVInt32(); |
| long indexStart = input.ReadVInt64(); |
| FieldInfo fieldInfo = fieldInfos.FieldInfo(field); |
| FieldIndexData previous = fields.Put(fieldInfo, new FieldIndexData(this, fieldInfo, indexStart)); |
| if (previous != null) |
| { |
| throw new CorruptIndexException("duplicate field: " + fieldInfo.Name + " (resource=" + input + ")"); |
| } |
| } |
| success = true; |
| } |
| finally |
| { |
| if (indexDivisor > 0) |
| { |
| input.Dispose(); |
| input = null; |
| if (success) |
| { |
| indexLoaded = true; |
| } |
| } |
| } |
| } |
| |
| public override int Divisor |
| { |
| get { return indexDivisor; } |
| } |
| |
| private int ReadHeader(IndexInput input) |
| { |
| int version = CodecUtil.CheckHeader(input, VariableGapTermsIndexWriter.CODEC_NAME, |
| VariableGapTermsIndexWriter.VERSION_START, VariableGapTermsIndexWriter.VERSION_CURRENT); |
| if (version < VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) |
| { |
| dirOffset = input.ReadInt64(); |
| } |
| return version; |
| } |
| |
| private class IndexEnum : FieldIndexEnum |
| { |
| private readonly BytesRefFSTEnum<long?> fstEnum; |
| private BytesRefFSTEnum.InputOutput<long?> current; |
| |
| public IndexEnum(FST<long?> fst) |
| { |
| fstEnum = new BytesRefFSTEnum<long?>(fst); |
| } |
| |
| public override BytesRef Term |
| { |
| get |
| { |
| if (current == null) |
| { |
| return null; |
| } |
| else |
| { |
| return current.Input; |
| } |
| } |
| } |
| |
| public override long Seek(BytesRef target) |
| { |
| //System.out.println("VGR: seek field=" + fieldInfo.name + " target=" + target); |
| current = fstEnum.SeekFloor(target); |
| //System.out.println(" got input=" + current.input + " output=" + current.output); |
| if (current.Output.HasValue) |
| { |
| return current.Output.Value; |
| } |
| else |
| { |
| throw new NullReferenceException("_current.Output is null"); // LUCENENET NOTE: NullReferenceException would be thrown in Java, so doing it here |
| } |
| } |
| |
| public override long Next() |
| { |
| //System.out.println("VGR: next field=" + fieldInfo.name); |
| current = fstEnum.Next(); |
| if (current == null) |
| { |
| //System.out.println(" eof"); |
| return -1; |
| } |
| else |
| { |
| if (current.Output.HasValue) |
| { |
| return current.Output.Value; |
| } |
| else |
| { |
| throw new NullReferenceException("_current.Output is null"); // LUCENENET NOTE: NullReferenceException would be thrown in Java, so doing it here |
| } |
| } |
| } |
| |
| public override long Ord |
| { |
| get { throw new NotSupportedException(); } |
| } |
| |
| public override long Seek(long ord) |
| { |
| throw new NotSupportedException(); |
| } |
| } |
| |
| public override bool SupportsOrd |
| { |
| get { return false; } |
| } |
| |
| private class FieldIndexData |
| { |
| private readonly VariableGapTermsIndexReader outerInstance; |
| |
| private readonly long indexStart; |
| // Set only if terms index is loaded: |
| internal volatile FST<long?> fst; |
| |
| public FieldIndexData(VariableGapTermsIndexReader outerInstance, FieldInfo fieldInfo, long indexStart) |
| { |
| this.outerInstance = outerInstance; |
| |
| this.indexStart = indexStart; |
| |
| if (outerInstance.indexDivisor > 0) |
| { |
| LoadTermsIndex(); |
| } |
| } |
| |
| private void LoadTermsIndex() |
| { |
| if (fst == null) |
| { |
| using (IndexInput clone = (IndexInput)outerInstance.input.Clone()) |
| { |
| clone.Seek(indexStart); |
| fst = new FST<long?>(clone, outerInstance.fstOutputs); |
| } // clone.Dispose(); |
| |
| /* |
| final String dotFileName = segment + "_" + fieldInfo.name + ".dot"; |
| Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName)); |
| Util.toDot(fst, w, false, false); |
| System.out.println("FST INDEX: SAVED to " + dotFileName); |
| w.close(); |
| */ |
| |
| if (outerInstance.indexDivisor > 1) |
| { |
| // subsample |
| Int32sRef scratchIntsRef = new Int32sRef(); |
| PositiveInt32Outputs outputs = PositiveInt32Outputs.Singleton; |
| Builder<long?> builder = new Builder<long?>(FST.INPUT_TYPE.BYTE1, outputs); |
| BytesRefFSTEnum<long?> fstEnum = new BytesRefFSTEnum<long?>(fst); |
| BytesRefFSTEnum.InputOutput<long?> result; |
| int count = outerInstance.indexDivisor; |
| while ((result = fstEnum.Next()) != null) |
| { |
| if (count == outerInstance.indexDivisor) |
| { |
| builder.Add(Util.Fst.Util.ToInt32sRef(result.Input, scratchIntsRef), result.Output); |
| count = 0; |
| } |
| count++; |
| } |
| fst = builder.Finish(); |
| } |
| } |
| } |
| |
| /// <summary>Returns approximate RAM bytes used.</summary> |
| public virtual long RamBytesUsed() |
| { |
| return fst == null ? 0 : fst.GetSizeInBytes(); |
| } |
| } |
| |
| public override FieldIndexEnum GetFieldEnum(FieldInfo fieldInfo) |
| { |
| FieldIndexData fieldData; |
| if (!fields.TryGetValue(fieldInfo, out fieldData) || fieldData == null || fieldData.fst == null) |
| { |
| return null; |
| } |
| else |
| { |
| return new IndexEnum(fieldData.fst); |
| } |
| } |
| |
| protected override void Dispose(bool disposing) |
| { |
| if (disposing) |
| { |
| if (input != null && !indexLoaded) |
| { |
| input.Dispose(); |
| } |
| } |
| } |
| |
| private void SeekDir(IndexInput input, long dirOffset) |
| { |
| if (version >= VariableGapTermsIndexWriter.VERSION_CHECKSUM) |
| { |
| input.Seek(input.Length - CodecUtil.FooterLength() - 8); |
| dirOffset = input.ReadInt64(); |
| } |
| else if (version >= VariableGapTermsIndexWriter.VERSION_APPEND_ONLY) |
| { |
| input.Seek(input.Length - 8); |
| dirOffset = input.ReadInt64(); |
| } |
| input.Seek(dirOffset); |
| } |
| |
| public override long RamBytesUsed() |
| { |
| long sizeInBytes = 0; |
| foreach (FieldIndexData entry in fields.Values) |
| { |
| sizeInBytes += entry.RamBytesUsed(); |
| } |
| return sizeInBytes; |
| } |
| } |
| } |