| using J2N.IO; |
| using J2N.IO.MemoryMappedFiles; |
| using J2N.Numerics; |
| using Lucene.Net.Diagnostics; |
| using System; |
| using System.IO; |
| using System.IO.MemoryMappedFiles; |
| |
| namespace Lucene.Net.Store |
| { |
| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| using Constants = Lucene.Net.Util.Constants; |
| |
| /// <summary> |
| /// File-based <see cref="Directory"/> implementation that uses |
| /// <see cref="MemoryMappedFile"/> for reading, and |
| /// <see cref="FSDirectory.FSIndexOutput"/> for writing. |
| /// |
| /// <para/><b>NOTE</b>: memory mapping uses up a portion of the |
| /// virtual memory address space in your process equal to the |
| /// size of the file being mapped. Before using this class, |
| /// be sure your have plenty of virtual address space, e.g. by |
| /// using a 64 bit runtime, or a 32 bit runtime with indexes that are |
| /// guaranteed to fit within the address space. |
| /// On 32 bit platforms also consult <see cref="MMapDirectory(DirectoryInfo, LockFactory, int)"/> |
| /// if you have problems with mmap failing because of fragmented |
| /// address space. If you get an <see cref="OutOfMemoryException"/>, it is recommended |
| /// to reduce the chunk size, until it works. |
| /// <para> |
| /// <b>NOTE:</b> Accessing this class either directly or |
| /// indirectly from a thread while it's interrupted can close the |
| /// underlying channel immediately if at the same time the thread is |
| /// blocked on IO. The channel will remain closed and subsequent access |
| /// to <see cref="MMapDirectory"/> will throw a <see cref="ObjectDisposedException"/>. |
| /// </para> |
| /// </summary> |
| public class MMapDirectory : FSDirectory |
| { |
| // LUCENENET specific - unmap hack not needed |
| |
| /// <summary> |
| /// Default max chunk size. </summary> |
| /// <seealso cref="MMapDirectory(DirectoryInfo, LockFactory, int)"/> |
| public static readonly int DEFAULT_MAX_BUFF = Constants.RUNTIME_IS_64BIT ? (1 << 30) : (1 << 28); |
| |
| private readonly int chunkSizePower; |
| |
| /// <summary> |
| /// Create a new <see cref="MMapDirectory"/> for the named location. |
| /// </summary> |
| /// <param name="path"> the path of the directory </param> |
| /// <param name="lockFactory"> the lock factory to use, or null for the default |
| /// (<see cref="NativeFSLockFactory"/>); </param> |
| /// <exception cref="IOException"> if there is a low-level I/O error </exception> |
| public MMapDirectory(DirectoryInfo path, LockFactory lockFactory) |
| : this(path, lockFactory, DEFAULT_MAX_BUFF) |
| { |
| } |
| |
| /// <summary> |
| /// Create a new <see cref="MMapDirectory"/> for the named location and <see cref="NativeFSLockFactory"/>. |
| /// </summary> |
| /// <param name="path"> the path of the directory </param> |
| /// <exception cref="IOException"> if there is a low-level I/O error </exception> |
| public MMapDirectory(DirectoryInfo path) |
| : this(path, null) |
| { |
| } |
| |
| /// <summary> |
| /// Create a new <see cref="MMapDirectory"/> for the named location, specifying the |
| /// maximum chunk size used for memory mapping. |
| /// </summary> |
| /// <param name="path"> the path of the directory </param> |
| /// <param name="lockFactory"> the lock factory to use, or <c>null</c> for the default |
| /// (<see cref="NativeFSLockFactory"/>); </param> |
| /// <param name="maxChunkSize"> maximum chunk size (default is 1 GiBytes for |
| /// 64 bit runtimes and 256 MiBytes for 32 bit runtimes) used for memory mapping. |
| /// <para/> |
| /// Especially on 32 bit platform, the address space can be very fragmented, |
| /// so large index files cannot be mapped. Using a lower chunk size makes |
| /// the directory implementation a little bit slower (as the correct chunk |
| /// may be resolved on lots of seeks) but the chance is higher that mmap |
| /// does not fail. On 64 bit platforms, this parameter should always |
| /// be <c>1 << 30</c>, as the address space is big enough. |
| /// <para/> |
| /// <b>Please note:</b> The chunk size is always rounded down to a power of 2. |
| /// </param> |
| /// <exception cref="IOException"> if there is a low-level I/O error </exception> |
| public MMapDirectory(DirectoryInfo path, LockFactory lockFactory, int maxChunkSize) |
| : base(path, lockFactory) |
| { |
| if (maxChunkSize <= 0) |
| { |
| throw new ArgumentException("Maximum chunk size for mmap must be >0"); |
| } |
| this.chunkSizePower = 31 - maxChunkSize.LeadingZeroCount(); |
| if (Debugging.AssertsEnabled) Debugging.Assert(this.chunkSizePower >= 0 && this.chunkSizePower <= 30); |
| } |
| |
| /// <summary> |
| /// Create a new <see cref="MMapDirectory"/> for the named location. |
| /// <para/> |
| /// LUCENENET specific overload for convenience using string instead of <see cref="DirectoryInfo"/>. |
| /// </summary> |
| /// <param name="path"> the path of the directory </param> |
| /// <param name="lockFactory"> the lock factory to use, or null for the default |
| /// (<see cref="NativeFSLockFactory"/>); </param> |
| /// <exception cref="IOException"> if there is a low-level I/O error </exception> |
| public MMapDirectory(string path, LockFactory lockFactory) |
| : this(path, lockFactory, DEFAULT_MAX_BUFF) |
| { |
| } |
| |
| /// <summary> |
| /// Create a new <see cref="MMapDirectory"/> for the named location and <see cref="NativeFSLockFactory"/>. |
| /// <para/> |
| /// LUCENENET specific overload for convenience using string instead of <see cref="DirectoryInfo"/>. |
| /// </summary> |
| /// <param name="path"> the path of the directory </param> |
| /// <exception cref="IOException"> if there is a low-level I/O error </exception> |
| public MMapDirectory(string path) |
| : this(path, null) |
| { |
| } |
| |
| /// <summary> |
| /// Create a new <see cref="MMapDirectory"/> for the named location, specifying the |
| /// maximum chunk size used for memory mapping. |
| /// <para/> |
| /// LUCENENET specific overload for convenience using string instead of <see cref="DirectoryInfo"/>. |
| /// </summary> |
| /// <param name="path"> the path of the directory </param> |
| /// <param name="lockFactory"> the lock factory to use, or <c>null</c> for the default |
| /// (<see cref="NativeFSLockFactory"/>); </param> |
| /// <param name="maxChunkSize"> maximum chunk size (default is 1 GiBytes for |
| /// 64 bit runtimes and 256 MiBytes for 32 bit runtimes) used for memory mapping. |
| /// <para/> |
| /// Especially on 32 bit platform, the address space can be very fragmented, |
| /// so large index files cannot be mapped. Using a lower chunk size makes |
| /// the directory implementation a little bit slower (as the correct chunk |
| /// may be resolved on lots of seeks) but the chance is higher that mmap |
| /// does not fail. On 64 bit platforms, this parameter should always |
| /// be <c>1 << 30</c>, as the address space is big enough. |
| /// <para/> |
| /// <b>Please note:</b> The chunk size is always rounded down to a power of 2. |
| /// </param> |
| /// <exception cref="IOException"> if there is a low-level I/O error </exception> |
| public MMapDirectory(string path, LockFactory lockFactory, int maxChunkSize) |
| : this(new DirectoryInfo(path), lockFactory, maxChunkSize) |
| { |
| } |
| |
| // LUCENENET specific - Some JREs had a bug that didn't allow them to unmap. |
| // But according to MSDN, the MemoryMappedFile.Dispose() method will |
| // indeed "release all resources". Therefore unmap hack is not needed in .NET. |
| |
| /// <summary> |
| /// Returns the current mmap chunk size. </summary> |
| /// <seealso cref="MMapDirectory(DirectoryInfo, LockFactory, int)"/> |
| public int MaxChunkSize => 1 << chunkSizePower; |
| |
| /// <summary> |
| /// Creates an <see cref="IndexInput"/> for the file with the given name. </summary> |
| public override IndexInput OpenInput(string name, IOContext context) |
| { |
| EnsureOpen(); |
| var file = new FileInfo(Path.Combine(Directory.FullName, name)); |
| var fc = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); |
| return new MMapIndexInput(this, "MMapIndexInput(path=\"" + file + "\")", fc); |
| } |
| |
| public override IndexInputSlicer CreateSlicer(string name, IOContext context) |
| { |
| var full = (MMapIndexInput)OpenInput(name, context); |
| return new IndexInputSlicerAnonymousClass(this, full); |
| } |
| |
| private class IndexInputSlicerAnonymousClass : IndexInputSlicer |
| { |
| private readonly MMapDirectory outerInstance; |
| |
| private readonly MMapIndexInput full; |
| |
| public IndexInputSlicerAnonymousClass(MMapDirectory outerInstance, MMapIndexInput full) |
| { |
| this.outerInstance = outerInstance; |
| this.full = full; |
| } |
| |
| public override IndexInput OpenSlice(string sliceDescription, long offset, long length) |
| { |
| outerInstance.EnsureOpen(); |
| return full.Slice(sliceDescription, offset, length); |
| } |
| |
| [Obsolete("Only for reading CFS files from 3.x indexes.")] |
| public override IndexInput OpenFullSlice() |
| { |
| outerInstance.EnsureOpen(); |
| return (IndexInput)full.Clone(); |
| } |
| |
| protected override void Dispose(bool disposing) |
| { |
| if (disposing) |
| { |
| full.Dispose(); |
| } |
| } |
| } |
| |
| public sealed class MMapIndexInput : ByteBufferIndexInput |
| { |
| internal MemoryMappedFile memoryMappedFile; // .NET port: this is equivalent to FileChannel.map |
| private readonly FileStream fc; |
| |
| internal MMapIndexInput(MMapDirectory outerInstance, string resourceDescription, FileStream fc) |
| : base(resourceDescription, null, fc.Length, outerInstance.chunkSizePower, true) |
| { |
| this.fc = fc ?? throw new ArgumentNullException(nameof(fc)); |
| this.SetBuffers(outerInstance.Map(this, fc, 0, fc.Length)); |
| } |
| |
| protected override sealed void Dispose(bool disposing) |
| { |
| try |
| { |
| if (disposing) |
| { |
| try |
| { |
| if (this.memoryMappedFile != null) |
| { |
| this.memoryMappedFile.Dispose(); |
| this.memoryMappedFile = null; |
| } |
| } |
| finally |
| { |
| // LUCENENET: If the file is 0 length we will not create a memoryMappedFile above |
| // so we must always ensure the FileStream is explicitly disposed. |
| this.fc.Dispose(); |
| } |
| } |
| } |
| finally |
| { |
| base.Dispose(disposing); |
| } |
| } |
| |
| /// <summary> |
| /// Try to unmap the buffer, this method silently fails if no support |
| /// for that in the runtime. On Windows, this leads to the fact, |
| /// that mmapped files cannot be modified or deleted. |
| /// </summary> |
| protected override void FreeBuffer(ByteBuffer buffer) |
| { |
| // LUCENENET specific: this should free the memory mapped view accessor |
| if (buffer is IDisposable disposable) |
| disposable.Dispose(); |
| |
| // LUCENENET specific: no need for UnmapHack |
| } |
| } |
| |
| /// <summary> |
| /// Maps a file into a set of buffers </summary> |
| internal virtual ByteBuffer[] Map(MMapIndexInput input, FileStream fc, long offset, long length) |
| { |
| if (length.TripleShift(chunkSizePower) >= int.MaxValue) |
| throw new ArgumentException("RandomAccessFile too big for chunk size: " + fc.ToString()); |
| |
| // LUCENENET specific: Return empty buffer if length is 0, rather than attempting to create a MemoryMappedFile. |
| // Part of a solution provided by Vincent Van Den Berghe: http://apache.markmail.org/message/hafnuhq2ydhfjmi2 |
| if (length == 0) |
| { |
| return new[] { ByteBuffer.Allocate(0).AsReadOnlyBuffer() }; |
| } |
| |
| long chunkSize = 1L << chunkSizePower; |
| |
| // we always allocate one more buffer, the last one may be a 0 byte one |
| int nrBuffers = (int)length.TripleShift(chunkSizePower) + 1; |
| |
| ByteBuffer[] buffers = new ByteBuffer[nrBuffers]; |
| |
| if (input.memoryMappedFile == null) |
| { |
| input.memoryMappedFile = MemoryMappedFile.CreateFromFile( |
| fileStream: fc, |
| mapName: null, |
| capacity: length, |
| access: MemoryMappedFileAccess.Read, |
| #if FEATURE_MEMORYMAPPEDFILESECURITY |
| memoryMappedFileSecurity: null, |
| #endif |
| inheritability: HandleInheritability.Inheritable, |
| leaveOpen: true); // LUCENENET: We explicitly dispose the FileStream separately. |
| } |
| |
| long bufferStart = 0L; |
| for (int bufNr = 0; bufNr < nrBuffers; bufNr++) |
| { |
| int bufSize = (int)((length > (bufferStart + chunkSize)) ? chunkSize : (length - bufferStart)); |
| |
| // LUCENENET: We get an UnauthorizedAccessException if we create a 0 byte file at the end of the range. |
| // See: https://stackoverflow.com/a/5501331 |
| // We can fix this by moving back 1 byte on the offset if the bufSize is 0. |
| int adjust = 0; |
| if (bufSize == 0 && bufNr == (nrBuffers - 1) && (offset + bufferStart) > 0) |
| { |
| adjust = 1; |
| } |
| |
| buffers[bufNr] = input.memoryMappedFile.CreateViewByteBuffer( |
| offset: (offset + bufferStart) - adjust, |
| size: bufSize, |
| access: MemoryMappedFileAccess.Read); |
| bufferStart += bufSize; |
| } |
| |
| return buffers; |
| } |
| } |
| } |