path: root/src/core/Store/MMapDirectory.cs

/* 
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

using System;

using Constants = Lucene.Net.Util.Constants;

namespace Lucene.Net.Store
{
	
	/// <summary>File-based <see cref="Directory" /> implementation that uses
	/// mmap for reading, and <see cref="SimpleFSDirectory.SimpleFSIndexOutput" />
	/// for writing.
	/// 
	/// <p/><b>NOTE</b>: memory mapping uses up a portion of the
	/// virtual memory address space in your process equal to the
	/// size of the file being mapped.  Before using this class,
	/// be sure your have plenty of virtual address space, e.g. by
	/// using a 64 bit JRE, or a 32 bit JRE with indexes that are
	/// guaranteed to fit within the address space.
	/// On 32 bit platforms also consult <see cref="MaxChunkSize" />
	/// if you have problems with mmap failing because of fragmented
	/// address space. If you get an OutOfMemoryException, it is recommened
	/// to reduce the chunk size, until it works.
	/// 
	/// <p/>Due to <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
	/// this bug</a> in Sun's JRE, MMapDirectory's <see cref="IndexInput.Close" />
	/// is unable to close the underlying OS file handle.  Only when GC
	/// finally collects the underlying objects, which could be quite
	/// some time later, will the file handle be closed.
	/// 
	/// <p/>This will consume additional transient disk usage: on Windows,
	/// attempts to delete or overwrite the files will result in an
	/// exception; on other platforms, which typically have a &quot;delete on
	/// last close&quot; semantics, while such operations will succeed, the bytes
	/// are still consuming space on disk.  For many applications this
	/// limitation is not a problem (e.g. if you have plenty of disk space,
	/// and you don't rely on overwriting files on Windows) but it's still
	/// an important limitation to be aware of.
	/// 
	/// <p/>This class supplies the workaround mentioned in the bug report
	/// (disabled by default, see <see cref="UseUnmap" />), which may fail on
	/// non-Sun JVMs. It forcefully unmaps the buffer on close by using
	/// an undocumented internal cleanup functionality.
	/// <see cref="UNMAP_SUPPORTED" /> is <c>true</c>, if the workaround
	/// can be enabled (with no guarantees).
	/// </summary>
	public class MMapDirectory:FSDirectory
	{
		private class AnonymousClassPrivilegedExceptionAction // : SupportClass.IPriviligedAction   // {{Aroush-2.9}}
		{
			public AnonymousClassPrivilegedExceptionAction(byte[] buffer, MMapDirectory enclosingInstance)
			{
				InitBlock(buffer, enclosingInstance);
			}
			private void  InitBlock(byte[] buffer, MMapDirectory enclosingInstance)
			{
				this.buffer = buffer;
				this.enclosingInstance = enclosingInstance;
			}
			private byte[] buffer;
			private MMapDirectory enclosingInstance;
			public MMapDirectory Enclosing_Instance
			{
				get
				{
					return enclosingInstance;
				}
				
			}
			public virtual System.Object Run()
			{
                // {{Aroush-2.9
                /*
				System.Reflection.MethodInfo getCleanerMethod = buffer.GetType().GetMethod("cleaner", (Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES == null)?new System.Type[0]:(System.Type[]) Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES);
                getCleanerMethod.SetAccessible(true);
				System.Object cleaner = getCleanerMethod.Invoke(buffer, (System.Object[]) Lucene.Net.Store.MMapDirectory.NO_PARAMS);
				if (cleaner != null)
				{
					cleaner.GetType().GetMethod("clean", (Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES == null)?new System.Type[0]:(System.Type[]) Lucene.Net.Store.MMapDirectory.NO_PARAM_TYPES).Invoke(cleaner, (System.Object[]) Lucene.Net.Store.MMapDirectory.NO_PARAMS);
				}
                */
                //System.Diagnostics.Debug.Fail("Port issue:", "sun.misc.Cleaner()"); // {{Aroush-2.9}}
                throw new NotImplementedException("Port issue: sun.misc.Cleaner()");
                // Aroush-2.9}}
				//return null; 
			}
		}
		private void  InitBlock()
		{
			maxBBuf = Constants.JRE_IS_64BIT?System.Int32.MaxValue:(256 * 1024 * 1024);
		}
		
        /// <summary>Create a new MMapDirectory for the named location.
        /// 
        /// </summary>
        /// <param name="path">the path of the directory
        /// </param>
        /// <param name="lockFactory">the lock factory to use, or null for the default.
        /// </param>
        /// <throws>  IOException </throws>
        public MMapDirectory(System.IO.DirectoryInfo path, LockFactory lockFactory)
            : base(path, lockFactory)
        {
            InitBlock();
        }

	    /// <summary>Create a new MMapDirectory for the named location and the default lock factory.
        /// 
        /// </summary>
        /// <param name="path">the path of the directory
        /// </param>
        /// <throws>  IOException </throws>
        public MMapDirectory(System.IO.DirectoryInfo path)
            : base(path, null)
	    {
	        InitBlock();
	    }
		
		private bool useUnmapHack = false;
		private int maxBBuf;
		
		/// <summary> <c>true</c>, if this platform supports unmapping mmaped files.</summary>
		public static bool UNMAP_SUPPORTED;

	    /// <summary> Enables or disables the workaround for unmapping the buffers
	    /// from address space after closing <see cref="IndexInput" />, that is
	    /// mentioned in the bug report. This hack may fail on non-Sun JVMs.
	    /// It forcefully unmaps the buffer on close by using
	    /// an undocumented internal cleanup functionality.
	    /// <p/><b>NOTE:</b> Enabling this is completely unsupported
	    /// by Java and may lead to JVM crashs if <c>IndexInput</c>
	    /// is closed while another thread is still accessing it (SIGSEGV).
	    /// </summary>
	    /// <throws>  IllegalArgumentException if <see cref="UNMAP_SUPPORTED" /> </throws>
	    /// <summary> is <c>false</c> and the workaround cannot be enabled.
	    /// </summary>
	    public virtual bool UseUnmap
	    {
	        get { return useUnmapHack; }
	        set
	        {
	            if (value && !UNMAP_SUPPORTED)
	                throw new System.ArgumentException("Unmap hack not supported on this platform!");
	            this.useUnmapHack = value;
	        }
	    }

	    /// <summary> Try to unmap the buffer, this method silently fails if no support
		/// for that in the JVM. On Windows, this leads to the fact,
		/// that mmapped files cannot be modified or deleted.
		/// </summary>
		internal void  CleanMapping(System.IO.MemoryStream buffer)
		{
			if (useUnmapHack)
			{
				try
				{
                    // {{Aroush-2.9}} Not converted: java.security.AccessController.doPrivileged()
                    //System.Diagnostics.Debug.Fail("Port issue:", "java.security.AccessController.doPrivileged()"); // {{Aroush-2.9}}
                    throw new NotImplementedException("Port issue: java.security.AccessController.doPrivileged()");
					// AccessController.DoPrivileged(new AnonymousClassPrivilegedExceptionAction(buffer, this));
				}
				catch (System.Exception e)
				{
					System.IO.IOException ioe = new System.IO.IOException("unable to unmap the mapped buffer", e.InnerException);
					throw ioe;
				}
			}
		}

	    /// <summary> Gets or sets the maximum chunk size (default is <see cref="int.MaxValue" /> for
	    /// 64 bit JVMs and 256 MiBytes for 32 bit JVMs) used for memory mapping.
	    /// Especially on 32 bit platform, the address space can be very fragmented,
	    /// so large index files cannot be mapped.
	    /// Using a lower chunk size makes the directory implementation a little
	    /// bit slower (as the correct chunk must be resolved on each seek)
	    /// but the chance is higher that mmap does not fail. On 64 bit
	    /// Java platforms, this parameter should always be <see cref="int.MaxValue" />,
	    /// as the adress space is big enough.
	    /// </summary>
	    public virtual int MaxChunkSize
	    {
	        get { return maxBBuf; }
	        set
	        {
	            if (value <= 0)
	                throw new System.ArgumentException("Maximum chunk size for mmap must be >0");
	            this.maxBBuf = value;
	        }
	    }

	    private class MMapIndexInput : IndexInput
		{
			private void  InitBlock(MMapDirectory enclosingInstance)
			{
				this.enclosingInstance = enclosingInstance;
			}
			private MMapDirectory enclosingInstance;
			public MMapDirectory Enclosing_Instance
			{
				get
				{
					return enclosingInstance;
				}
				
			}
			
			private System.IO.MemoryStream buffer;
			private long length;
			private bool isClone;
		    private bool isDisposed;
			
			internal MMapIndexInput(MMapDirectory enclosingInstance, System.IO.FileStream raf)
			{
                byte[] data = new byte[raf.Length];
                raf.Read(data, 0, (int) raf.Length);

				InitBlock(enclosingInstance);
				this.length = raf.Length;
				this.buffer = new System.IO.MemoryStream(data);
			}
			
			public override byte ReadByte()
			{
				try
				{
					return (byte) buffer.ReadByte();
				}
				catch (ObjectDisposedException)
				{
					throw new System.IO.IOException("read past EOF");
				}
			}
			
			public override void  ReadBytes(byte[] b, int offset, int len)
			{
				try
				{
					buffer.Read(b, offset, len);
				}
				catch (ObjectDisposedException)
				{
					throw new System.IO.IOException("read past EOF");
				}
			}

		    public override long FilePointer
		    {
		        get
		        {
		            return buffer.Position;
		        }
		    }

		    public override void  Seek(long pos)
			{
				buffer.Seek(pos, System.IO.SeekOrigin.Begin);
			}
			
			public override long Length()
			{
				return length;
			}
			
			public override System.Object Clone()
			{
                if (buffer == null)
                    throw new AlreadyClosedException("MMapIndexInput already closed");
				MMapIndexInput clone = (MMapIndexInput) base.Clone();
				clone.isClone = true;
				// clone.buffer = buffer.duplicate();   // {{Aroush-1.9}}
				return clone;
			}
			
			protected override void Dispose(bool isDisposing)
			{
                if (isDisposed) return;

                if (isDisposing)
                {
                    if (isClone || buffer == null)
                        return;
                    // unmap the buffer (if enabled) and at least unset it for GC
                    try
                    {
                        Enclosing_Instance.CleanMapping(buffer);
                    }
                    finally
                    {
                        buffer = null;
                    }
                }

			    isDisposed = true;
			}
		}
		
		// Because Java's ByteBuffer uses an int to address the
		// values, it's necessary to access a file >
		// Integer.MAX_VALUE in size using multiple byte buffers.
		protected internal class MultiMMapIndexInput:IndexInput, System.ICloneable
		{
			private void  InitBlock(MMapDirectory enclosingInstance)
			{
				this.enclosingInstance = enclosingInstance;
			}
			private MMapDirectory enclosingInstance;
			public MMapDirectory Enclosing_Instance
			{
				get
				{
					return enclosingInstance;
				}
				
			}
			
			private System.IO.MemoryStream[] buffers;
			private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
			
			private long length;

		    private bool isDisposed;
			
			private int curBufIndex;
			private int maxBufSize;
			
			private System.IO.MemoryStream curBuf; // redundant for speed: buffers[curBufIndex]
			private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position())
			
			private bool isClone = false;
			
			public MultiMMapIndexInput(MMapDirectory enclosingInstance, System.IO.FileStream raf, int maxBufSize)
			{
				InitBlock(enclosingInstance);
				this.length = raf.Length;
				this.maxBufSize = maxBufSize;
				
				if (maxBufSize <= 0)
					throw new System.ArgumentException("Non positive maxBufSize: " + maxBufSize);
				
				if ((length / maxBufSize) > System.Int32.MaxValue)
				{
					throw new System.ArgumentException("RandomAccessFile too big for maximum buffer size: " + raf.ToString());
				}
				
				int nrBuffers = (int) (length / maxBufSize);
				if (((long) nrBuffers * maxBufSize) < length)
					nrBuffers++;
				
				this.buffers = new System.IO.MemoryStream[nrBuffers];
				this.bufSizes = new int[nrBuffers];
				
				long bufferStart = 0;
				System.IO.FileStream rafc = raf;
				for (int bufNr = 0; bufNr < nrBuffers; bufNr++)
				{
                    byte[] data = new byte[rafc.Length];
                    raf.Read(data, 0, (int) rafc.Length);

					int bufSize = (length > (bufferStart + maxBufSize))?maxBufSize:(int) (length - bufferStart);
					this.buffers[bufNr] = new System.IO.MemoryStream(data);
					this.bufSizes[bufNr] = bufSize;
					bufferStart += bufSize;
				}
				Seek(0L);
			}
			
			public override byte ReadByte()
			{
				// Performance might be improved by reading ahead into an array of
				// e.g. 128 bytes and readByte() from there.
				if (curAvail == 0)
				{
					curBufIndex++;
					if (curBufIndex >= buffers.Length)
						throw new System.IO.IOException("read past EOF");
					curBuf = buffers[curBufIndex];
					curBuf.Seek(0, System.IO.SeekOrigin.Begin);
					curAvail = bufSizes[curBufIndex];
				}
				curAvail--;
				return (byte) curBuf.ReadByte();
			}
			
			public override void  ReadBytes(byte[] b, int offset, int len)
			{
				while (len > curAvail)
				{
					curBuf.Read(b, offset, curAvail);
					len -= curAvail;
					offset += curAvail;
					curBufIndex++;
					if (curBufIndex >= buffers.Length)
						throw new System.IO.IOException("read past EOF");
					curBuf = buffers[curBufIndex];
					curBuf.Seek(0, System.IO.SeekOrigin.Begin);
					curAvail = bufSizes[curBufIndex];
				}
				curBuf.Read(b, offset, len);
				curAvail -= len;
			}

		    public override long FilePointer
		    {
		        get { return ((long) curBufIndex*maxBufSize) + curBuf.Position; }
		    }

		    public override void  Seek(long pos)
			{
				curBufIndex = (int) (pos / maxBufSize);
				curBuf = buffers[curBufIndex];
				int bufOffset = (int) (pos - ((long) curBufIndex * maxBufSize));
				curBuf.Seek(bufOffset, System.IO.SeekOrigin.Begin);
				curAvail = bufSizes[curBufIndex] - bufOffset;
			}
			
			public override long Length()
			{
				return length;
			}
			
			public override System.Object Clone()
			{
				MultiMMapIndexInput clone = (MultiMMapIndexInput) base.Clone();
				clone.isClone = true;
				clone.buffers = new System.IO.MemoryStream[buffers.Length];
				// No need to clone bufSizes.
				// Since most clones will use only one buffer, duplicate() could also be
				// done lazy in clones, e.g. when adapting curBuf.
				for (int bufNr = 0; bufNr < buffers.Length; bufNr++)
				{
					clone.buffers[bufNr] = buffers[bufNr];    // clone.buffers[bufNr] = buffers[bufNr].duplicate();   // {{Aroush-1.9}} how do we clone?!
				}
				try
				{
					clone.Seek(FilePointer);
				}
				catch (System.IO.IOException ioe)
				{
					System.SystemException newException = new System.SystemException(ioe.Message, ioe);
					throw newException;
				}
				return clone;
			}

            protected override void Dispose(bool disposing)
            {
                if (isDisposed) return;
                if (isClone || buffers == null)
                    return;
                try
                {
                    for (int bufNr = 0; bufNr < buffers.Length; bufNr++)
                    {
                        // unmap the buffer (if enabled) and at least unset it for GC
                        try
                        {
                            Enclosing_Instance.CleanMapping(buffers[bufNr]);
                        }
                        finally
                        {
                            buffers[bufNr] = null;
                        }
                    }
                }
                finally
                {
                    buffers = null;
                }
                isDisposed = true;
            }
		}
		
		/// <summary>Creates an IndexInput for the file with the given name. </summary>
		public override IndexInput OpenInput(System.String name, int bufferSize)
		{
			EnsureOpen();
			System.String path = System.IO.Path.Combine(Directory.FullName, name);
			System.IO.FileStream raf = new System.IO.FileStream(path, System.IO.FileMode.Open, System.IO.FileAccess.Read);
			try
			{
				return (raf.Length <= (long) maxBBuf)?(IndexInput) new MMapIndexInput(this, raf):(IndexInput) new MultiMMapIndexInput(this, raf, maxBBuf);
			}
			finally
			{
				raf.Close();
			}
		}
		
		/// <summary>Creates an IndexOutput for the file with the given name. </summary>
		public override IndexOutput CreateOutput(System.String name)
		{
			InitOutput(name);
			return new SimpleFSDirectory.SimpleFSIndexOutput(new System.IO.FileInfo(System.IO.Path.Combine(internalDirectory.FullName, name)));
		}
		static MMapDirectory()
		{
			{
				bool v;
				try
				{
                    // {{Aroush-2.9
					/*
                    System.Type.GetType("sun.misc.Cleaner"); // {{Aroush-2.9}} port issue?
					System.Type.GetType("java.nio.DirectByteBuffer").GetMethod("cleaner", (NO_PARAM_TYPES == null)?new System.Type[0]:(System.Type[]) NO_PARAM_TYPES);
                    */
                    //System.Diagnostics.Debug.Fail("Port issue:", "sun.misc.Cleaner.clean()"); // {{Aroush-2.9}}
                    throw new NotImplementedException("Port issue: sun.misc.Cleaner.clean()");
                    // Aroush-2.9}}
					//v = true;
				}
				catch (System.Exception)
				{
					v = false;
				}
				UNMAP_SUPPORTED = v;
			}
		}
	}
}