path: root/src/core/Index/IndexFileDeleter.cs

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements.  See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License.  You may obtain a copy of the License at
* 
* http://www.apache.org/licenses/LICENSE-2.0
* 
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System;
using System.Collections.Generic;
using Lucene.Net.Support;
using Directory = Lucene.Net.Store.Directory;

namespace Lucene.Net.Index
{

    /// <summary>
    /// <para>This class keeps track of each SegmentInfos instance that
    /// is still "live", either because it corresponds to a
    /// segments_N file in the Directory (a "commit", i.e. a
    /// committed SegmentInfos) or because it's an in-memory
    /// SegmentInfos that a writer is actively updating but has
    /// not yet committed.  This class uses simple reference
    /// counting to map the live SegmentInfos instances to
    /// individual files in the Directory.</para>
    /// 
    /// <para>The same directory file may be referenced by more than
    /// one IndexCommit, i.e. more than one SegmentInfos.
    /// Therefore we count how many commits reference each file.
    /// When all the commits referencing a certain file have been
    /// deleted, the refcount for that file becomes zero, and the
    /// file is deleted.</para>
    ///
    /// <para>A separate deletion policy interface
    /// (IndexDeletionPolicy) is consulted on creation (onInit)
    /// and once per commit (onCommit), to decide when a commit
    /// should be removed.</para>
    /// 
    /// <para>It is the business of the IndexDeletionPolicy to choose
    /// when to delete commit points.  The actual mechanics of
    /// file deletion, retrying, etc, derived from the deletion
    /// of commit points is the business of the IndexFileDeleter.</para>
    /// 
    /// <para>The current default deletion policy is
    /// <see cref="KeepOnlyLastCommitDeletionPolicy"/>, which removes all
    /// prior commits when a new commit has completed.  This
    /// matches the behavior before 2.2.</para>
    ///
    /// <para>Note that you must hold the write.lock before
    /// instantiating this class.  It opens segments_N file(s)
    /// directly with no retry logic.</para>
    /// </summary>
	
	public sealed class IndexFileDeleter : IDisposable
	{
		
		//// Files that we tried to delete but failed (likely
		/// because they are open and we are running on Windows),
		/// so we will retry them again later: ////
		private IList<string> deletable;
		
		//// Reference count for all files in the index.  
		/// Counts how many existing commits reference a file.
		/// Maps String to RefCount (class below) instances: ////
		private IDictionary<string, RefCount> refCounts = new HashMap<string, RefCount>();
		
		//// Holds all commits (segments_N) currently in the index.
		/// This will have just 1 commit if you are using the
		/// default delete policy (KeepOnlyLastCommitDeletionPolicy).
		/// Other policies may leave commit points live for longer
		/// in which case this list would be longer than 1: ////
        private List<CommitPoint> commits = new List<CommitPoint>();
		
		//// Holds files we had incref'd from the previous
		/// non-commit checkpoint: ////
        private List<ICollection<string>> lastFiles = new List<ICollection<string>>();
		
		//// Commits that the IndexDeletionPolicy have decided to delete: ////
        private List<CommitPoint> commitsToDelete = new List<CommitPoint>();
		
		private System.IO.StreamWriter infoStream;
		private Directory directory;
		private IndexDeletionPolicy policy;
		private DocumentsWriter docWriter;
		
		internal bool startingCommitDeleted;
        private SegmentInfos lastSegmentInfos;

        private HashSet<string> synced;
		
		/// <summary>Change to true to see details of reference counts when
		/// infoStream != null 
		/// </summary>
		public static bool VERBOSE_REF_COUNTS = false;
		
		internal void  SetInfoStream(System.IO.StreamWriter infoStream)
		{
			this.infoStream = infoStream;
			if (infoStream != null)
			{
				Message("setInfoStream deletionPolicy=" + policy);
			}
		}
		
		private void  Message(System.String message)
		{
            infoStream.WriteLine("IFD [" + new DateTime().ToString() + "; " + ThreadClass.Current().Name + "]: " + message);
		}
		
		/// <summary> Initialize the deleter: find all previous commits in
		/// the Directory, incref the files they reference, call
		/// the policy to let it delete commits.  This will remove
		/// any files not referenced by any of the commits.
		/// </summary>
		/// <throws>  CorruptIndexException if the index is corrupt </throws>
		/// <throws>  IOException if there is a low-level IO error </throws>
        public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, System.IO.StreamWriter infoStream, DocumentsWriter docWriter, HashSet<string> synced)
		{
			
			this.docWriter = docWriter;
			this.infoStream = infoStream;
            this.synced = synced;
			
			if (infoStream != null)
			{
				Message("init: current segments file is \"" + segmentInfos.GetCurrentSegmentFileName() + "\"; deletionPolicy=" + policy);
			}
			
			this.policy = policy;
			this.directory = directory;
			
			// First pass: walk the files and initialize our ref
			// counts:
			long currentGen = segmentInfos.Generation;
			IndexFileNameFilter filter = IndexFileNameFilter.Filter;
			
			System.String[] files = directory.ListAll();
			
			CommitPoint currentCommitPoint = null;
			
			for (int i = 0; i < files.Length; i++)
			{
				
				System.String fileName = files[i];
				
				if (filter.Accept(null, fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
				{
					
					// Add this file to refCounts with initial count 0:
					GetRefCount(fileName);
					
					if (fileName.StartsWith(IndexFileNames.SEGMENTS))
					{
						
						// This is a commit (segments or segments_N), and
						// it's valid (<= the max gen).  Load it, then
						// incref all files it refers to:
                        if (infoStream != null)
                        {
                            Message("init: load commit \"" + fileName + "\"");
                        }
                        SegmentInfos sis = new SegmentInfos();
                        try
                        {
                            sis.Read(directory, fileName);
                        }
                        catch (System.IO.FileNotFoundException)
                        {
                            // LUCENE-948: on NFS (and maybe others), if
                            // you have writers switching back and forth
                            // between machines, it's very likely that the
                            // dir listing will be stale and will claim a
                            // file segments_X exists when in fact it
                            // doesn't.  So, we catch this and handle it
                            // as if the file does not exist
                            if (infoStream != null)
                            {
                                Message("init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
                            }
                            sis = null;
                        }
                        catch (System.IO.IOException)
                        {
                            if (SegmentInfos.GenerationFromSegmentsFileName(fileName) <= currentGen)
                            {
                                throw;
                            }
                            else
                            {
                                // Most likely we are opening an index that
                                // has an aborted "future" commit, so suppress
                                // exc in this case
                                sis = null;
                            }
                        }
                        if (sis != null)
                        {
                            CommitPoint commitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
                            if (sis.Generation == segmentInfos.Generation)
                            {
                                currentCommitPoint = commitPoint;
                            }
                            commits.Add(commitPoint);
                            IncRef(sis, true);

                            if (lastSegmentInfos == null || sis.Generation > lastSegmentInfos.Generation)
                            {
                                lastSegmentInfos = sis;
                            }
						}
					}
				}
			}
			
			if (currentCommitPoint == null)
			{
				// We did not in fact see the segments_N file
				// corresponding to the segmentInfos that was passed
				// in.  Yet, it must exist, because our caller holds
				// the write lock.  This can happen when the directory
				// listing was stale (eg when index accessed via NFS
				// client with stale directory listing cache).  So we
				// try now to explicitly open this commit point:
				SegmentInfos sis = new SegmentInfos();
				try
				{
					sis.Read(directory, segmentInfos.GetCurrentSegmentFileName());
				}
				catch (System.IO.IOException)
				{
					throw new CorruptIndexException("failed to locate current segments_N file");
				}
				if (infoStream != null)
					Message("forced open of current segments file " + segmentInfos.GetCurrentSegmentFileName());
				currentCommitPoint = new CommitPoint(this, commitsToDelete, directory, sis);
				commits.Add(currentCommitPoint);
				IncRef(sis, true);
			}
			
			// We keep commits list in sorted order (oldest to newest):
			commits.Sort();
			
			// Now delete anything with ref count at 0.  These are
			// presumably abandoned files eg due to crash of
			// IndexWriter.
			foreach(KeyValuePair<string, RefCount> entry in refCounts)
			{
                string fileName = entry.Key;
				RefCount rc = refCounts[fileName];
				if (0 == rc.count)
				{
					if (infoStream != null)
					{
						Message("init: removing unreferenced file \"" + fileName + "\"");
					}
					DeleteFile(fileName);
				}
			}
			
			// Finally, give policy a chance to remove things on
			// startup:
			policy.OnInit(commits);
			
			// Always protect the incoming segmentInfos since
			// sometime it may not be the most recent commit
			Checkpoint(segmentInfos, false);
			
			startingCommitDeleted = currentCommitPoint.IsDeleted;
			
			DeleteCommits();
		}

        public SegmentInfos LastSegmentInfos
        {
            get { return lastSegmentInfos; }
        }

        /// <summary> Remove the CommitPoints in the commitsToDelete List by
		/// DecRef'ing all files from each SegmentInfos.
		/// </summary>
		private void  DeleteCommits()
		{
			
			int size = commitsToDelete.Count;
			
			if (size > 0)
			{
				
				// First decref all files that had been referred to by
				// the now-deleted commits:
				for (int i = 0; i < size; i++)
				{
					CommitPoint commit = commitsToDelete[i];
					if (infoStream != null)
					{
						Message("deleteCommits: now decRef commit \"" + commit.SegmentsFileName + "\"");
					}
					foreach(string file in commit.files)
					{
						DecRef(file);
					}
				}
				commitsToDelete.Clear();
				
				// Now compact commits to remove deleted ones (preserving the sort):
				size = commits.Count;
				int readFrom = 0;
				int writeTo = 0;
				while (readFrom < size)
				{
					CommitPoint commit = commits[readFrom];
					if (!commit.deleted)
					{
						if (writeTo != readFrom)
						{
							commits[writeTo] = commits[readFrom];
						}
						writeTo++;
					}
					readFrom++;
				}
				
				while (size > writeTo)
				{
					commits.RemoveAt(size - 1);
					size--;
				}
			}
		}
		
		/// <summary> Writer calls this when it has hit an error and had to
		/// roll back, to tell us that there may now be
		/// unreferenced files in the filesystem.  So we re-list
		/// the filesystem and delete such files.  If segmentName
		/// is non-null, we will only delete files corresponding to
		/// that segment.
		/// </summary>
		public void  Refresh(System.String segmentName)
		{
			System.String[] files = directory.ListAll();
			IndexFileNameFilter filter = IndexFileNameFilter.Filter;
			System.String segmentPrefix1;
			System.String segmentPrefix2;
			if (segmentName != null)
			{
				segmentPrefix1 = segmentName + ".";
				segmentPrefix2 = segmentName + "_";
			}
			else
			{
				segmentPrefix1 = null;
				segmentPrefix2 = null;
			}
			
			for (int i = 0; i < files.Length; i++)
			{
				System.String fileName = files[i];
				if (filter.Accept(null, fileName) && (segmentName == null || fileName.StartsWith(segmentPrefix1) || fileName.StartsWith(segmentPrefix2)) && !refCounts.ContainsKey(fileName) && !fileName.Equals(IndexFileNames.SEGMENTS_GEN))
				{
					// Unreferenced file, so remove it
					if (infoStream != null)
					{
						Message("refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
					}
					DeleteFile(fileName);
				}
			}
		}
		
		public void  Refresh()
		{
			Refresh(null);
		}
		
		public void Dispose()
		{
            // Move to protected method if class becomes unsealed
			// DecRef old files from the last checkpoint, if any:
			int size = lastFiles.Count;
			if (size > 0)
			{
				for (int i = 0; i < size; i++)
					DecRef(lastFiles[i]);
				lastFiles.Clear();
			}
			
			DeletePendingFiles();
		}
		
		private void  DeletePendingFiles()
		{
			if (deletable != null)
			{
				IList<string> oldDeletable = deletable;
				deletable = null;
				int size = oldDeletable.Count;
				for (int i = 0; i < size; i++)
				{
					if (infoStream != null)
					{
						Message("delete pending file " + oldDeletable[i]);
					}
					DeleteFile(oldDeletable[i]);
				}
			}
		}
		
		/// <summary> For definition of "check point" see IndexWriter comments:
		/// "Clarification: Check Points (and commits)".
		/// 
		/// Writer calls this when it has made a "consistent
		/// change" to the index, meaning new files are written to
		/// the index and the in-memory SegmentInfos have been
		/// modified to point to those files.
		/// 
		/// This may or may not be a commit (segments_N may or may
		/// not have been written).
		/// 
		/// We simply incref the files referenced by the new
		/// SegmentInfos and decref the files we had previously
		/// seen (if any).
		/// 
		/// If this is a commit, we also call the policy to give it
		/// a chance to remove other commits.  If any commits are
		/// removed, we decref their files as well.
		/// </summary>
		public void  Checkpoint(SegmentInfos segmentInfos, bool isCommit)
		{
			
			if (infoStream != null)
			{
				Message("now checkpoint \"" + segmentInfos.GetCurrentSegmentFileName() + "\" [" + segmentInfos.Count + " segments " + "; isCommit = " + isCommit + "]");
			}
			
			// Try again now to delete any previously un-deletable
			// files (because they were in use, on Windows):
			DeletePendingFiles();
			
			// Incref the files:
			IncRef(segmentInfos, isCommit);
			
			if (isCommit)
			{
				// Append to our commits list:
				commits.Add(new CommitPoint(this, commitsToDelete, directory, segmentInfos));
				
				// Tell policy so it can remove commits:
				policy.OnCommit(commits);
				
				// Decref files for commits that were deleted by the policy:
				DeleteCommits();
			}
			else
			{
				
				IList<string> docWriterFiles;
				if (docWriter != null)
				{
					docWriterFiles = docWriter.OpenFiles();
					if (docWriterFiles != null)
					// We must incRef these files before decRef'ing
					// last files to make sure we don't accidentally
					// delete them:
						IncRef(docWriterFiles);
				}
				else
					docWriterFiles = null;
				
				// DecRef old files from the last checkpoint, if any:
				int size = lastFiles.Count;
				if (size > 0)
				{
					for (int i = 0; i < size; i++)
						DecRef(lastFiles[i]);
					lastFiles.Clear();
				}
				
				// Save files so we can decr on next checkpoint/commit:
                lastFiles.Add(segmentInfos.Files(directory, false));
				
                if (docWriterFiles != null)
                {
                    lastFiles.Add(docWriterFiles);
                }
			}
		}
		
		internal void  IncRef(SegmentInfos segmentInfos, bool isCommit)
		{
			// If this is a commit point, also incRef the
			// segments_N file:
			foreach(string fileName in segmentInfos.Files(directory, isCommit))
			{
				IncRef(fileName);
			}
		}

        internal void IncRef(ICollection<string> files)
		{
            foreach(string file in files)
			{
                IncRef(file);
			}
		}
		
		internal void  IncRef(string fileName)
		{
			RefCount rc = GetRefCount(fileName);
			if (infoStream != null && VERBOSE_REF_COUNTS)
			{
				Message("  IncRef \"" + fileName + "\": pre-incr count is " + rc.count);
			}
			rc.IncRef();
		}
		
		internal void  DecRef(ICollection<string> files)
		{
            foreach(string file in files)
            {
                DecRef(file);
            }
		}
		
		internal void  DecRef(System.String fileName)
		{
			RefCount rc = GetRefCount(fileName);
			if (infoStream != null && VERBOSE_REF_COUNTS)
			{
				Message("  DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
			}
			if (0 == rc.DecRef())
			{
				// This file is no longer referenced by any past
				// commit points nor by the in-memory SegmentInfos:
				DeleteFile(fileName);
				refCounts.Remove(fileName);

                if (synced != null) {
                    lock(synced) 
                    {
                      synced.Remove(fileName);
                    }
                }
			}
		}
		
		internal void  DecRef(SegmentInfos segmentInfos)
		{
			foreach(string file in segmentInfos.Files(directory, false))
			{
				DecRef(file);
			}
		}

        public bool Exists(String fileName)
        {
            if (!refCounts.ContainsKey(fileName))
            {
                return false;
            }
            else
            {
                return GetRefCount(fileName).count > 0;
            }
        }
		
		private RefCount GetRefCount(System.String fileName)
		{
			RefCount rc;
			if (!refCounts.ContainsKey(fileName))
			{
				rc = new RefCount(fileName);
				refCounts[fileName] = rc;
			}
			else
			{
				rc = refCounts[fileName];
			}
			return rc;
		}
		
		internal void  DeleteFiles(System.Collections.Generic.IList<string> files)
		{
			foreach(string file in files)
				DeleteFile(file);
		}
		
		/// <summary>Deletes the specified files, but only if they are new
		/// (have not yet been incref'd). 
		/// </summary>
        internal void DeleteNewFiles(System.Collections.Generic.ICollection<string> files)
		{
			foreach(string fileName in files)
			{
                if (!refCounts.ContainsKey(fileName))
                {
                    if (infoStream != null)
                    {
                        Message("delete new file \"" + fileName + "\"");
                    }
                    DeleteFile(fileName);
                }
			}
		}
		
		internal void  DeleteFile(System.String fileName)
		{
			try
			{
				if (infoStream != null)
				{
					Message("delete \"" + fileName + "\"");
				}
				directory.DeleteFile(fileName);
			}
			catch (System.IO.IOException e)
			{
				// if delete fails
				if (directory.FileExists(fileName))
				{
					
					// Some operating systems (e.g. Windows) don't
					// permit a file to be deleted while it is opened
					// for read (e.g. by another process or thread). So
					// we assume that when a delete fails it is because
					// the file is open in another process, and queue
					// the file for subsequent deletion.
					
					if (infoStream != null)
					{
						Message("IndexFileDeleter: unable to remove file \"" + fileName + "\": " + e.ToString() + "; Will re-try later.");
					}
					if (deletable == null)
					{
                        deletable = new List<string>();
					}
					deletable.Add(fileName); // add to deletable
				}
			}
		}
		
		/// <summary> Tracks the reference count for a single index file:</summary>
		sealed private class RefCount
		{
			
			// fileName used only for better assert error messages
			internal System.String fileName;
			internal bool initDone;
			internal RefCount(System.String fileName)
			{
				this.fileName = fileName;
			}
			
			internal int count;
			
			public int IncRef()
			{
				if (!initDone)
				{
					initDone = true;
				}
				else
				{
					System.Diagnostics.Debug.Assert(count > 0, "RefCount is 0 pre-increment for file " + fileName);
				}
				return ++count;
			}
			
			public int DecRef()
			{
				System.Diagnostics.Debug.Assert(count > 0, "RefCount is 0 pre-decrement for file " + fileName);
				return --count;
			}
		}
		
		/// <summary> Holds details for each commit point.  This class is
		/// also passed to the deletion policy.  Note: this class
		/// has a natural ordering that is inconsistent with
		/// equals.
		/// </summary>
		
		sealed private class CommitPoint:IndexCommit, System.IComparable<CommitPoint>
		{
            private void InitBlock(IndexFileDeleter enclosingInstance)
            {
                this.enclosingInstance = enclosingInstance;
            }
            private IndexFileDeleter enclosingInstance;
            public IndexFileDeleter Enclosing_Instance
            {
                get
                {
                    return enclosingInstance;
                }

            }
			
			internal long gen;
            internal ICollection<string> files;
			internal string segmentsFileName;
			internal bool deleted;
			internal Directory directory;
            internal ICollection<CommitPoint> commitsToDelete;
			internal long version;
			internal long generation;
			internal bool isOptimized;
            internal IDictionary<string, string> userData;
			
			public CommitPoint(IndexFileDeleter enclosingInstance, ICollection<CommitPoint> commitsToDelete, Directory directory, SegmentInfos segmentInfos)
			{
				InitBlock(enclosingInstance);
				this.directory = directory;
				this.commitsToDelete = commitsToDelete;
				userData = segmentInfos.UserData;
				segmentsFileName = segmentInfos.GetCurrentSegmentFileName();
				version = segmentInfos.Version;
				generation = segmentInfos.Generation;
                files = segmentInfos.Files(directory, true);
				gen = segmentInfos.Generation;
				isOptimized = segmentInfos.Count == 1 && !segmentInfos.Info(0).HasDeletions();
				
				System.Diagnostics.Debug.Assert(!segmentInfos.HasExternalSegments(directory));
			}

            public override string ToString()
            {
                return "IndexFileDeleter.CommitPoint(" + segmentsFileName + ")";
            }

		    public override bool IsOptimized
		    {
		        get { return isOptimized; }
		    }

		    public override string SegmentsFileName
		    {
		        get { return segmentsFileName; }
		    }

		    public override ICollection<string> FileNames
		    {
		        get { return files; }
		    }

		    public override Directory Directory
		    {
		        get { return directory; }
		    }

		    public override long Version
		    {
		        get { return version; }
		    }

		    public override long Generation
		    {
		        get { return generation; }
		    }

		    public override IDictionary<string, string> UserData
		    {
		        get { return userData; }
		    }

		    /// <summary> Called only be the deletion policy, to remove this
			/// commit point from the index.
			/// </summary>
			public override void  Delete()
			{
				if (!deleted)
				{
					deleted = true;
					Enclosing_Instance.commitsToDelete.Add(this);
				}
			}

		    public override bool IsDeleted
		    {
		        get { return deleted; }
		    }

		    public int CompareTo(CommitPoint commit)
			{
				if (gen < commit.gen)
				{
					return - 1;
				}
				else if (gen > commit.gen)
				{
					return 1;
				}
				else
				{
					return 0;
				}
			}
		}
	}
}