Fixed 'Bug 40991 - System.IO.IOException: Reading more than 2GB with

this call is not supported '

Unfortunately it's not possible to fix that in all occassions (regexes
only work on strings - not streams).

3 files changed, 124 insertions, 40 deletions

diff --git a/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FileProvider.cs b/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FileProvider.cs
index 23698d4a63..a0267de29e 100644
--- a/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FileProvider.cs
+++ b/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FileProvider.cs
@@ -76,22 +76,15 @@ namespace MonoDevelop.Ide.FindInFiles
 			SelectionEndPosition = selectionEndPosition;
 		}
 		
-		public string ReadString ()
+		public TextReader ReadString ()
 		{
 			return ReadString (false);
 		}
 
-		WeakReference cachedText;
-
-		public string ReadString (bool readBinaryFiles)
+		public TextReader ReadString (bool readBinaryFiles)
 		{
-			string result = cachedText != null ? cachedText.Target as string : null;
-			if (result != null) {
-				return result;
-			}
-
 			if (buffer != null) {
-				result = buffer.ToString ();
+				return new StringReader (buffer.ToString ());
 			} else {
 				Document doc = null;
 
@@ -100,25 +93,20 @@ namespace MonoDevelop.Ide.FindInFiles
 					doc = task.Result;
 
 				if (doc != null && doc.Editor != null) {
-					result = doc.Editor.Text;
-					encoding = doc.Editor.Encoding;
-					hadBom = doc.Editor.UseBOM;
+					return doc.Editor.CreateReader ();
 				} else {
 					try {
 						if (!File.Exists (FileName))
 							return null;
-						var content = File.ReadAllBytes (FileName);
-						if (!readBinaryFiles && TextFileUtility.IsBinary (content))
+						if (!readBinaryFiles && TextFileUtility.IsBinary (FileName))
 							return null;
-						result = TextFileUtility.GetText (content, out encoding, out hadBom);
+						return TextFileUtility.OpenStream (FileName);
 					} catch (Exception e) {
 						LoggingService.LogError ("Error while opening " + FileName, e);
 						return null;
 					}
 				}
 			}
-			cachedText = new WeakReference (result);
-			return result;
 		}
 		
 		async Task<Document> SearchDocument ()
diff --git a/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FindReplace.cs b/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FindReplace.cs
index aaae54a669..fe94b88198 100644
--- a/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FindReplace.cs
+++ b/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/FindReplace.cs
@@ -34,6 +34,7 @@ using Gtk;
 using System.Threading.Tasks;
 using System.Diagnostics;
 using System.Threading;
+using Microsoft.CodeAnalysis.CSharp.Syntax;
 
 namespace MonoDevelop.Ide.FindInFiles
 {
@@ -79,6 +80,22 @@ namespace MonoDevelop.Ide.FindInFiles
 			return true;
 		}
 
+		class FileSearchResult
+		{
+			public FileProvider Provider;
+			public TextReader Reader;
+			public List<SearchResult> Results;
+			public string Text { get; internal set; }
+
+			public FileSearchResult (FileProvider provider, TextReader reader, List<SearchResult> results)
+			{
+				Provider = provider;
+				Reader = reader;
+				Results = results;
+			}
+		}
+
+
 		public IEnumerable<SearchResult> FindAll (Scope scope, ProgressMonitor monitor, string pattern, string replacePattern, FilterOptions filter, CancellationToken token)
 		{
 			if (filter.RegexSearch) {
@@ -90,18 +107,20 @@ namespace MonoDevelop.Ide.FindInFiles
 			IsRunning = true;
 			FoundMatchesCount = SearchedFilesCount = 0;
 			monitor.BeginTask (scope.GetDescription (filter, pattern, replacePattern), 150);
+			Stopwatch sw = new Stopwatch ();
+			sw.Start ();
 			try {
 				int totalWork = scope.GetTotalWork (filter);
 				int step = Math.Max (1, totalWork / 50);
 
 
-				var contents = new List<Tuple<FileProvider, string, List<SearchResult>>>();
+				var contents = new List<FileSearchResult>();
 				foreach (var provider in scope.GetFiles (monitor, filter)) {
 					if (token.IsCancellationRequested)
 						return Enumerable.Empty<SearchResult> ();
 					try {
 						searchedFilesCount++;
-						contents.Add(Tuple.Create (provider, provider.ReadString (), new List<SearchResult> ()));
+						contents.Add(new FileSearchResult (provider, provider.ReadString (), new List<SearchResult> ()));
 						if (searchedFilesCount % step == 0)
 							monitor.Step (2); 
 					} catch (FileNotFoundException) {
@@ -114,7 +133,7 @@ namespace MonoDevelop.Ide.FindInFiles
 					foreach (var content in contents) {
 						if (token.IsCancellationRequested)
 							return Enumerable.Empty<SearchResult> ();
-						results.AddRange (RegexSearch (monitor, content.Item1, content.Item2, replacePattern, filter));
+						results.AddRange (RegexSearch (monitor, content.Provider, content.Reader, replacePattern, filter));
 					}
 				} else {
 					var options = new ParallelOptions ();
@@ -125,11 +144,15 @@ namespace MonoDevelop.Ide.FindInFiles
 							return;
 						try {
 							Interlocked.Increment (ref searchedFilesCount);
-							content.Item3.AddRange(FindAll (monitor, content.Item1, content.Item2, pattern, replacePattern, filter));
+							if (replacePattern != null) {
+								content.Text = content.Reader.ReadToEnd ();
+								content.Reader = new StringReader (content.Text);
+							}
+							content.Results.AddRange(FindAll (monitor, content.Provider, content.Reader, pattern, replacePattern, filter));
 							lock (results) {
-								results.AddRange (content.Item3);
+								results.AddRange (content.Results);
 							}
-							FoundMatchesCount += content.Item3.Count;
+							FoundMatchesCount += content.Results.Count;
 							if (searchedFilesCount % step == 0)
 								monitor.Step (1); 
 						} catch (Exception e) {
@@ -141,12 +164,12 @@ namespace MonoDevelop.Ide.FindInFiles
 						foreach (var content in contents) {
 							if (token.IsCancellationRequested)
 								return Enumerable.Empty<SearchResult> ();
-							if (content.Item3.Count == 0)
+							if (content.Results.Count == 0)
 								continue;
 							try {
-								content.Item1.BeginReplace (content.Item2);
-								Replace (content.Item1, content.Item3, replacePattern);
-								content.Item1.EndReplace ();
+								content.Provider.BeginReplace (content.Text);
+								Replace (content.Provider, content.Results, replacePattern);
+								content.Provider.EndReplace ();
 							} catch (Exception e) {
 								LoggingService.LogError("Exception during replace.", e);
 							}
@@ -157,11 +180,15 @@ namespace MonoDevelop.Ide.FindInFiles
 				return results;
 			} finally {
 				monitor.EndTask ();
+				sw.Stop ();
+				Console.WriteLine ("Took:" + sw.ElapsedMilliseconds);
 				IsRunning = false;
 			}
 		}
 
-		IEnumerable<SearchResult> FindAll (ProgressMonitor monitor, FileProvider provider, string content, string pattern, string replacePattern, FilterOptions filter)
+		// Took: 17743
+
+		IEnumerable<SearchResult> FindAll (ProgressMonitor monitor, FileProvider provider, TextReader content, string pattern, string replacePattern, FilterOptions filter)
 		{
 			if (string.IsNullOrEmpty (pattern))
 				return Enumerable.Empty<SearchResult> ();
@@ -172,8 +199,9 @@ namespace MonoDevelop.Ide.FindInFiles
 			return Search (provider, content, pattern, filter);
 		}
 
-		IEnumerable<SearchResult> RegexSearch (ProgressMonitor monitor, FileProvider provider, string content, string replacePattern, FilterOptions filter)
+		IEnumerable<SearchResult> RegexSearch (ProgressMonitor monitor, FileProvider provider, TextReader reader, string replacePattern, FilterOptions filter)
 		{
+			string content = reader.ReadToEnd ();
 			var results = new List<SearchResult> ();
 			if (replacePattern == null) {
 				foreach (Match match in regex.Matches (content)) {
@@ -212,18 +240,86 @@ namespace MonoDevelop.Ide.FindInFiles
 			return results;
 		}
 
-		public IEnumerable<SearchResult> Search (FileProvider provider, string content, string pattern, FilterOptions filter)
+		class RingBufferReader
 		{
-			if (string.IsNullOrEmpty (content))
+			int i, l;
+			char [] buffer;
+			TextReader reader;
+
+			public RingBufferReader (TextReader reader, int bufferSize)
+			{
+				this.reader = reader;
+				buffer = new char [bufferSize];
+			}
+
+			public int Next ()
+			{
+				if (l == 0) {
+					int ch = reader.Read ();
+					buffer [i] = (char)ch;
+					i = (i + 1) % buffer.Length;
+					return ch;
+				}
+				l--;
+				var result = buffer [i];
+				i = (i + 1) % buffer.Length;
+				return result;
+			}
+
+			public void TakeBack (int num)
+			{
+				l += num;
+				i = (i + buffer.Length - num) % buffer.Length;
+			}
+		}
+
+		public IEnumerable<SearchResult> Search (FileProvider provider, TextReader reader, string pattern, FilterOptions filter)
+		{
+			if (reader == null)
 				yield break;
-			int idx = provider.SelectionStartPosition < 0 ? 0 : Math.Max (0, provider.SelectionStartPosition);
-			var comparison = filter.CaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase;
-			int end = provider.SelectionEndPosition < 0 ? content.Length : Math.Min (content.Length, provider.SelectionEndPosition);
-			while ((idx = content.IndexOf (pattern, idx, end - idx, comparison)) >= 0) {
-				if (!filter.WholeWordsOnly || FilterOptions.IsWholeWordAt (content, idx, pattern.Length)) {
-					yield return new SearchResult (provider, idx, pattern.Length);
+			int i = provider.SelectionStartPosition < 0 ? 0 : Math.Max (0, provider.SelectionStartPosition);
+			var buffer = new RingBufferReader(reader, pattern.Length + 2);
+			bool wasSeparator = true;
+			if (!filter.CaseSensitive)
+				pattern = pattern.ToUpperInvariant ();
+			while (true) {
+				int next = buffer.Next ();
+				if (next < 0)
+					yield break;
+				char ch = (char)next;
+				if ((filter.CaseSensitive ? ch : char.ToUpperInvariant (ch)) == pattern [0] &&
+				    (!filter.WholeWordsOnly || wasSeparator)) {
+					bool isMatch = true;
+					for (int j = 1; j < pattern.Length; j++) {
+						next = buffer.Next ();
+						if (next < 0)
+							yield break;
+						if ((filter.CaseSensitive ? next : char.ToUpperInvariant ((char)next)) != pattern [j]) {
+							buffer.TakeBack (j);
+							isMatch = false;
+							break;
+						}
+					}
+					if (isMatch) {
+						if (filter.WholeWordsOnly) {
+							next = buffer.Next ();
+							if (next >= 0 && !FilterOptions.IsWordSeparator ((char)next)) {
+								buffer.TakeBack (pattern.Length);
+								i++;
+								continue;
+							}
+							buffer.TakeBack (1);
+						}
+
+						yield return new SearchResult (provider, i, pattern.Length);
+						i += pattern.Length - 1;
+					}
+				}
+
+				i++;
+				if (filter.WholeWordsOnly) {
+					wasSeparator = FilterOptions.IsWordSeparator ((char)ch);
 				}
-				idx += pattern.Length;
 			}
 		}
 
diff --git a/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/SearchResultWidget.cs b/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/SearchResultWidget.cs
index 3b9eb3c30c..e77549f160 100644
--- a/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/SearchResultWidget.cs
+++ b/main/src/core/MonoDevelop.Ide/MonoDevelop.Ide.FindInFiles/SearchResultWidget.cs
@@ -677,7 +677,7 @@ namespace MonoDevelop.Ide.FindInFiles
 				if (content == null)
 					return null;
 
-				doc = TextEditorFactory.CreateNewEditor (TextEditorFactory.CreateNewReadonlyDocument (new StringTextSource (content), result.FileName, DesktopService.GetMimeTypeForUri (result.FileName)));
+				doc = TextEditorFactory.CreateNewEditor (TextEditorFactory.CreateNewReadonlyDocument (new StringTextSource (content.ReadToEnd ()), result.FileName, DesktopService.GetMimeTypeForUri (result.FileName)));
 
 				documents [result.FileName] = doc;	
 			}