Sync with vs-editor-core@067bb659f

4 files changed, 135 insertions, 101 deletions

diff --git a/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs b/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs
index 5ae8b2f..66cee18 100644
--- a/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs
+++ b/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs
@@ -16,6 +16,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
     using System.IO.MemoryMappedFiles;
     using System.Text;
     using Microsoft.VisualStudio.Text.Differencing;
+    using Microsoft.VisualStudio.Text.Document;
     using Microsoft.VisualStudio.Text.Projection;
     using Microsoft.VisualStudio.Text.Projection.Implementation;
     using Microsoft.VisualStudio.Text.Utilities;
@@ -70,13 +71,16 @@ namespace Microsoft.VisualStudio.Text.Implementation
 
         [Import]
         internal IDifferenceService _differenceService { get; set; }
-        
+
         [Import]
         internal ITextDifferencingSelectorService _textDifferencingSelectorService { get; set; }
 
         [Import]
         internal GuardedOperations _guardedOperations { get; set; }
 
+        [Import]
+        internal IWhitespaceManagerFactory _whitespaceManagerFactory { get; set; }
+
         #endregion
 
         #region Private state
@@ -202,6 +206,11 @@ namespace Microsoft.VisualStudio.Text.Implementation
 
         public ITextBuffer CreateTextBuffer(TextReader reader, IContentType contentType, long length, string traceId)
         {
+            return this.CreateTextBuffer(reader, contentType, length, traceId, throwOnInvalidCharacters: false);
+        }
+
+        public ITextBuffer CreateTextBuffer(TextReader reader, IContentType contentType, long length, string traceId, bool throwOnInvalidCharacters)
+        {
             if (reader == null)
             {
                 throw new ArgumentNullException(nameof(reader));
@@ -215,25 +224,30 @@ namespace Microsoft.VisualStudio.Text.Implementation
                 throw new InvalidOperationException(Strings.FileTooLarge);
             }
 
-            bool hasConsistentLineEndings;
             int longestLineLength;
-            StringRebuilder content = TextImageLoader.Load(reader, length, out hasConsistentLineEndings, out longestLineLength);
+            StringRebuilder content = TextImageLoader.Load(
+                reader,
+                length,
+                out var newlineState,
+                out var leadingWhitespaceState,
+                out longestLineLength,
+                throwOnInvalidCharacters: throwOnInvalidCharacters);
 
             ITextBuffer buffer = Make(contentType, content, false);
-            if (!hasConsistentLineEndings)
-            {
-                // leave a sign that line endings are inconsistent. This is rather nasty but for now
-                // we don't want to pollute the API with this factoid
-                buffer.Properties.AddProperty("InconsistentLineEndings", true);
-            }
-            // leave a similar sign about the longest line in the buffer.
+
+            // Make the call to GetWhitespaceManager to add the manager to the properties. We don't need the return value here.
+            var _ = _whitespaceManagerFactory.GetOrCreateWhitespaceManager(buffer, newlineState, leadingWhitespaceState);
+
+            // Leave a sign about the longest line in the buffer. This is rather nasty, but for now
+            // we don't want to pollute the API with this factoid
+            buffer.Properties["LongestLineLength"] = longestLineLength;
 
             return buffer;
         }
 
         public ITextBuffer CreateTextBuffer(TextReader reader, IContentType contentType)
         {
-            return CreateTextBuffer(reader, contentType, -1, "legacy");
+            return CreateTextBuffer(reader, contentType, -1, "legacy", throwOnInvalidCharacters: false);
         }
 
         internal static StringRebuilder StringRebuilderFromSnapshotAndSpan(ITextSnapshot snapshot, Span span)
@@ -260,16 +274,19 @@ namespace Microsoft.VisualStudio.Text.Implementation
 
         internal static StringRebuilder AppendStringRebuildersFromSnapshotAndSpan(StringRebuilder content, ITextSnapshot snapshot, Span span)
         {
-            var baseSnapshot = snapshot as BaseSnapshot;
-            if (baseSnapshot != null)
+            if (span.Length != 0)
             {
-                content = content.Append(baseSnapshot.Content.GetSubText(span));
-            }
-            else
-            {
-                // The we don't know what to do fallback. This should never be called unless someone provides a new snapshot
-                // implementation.
-                content = content.Append(snapshot.GetText(span));
+                var baseSnapshot = snapshot as BaseSnapshot;
+                if (baseSnapshot != null)
+                {
+                    content = content.Append(baseSnapshot.Content.GetSubText(span));
+                }
+                else
+                {
+                    // The we don't know what to do fallback. This should never be called unless someone provides a new snapshot
+                    // implementation.
+                    content = content.Append(snapshot.GetText(span));
+                }
             }
 
             return content;
@@ -283,10 +300,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
 
         public ITextImage CreateTextImage(TextReader reader, long length)
         {
-            bool hasConsistentLineEndings;
-            int longestLineLength;
-
-            return CachingTextImage.Create(TextImageLoader.Load(reader, length, out hasConsistentLineEndings, out longestLineLength), null);
+            return CachingTextImage.Create(TextImageLoader.Load(reader, length, out var _, out var _, out var _), null);
         }
 
         public ITextImage CreateTextImage(MemoryMappedFile source)
@@ -310,7 +324,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
             return buffer;
         }
 
-        public IProjectionBuffer CreateProjectionBuffer(IProjectionEditResolver projectionEditResolver, 
+        public IProjectionBuffer CreateProjectionBuffer(IProjectionEditResolver projectionEditResolver,
                                                         IList<object> trackingSpans,
                                                         ProjectionBufferOptions options,
                                                         IContentType contentType)
@@ -324,7 +338,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
             {
                 throw new ArgumentNullException(nameof(contentType));
             }
-            IProjectionBuffer buffer = 
+            IProjectionBuffer buffer =
                 new ProjectionBuffer(this, projectionEditResolver, contentType, trackingSpans, _differenceService, _textDifferencingSelectorService.DefaultTextDifferencingService, options, _guardedOperations);
             RaiseProjectionBufferCreatedEvent(buffer);
             return buffer;
diff --git a/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs b/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs
index da7646d..5678506 100644
--- a/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs
+++ b/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs
@@ -17,13 +17,19 @@ namespace Microsoft.VisualStudio.Text.Implementation
         public const int BlockSize = 16384;
 
         internal static StringRebuilder Load(TextReader reader, long fileSize,
-                                             out bool hasConsistentLineEndings, out int longestLineLength,
+                                             out NewlineState newlineState,
+                                             out LeadingWhitespaceState leadingWhitespaceState,
+                                             out int longestLineLength,
                                              int blockSize = 0,
-                                             int minCompressedBlockSize = TextImageLoader.BlockSize)                                             // Exposed for unit tests
+                                             int minCompressedBlockSize = TextImageLoader.BlockSize,                                             // Exposed for unit tests
+                                             bool throwOnInvalidCharacters = false)
         {
-            LineEndingState lineEnding = LineEndingState.Unknown;
+            newlineState = new NewlineState();
+            leadingWhitespaceState = new LeadingWhitespaceState();
+
             int currentLineLength = 0;
             longestLineLength = 0;
+            char thresholdForInvalidCharacters = throwOnInvalidCharacters ? '\u0001' : '\0';    // Basically the only invalid character is \0, if we are looking for invalid characters.
 
             bool useCompressedStringRebuilders = (fileSize >= TextModelOptions.CompressedStorageFileSizeThreshold);
             if (blockSize == 0)
@@ -44,6 +50,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
             StringRebuilder content = StringRebuilderForChars.Empty;
             try
             {
+                bool nextCharIsStartOfLine = true;
                 while (true)
                 {
                     int read = TextImageLoader.LoadNextBlock(reader, buffer);
@@ -51,7 +58,15 @@ namespace Microsoft.VisualStudio.Text.Implementation
                     if (read == 0)
                         break;
 
-                    var lineBreaks = TextImageLoader.ParseBlock(buffer, read, ref lineEnding, ref currentLineLength, ref longestLineLength);
+                    var lineBreaks = TextImageLoader.ParseBlock(
+                        buffer,
+                        read,
+                        thresholdForInvalidCharacters,
+                        ref newlineState,
+                        ref leadingWhitespaceState,
+                        ref currentLineLength,
+                        ref longestLineLength,
+                        ref nextCharIsStartOfLine);
 
                     char[] bufferForStringBuilder = buffer;
                     if (read < (buffer.Length / 2))
@@ -83,8 +98,6 @@ namespace Microsoft.VisualStudio.Text.Implementation
                 }
             }
 
-            hasConsistentLineEndings = lineEnding != LineEndingState.Inconsistent;
-
             return content;
         }
 
@@ -109,8 +122,16 @@ namespace Microsoft.VisualStudio.Text.Implementation
             return read;
         }
 
-        private static ILineBreaks ParseBlock(char[] buffer, int length,
-                                              ref LineEndingState lineEnding, ref int currentLineLength, ref int longestLineLength)
+        // Evil performance hack (but we are on a hot path here):
+        //  thresholdForInvalidCharacters should be '\u0001' if we are throwing on invalid characters.
+        //                                should be '\0' if we are not.
+        // (otherwise we need to check both a throwOnInvalidCharacters boolean and that c == 0).
+        private static ILineBreaks ParseBlock(char[] buffer, int length, char thresholdForInvalidCharacters,
+                                              ref NewlineState newlineState,
+                                              ref LeadingWhitespaceState leadingWhitespaceState,
+                                              ref int currentLineLength,
+                                              ref int longestLineLength,
+                                              ref bool nextCharIsStartOfLine)
         {
             // Note that the lineBreaks created here will (internally) use the pooled list of line breaks.
             IPooledLineBreaksEditor lineBreaks = LineBreakManager.CreatePooledLineBreakEditor(length);
@@ -121,8 +142,32 @@ namespace Microsoft.VisualStudio.Text.Implementation
                 int breakLength = TextUtilities.LengthOfLineBreak(buffer, index, length);
                 if (breakLength == 0)
                 {
+                    char c = buffer[index];
+
+                    // If we are checking for invalid characters, throw if we encounter a \0
+                    if (c < thresholdForInvalidCharacters)
+                        throw new FileFormatException("File contains NUL characters");
+
                     ++currentLineLength;
                     ++index;
+
+                    if (nextCharIsStartOfLine)
+                    {
+                        switch (c)
+                        {
+                            case ' ':
+                                leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Space, 1);
+                                break;
+                            case '\t':
+                                leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Tab, 1);
+                                break;
+                            default:
+                                leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Printable, 1);
+                                break;
+                        }
+
+                        nextCharIsStartOfLine = false;
+                    }
                 }
                 else
                 {
@@ -130,38 +175,34 @@ namespace Microsoft.VisualStudio.Text.Implementation
                     longestLineLength = Math.Max(longestLineLength, currentLineLength);
                     currentLineLength = 0;
 
-                    if (lineEnding != LineEndingState.Inconsistent)
+
+                    if (breakLength == 2)
                     {
-                        if (breakLength == 2)
-                        {
-                            if (lineEnding == LineEndingState.Unknown)
-                                lineEnding = LineEndingState.CRLF;
-                            else if (lineEnding != LineEndingState.CRLF)
-                                lineEnding = LineEndingState.Inconsistent;
-                        }
-                        else
+                        newlineState.Increment(NewlineState.LineEnding.CRLF, 1);
+                    }
+                    else
+                    {
+                        switch (buffer[index])
                         {
-                            LineEndingState newLineEndingState;
-                            switch (buffer[index])
-                            {
-                                // This code needs to be kep consistent with TextUtilities.LengthOfLineBreak()
-                                case '\r': newLineEndingState = LineEndingState.CR; break;
-                                case '\n': newLineEndingState = LineEndingState.LF; break;
-                                case '\u0085': newLineEndingState = LineEndingState.NEL; break;
-                                case '\u2028': newLineEndingState = LineEndingState.LS; break;
-                                case '\u2029': newLineEndingState = LineEndingState.PS; break;
-                                default: throw new InvalidOperationException("Unexpected line ending");
-                            }
-
-                            if (lineEnding == LineEndingState.Unknown)
-                                lineEnding = newLineEndingState;
-                            else if (lineEnding != newLineEndingState)
-                                lineEnding = LineEndingState.Inconsistent;
+                            // This code needs to be kep consistent with TextUtilities.LengthOfLineBreak()
+                            case '\r': newlineState.Increment(NewlineState.LineEnding.CR, 1); break;
+                            case '\n': newlineState.Increment(NewlineState.LineEnding.LF, 1); break;
+                            case '\u0085': newlineState.Increment(NewlineState.LineEnding.NEL, 1); break;
+                            case '\u2028': newlineState.Increment(NewlineState.LineEnding.LS, 1); break;
+                            case '\u2029': newlineState.Increment(NewlineState.LineEnding.PS, 1); break;
+                            default: throw new InvalidOperationException("Unexpected line ending");
                         }
                     }
 
-                    index += breakLength;
+                    if (nextCharIsStartOfLine)
+                    {
+                        leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Empty, 1);
+                    }
+
+                    nextCharIsStartOfLine = true;
                 }
+
+                index += breakLength;
             }
 
             lineBreaks.ReleasePooledLineBreaks();
@@ -169,18 +210,6 @@ namespace Microsoft.VisualStudio.Text.Implementation
             return lineBreaks;
         }
 
-        internal enum LineEndingState
-        {
-            Unknown = 0,
-            CRLF = 1,
-            CR = 2,
-            LF = 3,
-            NEL = 4,            // unicode Next Line 0085
-            LS = 5,             // unicode Line Separator 2028
-            PS = 6,             // unicode Paragraph Separator 2029
-            Inconsistent = 7,
-        }
-
         private static char[] pooledBuffer;
 
         private static char[] AcquireBuffer(int size)
diff --git a/src/Editor/Text/Impl/TextModel/TextDocument.cs b/src/Editor/Text/Impl/TextModel/TextDocument.cs
index 62d0f5a..0ef25fd 100644
--- a/src/Editor/Text/Impl/TextModel/TextDocument.cs
+++ b/src/Editor/Text/Impl/TextModel/TextDocument.cs
@@ -13,12 +13,10 @@ namespace Microsoft.VisualStudio.Text.Implementation
     using System.Text;
     using Microsoft.VisualStudio.Text.Utilities;
     using Microsoft.VisualStudio.Utilities;
-    using Microsoft.VisualStudio.Text.Editor;
 
     internal sealed partial class TextDocument : ITextDocument
     {
         #region Private Members
-
         private readonly TextDocumentFactoryService _textDocumentFactoryService;
         private ITextBuffer _textBuffer;
         private Encoding _encoding;
@@ -145,24 +143,14 @@ namespace Microsoft.VisualStudio.Text.Implementation
                 TextBuffer concreteBuffer = _textBuffer as TextBuffer;
                 if (concreteBuffer != null)
                 {
-                    bool hasConsistentLineEndings;
                     int longestLineLength;
-                    StringRebuilder newContent = TextImageLoader.Load(streamReader, fileSize, out hasConsistentLineEndings, out longestLineLength);
+                    StringRebuilder newContent = TextImageLoader.Load(streamReader, fileSize, out var newlineState, out var leadingWhitespaceState, out longestLineLength);
 
-                    if (!hasConsistentLineEndings)
-                    {
-                        // leave a sign that line endings are inconsistent. This is rather nasty but for now
-                        // we don't want to pollute the API with this factoid.
-                        concreteBuffer.Properties["InconsistentLineEndings"] = true;
-                    }
-                    else
-                    {
-                        // this covers a really obscure case where on initial load the file had inconsistent line
-                        // endings, but the UI settings were such that it was ignored, and since then the file has
-                        // acquired consistent line endings and the UI settings have also changed.
-                        concreteBuffer.Properties.RemoveProperty("InconsistentLineEndings");
-                    }
-                    // leave a similar sign about the longest line in the buffer.
+                    // Make the call to GetWhitespaceManager to add the manager to the properties. We don't need the return value here.
+                    _textDocumentFactoryService.WhitespaceManagerFactory.GetOrCreateWhitespaceManager(concreteBuffer, newlineState, leadingWhitespaceState);
+
+                    // Leave a sign about the longest line in the buffer. This is rather nasty, but for now
+                    // we don't want to pollute the API with this factoid
                     concreteBuffer.Properties["LongestLineLength"] = longestLineLength;
 
                     concreteBuffer.ReloadContent(newContent, options, editTag: this);
@@ -210,8 +198,8 @@ namespace Microsoft.VisualStudio.Text.Implementation
                 using (var stream = TextDocumentFactoryService.OpenFileGuts(_filePath, out _lastModifiedTimeUtc, out fileSize))
                 {
                     var detectors = ExtensionSelector.SelectMatchingExtensions(_textDocumentFactoryService.OrderedEncodingDetectors, _textBuffer.ContentType);
-                    
-                    if(_explicitEncoding)
+
+                    if (_explicitEncoding)
                     {
                         // If the user explicitly chose their encoding, we want to respect it.
                         newEncoding = this.Encoding;
@@ -226,7 +214,6 @@ namespace Microsoft.VisualStudio.Text.Implementation
                         try
                         {
                             var detectorEncoding = new ExtendedCharacterDetector();
-
                             ReloadBufferFromStream(stream, fileSize, options, detectorEncoding);
 
                             if (detectorEncoding.DecodedExtendedCharacters)
@@ -262,18 +249,18 @@ namespace Microsoft.VisualStudio.Text.Implementation
                     }
 
                     //If there is no "Next" version of the original snapshot, we have not successfully reloaded the document
-                    if(beforeSnapshot.Version.Next == null)
+                    if (beforeSnapshot.Version.Next == null)
                     {
                         //We use this fall back detector to observe whether or not character substitutions 
                         //occur while we're reading the stream
                         var fallbackDetector = new FallbackDetector(newEncoding.DecoderFallback);
-                        var modifiedEncoding = (Encoding)newEncoding.Clone();
+                        var modifiedEncoding = (Encoding) newEncoding.Clone();
                         modifiedEncoding.DecoderFallback = fallbackDetector;
 
                         Debug.Assert(stream.Position == 0);
                         ReloadBufferFromStream(stream, fileSize, options, modifiedEncoding);
 
-                        if(fallbackDetector.FallbackOccurred)
+                        if (fallbackDetector.FallbackOccurred)
                         {
                             characterSubstitutionsOccurred = fallbackDetector.FallbackOccurred;
                         }
@@ -374,11 +361,11 @@ namespace Microsoft.VisualStudio.Text.Implementation
 
         public void SaveAs(string filePath, bool overwrite, bool createFolder, IContentType newContentType)
         {
-             if (newContentType == null)
+            if (newContentType == null)
             {
                 throw new ArgumentNullException(nameof(newContentType));
             }
-             SaveAs(filePath, overwrite, createFolder);
+            SaveAs(filePath, overwrite, createFolder);
             // content type won't be changed if the save fails (in which case SaveAs will throw an exception)
             _textBuffer.ChangeContentType(newContentType, null);
         }
@@ -459,7 +446,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
                 Encoding oldEncoding = _encoding;
 
                 _encoding = value;
-                
+
                 if (!_encoding.Equals(oldEncoding))
                 {
                     _textDocumentFactoryService.GuardedOperations.RaiseEvent(this, EncodingChanged, new EncodingChangedEventArgs(oldEncoding, _encoding));
@@ -567,12 +554,12 @@ namespace Microsoft.VisualStudio.Text.Implementation
                     }
                 }
 
-               _textDocumentFactoryService.GuardedOperations.RaiseEvent(this, FileActionOccurred, new TextDocumentFileActionEventArgs(filePath, actionTime, actionType));
+                _textDocumentFactoryService.GuardedOperations.RaiseEvent(this, FileActionOccurred, new TextDocumentFileActionEventArgs(filePath, actionTime, actionType));
 
             }
             finally
             {
-               _raisingFileActionChangedEvent = false;
+                _raisingFileActionChangedEvent = false;
             }
         }
 
diff --git a/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs b/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs
index bd78514..5c617cb 100644
--- a/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs
+++ b/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs
@@ -18,6 +18,7 @@ namespace Microsoft.VisualStudio.Text.Implementation
     using System.Diagnostics;
     using Microsoft.VisualStudio.Text.Editor;
     using System.Runtime.InteropServices;
+    using Microsoft.VisualStudio.Text.Document;
 
     [Export(typeof(ITextDocumentFactoryService))]
     internal sealed partial class TextDocumentFactoryService : ITextDocumentFactoryService
@@ -33,6 +34,9 @@ namespace Microsoft.VisualStudio.Text.Implementation
         [Import]
         internal GuardedOperations GuardedOperations { get; set; }
 
+        [Import]
+        internal IWhitespaceManagerFactory WhitespaceManagerFactory { get; set; }
+
         #endregion
 
         internal static Encoding DefaultEncoding = Encoding.Default; // Exposed for unit tests.