diff options
Diffstat (limited to 'src/Editor/Text/Impl')
4 files changed, 135 insertions, 101 deletions
diff --git a/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs b/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs index 5ae8b2f..66cee18 100644 --- a/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs +++ b/src/Editor/Text/Impl/TextModel/BufferFactoryService.cs @@ -16,6 +16,7 @@ namespace Microsoft.VisualStudio.Text.Implementation using System.IO.MemoryMappedFiles; using System.Text; using Microsoft.VisualStudio.Text.Differencing; + using Microsoft.VisualStudio.Text.Document; using Microsoft.VisualStudio.Text.Projection; using Microsoft.VisualStudio.Text.Projection.Implementation; using Microsoft.VisualStudio.Text.Utilities; @@ -70,13 +71,16 @@ namespace Microsoft.VisualStudio.Text.Implementation [Import] internal IDifferenceService _differenceService { get; set; } - + [Import] internal ITextDifferencingSelectorService _textDifferencingSelectorService { get; set; } [Import] internal GuardedOperations _guardedOperations { get; set; } + [Import] + internal IWhitespaceManagerFactory _whitespaceManagerFactory { get; set; } + #endregion #region Private state @@ -202,6 +206,11 @@ namespace Microsoft.VisualStudio.Text.Implementation public ITextBuffer CreateTextBuffer(TextReader reader, IContentType contentType, long length, string traceId) { + return this.CreateTextBuffer(reader, contentType, length, traceId, throwOnInvalidCharacters: false); + } + + public ITextBuffer CreateTextBuffer(TextReader reader, IContentType contentType, long length, string traceId, bool throwOnInvalidCharacters) + { if (reader == null) { throw new ArgumentNullException(nameof(reader)); @@ -215,25 +224,30 @@ namespace Microsoft.VisualStudio.Text.Implementation throw new InvalidOperationException(Strings.FileTooLarge); } - bool hasConsistentLineEndings; int longestLineLength; - StringRebuilder content = TextImageLoader.Load(reader, length, out hasConsistentLineEndings, out longestLineLength); + StringRebuilder content = TextImageLoader.Load( + reader, + length, + out var newlineState, + out var leadingWhitespaceState, + out longestLineLength, + throwOnInvalidCharacters: throwOnInvalidCharacters); ITextBuffer buffer = Make(contentType, content, false); - if (!hasConsistentLineEndings) - { - // leave a sign that line endings are inconsistent. This is rather nasty but for now - // we don't want to pollute the API with this factoid - buffer.Properties.AddProperty("InconsistentLineEndings", true); - } - // leave a similar sign about the longest line in the buffer. + + // Make the call to GetWhitespaceManager to add the manager to the properties. We don't need the return value here. + var _ = _whitespaceManagerFactory.GetOrCreateWhitespaceManager(buffer, newlineState, leadingWhitespaceState); + + // Leave a sign about the longest line in the buffer. This is rather nasty, but for now + // we don't want to pollute the API with this factoid + buffer.Properties["LongestLineLength"] = longestLineLength; return buffer; } public ITextBuffer CreateTextBuffer(TextReader reader, IContentType contentType) { - return CreateTextBuffer(reader, contentType, -1, "legacy"); + return CreateTextBuffer(reader, contentType, -1, "legacy", throwOnInvalidCharacters: false); } internal static StringRebuilder StringRebuilderFromSnapshotAndSpan(ITextSnapshot snapshot, Span span) @@ -260,16 +274,19 @@ namespace Microsoft.VisualStudio.Text.Implementation internal static StringRebuilder AppendStringRebuildersFromSnapshotAndSpan(StringRebuilder content, ITextSnapshot snapshot, Span span) { - var baseSnapshot = snapshot as BaseSnapshot; - if (baseSnapshot != null) + if (span.Length != 0) { - content = content.Append(baseSnapshot.Content.GetSubText(span)); - } - else - { - // The we don't know what to do fallback. This should never be called unless someone provides a new snapshot - // implementation. - content = content.Append(snapshot.GetText(span)); + var baseSnapshot = snapshot as BaseSnapshot; + if (baseSnapshot != null) + { + content = content.Append(baseSnapshot.Content.GetSubText(span)); + } + else + { + // The we don't know what to do fallback. This should never be called unless someone provides a new snapshot + // implementation. + content = content.Append(snapshot.GetText(span)); + } } return content; @@ -283,10 +300,7 @@ namespace Microsoft.VisualStudio.Text.Implementation public ITextImage CreateTextImage(TextReader reader, long length) { - bool hasConsistentLineEndings; - int longestLineLength; - - return CachingTextImage.Create(TextImageLoader.Load(reader, length, out hasConsistentLineEndings, out longestLineLength), null); + return CachingTextImage.Create(TextImageLoader.Load(reader, length, out var _, out var _, out var _), null); } public ITextImage CreateTextImage(MemoryMappedFile source) @@ -310,7 +324,7 @@ namespace Microsoft.VisualStudio.Text.Implementation return buffer; } - public IProjectionBuffer CreateProjectionBuffer(IProjectionEditResolver projectionEditResolver, + public IProjectionBuffer CreateProjectionBuffer(IProjectionEditResolver projectionEditResolver, IList<object> trackingSpans, ProjectionBufferOptions options, IContentType contentType) @@ -324,7 +338,7 @@ namespace Microsoft.VisualStudio.Text.Implementation { throw new ArgumentNullException(nameof(contentType)); } - IProjectionBuffer buffer = + IProjectionBuffer buffer = new ProjectionBuffer(this, projectionEditResolver, contentType, trackingSpans, _differenceService, _textDifferencingSelectorService.DefaultTextDifferencingService, options, _guardedOperations); RaiseProjectionBufferCreatedEvent(buffer); return buffer; diff --git a/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs b/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs index da7646d..5678506 100644 --- a/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs +++ b/src/Editor/Text/Impl/TextModel/Storage/TextImageLoader.cs @@ -17,13 +17,19 @@ namespace Microsoft.VisualStudio.Text.Implementation public const int BlockSize = 16384; internal static StringRebuilder Load(TextReader reader, long fileSize, - out bool hasConsistentLineEndings, out int longestLineLength, + out NewlineState newlineState, + out LeadingWhitespaceState leadingWhitespaceState, + out int longestLineLength, int blockSize = 0, - int minCompressedBlockSize = TextImageLoader.BlockSize) // Exposed for unit tests + int minCompressedBlockSize = TextImageLoader.BlockSize, // Exposed for unit tests + bool throwOnInvalidCharacters = false) { - LineEndingState lineEnding = LineEndingState.Unknown; + newlineState = new NewlineState(); + leadingWhitespaceState = new LeadingWhitespaceState(); + int currentLineLength = 0; longestLineLength = 0; + char thresholdForInvalidCharacters = throwOnInvalidCharacters ? '\u0001' : '\0'; // Basically the only invalid character is \0, if we are looking for invalid characters. bool useCompressedStringRebuilders = (fileSize >= TextModelOptions.CompressedStorageFileSizeThreshold); if (blockSize == 0) @@ -44,6 +50,7 @@ namespace Microsoft.VisualStudio.Text.Implementation StringRebuilder content = StringRebuilderForChars.Empty; try { + bool nextCharIsStartOfLine = true; while (true) { int read = TextImageLoader.LoadNextBlock(reader, buffer); @@ -51,7 +58,15 @@ namespace Microsoft.VisualStudio.Text.Implementation if (read == 0) break; - var lineBreaks = TextImageLoader.ParseBlock(buffer, read, ref lineEnding, ref currentLineLength, ref longestLineLength); + var lineBreaks = TextImageLoader.ParseBlock( + buffer, + read, + thresholdForInvalidCharacters, + ref newlineState, + ref leadingWhitespaceState, + ref currentLineLength, + ref longestLineLength, + ref nextCharIsStartOfLine); char[] bufferForStringBuilder = buffer; if (read < (buffer.Length / 2)) @@ -83,8 +98,6 @@ namespace Microsoft.VisualStudio.Text.Implementation } } - hasConsistentLineEndings = lineEnding != LineEndingState.Inconsistent; - return content; } @@ -109,8 +122,16 @@ namespace Microsoft.VisualStudio.Text.Implementation return read; } - private static ILineBreaks ParseBlock(char[] buffer, int length, - ref LineEndingState lineEnding, ref int currentLineLength, ref int longestLineLength) + // Evil performance hack (but we are on a hot path here): + // thresholdForInvalidCharacters should be '\u0001' if we are throwing on invalid characters. + // should be '\0' if we are not. + // (otherwise we need to check both a throwOnInvalidCharacters boolean and that c == 0). + private static ILineBreaks ParseBlock(char[] buffer, int length, char thresholdForInvalidCharacters, + ref NewlineState newlineState, + ref LeadingWhitespaceState leadingWhitespaceState, + ref int currentLineLength, + ref int longestLineLength, + ref bool nextCharIsStartOfLine) { // Note that the lineBreaks created here will (internally) use the pooled list of line breaks. IPooledLineBreaksEditor lineBreaks = LineBreakManager.CreatePooledLineBreakEditor(length); @@ -121,8 +142,32 @@ namespace Microsoft.VisualStudio.Text.Implementation int breakLength = TextUtilities.LengthOfLineBreak(buffer, index, length); if (breakLength == 0) { + char c = buffer[index]; + + // If we are checking for invalid characters, throw if we encounter a \0 + if (c < thresholdForInvalidCharacters) + throw new FileFormatException("File contains NUL characters"); + ++currentLineLength; ++index; + + if (nextCharIsStartOfLine) + { + switch (c) + { + case ' ': + leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Space, 1); + break; + case '\t': + leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Tab, 1); + break; + default: + leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Printable, 1); + break; + } + + nextCharIsStartOfLine = false; + } } else { @@ -130,38 +175,34 @@ namespace Microsoft.VisualStudio.Text.Implementation longestLineLength = Math.Max(longestLineLength, currentLineLength); currentLineLength = 0; - if (lineEnding != LineEndingState.Inconsistent) + + if (breakLength == 2) { - if (breakLength == 2) - { - if (lineEnding == LineEndingState.Unknown) - lineEnding = LineEndingState.CRLF; - else if (lineEnding != LineEndingState.CRLF) - lineEnding = LineEndingState.Inconsistent; - } - else + newlineState.Increment(NewlineState.LineEnding.CRLF, 1); + } + else + { + switch (buffer[index]) { - LineEndingState newLineEndingState; - switch (buffer[index]) - { - // This code needs to be kep consistent with TextUtilities.LengthOfLineBreak() - case '\r': newLineEndingState = LineEndingState.CR; break; - case '\n': newLineEndingState = LineEndingState.LF; break; - case '\u0085': newLineEndingState = LineEndingState.NEL; break; - case '\u2028': newLineEndingState = LineEndingState.LS; break; - case '\u2029': newLineEndingState = LineEndingState.PS; break; - default: throw new InvalidOperationException("Unexpected line ending"); - } - - if (lineEnding == LineEndingState.Unknown) - lineEnding = newLineEndingState; - else if (lineEnding != newLineEndingState) - lineEnding = LineEndingState.Inconsistent; + // This code needs to be kep consistent with TextUtilities.LengthOfLineBreak() + case '\r': newlineState.Increment(NewlineState.LineEnding.CR, 1); break; + case '\n': newlineState.Increment(NewlineState.LineEnding.LF, 1); break; + case '\u0085': newlineState.Increment(NewlineState.LineEnding.NEL, 1); break; + case '\u2028': newlineState.Increment(NewlineState.LineEnding.LS, 1); break; + case '\u2029': newlineState.Increment(NewlineState.LineEnding.PS, 1); break; + default: throw new InvalidOperationException("Unexpected line ending"); } } - index += breakLength; + if (nextCharIsStartOfLine) + { + leadingWhitespaceState.Increment(LeadingWhitespaceState.LineLeadingCharacter.Empty, 1); + } + + nextCharIsStartOfLine = true; } + + index += breakLength; } lineBreaks.ReleasePooledLineBreaks(); @@ -169,18 +210,6 @@ namespace Microsoft.VisualStudio.Text.Implementation return lineBreaks; } - internal enum LineEndingState - { - Unknown = 0, - CRLF = 1, - CR = 2, - LF = 3, - NEL = 4, // unicode Next Line 0085 - LS = 5, // unicode Line Separator 2028 - PS = 6, // unicode Paragraph Separator 2029 - Inconsistent = 7, - } - private static char[] pooledBuffer; private static char[] AcquireBuffer(int size) diff --git a/src/Editor/Text/Impl/TextModel/TextDocument.cs b/src/Editor/Text/Impl/TextModel/TextDocument.cs index 62d0f5a..0ef25fd 100644 --- a/src/Editor/Text/Impl/TextModel/TextDocument.cs +++ b/src/Editor/Text/Impl/TextModel/TextDocument.cs @@ -13,12 +13,10 @@ namespace Microsoft.VisualStudio.Text.Implementation using System.Text; using Microsoft.VisualStudio.Text.Utilities; using Microsoft.VisualStudio.Utilities; - using Microsoft.VisualStudio.Text.Editor; internal sealed partial class TextDocument : ITextDocument { #region Private Members - private readonly TextDocumentFactoryService _textDocumentFactoryService; private ITextBuffer _textBuffer; private Encoding _encoding; @@ -145,24 +143,14 @@ namespace Microsoft.VisualStudio.Text.Implementation TextBuffer concreteBuffer = _textBuffer as TextBuffer; if (concreteBuffer != null) { - bool hasConsistentLineEndings; int longestLineLength; - StringRebuilder newContent = TextImageLoader.Load(streamReader, fileSize, out hasConsistentLineEndings, out longestLineLength); + StringRebuilder newContent = TextImageLoader.Load(streamReader, fileSize, out var newlineState, out var leadingWhitespaceState, out longestLineLength); - if (!hasConsistentLineEndings) - { - // leave a sign that line endings are inconsistent. This is rather nasty but for now - // we don't want to pollute the API with this factoid. - concreteBuffer.Properties["InconsistentLineEndings"] = true; - } - else - { - // this covers a really obscure case where on initial load the file had inconsistent line - // endings, but the UI settings were such that it was ignored, and since then the file has - // acquired consistent line endings and the UI settings have also changed. - concreteBuffer.Properties.RemoveProperty("InconsistentLineEndings"); - } - // leave a similar sign about the longest line in the buffer. + // Make the call to GetWhitespaceManager to add the manager to the properties. We don't need the return value here. + _textDocumentFactoryService.WhitespaceManagerFactory.GetOrCreateWhitespaceManager(concreteBuffer, newlineState, leadingWhitespaceState); + + // Leave a sign about the longest line in the buffer. This is rather nasty, but for now + // we don't want to pollute the API with this factoid concreteBuffer.Properties["LongestLineLength"] = longestLineLength; concreteBuffer.ReloadContent(newContent, options, editTag: this); @@ -210,8 +198,8 @@ namespace Microsoft.VisualStudio.Text.Implementation using (var stream = TextDocumentFactoryService.OpenFileGuts(_filePath, out _lastModifiedTimeUtc, out fileSize)) { var detectors = ExtensionSelector.SelectMatchingExtensions(_textDocumentFactoryService.OrderedEncodingDetectors, _textBuffer.ContentType); - - if(_explicitEncoding) + + if (_explicitEncoding) { // If the user explicitly chose their encoding, we want to respect it. newEncoding = this.Encoding; @@ -226,7 +214,6 @@ namespace Microsoft.VisualStudio.Text.Implementation try { var detectorEncoding = new ExtendedCharacterDetector(); - ReloadBufferFromStream(stream, fileSize, options, detectorEncoding); if (detectorEncoding.DecodedExtendedCharacters) @@ -262,18 +249,18 @@ namespace Microsoft.VisualStudio.Text.Implementation } //If there is no "Next" version of the original snapshot, we have not successfully reloaded the document - if(beforeSnapshot.Version.Next == null) + if (beforeSnapshot.Version.Next == null) { //We use this fall back detector to observe whether or not character substitutions //occur while we're reading the stream var fallbackDetector = new FallbackDetector(newEncoding.DecoderFallback); - var modifiedEncoding = (Encoding)newEncoding.Clone(); + var modifiedEncoding = (Encoding) newEncoding.Clone(); modifiedEncoding.DecoderFallback = fallbackDetector; Debug.Assert(stream.Position == 0); ReloadBufferFromStream(stream, fileSize, options, modifiedEncoding); - if(fallbackDetector.FallbackOccurred) + if (fallbackDetector.FallbackOccurred) { characterSubstitutionsOccurred = fallbackDetector.FallbackOccurred; } @@ -374,11 +361,11 @@ namespace Microsoft.VisualStudio.Text.Implementation public void SaveAs(string filePath, bool overwrite, bool createFolder, IContentType newContentType) { - if (newContentType == null) + if (newContentType == null) { throw new ArgumentNullException(nameof(newContentType)); } - SaveAs(filePath, overwrite, createFolder); + SaveAs(filePath, overwrite, createFolder); // content type won't be changed if the save fails (in which case SaveAs will throw an exception) _textBuffer.ChangeContentType(newContentType, null); } @@ -459,7 +446,7 @@ namespace Microsoft.VisualStudio.Text.Implementation Encoding oldEncoding = _encoding; _encoding = value; - + if (!_encoding.Equals(oldEncoding)) { _textDocumentFactoryService.GuardedOperations.RaiseEvent(this, EncodingChanged, new EncodingChangedEventArgs(oldEncoding, _encoding)); @@ -567,12 +554,12 @@ namespace Microsoft.VisualStudio.Text.Implementation } } - _textDocumentFactoryService.GuardedOperations.RaiseEvent(this, FileActionOccurred, new TextDocumentFileActionEventArgs(filePath, actionTime, actionType)); + _textDocumentFactoryService.GuardedOperations.RaiseEvent(this, FileActionOccurred, new TextDocumentFileActionEventArgs(filePath, actionTime, actionType)); } finally { - _raisingFileActionChangedEvent = false; + _raisingFileActionChangedEvent = false; } } diff --git a/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs b/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs index bd78514..5c617cb 100644 --- a/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs +++ b/src/Editor/Text/Impl/TextModel/TextDocumentFactoryService.cs @@ -18,6 +18,7 @@ namespace Microsoft.VisualStudio.Text.Implementation using System.Diagnostics; using Microsoft.VisualStudio.Text.Editor; using System.Runtime.InteropServices; + using Microsoft.VisualStudio.Text.Document; [Export(typeof(ITextDocumentFactoryService))] internal sealed partial class TextDocumentFactoryService : ITextDocumentFactoryService @@ -33,6 +34,9 @@ namespace Microsoft.VisualStudio.Text.Implementation [Import] internal GuardedOperations GuardedOperations { get; set; } + [Import] + internal IWhitespaceManagerFactory WhitespaceManagerFactory { get; set; } + #endregion internal static Encoding DefaultEncoding = Encoding.Default; // Exposed for unit tests. |