diff options
author | Mike Krüger <mkrueger@xamarin.com> | 2012-03-14 16:58:24 +0400 |
---|---|---|
committer | Mike Krüger <mkrueger@xamarin.com> | 2012-03-14 16:58:24 +0400 |
commit | cc59ff1c5eeb0062ce3383a9e28351595f82b735 (patch) | |
tree | d64c7bfb607987e658a13b339cc899acd2f21436 /main/src | |
parent | 460d28caae43fe31989c752573c6327914cfbefe (diff) | |
parent | 8857b817e97eb5379d6423d98f321fd78630aee8 (diff) |
Merge branch 'master' into newresolver
Conflicts:
main/src/core/Mono.Texteditor/Mono.TextEditor.csproj
Diffstat (limited to 'main/src')
4 files changed, 363 insertions, 1 deletions
diff --git a/main/src/addins/VersionControl/MonoDevelop.VersionControl.Git/MonoDevelop.VersionControl.Git/GitRepository.cs b/main/src/addins/VersionControl/MonoDevelop.VersionControl.Git/MonoDevelop.VersionControl.Git/GitRepository.cs index 46624ac087..8ac86aa9fd 100644 --- a/main/src/addins/VersionControl/MonoDevelop.VersionControl.Git/MonoDevelop.VersionControl.Git/GitRepository.cs +++ b/main/src/addins/VersionControl/MonoDevelop.VersionControl.Git/MonoDevelop.VersionControl.Git/GitRepository.cs @@ -857,7 +857,7 @@ namespace MonoDevelop.VersionControl.Git string GetCommitTextContent (RevCommit c, FilePath file) { - return NGit.Util.RawParseUtils.Decode (GetCommitContent (c, file)); + return Mono.TextEditor.Utils.TextFileReader.GetText (GetCommitContent (c, file)); } string GenerateDiff (byte[] data1, byte[] data2) diff --git a/main/src/core/Mono.Texteditor/Makefile.am b/main/src/core/Mono.Texteditor/Makefile.am index c3c1ca437e..6c38f32953 100644 --- a/main/src/core/Mono.Texteditor/Makefile.am +++ b/main/src/core/Mono.Texteditor/Makefile.am @@ -56,6 +56,7 @@ FILES = \ Mono.TextEditor.Utils/Diff.cs \ Mono.TextEditor.Utils/RedBlackTree.cs \ Mono.TextEditor.Utils/TextBreaker.cs \ + Mono.TextEditor.Utils/TextFileReader.cs \ Mono.TextEditor.Vi/ViActionMaps.cs \ Mono.TextEditor.Vi/ViActions.cs \ Mono.TextEditor.Vi/ViBuilderContext.cs \ diff --git a/main/src/core/Mono.Texteditor/Mono.TextEditor.Utils/TextFileReader.cs b/main/src/core/Mono.Texteditor/Mono.TextEditor.Utils/TextFileReader.cs new file mode 100644 index 0000000000..df71c67824 --- /dev/null +++ b/main/src/core/Mono.Texteditor/Mono.TextEditor.Utils/TextFileReader.cs @@ -0,0 +1,360 @@ +// +// TextFileReader.cs +// +// Author: +// Mike Krüger <mkrueger@xamarin.com> +// +// Copyright (c) 2012 Xamarin Inc. (http://xamarin.com) +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Mono.TextEditor.Utils +{ + /// <summary> + /// This class handles text input from files, streams and byte arrays with auto-detect encoding. + /// </summary> + public static class TextFileReader + { + readonly static int maxBomLength = 0; + readonly static Encoding[] encodingsWithBom; + + static TextFileReader () + { + var encodings = new List<Encoding> (); + + foreach (var info in Encoding.GetEncodings ()) { + var encoding = info.GetEncoding (); + var bom = encoding.GetPreamble (); + if (bom == null || bom.Length == 0) + continue; + maxBomLength = System.Math.Max (maxBomLength, bom.Length); + encodings.Add (encoding); + } + encodingsWithBom = encodings.ToArray (); + } + + #region stream reader methods + public static StreamReader OpenStream (byte[] bytes) + { + bool hadBom; + return OpenStream (bytes, out hadBom); + } + + public static StreamReader OpenStream (byte[] bytes, out bool hadBom) + { + if (bytes == null) + throw new ArgumentNullException ("bytes"); + return OpenStream (new MemoryStream (bytes, false), out hadBom); + } + + public static StreamReader OpenStream (Stream stream) + { + bool hadBom; + return OpenStream (stream, out hadBom); + } + + public static StreamReader OpenStream (Stream stream, out bool hadBom) + { + if (stream == null) + throw new ArgumentNullException ("stream"); + byte[] possibleBom = new byte[maxBomLength]; + stream.Read (possibleBom, 0, System.Math.Min ((int)stream.Length, maxBomLength)); + + foreach (var encoding in encodingsWithBom) { + var bom = encoding.GetPreamble (); + bool invalid = false; + for (int i = 0; i < bom.Length; i++) { + if (bom[i] != possibleBom[i]) { + invalid = true; + break; + } + } + + if (!invalid) { + hadBom = true; + stream.Position = bom.Length; + return new StreamReader (stream, encoding); + } + } + stream.Position = 0; + hadBom = false; + return new StreamReader (stream, AutoDetectEncoding (stream)); + } + #endregion + + #region string methods + public static string GetText (byte[] bytes) + { + using (var stream = OpenStream (bytes)) { + return stream.ReadToEnd (); + } + } + + public static string GetText (byte[] bytes, out bool hadBom, out Encoding encoding) + { + if (bytes == null) + throw new ArgumentNullException ("bytes"); + using (var stream = OpenStream (bytes, out hadBom)) { + encoding = stream.CurrentEncoding; + return stream.ReadToEnd (); + } + } + + public static string GetText (Stream inputStream) + { + using (var stream = OpenStream (inputStream)) { + return stream.ReadToEnd (); + } + } + + public static string GetText (Stream inputStream, out bool hadBom, out Encoding encoding) + { + if (inputStream == null) + throw new ArgumentNullException ("inputStream"); + using (var stream = OpenStream (inputStream, out hadBom)) { + encoding = stream.CurrentEncoding; + return stream.ReadToEnd (); + } + } + + #endregion + + #region file methods + + public static string ReadAllText (string fileName) + { + bool hadBom; + Encoding encoding; + return ReadAllText (fileName, out hadBom, out encoding); + } + + public static string ReadAllText (string fileName, out bool hadBom, out Encoding encoding) + { + if (fileName == null) + throw new ArgumentNullException ("fileName"); + using (var stream = new FileStream (fileName, FileMode.Open, FileAccess.Read, FileShare.Read)) { + return GetText (stream, out hadBom, out encoding); + } + } + + #endregion + + #region Encoding autodetection + readonly static Encoding fallbackEncoding = Encoding.GetEncoding (1252); + + static Encoding AutoDetectEncoding (Stream stream) + { + try { + int max = (int)System.Math.Min (stream.Length, 50 * 1024); + + var utf8 = new Utf8Verifier (); + var utf16 = new UnicodeVerifier (); + var utf16BE = new BigEndianUnicodeVerifier (); + + for (int i = 0; i < max; i++) { + var b = (byte)stream.ReadByte (); + utf8.Advance (b); + utf16.Advance (b); + utf16BE.Advance (b); + } + if (utf8.IsValid) + return Encoding.UTF8; + if (utf16.IsValid) + return Encoding.Unicode; + if (utf16BE.IsValid) + return Encoding.BigEndianUnicode; + } catch (Exception) { + } + return fallbackEncoding; + } + + static bool IsBinary (byte[] bytes) + { + if (bytes == null) + throw new ArgumentNullException ("bytes"); + return IsBinary (new MemoryStream (bytes, false)); + } + + static bool IsBinary (string fileName) + { + if (fileName == null) + throw new ArgumentNullException ("fileName"); + using (var stream = new FileStream (fileName, FileMode.Open, FileAccess.Read, FileShare.Read)) { + return IsBinary (stream); + } + } + + static bool IsBinary (Stream stream) + { + if (stream == null) + throw new ArgumentNullException ("stream"); + + // Uses the same approach as GIT + const int FIRST_FEW_BYTES = 8 * 1024; + int length = System.Math.Min (FIRST_FEW_BYTES, (int)stream.Length); + for (int i = 0; i < length; i++) { + if (stream.ReadByte () == 0) + return true; + } + return false; + } + + #endregion + } + + class Utf8Verifier + { + const byte Error = 0; + + const byte UTF1 = 1; + const byte UTFTail1 = 2; + const byte UTFTail2 = 3; + const byte UTFTail3 = 4; + const byte UTF8_3_TailPre1 = 5; + const byte UTF8_3_TailPre2 = 6; + const byte UTF8_4_TailPre1 = 7; + const byte UTF8_4_TailPre2 = 8; + + const byte LAST = 8; + + byte state = UTF1; + + public bool IsValid { + get { + return state != Error; + } + } + + static byte[][] table; + + static Utf8Verifier () + { + table = new byte[LAST][]; + for (int i = 0; i < LAST; i++) + table[i] = new byte[(int)byte.MaxValue + 1]; + + // UTF8-1 = %x00-7F + for (int i = 0x00; i <= 0x7F; i++) { + table[UTF1][i] = UTF1; + } + + // UTF8-tail = %x80-BF + for (int i = 0x80; i <= 0xBF; i++) { + table[UTFTail1][i] = UTF1; + table[UTFTail2][i] = UTFTail1; + table[UTFTail3][i] = UTFTail2; + } + + // UTF8-2 = %xC2-DF UTF8-tail + for (int i = 0xC2; i <= 0xDF;i++) + table[UTF1][i] = UTFTail1; + + // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + for (int i = 0xA0; i <= 0xBF; i++) { + table[UTF8_3_TailPre1][i] = UTFTail1; + } + for (int i = 0x80; i <= 0x9F; i++) { + table[UTF8_3_TailPre2][i] = UTFTail1; + } + + table[UTF1][0xE0] = UTF8_3_TailPre1; + for (int i = 0xE1; i <= 0xEC;i++) + table[UTF1][i] = UTFTail2; + table[UTF1][0xED] = UTF8_3_TailPre2; + for (int i = 0xEE; i <= 0xEE;i++) + table[UTF1][i] = UTFTail2; + + // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + // %xF4 %x80-8F 2( UTF8-tail ) + + for (int i = 0x90; i <= 0xBF; i++) { + table[UTF8_4_TailPre1][i] = UTFTail2; + } + for (int i = 0x80; i <= 0xBF; i++) { + table[UTF8_4_TailPre2][i] = UTFTail2; + } + table[UTF1][0xF0] = UTF8_4_TailPre1; + for (int i = 0xF1; i <= 0xF3;i++) + table[UTF1][i] = UTFTail3; + table[UTF1][0xF4] = UTF8_4_TailPre2; + + // always invalid. + for (int i = 0; i < table.Length; i++) { + table[i][0xC0] = Error; + table[i][0xC1] = Error; + table[i][0xF5] = Error; + table[i][0xFF] = Error; + } + } + + public void Advance (byte b) + { + state = table[state][b]; + } + } + + class UnicodeVerifier + { + const byte Error = 0; + const byte Running = 1; + + byte state = Running; + + public bool IsValid { + get { + return state != Error; + } + } + int number = 0; + public void Advance (byte b) + { + if (number % 2 == 1 && b != 0) + state = Error; + number++; + } + } + + class BigEndianUnicodeVerifier + { + const byte Error = 0; + const byte Running = 1; + + byte state = Running; + + public bool IsValid { + get { + return state != Error; + } + } + int number = 0; + public void Advance (byte b) + { + if (number % 2 == 0 && b != 0) + state = Error; + number++; + } + } + +} + diff --git a/main/src/core/Mono.Texteditor/Mono.TextEditor.csproj b/main/src/core/Mono.Texteditor/Mono.TextEditor.csproj index 22551450fa..0e3adb43f8 100644 --- a/main/src/core/Mono.Texteditor/Mono.TextEditor.csproj +++ b/main/src/core/Mono.Texteditor/Mono.TextEditor.csproj @@ -180,6 +180,7 @@ <Compile Include="Mono.TextEditor\DocumentChangeEventArgs.cs" /> <Compile Include="Mono.TextEditor\TextDocument.cs" /> <Compile Include="Mono.TextEditor\TextSegment.cs" /> + <Compile Include="Mono.TextEditor.Utils\TextFileReader.cs" /> </ItemGroup> <ItemGroup> <None Include="Mono.TextEditor.dll.config"> |