// Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // // This library is free software; you can redistribute it and/or // modify it under the terms of the GNU Lesser General Public // License as published by the Free Software Foundation; either // version 2.1 of the License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using System.Text; using System.Text.RegularExpressions; using Duplicati.Library.Common.IO; using Duplicati.Library.Common; using System.Globalization; using System.Threading; namespace Duplicati.Library.Utility { public static class Utility { /// /// Size of buffers for copying stream /// public static long DEFAULT_BUFFER_SIZE => SystemContextSettings.Buffersize; /// /// A cache of the FileSystemCaseSensitive property, which is computed upon the first access. /// private static bool? CachedIsFSCaseSensitive; /// /// Gets the hash algorithm used for calculating a hash /// public static string HashAlgorithm => "SHA256"; /// /// The EPOCH offset (unix style) /// public static readonly DateTime EPOCH = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); /// /// The attribute value used to indicate error /// public const FileAttributes ATTRIBUTE_ERROR = (FileAttributes)(1 << 30); /// /// The callback delegate type used to collecting file information /// /// The path that the file enumeration started at /// The current element /// The attributes of the element /// A value indicating if the folder should be recursed, ignored for other types public delegate bool EnumerationFilterDelegate(string rootpath, string path, FileAttributes attributes); /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to public static long CopyStream(Stream source, Stream target) { return CopyStream(source, target, true); } /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to /// True if an attempt should be made to rewind the source stream, false otherwise /// Temporary buffer to use (optional) public static long CopyStream(Stream source, Stream target, bool tryRewindSource, byte[] buf = null) { if (tryRewindSource && source.CanSeek) try { source.Position = 0; } catch { // ignored } buf = buf ?? new byte[DEFAULT_BUFFER_SIZE]; int read; long total = 0; while ((read = source.Read(buf, 0, buf.Length)) != 0) { target.Write(buf, 0, read); total += read; } return total; } /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to /// Token to cancel the operation. public static async Task CopyStreamAsync(Stream source, Stream target, CancellationToken cancelToken) { return await CopyStreamAsync(source, target, tryRewindSource: true, cancelToken: cancelToken).ConfigureAwait(false); } /// /// Copies the content of one stream into another /// /// The stream to read from /// The stream to write to /// True if an attempt should be made to rewind the source stream, false otherwise /// Token to cancel the operation. /// Temporary buffer to use (optional) public static async Task CopyStreamAsync(Stream source, Stream target, bool tryRewindSource, CancellationToken cancelToken, byte[] buf = null) { if (tryRewindSource && source.CanSeek) try { source.Position = 0; } catch {} buf = buf ?? new byte[DEFAULT_BUFFER_SIZE]; int read; long total = 0; while (true) { read = await source.ReadAsync(buf, 0, buf.Length, cancelToken).ConfigureAwait(false); if (read == 0) break; await target.WriteAsync(buf, 0, read, cancelToken).ConfigureAwait(false); total += read; } return total; } /// /// These are characters that must be escaped when using a globbing expression /// private static readonly string BADCHARS = @"\\|\+|\||\{|\[|\(|\)|\]|\}|\^|\$|\#|\."; /// /// Most people will probably want to use fileglobbing, but RegExp's are more flexible. /// By converting from the weak globbing to the stronger regexp, we support both. /// /// /// public static string ConvertGlobbingToRegExp(string globexp) { //First escape all special characters globexp = Regex.Replace(globexp, BADCHARS, @"\$&"); //Replace the globbing expressions with the corresponding regular expressions globexp = globexp.Replace('?', '.').Replace("*", ".*"); return globexp; } /// /// Convert literal path to the equivalent regular expression. /// public static string ConvertLiteralToRegExp(string literalPath) { // Escape all special characters return Regex.Escape(literalPath); } /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// A list of the full filenames public static IEnumerable EnumerateFiles(string basepath) { return EnumerateFileSystemEntries(basepath).Where(x => !x.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal)); } /// /// Returns a list of folder names found in the given folder. /// The search is recursive. /// /// The folder to look in /// A list of the full paths public static IEnumerable EnumerateFolders(string basepath) { return EnumerateFileSystemEntries(basepath).Where(x => x.EndsWith(Util.DirectorySeparatorString, StringComparison.Ordinal)); } /// /// Returns a list of all files and subfolders found in the given folder. /// The search is recursive. /// /// The folder to look in. /// A list of the full filenames and foldernames. Foldernames ends with the directoryseparator char public static IEnumerable EnumerateFileSystemEntries(string basepath) { return EnumerateFileSystemEntries(basepath, (rootpath, path, attributes) => true, SystemIO.IO_OS.GetDirectories, Directory.GetFiles, null); } /// /// A callback delegate used for applying alternate enumeration of filesystems /// /// The path to return data from /// A list of paths public delegate string[] FileSystemInteraction(string path); /// /// A callback delegate used for extracting attributes from a file or folder /// /// The path to return data from /// Attributes for the file or folder public delegate FileAttributes ExtractFileAttributes(string path); /// /// A callback delegate used for extracting attributes from a file or folder /// /// The root folder where the path was found /// The path that produced the error /// The exception for the error public delegate void ReportAccessError(string rootpath, string path, Exception ex); /// /// Returns a list of all files found in the given folder. /// The search is recursive. /// /// The folder to look in /// The function to call with the filenames /// A function to call that lists all folders in the supplied folder /// A function to call that lists all files in the supplied folder /// A function to call that obtains the attributes for an element, set to null to avoid reading attributes /// An optional function to call with error messages. /// A list of the full filenames public static IEnumerable EnumerateFileSystemEntries(string rootpath, EnumerationFilterDelegate callback, FileSystemInteraction folderList, FileSystemInteraction fileList, ExtractFileAttributes attributeReader, ReportAccessError errorCallback = null) { var lst = new Stack(); if (IsFolder(rootpath, attributeReader)) { rootpath = Util.AppendDirSeparator(rootpath); try { var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Directory; if (callback(rootpath, rootpath, attr)) lst.Push(rootpath); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, rootpath, ex); callback(rootpath, rootpath, FileAttributes.Directory | ATTRIBUTE_ERROR); } while (lst.Count > 0) { var f = lst.Pop(); yield return f; try { foreach (var s in folderList(f)) { var sf = Util.AppendDirSeparator(s); try { var attr = attributeReader?.Invoke(sf) ?? FileAttributes.Directory; if (callback(rootpath, sf, attr)) lst.Push(sf); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, sf, ex); callback(rootpath, sf, FileAttributes.Directory | ATTRIBUTE_ERROR); } } } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } string[] files = null; if (fileList != null) { try { files = fileList(f); } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, f, ex); callback(rootpath, f, FileAttributes.Directory | ATTRIBUTE_ERROR); } } if (files != null) { foreach (var s in files) { try { var attr = attributeReader?.Invoke(s) ?? FileAttributes.Normal; if (!callback(rootpath, s, attr)) continue; } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, s, ex); callback(rootpath, s, ATTRIBUTE_ERROR); continue; } yield return s; } } } } else { try { var attr = attributeReader?.Invoke(rootpath) ?? FileAttributes.Normal; if (!callback(rootpath, rootpath, attr)) yield break; } catch (System.Threading.ThreadAbortException) { throw; } catch (Exception ex) { errorCallback?.Invoke(rootpath, rootpath, ex); callback(rootpath, rootpath, ATTRIBUTE_ERROR); yield break; } yield return rootpath; } } /// /// Test if specified path is a folder /// /// Path to test /// Function to use for testing path /// True if path is refers to a folder public static bool IsFolder(string path, ExtractFileAttributes attributeReader) { if (attributeReader == null) return true; try { return attributeReader(path).HasFlag(FileAttributes.Directory); } catch { return false; } } /// /// Tests if path refers to a file, or folder, below the parent folder /// /// File or folder to test /// Candidate parent folder /// True if below parent folder, false otherwise /// (note that this returns false if the two argument paths are identical!) public static bool IsPathBelowFolder(string fileOrFolderPath, string parentFolder) { var sanitizedParentFolder = Util.AppendDirSeparator(parentFolder); return fileOrFolderPath.StartsWith(sanitizedParentFolder, ClientFilenameStringComparison) && !fileOrFolderPath.Equals(sanitizedParentFolder, ClientFilenameStringComparison); } /// /// Returns parent folder of path /// /// Full file or folder path /// If true, return value always has trailing separator /// Parent folder of path (containing folder for file paths, parent folder for folder paths) public static string GetParent(string path, bool forceTrailingDirectorySeparator) { var len = path.Length - 1; if (len > 1 && path[len] == Path.DirectorySeparatorChar) { len--; } var last = path.LastIndexOf(Path.DirectorySeparatorChar, len); if (last == -1 || last == 0 && len == 0) return null; if (last == 0 && !Platform.IsClientWindows) return Util.DirectorySeparatorString; var parent = path.Substring(0, last); if (forceTrailingDirectorySeparator || Platform.IsClientWindows && parent.Length == 2 && parent[1] == ':' && char.IsLetter(parent[0])) { parent += Path.DirectorySeparatorChar; } return parent; } /// /// Given a collection of unique folders, returns only parent-most folders /// /// Collection of unique folders /// Parent-most folders of input collection public static IEnumerable SimplifyFolderList(ICollection folders) { if (!folders.Any()) return folders; var result = new LinkedList(); result.AddFirst(folders.First()); foreach (var folder1 in folders) { bool addFolder = true; LinkedListNode next; for (var node = result.First; node != null; node = next) { next = node.Next; var folder2 = node.Value; if (IsPathBelowFolder(folder1, folder2)) { // higher-level folder already present addFolder = false; break; } if (IsPathBelowFolder(folder2, folder1)) { // retain folder1 result.Remove(node); } } if (addFolder) { result.AddFirst(folder1); } } return result.Distinct(); } /// /// Given a collection of file paths, return those NOT contained within specified collection of folders /// /// Collection of files to filter /// Collection of folders to use as filter /// Files not in any of specified folders public static IEnumerable GetFilesNotInFolders(IEnumerable files, IEnumerable folders) { return files.Where(x => folders.All(folder => !IsPathBelowFolder(x, folder))); } /// /// Calculates the size of files in a given folder /// /// The folder to examine /// The combined size of all files that match the filter public static long GetDirectorySize(string folder) { return EnumerateFolders(folder).Sum((path) => new FileInfo(path).Length); } /// /// Some streams can return a number that is less than the requested number of bytes. /// This is usually due to fragmentation, and is solved by issuing a new read. /// This function wraps that functionality. /// /// The stream to read /// The buffer to read into /// The amount of bytes to read /// The actual number of bytes read public static int ForceStreamRead(Stream stream, byte[] buf, int count) { int a; int index = 0; do { a = stream.Read(buf, index, count); index += a; count -= a; } while (a != 0 && count > 0); return index; } /// /// Some streams can return a number that is less than the requested number of bytes. /// This is usually due to fragmentation, and is solved by issuing a new read. /// This function wraps that functionality. /// /// The stream to read. /// The buffer to read into. /// The amount of bytes to read. /// The number of bytes read public static async Task ForceStreamReadAsync(this System.IO.Stream stream, byte[] buf, int count) { int a; int index = 0; do { a = await stream.ReadAsync(buf, index, count).ConfigureAwait(false); index += a; count -= a; } while (a != 0 && count > 0); return index; } /// /// Compares two streams to see if they are binary equals /// /// One stream /// Another stream /// True if the length of the two streams should be compared /// True if they are equal, false otherwise public static bool CompareStreams(Stream stream1, Stream stream2, bool checkLength) { if (checkLength) { try { if (stream1.Length != stream2.Length) return false; } catch { //We must read along, trying to determine if they are equals } } int longSize = BitConverter.GetBytes((long)0).Length; byte[] buf1 = new byte[longSize * 512]; byte[] buf2 = new byte[buf1.Length]; int a1, a2; while ((a1 = ForceStreamRead(stream1, buf1, buf1.Length)) == (a2 = ForceStreamRead(stream2, buf2, buf2.Length))) { int ix = 0; for (int i = 0; i < a1 / longSize; i++) if (BitConverter.ToUInt64(buf1, ix) != BitConverter.ToUInt64(buf2, ix)) return false; else ix += longSize; for (int i = 0; i < a1 % longSize; i++) if (buf1[ix] != buf2[ix]) return false; else ix++; if (a1 == 0) break; } return a1 == a2; } /// /// Calculates the hash of a given stream, and returns the results as an base64 encoded string /// /// The stream to calculate the hash for /// The base64 encoded hash public static string CalculateHash(Stream stream) { return Convert.ToBase64String(HashAlgorithmHelper.Create(HashAlgorithm).ComputeHash(stream)); } /// /// Reads a file, attempts to detect encoding /// /// The path to the file to read /// The file contents public static string ReadFileWithDefaultEncoding(string filename) { // Since StreamReader defaults to UTF8 and most text files will NOT be UTF8 without BOM, // we need to detect the encoding (at least that it's not UTF8). // So we read the first 4096 bytes and try to decode them as UTF8. var buffer = new byte[4096]; using (var file = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) { Utility.ForceStreamRead(file, buffer, 4096); } var enc = Encoding.UTF8; try { // this will throw an error if not really UTF8 // ReSharper disable once ReturnValueOfPureMethodIsNotUsed new UTF8Encoding(false, true).GetString(buffer); } catch (Exception) { enc = Encoding.Default; } // This will load the text using the BOM, or the detected encoding if no BOM. using (var reader = new StreamReader(filename, enc, true)) { // Remove all \r from the file and split on \n, then pass directly to ExtractOptions return reader.ReadToEnd(); } } /// /// Formats a size into a human readable format, eg. 2048 becomes "2 KB" or -2283 becomes "-2.23 KB%quot. /// /// The size to format /// A human readable string representing the size public static string FormatSizeString(double size) { double sizeAbs = Math.Abs(size); // Allow formatting of negative sizes if (sizeAbs >= 1024 * 1024 * 1024 * 1024L) return Strings.Utility.FormatStringTB(size / (1024 * 1024 * 1024 * 1024L)); else if (sizeAbs >= 1024 * 1024 * 1024) return Strings.Utility.FormatStringGB(size / (1024 * 1024 * 1024)); else if (sizeAbs >= 1024 * 1024) return Strings.Utility.FormatStringMB(size / (1024 * 1024)); else if (sizeAbs >= 1024) return Strings.Utility.FormatStringKB(size / 1024); else return Strings.Utility.FormatStringB((long) size); // safe to cast because lower than 1024 and thus well within range of long } public static System.Threading.ThreadPriority ParsePriority(string value) { if (string.IsNullOrEmpty(value) || value.Trim().Length == 0) return System.Threading.ThreadPriority.Normal; switch (value.ToLower(CultureInfo.InvariantCulture).Trim()) { case "+2": case "high": case "highest": return System.Threading.ThreadPriority.Highest; case "+1": case "abovenormal": case "above normal": return System.Threading.ThreadPriority.AboveNormal; case "-1": case "belownormal": case "below normal": return System.Threading.ThreadPriority.BelowNormal; case "-2": case "low": case "lowest": case "idle": return System.Threading.ThreadPriority.Lowest; default: return System.Threading.ThreadPriority.Normal; } } /// /// Parses a string into a boolean value. /// /// The value to parse. /// A delegate that returns the default value if is not a valid boolean value. /// The parsed value, or the value returned by . public static bool ParseBool(string value, Func defaultFunc) { if (String.IsNullOrWhiteSpace(value)) { return defaultFunc(); } switch (value.Trim().ToLower(CultureInfo.InvariantCulture)) { case "1": case "on": case "true": case "yes": return true; case "0": case "off": case "false": case "no": return false; default: return defaultFunc(); } } /// /// Parses a string into a boolean value. /// /// The value to parse. /// The default value, in case is not a valid boolean value. /// The parsed value, or the default value. public static bool ParseBool(string value, bool @default) { return ParseBool(value, () => @default); } /// /// Parses an option from the option set, using the convention that if the option is set, it is true unless it parses to false, and false otherwise /// /// The set of options to look for the setting in /// The value to look for in the settings /// public static bool ParseBoolOption(IDictionary options, string value) { string opt; if (options.TryGetValue(value, out opt)) return ParseBool(opt, true); else return false; } /// /// Parses an enum found in the options dictionary /// /// The parsed or default enum value. /// The set of options to look for the setting in /// The value to look for in the settings /// The default value to return if there are no matches. /// The enum type parameter. public static T ParseEnumOption(IDictionary options, string value, T @default) { return options.TryGetValue(value, out var opt) ? ParseEnum(opt, @default) : @default; } /// /// Attempts to parse an enum with case-insensitive lookup, returning the default value if there was no match /// /// The parsed or default enum value. /// The string to parse. /// The default value to return if there are no matches. /// The enum type parameter. public static T ParseEnum(string value, T @default) { foreach (var s in Enum.GetNames(typeof(T))) if (s.Equals(value, StringComparison.OrdinalIgnoreCase)) return (T)Enum.Parse(typeof(T), s); return @default; } /// /// Converts a sequence of bytes to a hex string /// /// The array as hex string. /// The data to convert public static string ByteArrayAsHexString(byte[] data) { return BitConverter.ToString(data).Replace("-", string.Empty); } /// /// Converts a hex string to a byte array /// /// The string as byte array. /// The hex string /// The parsed data public static void HexStringAsByteArray(string hex, byte[] data) { for (var i = 0; i < hex.Length; i += 2) data[i / 2] = Convert.ToByte(hex.Substring(i, 2), 16); } public static bool Which(string appname) { if (!Platform.IsClientPosix) return false; try { var psi = new System.Diagnostics.ProcessStartInfo("which", appname) { RedirectStandardOutput = true, RedirectStandardError = false, RedirectStandardInput = false, UseShellExecute = false }; var pi = System.Diagnostics.Process.Start(psi); pi.WaitForExit(5000); if (pi.HasExited) return pi.ExitCode == 0; else return false; } catch { } return false; } /// /// Returns a value indicating if the filesystem, is case sensitive /// public static bool IsFSCaseSensitive { get { if (!CachedIsFSCaseSensitive.HasValue) { var str = Environment.GetEnvironmentVariable("FILESYSTEM_CASE_SENSITIVE"); // TODO: This should probably be determined by filesystem rather than OS, // OSX can actually have the disks formatted as Case Sensitive, but insensitive is default CachedIsFSCaseSensitive = ParseBool(str, () => Platform.IsClientPosix && !Platform.IsClientOSX); } return CachedIsFSCaseSensitive.Value; } } /// /// Returns a value indicating if the app is running under Mono /// public static bool IsMono => Type.GetType("Mono.Runtime") != null; /// /// Gets the current Mono runtime version, will return 0.0 if not running Mono /// public static Version MonoVersion { get { try { var v = MonoDisplayVersion; if (v != null) { var regex = new Regex(@"\d+\.\d+(\.\d+)?(\.\d+)?"); var match = regex.Match(v); if (match.Success) return new Version(match.Value); } } catch { // ignored } return new Version(); } } /// /// Gets the Mono display version, or null if not running Mono /// public static string MonoDisplayVersion { get { try { var t = Type.GetType("Mono.Runtime"); if (t != null) { var mi = t.GetMethod("GetDisplayName", System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic); if (mi != null) return (string)mi.Invoke(null, null); } } catch { // ignored } return null; } } /// /// Gets a string comparer that matches the client filesystems case sensitivity /// public static StringComparer ClientFilenameStringComparer => IsFSCaseSensitive ? StringComparer.Ordinal : StringComparer.OrdinalIgnoreCase; /// /// Gets the string comparision that matches the client filesystems case sensitivity /// public static StringComparison ClientFilenameStringComparison => IsFSCaseSensitive ? StringComparison.Ordinal : StringComparison.OrdinalIgnoreCase; /// /// The path to the users home directory /// public static readonly string HOME_PATH = Environment.GetFolderPath(Platform.IsClientPosix ? Environment.SpecialFolder.Personal : Environment.SpecialFolder.UserProfile); /// /// Regexp for matching environment variables on Windows (%VAR%) /// private static readonly Regex ENVIRONMENT_VARIABLE_MATCHER_WINDOWS = new Regex(@"\%(?\w+)\%"); /// /// Expands environment variables in a RegExp safe format /// /// The expanded string. /// The string to expand. /// A lookup method that converts an environment key to an expanded string public static string ExpandEnvironmentVariablesRegexp(string str, Func lookup = null) { if (lookup == null) lookup = Environment.GetEnvironmentVariable; return // TODO: Should we switch to using the native format ($VAR or ${VAR}), instead of following the Windows scheme? // IsClientLinux ? new Regex(@"\$(?\w+)|(\{(?[^\}]+)\})") : ENVIRONMENT_VARIABLE_MATCHER_WINDOWS ENVIRONMENT_VARIABLE_MATCHER_WINDOWS.Replace(str, m => Regex.Escape(lookup(m.Groups["name"].Value))); } /// /// Normalizes a DateTime instance by converting to UTC and flooring to seconds. /// /// The normalized date time /// The input time public static DateTime NormalizeDateTime(DateTime input) { var ticks = input.ToUniversalTime().Ticks; ticks -= ticks % TimeSpan.TicksPerSecond; return new DateTime(ticks, DateTimeKind.Utc); } /// /// Given a DateTime instance, return the number of elapsed seconds since the Unix epoch /// /// The number of elapsed seconds since the Unix epoch /// The input time public static long NormalizeDateTimeToEpochSeconds(DateTime input) { // Note that we cannot return (new DateTimeOffset(input)).ToUnixTimeSeconds() here. // The DateTimeOffset constructor will convert the provided DateTime to the UTC // equivalent. However, if DateTime.MinValue is provided (for example, when creating // a new backup), this can result in values that fall outside the DateTimeOffset.MinValue // and DateTimeOffset.MaxValue bounds. return (long) Math.Floor((NormalizeDateTime(input) - EPOCH).TotalSeconds); } /// /// The format string for a DateTime /// //Note: Actually the K should be Z which is more correct as it is forced to be Z, but Z as a format specifier is fairly undocumented public static string SERIALIZED_DATE_TIME_FORMAT = "yyyyMMdd'T'HHmmssK"; /// /// Returns a string representation of a in UTC format /// /// The instance /// A string representing the time public static string SerializeDateTime(DateTime dt) { return dt.ToUniversalTime().ToString(SERIALIZED_DATE_TIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture); } /// /// Parses a serialized instance /// /// The string to parse /// The parsed instance public static bool TryDeserializeDateTime(string str, out DateTime dt) { return DateTime.TryParseExact(str, SERIALIZED_DATE_TIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal, out dt); } /// /// Parses a serialized instance /// /// The string to parse /// The parsed instance public static DateTime DeserializeDateTime(string str) { if (!TryDeserializeDateTime(str, out var dt)) throw new Exception(Strings.Utility.InvalidDateError(str)); return dt; } /// /// Gets the unique items from a collection. /// /// The type of the elements in . /// The collection to remove duplicate items from. /// The duplicate items in . /// The unique items from . public static ISet GetUniqueItems(IEnumerable collection, out ISet duplicateItems) { return GetUniqueItems(collection, EqualityComparer.Default, out duplicateItems); } /// /// Gets the unique items from a collection. /// /// The type of the elements in . /// The collection to remove duplicate items from. /// The implementation to use when comparing values in the collection. /// The duplicate items in . /// The unique items from . public static ISet GetUniqueItems(IEnumerable collection, IEqualityComparer comparer, out ISet duplicateItems) { var uniqueItems = new HashSet(comparer); duplicateItems = new HashSet(comparer); foreach (var item in collection) { if (!uniqueItems.Add(item)) duplicateItems.Add(item); } return uniqueItems; } // // Returns the entry assembly or reasonable approximation if no entry assembly is available. // This is the case in NUnit tests. The following approach does not work w/ Mono due to unimplemented members: // http://social.msdn.microsoft.com/Forums/nb-NO/clr/thread/db44fe1a-3bb4-41d4-a0e0-f3021f30e56f // so this layer of indirection is necessary // // entry assembly or reasonable approximation public static System.Reflection.Assembly getEntryAssembly() { return System.Reflection.Assembly.GetEntryAssembly() ?? System.Reflection.Assembly.GetExecutingAssembly(); } /// /// Converts a Base64 encoded string to "base64 for url" /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The base64 encoded string /// The base64 for url encoded string public static string Base64PlainToBase64Url(string data) { return data.Replace('+', '-').Replace('/', '_'); } /// /// Converts a "base64 for url" encoded string to a Base64 encoded string. /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The base64 for url encoded string /// The base64 encoded string public static string Base64UrlToBase64Plain(string data) { return data.Replace('-', '+').Replace('_', '/'); } /// /// Encodes a byte array into a "base64 for url" encoded string. /// See https://en.wikipedia.org/wiki/Base64#URL_applications /// /// The data to encode /// The base64 for url encoded string public static string Base64UrlEncode(byte[] data) { return Base64PlainToBase64Url(Convert.ToBase64String(data)); } /// /// Converts a DateTime instance to a Unix timestamp /// /// The Unix timestamp. /// The DateTime instance to convert. public static long ToUnixTimestamp(DateTime input) { var ticks = input.ToUniversalTime().Ticks; ticks -= ticks % TimeSpan.TicksPerSecond; input = new DateTime(ticks, DateTimeKind.Utc); return (long)Math.Floor((input - EPOCH).TotalSeconds); } /// /// Returns a value indicating if the given type should be treated as a primitive /// /// true, if type is primitive for serialization, false otherwise. /// The type to check. private static bool IsPrimitiveTypeForSerialization(Type t) { return t.IsPrimitive || t.IsEnum || t == typeof(string) || t == typeof(DateTime) || t == typeof(TimeSpan); } /// /// Writes a primitive to the output, or returns false if the input is not primitive /// /// true, the item was printed, false otherwise. /// The item to write. /// The target writer. private static bool PrintSerializeIfPrimitive(object item, TextWriter writer) { if (item == null) { writer.Write("null"); return true; } if (IsPrimitiveTypeForSerialization(item.GetType())) { if (item is DateTime time) { writer.Write(time.ToLocalTime()); writer.Write(" ("); writer.Write(ToUnixTimestamp(time)); writer.Write(")"); } else writer.Write(item); return true; } return false; } /// /// Prints the object to a stream, which can be used for display or logging /// /// The serialized object /// The object to serialize /// The writer to write the results to /// A filter applied to properties to decide if they are omitted or not /// A value indicating if non-primitive values are recursed /// The string indentation /// A lookup table with visited objects, used to avoid infinite recursion /// The maximum number of items to report from an IEnumerable instance public static void PrintSerializeObject(object item, TextWriter writer, Func filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 0, Dictionary visited = null) { visited = visited ?? new Dictionary(); var indentstring = new string(' ', indentation); var first = true; if (item == null || IsPrimitiveTypeForSerialization(item.GetType())) { writer.Write(indentstring); if (PrintSerializeIfPrimitive(item, writer)) return; } foreach (var p in item.GetType().GetProperties()) { if (filter != null && !filter(p, item)) continue; if (IsPrimitiveTypeForSerialization(p.PropertyType)) { if (first) first = false; else writer.WriteLine(); writer.Write("{0}{1}: ", indentstring, p.Name); PrintSerializeIfPrimitive(p.GetValue(item, null), writer); } else if (typeof(Task).IsAssignableFrom(p.PropertyType) || p.Name == "TaskReader") { // Ignore Task items continue; } else if (typeof(System.Collections.IEnumerable).IsAssignableFrom(p.PropertyType)) { var enumerable = (System.Collections.IEnumerable)p.GetValue(item, null); var any = false; if (enumerable != null) { var enumerator = enumerable.GetEnumerator(); if (enumerator != null) { var remain = collectionlimit; if (first) first = false; else writer.WriteLine(); writer.Write("{0}{1}: [", indentstring, p.Name); if (enumerator.MoveNext()) { any = true; writer.WriteLine(); PrintSerializeObject(enumerator.Current, writer, filter, recurseobjects, indentation + 4, collectionlimit, visited); remain--; while (enumerator.MoveNext()) { writer.WriteLine(","); if (remain == 0) { writer.Write("..."); break; } PrintSerializeObject(enumerator.Current, writer, filter, recurseobjects, indentation + 4, collectionlimit, visited); remain--; } } if (any) { writer.WriteLine(); writer.Write(indentstring); } writer.Write("]"); } } } else if (recurseobjects) { var value = p.GetValue(item, null); if (value == null) { if (first) first = false; else writer.WriteLine(); writer.Write("{0}{1}: null", indentstring, p.Name); } else if (!visited.ContainsKey(value)) { if (first) first = false; else writer.WriteLine(); writer.WriteLine("{0}{1}:", indentstring, p.Name); visited[value] = null; PrintSerializeObject(value, writer, filter, recurseobjects, indentation + 4, collectionlimit, visited); } } } writer.Flush(); } /// /// Returns a string representing the object, which can be used for display or logging /// /// The serialized object /// The object to serialize /// A filter applied to properties to decide if they are omitted or not /// A value indicating if non-primitive values are recursed /// The string indentation /// The maximum number of items to report from an IEnumerable instance, set to zero or less for reporting all public static StringBuilder PrintSerializeObject(object item, StringBuilder sb = null, Func filter = null, bool recurseobjects = false, int indentation = 0, int collectionlimit = 10) { sb = sb ?? new StringBuilder(); using (var sw = new StringWriter(sb)) PrintSerializeObject(item, sw, filter, recurseobjects, indentation, collectionlimit); return sb; } /// /// Repeatedly hash a value with a salt. /// This effectively masks the original value, /// and destroys lookup methods, like rainbow tables /// /// The data to hash /// The salt to apply /// The number of times to repeat the hashing /// The salted hash public static byte[] RepeatedHashWithSalt(string data, string salt, int repeats = 1200) { return RepeatedHashWithSalt( Encoding.UTF8.GetBytes(data ?? ""), Encoding.UTF8.GetBytes(salt ?? ""), repeats); } /// /// Repeatedly hash a value with a salt. /// This effectively masks the original value, /// and destroys lookup methods, like rainbow tables /// /// The data to hash /// The salt to apply /// The salted hash public static byte[] RepeatedHashWithSalt(byte[] data, byte[] salt, int repeats = 1200) { // We avoid storing the passphrase directly, // instead we salt and rehash repeatedly using (var h = System.Security.Cryptography.SHA256.Create()) { h.Initialize(); h.TransformBlock(salt, 0, salt.Length, salt, 0); h.TransformFinalBlock(data, 0, data.Length); var buf = h.Hash; for (var i = 0; i < repeats; i++) { h.Initialize(); h.TransformBlock(salt, 0, salt.Length, salt, 0); h.TransformFinalBlock(buf, 0, buf.Length); buf = h.Hash; } return buf; } } /// /// Gets the drive letter from the given volume guid. /// This method cannot be inlined since the System.Management types are not implemented in Mono /// /// Volume guid /// Drive letter, as a single character, or null if the volume wasn't found [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public static string GetDriveLetterFromVolumeGuid(Guid volumeGuid) { // Based on this answer: // https://stackoverflow.com/questions/10186277/how-to-get-drive-information-by-volume-id using (System.Management.ManagementObjectSearcher searcher = new System.Management.ManagementObjectSearcher("Select * from Win32_Volume")) { string targetId = string.Format(@"\\?\Volume{{{0}}}\", volumeGuid); foreach (System.Management.ManagementObject obj in searcher.Get()) { if (string.Equals(obj["DeviceID"].ToString(), targetId, StringComparison.OrdinalIgnoreCase)) { object driveLetter = obj["DriveLetter"]; if (driveLetter != null) { return obj["DriveLetter"].ToString(); } else { // The volume was found, but doesn't have a drive letter associated with it. break; } } } return null; } } /// /// Gets all volume guids and their associated drive letters. /// This method cannot be inlined since the System.Management types are not implemented in Mono /// /// Pairs of drive letter to volume guids [System.Runtime.CompilerServices.MethodImpl(System.Runtime.CompilerServices.MethodImplOptions.NoInlining)] public static IEnumerable> GetVolumeGuidsAndDriveLetters() { using (var searcher = new System.Management.ManagementObjectSearcher("Select * from Win32_Volume")) { foreach (var obj in searcher.Get()) { var deviceIdObj = obj["DeviceID"]; var driveLetterObj = obj["DriveLetter"]; if (deviceIdObj != null && driveLetterObj != null) { var deviceId = deviceIdObj.ToString(); var driveLetter = driveLetterObj.ToString(); if (!string.IsNullOrEmpty(deviceId) && !string.IsNullOrEmpty(driveLetter)) { yield return new KeyValuePair(driveLetter + @"\", deviceId); } } } } } /// /// The regular expression matching all know non-quoted commandline characters /// private static readonly Regex COMMANDLINE_SAFE = new Regex(@"[A-Za-z0-9\-_/:\.]*"); /// /// Special characters that needs to be escaped on Linux /// private static readonly Regex COMMANDLINE_ESCAPED_LINUX = new Regex(@"[""$`\\!]"); /// /// Wraps a single argument in quotes suitable for the passing on the commandline /// /// The wrapped commandline element. /// The argument to wrap. /// A flag indicating if environment variables are allowed to be expanded public static string WrapCommandLineElement(string arg, bool allowEnvExpansion) { if (string.IsNullOrWhiteSpace(arg)) return arg; if (!Platform.IsClientWindows) { // We could consider using single quotes that prevents all expansions //if (!allowEnvExpansion) // return "'" + arg.Replace("'", "\\'") + "'"; // Linux is using backslash to escape, except for ! arg = COMMANDLINE_ESCAPED_LINUX.Replace(arg, (match) => { if (match.Value == "!") return @"""'!'"""; if (match.Value == "$" && allowEnvExpansion) return match.Value; return @"\" + match.Value; }); } else { // Windows needs only needs " replaced with "", // but is prone to %var% expansion when used in // immediate mode (i.e. from command prompt) // Fortunately it does not expand when processes // are started from within .Net // TODO: I have not found a way to avoid escaping %varname%, // and sadly it expands only if the variable exists // making it even rarer and harder to diagnose when // it happens arg = arg.Replace(@"""", @""""""); // Also fix the case where the argument ends with a slash if (arg[arg.Length - 1] == '\\') arg += @"\"; } // Check that all characters are in the safe set if (COMMANDLINE_SAFE.Match(arg).Length != arg.Length) return @"""" + arg + @""""; else return arg; } /// /// Wrap a set of commandline arguments suitable for the commandline /// /// A commandline string. /// The arguments to create into a commandline. /// A flag indicating if environment variables are allowed to be expanded public static string WrapAsCommandLine(IEnumerable args, bool allowEnvExpansion = false) { return string.Join(" ", args.Select(x => WrapCommandLineElement(x, allowEnvExpansion))); } /// /// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async. /// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions. /// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.) /// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult /// /// Task to await public static void Await(this Task task) { task.GetAwaiter().GetResult(); } /// /// Utility method that emulates C#'s built in await keyword without requiring the calling method to be async. /// This method should be preferred over using Task.Result, as it doesn't wrap singular exceptions in AggregateExceptions. /// (It uses Task.GetAwaiter().GetResult(), which is the same thing that await uses under the covers.) /// https://stackoverflow.com/questions/17284517/is-task-result-the-same-as-getawaiter-getresult /// /// Result type /// Task to await /// Task result public static T Await(this Task task) { return task.GetAwaiter().GetResult(); } } }