Fix encoding of Digest authentication headers (#36627) (#399)

* Fix encoding of Digest authentication headers for servers that don't understand RFC 5987 encoding

* Quote-prefix special characters when encoding Digest headers

* Address PR feedback

4 files changed, 77 insertions, 52 deletions

diff --git a/src/System.Net.Http/src/System/Net/Http/Headers/ContentDispositionHeaderValue.cs b/src/System.Net.Http/src/System/Net/Http/Headers/ContentDispositionHeaderValue.cs
index f6e99d800f..065b934db2 100644
--- a/src/System.Net.Http/src/System/Net/Http/Headers/ContentDispositionHeaderValue.cs
+++ b/src/System.Net.Http/src/System/Net/Http/Headers/ContentDispositionHeaderValue.cs
@@ -5,7 +5,6 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
-using System.IO;
 using System.Text;
 
 namespace System.Net.Http.Headers
@@ -425,7 +424,7 @@ namespace System.Net.Http.Headers
         }
 
         // Returns input for decoding failures, as the content might not be encoded.
-        private string EncodeAndQuoteMime(string input)
+        private static string EncodeAndQuoteMime(string input)
         {
             string result = input;
             bool needsQuotes = false;
@@ -441,7 +440,7 @@ namespace System.Net.Http.Headers
                 throw new ArgumentException(string.Format(CultureInfo.InvariantCulture,
                     SR.net_http_headers_invalid_value, input));
             }
-            else if (RequiresEncoding(result))
+            else if (HeaderUtilities.ContainsNonAscii(result))
             {
                 needsQuotes = true; // Encoded data must always be quoted, the equals signs are invalid in tokens.
                 result = EncodeMime(result); // =?utf-8?B?asdfasdfaesdf?=
@@ -460,7 +459,7 @@ namespace System.Net.Http.Headers
         }
 
         // Returns true if the value starts and ends with a quote.
-        private bool IsQuoted(string value)
+        private static bool IsQuoted(string value)
         {
             Debug.Assert(value != null);
 
@@ -468,23 +467,8 @@ namespace System.Net.Http.Headers
                 && value.EndsWith("\"", StringComparison.Ordinal);
         }
 
-        // tspecials are required to be in a quoted string.  Only non-ascii needs to be encoded.
-        private bool RequiresEncoding(string input)
-        {
-            Debug.Assert(input != null);
-
-            foreach (char c in input)
-            {
-                if ((int)c > 0x7f)
-                {
-                    return true;
-                }
-            }
-            return false;
-        }
-
         // Encode using MIME encoding.
-        private string EncodeMime(string input)
+        private static string EncodeMime(string input)
         {
             byte[] buffer = Encoding.UTF8.GetBytes(input);
             string encodedName = Convert.ToBase64String(buffer);
@@ -492,7 +476,7 @@ namespace System.Net.Http.Headers
         }
 
         // Attempt to decode MIME encoded strings.
-        private bool TryDecodeMime(string input, out string output)
+        private static bool TryDecodeMime(string input, out string output)
         {
             Debug.Assert(input != null);
 
@@ -535,7 +519,7 @@ namespace System.Net.Http.Headers
 
         // Attempt to decode using RFC 5987 encoding.
         // encoding'language'my%20string
-        private bool TryDecode5987(string input, out string output)
+        private static bool TryDecode5987(string input, out string output)
         {
             output = null;
             
diff --git a/src/System.Net.Http/src/System/Net/Http/Headers/HeaderUtilities.cs b/src/System.Net.Http/src/System/Net/Http/Headers/HeaderUtilities.cs
index 75a8ef8e09..bfb3815d5f 100644
--- a/src/System.Net.Http/src/System/Net/Http/Headers/HeaderUtilities.cs
+++ b/src/System.Net.Http/src/System/Net/Http/Headers/HeaderUtilities.cs
@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Buffers;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.Contracts;
@@ -66,58 +67,63 @@ namespace System.Net.Http.Headers
             }
         }
 
-        // Encode a string using RFC 5987 encoding.
-        // encoding'lang'PercentEncodedSpecials
-        internal static string Encode5987(string input)
+        internal static bool ContainsNonAscii(string input)
         {
-            string output;
-            IsInputEncoded5987(input, out output);
+            Debug.Assert(input != null);
 
-            return output;
+            foreach (char c in input)
+            {
+                if ((int)c > 0x7f)
+                {
+                    return true;
+                }
+            }
+            return false;
         }
 
-        internal static bool IsInputEncoded5987(string input, out string output)
+        // Encode a string using RFC 5987 encoding.
+        // encoding'lang'PercentEncodedSpecials
+        internal static string Encode5987(string input)
         {
             // Encode a string using RFC 5987 encoding.
             // encoding'lang'PercentEncodedSpecials
-            bool wasEncoded = false;
             StringBuilder builder = StringBuilderCache.Acquire();
+            byte[] utf8bytes = ArrayPool<byte>.Shared.Rent(Encoding.UTF8.GetMaxByteCount(input.Length));
+            int utf8length = Encoding.UTF8.GetBytes(input, 0, input.Length, utf8bytes, 0);
+
             builder.Append("utf-8\'\'");
-            foreach (char c in input)
+            for (int i = 0; i < utf8length; i++)
             {
+                byte utf8byte = utf8bytes[i];
+
                 // attr-char = ALPHA / DIGIT / "!" / "#" / "$" / "&" / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~"
                 //      ; token except ( "*" / "'" / "%" )
-                if (c > 0x7F) // Encodes as multiple utf-8 bytes
+                if (utf8byte > 0x7F) // Encodes as multiple utf-8 bytes
                 {
-                    byte[] bytes = Encoding.UTF8.GetBytes(c.ToString());
-                    foreach (byte b in bytes)
-                    {
-                        AddHexEscaped((char)b, builder);
-                        wasEncoded = true;
-                    }
+                    AddHexEscaped(utf8byte, builder);
                 }
-                else if (!HttpRuleParser.IsTokenChar(c) || c == '*' || c == '\'' || c == '%')
+                else if (!HttpRuleParser.IsTokenChar((char)utf8byte) || utf8byte == '*' || utf8byte == '\'' || utf8byte == '%')
                 {
                     // ASCII - Only one encoded byte.
-                    AddHexEscaped(c, builder);
-                    wasEncoded = true;
+                    AddHexEscaped(utf8byte, builder);
                 }
                 else
                 {
-                    builder.Append(c);
+                    builder.Append((char)utf8byte);
                 }
 
             }
 
-            output = StringBuilderCache.GetStringAndRelease(builder);
-            return wasEncoded;
+            Array.Clear(utf8bytes, 0, utf8length);
+            ArrayPool<byte>.Shared.Return(utf8bytes);
+
+            return StringBuilderCache.GetStringAndRelease(builder);
         }
 
         /// <summary>Transforms an ASCII character into its hexadecimal representation, adding the characters to a StringBuilder.</summary>
-        private static void AddHexEscaped(char c, StringBuilder destination)
+        private static void AddHexEscaped(byte c, StringBuilder destination)
         {
             Debug.Assert(destination != null);
-            Debug.Assert(c <= 0xFF);
 
             destination.Append('%');
             destination.Append(s_hexUpperChars[(c & 0xf0) >> 4]);
diff --git a/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.Digest.cs b/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.Digest.cs
index b156134285..b65b240e80 100644
--- a/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.Digest.cs
+++ b/src/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/AuthenticationHelper.Digest.cs
@@ -91,9 +91,9 @@ namespace System.Net.Http
             }
             else
             {
-                string usernameStar;
-                if (HeaderUtilities.IsInputEncoded5987(credential.UserName, out usernameStar))
+                if (HeaderUtilities.ContainsNonAscii(credential.UserName))
                 {
+                    string usernameStar = HeaderUtilities.Encode5987(credential.UserName);
                     sb.AppendKeyValue(UsernameStar, usernameStar, includeQuotes: false);
                 }
                 else
@@ -408,6 +408,9 @@ namespace System.Net.Http
 
     internal static class StringBuilderExtensions
     {
+        // Characters that require escaping in quoted string
+        private static readonly char[] SpecialCharacters = new[] { '"', '\\' };
+
         public static void AppendKeyValue(this StringBuilder sb, string key, string value, bool includeQuotes = true, bool includeComma = true)
         {
             sb.Append(key);
@@ -415,12 +418,29 @@ namespace System.Net.Http
             if (includeQuotes)
             {
                 sb.Append('"');
+                int lastSpecialIndex = 0;
+                int specialIndex;
+                while (true)
+                {
+                    specialIndex = value.IndexOfAny(SpecialCharacters, lastSpecialIndex);
+                    if (specialIndex >= 0)
+                    {
+                        sb.Append(value, lastSpecialIndex, specialIndex - lastSpecialIndex);
+                        sb.Append('\\');
+                        sb.Append(value[specialIndex]);
+                        lastSpecialIndex = specialIndex + 1;
+                    }
+                    else
+                    {
+                        sb.Append(value, lastSpecialIndex, value.Length - lastSpecialIndex);
+                        break;
+                    }
+                }
+                sb.Append('"');
             }
-
-            sb.Append(value);
-            if (includeQuotes)
+            else
             {
-                sb.Append('"');
+                sb.Append(value);
             }
 
             if (includeComma)
diff --git a/src/System.Net.Http/tests/UnitTests/DigestAuthenticationTests.cs b/src/System.Net.Http/tests/UnitTests/DigestAuthenticationTests.cs
index f339ba78e4..45d03712a7 100644
--- a/src/System.Net.Http/tests/UnitTests/DigestAuthenticationTests.cs
+++ b/src/System.Net.Http/tests/UnitTests/DigestAuthenticationTests.cs
@@ -56,5 +56,20 @@ namespace System.Net.Http.Tests
 
             Assert.Equal(expectedResult, parameter != null);
         }
+
+        [Theory]
+        [InlineData("test", "username=\"test\"")]
+        [InlineData("test@example.org", "username=\"test@example.org\"")]
+        [InlineData("test\"example.org", "username=\"test\\\"example.org\"")]
+        [InlineData("t\u00E6st", "username*=utf-8''t%C3%A6st")]
+        [InlineData("\uD834\uDD1E", "username*=utf-8''%F0%9D%84%9E")]
+        public async void DigestResponse_UserName_Encoding(string username, string encodedUserName)
+        {
+            NetworkCredential credential = new NetworkCredential(username, "bar");
+            AuthenticationHelper.DigestResponse digestResponse = new AuthenticationHelper.DigestResponse("realm=\"NetCore\", nonce=\"qMRqWgAAAAAQMjIABgAAAFwEiEwAAAAA\"");
+            HttpRequestMessage request = new HttpRequestMessage(HttpMethod.Get, "http://microsoft.com/");
+            string parameter = await AuthenticationHelper.GetDigestTokenForCredential(credential, request, digestResponse).ConfigureAwait(false);
+            Assert.StartsWith(encodedUserName, parameter);
+        }
     }
 }