SerialPort: Clear fAbortOnError when port opens (#17662)

* Added failing test for fAbortOnError not cleared

Issue #17396

* Ensure fAbortOnError is clear when opening port

3 files changed, 75 insertions, 0 deletions

diff --git a/src/System.IO.Ports/src/System/IO/Ports/SerialStream.cs b/src/System.IO.Ports/src/System/IO/Ports/SerialStream.cs
index cf0015923b..a5029bfc78 100644
--- a/src/System.IO.Ports/src/System/IO/Ports/SerialStream.cs
+++ b/src/System.IO.Ports/src/System/IO/Ports/SerialStream.cs
@@ -1307,6 +1307,8 @@ namespace System.IO.Ports
             // Later the user may change this via the NullDiscard property.
             SetDcbFlag(NativeMethods.FNULL, discardNull ? 1 : 0);
 
+            // SerialStream does not handle the fAbortOnError behaviour, so we must make sure it's not enabled
+            SetDcbFlag(NativeMethods.FABORTONOERROR, 0);
 
             // Setting RTS control, which is RTS_CONTROL_HANDSHAKE if RTS / RTS-XOnXOff handshaking
             // used, RTS_ENABLE (RTS pin used during operation) if rtsEnable true but XOnXoff / No handshaking
diff --git a/src/System.IO.Ports/tests/SerialPort/AbortOnError.cs b/src/System.IO.Ports/tests/SerialPort/AbortOnError.cs
new file mode 100644
index 0000000000..b76a57efc5
--- /dev/null
+++ b/src/System.IO.Ports/tests/SerialPort/AbortOnError.cs
@@ -0,0 +1,72 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System.Diagnostics;
+using System.IO.PortsTests;
+using System.Reflection;
+using Legacy.Support;
+using Xunit;
+
+namespace System.IO.Ports.Tests
+{
+    public class AbortOnError : PortsTest
+    {
+        /// <summary>
+        /// This is a test for GH issue 17396, Connect issue 584116
+        /// 
+        /// The situation is as follows:
+        /// * Some user of the port causes DCB.fAbortOnError to be set at some point
+        /// * The port is later opened by SerialPort - which doesn't clear fAbortOnError
+        ///   when it's initialising the port
+        /// * Subsequent errors on the port are mishandled, because fAbortOnError causes
+        ///   the error behaviour to be different, and SerialPort isn't prepared for this
+        /// 
+        /// To test this, we need to do the following
+        /// 1. Open the port, then use a private SerialStream method to set fAbortOnError to TRUE and then close the port
+        /// 2. Reopen the port
+        /// 3. Verify that the fAbortOnError flag is clear
+        /// 
+        /// </summary>
+        [ConditionalFact(nameof(HasOneSerialPort))]
+        public void AbortOnErrorShouldBeClearedOnOpen()
+        {
+            // Open the port, set the fAbortOnError flag and then close the port
+            SetAbortOnError(TCSupport.LocalMachineSerialInfo.FirstAvailablePortName);
+
+            using (SerialPort com = new SerialPort(TCSupport.LocalMachineSerialInfo.FirstAvailablePortName))
+            {
+                com.Open();
+
+                Assert.False(ReadAbortOnErrorFlag(com), "fAbortOnError should be clear when port is opened");
+            }
+        }
+
+        private static bool ReadAbortOnErrorFlag(SerialPort com)
+        {
+            Stream baseStream = com.BaseStream;
+            int flagValue = (int)baseStream.GetType()
+                .GetMethod("GetDcbFlag", BindingFlags.NonPublic | BindingFlags.Instance)
+                .Invoke(baseStream, new object[] {14});
+            return flagValue != 0;
+        }
+
+        private void SetAbortOnError(string port)
+        {
+            using (SerialPort com = new SerialPort(port))
+            {
+                com.Open();
+                Stream baseStream = com.BaseStream;
+
+                // Invoke the private method SetDcbFlag to set bit 14 in the DCB flags
+                // This just updates the _dcb.Flags member of SerialStream
+                baseStream.GetType().GetMethod("SetDcbFlag", BindingFlags.NonPublic | BindingFlags.Instance)
+                    .Invoke(baseStream, new object[] { 14, 1 });
+
+                // Force the DCB to get written to the driver
+                com.DataBits = 7;
+                com.DataBits = 8;
+            }
+        }
+    }
+}
diff --git a/src/System.IO.Ports/tests/System.IO.Ports.Tests.csproj b/src/System.IO.Ports/tests/System.IO.Ports.Tests.csproj
index 90af1b654d..6b46808881 100644
--- a/src/System.IO.Ports/tests/System.IO.Ports.Tests.csproj
+++ b/src/System.IO.Ports/tests/System.IO.Ports.Tests.csproj
@@ -42,6 +42,7 @@
     <Compile Include="SerialPort\Handshake.cs" />
     <Compile Include="SerialPort\IsOpen.cs" />
     <Compile Include="SerialPort\NewLine.cs" />
+    <Compile Include="SerialPort\AbortOnError.cs" />
     <Compile Include="SerialPort\Open.cs" />
     <Compile Include="SerialPort\OpenDevices.cs" />
     <Compile Include="SerialPort\Open_Stress.cs" />