Fix SuperPMI unit test error checking (#76411)

* Fix SuperPMI unit test error checking

We were ignoring test failures, and even ignoring if the test
we wanted to run under collection actually existed. This means
that at some point during recent test reconfigurations, we
stopped building 10w5d_cs_do.cmd/sh but we didn't notice. Because
Bytemark kept being built as before, we got at least some .mc
files.

Add more checking that the programs we want to run exist, and
that the tests we invoke pass.

* Adjust superpmi unit test collection tests

Add CscBench to get better C# coverage.

Disable 10w5d_cs_do.csproj due to https://github.com/dotnet/runtime/issues/76421

2 files changed, 18 insertions, 15 deletions

diff --git a/src/tests/JIT/superpmi/superpmicollect.cs b/src/tests/JIT/superpmi/superpmicollect.cs
index 9e92df88ede..2377a3eedb7 100644
--- a/src/tests/JIT/superpmi/superpmicollect.cs
+++ b/src/tests/JIT/superpmi/superpmicollect.cs
@@ -161,6 +161,11 @@ namespace SuperPMICollection
 
         private static int RunProgram(string program, string arguments)
         {
+            if (!File.Exists(program))
+            {
+                throw new SpmiException("program " + program + " not found");
+            }
+
             // If the program is a script, move the program name into the arguments, and run it
             // under the appropriate shell.
             if (Global.IsWindows)
@@ -238,7 +243,11 @@ namespace SuperPMICollection
 
             try
             {
-                RunProgram(testName, "");
+                int retval = RunProgram(testName, "");
+                if (retval != 0)
+                {
+                    throw new SpmiException("Test " + testName + " failed");
+                }
             }
             finally
             {
@@ -269,22 +278,9 @@ namespace SuperPMICollection
         // Run all the programs from the CoreCLR test binary drop we wish to run while collecting MC files.
         private static void RunTestProgramsWhileCollecting()
         {
-
-            // Run the tests
             foreach (string spmiTestPath in GetSpmiTestFullPaths())
             {
-                try
-                {
-                    RunTest(spmiTestPath);
-                }
-                catch (SpmiException ex)
-                {
-                    // Ignore failures running the test. We don't really care if they pass or not
-                    // as long as they generate some .MC files. Plus, I'm not sure how confident
-                    // we can be in getting a correct error code.
-
-                    Console.Error.WriteLine("WARNING: test failed (ignoring): " + ex.Message);
-                }
+                RunTest(spmiTestPath);
             }
         }
 
diff --git a/src/tests/JIT/superpmi/superpmicollect.csproj b/src/tests/JIT/superpmi/superpmicollect.csproj
index 86ddca45522..ed252f5785a 100644
--- a/src/tests/JIT/superpmi/superpmicollect.csproj
+++ b/src/tests/JIT/superpmi/superpmicollect.csproj
@@ -24,7 +24,14 @@
 
   <ItemGroup>
     <_SpmiTestProjects Include="..\Performance\CodeQuality\Bytemark\Bytemark.csproj" />
+    <_SpmiTestProjects Include="..\Performance\CodeQuality\Roslyn\CscBench.csproj" />
+    <!-- The reference to 10w5d_cs_do.csproj is commented out due to https://github.com/dotnet/runtime/issues/76421: 
+        merged tests (those built with BuildAsStandalone=false) don't get .cmd/.sh files, which are needed to
+        run the test by the superpmicollect driver.
+    -->
+    <!--
     <_SpmiTestProjects Include="..\Methodical\fp\exgen\10w5d_cs_do.csproj" />
+    -->
     <ProjectReference Include="@(_SpmiTestProjects)">
       <Targets>Build</Targets>
       <OutputItemType>_SpmiTest</OutputItemType>