Adds babysitter script for PR tests

I see two problems with our current automated PR tests:
- Many false negatives— PR tests often fail due to recurring failures
  unrelated to the PR
- Information about what failed is awkward to retrieve, scattered in
  multiple places

This commit adds a Python script, `scripts/babysitter`. Currently our
Jenkins test suites are each run wrapped with the `timeout` command.
The babysitter acts as a drop-in replacement for GNU timeout but adds
the following features:
- Logs machine readable output about each test suite (as
  line-delimited json)
- If the test suite uses NUnit, can detect if a test case failed or
  crashed (terminated mono in mid-test), and retry unsuccessful tests
  (up to a limit).

The reasoning here is that tests which fail inconsistently are
currently most likely due to one of our outstanding recurring failures
rather than the change made in the PR. Therefore, if a failing test
succeeds on retry, the PR itself is probably valid (although the
failure should be logged and looked at).

In addition to the script itself, changes to NUnit were required in
order to support the retry feature and allow fine-grained logging for
NUnit suites.

Major TODOs:
- Add retry support for our non-NUnit-based test suites
- Save the XML files NUnit produces (since reruns stomp XMLs from
  previous runs)
- Add some kind of sensible feature for dealing with timeouts

8 files changed, 129 insertions, 0 deletions

diff --git a/mcs/nunit24/ConsoleRunner/nunit-console/ConsoleUi.cs b/mcs/nunit24/ConsoleRunner/nunit-console/ConsoleUi.cs
index 072b1efc9c1..24a0b570d32 100644
--- a/mcs/nunit24/ConsoleRunner/nunit-console/ConsoleUi.cs
+++ b/mcs/nunit24/ConsoleRunner/nunit-console/ConsoleUi.cs
@@ -97,6 +97,8 @@ namespace NUnit.ConsoleRunner
 						testFilter = new AndFilter( testFilter, excludeFilter );

 				}

 

+				testFilter = BabysitterSupport.AddBabysitterFilter(testFilter);

+

 				TestResult result = null;

 				string savedDirectory = Environment.CurrentDirectory;

 				TextWriter savedOut = Console.Out;

diff --git a/mcs/nunit24/ConsoleRunner/nunit-console/EventCollector.cs b/mcs/nunit24/ConsoleRunner/nunit-console/EventCollector.cs
index 7e3d582688a..c3f8db9c3d4 100644
--- a/mcs/nunit24/ConsoleRunner/nunit-console/EventCollector.cs
+++ b/mcs/nunit24/ConsoleRunner/nunit-console/EventCollector.cs
@@ -98,6 +98,8 @@ namespace NUnit.ConsoleRunner
 							}

 						}

 					}

+

+					BabysitterSupport.RecordFailedTest(currentTestName);

 				}

 			}

 			else

diff --git a/mcs/nunit24/NUnitCore/core/BabysitterSupport.cs b/mcs/nunit24/NUnitCore/core/BabysitterSupport.cs
new file mode 100644
index 00000000000..e9ea14b2565
--- /dev/null
+++ b/mcs/nunit24/NUnitCore/core/BabysitterSupport.cs
@@ -0,0 +1,104 @@
+// Added to NUnit by Andi McClure to add special support for the test babysitter script.
+// See scripts/babysitter in Mono repository.
+
+namespace NUnit.Core
+{
+	using System;
+	using System.IO;
+	using System.Collections.Generic;
+	using NUnit.Core.Filters;
+
+	public class BabysitterSupport
+	{
+		enum OverrideMode {None, Run, Exclude};
+		public static Dictionary<string, bool> OverrideTests = new Dictionary<string, bool>();
+		private static OverrideMode Override = OverrideMode.None;
+		private static string CurrentTestFile = null, RanTestFile = null, FailedTestFile = null;
+
+		private static void DeleteFile(string path)
+		{
+			try {
+				File.Delete(path);
+			} catch (Exception e) {}
+		}
+		private static void WriteFile(string path, string contents)
+		{
+			DeleteFile(path);
+			File.AppendAllText(path, contents);
+		}
+
+		// Environment variables are available from process start, so safe to do setup in a static constructor
+		static BabysitterSupport()
+		{
+			string overrideModeString = Environment.GetEnvironmentVariable("NUNIT_BABYSITTER_RUN_MODE");
+			string overrideTestString = Environment.GetEnvironmentVariable("NUNIT_BABYSITTER_RUN_TEST");
+			if (overrideModeString == "RUN")
+				Override = OverrideMode.Run;
+			else if (overrideModeString == "EXCLUDE")
+				Override = OverrideMode.Exclude;
+			if (Override != OverrideMode.None)
+			{
+				char [] semicolon = {';'};
+				foreach (string s in overrideTestString.Split(semicolon, StringSplitOptions.RemoveEmptyEntries))
+					OverrideTests[s] = true;
+			}
+
+			CurrentTestFile = Environment.GetEnvironmentVariable("NUNIT_BABYSITTER_CURRENT_TEST_FILE");
+			RanTestFile = Environment.GetEnvironmentVariable("NUNIT_BABYSITTER_RAN_TEST_FILE");
+			FailedTestFile = Environment.GetEnvironmentVariable("NUNIT_BABYSITTER_FAILED_TEST_FILE");
+		}
+
+		// Entry points
+
+		public static void RecordEnterTest( string testName )
+		{
+			if (CurrentTestFile != null)
+				WriteFile(CurrentTestFile, testName);
+			if (RanTestFile != null)
+				File.AppendAllText(RanTestFile, testName + "\n");
+		}
+
+		public static void RecordLeaveTest( string testName )
+		{
+			if (CurrentTestFile != null)
+				DeleteFile(CurrentTestFile);
+		}
+
+		public static void RecordFailedTest( string testName )
+		{
+			if (FailedTestFile != null)
+				File.AppendAllText(FailedTestFile, testName + "\n");
+		}
+
+		public static TestFilter AddBabysitterFilter(TestFilter currentFilter)
+		{
+			if (Override == OverrideMode.None)
+				return currentFilter;
+			return new AndFilter(currentFilter, new BabysitterFilter());
+		}
+
+		[Serializable]
+		private class BabysitterFilter : TestFilter
+		{
+			public override bool Match(ITest test)
+			{
+				if (test.IsSuite) // A suite returning true will automatically run ALL contents, filters ignored
+					return false;
+				bool inList = OverrideTests.ContainsKey(test.TestName.FullName);
+				bool allow = true;
+				switch (Override)
+				{
+					case OverrideMode.None:
+						break;
+					case OverrideMode.Run:
+						allow = inList;
+						break;
+					case OverrideMode.Exclude:
+						allow = !inList;
+						break;
+				}
+				return allow;
+			}
+		}
+	}
+}
\ No newline at end of file
diff --git a/mcs/nunit24/NUnitCore/core/TestCase.cs b/mcs/nunit24/NUnitCore/core/TestCase.cs
index 4722bcb954f..d43714f3ad5 100644
--- a/mcs/nunit24/NUnitCore/core/TestCase.cs
+++ b/mcs/nunit24/NUnitCore/core/TestCase.cs
@@ -49,6 +49,13 @@ namespace NUnit.Core
 				TestCaseResult testResult = MakeTestCaseResult();

 

 				listener.TestStarted( this.TestName );

+

+				// The babysitter's enter/leave "listeners" specifically exist to track crashes,

+				// so unfortunately they can't work through the (asynchronous) listener interface.

+				bool willRun = this.RunState == RunState.Runnable || this.RunState == RunState.Explicit;

+				if (willRun)

+					BabysitterSupport.RecordEnterTest(this.TestName.FullName);

+

 				long startTime = DateTime.Now.Ticks;

 

 				switch (this.RunState)

@@ -68,6 +75,10 @@ namespace NUnit.Core
 				}

 

 				long stopTime = DateTime.Now.Ticks;

+

+				if (willRun)

+					BabysitterSupport.RecordLeaveTest(this.TestName.FullName);

+

 				double time = ((double)(stopTime - startTime)) / (double)TimeSpan.TicksPerSecond;

 				testResult.Time = time;

 

diff --git a/mcs/nunit24/NUnitCore/core/nunit.core.build b/mcs/nunit24/NUnitCore/core/nunit.core.build
index 49712e32fd8..2651b22376c 100644
--- a/mcs/nunit24/NUnitCore/core/nunit.core.build
+++ b/mcs/nunit24/NUnitCore/core/nunit.core.build
@@ -11,6 +11,7 @@
         <include name="AssemblyInfo.cs"/>

         <include name="AssemblyReader.cs"/>

         <include name="AssemblyResolver.cs"/>

+        <include name="BabysitterSupport.cs"/>

         <include name="CoreExtensions.cs"/>

         <include name="CultureDetector.cs"/>

         <include name="DirectorySwapper.cs"/>

diff --git a/mcs/nunit24/NUnitCore/core/nunit.core.dll.csproj b/mcs/nunit24/NUnitCore/core/nunit.core.dll.csproj
index 8502368a78a..2901792b63e 100644
--- a/mcs/nunit24/NUnitCore/core/nunit.core.dll.csproj
+++ b/mcs/nunit24/NUnitCore/core/nunit.core.dll.csproj
@@ -109,6 +109,11 @@
                     BuildAction = "Compile"

                 />

                 <File

+                    RelPath = "BabysitterSupport.cs"

+                    SubType = "Code"

+                    BuildAction = "Compile"

+                />

+                <File

                     RelPath = "CommonAssemblyInfo.cs"

                     Link = "..\..\CommonAssemblyInfo.cs"

                     SubType = "Code"

diff --git a/mcs/nunit24/NUnitCore/core/nunit.core.dll.sources b/mcs/nunit24/NUnitCore/core/nunit.core.dll.sources
index 0efe3c0fd64..4fa186b9206 100644
--- a/mcs/nunit24/NUnitCore/core/nunit.core.dll.sources
+++ b/mcs/nunit24/NUnitCore/core/nunit.core.dll.sources
@@ -3,6 +3,7 @@ AbstractTestCaseDecoration.cs
 AssemblyInfo.cs
 AssemblyReader.cs
 AssemblyResolver.cs
+BabysitterSupport.cs
 Builders/AbstractFixtureBuilder.cs
 Builders/AbstractTestCaseBuilder.cs
 Builders/LegacySuiteBuilder.cs
diff --git a/mcs/nunit24/NUnitCore/core/nunit.core.dll_VS2005.csproj b/mcs/nunit24/NUnitCore/core/nunit.core.dll_VS2005.csproj
index 00e25771939..eaf015df06b 100644
--- a/mcs/nunit24/NUnitCore/core/nunit.core.dll_VS2005.csproj
+++ b/mcs/nunit24/NUnitCore/core/nunit.core.dll_VS2005.csproj
@@ -65,6 +65,9 @@
     <Compile Include="AssemblyResolver.cs">

       <SubType>Code</SubType>

     </Compile>

+    <Compile Include="BabysitterSupport.cs">

+      <SubType>Code</SubType>

+    </Compile>

     <Compile Include="CoreExtensions.cs" />

     <Compile Include="CultureDetector.cs" />

     <Compile Include="ExtensionPoint.cs" />