[acceptance-tests] Some improvements to the profiler stress runner.

* Print combined stdout and stderr in console output.
* Print the RNG seed used for a test run on startup and in the test report.
* Set the individual test timeout back to 6 hours.
* Use the nodefaults profiler option.
* Allow configuration through environment variables:
 - MONO_PROFILER_STRESS_OPTIONS: Specify a set of profiler options to use
   instead of the randomly generated ones.
 - MONO_PROFILER_STRESS_REGEX: Specify a regular expression to filter which
   tests should be run.
 - MONO_PROFILER_STRESS_SEED: Specify the RNG seed. Useful for reproducing full
   test runs at a later point.
 - MONO_PROFILER_STRESS_TIMEOUT: Specify a different individual test timeout
   from the default.

1 files changed, 59 insertions, 25 deletions

diff --git a/acceptance-tests/profiler-stress/runner.cs b/acceptance-tests/profiler-stress/runner.cs
index 71295fc9b56..84bb9846a56 100644
--- a/acceptance-tests/profiler-stress/runner.cs
+++ b/acceptance-tests/profiler-stress/runner.cs
@@ -40,6 +40,7 @@ namespace Mono.Profiling.Tests.Stress {
 		public int? ExitCode { get; set; }
 		public string StandardOutput { get; set; }
 		public string StandardError { get; set; }
+		public string CombinedOutput { get; set; }
 	}
 
 	static class Program {
@@ -57,7 +58,9 @@ namespace Mono.Profiling.Tests.Stress {
 			"jit",
 		};
 
-		static readonly TimeSpan _timeout = TimeSpan.FromHours (9);
+		static readonly TimeSpan _timeout = TimeSpan.FromHours (6);
+
+		const RegexOptions _regexOptions = RegexOptions.Compiled | RegexOptions.Singleline;
 
 		static readonly Dictionary<string, Predicate<Benchmark>> _filters = new Dictionary<string, Predicate<Benchmark>> {
 			{ "ironjs-v8", FilterNotOnArm },
@@ -89,26 +92,50 @@ namespace Mono.Profiling.Tests.Stress {
 
 		static int Main ()
 		{
+			var regex = new Regex (".*", _regexOptions);
+
+			if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_REGEX") is string envRegex)
+				regex = new Regex (envRegex, _regexOptions);
+
 			var depDir = Path.Combine ("..", "external", "benchmarker");
 			var benchDir = Path.Combine (depDir, "benchmarks");
 			var testDir = Path.Combine (depDir, "tests");
 
 			var benchmarks = Directory.EnumerateFiles (benchDir, "*.benchmark")
 			                 .Select (Benchmark.Load)
-			                 .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b))
+			                 .Where (b => !b.OnlyExplicit && b.ClientCommandLine == null && IsSupported (b) && regex.IsMatch (b.Name))
 			                 .OrderBy (b => b.Name)
 			                 .ToArray ();
 
 			var monoPath = Path.GetFullPath (Path.Combine ("..", "..", "runtime", "mono-wrapper"));
 			var classDir = Path.GetFullPath (Path.Combine ("..", "..", "mcs", "class", "lib", "net_4_x"));
 
-			var rand = new Random ();
-			var cpus = Environment.ProcessorCount;
+			var seed = Environment.TickCount;
 
-			var results = new List<TestResult> (benchmarks.Length);
+			if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_SEED") is string envSeed)
+				seed = int.Parse (envSeed);
+
+			var timeout = (int) _timeout.TotalMilliseconds;
+
+			if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_TIMEOUT") is string envTimeout)
+				timeout = (int) TimeSpan.Parse (envTimeout).TotalMilliseconds;
+
+			string options = null;
+
+			if (Environment.GetEnvironmentVariable ("MONO_PROFILER_STRESS_OPTIONS") is string envOptions)
+				options = envOptions;
+
+			var rand = new Random (seed);
+			var cpus = Environment.ProcessorCount;
 
 			var sw = Stopwatch.StartNew ();
 
+			Console.ForegroundColor = ConsoleColor.Magenta;
+			Console.WriteLine ($"[{sw.Elapsed.ToString ("G")}] Starting test session with seed: {seed}");
+			Console.ResetColor ();
+
+			var results = new List<TestResult> (benchmarks.Length);
+
 			for (var i = 0; i < benchmarks.Length; i++) {
 				var bench = benchmarks [i];
 
@@ -117,17 +144,24 @@ namespace Mono.Profiling.Tests.Stress {
 				var maxSamples = rand.Next (0, cpus * 2000 + 1);
 				var heapShotFreq = rand.Next (-10, 11);
 				var maxFrames = rand.Next (0, 33);
-				var options = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
-				                      .Select (x => (x.Value ? string.Empty : "no") + x.Key)
-				                      .ToArray ();
+				var flags = _options.ToDictionary (x => x, _ => rand.Next (0, 2) == 1)
+				                    .Select (x => (x.Value ? string.Empty : "no") + x.Key)
+				                    .ToArray ();
+
+				var profOptions = "nodefaults,output=/dev/null,";
 
-				var profOptions = $"maxframes={maxFrames},{string.Join (",", options)},output=/dev/null";
+				if (options == null) {
+					profOptions += $"maxframes={maxFrames},";
 
-				if (sampleFreq > 0)
-					profOptions += $",sample{sampleMode}={sampleFreq},maxsamples={maxSamples}";
+					if (sampleFreq > 0)
+						profOptions += $"sample{sampleMode}={sampleFreq},maxsamples={maxSamples},";
 
-				if (heapShotFreq > 0)
-					profOptions += $",heapshot={heapShotFreq}gc";
+					if (heapShotFreq > 0)
+						profOptions += $"heapshot={heapShotFreq}gc,";
+
+					profOptions += string.Join (",", flags);
+				} else
+					profOptions += options;
 
 				var info = new ProcessStartInfo {
 					UseShellExecute = false,
@@ -157,17 +191,22 @@ namespace Mono.Profiling.Tests.Stress {
 
 					var stdout = new StringBuilder ();
 					var stderr = new StringBuilder ();
+					var combined = new StringBuilder ();
 
 					proc.OutputDataReceived += (sender, args) => {
 						if (args.Data != null)
-							lock (result)
+							lock (result) {
 								stdout.AppendLine (args.Data);
+								combined.AppendLine (args.Data);
+							}
 					};
 
 					proc.ErrorDataReceived += (sender, args) => {
 						if (args.Data != null)
-							lock (result)
+							lock (result) {
 								stderr.AppendLine (args.Data);
+								combined.AppendLine (args.Data);
+							}
 					};
 
 					result.Stopwatch.Start ();
@@ -177,7 +216,7 @@ namespace Mono.Profiling.Tests.Stress {
 					proc.BeginOutputReadLine ();
 					proc.BeginErrorReadLine ();
 
-					if (!proc.WaitForExit ((int) _timeout.TotalMilliseconds)) {
+					if (!proc.WaitForExit (timeout)) {
 						// Force a thread dump.
 						Syscall.kill (proc.Id, Signum.SIGQUIT);
 						Thread.Sleep (1000);
@@ -194,6 +233,7 @@ namespace Mono.Profiling.Tests.Stress {
 					lock (result) {
 						result.StandardOutput = stdout.ToString ();
 						result.StandardError = stderr.ToString ();
+						result.CombinedOutput = combined.ToString ();
 					}
 				}
 
@@ -205,16 +245,10 @@ namespace Mono.Profiling.Tests.Stress {
 
 				if (result.ExitCode != 0) {
 					Console.ForegroundColor = ConsoleColor.Red;
-					Console.WriteLine ("===== stdout =====");
-					Console.ResetColor ();
-
-					Console.WriteLine (result.StandardOutput);
-
-					Console.ForegroundColor = ConsoleColor.Red;
-					Console.WriteLine ("===== stderr =====");
+					Console.WriteLine ("===== stdout + stderr =====");
 					Console.ResetColor ();
 
-					Console.WriteLine (result.StandardError);
+					Console.WriteLine (result.CombinedOutput);
 				}
 
 				results.Add (result);
@@ -233,7 +267,7 @@ namespace Mono.Profiling.Tests.Stress {
 
 			using (var writer = XmlWriter.Create ("TestResult-profiler-stress.xml", settings)) {
 				writer.WriteStartDocument ();
-				writer.WriteComment ("This file represents the results of running a test suite");
+				writer.WriteComment ($"This file represents the results of running a test suite (seed: {seed})");
 
 				writer.WriteStartElement ("test-results");
 				writer.WriteAttributeString ("name", "profiler-stress-tests.dummy");