IOS SDKs (#6257)

* [ios-sdk] Rename test-runner directory to 'runtime' to be consistent with xamarin.ios.

* [ios-sdk] Add missing xamarin_log () icall used by Console.Write ().

* [ios-sdk] Change the run-ios-sim target so it works on ios11 as well, the --console option to simctl appears to be broken.

* [ios-sdk] Add a test harness.

The test harness is resposible for starting the emulator, installing the app on the emulator, running it, and
collecting application output using the osx logging facilities.

* [ios-sdk] Fix detection of the app exiting with an unhandled exception.

* [ios-sdk] Add CI support for running some test suites on the simulator.

* [ios-sdk] Kill the log reader process on shutdown.

* [ios-sdk] Disable android on CI because the make targets error out if the android sdk is not installed.

* [ios-sdk] Fix error checking in make targets.

* [ios-sdk] Remove accidently commited binary.

* [ios-sdk] Switch to using os_log instead of NSLog (), the latter doesn't seem to show up in the logs anymore. Switch to using the default ios sdk instead of hardcoding 11.1.

* [ios-sdk] Don't hardcode the ios version when creating a simulator.

* [ios-sdk] Parse the output of simctl list runtimes as json.

* [ios-sdk] Use 'syslog' style logging, its supported by older osx versions.

* [ios-sdk] Fix logfile parsing.

* [ios-sdk] Fix log filtering.

* [ios-sdk] Add test exclusions file for System.

* [ios-sdk] Redirect runtime logging to os_log ().

* [ios-sdk] Link libMonoPosixHelper and zlib into the test runner.

* [ios-sdk] Fix the target64 and cross64 builds.

* [ios-sdk] Build target64/cross64 on CI.

* [ios-sdk] Add device support to the runtime lib. Mark strings in os_log () calls as public.

* [ios-sdk] Log stdout messages using a separate subsystem for easier filtering.

* [ios-sdk] Add an 'appbuilder' tool which can generate simulator and device apps.

* [ios-sdk] Build device tests on CI.

* [ios-sdk] Add a test-runner instead of using nunit-lite-console.exe. The two are identical right now, but they might diverge in the future.

* [ios-sdk] Add a script to download prebuild llvm binaries.

* [ios-sdk] Run the runtime initialization on a separate thread, not the UI thread.

* [ios-sdks] Fix the build.

* [ios-sdk] Build the cross compiler against llvm.

* [ios-sdk] Add missing file.

* [ios-sdk] Add llvm support, enabled by passing LLVM=1 to make build-ios-dev-<app>.

* [bcl] Enable System.Security tests on the MOBILE profile.

* [ios-sdk] Enable more test suites.

* [ios-sdk] Fix device builds.

* [ios-sdk] Make some test steps non-fatal.

* [ios-dev] Use ad-hoc signing to sign device apps.

* [ios-sdk] Decrease the timeout on run-sim to 20m.

* [ios-sdk] Send back the test results using a tcp connection, parsing the ios logs is too fragile.

* [ios-sdk] Mark System.Net.Http tests as notworking, they seem to fail on CI.

* [ios-sdk] Compile the 32 bit cross compiler against llvm as well.

* [ios-sdk] Package up the binaries which can be uploaded to storage.

* [ios-sdk] Fix the download-llvm.sh script.

* [ios-sdk] Compile cross32 on CI.

* [ios-sdk] Use Options.cs from the mcs sources instead of making a copy.

* [ios-sdk] Avoid hardcoding the xcode sysroot, obtain it using 'xcode-select -p'.

* [ios-sdk] Fix the previous change.

* [ios-sdk] Add an option to appbuilder to cache aot compilation results between building different apps.

* [ios-sdk] Build device apps with llvm as well on CI.

* [ios-skd] Use XCODE_ROOT in a few more places, update README.md.

1 files changed, 196 insertions, 0 deletions

diff --git a/sdks/ios/harness/harness.cs b/sdks/ios/harness/harness.cs
new file mode 100644
index 00000000000..cd838be4ed2
--- /dev/null
+++ b/sdks/ios/harness/harness.cs
@@ -0,0 +1,196 @@
+using System;
+using System.IO;
+using System.Json;
+using System.Threading;
+using System.Diagnostics;
+using System.Net;
+using System.Net.Sockets;
+using Mono.Options;
+
+public class Harness
+{
+	public const string SIM_NAME = "xamarin.ios-sdk.sim";
+
+	static void Usage () {
+		Console.WriteLine ("Usage: mono harness.exe <options> <app dir>");
+		Console.WriteLine ("Where options are:");
+		Console.WriteLine ("\t--run-sim");
+		Console.WriteLine ("\t--app=<app bundle id>");
+		Console.WriteLine ("\t--logfile=<log file name>");
+	}
+
+	public static int Main (string[] args) {
+		new Harness ().Run (args);
+		return 0;
+	}
+
+	string bundle_id;
+	string bundle_dir;
+	string logfile_name;
+	string[] new_args;
+
+	public void Run (string[] args) {
+		string action = "";
+		bundle_id = "";
+		bundle_dir = "";
+		logfile_name = "";
+
+		var p = new OptionSet () {
+		  { "start-sim", s => action = "start-sim" },
+		  { "run-sim", s => action = "run-sim" },
+		  { "bundle-id=", s => bundle_id = s },
+		  { "bundle-dir=", s => bundle_dir = s },
+		  { "logfile=", s => logfile_name = s },
+		};
+		new_args = p.Parse (args).ToArray ();
+
+		if (action == "start-sim") {
+			StartSim ();
+		} else if (action == "run-sim") {
+			if (bundle_id == "" || bundle_dir == "") {
+				Console.WriteLine ("The --bundle-id and --bundle-dir arguments are mandatory.");
+				Environment.Exit (1);
+			}
+			RunSim ();
+		} else {
+			Usage ();
+			Environment.Exit (1);
+		}
+	}
+
+	void StartSim () {
+		// Check whenever our simulator instance exists
+		string state_line = "";
+		{
+			var args = "simctl list devices";
+			Console.WriteLine ("Running: " + "xcrun " + args);
+			var start_info = new ProcessStartInfo ("xcrun", args);
+			start_info.RedirectStandardOutput = true;
+			start_info.UseShellExecute = false;
+			var process = Process.Start (start_info);
+			var stream = process.StandardOutput;
+			string line = "";
+			while (true) {
+				line = stream.ReadLine ();
+				if (line == null)
+					break;
+				if (line.Contains (SIM_NAME)) {
+					state_line = line;
+					break;
+				}
+			}
+			process.WaitForExit ();
+			if (process.ExitCode != 0)
+				Environment.Exit (1);
+		}
+
+		bool need_start = false;
+		if (state_line == "") {
+			// Get the runtime type
+			var args = "simctl list -j runtimes";
+			Console.WriteLine ("Running: " + "xcrun " + args);
+			var start_info = new ProcessStartInfo ("xcrun", args);
+			start_info.RedirectStandardOutput = true;
+			start_info.UseShellExecute = false;
+			var process = Process.Start (start_info);
+			var stream = process.StandardOutput;
+			JsonObject value = JsonValue.Parse (stream.ReadToEnd ()) as JsonObject;
+			string runtime = value ["runtimes"][0]["identifier"];
+
+			// Create the simulator
+			args = "simctl create " + SIM_NAME + " 'iPhone 7' " + runtime;
+			Console.WriteLine ("Running: " + "xcrun " + args);
+			process = Process.Start ("xcrun", args);
+			process.WaitForExit ();
+			if (process.ExitCode != 0)
+				Environment.Exit (1);
+			need_start = true;
+		} else if (state_line.Contains ("(Shutdown)")) {
+			need_start = true;
+		}
+
+		if (need_start) {
+			var args = "simctl boot " + SIM_NAME;
+			Console.WriteLine ("Running: " + "xcrun " + args);
+			var process = Process.Start ("xcrun", args);
+			process.WaitForExit ();
+			if (process.ExitCode != 0)
+				Environment.Exit (1);
+		}
+	}
+
+	void RunSim () {
+		Console.WriteLine ("App: " + bundle_id);
+
+		StartSim ();
+
+		// Install the app
+		// We do this all the time since its cheap
+		string exe = "xcrun";
+		string args = "simctl install " + SIM_NAME + " " + bundle_dir;
+		Console.WriteLine ("Running: " + exe + " " + args);
+		var process = Process.Start (exe, args);
+		process.WaitForExit ();
+		if (process.ExitCode != 0)
+			Environment.Exit (1);
+
+		//
+		// Test results are returned using an socket connection.
+		//
+		var host = Dns.GetHostEntry (Dns.GetHostName ());
+		var server = new TcpListener (System.Net.IPAddress.Loopback, 0);
+		server.Start ();
+		int port = ((IPEndPoint)server.LocalEndpoint).Port;
+
+		string app_args = "";
+		foreach (var a in new_args)
+			app_args += a + " ";
+		if (!app_args.Contains ("CONNSTR"))
+			throw new Exception ();
+		app_args = app_args.Replace ("CONNSTR", $"tcp:localhost:{port}");
+
+		// Terminate previous app
+		exe = "xcrun";
+		args = "simctl terminate " + SIM_NAME + " " + bundle_id;
+		Console.WriteLine ("Running: " + exe + " " + args);
+		process = Process.Start (exe, args);
+		process.WaitForExit ();
+		if (process.ExitCode != 0)
+			Environment.Exit (1);
+
+		// Launch new app
+		exe = "xcrun";
+		args = "simctl launch " + SIM_NAME + " " + bundle_id + " " + app_args;
+		Console.WriteLine ("Running: " + exe + " " + args);
+		process = Process.Start (exe, args);
+		process.WaitForExit ();
+		if (process.ExitCode != 0)
+			Environment.Exit (1);
+
+		//
+		// Read test results from the tcp connection
+		//
+		TextWriter w = new StreamWriter (logfile_name);
+		string result_line = null;
+		var client = server.AcceptTcpClient ();
+		var stream = client.GetStream ();
+		var reader = new StreamReader (stream);
+		while (true) {
+			var line = reader.ReadLine ();
+			if (line == null)
+				break;
+			Console.WriteLine (line);
+			w.WriteLine (line);
+			if (line.Contains ("Tests run:"))
+				result_line = line;
+			// Printed by the runtime
+			if (line.Contains ("Exit code:"))
+				break;
+		}
+
+		if (result_line != null && result_line.Contains ("Errors: 0"))
+			Environment.Exit (0);
+		else
+			Environment.Exit (1);
+	}
+}