build: convert V8 test JSON to JUnit XML

This introduces some code to convert from V8's test JSON output to JUnit
XML. We need this because V8's latest refactor of their test runner has
made it difficult to float our JUnit reporter patch on top (see the
referenced issue).

I also think that there needs to be the same changes to vcbuild.bat, but
I don't know how to do test those yet. I can create a Windows VM and
test it if we decide to go with this approach.

Refs: https://github.com/nodejs/node-v8/issues/236
PR-URL: https://github.com/nodejs/node/pull/44049
Fixes: https://github.com/nodejs/node-v8/issues/236
Reviewed-By: Ben Noordhuis <info@bnoordhuis.nl>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>

2 files changed, 133 insertions, 6 deletions

diff --git a/tools/test-v8.bat b/tools/test-v8.bat
index d322c31a38d..64265157f00 100644
--- a/tools/test-v8.bat
+++ b/tools/test-v8.bat
@@ -20,18 +20,21 @@ if errorlevel 1 set ERROR_STATUS=1&goto test-v8-exit
 set path=%savedpath%
 
 if not defined test_v8 goto test-v8-intl
-echo running 'python tools\run-tests.py %common_v8_test_options% %v8_test_options% --junitout ./v8-tap.xml'
-call python tools\run-tests.py %common_v8_test_options% %v8_test_options% --junitout ./v8-tap.xml
+echo running 'python tools\run-tests.py %common_v8_test_options% %v8_test_options% --slow-tests-cutoff 1000000 --json-test-results v8-tap.xml'
+call python tools\run-tests.py %common_v8_test_options% %v8_test_options% --slow-tests-cutoff 1000000 --json-test-results v8-tap.xml
+call python ..\..\tools\v8-json-to-junit.py < v8-tap.xml > v8-tap.json
 
 :test-v8-intl
 if not defined test_v8_intl goto test-v8-benchmarks
-echo running 'python tools\run-tests.py %common_v8_test_options% intl --junitout ./v8-intl-tap.xml'
-call python tools\run-tests.py %common_v8_test_options% intl --junitout ./v8-intl-tap.xml
+echo running 'python tools\run-tests.py %common_v8_test_options% intl --slow-tests-cutoff 1000000 --json-test-results v8-intl-tap.xml'
+call python tools\run-tests.py %common_v8_test_options% intl --slow-tests-cutoff 1000000 --json-test-results ./v8-intl-tap.xml
+call python ..\..\tools\v8-json-to-junit.py < v8-intl-tap.xml > v8-intl-tap.json
 
 :test-v8-benchmarks
 if not defined test_v8_benchmarks goto test-v8-exit
-echo running 'python tools\run-tests.py %common_v8_test_options% benchmarks --junitout ./v8-benchmarks-tap.xml'
-call python tools\run-tests.py %common_v8_test_options% benchmarks --junitout ./v8-benchmarks-tap.xml
+echo running 'python tools\run-tests.py %common_v8_test_options% benchmarks --slow-tests-cutoff 1000000 --json-test-results v8-benchmarks-tap.xml'
+call python tools\run-tests.py %common_v8_test_options% benchmarks --slow-tests-cutoff 1000000 --json-test-results ./v8-benchmarks-tap.xml
+call python ..\..\tools\v8-json-to-junit.py < v8-benchmarks-tap.xml > v8-benchmarks-tap.json
 goto test-v8-exit
 
 :test-v8-exit
diff --git a/tools/v8-json-to-junit.py b/tools/v8-json-to-junit.py
new file mode 100755
index 00000000000..3d94df58087
--- /dev/null
+++ b/tools/v8-json-to-junit.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+# Large parts of this file are modified from
+# deps/v8/tools/testrunner/local/junit_output.py, which no longer exists in
+# latest V8.
+#
+# Copyright 2013 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import utils
+import signal
+import sys
+import xml.etree.ElementTree as xml
+
+def IsExitCodeCrashing(exit_code):
+  if utils.IsWindows():
+    return 0x80000000 & exit_code and not (0x3FFFFF00 & exit_code)
+  return exit_code < 0 and exit_code != -signal.SIGABRT
+
+
+class JUnitTestOutput:
+  def __init__(self, test_suite_name):
+    self.root = xml.Element("testsuite")
+    self.root.attrib["name"] = test_suite_name
+
+  def HasRunTest(self, test_name, test_cmd, test_duration, test_failure):
+    test_case_element = xml.Element("testcase")
+    test_case_element.attrib["name"] = test_name
+    test_case_element.attrib["cmd"] = test_cmd
+    test_case_element.attrib["time"] = str(round(test_duration, 3))
+    if test_failure is not None:
+      failure_element = xml.Element("failure")
+      failure_element.text = test_failure
+      test_case_element.append(failure_element)
+    self.root.append(test_case_element)
+
+  def FinishAndWrite(self, f):
+    xml.ElementTree(self.root).write(f, "UTF-8")
+
+
+def Main():
+  test_results = json.load(sys.stdin)
+
+  # V8's JSON test runner only logs failing and flaky tests under "results". We
+  # assume the caller has put a large number for --slow-tests-cutoff, to ensure
+  # that all the tests appear under "slowest_tests".
+
+  failing_tests = {result["name"]: result for result in test_results["results"]}
+  all_tests = {result["name"]: result for result in test_results["slowest_tests"]}
+  passing_tests = {
+    name: result for name, result in all_tests.items() if name not in failing_tests
+  }
+
+  # These check that --slow-tests-cutoff was passed correctly.
+  assert len(failing_tests) + len(passing_tests) == len(all_tests)
+  assert len(all_tests) == len(test_results["slowest_tests"])
+
+  output = JUnitTestOutput("v8tests")
+
+  for name, failing_test in failing_tests.items():
+    failing_output = []
+
+    stdout = failing_test["stdout"].strip()
+    if len(stdout):
+      failing_output.append("stdout:")
+      failing_output.append(stdout)
+
+    stderr = failing_test["stderr"].strip()
+    if len(stderr):
+      failing_output.append("stderr:")
+      failing_output.append(stderr)
+
+    failing_output.append("Command: " + failing_test["command"])
+
+    exit_code = failing_test["exit_code"]
+    if failing_test["result"] == "TIMEOUT":
+      failing_output.append("--- TIMEOUT ---")
+    elif IsExitCodeCrashing(exit_code):
+      failing_output.append("exit code: " + str(exit_code))
+      failing_output.append("--- CRASHED ---")
+
+    output.HasRunTest(
+      test_name=name,
+      test_cmd=failing_test["command"],
+      test_duration=failing_test["duration"],
+      test_failure="\n".join(failing_output),
+    )
+
+  for name, passing_test in passing_tests.items():
+    output.HasRunTest(
+      test_name=name,
+      test_cmd=passing_test["command"],
+      test_duration=passing_test["duration"],
+      test_failure=None,
+    )
+
+  output.FinishAndWrite(sys.stdout.buffer)
+
+if __name__ == '__main__':
+  Main()