Add support for Chromium's JSON test result format.

Also adds a flag to runner.go to make it more suitable for printing to a pipe.

Change-Id: I26fae21f3e4910028f6b8bfc4821c8c595525504
Reviewed-on: https://boringssl-review.googlesource.com/3490
Reviewed-by: Adam Langley <agl@google.com>

3 files changed, 193 insertions, 25 deletions

diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go
index 99c66a45..ae7e0e2c 100644
--- a/ssl/test/runner/runner.go
+++ b/ssl/test/runner/runner.go
@@ -24,11 +24,13 @@ import (
 )
 
 var (
-	useValgrind            = flag.Bool("valgrind", false, "If true, run code under valgrind")
-	useGDB                 = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
-	flagDebug       *bool  = flag.Bool("debug", false, "Hexdump the contents of the connection")
-	mallocTest      *int64 = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
-	mallocTestDebug *bool  = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
+	useValgrind     = flag.Bool("valgrind", false, "If true, run code under valgrind")
+	useGDB          = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
+	flagDebug       = flag.Bool("debug", false, "Hexdump the contents of the connection")
+	mallocTest      = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
+	mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
+	jsonOutput      = flag.String("json-output", "", "The file to output JSON results to.")
+	pipe            = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
 )
 
 const (
@@ -1798,7 +1800,7 @@ func addStateMachineCoverageTests(async, splitHandshake bool, protocol protocol)
 
 		// False Start without session tickets.
 		testCases = append(testCases, testCase{
-			name: "FalseStart-SessionTicketsDisabled",
+			name: "FalseStart-SessionTicketsDisabled" + suffix,
 			config: Config{
 				CipherSuites:           []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
 				NextProtos:             []string{"foo"},
@@ -2134,7 +2136,7 @@ func addExtensionTests() {
 	})
 	testCases = append(testCases, testCase{
 		testType: clientTest,
-		name:     "ServerNameExtensionClient",
+		name:     "ServerNameExtensionClientMismatch",
 		config: Config{
 			Bugs: ProtocolBugs{
 				ExpectServerName: "mismatch.com",
@@ -2146,7 +2148,7 @@ func addExtensionTests() {
 	})
 	testCases = append(testCases, testCase{
 		testType: clientTest,
-		name:     "ServerNameExtensionClient",
+		name:     "ServerNameExtensionClientMissing",
 		config: Config{
 			Bugs: ProtocolBugs{
 				ExpectServerName: "missing.com",
@@ -2576,7 +2578,7 @@ func addFastRadioPaddingTests() {
 	})
 	testCases = append(testCases, testCase{
 		protocol: dtls,
-		name:     "FastRadio-Padding",
+		name:     "FastRadio-Padding-DTLS",
 		config: Config{
 			Bugs: ProtocolBugs{
 				RequireFastradioPadding: true,
@@ -2805,27 +2807,43 @@ type statusMsg struct {
 	err     error
 }
 
-func statusPrinter(doneChan chan struct{}, statusChan chan statusMsg, total int) {
+func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) {
 	var started, done, failed, lineLen int
-	defer close(doneChan)
 
+	testOutput := newTestOutput()
 	for msg := range statusChan {
+		if !*pipe {
+			// Erase the previous status line.
+			fmt.Printf("\x1b[%dD\x1b[K", lineLen)
+		}
+
 		if msg.started {
 			started++
 		} else {
 			done++
-		}
 
-		fmt.Printf("\x1b[%dD\x1b[K", lineLen)
+			if msg.err != nil {
+				fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
+				failed++
+				testOutput.addResult(msg.test.name, "FAIL")
+			} else {
+				if *pipe {
+					// Print each test instead of a status line.
+					fmt.Printf("PASSED (%s)\n", msg.test.name)
+				}
+				testOutput.addResult(msg.test.name, "PASS")
+			}
+		}
 
-		if msg.err != nil {
-			fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
-			failed++
+		if !*pipe {
+			// Print a new status line.
+			line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total)
+			lineLen = len(line)
+			os.Stdout.WriteString(line)
 		}
-		line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total)
-		lineLen = len(line)
-		os.Stdout.WriteString(line)
 	}
+
+	doneChan <- testOutput
 }
 
 func main() {
@@ -2865,7 +2883,7 @@ func main() {
 
 	statusChan := make(chan statusMsg, numWorkers)
 	testChan := make(chan *testCase, numWorkers)
-	doneChan := make(chan struct{})
+	doneChan := make(chan *testOutput)
 
 	go statusPrinter(doneChan, statusChan, len(testCases))
 
@@ -2883,7 +2901,13 @@ func main() {
 	close(testChan)
 	wg.Wait()
 	close(statusChan)
-	<-doneChan
+	testOutput := <-doneChan
 
 	fmt.Printf("\n")
+
+	if *jsonOutput != "" {
+		if err := testOutput.writeTo(*jsonOutput); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+		}
+	}
 }
diff --git a/ssl/test/runner/test_output.go b/ssl/test/runner/test_output.go
new file mode 100644
index 00000000..57fbbe6f
--- /dev/null
+++ b/ssl/test/runner/test_output.go
@@ -0,0 +1,69 @@
+/* Copyright (c) 2015, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+package main
+
+import (
+	"encoding/json"
+	"os"
+	"time"
+)
+
+// testOutput is a representation of Chromium's JSON test result format. See
+// https://www.chromium.org/developers/the-json-test-results-format
+type testOutput struct {
+	Version           int                   `json:"version"`
+	Interrupted       bool                  `json:"interrupted"`
+	PathDelimiter     string                `json:"path_delimiter"`
+	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
+	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
+	Tests             map[string]testResult `json:"tests"`
+}
+
+type testResult struct {
+	Actual   string `json:"actual"`
+	Expected string `json:"expected"`
+}
+
+func newTestOutput() *testOutput {
+	return &testOutput{
+		Version:           3,
+		PathDelimiter:     ".",
+		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
+		NumFailuresByType: make(map[string]int),
+		Tests:             make(map[string]testResult),
+	}
+}
+
+func (t *testOutput) addResult(name, result string) {
+	if _, found := t.Tests[name]; found {
+		panic(name)
+	}
+	t.Tests[name] = testResult{Actual: result, Expected: "PASS"}
+	t.NumFailuresByType[result]++
+}
+
+func (t *testOutput) writeTo(name string) error {
+	file, err := os.Create(name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	out, err := json.MarshalIndent(t, "", "  ")
+	if err != nil {
+		return err
+	}
+	_, err = file.Write(out)
+	return err
+}
diff --git a/util/all_tests.go b/util/all_tests.go
index 13f83021..907dc48a 100644
--- a/util/all_tests.go
+++ b/util/all_tests.go
@@ -16,12 +16,14 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"strings"
+	"time"
 )
 
 // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
@@ -29,6 +31,7 @@ import (
 var (
 	useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind")
 	buildDir    = flag.String("build-dir", "build", "The build directory to run the tests from.")
+	jsonOutput  = flag.String("json-output", "", "The file to output JSON results to.")
 )
 
 type test []string
@@ -83,6 +86,54 @@ var tests = []test{
 	{"ssl/ssl_test"},
 }
 
+// testOutput is a representation of Chromium's JSON test result format. See
+// https://www.chromium.org/developers/the-json-test-results-format
+type testOutput struct {
+	Version           int                   `json:"version"`
+	Interrupted       bool                  `json:"interrupted"`
+	PathDelimiter     string                `json:"path_delimiter"`
+	SecondsSinceEpoch float64               `json:"seconds_since_epoch"`
+	NumFailuresByType map[string]int        `json:"num_failures_by_type"`
+	Tests             map[string]testResult `json:"tests"`
+}
+
+type testResult struct {
+	Actual   string `json:"actual"`
+	Expected string `json:"expected"`
+}
+
+func newTestOutput() *testOutput {
+	return &testOutput{
+		Version:           3,
+		PathDelimiter:     ".",
+		SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
+		NumFailuresByType: make(map[string]int),
+		Tests:             make(map[string]testResult),
+	}
+}
+
+func (t *testOutput) addResult(name, result string) {
+	if _, found := t.Tests[name]; found {
+		panic(name)
+	}
+	t.Tests[name] = testResult{Actual: result, Expected: "PASS"}
+	t.NumFailuresByType[result]++
+}
+
+func (t *testOutput) writeTo(name string) error {
+	file, err := os.Create(name)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	out, err := json.MarshalIndent(t, "", "  ")
+	if err != nil {
+		return err
+	}
+	_, err = file.Write(out)
+	return err
+}
+
 func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
 	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"}
 	if dbAttach {
@@ -124,21 +175,39 @@ func runTest(test test) (passed bool, err error) {
 	return false, nil
 }
 
+// shortTestName returns the short name of a test. It assumes that any argument
+// which ends in .txt is a path to a data file and not relevant to the test's
+// uniqueness.
+func shortTestName(test test) string {
+	var args []string
+	for _, arg := range test {
+		if !strings.HasSuffix(arg, ".txt") {
+			args = append(args, arg)
+		}
+	}
+	return strings.Join(args, " ")
+}
+
 func main() {
 	flag.Parse()
 
+	testOutput := newTestOutput()
 	var failed []test
 	for _, test := range tests {
 		fmt.Printf("%s\n", strings.Join([]string(test), " "))
+
+		name := shortTestName(test)
 		passed, err := runTest(test)
 		if err != nil {
-			fmt.Printf("%s failed to complete: %s\n", test[0], err.Error())
+			fmt.Printf("%s failed to complete: %s\n", test[0], err)
 			failed = append(failed, test)
-			continue
-		}
-		if !passed {
+			testOutput.addResult(name, "CRASHED")
+		} else if !passed {
 			fmt.Printf("%s failed to print PASS on the last line.\n", test[0])
 			failed = append(failed, test)
+			testOutput.addResult(name, "FAIL")
+		} else {
+			testOutput.addResult(name, "PASS")
 		}
 	}
 
@@ -150,4 +219,10 @@ func main() {
 			fmt.Printf("\t%s\n", strings.Join([]string(test), " "))
 		}
 	}
+
+	if *jsonOutput != "" {
+		if err := testOutput.writeTo(*jsonOutput); err != nil {
+			fmt.Fprintf(os.Stderr, "Error: %s\n", err)
+		}
+	}
 }