diff options
author | David Benjamin <davidben@chromium.org> | 2015-02-12 01:14:15 +0300 |
---|---|---|
committer | Adam Langley <agl@google.com> | 2015-02-18 02:37:12 +0300 |
commit | 5f237bc843f4a4791667493d8748c903b8e540e4 (patch) | |
tree | dc5afdd55a979d4fffc52eb16901a611e351a967 | |
parent | 1b5cfb5ba39b1f3988d36926fb1d30794d338d69 (diff) |
Add support for Chromium's JSON test result format.
Also adds a flag to runner.go to make it more suitable for printing to a pipe.
Change-Id: I26fae21f3e4910028f6b8bfc4821c8c595525504
Reviewed-on: https://boringssl-review.googlesource.com/3490
Reviewed-by: Adam Langley <agl@google.com>
-rw-r--r-- | ssl/test/runner/runner.go | 66 | ||||
-rw-r--r-- | ssl/test/runner/test_output.go | 69 | ||||
-rw-r--r-- | util/all_tests.go | 83 |
3 files changed, 193 insertions, 25 deletions
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go index 99c66a45..ae7e0e2c 100644 --- a/ssl/test/runner/runner.go +++ b/ssl/test/runner/runner.go @@ -24,11 +24,13 @@ import ( ) var ( - useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") - useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") - flagDebug *bool = flag.Bool("debug", false, "Hexdump the contents of the connection") - mallocTest *int64 = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") - mallocTestDebug *bool = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") + useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") + useGDB = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb") + flagDebug = flag.Bool("debug", false, "Hexdump the contents of the connection") + mallocTest = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.") + mallocTestDebug = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.") + jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") + pipe = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.") ) const ( @@ -1798,7 +1800,7 @@ func addStateMachineCoverageTests(async, splitHandshake bool, protocol protocol) // False Start without session tickets. testCases = append(testCases, testCase{ - name: "FalseStart-SessionTicketsDisabled", + name: "FalseStart-SessionTicketsDisabled" + suffix, config: Config{ CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256}, NextProtos: []string{"foo"}, @@ -2134,7 +2136,7 @@ func addExtensionTests() { }) testCases = append(testCases, testCase{ testType: clientTest, - name: "ServerNameExtensionClient", + name: "ServerNameExtensionClientMismatch", config: Config{ Bugs: ProtocolBugs{ ExpectServerName: "mismatch.com", @@ -2146,7 +2148,7 @@ func addExtensionTests() { }) testCases = append(testCases, testCase{ testType: clientTest, - name: "ServerNameExtensionClient", + name: "ServerNameExtensionClientMissing", config: Config{ Bugs: ProtocolBugs{ ExpectServerName: "missing.com", @@ -2576,7 +2578,7 @@ func addFastRadioPaddingTests() { }) testCases = append(testCases, testCase{ protocol: dtls, - name: "FastRadio-Padding", + name: "FastRadio-Padding-DTLS", config: Config{ Bugs: ProtocolBugs{ RequireFastradioPadding: true, @@ -2805,27 +2807,43 @@ type statusMsg struct { err error } -func statusPrinter(doneChan chan struct{}, statusChan chan statusMsg, total int) { +func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) { var started, done, failed, lineLen int - defer close(doneChan) + testOutput := newTestOutput() for msg := range statusChan { + if !*pipe { + // Erase the previous status line. + fmt.Printf("\x1b[%dD\x1b[K", lineLen) + } + if msg.started { started++ } else { done++ - } - fmt.Printf("\x1b[%dD\x1b[K", lineLen) + if msg.err != nil { + fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err) + failed++ + testOutput.addResult(msg.test.name, "FAIL") + } else { + if *pipe { + // Print each test instead of a status line. + fmt.Printf("PASSED (%s)\n", msg.test.name) + } + testOutput.addResult(msg.test.name, "PASS") + } + } - if msg.err != nil { - fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err) - failed++ + if !*pipe { + // Print a new status line. + line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total) + lineLen = len(line) + os.Stdout.WriteString(line) } - line := fmt.Sprintf("%d/%d/%d/%d", failed, done, started, total) - lineLen = len(line) - os.Stdout.WriteString(line) } + + doneChan <- testOutput } func main() { @@ -2865,7 +2883,7 @@ func main() { statusChan := make(chan statusMsg, numWorkers) testChan := make(chan *testCase, numWorkers) - doneChan := make(chan struct{}) + doneChan := make(chan *testOutput) go statusPrinter(doneChan, statusChan, len(testCases)) @@ -2883,7 +2901,13 @@ func main() { close(testChan) wg.Wait() close(statusChan) - <-doneChan + testOutput := <-doneChan fmt.Printf("\n") + + if *jsonOutput != "" { + if err := testOutput.writeTo(*jsonOutput); err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + } + } } diff --git a/ssl/test/runner/test_output.go b/ssl/test/runner/test_output.go new file mode 100644 index 00000000..57fbbe6f --- /dev/null +++ b/ssl/test/runner/test_output.go @@ -0,0 +1,69 @@ +/* Copyright (c) 2015, Google Inc. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +package main + +import ( + "encoding/json" + "os" + "time" +) + +// testOutput is a representation of Chromium's JSON test result format. See +// https://www.chromium.org/developers/the-json-test-results-format +type testOutput struct { + Version int `json:"version"` + Interrupted bool `json:"interrupted"` + PathDelimiter string `json:"path_delimiter"` + SecondsSinceEpoch float64 `json:"seconds_since_epoch"` + NumFailuresByType map[string]int `json:"num_failures_by_type"` + Tests map[string]testResult `json:"tests"` +} + +type testResult struct { + Actual string `json:"actual"` + Expected string `json:"expected"` +} + +func newTestOutput() *testOutput { + return &testOutput{ + Version: 3, + PathDelimiter: ".", + SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), + NumFailuresByType: make(map[string]int), + Tests: make(map[string]testResult), + } +} + +func (t *testOutput) addResult(name, result string) { + if _, found := t.Tests[name]; found { + panic(name) + } + t.Tests[name] = testResult{Actual: result, Expected: "PASS"} + t.NumFailuresByType[result]++ +} + +func (t *testOutput) writeTo(name string) error { + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + out, err := json.MarshalIndent(t, "", " ") + if err != nil { + return err + } + _, err = file.Write(out) + return err +} diff --git a/util/all_tests.go b/util/all_tests.go index 13f83021..907dc48a 100644 --- a/util/all_tests.go +++ b/util/all_tests.go @@ -16,12 +16,14 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "os" "os/exec" "path" "strings" + "time" ) // TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner. @@ -29,6 +31,7 @@ import ( var ( useValgrind = flag.Bool("valgrind", false, "If true, run code under valgrind") buildDir = flag.String("build-dir", "build", "The build directory to run the tests from.") + jsonOutput = flag.String("json-output", "", "The file to output JSON results to.") ) type test []string @@ -83,6 +86,54 @@ var tests = []test{ {"ssl/ssl_test"}, } +// testOutput is a representation of Chromium's JSON test result format. See +// https://www.chromium.org/developers/the-json-test-results-format +type testOutput struct { + Version int `json:"version"` + Interrupted bool `json:"interrupted"` + PathDelimiter string `json:"path_delimiter"` + SecondsSinceEpoch float64 `json:"seconds_since_epoch"` + NumFailuresByType map[string]int `json:"num_failures_by_type"` + Tests map[string]testResult `json:"tests"` +} + +type testResult struct { + Actual string `json:"actual"` + Expected string `json:"expected"` +} + +func newTestOutput() *testOutput { + return &testOutput{ + Version: 3, + PathDelimiter: ".", + SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond), + NumFailuresByType: make(map[string]int), + Tests: make(map[string]testResult), + } +} + +func (t *testOutput) addResult(name, result string) { + if _, found := t.Tests[name]; found { + panic(name) + } + t.Tests[name] = testResult{Actual: result, Expected: "PASS"} + t.NumFailuresByType[result]++ +} + +func (t *testOutput) writeTo(name string) error { + file, err := os.Create(name) + if err != nil { + return err + } + defer file.Close() + out, err := json.MarshalIndent(t, "", " ") + if err != nil { + return err + } + _, err = file.Write(out) + return err +} + func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd { valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full"} if dbAttach { @@ -124,21 +175,39 @@ func runTest(test test) (passed bool, err error) { return false, nil } +// shortTestName returns the short name of a test. It assumes that any argument +// which ends in .txt is a path to a data file and not relevant to the test's +// uniqueness. +func shortTestName(test test) string { + var args []string + for _, arg := range test { + if !strings.HasSuffix(arg, ".txt") { + args = append(args, arg) + } + } + return strings.Join(args, " ") +} + func main() { flag.Parse() + testOutput := newTestOutput() var failed []test for _, test := range tests { fmt.Printf("%s\n", strings.Join([]string(test), " ")) + + name := shortTestName(test) passed, err := runTest(test) if err != nil { - fmt.Printf("%s failed to complete: %s\n", test[0], err.Error()) + fmt.Printf("%s failed to complete: %s\n", test[0], err) failed = append(failed, test) - continue - } - if !passed { + testOutput.addResult(name, "CRASHED") + } else if !passed { fmt.Printf("%s failed to print PASS on the last line.\n", test[0]) failed = append(failed, test) + testOutput.addResult(name, "FAIL") + } else { + testOutput.addResult(name, "PASS") } } @@ -150,4 +219,10 @@ func main() { fmt.Printf("\t%s\n", strings.Join([]string(test), " ")) } } + + if *jsonOutput != "" { + if err := testOutput.writeTo(*jsonOutput); err != nil { + fmt.Fprintf(os.Stderr, "Error: %s\n", err) + } + } } |