Move JSON test results code into a common module.
We can actually use modules now.
Change-Id: I0bd8abaf4e3318069f93fa17e89b4804d03944eb
Reviewed-on: https://boringssl-review.googlesource.com/c/33205
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go
index 9d5ba7f..95fd27c 100644
--- a/ssl/test/runner/runner.go
+++ b/ssl/test/runner/runner.go
@@ -42,6 +42,8 @@
"sync"
"syscall"
"time"
+
+ "boringssl.googlesource.com/boringssl/util/testresult"
)
var (
@@ -14661,10 +14663,10 @@
err error
}
-func statusPrinter(doneChan chan *testOutput, statusChan chan statusMsg, total int) {
+func statusPrinter(doneChan chan *testresult.Results, statusChan chan statusMsg, total int) {
var started, done, failed, unimplemented, lineLen int
- testOutput := newTestOutput()
+ testOutput := testresult.NewResults()
for msg := range statusChan {
if !*pipe {
// Erase the previous status line.
@@ -14687,18 +14689,22 @@
fmt.Printf("UNIMPLEMENTED (%s)\n", msg.test.name)
}
unimplemented++
- testOutput.addResult(msg.test.name, "UNIMPLEMENTED")
+ if *allowUnimplemented {
+ testOutput.AddSkip(msg.test.name)
+ } else {
+ testOutput.AddResult(msg.test.name, "SKIP")
+ }
} else {
fmt.Printf("FAILED (%s)\n%s\n", msg.test.name, msg.err)
failed++
- testOutput.addResult(msg.test.name, "FAIL")
+ testOutput.AddResult(msg.test.name, "FAIL")
}
} else {
if *pipe {
// Print each test instead of a status line.
fmt.Printf("PASSED (%s)\n", msg.test.name)
}
- testOutput.addResult(msg.test.name, "PASS")
+ testOutput.AddResult(msg.test.name, "PASS")
}
}
@@ -14762,7 +14768,7 @@
statusChan := make(chan statusMsg, *numWorkers)
testChan := make(chan *testCase, *numWorkers)
- doneChan := make(chan *testOutput)
+ doneChan := make(chan *testresult.Results)
if len(*shimConfigFile) != 0 {
encoded, err := ioutil.ReadFile(*shimConfigFile)
@@ -14835,16 +14841,12 @@
fmt.Printf("\n")
if *jsonOutput != "" {
- if err := testOutput.writeTo(*jsonOutput); err != nil {
+ if err := testOutput.WriteToFile(*jsonOutput); err != nil {
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
}
}
- if !*allowUnimplemented && testOutput.NumFailuresByType["UNIMPLEMENTED"] > 0 {
- os.Exit(1)
- }
-
- if !testOutput.noneFailed {
+ if !testOutput.HasUnexpectedResults() {
os.Exit(1)
}
}
diff --git a/ssl/test/runner/test_output.go b/ssl/test/runner/test_output.go
deleted file mode 100644
index eb54638..0000000
--- a/ssl/test/runner/test_output.go
+++ /dev/null
@@ -1,79 +0,0 @@
-/* Copyright (c) 2015, Google Inc.
- *
- * Permission to use, copy, modify, and/or distribute this software for any
- * purpose with or without fee is hereby granted, provided that the above
- * copyright notice and this permission notice appear in all copies.
- *
- * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
- * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
- * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
-
-package runner
-
-import (
- "encoding/json"
- "os"
- "time"
-)
-
-// testOutput is a representation of Chromium's JSON test result format. See
-// https://www.chromium.org/developers/the-json-test-results-format
-type testOutput struct {
- Version int `json:"version"`
- Interrupted bool `json:"interrupted"`
- PathDelimiter string `json:"path_delimiter"`
- SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
- NumFailuresByType map[string]int `json:"num_failures_by_type"`
- Tests map[string]testResult `json:"tests"`
- noneFailed bool
-}
-
-type testResult struct {
- Actual string `json:"actual"`
- Expected string `json:"expected"`
- IsUnexpected bool `json:"is_unexpected"`
-}
-
-func newTestOutput() *testOutput {
- return &testOutput{
- Version: 3,
- PathDelimiter: ".",
- SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
- NumFailuresByType: make(map[string]int),
- Tests: make(map[string]testResult),
- noneFailed: true,
- }
-}
-
-func (t *testOutput) addResult(name, result string) {
- if _, found := t.Tests[name]; found {
- panic(name)
- }
- t.Tests[name] = testResult{
- Actual: result,
- Expected: "PASS",
- IsUnexpected: result != "PASS",
- }
- t.NumFailuresByType[result]++
- if result != "PASS" && result != "UNIMPLEMENTED" {
- t.noneFailed = false
- }
-}
-
-func (t *testOutput) writeTo(name string) error {
- file, err := os.Create(name)
- if err != nil {
- return err
- }
- defer file.Close()
- out, err := json.MarshalIndent(t, "", " ")
- if err != nil {
- return err
- }
- _, err = file.Write(out)
- return err
-}
diff --git a/util/all_tests.go b/util/all_tests.go
index 8332b1f..55e1921 100644
--- a/util/all_tests.go
+++ b/util/all_tests.go
@@ -30,7 +30,8 @@
"strings"
"sync"
"syscall"
- "time"
+
+ "boringssl.googlesource.com/boringssl/util/testresult"
)
// TODO(davidben): Link tests with the malloc shim and port -malloc-test to this runner.
@@ -68,23 +69,6 @@
Error error
}
-// testOutput is a representation of Chromium's JSON test result format. See
-// https://www.chromium.org/developers/the-json-test-results-format
-type testOutput struct {
- Version int `json:"version"`
- Interrupted bool `json:"interrupted"`
- PathDelimiter string `json:"path_delimiter"`
- SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
- NumFailuresByType map[string]int `json:"num_failures_by_type"`
- Tests map[string]testResult `json:"tests"`
-}
-
-type testResult struct {
- Actual string `json:"actual"`
- Expected string `json:"expected"`
- IsUnexpected bool `json:"is_unexpected"`
-}
-
// sdeCPUs contains a list of CPU code that we run all tests under when *useSDE
// is true.
var sdeCPUs = []string{
@@ -113,46 +97,6 @@
"crypto", // Support for NEON and crypto extensions.
}
-func newTestOutput() *testOutput {
- return &testOutput{
- Version: 3,
- PathDelimiter: ".",
- SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
- NumFailuresByType: make(map[string]int),
- Tests: make(map[string]testResult),
- }
-}
-
-func (t *testOutput) addResult(name, result string) {
- if _, found := t.Tests[name]; found {
- panic(name)
- }
- expected := "PASS"
- if result == "SKIP" {
- expected = "SKIP"
- }
- t.Tests[name] = testResult{
- Actual: result,
- Expected: expected,
- IsUnexpected: result != expected,
- }
- t.NumFailuresByType[result]++
-}
-
-func (t *testOutput) writeTo(name string) error {
- file, err := os.Create(name)
- if err != nil {
- return err
- }
- defer file.Close()
- out, err := json.MarshalIndent(t, "", " ")
- if err != nil {
- return err
- }
- _, err = file.Write(out)
- return err
-}
-
func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"}
if dbAttach {
@@ -480,7 +424,7 @@
close(results)
}()
- testOutput := newTestOutput()
+ testOutput := testresult.NewResults()
var failed, skipped []test
for testResult := range results {
test := testResult.Test
@@ -490,25 +434,25 @@
fmt.Printf("%s\n", test.longName())
fmt.Printf("%s was skipped\n", args[0])
skipped = append(skipped, test)
- testOutput.addResult(test.longName(), "SKIP")
+ testOutput.AddSkip(test.longName())
} else if testResult.Error != nil {
fmt.Printf("%s\n", test.longName())
fmt.Printf("%s failed to complete: %s\n", args[0], testResult.Error)
failed = append(failed, test)
- testOutput.addResult(test.longName(), "CRASH")
+ testOutput.AddResult(test.longName(), "CRASH")
} else if !testResult.Passed {
fmt.Printf("%s\n", test.longName())
fmt.Printf("%s failed to print PASS on the last line.\n", args[0])
failed = append(failed, test)
- testOutput.addResult(test.longName(), "FAIL")
+ testOutput.AddResult(test.longName(), "FAIL")
} else {
fmt.Printf("%s\n", test.shortName())
- testOutput.addResult(test.longName(), "PASS")
+ testOutput.AddResult(test.longName(), "PASS")
}
}
if *jsonOutput != "" {
- if err := testOutput.writeTo(*jsonOutput); err != nil {
+ if err := testOutput.WriteToFile(*jsonOutput); err != nil {
fmt.Fprintf(os.Stderr, "Error: %s\n", err)
}
}
diff --git a/util/testresult/testresult.go b/util/testresult/testresult.go
new file mode 100644
index 0000000..5226716
--- /dev/null
+++ b/util/testresult/testresult.go
@@ -0,0 +1,95 @@
+/* Copyright (c) 2018, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+// testresult is an implementation of Chromium's JSON test result format. See
+// https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
+package testresult
+
+import (
+ "encoding/json"
+ "os"
+ "time"
+)
+
+// Results stores the top-level test results.
+type Results struct {
+ Version int `json:"version"`
+ Interrupted bool `json:"interrupted"`
+ PathDelimiter string `json:"path_delimiter"`
+ SecondsSinceEpoch float64 `json:"seconds_since_epoch"`
+ NumFailuresByType map[string]int `json:"num_failures_by_type"`
+ Tests map[string]Result `json:"tests"`
+}
+
+func NewResults() *Results {
+ return &Results{
+ Version: 3,
+ PathDelimiter: ".",
+ SecondsSinceEpoch: float64(time.Now().UnixNano()) / float64(time.Second/time.Nanosecond),
+ NumFailuresByType: make(map[string]int),
+ Tests: make(map[string]Result),
+ }
+}
+
+func (t *Results) addResult(name, result, expected string) {
+ if _, found := t.Tests[name]; found {
+ panic(name)
+ }
+ t.Tests[name] = Result{
+ Actual: result,
+ Expected: expected,
+ IsUnexpected: result != expected,
+ }
+ t.NumFailuresByType[result]++
+}
+
+// AddResult records a test result with the given result string. The test is a
+// failure if the result is not "PASS".
+func (t *Results) AddResult(name, result string) {
+ t.addResult(name, result, "PASS")
+}
+
+// AddSkip marks a test as being skipped. It is not considered a failure.
+func (t *Results) AddSkip(name string) {
+ t.addResult(name, "SKIP", "SKIP")
+}
+
+func (t *Results) HasUnexpectedResults() bool {
+ for _, r := range t.Tests {
+ if r.IsUnexpected {
+ return false
+ }
+ }
+ return true
+}
+
+func (t *Results) WriteToFile(name string) error {
+ file, err := os.Create(name)
+ if err != nil {
+ return err
+ }
+ defer file.Close()
+ out, err := json.MarshalIndent(t, "", " ")
+ if err != nil {
+ return err
+ }
+ _, err = file.Write(out)
+ return err
+}
+
+type Result struct {
+ Actual string `json:"actual"`
+ Expected string `json:"expected"`
+ IsUnexpected bool `json:"is_unexpected"`
+}
