aboutsummaryrefslogtreecommitdiff
path: root/src/testing
diff options
context:
space:
mode:
Diffstat (limited to 'src/testing')
-rw-r--r--src/testing/benchmark.go45
-rw-r--r--src/testing/match.go71
-rw-r--r--src/testing/match_test.go118
-rw-r--r--src/testing/sub_test.go180
-rw-r--r--src/testing/testing.go16
5 files changed, 356 insertions, 74 deletions
diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go
index c935bc5e06..5d58b85e78 100644
--- a/src/testing/benchmark.go
+++ b/src/testing/benchmark.go
@@ -208,7 +208,24 @@ func (b *B) run1() bool {
b.runN(1)
}()
<-b.signal
- return !b.hasSub
+ if b.failed {
+ fmt.Fprintf(b.w, "--- FAIL: %s\n%s", b.name, b.output)
+ return false
+ }
+ // Only print the output if we know we are not going to proceed.
+ // Otherwise it is printed in processBench.
+ if b.hasSub || b.finished {
+ tag := "BENCH"
+ if b.skipped {
+ tag = "SKIP"
+ }
+ if b.chatty && (len(b.output) > 0 || b.finished) {
+ b.trimOutput()
+ fmt.Fprintf(b.w, "--- %s: %s\n%s", tag, b.name, b.output)
+ }
+ return false
+ }
+ return true
}
// run executes the benchmark in a separate goroutine, including all of its
@@ -372,7 +389,11 @@ func runBenchmarksInternal(matchString func(pat, str string) (bool, error), benc
}
}
main := &B{
- common: common{name: "Main"},
+ common: common{
+ name: "Main",
+ w: os.Stdout,
+ chatty: *chatty,
+ },
benchFunc: func(b *B) {
for _, Benchmark := range bs {
b.Run(Benchmark.Name, Benchmark.F)
@@ -390,13 +411,15 @@ func (ctx *benchContext) processBench(b *B) {
for i, procs := range cpuList {
runtime.GOMAXPROCS(procs)
benchName := benchmarkName(b.name, procs)
- fmt.Printf("%-*s\t", ctx.maxLen, benchName)
+ fmt.Fprintf(b.w, "%-*s\t", ctx.maxLen, benchName)
// Recompute the running time for all but the first iteration.
if i > 0 {
b = &B{
common: common{
signal: make(chan bool),
name: b.name,
+ w: b.w,
+ chatty: b.chatty,
},
benchFunc: b.benchFunc,
benchTime: b.benchTime,
@@ -408,19 +431,19 @@ func (ctx *benchContext) processBench(b *B) {
// The output could be very long here, but probably isn't.
// We print it all, regardless, because we don't want to trim the reason
// the benchmark failed.
- fmt.Printf("--- FAIL: %s\n%s", benchName, b.output)
+ fmt.Fprintf(b.w, "--- FAIL: %s\n%s", benchName, b.output)
continue
}
results := r.String()
if *benchmarkMemory || b.showAllocResult {
results += "\t" + r.MemString()
}
- fmt.Println(results)
+ fmt.Fprintln(b.w, results)
// Unlike with tests, we ignore the -chatty flag and always print output for
// benchmarks since the output generation time will skew the results.
if len(b.output) > 0 {
b.trimOutput()
- fmt.Printf("--- BENCH: %s\n%s", benchName, b.output)
+ fmt.Fprintf(b.w, "--- BENCH: %s\n%s", benchName, b.output)
}
if p := runtime.GOMAXPROCS(-1); p != procs {
fmt.Fprintf(os.Stderr, "testing: %s left GOMAXPROCS set to %d\n", benchName, p)
@@ -453,6 +476,8 @@ func (b *B) Run(name string, f func(b *B)) bool {
name: benchName,
parent: &b.common,
level: b.level + 1,
+ w: b.w,
+ chatty: b.chatty,
},
benchFunc: f,
benchTime: b.benchTime,
@@ -597,9 +622,17 @@ func Benchmark(f func(b *B)) BenchmarkResult {
b := &B{
common: common{
signal: make(chan bool),
+ w: discard{},
},
benchFunc: f,
benchTime: *benchTime,
}
+ if !b.run1() {
+ return BenchmarkResult{}
+ }
return b.run()
}
+
+type discard struct{}
+
+func (discard) Write(b []byte) (n int, err error) { return len(b), nil }
diff --git a/src/testing/match.go b/src/testing/match.go
index d0c52142ba..7751035760 100644
--- a/src/testing/match.go
+++ b/src/testing/match.go
@@ -8,29 +8,40 @@ import (
"fmt"
"os"
"strconv"
+ "strings"
"sync"
)
// matcher sanitizes, uniques, and filters names of subtests and subbenchmarks.
type matcher struct {
- filter string
+ filter []string
matchFunc func(pat, str string) (bool, error)
mu sync.Mutex
subNames map[string]int64
}
-// TODO: fix test_main to avoid race and improve caching.
+// TODO: fix test_main to avoid race and improve caching, also allowing to
+// eliminate this Mutex.
var matchMutex sync.Mutex
-func newMatcher(matchString func(pat, str string) (bool, error), pattern, name string) *matcher {
- // Verify filters before doing any processing.
- if _, err := matchString(pattern, "non-empty"); err != nil {
- fmt.Fprintf(os.Stderr, "testing: invalid regexp for %s: %s\n", name, err)
- os.Exit(1)
+func newMatcher(matchString func(pat, str string) (bool, error), patterns, name string) *matcher {
+ var filter []string
+ if patterns != "" {
+ filter = splitRegexp(patterns)
+ for i, s := range filter {
+ filter[i] = rewrite(s)
+ }
+ // Verify filters before doing any processing.
+ for i, s := range filter {
+ if _, err := matchString(s, "non-empty"); err != nil {
+ fmt.Fprintf(os.Stderr, "testing: invalid regexp for element %d of %s (%q): %s\n", i, name, s, err)
+ os.Exit(1)
+ }
+ }
}
return &matcher{
- filter: pattern,
+ filter: filter,
matchFunc: matchString,
subNames: map[string]int64{},
}
@@ -49,14 +60,54 @@ func (m *matcher) fullName(c *common, subname string) (name string, ok bool) {
matchMutex.Lock()
defer matchMutex.Unlock()
- if c != nil && c.level == 0 {
- if matched, _ := m.matchFunc(m.filter, subname); !matched {
+ // We check the full array of paths each time to allow for the case that
+ // a pattern contains a '/'.
+ for i, s := range strings.Split(name, "/") {
+ if i >= len(m.filter) {
+ break
+ }
+ if ok, _ := m.matchFunc(m.filter[i], s); !ok {
return name, false
}
}
return name, true
}
+func splitRegexp(s string) []string {
+ a := make([]string, 0, strings.Count(s, "/"))
+ cs := 0
+ cp := 0
+ for i := 0; i < len(s); {
+ switch s[i] {
+ case '[':
+ cs++
+ case ']':
+ if cs--; cs < 0 { // An unmatched ']' is legal.
+ cs = 0
+ }
+ case '(':
+ if cs == 0 {
+ cp++
+ }
+ case ')':
+ if cs == 0 {
+ cp--
+ }
+ case '\\':
+ i++
+ case '/':
+ if cs == 0 && cp == 0 {
+ a = append(a, s[:i])
+ s = s[i+1:]
+ i = 0
+ continue
+ }
+ }
+ i++
+ }
+ return append(a, s)
+}
+
// unique creates a unique name for the given parent and subname by affixing it
// with one ore more counts, if necessary.
func (m *matcher) unique(parent, subname string) string {
diff --git a/src/testing/match_test.go b/src/testing/match_test.go
index 68f3e9e867..d19036c72d 100644
--- a/src/testing/match_test.go
+++ b/src/testing/match_test.go
@@ -5,6 +5,7 @@
package testing
import (
+ "reflect"
"regexp"
"unicode"
)
@@ -23,6 +24,123 @@ func TestIsSpace(t *T) {
}
}
+func TestSplitRegexp(t *T) {
+ res := func(s ...string) []string { return s }
+ testCases := []struct {
+ pattern string
+ result []string
+ }{
+ // Correct patterns
+ // If a regexp pattern is correct, all split regexps need to be correct
+ // as well.
+ {"", res("")},
+ {"/", res("", "")},
+ {"//", res("", "", "")},
+ {"A", res("A")},
+ {"A/B", res("A", "B")},
+ {"A/B/", res("A", "B", "")},
+ {"/A/B/", res("", "A", "B", "")},
+ {"[A]/(B)", res("[A]", "(B)")},
+ {"[/]/[/]", res("[/]", "[/]")},
+ {"[/]/[:/]", res("[/]", "[:/]")},
+ {"/]", res("", "]")},
+ {"]/", res("]", "")},
+ {"]/[/]", res("]", "[/]")},
+ {`([)/][(])`, res(`([)/][(])`)},
+ {"[(]/[)]", res("[(]", "[)]")},
+
+ // Faulty patterns
+ // Errors in original should produce at least one faulty regexp in results.
+ {")/", res(")/")},
+ {")/(/)", res(")/(", ")")},
+ {"a[/)b", res("a[/)b")},
+ {"(/]", res("(/]")},
+ {"(/", res("(/")},
+ {"[/]/[/", res("[/]", "[/")},
+ {`\p{/}`, res(`\p{`, "}")},
+ {`\p/`, res(`\p`, "")},
+ {`[[:/:]]`, res(`[[:/:]]`)},
+ }
+ for _, tc := range testCases {
+ a := splitRegexp(tc.pattern)
+ if !reflect.DeepEqual(a, tc.result) {
+ t.Errorf("splitRegexp(%q) = %#v; want %#v", tc.pattern, a, tc.result)
+ }
+
+ // If there is any error in the pattern, one of the returned subpatterns
+ // needs to have an error as well.
+ if _, err := regexp.Compile(tc.pattern); err != nil {
+ ok := true
+ for _, re := range a {
+ if _, err := regexp.Compile(re); err != nil {
+ ok = false
+ }
+ }
+ if ok {
+ t.Errorf("%s: expected error in any of %q", tc.pattern, a)
+ }
+ }
+ }
+}
+
+func TestMatcher(t *T) {
+ testCases := []struct {
+ pattern string
+ parent, sub string
+ ok bool
+ }{
+ // Behavior without subtests.
+ {"", "", "TestFoo", true},
+ {"TestFoo", "", "TestFoo", true},
+ {"TestFoo/", "", "TestFoo", true},
+ {"TestFoo/bar/baz", "", "TestFoo", true},
+ {"TestFoo", "", "TestBar", false},
+ {"TestFoo/", "", "TestBar", false},
+ {"TestFoo/bar/baz", "", "TestBar/bar/baz", false},
+
+ // with subtests
+ {"", "TestFoo", "x", true},
+ {"TestFoo", "TestFoo", "x", true},
+ {"TestFoo/", "TestFoo", "x", true},
+ {"TestFoo/bar/baz", "TestFoo", "bar", true},
+ // Subtest with a '/' in its name still allows for copy and pasted names
+ // to match.
+ {"TestFoo/bar/baz", "TestFoo", "bar/baz", true},
+ {"TestFoo/bar/baz", "TestFoo/bar", "baz", true},
+ {"TestFoo/bar/baz", "TestFoo", "x", false},
+ {"TestFoo", "TestBar", "x", false},
+ {"TestFoo/", "TestBar", "x", false},
+ {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false},
+
+ // subtests only
+ {"", "TestFoo", "x", true},
+ {"/", "TestFoo", "x", true},
+ {"./", "TestFoo", "x", true},
+ {"./.", "TestFoo", "x", true},
+ {"/bar/baz", "TestFoo", "bar", true},
+ {"/bar/baz", "TestFoo", "bar/baz", true},
+ {"//baz", "TestFoo", "bar/baz", true},
+ {"//", "TestFoo", "bar/baz", true},
+ {"/bar/baz", "TestFoo/bar", "baz", true},
+ {"//foo", "TestFoo", "bar/baz", false},
+ {"/bar/baz", "TestFoo", "x", false},
+ {"/bar/baz", "TestBar", "x/bar/baz", false},
+ }
+
+ for _, tc := range testCases {
+ m := newMatcher(regexp.MatchString, tc.pattern, "-test.run")
+
+ parent := &common{name: tc.parent}
+ if tc.parent != "" {
+ parent.level = 1
+ }
+ if n, ok := m.fullName(parent, tc.sub); ok != tc.ok {
+ t.Errorf("pattern: %q, parent: %q, sub %q: got %v; want %v",
+ tc.pattern, tc.parent, tc.sub, ok, tc.ok, n)
+ }
+ }
+}
+
func TestNaming(t *T) {
m := newMatcher(regexp.MatchString, "", "")
diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go
index 264d77c2cf..2804550737 100644
--- a/src/testing/sub_test.go
+++ b/src/testing/sub_test.go
@@ -5,8 +5,9 @@
package testing
import (
- "io/ioutil"
+ "bytes"
"regexp"
+ "strings"
"sync/atomic"
"time"
)
@@ -113,11 +114,17 @@ func TestTRun(t *T) {
desc string
ok bool
maxPar int
+ chatty bool
+ output string
f func(*T)
}{{
desc: "failnow skips future sequential and parallel tests at same level",
ok: false,
maxPar: 1,
+ output: `
+--- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs)
+ --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs)
+ `,
f: func(t *T) {
ranSeq := false
ranPar := false
@@ -149,6 +156,11 @@ func TestTRun(t *T) {
desc: "failure in parallel test propagates upwards",
ok: false,
maxPar: 1,
+ output: `
+--- FAIL: failure in parallel test propagates upwards (N.NNs)
+ --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs)
+ --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs)
+ `,
f: func(t *T) {
t.Run("", func(t *T) {
t.Parallel()
@@ -159,6 +171,44 @@ func TestTRun(t *T) {
})
},
}, {
+ desc: "skipping without message, chatty",
+ ok: true,
+ chatty: true,
+ output: `
+=== RUN skipping without message, chatty
+--- SKIP: skipping without message, chatty (N.NNs)`,
+ f: func(t *T) { t.SkipNow() },
+ }, {
+ desc: "chatty with recursion",
+ ok: true,
+ chatty: true,
+ output: `
+=== RUN chatty with recursion
+=== RUN chatty with recursion/#00
+=== RUN chatty with recursion/#00/#00
+--- PASS: chatty with recursion (N.NNs)
+ --- PASS: chatty with recursion/#00 (N.NNs)
+ --- PASS: chatty with recursion/#00/#00 (N.NNs)`,
+ f: func(t *T) {
+ t.Run("", func(t *T) {
+ t.Run("", func(t *T) {})
+ })
+ },
+ }, {
+ desc: "skipping without message, not chatty",
+ ok: true,
+ f: func(t *T) { t.SkipNow() },
+ }, {
+ desc: "skipping after error",
+ output: `
+--- FAIL: skipping after error (N.NNs)
+ sub_test.go:NNN: an error
+ sub_test.go:NNN: skipped`,
+ f: func(t *T) {
+ t.Error("an error")
+ t.Skip("skipped")
+ },
+ }, {
desc: "use Run to locally synchronize parallelism",
ok: true,
maxPar: 1,
@@ -177,57 +227,6 @@ func TestTRun(t *T) {
}
},
}, {
- desc: "run no more than *parallel tests concurrently",
- ok: true,
- maxPar: 4,
- f: func(t *T) {
- max := 0
- in := make(chan int)
- out := make(chan int)
- ctx := t.context
- t.Run("wait", func(t *T) {
- t.Run("controller", func(t *T) {
- // Verify sequential tests don't skew counts.
- t.Run("seq1", func(t *T) {})
- t.Run("seq2", func(t *T) {})
- t.Run("seq3", func(t *T) {})
- t.Parallel()
- for i := 0; i < 80; i++ {
- ctx.mu.Lock()
- if ctx.running > max {
- max = ctx.running
- }
- ctx.mu.Unlock()
- <-in
- // force a minimum to avoid a race, although it works
- // without it.
- if i >= ctx.maxParallel-2 { // max - this - 1
- out <- i
- }
- }
- close(out)
- })
- // Ensure we don't exceed the maximum even with nested parallelism.
- for i := 0; i < 2; i++ {
- t.Run("", func(t *T) {
- t.Parallel()
- for j := 0; j < 40; j++ {
- t.Run("", func(t *T) {
- t.Run("seq1", func(t *T) {})
- t.Run("seq2", func(t *T) {})
- t.Parallel()
- in <- j
- <-out
- })
- }
- })
- }
- })
- if max != ctx.maxParallel {
- realTest.Errorf("max: got %d; want: %d", max, ctx.maxParallel)
- }
- },
- }, {
desc: "alternate sequential and parallel",
// Sequential tests should partake in the counting of running threads.
// Otherwise, if one runs parallel subtests in sequential tests that are
@@ -301,14 +300,23 @@ func TestTRun(t *T) {
})
}
},
+ }, {
+ desc: "skip output",
+ ok: true,
+ maxPar: 4,
+ f: func(t *T) {
+ t.Skip()
+ },
}}
for _, tc := range testCases {
ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", ""))
+ buf := &bytes.Buffer{}
root := &T{
common: common{
signal: make(chan bool),
name: "Test",
- w: ioutil.Discard,
+ w: buf,
+ chatty: tc.chatty,
},
context: ctx,
}
@@ -324,6 +332,12 @@ func TestTRun(t *T) {
if ctx.running != 0 || ctx.numWaiting != 0 {
t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting)
}
+ got := strings.TrimSpace(buf.String())
+ want := strings.TrimSpace(tc.output)
+ re := makeRegexp(want)
+ if ok, err := regexp.MatchString(re, got); !ok || err != nil {
+ t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
+ }
}
}
@@ -336,6 +350,8 @@ func TestBRun(t *T) {
testCases := []struct {
desc string
failed bool
+ chatty bool
+ output string
f func(*B)
}{{
desc: "simulate sequential run of subbenchmarks.",
@@ -371,8 +387,43 @@ func TestBRun(t *T) {
}, {
desc: "failure carried over to root",
failed: true,
+ output: "--- FAIL: root",
f: func(b *B) { b.Fail() },
}, {
+ desc: "skipping without message, chatty",
+ chatty: true,
+ output: "--- SKIP: root",
+ f: func(b *B) { b.SkipNow() },
+ }, {
+ desc: "skipping with message, chatty",
+ chatty: true,
+ output: `
+--- SKIP: root
+ sub_test.go:NNN: skipping`,
+ f: func(b *B) { b.Skip("skipping") },
+ }, {
+ desc: "chatty with recursion",
+ chatty: true,
+ f: func(b *B) {
+ b.Run("", func(b *B) {
+ b.Run("", func(b *B) {})
+ })
+ },
+ }, {
+ desc: "skipping without message, not chatty",
+ f: func(b *B) { b.SkipNow() },
+ }, {
+ desc: "skipping after error",
+ failed: true,
+ output: `
+--- FAIL: root
+ sub_test.go:NNN: an error
+ sub_test.go:NNN: skipped`,
+ f: func(b *B) {
+ b.Error("an error")
+ b.Skip("skipped")
+ },
+ }, {
desc: "memory allocation",
f: func(b *B) {
const bufSize = 256
@@ -398,11 +449,15 @@ func TestBRun(t *T) {
}}
for _, tc := range testCases {
var ok bool
+ buf := &bytes.Buffer{}
// This is almost like the Benchmark function, except that we override
// the benchtime and catch the failure result of the subbenchmark.
root := &B{
common: common{
signal: make(chan bool),
+ name: "root",
+ w: buf,
+ chatty: tc.chatty,
},
benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
benchTime: time.Microsecond,
@@ -418,5 +473,24 @@ func TestBRun(t *T) {
if root.result.N != 1 {
t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N)
}
+ got := strings.TrimSpace(buf.String())
+ want := strings.TrimSpace(tc.output)
+ re := makeRegexp(want)
+ if ok, err := regexp.MatchString(re, got); !ok || err != nil {
+ t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want)
+ }
}
}
+
+func makeRegexp(s string) string {
+ s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1)
+ s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1)
+ return s
+}
+
+func TestBenchmarkOutput(t *T) {
+ // Ensure Benchmark initialized common.w by invoking it with an error and
+ // normal case.
+ Benchmark(func(b *B) { b.Error("do not print this output") })
+ Benchmark(func(b *B) {})
+}
diff --git a/src/testing/testing.go b/src/testing/testing.go
index f9bb43b618..3a7a135a3c 100644
--- a/src/testing/testing.go
+++ b/src/testing/testing.go
@@ -199,6 +199,7 @@ type common struct {
mu sync.RWMutex // guards output and failed
output []byte // Output generated by test or benchmark.
w io.Writer // For flushToParent.
+ chatty bool // A copy of the chatty flag.
failed bool // Test or benchmark has failed.
skipped bool // Test of benchmark has been skipped.
finished bool
@@ -265,7 +266,6 @@ func (c *common) flushToParent(format string, args ...interface{}) {
defer p.mu.Unlock()
fmt.Fprintf(p.w, format, args...)
- fmt.Fprintln(p.w)
c.mu.Lock()
defer c.mu.Unlock()
@@ -562,13 +562,18 @@ func (t *T) Run(name string, f func(t *T)) bool {
name: testName,
parent: &t.common,
level: t.level + 1,
+ chatty: t.chatty,
},
context: t.context,
}
t.w = indenter{&t.common}
- if *chatty {
- fmt.Printf("=== RUN %s\n", t.name)
+ if t.chatty {
+ // Print directly to root's io.Writer so there is no delay.
+ root := t.parent
+ for ; root.parent != nil; root = root.parent {
+ }
+ fmt.Fprintf(root.w, "=== RUN %s\n", t.name)
}
// Instead of reducing the running count of this test before calling the
// tRunner and increasing it afterwards, we rely on tRunner keeping the
@@ -690,10 +695,10 @@ func (t *T) report() {
return
}
dstr := fmtDuration(t.duration)
- format := "--- %s: %s (%s)"
+ format := "--- %s: %s (%s)\n"
if t.Failed() {
t.flushToParent(format, "FAIL", t.name, dstr)
- } else if *chatty {
+ } else if t.chatty {
if t.Skipped() {
t.flushToParent(format, "SKIP", t.name, dstr)
} else {
@@ -716,6 +721,7 @@ func RunTests(matchString func(pat, str string) (bool, error), tests []InternalT
signal: make(chan bool),
barrier: make(chan bool),
w: os.Stdout,
+ chatty: *chatty,
},
context: ctx,
}
generated by cgit v1.3 (git 2.53.0) at 2026-04-16 03:38:06 +0000