From 7e0d66020c49ef56158346ce18dc3f538393829e Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Mon, 4 Apr 2016 18:22:29 +0200 Subject: testing: improve output This introduces a few changes - Skipped benchmarks now print a SKIP line, also if there was no output - The benchmark name is only printed if there the benchmark was not skipped or did not fail in the probe phase. It also fixes a bug of doubling a skip message in chatty mode in absense of a failure. The chatty flag is now passed in the common struct to allow for testing of the printed messages. Fixes #14799 Change-Id: Ia8eb140c2e5bb467e66b8ef20a2f98f5d95415d5 Reviewed-on: https://go-review.googlesource.com/21504 Reviewed-by: Brad Fitzpatrick Run-TryBot: Marcel van Lohuizen TryBot-Result: Gobot Gobot --- src/testing/benchmark.go | 45 ++++++++++++++++--- src/testing/sub_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++- src/testing/testing.go | 16 ++++--- 3 files changed, 158 insertions(+), 13 deletions(-) (limited to 'src/testing') diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go index c935bc5e06..5d58b85e78 100644 --- a/src/testing/benchmark.go +++ b/src/testing/benchmark.go @@ -208,7 +208,24 @@ func (b *B) run1() bool { b.runN(1) }() <-b.signal - return !b.hasSub + if b.failed { + fmt.Fprintf(b.w, "--- FAIL: %s\n%s", b.name, b.output) + return false + } + // Only print the output if we know we are not going to proceed. + // Otherwise it is printed in processBench. + if b.hasSub || b.finished { + tag := "BENCH" + if b.skipped { + tag = "SKIP" + } + if b.chatty && (len(b.output) > 0 || b.finished) { + b.trimOutput() + fmt.Fprintf(b.w, "--- %s: %s\n%s", tag, b.name, b.output) + } + return false + } + return true } // run executes the benchmark in a separate goroutine, including all of its @@ -372,7 +389,11 @@ func runBenchmarksInternal(matchString func(pat, str string) (bool, error), benc } } main := &B{ - common: common{name: "Main"}, + common: common{ + name: "Main", + w: os.Stdout, + chatty: *chatty, + }, benchFunc: func(b *B) { for _, Benchmark := range bs { b.Run(Benchmark.Name, Benchmark.F) @@ -390,13 +411,15 @@ func (ctx *benchContext) processBench(b *B) { for i, procs := range cpuList { runtime.GOMAXPROCS(procs) benchName := benchmarkName(b.name, procs) - fmt.Printf("%-*s\t", ctx.maxLen, benchName) + fmt.Fprintf(b.w, "%-*s\t", ctx.maxLen, benchName) // Recompute the running time for all but the first iteration. if i > 0 { b = &B{ common: common{ signal: make(chan bool), name: b.name, + w: b.w, + chatty: b.chatty, }, benchFunc: b.benchFunc, benchTime: b.benchTime, @@ -408,19 +431,19 @@ func (ctx *benchContext) processBench(b *B) { // The output could be very long here, but probably isn't. // We print it all, regardless, because we don't want to trim the reason // the benchmark failed. - fmt.Printf("--- FAIL: %s\n%s", benchName, b.output) + fmt.Fprintf(b.w, "--- FAIL: %s\n%s", benchName, b.output) continue } results := r.String() if *benchmarkMemory || b.showAllocResult { results += "\t" + r.MemString() } - fmt.Println(results) + fmt.Fprintln(b.w, results) // Unlike with tests, we ignore the -chatty flag and always print output for // benchmarks since the output generation time will skew the results. if len(b.output) > 0 { b.trimOutput() - fmt.Printf("--- BENCH: %s\n%s", benchName, b.output) + fmt.Fprintf(b.w, "--- BENCH: %s\n%s", benchName, b.output) } if p := runtime.GOMAXPROCS(-1); p != procs { fmt.Fprintf(os.Stderr, "testing: %s left GOMAXPROCS set to %d\n", benchName, p) @@ -453,6 +476,8 @@ func (b *B) Run(name string, f func(b *B)) bool { name: benchName, parent: &b.common, level: b.level + 1, + w: b.w, + chatty: b.chatty, }, benchFunc: f, benchTime: b.benchTime, @@ -597,9 +622,17 @@ func Benchmark(f func(b *B)) BenchmarkResult { b := &B{ common: common{ signal: make(chan bool), + w: discard{}, }, benchFunc: f, benchTime: *benchTime, } + if !b.run1() { + return BenchmarkResult{} + } return b.run() } + +type discard struct{} + +func (discard) Write(b []byte) (n int, err error) { return len(b), nil } diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 264d77c2cf..e053a3c348 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -5,8 +5,9 @@ package testing import ( - "io/ioutil" + "bytes" "regexp" + "strings" "sync/atomic" "time" ) @@ -113,11 +114,17 @@ func TestTRun(t *T) { desc string ok bool maxPar int + chatty bool + output string f func(*T) }{{ desc: "failnow skips future sequential and parallel tests at same level", ok: false, maxPar: 1, + output: ` +--- FAIL: failnow skips future sequential and parallel tests at same level (0.00s) + --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (0.00s) + `, f: func(t *T) { ranSeq := false ranPar := false @@ -149,6 +156,11 @@ func TestTRun(t *T) { desc: "failure in parallel test propagates upwards", ok: false, maxPar: 1, + output: ` +--- FAIL: failure in parallel test propagates upwards (0.00s) + --- FAIL: failure in parallel test propagates upwards/#00 (0.00s) + --- FAIL: failure in parallel test propagates upwards/#00/par (0.00s) + `, f: func(t *T) { t.Run("", func(t *T) { t.Parallel() @@ -158,6 +170,28 @@ func TestTRun(t *T) { }) }) }, + }, { + desc: "skipping without message, chatty", + ok: true, + chatty: true, + output: ` +=== RUN skipping without message, chatty +--- SKIP: skipping without message, chatty (0.00s)`, + f: func(t *T) { t.SkipNow() }, + }, { + desc: "skipping without message, not chatty", + ok: true, + f: func(t *T) { t.SkipNow() }, + }, { + desc: "skipping after error", + output: ` +--- FAIL: skipping after error (0.00s) + sub_test.go:nnn: an error + sub_test.go:nnn: skipped`, + f: func(t *T) { + t.Error("an error") + t.Skip("skipped") + }, }, { desc: "use Run to locally synchronize parallelism", ok: true, @@ -301,14 +335,23 @@ func TestTRun(t *T) { }) } }, + }, { + desc: "skip output", + ok: true, + maxPar: 4, + f: func(t *T) { + t.Skip() + }, }} for _, tc := range testCases { ctx := newTestContext(tc.maxPar, newMatcher(regexp.MatchString, "", "")) + buf := &bytes.Buffer{} root := &T{ common: common{ signal: make(chan bool), name: "Test", - w: ioutil.Discard, + w: buf, + chatty: tc.chatty, }, context: ctx, } @@ -324,6 +367,11 @@ func TestTRun(t *T) { if ctx.running != 0 || ctx.numWaiting != 0 { t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting) } + got := sanitizeLog(buf.String()) + want := sanitizeLog(tc.output) + if got != want { + t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) + } } } @@ -336,6 +384,8 @@ func TestBRun(t *T) { testCases := []struct { desc string failed bool + chatty bool + output string f func(*B) }{{ desc: "simulate sequential run of subbenchmarks.", @@ -371,7 +421,34 @@ func TestBRun(t *T) { }, { desc: "failure carried over to root", failed: true, + output: "--- FAIL: root", f: func(b *B) { b.Fail() }, + }, { + desc: "skipping without message, chatty", + chatty: true, + output: "--- SKIP: root", + f: func(b *B) { b.SkipNow() }, + }, { + desc: "skipping with message, chatty", + chatty: true, + output: ` +--- SKIP: root + sub_test.go:: skipping`, + f: func(b *B) { b.Skip("skipping") }, + }, { + desc: "skipping without message, not chatty", + f: func(b *B) { b.SkipNow() }, + }, { + desc: "skipping after error", + failed: true, + output: ` +--- FAIL: root + sub_test.go:nnn: an error + sub_test.go:nnn: skipped`, + f: func(b *B) { + b.Error("an error") + b.Skip("skipped") + }, }, { desc: "memory allocation", f: func(b *B) { @@ -398,11 +475,15 @@ func TestBRun(t *T) { }} for _, tc := range testCases { var ok bool + buf := &bytes.Buffer{} // This is almost like the Benchmark function, except that we override // the benchtime and catch the failure result of the subbenchmark. root := &B{ common: common{ signal: make(chan bool), + name: "root", + w: buf, + chatty: tc.chatty, }, benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure. benchTime: time.Microsecond, @@ -418,5 +499,30 @@ func TestBRun(t *T) { if root.result.N != 1 { t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N) } + got := sanitizeLog(buf.String()) + want := sanitizeLog(tc.output) + if got != want { + t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) + } } } + +// sanitizeLog removes line numbers from log entries. +func sanitizeLog(s string) string { + s = strings.TrimSpace(s) + lines := strings.Split(s, "\n") + for i, line := range lines { + p := strings.IndexByte(line, ':') + if p > 0 && line[p+4] == ':' { // assuming 3-digit file positions + lines[i] = line[:p+1] + line[p+4:] + } + } + return strings.Join(lines, "\n") +} + +func TestBenchmarkOutput(t *T) { + // Ensure Benchmark initialized common.w by invoking it with an error and + // normal case. + Benchmark(func(b *B) { b.Error("do not print this output") }) + Benchmark(func(b *B) {}) +} diff --git a/src/testing/testing.go b/src/testing/testing.go index f9bb43b618..8e16db321d 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -199,6 +199,7 @@ type common struct { mu sync.RWMutex // guards output and failed output []byte // Output generated by test or benchmark. w io.Writer // For flushToParent. + chatty bool // A copy of the chatty flag. failed bool // Test or benchmark has failed. skipped bool // Test of benchmark has been skipped. finished bool @@ -265,7 +266,6 @@ func (c *common) flushToParent(format string, args ...interface{}) { defer p.mu.Unlock() fmt.Fprintf(p.w, format, args...) - fmt.Fprintln(p.w) c.mu.Lock() defer c.mu.Unlock() @@ -562,13 +562,18 @@ func (t *T) Run(name string, f func(t *T)) bool { name: testName, parent: &t.common, level: t.level + 1, + chatty: t.chatty, }, context: t.context, } t.w = indenter{&t.common} - if *chatty { - fmt.Printf("=== RUN %s\n", t.name) + if t.chatty { + // Print directly to root's io.Writer so there is no delay. + root := t.parent + for ; root.parent != nil; root = t.parent { + } + fmt.Fprintf(root.w, "=== RUN %s\n", t.name) } // Instead of reducing the running count of this test before calling the // tRunner and increasing it afterwards, we rely on tRunner keeping the @@ -690,10 +695,10 @@ func (t *T) report() { return } dstr := fmtDuration(t.duration) - format := "--- %s: %s (%s)" + format := "--- %s: %s (%s)\n" if t.Failed() { t.flushToParent(format, "FAIL", t.name, dstr) - } else if *chatty { + } else if t.chatty { if t.Skipped() { t.flushToParent(format, "SKIP", t.name, dstr) } else { @@ -716,6 +721,7 @@ func RunTests(matchString func(pat, str string) (bool, error), tests []InternalT signal: make(chan bool), barrier: make(chan bool), w: os.Stdout, + chatty: *chatty, }, context: ctx, } -- cgit v1.3 From 63cea5ac2b8ed0cf257c7bfe7ed13bdd42373a0c Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Wed, 6 Apr 2016 09:59:32 +0200 Subject: testing: fixed bug introduced by CL 21504 This broke T.Run Change-Id: I12c8fe3612f3fa2caa83049c1c7003056daf2b0c Reviewed-on: https://go-review.googlesource.com/21600 Run-TryBot: Marcel van Lohuizen TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/testing/sub_test.go | 24 ++++++++++++++++++++++++ src/testing/testing.go | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) (limited to 'src/testing') diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index e053a3c348..7fe0fffd8f 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -178,6 +178,22 @@ func TestTRun(t *T) { === RUN skipping without message, chatty --- SKIP: skipping without message, chatty (0.00s)`, f: func(t *T) { t.SkipNow() }, + }, { + desc: "chatty with recursion", + ok: true, + chatty: true, + output: ` +=== RUN chatty with recursion +=== RUN chatty with recursion/#00 +=== RUN chatty with recursion/#00/#00 +--- PASS: chatty with recursion (0.00s) + --- PASS: chatty with recursion/#00 (0.00s) + --- PASS: chatty with recursion/#00/#00 (0.00s)`, + f: func(t *T) { + t.Run("", func(t *T) { + t.Run("", func(t *T) {}) + }) + }, }, { desc: "skipping without message, not chatty", ok: true, @@ -435,6 +451,14 @@ func TestBRun(t *T) { --- SKIP: root sub_test.go:: skipping`, f: func(b *B) { b.Skip("skipping") }, + }, { + desc: "chatty with recursion", + chatty: true, + f: func(b *B) { + b.Run("", func(b *B) { + b.Run("", func(b *B) {}) + }) + }, }, { desc: "skipping without message, not chatty", f: func(b *B) { b.SkipNow() }, diff --git a/src/testing/testing.go b/src/testing/testing.go index 8e16db321d..3a7a135a3c 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -571,7 +571,7 @@ func (t *T) Run(name string, f func(t *T)) bool { if t.chatty { // Print directly to root's io.Writer so there is no delay. root := t.parent - for ; root.parent != nil; root = t.parent { + for ; root.parent != nil; root = root.parent { } fmt.Fprintf(root.w, "=== RUN %s\n", t.name) } -- cgit v1.3 From 5176a4b39b4595e5d9025e7aaf19146c29b7e349 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Wed, 6 Apr 2016 15:01:50 +0200 Subject: testing: fix flakey test on plan9 allow for more than 0.00s. Fixes #15149 Change-Id: I1d428a9b3c9bb3d1db8682c53b86e44cecc1dde1 Reviewed-on: https://go-review.googlesource.com/21602 Reviewed-by: Brad Fitzpatrick Run-TryBot: Marcel van Lohuizen TryBot-Result: Gobot Gobot --- src/testing/sub_test.go | 59 ++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) (limited to 'src/testing') diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 7fe0fffd8f..4f26a53ab6 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -122,8 +122,8 @@ func TestTRun(t *T) { ok: false, maxPar: 1, output: ` ---- FAIL: failnow skips future sequential and parallel tests at same level (0.00s) - --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (0.00s) +--- FAIL: failnow skips future sequential and parallel tests at same level (N.NNs) + --- FAIL: failnow skips future sequential and parallel tests at same level/#00 (N.NNs) `, f: func(t *T) { ranSeq := false @@ -157,9 +157,9 @@ func TestTRun(t *T) { ok: false, maxPar: 1, output: ` ---- FAIL: failure in parallel test propagates upwards (0.00s) - --- FAIL: failure in parallel test propagates upwards/#00 (0.00s) - --- FAIL: failure in parallel test propagates upwards/#00/par (0.00s) +--- FAIL: failure in parallel test propagates upwards (N.NNs) + --- FAIL: failure in parallel test propagates upwards/#00 (N.NNs) + --- FAIL: failure in parallel test propagates upwards/#00/par (N.NNs) `, f: func(t *T) { t.Run("", func(t *T) { @@ -176,7 +176,7 @@ func TestTRun(t *T) { chatty: true, output: ` === RUN skipping without message, chatty ---- SKIP: skipping without message, chatty (0.00s)`, +--- SKIP: skipping without message, chatty (N.NNs)`, f: func(t *T) { t.SkipNow() }, }, { desc: "chatty with recursion", @@ -186,9 +186,9 @@ func TestTRun(t *T) { === RUN chatty with recursion === RUN chatty with recursion/#00 === RUN chatty with recursion/#00/#00 ---- PASS: chatty with recursion (0.00s) - --- PASS: chatty with recursion/#00 (0.00s) - --- PASS: chatty with recursion/#00/#00 (0.00s)`, +--- PASS: chatty with recursion (N.NNs) + --- PASS: chatty with recursion/#00 (N.NNs) + --- PASS: chatty with recursion/#00/#00 (N.NNs)`, f: func(t *T) { t.Run("", func(t *T) { t.Run("", func(t *T) {}) @@ -201,9 +201,9 @@ func TestTRun(t *T) { }, { desc: "skipping after error", output: ` ---- FAIL: skipping after error (0.00s) - sub_test.go:nnn: an error - sub_test.go:nnn: skipped`, +--- FAIL: skipping after error (N.NNs) + sub_test.go:NNN: an error + sub_test.go:NNN: skipped`, f: func(t *T) { t.Error("an error") t.Skip("skipped") @@ -383,9 +383,10 @@ func TestTRun(t *T) { if ctx.running != 0 || ctx.numWaiting != 0 { t.Errorf("%s:running and waiting non-zero: got %d and %d", tc.desc, ctx.running, ctx.numWaiting) } - got := sanitizeLog(buf.String()) - want := sanitizeLog(tc.output) - if got != want { + got := strings.TrimSpace(buf.String()) + want := strings.TrimSpace(tc.output) + re := makeRegexp(want) + if ok, err := regexp.MatchString(re, got); !ok || err != nil { t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) } } @@ -449,7 +450,7 @@ func TestBRun(t *T) { chatty: true, output: ` --- SKIP: root - sub_test.go:: skipping`, + sub_test.go:NNN: skipping`, f: func(b *B) { b.Skip("skipping") }, }, { desc: "chatty with recursion", @@ -467,8 +468,8 @@ func TestBRun(t *T) { failed: true, output: ` --- FAIL: root - sub_test.go:nnn: an error - sub_test.go:nnn: skipped`, + sub_test.go:NNN: an error + sub_test.go:NNN: skipped`, f: func(b *B) { b.Error("an error") b.Skip("skipped") @@ -523,25 +524,19 @@ func TestBRun(t *T) { if root.result.N != 1 { t.Errorf("%s: N for parent benchmark was %d; want 1", tc.desc, root.result.N) } - got := sanitizeLog(buf.String()) - want := sanitizeLog(tc.output) - if got != want { + got := strings.TrimSpace(buf.String()) + want := strings.TrimSpace(tc.output) + re := makeRegexp(want) + if ok, err := regexp.MatchString(re, got); !ok || err != nil { t.Errorf("%s:ouput:\ngot:\n%s\nwant:\n%s", tc.desc, got, want) } } } -// sanitizeLog removes line numbers from log entries. -func sanitizeLog(s string) string { - s = strings.TrimSpace(s) - lines := strings.Split(s, "\n") - for i, line := range lines { - p := strings.IndexByte(line, ':') - if p > 0 && line[p+4] == ':' { // assuming 3-digit file positions - lines[i] = line[:p+1] + line[p+4:] - } - } - return strings.Join(lines, "\n") +func makeRegexp(s string) string { + s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1) + s = strings.Replace(s, "(N.NNs)", `\(\d*\.\d*s\)`, -1) + return s } func TestBenchmarkOutput(t *T) { -- cgit v1.3 From 0ec6d7c0bbfceb7b8e4857b775686ae5cf699e54 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Thu, 14 Apr 2016 15:44:48 +0800 Subject: testing: removed flakey test The synchronization in this test is a bit complicated and likely incorrect, judging from the sporadically hanging trybots. Most of what this is supposed to test is already tested in TestTestContext, so I'll just remove it. Fixes #15170 Change-Id: If54db977503caa109cec4516974eda9191051888 Reviewed-on: https://go-review.googlesource.com/22080 Run-TryBot: Marcel van Lohuizen TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/testing/sub_test.go | 51 ------------------------------------------------- 1 file changed, 51 deletions(-) (limited to 'src/testing') diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 4f26a53ab6..2804550737 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -226,57 +226,6 @@ func TestTRun(t *T) { t.Errorf("count was %d; want 4", count) } }, - }, { - desc: "run no more than *parallel tests concurrently", - ok: true, - maxPar: 4, - f: func(t *T) { - max := 0 - in := make(chan int) - out := make(chan int) - ctx := t.context - t.Run("wait", func(t *T) { - t.Run("controller", func(t *T) { - // Verify sequential tests don't skew counts. - t.Run("seq1", func(t *T) {}) - t.Run("seq2", func(t *T) {}) - t.Run("seq3", func(t *T) {}) - t.Parallel() - for i := 0; i < 80; i++ { - ctx.mu.Lock() - if ctx.running > max { - max = ctx.running - } - ctx.mu.Unlock() - <-in - // force a minimum to avoid a race, although it works - // without it. - if i >= ctx.maxParallel-2 { // max - this - 1 - out <- i - } - } - close(out) - }) - // Ensure we don't exceed the maximum even with nested parallelism. - for i := 0; i < 2; i++ { - t.Run("", func(t *T) { - t.Parallel() - for j := 0; j < 40; j++ { - t.Run("", func(t *T) { - t.Run("seq1", func(t *T) {}) - t.Run("seq2", func(t *T) {}) - t.Parallel() - in <- j - <-out - }) - } - }) - } - }) - if max != ctx.maxParallel { - realTest.Errorf("max: got %d; want: %d", max, ctx.maxParallel) - } - }, }, { desc: "alternate sequential and parallel", // Sequential tests should partake in the counting of running threads. -- cgit v1.3 From a683c385ad874b0066787dc010cacba8aaff894c Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Fri, 29 Jan 2016 16:16:03 +0100 Subject: testing: add matching of subtest Allows passing regexps per subtest to --test.run and --test.bench Note that the documentation explicitly states that the split regular expressions match the correpsonding parts (path components) of the bench/test identifier. This is intended and slightly different from the i'th RE matching the subtest/subbench at the respective level. Picking this semantics allows guaranteeing that a test or benchmark identifier as printed by go test can be passed verbatim (possibly quoted) to, respectively, -run or -bench: subtests and subbenches might have a '/' in their name, causing a misaligment if their ID is passed to -run or -bench as is. This semantics has other benefits, but this is the main motivation. Fixes golang.go#15126 Change-Id: If72e6d3f54db1df6bc2729ac6edc7ab3c740e7c3 Reviewed-on: https://go-review.googlesource.com/19122 Reviewed-by: Russ Cox Run-TryBot: Marcel van Lohuizen TryBot-Result: Gobot Gobot --- src/cmd/go/alldocs.go | 17 ++++--- src/cmd/go/test.go | 17 ++++--- src/testing/match.go | 71 ++++++++++++++++++++++++---- src/testing/match_test.go | 118 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 24 deletions(-) (limited to 'src/testing') diff --git a/src/cmd/go/alldocs.go b/src/cmd/go/alldocs.go index 090b207db7..aa1f029939 100644 --- a/src/cmd/go/alldocs.go +++ b/src/cmd/go/alldocs.go @@ -1348,9 +1348,12 @@ The following flags are recognized by the 'go test' command and control the execution of any test: -bench regexp - Run benchmarks matching the regular expression. - By default, no benchmarks run. - To run all benchmarks, use '-bench=.'. + Run (sub)benchmarks matching a regular expression. + The given regular expression is split into smaller ones by + top-level '/', where each must match the corresponding part of a + benchmark's identifier. + By default, no benchmarks run. To run all benchmarks, + use '-bench .' or '-bench=.'. -benchmem Print memory allocation statistics for benchmarks. @@ -1436,10 +1439,10 @@ control the execution of any test: (see 'go help build'). -run regexp - Run only those tests and examples matching the regular - expression. By default, all tests run. - To skip all tests, use a pattern that matches no test names, - such as '-run=^$'. + Run only those tests and examples matching the regular expression. + For tests the regular expression is split into smaller ones by + top-level '/', where each must match the corresponding part of a + test's identifier. -short Tell long-running tests to shorten their run time. diff --git a/src/cmd/go/test.go b/src/cmd/go/test.go index 8dbd9e22bf..0c87fca556 100644 --- a/src/cmd/go/test.go +++ b/src/cmd/go/test.go @@ -125,9 +125,12 @@ control the execution of any test: const testFlag2 = ` -bench regexp - Run benchmarks matching the regular expression. - By default, no benchmarks run. - To run all benchmarks, use '-bench=.'. + Run (sub)benchmarks matching a regular expression. + The given regular expression is split into smaller ones by + top-level '/', where each must match the corresponding part of a + benchmark's identifier. + By default, no benchmarks run. To run all benchmarks, + use '-bench .' or '-bench=.'. -benchmem Print memory allocation statistics for benchmarks. @@ -213,10 +216,10 @@ const testFlag2 = ` (see 'go help build'). -run regexp - Run only those tests and examples matching the regular - expression. By default, all tests run. - To skip all tests, use a pattern that matches no test names, - such as '-run=^$'. + Run only those tests and examples matching the regular expression. + For tests the regular expression is split into smaller ones by + top-level '/', where each must match the corresponding part of a + test's identifier. -short Tell long-running tests to shorten their run time. diff --git a/src/testing/match.go b/src/testing/match.go index d0c52142ba..7751035760 100644 --- a/src/testing/match.go +++ b/src/testing/match.go @@ -8,29 +8,40 @@ import ( "fmt" "os" "strconv" + "strings" "sync" ) // matcher sanitizes, uniques, and filters names of subtests and subbenchmarks. type matcher struct { - filter string + filter []string matchFunc func(pat, str string) (bool, error) mu sync.Mutex subNames map[string]int64 } -// TODO: fix test_main to avoid race and improve caching. +// TODO: fix test_main to avoid race and improve caching, also allowing to +// eliminate this Mutex. var matchMutex sync.Mutex -func newMatcher(matchString func(pat, str string) (bool, error), pattern, name string) *matcher { - // Verify filters before doing any processing. - if _, err := matchString(pattern, "non-empty"); err != nil { - fmt.Fprintf(os.Stderr, "testing: invalid regexp for %s: %s\n", name, err) - os.Exit(1) +func newMatcher(matchString func(pat, str string) (bool, error), patterns, name string) *matcher { + var filter []string + if patterns != "" { + filter = splitRegexp(patterns) + for i, s := range filter { + filter[i] = rewrite(s) + } + // Verify filters before doing any processing. + for i, s := range filter { + if _, err := matchString(s, "non-empty"); err != nil { + fmt.Fprintf(os.Stderr, "testing: invalid regexp for element %d of %s (%q): %s\n", i, name, s, err) + os.Exit(1) + } + } } return &matcher{ - filter: pattern, + filter: filter, matchFunc: matchString, subNames: map[string]int64{}, } @@ -49,14 +60,54 @@ func (m *matcher) fullName(c *common, subname string) (name string, ok bool) { matchMutex.Lock() defer matchMutex.Unlock() - if c != nil && c.level == 0 { - if matched, _ := m.matchFunc(m.filter, subname); !matched { + // We check the full array of paths each time to allow for the case that + // a pattern contains a '/'. + for i, s := range strings.Split(name, "/") { + if i >= len(m.filter) { + break + } + if ok, _ := m.matchFunc(m.filter[i], s); !ok { return name, false } } return name, true } +func splitRegexp(s string) []string { + a := make([]string, 0, strings.Count(s, "/")) + cs := 0 + cp := 0 + for i := 0; i < len(s); { + switch s[i] { + case '[': + cs++ + case ']': + if cs--; cs < 0 { // An unmatched ']' is legal. + cs = 0 + } + case '(': + if cs == 0 { + cp++ + } + case ')': + if cs == 0 { + cp-- + } + case '\\': + i++ + case '/': + if cs == 0 && cp == 0 { + a = append(a, s[:i]) + s = s[i+1:] + i = 0 + continue + } + } + i++ + } + return append(a, s) +} + // unique creates a unique name for the given parent and subname by affixing it // with one ore more counts, if necessary. func (m *matcher) unique(parent, subname string) string { diff --git a/src/testing/match_test.go b/src/testing/match_test.go index 68f3e9e867..d19036c72d 100644 --- a/src/testing/match_test.go +++ b/src/testing/match_test.go @@ -5,6 +5,7 @@ package testing import ( + "reflect" "regexp" "unicode" ) @@ -23,6 +24,123 @@ func TestIsSpace(t *T) { } } +func TestSplitRegexp(t *T) { + res := func(s ...string) []string { return s } + testCases := []struct { + pattern string + result []string + }{ + // Correct patterns + // If a regexp pattern is correct, all split regexps need to be correct + // as well. + {"", res("")}, + {"/", res("", "")}, + {"//", res("", "", "")}, + {"A", res("A")}, + {"A/B", res("A", "B")}, + {"A/B/", res("A", "B", "")}, + {"/A/B/", res("", "A", "B", "")}, + {"[A]/(B)", res("[A]", "(B)")}, + {"[/]/[/]", res("[/]", "[/]")}, + {"[/]/[:/]", res("[/]", "[:/]")}, + {"/]", res("", "]")}, + {"]/", res("]", "")}, + {"]/[/]", res("]", "[/]")}, + {`([)/][(])`, res(`([)/][(])`)}, + {"[(]/[)]", res("[(]", "[)]")}, + + // Faulty patterns + // Errors in original should produce at least one faulty regexp in results. + {")/", res(")/")}, + {")/(/)", res(")/(", ")")}, + {"a[/)b", res("a[/)b")}, + {"(/]", res("(/]")}, + {"(/", res("(/")}, + {"[/]/[/", res("[/]", "[/")}, + {`\p{/}`, res(`\p{`, "}")}, + {`\p/`, res(`\p`, "")}, + {`[[:/:]]`, res(`[[:/:]]`)}, + } + for _, tc := range testCases { + a := splitRegexp(tc.pattern) + if !reflect.DeepEqual(a, tc.result) { + t.Errorf("splitRegexp(%q) = %#v; want %#v", tc.pattern, a, tc.result) + } + + // If there is any error in the pattern, one of the returned subpatterns + // needs to have an error as well. + if _, err := regexp.Compile(tc.pattern); err != nil { + ok := true + for _, re := range a { + if _, err := regexp.Compile(re); err != nil { + ok = false + } + } + if ok { + t.Errorf("%s: expected error in any of %q", tc.pattern, a) + } + } + } +} + +func TestMatcher(t *T) { + testCases := []struct { + pattern string + parent, sub string + ok bool + }{ + // Behavior without subtests. + {"", "", "TestFoo", true}, + {"TestFoo", "", "TestFoo", true}, + {"TestFoo/", "", "TestFoo", true}, + {"TestFoo/bar/baz", "", "TestFoo", true}, + {"TestFoo", "", "TestBar", false}, + {"TestFoo/", "", "TestBar", false}, + {"TestFoo/bar/baz", "", "TestBar/bar/baz", false}, + + // with subtests + {"", "TestFoo", "x", true}, + {"TestFoo", "TestFoo", "x", true}, + {"TestFoo/", "TestFoo", "x", true}, + {"TestFoo/bar/baz", "TestFoo", "bar", true}, + // Subtest with a '/' in its name still allows for copy and pasted names + // to match. + {"TestFoo/bar/baz", "TestFoo", "bar/baz", true}, + {"TestFoo/bar/baz", "TestFoo/bar", "baz", true}, + {"TestFoo/bar/baz", "TestFoo", "x", false}, + {"TestFoo", "TestBar", "x", false}, + {"TestFoo/", "TestBar", "x", false}, + {"TestFoo/bar/baz", "TestBar", "x/bar/baz", false}, + + // subtests only + {"", "TestFoo", "x", true}, + {"/", "TestFoo", "x", true}, + {"./", "TestFoo", "x", true}, + {"./.", "TestFoo", "x", true}, + {"/bar/baz", "TestFoo", "bar", true}, + {"/bar/baz", "TestFoo", "bar/baz", true}, + {"//baz", "TestFoo", "bar/baz", true}, + {"//", "TestFoo", "bar/baz", true}, + {"/bar/baz", "TestFoo/bar", "baz", true}, + {"//foo", "TestFoo", "bar/baz", false}, + {"/bar/baz", "TestFoo", "x", false}, + {"/bar/baz", "TestBar", "x/bar/baz", false}, + } + + for _, tc := range testCases { + m := newMatcher(regexp.MatchString, tc.pattern, "-test.run") + + parent := &common{name: tc.parent} + if tc.parent != "" { + parent.level = 1 + } + if n, ok := m.fullName(parent, tc.sub); ok != tc.ok { + t.Errorf("pattern: %q, parent: %q, sub %q: got %v; want %v", + tc.pattern, tc.parent, tc.sub, ok, tc.ok, n) + } + } +} + func TestNaming(t *T) { m := newMatcher(regexp.MatchString, "", "") -- cgit v1.3