From cbb3f090477de92a7e158050803ef71a5ea825ee Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Wed, 14 Apr 2021 23:55:38 +0300 Subject: testing: add -shuffle=off|on|N to alter the execution order of tests and benchmarks This CL adds a new flag to the testing package and the go test command which randomizes the execution order for tests and benchmarks. This can be useful for identifying unwanted dependencies between test or benchmark functions. The flag is off by default. If `-shuffle` is set to `on` then the system clock will be used as the seed value. If `-shuffle` is set to an integer N, then N will be used as the seed value. In both cases, the seed will be reported for failed runs so that they can reproduced later on. Fixes #28592 Change-Id: I62e7dfae5f63f97a0cbd7830ea844d9f7beac335 Reviewed-on: https://go-review.googlesource.com/c/go/+/310033 Run-TryBot: Ian Lance Taylor TryBot-Result: Go Bot Reviewed-by: Emmanuel Odeke Trust: Bryan C. Mills --- src/testing/testing.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'src/testing') diff --git a/src/testing/testing.go b/src/testing/testing.go index 1562eadef0..85a7fec65f 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -242,6 +242,7 @@ import ( "fmt" "internal/race" "io" + "math/rand" "os" "runtime" "runtime/debug" @@ -299,6 +300,7 @@ func Init() { cpuListStr = flag.String("test.cpu", "", "comma-separated `list` of cpu counts to run each test with") parallel = flag.Int("test.parallel", runtime.GOMAXPROCS(0), "run at most `n` tests in parallel") testlog = flag.String("test.testlogfile", "", "write test action log to `file` (for use only by cmd/go)") + shuffle = flag.String("test.shuffle", "off", "randomize the execution order of tests and benchmarks") initBenchmarkFlags() } @@ -325,6 +327,7 @@ var ( timeout *time.Duration cpuListStr *string parallel *int + shuffle *string testlog *string haveExamples bool // are there examples? @@ -1456,6 +1459,25 @@ func (m *M) Run() (code int) { return } + if *shuffle != "off" { + var n int64 + var err error + if *shuffle == "on" { + n = time.Now().UnixNano() + } else { + n, err = strconv.ParseInt(*shuffle, 10, 64) + if err != nil { + fmt.Fprintln(os.Stderr, `testing: -shuffle should be "off", "on", or a valid integer:`, err) + m.exitCode = 2 + return + } + } + fmt.Println("-test.shuffle", n) + rng := rand.New(rand.NewSource(n)) + rng.Shuffle(len(m.tests), func(i, j int) { m.tests[i], m.tests[j] = m.tests[j], m.tests[i] }) + rng.Shuffle(len(m.benchmarks), func(i, j int) { m.benchmarks[i], m.benchmarks[j] = m.benchmarks[j], m.benchmarks[i] }) + } + parseCpuList() m.before() -- cgit v1.3