[dev.fuzz] testing: let -fuzztime specify a number of executions

-fuzztime now works similarly to -benchtime: if it's given a string
with an "x" suffix (as opposed to "s" or some other unit of
duration), the fuzzing system will generate and run a maximum number
of values.

This CL also implements tracking and printing counts, since most of
the work was already done.

Change-Id: I013007984b5adfc1a751c379dc98c8d46b4a97e9
Reviewed-on: https://go-review.googlesource.com/c/go/+/306909
Trust: Jay Conrod <jayconrod@google.com>
Trust: Katie Hockman <katie@golang.org>
Run-TryBot: Jay Conrod <jayconrod@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Katie Hockman <katie@golang.org>

5 files changed, 16 insertions, 22 deletions

diff --git a/src/testing/benchmark.go b/src/testing/benchmark.go
index a8f75e9712..ac22ac5b26 100644
--- a/src/testing/benchmark.go
+++ b/src/testing/benchmark.go
@@ -32,35 +32,35 @@ var (
 	matchBenchmarks *string
 	benchmarkMemory *bool
 
-	benchTime = benchTimeFlag{d: 1 * time.Second} // changed during test of testing package
+	benchTime = durationOrCountFlag{d: 1 * time.Second} // changed during test of testing package
 )
 
-type benchTimeFlag struct {
+type durationOrCountFlag struct {
 	d time.Duration
 	n int
 }
 
-func (f *benchTimeFlag) String() string {
+func (f *durationOrCountFlag) String() string {
 	if f.n > 0 {
 		return fmt.Sprintf("%dx", f.n)
 	}
 	return time.Duration(f.d).String()
 }
 
-func (f *benchTimeFlag) Set(s string) error {
+func (f *durationOrCountFlag) Set(s string) error {
 	if strings.HasSuffix(s, "x") {
 		n, err := strconv.ParseInt(s[:len(s)-1], 10, 0)
 		if err != nil || n <= 0 {
 			return fmt.Errorf("invalid count")
 		}
-		*f = benchTimeFlag{n: int(n)}
+		*f = durationOrCountFlag{n: int(n)}
 		return nil
 	}
 	d, err := time.ParseDuration(s)
 	if err != nil || d <= 0 {
 		return fmt.Errorf("invalid duration")
 	}
-	*f = benchTimeFlag{d: d}
+	*f = durationOrCountFlag{d: d}
 	return nil
 }
 
@@ -98,7 +98,7 @@ type B struct {
 	previousN        int           // number of iterations in the previous run
 	previousDuration time.Duration // total duration of the previous run
 	benchFunc        func(b *B)
-	benchTime        benchTimeFlag
+	benchTime        durationOrCountFlag
 	bytes            int64
 	missingBytes     bool // one of the subbenchmarks does not have bytes set.
 	timerOn          bool
diff --git a/src/testing/fuzz.go b/src/testing/fuzz.go
index 73ac59cfb4..0c1280c656 100644
--- a/src/testing/fuzz.go
+++ b/src/testing/fuzz.go
@@ -18,14 +18,14 @@ import (
 
 func initFuzzFlags() {
 	matchFuzz = flag.String("test.fuzz", "", "run the fuzz target matching `regexp`")
-	fuzzDuration = flag.Duration("test.fuzztime", 0, "time to spend fuzzing; default (0) is to run indefinitely")
+	flag.Var(&fuzzDuration, "test.fuzztime", "time to spend fuzzing default is to run indefinitely")
 	fuzzCacheDir = flag.String("test.fuzzcachedir", "", "directory where interesting fuzzing inputs are stored")
 	isFuzzWorker = flag.Bool("test.fuzzworker", false, "coordinate with the parent process to fuzz random values")
 }
 
 var (
 	matchFuzz    *string
-	fuzzDuration *time.Duration
+	fuzzDuration durationOrCountFlag
 	fuzzCacheDir *string
 	isFuzzWorker *bool
 
@@ -358,7 +358,7 @@ func (f *F) Fuzz(ff interface{}) {
 		// actual fuzzing.
 		corpusTargetDir := filepath.Join(corpusDir, f.name)
 		cacheTargetDir := filepath.Join(*fuzzCacheDir, f.name)
-		err := f.fuzzContext.coordinateFuzzing(*fuzzDuration, *parallel, f.corpus, types, corpusTargetDir, cacheTargetDir)
+		err := f.fuzzContext.coordinateFuzzing(fuzzDuration.d, int64(fuzzDuration.n), *parallel, f.corpus, types, corpusTargetDir, cacheTargetDir)
 		if err != nil {
 			f.result = FuzzResult{Error: err}
 			f.Fail()
@@ -452,7 +452,7 @@ type fuzzCrashError interface {
 // fuzzContext holds all fields that are common to all fuzz targets.
 type fuzzContext struct {
 	importPath        func() string
-	coordinateFuzzing func(time.Duration, int, []corpusEntry, []reflect.Type, string, string) error
+	coordinateFuzzing func(time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error
 	runFuzzWorker     func(func(corpusEntry) error) error
 	readCorpus        func(string, []reflect.Type) ([]corpusEntry, error)
 }
diff --git a/src/testing/internal/testdeps/deps.go b/src/testing/internal/testdeps/deps.go
index c77aca3da8..73c61fb54f 100644
--- a/src/testing/internal/testdeps/deps.go
+++ b/src/testing/internal/testdeps/deps.go
@@ -133,13 +133,13 @@ func (TestDeps) SetPanicOnExit0(v bool) {
 	testlog.SetPanicOnExit0(v)
 }
 
-func (TestDeps) CoordinateFuzzing(timeout time.Duration, parallel int, seed []fuzz.CorpusEntry, types []reflect.Type, corpusDir, cacheDir string) (err error) {
+func (TestDeps) CoordinateFuzzing(timeout time.Duration, count int64, parallel int, seed []fuzz.CorpusEntry, types []reflect.Type, corpusDir, cacheDir string) (err error) {
 	// Fuzzing may be interrupted with a timeout or if the user presses ^C.
 	// In either case, we'll stop worker processes gracefully and save
 	// crashers and interesting values.
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer cancel()
-	err = fuzz.CoordinateFuzzing(ctx, timeout, parallel, seed, types, corpusDir, cacheDir)
+	err = fuzz.CoordinateFuzzing(ctx, os.Stderr, timeout, count, parallel, seed, types, corpusDir, cacheDir)
 	if err == ctx.Err() {
 		return nil
 	}
diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go
index 5b226f85ad..d2b966dcf9 100644
--- a/src/testing/sub_test.go
+++ b/src/testing/sub_test.go
@@ -669,7 +669,7 @@ func TestBRun(t *T) {
 					w:      buf,
 				},
 				benchFunc: func(b *B) { ok = b.Run("test", tc.f) }, // Use Run to catch failure.
-				benchTime: benchTimeFlag{d: 1 * time.Microsecond},
+				benchTime: durationOrCountFlag{d: 1 * time.Microsecond},
 			}
 			if tc.chatty {
 				root.chatty = newChattyPrinter(root.w)
diff --git a/src/testing/testing.go b/src/testing/testing.go
index 2ba93ad63d..48e9ee089f 100644
--- a/src/testing/testing.go
+++ b/src/testing/testing.go
@@ -1326,7 +1326,7 @@ func (f matchStringOnly) ImportPath() string                          { return "
 func (f matchStringOnly) StartTestLog(io.Writer)                      {}
 func (f matchStringOnly) StopTestLog() error                          { return errMain }
 func (f matchStringOnly) SetPanicOnExit0(bool)                        {}
-func (f matchStringOnly) CoordinateFuzzing(time.Duration, int, []corpusEntry, []reflect.Type, string, string) error {
+func (f matchStringOnly) CoordinateFuzzing(time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error {
 	return errMain
 }
 func (f matchStringOnly) RunFuzzWorker(func(corpusEntry) error) error { return errMain }
@@ -1375,7 +1375,7 @@ type testDeps interface {
 	StartTestLog(io.Writer)
 	StopTestLog() error
 	WriteProfileTo(string, io.Writer, int) error
-	CoordinateFuzzing(time.Duration, int, []corpusEntry, []reflect.Type, string, string) error
+	CoordinateFuzzing(time.Duration, int64, int, []corpusEntry, []reflect.Type, string, string) error
 	RunFuzzWorker(func(corpusEntry) error) error
 	ReadCorpus(string, []reflect.Type) ([]corpusEntry, error)
 }
@@ -1417,12 +1417,6 @@ func (m *M) Run() (code int) {
 		m.exitCode = 2
 		return
 	}
-	if *fuzzDuration < 0 {
-		fmt.Fprintln(os.Stderr, "testing: -fuzztime can only be given a positive duration, or zero to run indefinitely")
-		flag.Usage()
-		m.exitCode = 2
-		return
-	}
 	if *matchFuzz != "" && *fuzzCacheDir == "" {
 		fmt.Fprintln(os.Stderr, "testing: internal error: -test.fuzzcachedir must be set if -test.fuzz is set")
 		flag.Usage()