cmd/internal/test2json: generate and validate test artifacts

Adds a mechanism for generating test2json test artifacts from and validating them against a real test. If a .test file has a corresponding .src file, TestGolden will now treat the .src file as a script test, executing it and verifying that the output matches the contents of the .test file. Running TestGolden with the -update flag will also regenerate .test files if they have a corresponding .src file.

Capturing the output of the script test in this way required making minor changes to cmd/internal/script/scripttest.

This was motivated by CL 601535 (golang/go#62728). Specifically, testing that CL required adding src/cmd/internal/test2json/testdata/frameescape.test which has a multitude of non-printing characters and thus must be generated by executing `go test`. Using a script test to generate the test file is more reliable than doing it by hand.

Change-Id: I60456700e37e21a42d0514be2ce86dc6df2bccb0
Reviewed-on: https://go-review.googlesource.com/c/go/+/628615
Reviewed-by: Michael Matloob <matloob@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Matloob <matloob@google.com>
Reviewed-by: Damien Neil <dneil@google.com>
Auto-Submit: Damien Neil <dneil@google.com>

1 files changed, 62 insertions, 33 deletions

diff --git a/src/cmd/internal/script/scripttest/run.go b/src/cmd/internal/script/scripttest/run.go
index 2a909f656d..d235246a11 100644
--- a/src/cmd/internal/script/scripttest/run.go
+++ b/src/cmd/internal/script/scripttest/run.go
@@ -29,12 +29,9 @@ type ToolReplacement struct {
 	EnvVar          string // env var setting (e.g. "FOO=BAR")
 }
 
-// RunToolScriptTest kicks off a set of script tests runs for
-// a tool of some sort (compiler, linker, etc). The expectation
-// is that we'll be called from the top level cmd/X dir for tool X,
-// and that instead of executing the install tool X we'll use the
-// test binary instead.
-func RunToolScriptTest(t *testing.T, repls []ToolReplacement, scriptsdir string, fixReadme bool) {
+// NewEngine constructs a new [script.Engine] and environment to be used with
+// [RunTests].
+func NewEngine(t *testing.T, repls []ToolReplacement) (*script.Engine, []string) {
 	// Nearly all script tests involve doing builds, so don't
 	// bother here if we don't have "go build".
 	testenv.MustHaveGoBuild(t)
@@ -156,6 +153,23 @@ func RunToolScriptTest(t *testing.T, repls []ToolReplacement, scriptsdir string,
 		Quiet: !testing.Verbose(),
 	}
 
+	return engine, env
+}
+
+// RunToolScriptTest kicks off a set of script tests runs for
+// a tool of some sort (compiler, linker, etc). The expectation
+// is that we'll be called from the top level cmd/X dir for tool X,
+// and that instead of executing the install tool X we'll use the
+// test binary instead.
+func RunToolScriptTest(t *testing.T, repls []ToolReplacement, scriptsdir string, fixReadme bool) {
+	// Locate our Go tool.
+	gotool, err := testenv.GoTool()
+	if err != nil {
+		t.Fatalf("locating go tool: %v", err)
+	}
+
+	engine, env := NewEngine(t, repls)
+
 	t.Run("README", func(t *testing.T) {
 		checkScriptReadme(t, engine, env, scriptsdir, gotool, fixReadme)
 	})
@@ -166,36 +180,46 @@ func RunToolScriptTest(t *testing.T, repls []ToolReplacement, scriptsdir string,
 	RunTests(t, ctx, engine, env, pattern)
 }
 
+// ScriptTestContext returns a context with a grace period for cleaning up
+// subprocesses of a script test.
+//
+// When we run commands that execute subprocesses, we want to reserve two grace
+// periods to clean up. We will send the first termination signal when the
+// context expires, then wait one grace period for the process to produce
+// whatever useful output it can (such as a stack trace). After the first grace
+// period expires, we'll escalate to os.Kill, leaving the second grace period
+// for the test function to record its output before the test process itself
+// terminates.
+//
+// The grace period is 100ms or 5% of the time remaining until
+// [testing.T.Deadline], whichever is greater.
+func ScriptTestContext(t *testing.T, ctx context.Context) context.Context {
+	deadline, ok := t.Deadline()
+	if !ok {
+		return ctx
+	}
+
+	gracePeriod := 100 * time.Millisecond
+	timeout := time.Until(deadline)
+
+	// If time allows, increase the termination grace period to 5% of the
+	// remaining time.
+	gracePeriod = max(gracePeriod, timeout/20)
+
+	// Reserve two grace periods to clean up
+	timeout -= 2 * gracePeriod
+
+	ctx, cancel := context.WithTimeout(ctx, timeout)
+	t.Cleanup(cancel)
+	return ctx
+}
+
 // RunTests kicks off one or more script-based tests using the
 // specified engine, running all test files that match pattern.
 // This function adapted from Russ's rsc.io/script/scripttest#Run
 // function, which was in turn forked off cmd/go's runner.
 func RunTests(t *testing.T, ctx context.Context, engine *script.Engine, env []string, pattern string) {
-	gracePeriod := 100 * time.Millisecond
-	if deadline, ok := t.Deadline(); ok {
-		timeout := time.Until(deadline)
-
-		// If time allows, increase the termination grace period to 5% of the
-		// remaining time.
-		if gp := timeout / 20; gp > gracePeriod {
-			gracePeriod = gp
-		}
-
-		// When we run commands that execute subprocesses, we want to
-		// reserve two grace periods to clean up. We will send the
-		// first termination signal when the context expires, then
-		// wait one grace period for the process to produce whatever
-		// useful output it can (such as a stack trace). After the
-		// first grace period expires, we'll escalate to os.Kill,
-		// leaving the second grace period for the test function to
-		// record its output before the test process itself
-		// terminates.
-		timeout -= 2 * gracePeriod
-
-		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, timeout)
-		t.Cleanup(cancel)
-	}
+	ctx = ScriptTestContext(t, ctx)
 
 	files, _ := filepath.Glob(pattern)
 	if len(files) == 0 {
@@ -218,7 +242,7 @@ func RunTests(t *testing.T, ctx context.Context, engine *script.Engine, env []st
 			if err != nil {
 				t.Fatal(err)
 			}
-			initScriptDirs(t, s)
+			InitScriptDirs(t, s)
 			if err := s.ExtractFiles(a); err != nil {
 				t.Fatal(err)
 			}
@@ -237,7 +261,12 @@ func RunTests(t *testing.T, ctx context.Context, engine *script.Engine, env []st
 	}
 }
 
-func initScriptDirs(t testing.TB, s *script.State) {
+// InitScriptDirs sets up directories for executing a script test.
+//
+//   - WORK (env var) is set to the current working directory.
+//   - TMPDIR (env var; TMP on Windows) is set to $WORK/tmp.
+//   - $TMPDIR is created.
+func InitScriptDirs(t testing.TB, s *script.State) {
 	must := func(err error) {
 		if err != nil {
 			t.Helper()