cmd/dist: make toolchain build reproducible

- Build cmd with CGO_ENABLED=0. Doing so removes the C compiler
  toolchain from the reproducibility perimeter and also results in
  cmd/go and cmd/pprof binaries that are statically linked,
  so that they will run on a wider variety of systems.
  In particular the Linux versions will run on Alpine and NixOS
  without needing a simulation of libc.so.6.

  The potential downside of disabling cgo is that cmd/go and cmd/pprof
  use the pure Go network resolver instead of the host resolver on
  Unix systems. This means they will not be able to use non-DNS
  resolver mechanisms that may be specified in /etc/resolv.conf,
  such as mDNS. Neither program seems likely to need non-DNS names
  like those, however.

  macOS and Windows systems still use the host resolver, which they
  access without cgo.

- Build cmd with -trimpath when building a release.
  Doing so removes $GOPATH from the file name prefixes stored in the
  binary, so that the build directory does not leak into the final artifacts.

- When CC and CXX are empty, do not pick values to hard-code into
  the source tree and binaries. Instead, emit code that makes the
  right decision at runtime. In addition to reproducibility, this
  makes cross-compiled toolchains work better. A macOS toolchain
  cross-compiled on Linux will now correctly look for clang,
  instead of looking for gcc because it was built on Linux.

- Convert \ to / in file names stored in .a files.
  These are converted to / in the final binaries, but the hashes of
  the .a files affect the final build ID of the binaries. Without this
  change, builds of a Windows toolchain on Windows and non-Windows
  machines produce identical binaries except for the input hash part
  of the build ID.

- Due to the conversion of \ to / in .a files, convert back when
  reading inline bodies on Windows to preserve output file names
  in error messages.

Combined, these four changes (along with Go 1.20's removal of
installed pkg/**.a files and conversion of macOS net away from cgo)
make the output of make.bash fully reproducible, even when
cross-compiling: a released macOS toolchain built on Linux or Windows
will contain exactly the same bits as a released macOS toolchain
built on macOS.

The word "released" in the previous sentence is important.
For the build IDs in the binaries to work out the same on
both systems, a VERSION file must exist to provide a consistent
compiler build ID (instead of using a content hash of the binary).

For #24904.
Fixes #57007.

Change-Id: I665e1ef4ff207d6ff469452347dca5bfc81050e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/454836
Reviewed-by: Bryan Mills <bcmills@google.com>
Run-TryBot: Russ Cox <rsc@golang.org>
Auto-Submit: Russ Cox <rsc@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>

5 files changed, 131 insertions, 61 deletions

diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go
index 82a284c208..c15515f695 100644
--- a/src/cmd/dist/build.go
+++ b/src/cmd/dist/build.go
@@ -52,9 +52,8 @@ var (
 	defaultpkgconfig string
 	defaultldso      string
 
-	rebuildall   bool
-	defaultclang bool
-	noOpt        bool
+	rebuildall bool
+	noOpt      bool
 
 	vflag int // verbosity
 )
@@ -210,12 +209,8 @@ func xinit() {
 	gogcflags = os.Getenv("BOOT_GO_GCFLAGS")
 	goldflags = os.Getenv("BOOT_GO_LDFLAGS")
 
-	cc, cxx := "gcc", "g++"
-	if defaultclang {
-		cc, cxx = "clang", "clang++"
-	}
-	defaultcc = compilerEnv("CC", cc)
-	defaultcxx = compilerEnv("CXX", cxx)
+	defaultcc = compilerEnv("CC", "")
+	defaultcxx = compilerEnv("CXX", "")
 
 	b = os.Getenv("PKG_CONFIG")
 	if b == "" {
@@ -308,12 +303,34 @@ func compilerEnv(envName, def string) map[string]string {
 	return m
 }
 
+// clangos lists the operating systems where we prefer clang to gcc.
+var clangos = []string{
+	"darwin",  // macOS 10.9 and later require clang
+	"freebsd", // FreeBSD 10 and later do not ship gcc
+	"openbsd", // OpenBSD ships with GCC 4.2, which is now quite old.
+}
+
 // compilerEnvLookup returns the compiler settings for goos/goarch in map m.
-func compilerEnvLookup(m map[string]string, goos, goarch string) string {
+// kind is "CC" or "CXX".
+func compilerEnvLookup(kind string, m map[string]string, goos, goarch string) string {
 	if cc := m[goos+"/"+goarch]; cc != "" {
 		return cc
 	}
-	return m[""]
+	if cc := m[""]; cc != "" {
+		return cc
+	}
+	for _, os := range clangos {
+		if goos == os {
+			if kind == "CXX" {
+				return "clang++"
+			}
+			return "clang"
+		}
+	}
+	if kind == "CXX" {
+		return "g++"
+	}
+	return "gcc"
 }
 
 // rmworkdir deletes the work directory.
@@ -524,15 +541,25 @@ func setup() {
 		xremove(pathf("%s/bin/%s", goroot, old))
 	}
 
-	// For release, make sure excluded things are excluded.
+	// Special release-specific setup.
 	goversion := findgoversion()
-	if strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta")) {
+	isRelease := strings.HasPrefix(goversion, "release.") || (strings.HasPrefix(goversion, "go") && !strings.Contains(goversion, "beta"))
+	if isRelease {
+		// Make sure release-excluded things are excluded.
 		for _, dir := range unreleased {
 			if p := pathf("%s/%s", goroot, dir); isdir(p) {
 				fatalf("%s should not exist in release build", p)
 			}
 		}
 	}
+	if isRelease || os.Getenv("GO_BUILDER_NAME") != "" {
+		// Add -trimpath for reproducible builds of releases.
+		// Include builders so that -trimpath is well-tested ahead of releases.
+		// Do not include local development, so that people working in the
+		// main branch for day-to-day work on the Go toolchain itself can
+		// still have full paths for stack traces for compiler crashes and the like.
+		// toolenv = append(toolenv, "GOFLAGS=-trimpath")
+	}
 }
 
 /*
@@ -675,7 +702,7 @@ func runInstall(pkg string, ch chan struct{}) {
 		if goldflags != "" {
 			link = append(link, goldflags)
 		}
-		link = append(link, "-extld="+compilerEnvLookup(defaultcc, goos, goarch))
+		link = append(link, "-extld="+compilerEnvLookup("CC", defaultcc, goos, goarch))
 		link = append(link, "-L="+pathf("%s/pkg/obj/go-bootstrap/%s_%s", goroot, goos, goarch))
 		link = append(link, "-o", pathf("%s/%s%s", tooldir, elem, exe))
 		targ = len(link) - 1
@@ -1263,6 +1290,15 @@ func timelog(op, name string) {
 	fmt.Fprintf(timeLogFile, "%s %+.1fs %s %s\n", t.Format(time.UnixDate), t.Sub(timeLogStart).Seconds(), op, name)
 }
 
+// toolenv is the environment to use when building cmd.
+// We disable cgo to get static binaries for cmd/go and cmd/pprof,
+// so that they work on systems without the same dynamic libraries
+// as the original build system.
+// In release branches, we add -trimpath for reproducible builds.
+// In the main branch we leave it off, so that compiler crashes and
+// the like have full path names for easier navigation to source files.
+var toolenv = []string{"CGO_ENABLED=0"}
+
 var toolchain = []string{"cmd/asm", "cmd/cgo", "cmd/compile", "cmd/link"}
 
 // The bootstrap command runs a build from scratch,
@@ -1388,10 +1424,10 @@ func cmdbootstrap() {
 		xprintf("\n")
 	}
 	xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n")
-	os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
+	os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch))
 	// Now that cmd/go is in charge of the build process, enable GOEXPERIMENT.
 	os.Setenv("GOEXPERIMENT", goexperiment)
-	goInstall(goBootstrap, toolchain...)
+	goInstall(toolenv, goBootstrap, toolchain...)
 	if debug {
 		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
 		copyfile(pathf("%s/compile2", tooldir), pathf("%s/compile", tooldir), writeExec)
@@ -1418,7 +1454,7 @@ func cmdbootstrap() {
 		xprintf("\n")
 	}
 	xprintf("Building Go toolchain3 using go_bootstrap and Go toolchain2.\n")
-	goInstall(goBootstrap, append([]string{"-a"}, toolchain...)...)
+	goInstall(toolenv, goBootstrap, append([]string{"-a"}, toolchain...)...)
 	if debug {
 		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
 		copyfile(pathf("%s/compile3", tooldir), pathf("%s/compile", tooldir), writeExec)
@@ -1440,9 +1476,12 @@ func cmdbootstrap() {
 			xprintf("\n")
 		}
 		xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch)
-		goInstall(goBootstrap, "std", "cmd")
-		checkNotStale(goBootstrap, "std", "cmd")
-		checkNotStale(cmdGo, "std", "cmd")
+		goInstall(nil, goBootstrap, "std")
+		goInstall(toolenv, goBootstrap, "cmd")
+		checkNotStale(nil, goBootstrap, "std")
+		checkNotStale(toolenv, goBootstrap, "cmd")
+		checkNotStale(nil, cmdGo, "std")
+		checkNotStale(toolenv, cmdGo, "cmd")
 
 		timelog("build", "target toolchain")
 		if vflag > 0 {
@@ -1452,17 +1491,19 @@ func cmdbootstrap() {
 		goarch = oldgoarch
 		os.Setenv("GOOS", goos)
 		os.Setenv("GOARCH", goarch)
-		os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch))
+		os.Setenv("CC", compilerEnvLookup("CC", defaultcc, goos, goarch))
 		xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch)
 	}
-	targets := []string{"std", "cmd"}
-	goInstall(goBootstrap, targets...)
-	checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
-	checkNotStale(goBootstrap, targets...)
-	checkNotStale(cmdGo, targets...)
+	goInstall(nil, goBootstrap, "std")
+	goInstall(toolenv, goBootstrap, "cmd")
+	checkNotStale(toolenv, goBootstrap, append(toolchain, "runtime/internal/sys")...)
+	checkNotStale(nil, goBootstrap, "std")
+	checkNotStale(toolenv, goBootstrap, "cmd")
+	checkNotStale(nil, cmdGo, "std")
+	checkNotStale(toolenv, cmdGo, "cmd")
 	if debug {
 		run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full")
-		checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...)
+		checkNotStale(toolenv, goBootstrap, append(toolchain, "runtime/internal/sys")...)
 		copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec)
 	}
 
@@ -1492,8 +1533,8 @@ func cmdbootstrap() {
 		oldcc := os.Getenv("CC")
 		os.Setenv("GOOS", gohostos)
 		os.Setenv("GOARCH", gohostarch)
-		os.Setenv("CC", compilerEnvLookup(defaultcc, gohostos, gohostarch))
-		goCmd(cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath)
+		os.Setenv("CC", compilerEnvLookup("CC", defaultcc, gohostos, gohostarch))
+		goCmd(nil, cmdGo, "build", "-o", pathf("%s/go_%s_%s_exec%s", gorootBin, goos, goarch, exe), wrapperPath)
 		// Restore environment.
 		// TODO(elias.naur): support environment variables in goCmd?
 		os.Setenv("GOOS", goos)
@@ -1521,8 +1562,8 @@ func wrapperPathFor(goos, goarch string) string {
 	return ""
 }
 
-func goInstall(goBinary string, args ...string) {
-	goCmd(goBinary, "install", args...)
+func goInstall(env []string, goBinary string, args ...string) {
+	goCmd(env, goBinary, "install", args...)
 }
 
 func appendCompilerFlags(args []string) []string {
@@ -1535,7 +1576,7 @@ func appendCompilerFlags(args []string) []string {
 	return args
 }
 
-func goCmd(goBinary string, cmd string, args ...string) {
+func goCmd(env []string, goBinary string, cmd string, args ...string) {
 	goCmd := []string{goBinary, cmd}
 	if noOpt {
 		goCmd = append(goCmd, "-tags=noopt")
@@ -1550,10 +1591,10 @@ func goCmd(goBinary string, cmd string, args ...string) {
 		goCmd = append(goCmd, "-p=1")
 	}
 
-	run(workdir, ShowOutput|CheckExit, append(goCmd, args...)...)
+	runEnv(workdir, ShowOutput|CheckExit, env, append(goCmd, args...)...)
 }
 
-func checkNotStale(goBinary string, targets ...string) {
+func checkNotStale(env []string, goBinary string, targets ...string) {
 	goCmd := []string{goBinary, "list"}
 	if noOpt {
 		goCmd = append(goCmd, "-tags=noopt")
@@ -1561,7 +1602,7 @@ func checkNotStale(goBinary string, targets ...string) {
 	goCmd = appendCompilerFlags(goCmd)
 	goCmd = append(goCmd, "-f={{if .Stale}}\tSTALE {{.ImportPath}}: {{.StaleReason}}{{end}}")
 
-	out := run(workdir, CheckExit, append(goCmd, targets...)...)
+	out := runEnv(workdir, CheckExit, env, append(goCmd, targets...)...)
 	if strings.Contains(out, "\tSTALE ") {
 		os.Setenv("GODEBUG", "gocachehash=1")
 		for _, target := range []string{"runtime/internal/sys", "cmd/dist", "cmd/link"} {
@@ -1664,7 +1705,17 @@ func checkCC() {
 	if !needCC() {
 		return
 	}
-	cc, err := quotedSplit(defaultcc[""])
+	cc1 := defaultcc[""]
+	if cc1 == "" {
+		cc1 = "gcc"
+		for _, os := range clangos {
+			if gohostos == os {
+				cc1 = "clang"
+				break
+			}
+		}
+	}
+	cc, err := quotedSplit(cc1)
 	if err != nil {
 		fatalf("split CC: %v", err)
 	}
diff --git a/src/cmd/dist/buildgo.go b/src/cmd/dist/buildgo.go
index e56d72c8b1..495244a3a1 100644
--- a/src/cmd/dist/buildgo.go
+++ b/src/cmd/dist/buildgo.go
@@ -66,7 +66,26 @@ func defaultCCFunc(name string, defaultcc map[string]string) string {
 		fmt.Fprintf(&buf, "\tcase %q:\n\t\treturn %q\n", k, defaultcc[k])
 	}
 	fmt.Fprintf(&buf, "\t}\n")
-	fmt.Fprintf(&buf, "\treturn %q\n", defaultcc[""])
+	if cc := defaultcc[""]; cc != "" {
+		fmt.Fprintf(&buf, "\treturn %q\n", cc)
+	} else {
+		clang, gcc := "clang", "gcc"
+		if strings.HasSuffix(name, "CXX") {
+			clang, gcc = "clang++", "g++"
+		}
+		fmt.Fprintf(&buf, "\tswitch goos {\n")
+		fmt.Fprintf(&buf, "\tcase ")
+		for i, os := range clangos {
+			if i > 0 {
+				fmt.Fprintf(&buf, ", ")
+			}
+			fmt.Fprintf(&buf, "%q", os)
+		}
+		fmt.Fprintf(&buf, ":\n")
+		fmt.Fprintf(&buf, "\t\treturn %q\n", clang)
+		fmt.Fprintf(&buf, "\t}\n")
+		fmt.Fprintf(&buf, "\treturn %q\n", gcc)
+	}
 	fmt.Fprintf(&buf, "}\n")
 
 	return buf.String()
diff --git a/src/cmd/dist/main.go b/src/cmd/dist/main.go
index 6194ea901c..31a348e638 100644
--- a/src/cmd/dist/main.go
+++ b/src/cmd/dist/main.go
@@ -59,15 +59,6 @@ func main() {
 	case "aix":
 		// uname -m doesn't work under AIX
 		gohostarch = "ppc64"
-	case "darwin":
-		// macOS 10.9 and later require clang
-		defaultclang = true
-	case "freebsd":
-		// Since FreeBSD 10 gcc is no longer part of the base system.
-		defaultclang = true
-	case "openbsd":
-		// OpenBSD ships with GCC 4.2, which is now quite old.
-		defaultclang = true
 	case "plan9":
 		gohostarch = os.Getenv("objtype")
 		if gohostarch == "" {
diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go
index 9f2660631d..9700e15738 100644
--- a/src/cmd/dist/test.go
+++ b/src/cmd/dist/test.go
@@ -149,7 +149,7 @@ func (t *tester) run() {
 	if t.rebuild {
 		t.out("Building packages and commands.")
 		// Force rebuild the whole toolchain.
-		goInstall("go", append([]string{"-a"}, toolchain...)...)
+		goInstall(toolenv, "go", append([]string{"-a"}, toolchain...)...)
 	}
 
 	if !t.listMode {
@@ -166,9 +166,10 @@ func (t *tester) run() {
 			// to break if we don't automatically refresh things here.
 			// Rebuilding is a shortened bootstrap.
 			// See cmdbootstrap for a description of the overall process.
-			goInstall("go", toolchain...)
-			goInstall("go", toolchain...)
-			goInstall("go", "std", "cmd")
+			goInstall(toolenv, "go", toolchain...)
+			goInstall(toolenv, "go", toolchain...)
+			goInstall(toolenv, "go", "cmd")
+			goInstall(nil, "go", "std")
 		} else {
 			// The Go builder infrastructure should always begin running tests from a
 			// clean, non-stale state, so there is no need to rebuild the world.
@@ -178,15 +179,15 @@ func (t *tester) run() {
 			// The cache used by dist when building is different from that used when
 			// running dist test, so rebuild (but don't install) std and cmd to make
 			// sure packages without install targets are cached so they are not stale.
-			goCmd("go", "build", "std", "cmd") // make sure dependencies of targets are cached
-			if builder == "aix-ppc64" {
+			goCmd(toolenv, "go", "build", "cmd") // make sure dependencies of targets are cached
+			goCmd(nil, "go", "build", "std")
+			checkNotStale(nil, "go", "std")
+			if builder != "aix-ppc64" {
 				// The aix-ppc64 builder for some reason does not have deterministic cgo
 				// builds, so "cmd" is stale. Fortunately, most of the tests don't care.
 				// TODO(#56896): remove this special case once the builder supports
 				// determistic cgo builds.
-				checkNotStale("go", "std")
-			} else {
-				checkNotStale("go", "std", "cmd")
+				checkNotStale(toolenv, "go", "cmd")
 			}
 		}
 	}
@@ -1315,7 +1316,7 @@ func (t *tester) registerCgoTests() {
 			// Check for static linking support
 			var staticCheck rtPreFunc
 			cmd := t.dirCmd("misc/cgo/test",
-				compilerEnvLookup(defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
+				compilerEnvLookup("CC", defaultcc, goos, goarch), "-xc", "-o", "/dev/null", "-static", "-")
 			cmd.Stdin = strings.NewReader("int main() {}")
 			cmd.Stdout, cmd.Stderr = nil, nil // Discard output
 			if err := cmd.Run(); err != nil {
@@ -1365,7 +1366,7 @@ func (t *tester) registerCgoTests() {
 // running in parallel with earlier tests, or if it has some other reason
 // for needing the earlier tests to be done.
 func (t *tester) runPending(nextTest *distTest) {
-	checkNotStale("go", "std")
+	checkNotStale(nil, "go", "std")
 	worklist := t.worklist
 	t.worklist = nil
 	for _, w := range worklist {
@@ -1423,7 +1424,7 @@ func (t *tester) runPending(nextTest *distTest) {
 			log.Printf("Failed: %v", w.err)
 			t.failed = true
 		}
-		checkNotStale("go", "std")
+		checkNotStale(nil, "go", "std")
 	}
 	if t.failed && !t.keepGoing {
 		fatalf("FAILED")
@@ -1449,7 +1450,7 @@ func (t *tester) hasBash() bool {
 }
 
 func (t *tester) hasCxx() bool {
-	cxx, _ := exec.LookPath(compilerEnvLookup(defaultcxx, goos, goarch))
+	cxx, _ := exec.LookPath(compilerEnvLookup("CXX", defaultcxx, goos, goarch))
 	return cxx != ""
 }
 
diff --git a/src/cmd/dist/util.go b/src/cmd/dist/util.go
index fe36230207..d951abd556 100644
--- a/src/cmd/dist/util.go
+++ b/src/cmd/dist/util.go
@@ -58,14 +58,19 @@ const (
 
 var outputLock sync.Mutex
 
-// run runs the command line cmd in dir.
+// run is like runEnv with no additional environment.
+func run(dir string, mode int, cmd ...string) string {
+	return runEnv(dir, mode, nil, cmd...)
+}
+
+// runEnv runs the command line cmd in dir with additional environment env.
 // If mode has ShowOutput set and Background unset, run passes cmd's output to
 // stdout/stderr directly. Otherwise, run returns cmd's output as a string.
 // If mode has CheckExit set and the command fails, run calls fatalf.
 // If mode has Background set, this command is being run as a
 // Background job. Only bgrun should use the Background mode,
 // not other callers.
-func run(dir string, mode int, cmd ...string) string {
+func runEnv(dir string, mode int, env []string, cmd ...string) string {
 	if vflag > 1 {
 		errprintf("run: %s\n", strings.Join(cmd, " "))
 	}
@@ -75,6 +80,9 @@ func run(dir string, mode int, cmd ...string) string {
 		bin = gorootBinGo
 	}
 	xcmd := exec.Command(bin, cmd[1:]...)
+	if env != nil {
+		xcmd.Env = append(os.Environ(), env...)
+	}
 	setDir(xcmd, dir)
 	var data []byte
 	var err error