[dev.unified] cmd/compile: implement simple inline body pruning heuristic

An important optimization in the existing export data format is the
pruning of unreachable inline bodies. That is, when re-exporting
transitively imported types, omitting the inline bodies for methods
that can't actually be needed due to importing that package.

The existing logic (implemented in typecheck/crawler.go) is fairly
sophisticated, but also relies on actually expanding inline bodies in
the process, which is undesirable. However, including all inline
bodies is also prohibitive for testing GOEXPERIMENT=unified against
very large Go code bases that impose size limits on build action
inputs.

As a short-term solution, this CL implements a simple heuristic for
GOEXPERIMENT=unified: include the inline bodies for all
locally-declared functions/methods, and for any imported
functions/methods that were inlined into this package.

Change-Id: I686964a0cd9262b77d3d5587f89cfbcfe8b2e521
Reviewed-on: https://go-review.googlesource.com/c/go/+/419675
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: David Chase <drchase@google.com>

1 files changed, 72 insertions, 20 deletions

diff --git a/src/cmd/compile/internal/noder/unified.go b/src/cmd/compile/internal/noder/unified.go
index 95486af66c..d9b15ab385 100644
--- a/src/cmd/compile/internal/noder/unified.go
+++ b/src/cmd/compile/internal/noder/unified.go
@@ -85,7 +85,7 @@ func unified(noders []*noder) {
 	typecheck.TypecheckAllowed = true
 
 	localPkgReader = newPkgReader(pkgbits.NewPkgDecoder(types.LocalPkg.Path, data))
-	readPackage(localPkgReader, types.LocalPkg)
+	readPackage(localPkgReader, types.LocalPkg, true)
 
 	r := localPkgReader.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
 	r.pkgInit(types.LocalPkg, target)
@@ -226,29 +226,54 @@ func freePackage(pkg *types2.Package) {
 
 // readPackage reads package export data from pr to populate
 // importpkg.
-func readPackage(pr *pkgReader, importpkg *types.Pkg) {
-	r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
+//
+// localStub indicates whether pr is reading the stub export data for
+// the local package, as opposed to relocated export data for an
+// import.
+func readPackage(pr *pkgReader, importpkg *types.Pkg, localStub bool) {
+	{
+		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PublicRootIdx, pkgbits.SyncPublic)
+
+		pkg := r.pkg()
+		base.Assertf(pkg == importpkg, "have package %q (%p), want package %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
+
+		if r.Bool() {
+			sym := pkg.Lookup(".inittask")
+			task := ir.NewNameAt(src.NoXPos, sym)
+			task.Class = ir.PEXTERN
+			sym.Def = task
+		}
+
+		for i, n := 0, r.Len(); i < n; i++ {
+			r.Sync(pkgbits.SyncObject)
+			assert(!r.Bool())
+			idx := r.Reloc(pkgbits.RelocObj)
+			assert(r.Len() == 0)
 
-	pkg := r.pkg()
-	base.Assertf(pkg == importpkg, "have package %q (%p), want package %q (%p)", pkg.Path, pkg, importpkg.Path, importpkg)
+			path, name, code := r.p.PeekObj(idx)
+			if code != pkgbits.ObjStub {
+				objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil}
+			}
+		}
 
-	if r.Bool() {
-		sym := pkg.Lookup(".inittask")
-		task := ir.NewNameAt(src.NoXPos, sym)
-		task.Class = ir.PEXTERN
-		sym.Def = task
+		r.Sync(pkgbits.SyncEOF)
 	}
 
-	for i, n := 0, r.Len(); i < n; i++ {
-		r.Sync(pkgbits.SyncObject)
-		assert(!r.Bool())
-		idx := r.Reloc(pkgbits.RelocObj)
-		assert(r.Len() == 0)
+	if !localStub {
+		r := pr.newReader(pkgbits.RelocMeta, pkgbits.PrivateRootIdx, pkgbits.SyncPrivate)
+
+		for i, n := 0, r.Len(); i < n; i++ {
+			path := r.String()
+			name := r.String()
+			idx := r.Reloc(pkgbits.RelocBody)
 
-		path, name, code := r.p.PeekObj(idx)
-		if code != pkgbits.ObjStub {
-			objReader[types.NewPkg(path, "").Lookup(name)] = pkgReaderIndex{pr, idx, nil}
+			sym := types.NewPkg(path, "").Lookup(name)
+			if _, ok := importBodyReader[sym]; !ok {
+				importBodyReader[sym] = pkgReaderIndex{pr, idx, nil}
+			}
 		}
+
+		r.Sync(pkgbits.SyncEOF)
 	}
 }
 
@@ -258,12 +283,15 @@ func writeUnifiedExport(out io.Writer) {
 	l := linker{
 		pw: pkgbits.NewPkgEncoder(base.Debug.SyncFrames),
 
-		pkgs:  make(map[string]pkgbits.Index),
-		decls: make(map[*types.Sym]pkgbits.Index),
+		pkgs:   make(map[string]pkgbits.Index),
+		decls:  make(map[*types.Sym]pkgbits.Index),
+		bodies: make(map[*types.Sym]pkgbits.Index),
 	}
 
 	publicRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPublic)
+	privateRootWriter := l.pw.NewEncoder(pkgbits.RelocMeta, pkgbits.SyncPrivate)
 	assert(publicRootWriter.Idx == pkgbits.PublicRootIdx)
+	assert(privateRootWriter.Idx == pkgbits.PrivateRootIdx)
 
 	var selfPkgIdx pkgbits.Index
 
@@ -320,5 +348,29 @@ func writeUnifiedExport(out io.Writer) {
 		w.Flush()
 	}
 
+	{
+		type symIdx struct {
+			sym *types.Sym
+			idx pkgbits.Index
+		}
+		var bodies []symIdx
+		for sym, idx := range l.bodies {
+			bodies = append(bodies, symIdx{sym, idx})
+		}
+		sort.Slice(bodies, func(i, j int) bool { return bodies[i].idx < bodies[j].idx })
+
+		w := privateRootWriter
+
+		w.Len(len(bodies))
+		for _, body := range bodies {
+			w.String(body.sym.Pkg.Path)
+			w.String(body.sym.Name)
+			w.Reloc(pkgbits.RelocBody, body.idx)
+		}
+
+		w.Sync(pkgbits.SyncEOF)
+		w.Flush()
+	}
+
 	base.Ctxt.Fingerprint = l.pw.DumpTo(out)
 }