cmd/compile: fix PGO cross-package inlining

With CL 447015, we identify hot callees from edge weights, but
the code only traverses edges for calls from the current package.
If the callee is in a different package, when compiling that
package, the edge was not visited, so the callee was not actually
marked inline candidate. This CL fixes it by traversing all hot
edges.

Change-Id: If668c1a16ebe34e3474376b88ab3a84be76b8562
Reviewed-on: https://go-review.googlesource.com/c/go/+/448015
Run-TryBot: Cherry Mui <cherryyz@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>

1 files changed, 16 insertions, 6 deletions

diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go
index 3f7ad34af3..c7f56d360d 100644
--- a/src/cmd/compile/internal/inline/inl.go
+++ b/src/cmd/compile/internal/inline/inl.go
@@ -87,7 +87,8 @@ func pgoInlinePrologue(p *pgo.Profile) {
 	if s, err := strconv.ParseFloat(base.Debug.InlineHotCallSiteCDFThreshold, 64); err == nil {
 		inlineCDFHotCallSiteThresholdPercent = s
 	}
-	inlineHotCallSiteThresholdPercent = computeThresholdFromCDF(p)
+	var hotCallsites []pgo.NodeMapKey
+	inlineHotCallSiteThresholdPercent, hotCallsites = computeThresholdFromCDF(p)
 	if base.Debug.PGOInline > 0 {
 		fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
 	}
@@ -96,6 +97,13 @@ func pgoInlinePrologue(p *pgo.Profile) {
 		inlineHotMaxBudget = int32(base.Debug.InlineHotBudget)
 	}
 
+	// mark inlineable callees from hot edges
+	for _, n := range hotCallsites {
+		if fn := p.WeightedCG.IRNodes[n.CalleeName]; fn != nil {
+			candHotCalleeMap[fn] = struct{}{}
+		}
+	}
+	// mark hot call sites
 	ir.VisitFuncsBottomUp(typecheck.Target.Decls, func(list []*ir.Func, recursive bool) {
 		for _, f := range list {
 			name := ir.PkgFuncName(f)
@@ -107,7 +115,6 @@ func pgoInlinePrologue(p *pgo.Profile) {
 							csi := pgo.CallSiteInfo{Line: e.CallSite, Caller: n.AST}
 							if _, ok := candHotEdgeMap[csi]; !ok {
 								candHotEdgeMap[csi] = struct{}{}
-								candHotCalleeMap[e.Dst] = struct{}{}
 							}
 						}
 					}
@@ -121,7 +128,10 @@ func pgoInlinePrologue(p *pgo.Profile) {
 	}
 }
 
-func computeThresholdFromCDF(p *pgo.Profile) float64 {
+// computeThresholdFromCDF computes an edge weight threshold based on the
+// CDF of edge weights from the profile. Returns the threshold, and the
+// list of edges that make up the given percentage of the CDF.
+func computeThresholdFromCDF(p *pgo.Profile) (float64, []pgo.NodeMapKey) {
 	nodes := make([]pgo.NodeMapKey, len(p.NodeMap))
 	i := 0
 	for n := range p.NodeMap {
@@ -143,14 +153,14 @@ func computeThresholdFromCDF(p *pgo.Profile) float64 {
 		return ni.CallSite < nj.CallSite
 	})
 	cum := int64(0)
-	for _, n := range nodes {
+	for i, n := range nodes {
 		w := p.NodeMap[n].EWeight
 		cum += w
 		if pgo.WeightInPercentage(cum, p.TotalEdgeWeight) > inlineCDFHotCallSiteThresholdPercent {
-			return pgo.WeightInPercentage(w, p.TotalEdgeWeight)
+			return pgo.WeightInPercentage(w, p.TotalEdgeWeight), nodes[:i]
 		}
 	}
-	return 100
+	return 100, nil
 }
 
 // pgoInlineEpilogue updates IRGraph after inlining.