From fb4f7fdb26da9ed0fee6beab280c84b399edaa42 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Tue, 1 Nov 2022 11:40:29 -0400 Subject: cmd/compile: use edge weights to decide inlineability in PGO Currently, with PGO, the inliner uses node weights to decide if a function is inlineable (with a larger budget). But the actual inlining is determined by the weight of the call edge. There is a discrepancy that, if a callee node is hot but the call edge is not, it would not inlined, and marking the callee inlineable would of no use. Instead of using two kinds of weights, we just use the edge weights to decide inlineability. If a function is the callee of a hot call edge, its inlineability is determined with a larger threshold. For a function that exceeds the regular inlining budget, it is still inlined only when the call edge is hot, as it would exceed the regular inlining cost for non-hot call sites, even if it is marked inlineable. For #55022. Change-Id: I93fa9919fc6bcbb394e6cfe54ec96a96eede08f7 Reviewed-on: https://go-review.googlesource.com/c/go/+/447015 Run-TryBot: Cherry Mui Reviewed-by: Michael Pratt TryBot-Result: Gopher Robot --- src/cmd/compile/internal/inline/inl.go | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) (limited to 'src/cmd/compile/internal/inline') diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 335bb23ecb..98bfb4e382 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -56,9 +56,9 @@ const ( ) var ( - // List of all hot nodes. + // List of all hot callee nodes. // TODO(prattmic): Make this non-global. - candHotNodeMap = make(map[*pgo.IRNode]struct{}) + candHotCalleeMap = make(map[*pgo.IRNode]struct{}) // List of all hot call sites. CallSiteInfo.Callee is always nil. // TODO(prattmic): Make this non-global. @@ -68,9 +68,6 @@ var ( // TODO(prattmic): Make this non-global. inlinedCallSites = make(map[pgo.CallSiteInfo]struct{}) - // Threshold in percentage for hot function inlining. - inlineHotFuncThresholdPercent = float64(2) - // Threshold in percentage for hot callsite inlining. inlineHotCallSiteThresholdPercent = float64(0.1) @@ -80,13 +77,6 @@ var ( // pgoInlinePrologue records the hot callsites from ir-graph. func pgoInlinePrologue(p *pgo.Profile) { - if s, err := strconv.ParseFloat(base.Debug.InlineHotFuncThreshold, 64); err == nil { - inlineHotFuncThresholdPercent = s - if base.Debug.PGOInline > 0 { - fmt.Printf("hot-node-thres=%v\n", inlineHotFuncThresholdPercent) - } - } - if s, err := strconv.ParseFloat(base.Debug.InlineHotCallSiteThreshold, 64); err == nil { inlineHotCallSiteThresholdPercent = s if base.Debug.PGOInline > 0 { @@ -102,10 +92,6 @@ func pgoInlinePrologue(p *pgo.Profile) { for _, f := range list { name := ir.PkgFuncName(f) if n, ok := p.WeightedCG.IRNodes[name]; ok { - nodeweight := pgo.WeightInPercentage(n.Flat, p.TotalNodeWeight) - if nodeweight > inlineHotFuncThresholdPercent { - candHotNodeMap[n] = struct{}{} - } for _, e := range p.WeightedCG.OutEdges[n] { if e.Weight != 0 { edgeweightpercent := pgo.WeightInPercentage(e.Weight, p.TotalEdgeWeight) @@ -113,6 +99,7 @@ func pgoInlinePrologue(p *pgo.Profile) { csi := pgo.CallSiteInfo{Line: e.CallSite, Caller: n.AST} if _, ok := candHotEdgeMap[csi]; !ok { candHotEdgeMap[csi] = struct{}{} + candHotCalleeMap[e.Dst] = struct{}{} } } } @@ -122,7 +109,7 @@ func pgoInlinePrologue(p *pgo.Profile) { }) if base.Debug.PGOInline > 0 { fmt.Printf("hot-cg before inline in dot format:") - p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) + p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent) } } @@ -139,7 +126,7 @@ func pgoInlineEpilogue(p *pgo.Profile) { }) // Print the call-graph after inlining. This is a debugging feature. fmt.Printf("hot-cg after inline in dot:") - p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) + p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent) } } @@ -270,7 +257,7 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) { budget := int32(inlineMaxBudget) if profile != nil { if n, ok := profile.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok { - if _, ok := candHotNodeMap[n]; ok { + if _, ok := candHotCalleeMap[n]; ok { budget = int32(inlineHotMaxBudget) if base.Debug.PGOInline > 0 { fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn)) -- cgit v1.3