inl.go

Documentation: cmd/compile/internal/inline

     1  // Copyright 2011 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  //
     5  // The inlining facility makes 2 passes: first CanInline determines which
     6  // functions are suitable for inlining, and for those that are it
     7  // saves a copy of the body. Then InlineCalls walks each function body to
     8  // expand calls to inlinable functions.
     9  //
    10  // The Debug.l flag controls the aggressiveness. Note that main() swaps level 0 and 1,
    11  // making 1 the default and -l disable. Additional levels (beyond -l) may be buggy and
    12  // are not supported.
    13  //      0: disabled
    14  //      1: 80-nodes leaf functions, oneliners, panic, lazy typechecking (default)
    15  //      2: (unassigned)
    16  //      3: (unassigned)
    17  //      4: allow non-leaf functions
    18  //
    19  // At some point this may get another default and become switch-offable with -N.
    20  //
    21  // The -d typcheckinl flag enables early typechecking of all imported bodies,
    22  // which is useful to flush out bugs.
    23  //
    24  // The Debug.m flag enables diagnostic output.  a single -m is useful for verifying
    25  // which calls get inlined or not, more is for debugging, and may go away at any point.
    26  
    27  package inline
    28  
    29  import (
    30  	"fmt"
    31  	"go/constant"
    32  	"internal/buildcfg"
    33  	"strconv"
    34  
    35  	"cmd/compile/internal/base"
    36  	"cmd/compile/internal/inline/inlheur"
    37  	"cmd/compile/internal/ir"
    38  	"cmd/compile/internal/logopt"
    39  	"cmd/compile/internal/pgo"
    40  	"cmd/compile/internal/typecheck"
    41  	"cmd/compile/internal/types"
    42  	"cmd/internal/obj"
    43  )
    44  
    45  // Inlining budget parameters, gathered in one place
    46  const (
    47  	inlineMaxBudget       = 80
    48  	inlineExtraAppendCost = 0
    49  	// default is to inline if there's at most one call. -l=4 overrides this by using 1 instead.
    50  	inlineExtraCallCost  = 57              // 57 was benchmarked to provided most benefit with no bad surprises; see https://github.com/golang/go/issues/19348#issuecomment-439370742
    51  	inlineExtraPanicCost = 1               // do not penalize inlining panics.
    52  	inlineExtraThrowCost = inlineMaxBudget // with current (2018-05/1.11) code, inlining runtime.throw does not help.
    53  
    54  	inlineBigFunctionNodes   = 5000 // Functions with this many nodes are considered "big".
    55  	inlineBigFunctionMaxCost = 20   // Max cost of inlinee when inlining into a "big" function.
    56  )
    57  
    58  var (
    59  	// List of all hot callee nodes.
    60  	// TODO(prattmic): Make this non-global.
    61  	candHotCalleeMap = make(map[*pgo.IRNode]struct{})
    62  
    63  	// List of all hot call sites. CallSiteInfo.Callee is always nil.
    64  	// TODO(prattmic): Make this non-global.
    65  	candHotEdgeMap = make(map[pgo.CallSiteInfo]struct{})
    66  
    67  	// Threshold in percentage for hot callsite inlining.
    68  	inlineHotCallSiteThresholdPercent float64
    69  
    70  	// Threshold in CDF percentage for hot callsite inlining,
    71  	// that is, for a threshold of X the hottest callsites that
    72  	// make up the top X% of total edge weight will be
    73  	// considered hot for inlining candidates.
    74  	inlineCDFHotCallSiteThresholdPercent = float64(99)
    75  
    76  	// Budget increased due to hotness.
    77  	inlineHotMaxBudget int32 = 2000
    78  )
    79  
    80  // PGOInlinePrologue records the hot callsites from ir-graph.
    81  func PGOInlinePrologue(p *pgo.Profile, funcs []*ir.Func) {
    82  	if base.Debug.PGOInlineCDFThreshold != "" {
    83  		if s, err := strconv.ParseFloat(base.Debug.PGOInlineCDFThreshold, 64); err == nil && s >= 0 && s <= 100 {
    84  			inlineCDFHotCallSiteThresholdPercent = s
    85  		} else {
    86  			base.Fatalf("invalid PGOInlineCDFThreshold, must be between 0 and 100")
    87  		}
    88  	}
    89  	var hotCallsites []pgo.NamedCallEdge
    90  	inlineHotCallSiteThresholdPercent, hotCallsites = hotNodesFromCDF(p)
    91  	if base.Debug.PGODebug > 0 {
    92  		fmt.Printf("hot-callsite-thres-from-CDF=%v\n", inlineHotCallSiteThresholdPercent)
    93  	}
    94  
    95  	if x := base.Debug.PGOInlineBudget; x != 0 {
    96  		inlineHotMaxBudget = int32(x)
    97  	}
    98  
    99  	for _, n := range hotCallsites {
   100  		// mark inlineable callees from hot edges
   101  		if callee := p.WeightedCG.IRNodes[n.CalleeName]; callee != nil {
   102  			candHotCalleeMap[callee] = struct{}{}
   103  		}
   104  		// mark hot call sites
   105  		if caller := p.WeightedCG.IRNodes[n.CallerName]; caller != nil && caller.AST != nil {
   106  			csi := pgo.CallSiteInfo{LineOffset: n.CallSiteOffset, Caller: caller.AST}
   107  			candHotEdgeMap[csi] = struct{}{}
   108  		}
   109  	}
   110  
   111  	if base.Debug.PGODebug >= 3 {
   112  		fmt.Printf("hot-cg before inline in dot format:")
   113  		p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent)
   114  	}
   115  }
   116  
   117  // hotNodesFromCDF computes an edge weight threshold and the list of hot
   118  // nodes that make up the given percentage of the CDF. The threshold, as
   119  // a percent, is the lower bound of weight for nodes to be considered hot
   120  // (currently only used in debug prints) (in case of equal weights,
   121  // comparing with the threshold may not accurately reflect which nodes are
   122  // considiered hot).
   123  func hotNodesFromCDF(p *pgo.Profile) (float64, []pgo.NamedCallEdge) {
   124  	cum := int64(0)
   125  	for i, n := range p.NamedEdgeMap.ByWeight {
   126  		w := p.NamedEdgeMap.Weight[n]
   127  		cum += w
   128  		if pgo.WeightInPercentage(cum, p.TotalWeight) > inlineCDFHotCallSiteThresholdPercent {
   129  			// nodes[:i+1] to include the very last node that makes it to go over the threshold.
   130  			// (Say, if the CDF threshold is 50% and one hot node takes 60% of weight, we want to
   131  			// include that node instead of excluding it.)
   132  			return pgo.WeightInPercentage(w, p.TotalWeight), p.NamedEdgeMap.ByWeight[:i+1]
   133  		}
   134  	}
   135  	return 0, p.NamedEdgeMap.ByWeight
   136  }
   137  
   138  // CanInlineFuncs computes whether a batch of functions are inlinable.
   139  func CanInlineFuncs(funcs []*ir.Func, profile *pgo.Profile) {
   140  	if profile != nil {
   141  		PGOInlinePrologue(profile, funcs)
   142  	}
   143  
   144  	ir.VisitFuncsBottomUp(funcs, func(list []*ir.Func, recursive bool) {
   145  		CanInlineSCC(list, recursive, profile)
   146  	})
   147  }
   148  
   149  // CanInlineSCC computes the inlinability of functions within an SCC
   150  // (strongly connected component).
   151  //
   152  // CanInlineSCC is designed to be used by ir.VisitFuncsBottomUp
   153  // callbacks.
   154  func CanInlineSCC(funcs []*ir.Func, recursive bool, profile *pgo.Profile) {
   155  	if base.Flag.LowerL == 0 {
   156  		return
   157  	}
   158  
   159  	numfns := numNonClosures(funcs)
   160  
   161  	for _, fn := range funcs {
   162  		if !recursive || numfns > 1 {
   163  			// We allow inlining if there is no
   164  			// recursion, or the recursion cycle is
   165  			// across more than one function.
   166  			CanInline(fn, profile)
   167  		} else {
   168  			if base.Flag.LowerM > 1 && fn.OClosure == nil {
   169  				fmt.Printf("%v: cannot inline %v: recursive\n", ir.Line(fn), fn.Nname)
   170  			}
   171  		}
   172  		if inlheur.Enabled() {
   173  			analyzeFuncProps(fn, profile)
   174  		}
   175  	}
   176  }
   177  
   178  // GarbageCollectUnreferencedHiddenClosures makes a pass over all the
   179  // top-level (non-hidden-closure) functions looking for nested closure
   180  // functions that are reachable, then sweeps through the Target.Decls
   181  // list and marks any non-reachable hidden closure function as dead.
   182  // See issues #59404 and #59638 for more context.
   183  func GarbageCollectUnreferencedHiddenClosures() {
   184  
   185  	liveFuncs := make(map[*ir.Func]bool)
   186  
   187  	var markLiveFuncs func(fn *ir.Func)
   188  	markLiveFuncs = func(fn *ir.Func) {
   189  		if liveFuncs[fn] {
   190  			return
   191  		}
   192  		liveFuncs[fn] = true
   193  		ir.Visit(fn, func(n ir.Node) {
   194  			if clo, ok := n.(*ir.ClosureExpr); ok {
   195  				markLiveFuncs(clo.Func)
   196  			}
   197  		})
   198  	}
   199  
   200  	for i := 0; i < len(typecheck.Target.Funcs); i++ {
   201  		fn := typecheck.Target.Funcs[i]
   202  		if fn.IsHiddenClosure() {
   203  			continue
   204  		}
   205  		markLiveFuncs(fn)
   206  	}
   207  
   208  	for i := 0; i < len(typecheck.Target.Funcs); i++ {
   209  		fn := typecheck.Target.Funcs[i]
   210  		if !fn.IsHiddenClosure() {
   211  			continue
   212  		}
   213  		if fn.IsDeadcodeClosure() {
   214  			continue
   215  		}
   216  		if liveFuncs[fn] {
   217  			continue
   218  		}
   219  		fn.SetIsDeadcodeClosure(true)
   220  		if base.Flag.LowerM > 2 {
   221  			fmt.Printf("%v: unreferenced closure %v marked as dead\n", ir.Line(fn), fn)
   222  		}
   223  		if fn.Inl != nil && fn.LSym == nil {
   224  			ir.InitLSym(fn, true)
   225  		}
   226  	}
   227  }
   228  
   229  // inlineBudget determines the max budget for function 'fn' prior to
   230  // analyzing the hairyness of the body of 'fn'. We pass in the pgo
   231  // profile if available (which can change the budget), also a
   232  // 'relaxed' flag, which expands the budget slightly to allow for the
   233  // possibility that a call to the function might have its score
   234  // adjusted downwards. If 'verbose' is set, then print a remark where
   235  // we boost the budget due to PGO.
   236  func inlineBudget(fn *ir.Func, profile *pgo.Profile, relaxed bool, verbose bool) int32 {
   237  	// Update the budget for profile-guided inlining.
   238  	budget := int32(inlineMaxBudget)
   239  	if profile != nil {
   240  		if n, ok := profile.WeightedCG.IRNodes[ir.LinkFuncName(fn)]; ok {
   241  			if _, ok := candHotCalleeMap[n]; ok {
   242  				budget = int32(inlineHotMaxBudget)
   243  				if verbose {
   244  					fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn))
   245  				}
   246  			}
   247  		}
   248  	}
   249  	if relaxed {
   250  		budget += inlheur.BudgetExpansion(inlineMaxBudget)
   251  	}
   252  	return budget
   253  }
   254  
   255  // CanInline determines whether fn is inlineable.
   256  // If so, CanInline saves copies of fn.Body and fn.Dcl in fn.Inl.
   257  // fn and fn.Body will already have been typechecked.
   258  func CanInline(fn *ir.Func, profile *pgo.Profile) {
   259  	if fn.Nname == nil {
   260  		base.Fatalf("CanInline no nname %+v", fn)
   261  	}
   262  
   263  	var reason string // reason, if any, that the function was not inlined
   264  	if base.Flag.LowerM > 1 || logopt.Enabled() {
   265  		defer func() {
   266  			if reason != "" {
   267  				if base.Flag.LowerM > 1 {
   268  					fmt.Printf("%v: cannot inline %v: %s\n", ir.Line(fn), fn.Nname, reason)
   269  				}
   270  				if logopt.Enabled() {
   271  					logopt.LogOpt(fn.Pos(), "cannotInlineFunction", "inline", ir.FuncName(fn), reason)
   272  				}
   273  			}
   274  		}()
   275  	}
   276  
   277  	reason = InlineImpossible(fn)
   278  	if reason != "" {
   279  		return
   280  	}
   281  	if fn.Typecheck() == 0 {
   282  		base.Fatalf("CanInline on non-typechecked function %v", fn)
   283  	}
   284  
   285  	n := fn.Nname
   286  	if n.Func.InlinabilityChecked() {
   287  		return
   288  	}
   289  	defer n.Func.SetInlinabilityChecked(true)
   290  
   291  	cc := int32(inlineExtraCallCost)
   292  	if base.Flag.LowerL == 4 {
   293  		cc = 1 // this appears to yield better performance than 0.
   294  	}
   295  
   296  	// Used a "relaxed" inline budget if the new inliner is enabled.
   297  	relaxed := inlheur.Enabled()
   298  
   299  	// Compute the inline budget for this func.
   300  	budget := inlineBudget(fn, profile, relaxed, base.Debug.PGODebug > 0)
   301  
   302  	// At this point in the game the function we're looking at may
   303  	// have "stale" autos, vars that still appear in the Dcl list, but
   304  	// which no longer have any uses in the function body (due to
   305  	// elimination by deadcode). We'd like to exclude these dead vars
   306  	// when creating the "Inline.Dcl" field below; to accomplish this,
   307  	// the hairyVisitor below builds up a map of used/referenced
   308  	// locals, and we use this map to produce a pruned Inline.Dcl
   309  	// list. See issue 25459 for more context.
   310  
   311  	visitor := hairyVisitor{
   312  		curFunc:       fn,
   313  		isBigFunc:     IsBigFunc(fn),
   314  		budget:        budget,
   315  		maxBudget:     budget,
   316  		extraCallCost: cc,
   317  		profile:       profile,
   318  	}
   319  	if visitor.tooHairy(fn) {
   320  		reason = visitor.reason
   321  		return
   322  	}
   323  
   324  	n.Func.Inl = &ir.Inline{
   325  		Cost:    budget - visitor.budget,
   326  		Dcl:     pruneUnusedAutos(n.Func.Dcl, &visitor),
   327  		HaveDcl: true,
   328  
   329  		CanDelayResults: canDelayResults(fn),
   330  	}
   331  	if base.Flag.LowerM != 0 || logopt.Enabled() {
   332  		noteInlinableFunc(n, fn, budget-visitor.budget)
   333  	}
   334  }
   335  
   336  // noteInlinableFunc issues a message to the user that the specified
   337  // function is inlinable.
   338  func noteInlinableFunc(n *ir.Name, fn *ir.Func, cost int32) {
   339  	if base.Flag.LowerM > 1 {
   340  		fmt.Printf("%v: can inline %v with cost %d as: %v { %v }\n", ir.Line(fn), n, cost, fn.Type(), ir.Nodes(fn.Body))
   341  	} else if base.Flag.LowerM != 0 {
   342  		fmt.Printf("%v: can inline %v\n", ir.Line(fn), n)
   343  	}
   344  	// JSON optimization log output.
   345  	if logopt.Enabled() {
   346  		logopt.LogOpt(fn.Pos(), "canInlineFunction", "inline", ir.FuncName(fn), fmt.Sprintf("cost: %d", cost))
   347  	}
   348  }
   349  
   350  // InlineImpossible returns a non-empty reason string if fn is impossible to
   351  // inline regardless of cost or contents.
   352  func InlineImpossible(fn *ir.Func) string {
   353  	var reason string // reason, if any, that the function can not be inlined.
   354  	if fn.Nname == nil {
   355  		reason = "no name"
   356  		return reason
   357  	}
   358  
   359  	// If marked "go:noinline", don't inline.
   360  	if fn.Pragma&ir.Noinline != 0 {
   361  		reason = "marked go:noinline"
   362  		return reason
   363  	}
   364  
   365  	// If marked "go:norace" and -race compilation, don't inline.
   366  	if base.Flag.Race && fn.Pragma&ir.Norace != 0 {
   367  		reason = "marked go:norace with -race compilation"
   368  		return reason
   369  	}
   370  
   371  	// If marked "go:nocheckptr" and -d checkptr compilation, don't inline.
   372  	if base.Debug.Checkptr != 0 && fn.Pragma&ir.NoCheckPtr != 0 {
   373  		reason = "marked go:nocheckptr"
   374  		return reason
   375  	}
   376  
   377  	// If marked "go:cgo_unsafe_args", don't inline, since the function
   378  	// makes assumptions about its argument frame layout.
   379  	if fn.Pragma&ir.CgoUnsafeArgs != 0 {
   380  		reason = "marked go:cgo_unsafe_args"
   381  		return reason
   382  	}
   383  
   384  	// If marked as "go:uintptrkeepalive", don't inline, since the keep
   385  	// alive information is lost during inlining.
   386  	//
   387  	// TODO(prattmic): This is handled on calls during escape analysis,
   388  	// which is after inlining. Move prior to inlining so the keep-alive is
   389  	// maintained after inlining.
   390  	if fn.Pragma&ir.UintptrKeepAlive != 0 {
   391  		reason = "marked as having a keep-alive uintptr argument"
   392  		return reason
   393  	}
   394  
   395  	// If marked as "go:uintptrescapes", don't inline, since the escape
   396  	// information is lost during inlining.
   397  	if fn.Pragma&ir.UintptrEscapes != 0 {
   398  		reason = "marked as having an escaping uintptr argument"
   399  		return reason
   400  	}
   401  
   402  	// The nowritebarrierrec checker currently works at function
   403  	// granularity, so inlining yeswritebarrierrec functions can confuse it
   404  	// (#22342). As a workaround, disallow inlining them for now.
   405  	if fn.Pragma&ir.Yeswritebarrierrec != 0 {
   406  		reason = "marked go:yeswritebarrierrec"
   407  		return reason
   408  	}
   409  
   410  	// If a local function has no fn.Body (is defined outside of Go), cannot inline it.
   411  	// Imported functions don't have fn.Body but might have inline body in fn.Inl.
   412  	if len(fn.Body) == 0 && !typecheck.HaveInlineBody(fn) {
   413  		reason = "no function body"
   414  		return reason
   415  	}
   416  
   417  	return ""
   418  }
   419  
   420  // canDelayResults reports whether inlined calls to fn can delay
   421  // declaring the result parameter until the "return" statement.
   422  func canDelayResults(fn *ir.Func) bool {
   423  	// We can delay declaring+initializing result parameters if:
   424  	// (1) there's exactly one "return" statement in the inlined function;
   425  	// (2) it's not an empty return statement (#44355); and
   426  	// (3) the result parameters aren't named.
   427  
   428  	nreturns := 0
   429  	ir.VisitList(fn.Body, func(n ir.Node) {
   430  		if n, ok := n.(*ir.ReturnStmt); ok {
   431  			nreturns++
   432  			if len(n.Results) == 0 {
   433  				nreturns++ // empty return statement (case 2)
   434  			}
   435  		}
   436  	})
   437  
   438  	if nreturns != 1 {
   439  		return false // not exactly one return statement (case 1)
   440  	}
   441  
   442  	// temporaries for return values.
   443  	for _, param := range fn.Type().Results() {
   444  		if sym := param.Sym; sym != nil && !sym.IsBlank() {
   445  			return false // found a named result parameter (case 3)
   446  		}
   447  	}
   448  
   449  	return true
   450  }
   451  
   452  // hairyVisitor visits a function body to determine its inlining
   453  // hairiness and whether or not it can be inlined.
   454  type hairyVisitor struct {
   455  	// This is needed to access the current caller in the doNode function.
   456  	curFunc       *ir.Func
   457  	isBigFunc     bool
   458  	budget        int32
   459  	maxBudget     int32
   460  	reason        string
   461  	extraCallCost int32
   462  	usedLocals    ir.NameSet
   463  	do            func(ir.Node) bool
   464  	profile       *pgo.Profile
   465  }
   466  
   467  func (v *hairyVisitor) tooHairy(fn *ir.Func) bool {
   468  	v.do = v.doNode // cache closure
   469  	if ir.DoChildren(fn, v.do) {
   470  		return true
   471  	}
   472  	if v.budget < 0 {
   473  		v.reason = fmt.Sprintf("function too complex: cost %d exceeds budget %d", v.maxBudget-v.budget, v.maxBudget)
   474  		return true
   475  	}
   476  	return false
   477  }
   478  
   479  // doNode visits n and its children, updates the state in v, and returns true if
   480  // n makes the current function too hairy for inlining.
   481  func (v *hairyVisitor) doNode(n ir.Node) bool {
   482  	if n == nil {
   483  		return false
   484  	}
   485  opSwitch:
   486  	switch n.Op() {
   487  	// Call is okay if inlinable and we have the budget for the body.
   488  	case ir.OCALLFUNC:
   489  		n := n.(*ir.CallExpr)
   490  		// Functions that call runtime.getcaller{pc,sp} can not be inlined
   491  		// because getcaller{pc,sp} expect a pointer to the caller's first argument.
   492  		//
   493  		// runtime.throw is a "cheap call" like panic in normal code.
   494  		var cheap bool
   495  		if n.Fun.Op() == ir.ONAME {
   496  			name := n.Fun.(*ir.Name)
   497  			if name.Class == ir.PFUNC {
   498  				switch fn := types.RuntimeSymName(name.Sym()); fn {
   499  				case "getcallerpc", "getcallersp":
   500  					v.reason = "call to " + fn
   501  					return true
   502  				case "throw":
   503  					v.budget -= inlineExtraThrowCost
   504  					break opSwitch
   505  				case "panicrangeexit":
   506  					cheap = true
   507  				}
   508  				// Special case for reflect.noescape. It does just type
   509  				// conversions to appease the escape analysis, and doesn't
   510  				// generate code.
   511  				if types.ReflectSymName(name.Sym()) == "noescape" {
   512  					cheap = true
   513  				}
   514  			}
   515  			// Special case for coverage counter updates; although
   516  			// these correspond to real operations, we treat them as
   517  			// zero cost for the moment. This is due to the existence
   518  			// of tests that are sensitive to inlining-- if the
   519  			// insertion of coverage instrumentation happens to tip a
   520  			// given function over the threshold and move it from
   521  			// "inlinable" to "not-inlinable", this can cause changes
   522  			// in allocation behavior, which can then result in test
   523  			// failures (a good example is the TestAllocations in
   524  			// crypto/ed25519).
   525  			if isAtomicCoverageCounterUpdate(n) {
   526  				return false
   527  			}
   528  		}
   529  		if n.Fun.Op() == ir.OMETHEXPR {
   530  			if meth := ir.MethodExprName(n.Fun); meth != nil {
   531  				if fn := meth.Func; fn != nil {
   532  					s := fn.Sym()
   533  					if types.RuntimeSymName(s) == "heapBits.nextArena" {
   534  						// Special case: explicitly allow mid-stack inlining of
   535  						// runtime.heapBits.next even though it calls slow-path
   536  						// runtime.heapBits.nextArena.
   537  						cheap = true
   538  					}
   539  					// Special case: on architectures that can do unaligned loads,
   540  					// explicitly mark encoding/binary methods as cheap,
   541  					// because in practice they are, even though our inlining
   542  					// budgeting system does not see that. See issue 42958.
   543  					if base.Ctxt.Arch.CanMergeLoads && s.Pkg.Path == "encoding/binary" {
   544  						switch s.Name {
   545  						case "littleEndian.Uint64", "littleEndian.Uint32", "littleEndian.Uint16",
   546  							"bigEndian.Uint64", "bigEndian.Uint32", "bigEndian.Uint16",
   547  							"littleEndian.PutUint64", "littleEndian.PutUint32", "littleEndian.PutUint16",
   548  							"bigEndian.PutUint64", "bigEndian.PutUint32", "bigEndian.PutUint16",
   549  							"littleEndian.AppendUint64", "littleEndian.AppendUint32", "littleEndian.AppendUint16",
   550  							"bigEndian.AppendUint64", "bigEndian.AppendUint32", "bigEndian.AppendUint16":
   551  							cheap = true
   552  						}
   553  					}
   554  				}
   555  			}
   556  		}
   557  		if cheap {
   558  			break // treat like any other node, that is, cost of 1
   559  		}
   560  
   561  		if ir.IsIntrinsicCall(n) {
   562  			// Treat like any other node.
   563  			break
   564  		}
   565  
   566  		if callee := inlCallee(v.curFunc, n.Fun, v.profile); callee != nil && typecheck.HaveInlineBody(callee) {
   567  			// Check whether we'd actually inline this call. Set
   568  			// log == false since we aren't actually doing inlining
   569  			// yet.
   570  			if ok, _ := canInlineCallExpr(v.curFunc, n, callee, v.isBigFunc, false); ok {
   571  				// mkinlcall would inline this call [1], so use
   572  				// the cost of the inline body as the cost of
   573  				// the call, as that is what will actually
   574  				// appear in the code.
   575  				//
   576  				// [1] This is almost a perfect match to the
   577  				// mkinlcall logic, except that
   578  				// canInlineCallExpr considers inlining cycles
   579  				// by looking at what has already been inlined.
   580  				// Since we haven't done any inlining yet we
   581  				// will miss those.
   582  				v.budget -= callee.Inl.Cost
   583  				break
   584  			}
   585  		}
   586  
   587  		// Call cost for non-leaf inlining.
   588  		v.budget -= v.extraCallCost
   589  
   590  	case ir.OCALLMETH:
   591  		base.FatalfAt(n.Pos(), "OCALLMETH missed by typecheck")
   592  
   593  	// Things that are too hairy, irrespective of the budget
   594  	case ir.OCALL, ir.OCALLINTER:
   595  		// Call cost for non-leaf inlining.
   596  		v.budget -= v.extraCallCost
   597  
   598  	case ir.OPANIC:
   599  		n := n.(*ir.UnaryExpr)
   600  		if n.X.Op() == ir.OCONVIFACE && n.X.(*ir.ConvExpr).Implicit() {
   601  			// Hack to keep reflect.flag.mustBe inlinable for TestIntendedInlining.
   602  			// Before CL 284412, these conversions were introduced later in the
   603  			// compiler, so they didn't count against inlining budget.
   604  			v.budget++
   605  		}
   606  		v.budget -= inlineExtraPanicCost
   607  
   608  	case ir.ORECOVER:
   609  		base.FatalfAt(n.Pos(), "ORECOVER missed typecheck")
   610  	case ir.ORECOVERFP:
   611  		// recover matches the argument frame pointer to find
   612  		// the right panic value, so it needs an argument frame.
   613  		v.reason = "call to recover"
   614  		return true
   615  
   616  	case ir.OCLOSURE:
   617  		if base.Debug.InlFuncsWithClosures == 0 {
   618  			v.reason = "not inlining functions with closures"
   619  			return true
   620  		}
   621  
   622  		// TODO(danscales): Maybe make budget proportional to number of closure
   623  		// variables, e.g.:
   624  		//v.budget -= int32(len(n.(*ir.ClosureExpr).Func.ClosureVars) * 3)
   625  		// TODO(austin): However, if we're able to inline this closure into
   626  		// v.curFunc, then we actually pay nothing for the closure captures. We
   627  		// should try to account for that if we're going to account for captures.
   628  		v.budget -= 15
   629  
   630  	case ir.OGO, ir.ODEFER, ir.OTAILCALL:
   631  		v.reason = "unhandled op " + n.Op().String()
   632  		return true
   633  
   634  	case ir.OAPPEND:
   635  		v.budget -= inlineExtraAppendCost
   636  
   637  	case ir.OADDR:
   638  		n := n.(*ir.AddrExpr)
   639  		// Make "&s.f" cost 0 when f's offset is zero.
   640  		if dot, ok := n.X.(*ir.SelectorExpr); ok && (dot.Op() == ir.ODOT || dot.Op() == ir.ODOTPTR) {
   641  			if _, ok := dot.X.(*ir.Name); ok && dot.Selection.Offset == 0 {
   642  				v.budget += 2 // undo ir.OADDR+ir.ODOT/ir.ODOTPTR
   643  			}
   644  		}
   645  
   646  	case ir.ODEREF:
   647  		// *(*X)(unsafe.Pointer(&x)) is low-cost
   648  		n := n.(*ir.StarExpr)
   649  
   650  		ptr := n.X
   651  		for ptr.Op() == ir.OCONVNOP {
   652  			ptr = ptr.(*ir.ConvExpr).X
   653  		}
   654  		if ptr.Op() == ir.OADDR {
   655  			v.budget += 1 // undo half of default cost of ir.ODEREF+ir.OADDR
   656  		}
   657  
   658  	case ir.OCONVNOP:
   659  		// This doesn't produce code, but the children might.
   660  		v.budget++ // undo default cost
   661  
   662  	case ir.OFALL, ir.OTYPE:
   663  		// These nodes don't produce code; omit from inlining budget.
   664  		return false
   665  
   666  	case ir.OIF:
   667  		n := n.(*ir.IfStmt)
   668  		if ir.IsConst(n.Cond, constant.Bool) {
   669  			// This if and the condition cost nothing.
   670  			if doList(n.Init(), v.do) {
   671  				return true
   672  			}
   673  			if ir.BoolVal(n.Cond) {
   674  				return doList(n.Body, v.do)
   675  			} else {
   676  				return doList(n.Else, v.do)
   677  			}
   678  		}
   679  
   680  	case ir.ONAME:
   681  		n := n.(*ir.Name)
   682  		if n.Class == ir.PAUTO {
   683  			v.usedLocals.Add(n)
   684  		}
   685  
   686  	case ir.OBLOCK:
   687  		// The only OBLOCK we should see at this point is an empty one.
   688  		// In any event, let the visitList(n.List()) below take care of the statements,
   689  		// and don't charge for the OBLOCK itself. The ++ undoes the -- below.
   690  		v.budget++
   691  
   692  	case ir.OMETHVALUE, ir.OSLICELIT:
   693  		v.budget-- // Hack for toolstash -cmp.
   694  
   695  	case ir.OMETHEXPR:
   696  		v.budget++ // Hack for toolstash -cmp.
   697  
   698  	case ir.OAS2:
   699  		n := n.(*ir.AssignListStmt)
   700  
   701  		// Unified IR unconditionally rewrites:
   702  		//
   703  		//	a, b = f()
   704  		//
   705  		// into:
   706  		//
   707  		//	DCL tmp1
   708  		//	DCL tmp2
   709  		//	tmp1, tmp2 = f()
   710  		//	a, b = tmp1, tmp2
   711  		//
   712  		// so that it can insert implicit conversions as necessary. To
   713  		// minimize impact to the existing inlining heuristics (in
   714  		// particular, to avoid breaking the existing inlinability regress
   715  		// tests), we need to compensate for this here.
   716  		//
   717  		// See also identical logic in IsBigFunc.
   718  		if len(n.Rhs) > 0 {
   719  			if init := n.Rhs[0].Init(); len(init) == 1 {
   720  				if _, ok := init[0].(*ir.AssignListStmt); ok {
   721  					// 4 for each value, because each temporary variable now
   722  					// appears 3 times (DCL, LHS, RHS), plus an extra DCL node.
   723  					//
   724  					// 1 for the extra "tmp1, tmp2 = f()" assignment statement.
   725  					v.budget += 4*int32(len(n.Lhs)) + 1
   726  				}
   727  			}
   728  		}
   729  
   730  	case ir.OAS:
   731  		// Special case for coverage counter updates and coverage
   732  		// function registrations. Although these correspond to real
   733  		// operations, we treat them as zero cost for the moment. This
   734  		// is primarily due to the existence of tests that are
   735  		// sensitive to inlining-- if the insertion of coverage
   736  		// instrumentation happens to tip a given function over the
   737  		// threshold and move it from "inlinable" to "not-inlinable",
   738  		// this can cause changes in allocation behavior, which can
   739  		// then result in test failures (a good example is the
   740  		// TestAllocations in crypto/ed25519).
   741  		n := n.(*ir.AssignStmt)
   742  		if n.X.Op() == ir.OINDEX && isIndexingCoverageCounter(n.X) {
   743  			return false
   744  		}
   745  	}
   746  
   747  	v.budget--
   748  
   749  	// When debugging, don't stop early, to get full cost of inlining this function
   750  	if v.budget < 0 && base.Flag.LowerM < 2 && !logopt.Enabled() {
   751  		v.reason = "too expensive"
   752  		return true
   753  	}
   754  
   755  	return ir.DoChildren(n, v.do)
   756  }
   757  
   758  // IsBigFunc reports whether fn is a "big" function.
   759  //
   760  // Note: The criteria for "big" is heuristic and subject to change.
   761  func IsBigFunc(fn *ir.Func) bool {
   762  	budget := inlineBigFunctionNodes
   763  	return ir.Any(fn, func(n ir.Node) bool {
   764  		// See logic in hairyVisitor.doNode, explaining unified IR's
   765  		// handling of "a, b = f()" assignments.
   766  		if n, ok := n.(*ir.AssignListStmt); ok && n.Op() == ir.OAS2 && len(n.Rhs) > 0 {
   767  			if init := n.Rhs[0].Init(); len(init) == 1 {
   768  				if _, ok := init[0].(*ir.AssignListStmt); ok {
   769  					budget += 4*len(n.Lhs) + 1
   770  				}
   771  			}
   772  		}
   773  
   774  		budget--
   775  		return budget <= 0
   776  	})
   777  }
   778  
   779  // TryInlineCall returns an inlined call expression for call, or nil
   780  // if inlining is not possible.
   781  func TryInlineCall(callerfn *ir.Func, call *ir.CallExpr, bigCaller bool, profile *pgo.Profile) *ir.InlinedCallExpr {
   782  	if base.Flag.LowerL == 0 {
   783  		return nil
   784  	}
   785  	if call.Op() != ir.OCALLFUNC {
   786  		return nil
   787  	}
   788  	if call.GoDefer || call.NoInline {
   789  		return nil
   790  	}
   791  
   792  	// Prevent inlining some reflect.Value methods when using checkptr,
   793  	// even when package reflect was compiled without it (#35073).
   794  	if base.Debug.Checkptr != 0 && call.Fun.Op() == ir.OMETHEXPR {
   795  		if method := ir.MethodExprName(call.Fun); method != nil {
   796  			switch types.ReflectSymName(method.Sym()) {
   797  			case "Value.UnsafeAddr", "Value.Pointer":
   798  				return nil
   799  			}
   800  		}
   801  	}
   802  
   803  	if base.Flag.LowerM > 3 {
   804  		fmt.Printf("%v:call to func %+v\n", ir.Line(call), call.Fun)
   805  	}
   806  	if ir.IsIntrinsicCall(call) {
   807  		return nil
   808  	}
   809  	if fn := inlCallee(callerfn, call.Fun, profile); fn != nil && typecheck.HaveInlineBody(fn) {
   810  		return mkinlcall(callerfn, call, fn, bigCaller)
   811  	}
   812  	return nil
   813  }
   814  
   815  // inlCallee takes a function-typed expression and returns the underlying function ONAME
   816  // that it refers to if statically known. Otherwise, it returns nil.
   817  func inlCallee(caller *ir.Func, fn ir.Node, profile *pgo.Profile) (res *ir.Func) {
   818  	fn = ir.StaticValue(fn)
   819  	switch fn.Op() {
   820  	case ir.OMETHEXPR:
   821  		fn := fn.(*ir.SelectorExpr)
   822  		n := ir.MethodExprName(fn)
   823  		// Check that receiver type matches fn.X.
   824  		// TODO(mdempsky): Handle implicit dereference
   825  		// of pointer receiver argument?
   826  		if n == nil || !types.Identical(n.Type().Recv().Type, fn.X.Type()) {
   827  			return nil
   828  		}
   829  		return n.Func
   830  	case ir.ONAME:
   831  		fn := fn.(*ir.Name)
   832  		if fn.Class == ir.PFUNC {
   833  			return fn.Func
   834  		}
   835  	case ir.OCLOSURE:
   836  		fn := fn.(*ir.ClosureExpr)
   837  		c := fn.Func
   838  		if len(c.ClosureVars) != 0 && c.ClosureVars[0].Outer.Curfn != caller {
   839  			return nil // inliner doesn't support inlining across closure frames
   840  		}
   841  		CanInline(c, profile)
   842  		return c
   843  	}
   844  	return nil
   845  }
   846  
   847  var inlgen int
   848  
   849  // SSADumpInline gives the SSA back end a chance to dump the function
   850  // when producing output for debugging the compiler itself.
   851  var SSADumpInline = func(*ir.Func) {}
   852  
   853  // InlineCall allows the inliner implementation to be overridden.
   854  // If it returns nil, the function will not be inlined.
   855  var InlineCall = func(callerfn *ir.Func, call *ir.CallExpr, fn *ir.Func, inlIndex int) *ir.InlinedCallExpr {
   856  	base.Fatalf("inline.InlineCall not overridden")
   857  	panic("unreachable")
   858  }
   859  
   860  // inlineCostOK returns true if call n from caller to callee is cheap enough to
   861  // inline. bigCaller indicates that caller is a big function.
   862  //
   863  // In addition to the "cost OK" boolean, it also returns the "max
   864  // cost" limit used to make the decision (which may differ depending
   865  // on func size), and the score assigned to this specific callsite.
   866  func inlineCostOK(n *ir.CallExpr, caller, callee *ir.Func, bigCaller bool) (bool, int32, int32) {
   867  	maxCost := int32(inlineMaxBudget)
   868  	if bigCaller {
   869  		// We use this to restrict inlining into very big functions.
   870  		// See issue 26546 and 17566.
   871  		maxCost = inlineBigFunctionMaxCost
   872  	}
   873  
   874  	metric := callee.Inl.Cost
   875  	if inlheur.Enabled() {
   876  		score, ok := inlheur.GetCallSiteScore(caller, n)
   877  		if ok {
   878  			metric = int32(score)
   879  		}
   880  	}
   881  
   882  	if metric <= maxCost {
   883  		// Simple case. Function is already cheap enough.
   884  		return true, 0, metric
   885  	}
   886  
   887  	// We'll also allow inlining of hot functions below inlineHotMaxBudget,
   888  	// but only in small functions.
   889  
   890  	lineOffset := pgo.NodeLineOffset(n, caller)
   891  	csi := pgo.CallSiteInfo{LineOffset: lineOffset, Caller: caller}
   892  	if _, ok := candHotEdgeMap[csi]; !ok {
   893  		// Cold
   894  		return false, maxCost, metric
   895  	}
   896  
   897  	// Hot
   898  
   899  	if bigCaller {
   900  		if base.Debug.PGODebug > 0 {
   901  			fmt.Printf("hot-big check disallows inlining for call %s (cost %d) at %v in big function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
   902  		}
   903  		return false, maxCost, metric
   904  	}
   905  
   906  	if metric > inlineHotMaxBudget {
   907  		return false, inlineHotMaxBudget, metric
   908  	}
   909  
   910  	if !base.PGOHash.MatchPosWithInfo(n.Pos(), "inline", nil) {
   911  		// De-selected by PGO Hash.
   912  		return false, maxCost, metric
   913  	}
   914  
   915  	if base.Debug.PGODebug > 0 {
   916  		fmt.Printf("hot-budget check allows inlining for call %s (cost %d) at %v in function %s\n", ir.PkgFuncName(callee), callee.Inl.Cost, ir.Line(n), ir.PkgFuncName(caller))
   917  	}
   918  
   919  	return true, 0, metric
   920  }
   921  
   922  // canInlineCallsite returns true if the call n from caller to callee
   923  // can be inlined, plus the score computed for the call expr in
   924  // question. bigCaller indicates that caller is a big function. log
   925  // indicates that the 'cannot inline' reason should be logged.
   926  //
   927  // Preconditions: CanInline(callee) has already been called.
   928  func canInlineCallExpr(callerfn *ir.Func, n *ir.CallExpr, callee *ir.Func, bigCaller bool, log bool) (bool, int32) {
   929  	if callee.Inl == nil {
   930  		// callee is never inlinable.
   931  		if log && logopt.Enabled() {
   932  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   933  				fmt.Sprintf("%s cannot be inlined", ir.PkgFuncName(callee)))
   934  		}
   935  		return false, 0
   936  	}
   937  
   938  	ok, maxCost, callSiteScore := inlineCostOK(n, callerfn, callee, bigCaller)
   939  	if !ok {
   940  		// callee cost too high for this call site.
   941  		if log && logopt.Enabled() {
   942  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   943  				fmt.Sprintf("cost %d of %s exceeds max caller cost %d", callee.Inl.Cost, ir.PkgFuncName(callee), maxCost))
   944  		}
   945  		return false, 0
   946  	}
   947  
   948  	if callee == callerfn {
   949  		// Can't recursively inline a function into itself.
   950  		if log && logopt.Enabled() {
   951  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", fmt.Sprintf("recursive call to %s", ir.FuncName(callerfn)))
   952  		}
   953  		return false, 0
   954  	}
   955  
   956  	if base.Flag.Cfg.Instrumenting && types.IsNoInstrumentPkg(callee.Sym().Pkg) {
   957  		// Runtime package must not be instrumented.
   958  		// Instrument skips runtime package. However, some runtime code can be
   959  		// inlined into other packages and instrumented there. To avoid this,
   960  		// we disable inlining of runtime functions when instrumenting.
   961  		// The example that we observed is inlining of LockOSThread,
   962  		// which lead to false race reports on m contents.
   963  		if log && logopt.Enabled() {
   964  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   965  				fmt.Sprintf("call to runtime function %s in instrumented build", ir.PkgFuncName(callee)))
   966  		}
   967  		return false, 0
   968  	}
   969  
   970  	if base.Flag.Race && types.IsNoRacePkg(callee.Sym().Pkg) {
   971  		if log && logopt.Enabled() {
   972  			logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   973  				fmt.Sprintf(`call to into "no-race" package function %s in race build`, ir.PkgFuncName(callee)))
   974  		}
   975  		return false, 0
   976  	}
   977  
   978  	// Check if we've already inlined this function at this particular
   979  	// call site, in order to stop inlining when we reach the beginning
   980  	// of a recursion cycle again. We don't inline immediately recursive
   981  	// functions, but allow inlining if there is a recursion cycle of
   982  	// many functions. Most likely, the inlining will stop before we
   983  	// even hit the beginning of the cycle again, but this catches the
   984  	// unusual case.
   985  	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
   986  	sym := callee.Linksym()
   987  	for inlIndex := parent; inlIndex >= 0; inlIndex = base.Ctxt.InlTree.Parent(inlIndex) {
   988  		if base.Ctxt.InlTree.InlinedFunction(inlIndex) == sym {
   989  			if log {
   990  				if base.Flag.LowerM > 1 {
   991  					fmt.Printf("%v: cannot inline %v into %v: repeated recursive cycle\n", ir.Line(n), callee, ir.FuncName(callerfn))
   992  				}
   993  				if logopt.Enabled() {
   994  					logopt.LogOpt(n.Pos(), "cannotInlineCall", "inline", ir.FuncName(callerfn),
   995  						fmt.Sprintf("repeated recursive cycle to %s", ir.PkgFuncName(callee)))
   996  				}
   997  			}
   998  			return false, 0
   999  		}
  1000  	}
  1001  
  1002  	return true, callSiteScore
  1003  }
  1004  
  1005  // mkinlcall returns an OINLCALL node that can replace OCALLFUNC n, or
  1006  // nil if it cannot be inlined. callerfn is the function that contains
  1007  // n, and fn is the function being called.
  1008  //
  1009  // The result of mkinlcall MUST be assigned back to n, e.g.
  1010  //
  1011  //	n.Left = mkinlcall(n.Left, fn, isddd)
  1012  func mkinlcall(callerfn *ir.Func, n *ir.CallExpr, fn *ir.Func, bigCaller bool) *ir.InlinedCallExpr {
  1013  	ok, score := canInlineCallExpr(callerfn, n, fn, bigCaller, true)
  1014  	if !ok {
  1015  		return nil
  1016  	}
  1017  	typecheck.AssertFixedCall(n)
  1018  
  1019  	parent := base.Ctxt.PosTable.Pos(n.Pos()).Base().InliningIndex()
  1020  	sym := fn.Linksym()
  1021  	inlIndex := base.Ctxt.InlTree.Add(parent, n.Pos(), sym, ir.FuncName(fn))
  1022  
  1023  	closureInitLSym := func(n *ir.CallExpr, fn *ir.Func) {
  1024  		// The linker needs FuncInfo metadata for all inlined
  1025  		// functions. This is typically handled by gc.enqueueFunc
  1026  		// calling ir.InitLSym for all function declarations in
  1027  		// typecheck.Target.Decls (ir.UseClosure adds all closures to
  1028  		// Decls).
  1029  		//
  1030  		// However, non-trivial closures in Decls are ignored, and are
  1031  		// insteaded enqueued when walk of the calling function
  1032  		// discovers them.
  1033  		//
  1034  		// This presents a problem for direct calls to closures.
  1035  		// Inlining will replace the entire closure definition with its
  1036  		// body, which hides the closure from walk and thus suppresses
  1037  		// symbol creation.
  1038  		//
  1039  		// Explicitly create a symbol early in this edge case to ensure
  1040  		// we keep this metadata.
  1041  		//
  1042  		// TODO: Refactor to keep a reference so this can all be done
  1043  		// by enqueueFunc.
  1044  
  1045  		if n.Op() != ir.OCALLFUNC {
  1046  			// Not a standard call.
  1047  			return
  1048  		}
  1049  		if n.Fun.Op() != ir.OCLOSURE {
  1050  			// Not a direct closure call.
  1051  			return
  1052  		}
  1053  
  1054  		clo := n.Fun.(*ir.ClosureExpr)
  1055  		if ir.IsTrivialClosure(clo) {
  1056  			// enqueueFunc will handle trivial closures anyways.
  1057  			return
  1058  		}
  1059  
  1060  		ir.InitLSym(fn, true)
  1061  	}
  1062  
  1063  	closureInitLSym(n, fn)
  1064  
  1065  	if base.Flag.GenDwarfInl > 0 {
  1066  		if !sym.WasInlined() {
  1067  			base.Ctxt.DwFixups.SetPrecursorFunc(sym, fn)
  1068  			sym.Set(obj.AttrWasInlined, true)
  1069  		}
  1070  	}
  1071  
  1072  	if base.Flag.LowerM != 0 {
  1073  		if buildcfg.Experiment.NewInliner {
  1074  			fmt.Printf("%v: inlining call to %v with score %d\n",
  1075  				ir.Line(n), fn, score)
  1076  		} else {
  1077  			fmt.Printf("%v: inlining call to %v\n", ir.Line(n), fn)
  1078  		}
  1079  	}
  1080  	if base.Flag.LowerM > 2 {
  1081  		fmt.Printf("%v: Before inlining: %+v\n", ir.Line(n), n)
  1082  	}
  1083  
  1084  	res := InlineCall(callerfn, n, fn, inlIndex)
  1085  
  1086  	if res == nil {
  1087  		base.FatalfAt(n.Pos(), "inlining call to %v failed", fn)
  1088  	}
  1089  
  1090  	if base.Flag.LowerM > 2 {
  1091  		fmt.Printf("%v: After inlining %+v\n\n", ir.Line(res), res)
  1092  	}
  1093  
  1094  	if inlheur.Enabled() {
  1095  		inlheur.UpdateCallsiteTable(callerfn, n, res)
  1096  	}
  1097  
  1098  	return res
  1099  }
  1100  
  1101  // CalleeEffects appends any side effects from evaluating callee to init.
  1102  func CalleeEffects(init *ir.Nodes, callee ir.Node) {
  1103  	for {
  1104  		init.Append(ir.TakeInit(callee)...)
  1105  
  1106  		switch callee.Op() {
  1107  		case ir.ONAME, ir.OCLOSURE, ir.OMETHEXPR:
  1108  			return // done
  1109  
  1110  		case ir.OCONVNOP:
  1111  			conv := callee.(*ir.ConvExpr)
  1112  			callee = conv.X
  1113  
  1114  		case ir.OINLCALL:
  1115  			ic := callee.(*ir.InlinedCallExpr)
  1116  			init.Append(ic.Body.Take()...)
  1117  			callee = ic.SingleResult()
  1118  
  1119  		default:
  1120  			base.FatalfAt(callee.Pos(), "unexpected callee expression: %v", callee)
  1121  		}
  1122  	}
  1123  }
  1124  
  1125  func pruneUnusedAutos(ll []*ir.Name, vis *hairyVisitor) []*ir.Name {
  1126  	s := make([]*ir.Name, 0, len(ll))
  1127  	for _, n := range ll {
  1128  		if n.Class == ir.PAUTO {
  1129  			if !vis.usedLocals.Has(n) {
  1130  				// TODO(mdempsky): Simplify code after confident that this
  1131  				// never happens anymore.
  1132  				base.FatalfAt(n.Pos(), "unused auto: %v", n)
  1133  				continue
  1134  			}
  1135  		}
  1136  		s = append(s, n)
  1137  	}
  1138  	return s
  1139  }
  1140  
  1141  // numNonClosures returns the number of functions in list which are not closures.
  1142  func numNonClosures(list []*ir.Func) int {
  1143  	count := 0
  1144  	for _, fn := range list {
  1145  		if fn.OClosure == nil {
  1146  			count++
  1147  		}
  1148  	}
  1149  	return count
  1150  }
  1151  
  1152  func doList(list []ir.Node, do func(ir.Node) bool) bool {
  1153  	for _, x := range list {
  1154  		if x != nil {
  1155  			if do(x) {
  1156  				return true
  1157  			}
  1158  		}
  1159  	}
  1160  	return false
  1161  }
  1162  
  1163  // isIndexingCoverageCounter returns true if the specified node 'n' is indexing
  1164  // into a coverage counter array.
  1165  func isIndexingCoverageCounter(n ir.Node) bool {
  1166  	if n.Op() != ir.OINDEX {
  1167  		return false
  1168  	}
  1169  	ixn := n.(*ir.IndexExpr)
  1170  	if ixn.X.Op() != ir.ONAME || !ixn.X.Type().IsArray() {
  1171  		return false
  1172  	}
  1173  	nn := ixn.X.(*ir.Name)
  1174  	return nn.CoverageCounter()
  1175  }
  1176  
  1177  // isAtomicCoverageCounterUpdate examines the specified node to
  1178  // determine whether it represents a call to sync/atomic.AddUint32 to
  1179  // increment a coverage counter.
  1180  func isAtomicCoverageCounterUpdate(cn *ir.CallExpr) bool {
  1181  	if cn.Fun.Op() != ir.ONAME {
  1182  		return false
  1183  	}
  1184  	name := cn.Fun.(*ir.Name)
  1185  	if name.Class != ir.PFUNC {
  1186  		return false
  1187  	}
  1188  	fn := name.Sym().Name
  1189  	if name.Sym().Pkg.Path != "sync/atomic" ||
  1190  		(fn != "AddUint32" && fn != "StoreUint32") {
  1191  		return false
  1192  	}
  1193  	if len(cn.Args) != 2 || cn.Args[0].Op() != ir.OADDR {
  1194  		return false
  1195  	}
  1196  	adn := cn.Args[0].(*ir.AddrExpr)
  1197  	v := isIndexingCoverageCounter(adn.X)
  1198  	return v
  1199  }
  1200  
  1201  func PostProcessCallSites(profile *pgo.Profile) {
  1202  	if base.Debug.DumpInlCallSiteScores != 0 {
  1203  		budgetCallback := func(fn *ir.Func, prof *pgo.Profile) (int32, bool) {
  1204  			v := inlineBudget(fn, prof, false, false)
  1205  			return v, v == inlineHotMaxBudget
  1206  		}
  1207  		inlheur.DumpInlCallSiteScores(profile, budgetCallback)
  1208  	}
  1209  }
  1210  
  1211  func analyzeFuncProps(fn *ir.Func, p *pgo.Profile) {
  1212  	canInline := func(fn *ir.Func) { CanInline(fn, p) }
  1213  	budgetForFunc := func(fn *ir.Func) int32 {
  1214  		return inlineBudget(fn, p, true, false)
  1215  	}
  1216  	inlheur.AnalyzeFunc(fn, canInline, budgetForFunc, inlineMaxBudget)
  1217  }
  1218
View as plain text