Source file src/runtime/stack.go

Documentation: runtime

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/cpu"
    10  	"internal/goarch"
    11  	"internal/goos"
    12  	"runtime/internal/atomic"
    13  	"runtime/internal/sys"
    14  	"unsafe"
    15  )
    16  
    17  /*
    18  Stack layout parameters.
    19  Included both by runtime (compiled via 6c) and linkers (compiled via gcc).
    20  
    21  The per-goroutine g->stackguard is set to point StackGuard bytes
    22  above the bottom of the stack.  Each function compares its stack
    23  pointer against g->stackguard to check for overflow.  To cut one
    24  instruction from the check sequence for functions with tiny frames,
    25  the stack is allowed to protrude StackSmall bytes below the stack
    26  guard.  Functions with large frames don't bother with the check and
    27  always call morestack.  The sequences are (for amd64, others are
    28  similar):
    29  
    30  	guard = g->stackguard
    31  	frame = function's stack frame size
    32  	argsize = size of function arguments (call + return)
    33  
    34  	stack frame size <= StackSmall:
    35  		CMPQ guard, SP
    36  		JHI 3(PC)
    37  		MOVQ m->morearg, $(argsize << 32)
    38  		CALL morestack(SB)
    39  
    40  	stack frame size > StackSmall but < StackBig
    41  		LEAQ (frame-StackSmall)(SP), R0
    42  		CMPQ guard, R0
    43  		JHI 3(PC)
    44  		MOVQ m->morearg, $(argsize << 32)
    45  		CALL morestack(SB)
    46  
    47  	stack frame size >= StackBig:
    48  		MOVQ m->morearg, $((argsize << 32) | frame)
    49  		CALL morestack(SB)
    50  
    51  The bottom StackGuard - StackSmall bytes are important: there has
    52  to be enough room to execute functions that refuse to check for
    53  stack overflow, either because they need to be adjacent to the
    54  actual caller's frame (deferproc) or because they handle the imminent
    55  stack overflow (morestack).
    56  
    57  For example, deferproc might call malloc, which does one of the
    58  above checks (without allocating a full frame), which might trigger
    59  a call to morestack.  This sequence needs to fit in the bottom
    60  section of the stack.  On amd64, morestack's frame is 40 bytes, and
    61  deferproc's frame is 56 bytes.  That fits well within the
    62  StackGuard - StackSmall bytes at the bottom.
    63  The linkers explore all possible call traces involving non-splitting
    64  functions to make sure that this limit cannot be violated.
    65  */
    66  
    67  const (
    68  	// stackSystem is a number of additional bytes to add
    69  	// to each stack below the usual guard area for OS-specific
    70  	// purposes like signal handling. Used on Windows, Plan 9,
    71  	// and iOS because they do not use a separate stack.
    72  	stackSystem = goos.IsWindows*512*goarch.PtrSize + goos.IsPlan9*512 + goos.IsIos*goarch.IsArm64*1024
    73  
    74  	// The minimum size of stack used by Go code
    75  	stackMin = 2048
    76  
    77  	// The minimum stack size to allocate.
    78  	// The hackery here rounds fixedStack0 up to a power of 2.
    79  	fixedStack0 = stackMin + stackSystem
    80  	fixedStack1 = fixedStack0 - 1
    81  	fixedStack2 = fixedStack1 | (fixedStack1 >> 1)
    82  	fixedStack3 = fixedStack2 | (fixedStack2 >> 2)
    83  	fixedStack4 = fixedStack3 | (fixedStack3 >> 4)
    84  	fixedStack5 = fixedStack4 | (fixedStack4 >> 8)
    85  	fixedStack6 = fixedStack5 | (fixedStack5 >> 16)
    86  	fixedStack  = fixedStack6 + 1
    87  
    88  	// stackNosplit is the maximum number of bytes that a chain of NOSPLIT
    89  	// functions can use.
    90  	// This arithmetic must match that in cmd/internal/objabi/stack.go:StackNosplit.
    91  	stackNosplit = abi.StackNosplitBase * sys.StackGuardMultiplier
    92  
    93  	// The stack guard is a pointer this many bytes above the
    94  	// bottom of the stack.
    95  	//
    96  	// The guard leaves enough room for a stackNosplit chain of NOSPLIT calls
    97  	// plus one stackSmall frame plus stackSystem bytes for the OS.
    98  	// This arithmetic must match that in cmd/internal/objabi/stack.go:StackLimit.
    99  	stackGuard = stackNosplit + stackSystem + abi.StackSmall
   100  )
   101  
   102  const (
   103  	// stackDebug == 0: no logging
   104  	//            == 1: logging of per-stack operations
   105  	//            == 2: logging of per-frame operations
   106  	//            == 3: logging of per-word updates
   107  	//            == 4: logging of per-word reads
   108  	stackDebug       = 0
   109  	stackFromSystem  = 0 // allocate stacks from system memory instead of the heap
   110  	stackFaultOnFree = 0 // old stacks are mapped noaccess to detect use after free
   111  	stackNoCache     = 0 // disable per-P small stack caches
   112  
   113  	// check the BP links during traceback.
   114  	debugCheckBP = false
   115  )
   116  
   117  var (
   118  	stackPoisonCopy = 0 // fill stack that should not be accessed with garbage, to detect bad dereferences during copy
   119  )
   120  
   121  const (
   122  	uintptrMask = 1<<(8*goarch.PtrSize) - 1
   123  
   124  	// The values below can be stored to g.stackguard0 to force
   125  	// the next stack check to fail.
   126  	// These are all larger than any real SP.
   127  
   128  	// Goroutine preemption request.
   129  	// 0xfffffade in hex.
   130  	stackPreempt = uintptrMask & -1314
   131  
   132  	// Thread is forking. Causes a split stack check failure.
   133  	// 0xfffffb2e in hex.
   134  	stackFork = uintptrMask & -1234
   135  
   136  	// Force a stack movement. Used for debugging.
   137  	// 0xfffffeed in hex.
   138  	stackForceMove = uintptrMask & -275
   139  
   140  	// stackPoisonMin is the lowest allowed stack poison value.
   141  	stackPoisonMin = uintptrMask & -4096
   142  )
   143  
   144  // Global pool of spans that have free stacks.
   145  // Stacks are assigned an order according to size.
   146  //
   147  //	order = log_2(size/FixedStack)
   148  //
   149  // There is a free list for each order.
   150  var stackpool [_NumStackOrders]struct {
   151  	item stackpoolItem
   152  	_    [(cpu.CacheLinePadSize - unsafe.Sizeof(stackpoolItem{})%cpu.CacheLinePadSize) % cpu.CacheLinePadSize]byte
   153  }
   154  
   155  type stackpoolItem struct {
   156  	_    sys.NotInHeap
   157  	mu   mutex
   158  	span mSpanList
   159  }
   160  
   161  // Global pool of large stack spans.
   162  var stackLarge struct {
   163  	lock mutex
   164  	free [heapAddrBits - pageShift]mSpanList // free lists by log_2(s.npages)
   165  }
   166  
   167  func stackinit() {
   168  	if _StackCacheSize&_PageMask != 0 {
   169  		throw("cache size must be a multiple of page size")
   170  	}
   171  	for i := range stackpool {
   172  		stackpool[i].item.span.init()
   173  		lockInit(&stackpool[i].item.mu, lockRankStackpool)
   174  	}
   175  	for i := range stackLarge.free {
   176  		stackLarge.free[i].init()
   177  		lockInit(&stackLarge.lock, lockRankStackLarge)
   178  	}
   179  }
   180  
   181  // stacklog2 returns ⌊log_2(n)⌋.
   182  func stacklog2(n uintptr) int {
   183  	log2 := 0
   184  	for n > 1 {
   185  		n >>= 1
   186  		log2++
   187  	}
   188  	return log2
   189  }
   190  
   191  // Allocates a stack from the free pool. Must be called with
   192  // stackpool[order].item.mu held.
   193  func stackpoolalloc(order uint8) gclinkptr {
   194  	list := &stackpool[order].item.span
   195  	s := list.first
   196  	lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
   197  	if s == nil {
   198  		// no free stacks. Allocate another span worth.
   199  		s = mheap_.allocManual(_StackCacheSize>>_PageShift, spanAllocStack)
   200  		if s == nil {
   201  			throw("out of memory")
   202  		}
   203  		if s.allocCount != 0 {
   204  			throw("bad allocCount")
   205  		}
   206  		if s.manualFreeList.ptr() != nil {
   207  			throw("bad manualFreeList")
   208  		}
   209  		osStackAlloc(s)
   210  		s.elemsize = fixedStack << order
   211  		for i := uintptr(0); i < _StackCacheSize; i += s.elemsize {
   212  			x := gclinkptr(s.base() + i)
   213  			x.ptr().next = s.manualFreeList
   214  			s.manualFreeList = x
   215  		}
   216  		list.insert(s)
   217  	}
   218  	x := s.manualFreeList
   219  	if x.ptr() == nil {
   220  		throw("span has no free stacks")
   221  	}
   222  	s.manualFreeList = x.ptr().next
   223  	s.allocCount++
   224  	if s.manualFreeList.ptr() == nil {
   225  		// all stacks in s are allocated.
   226  		list.remove(s)
   227  	}
   228  	return x
   229  }
   230  
   231  // Adds stack x to the free pool. Must be called with stackpool[order].item.mu held.
   232  func stackpoolfree(x gclinkptr, order uint8) {
   233  	s := spanOfUnchecked(uintptr(x))
   234  	if s.state.get() != mSpanManual {
   235  		throw("freeing stack not in a stack span")
   236  	}
   237  	if s.manualFreeList.ptr() == nil {
   238  		// s will now have a free stack
   239  		stackpool[order].item.span.insert(s)
   240  	}
   241  	x.ptr().next = s.manualFreeList
   242  	s.manualFreeList = x
   243  	s.allocCount--
   244  	if gcphase == _GCoff && s.allocCount == 0 {
   245  		// Span is completely free. Return it to the heap
   246  		// immediately if we're sweeping.
   247  		//
   248  		// If GC is active, we delay the free until the end of
   249  		// GC to avoid the following type of situation:
   250  		//
   251  		// 1) GC starts, scans a SudoG but does not yet mark the SudoG.elem pointer
   252  		// 2) The stack that pointer points to is copied
   253  		// 3) The old stack is freed
   254  		// 4) The containing span is marked free
   255  		// 5) GC attempts to mark the SudoG.elem pointer. The
   256  		//    marking fails because the pointer looks like a
   257  		//    pointer into a free span.
   258  		//
   259  		// By not freeing, we prevent step #4 until GC is done.
   260  		stackpool[order].item.span.remove(s)
   261  		s.manualFreeList = 0
   262  		osStackFree(s)
   263  		mheap_.freeManual(s, spanAllocStack)
   264  	}
   265  }
   266  
   267  // stackcacherefill/stackcacherelease implement a global pool of stack segments.
   268  // The pool is required to prevent unlimited growth of per-thread caches.
   269  //
   270  //go:systemstack
   271  func stackcacherefill(c *mcache, order uint8) {
   272  	if stackDebug >= 1 {
   273  		print("stackcacherefill order=", order, "\n")
   274  	}
   275  
   276  	// Grab some stacks from the global cache.
   277  	// Grab half of the allowed capacity (to prevent thrashing).
   278  	var list gclinkptr
   279  	var size uintptr
   280  	lock(&stackpool[order].item.mu)
   281  	for size < _StackCacheSize/2 {
   282  		x := stackpoolalloc(order)
   283  		x.ptr().next = list
   284  		list = x
   285  		size += fixedStack << order
   286  	}
   287  	unlock(&stackpool[order].item.mu)
   288  	c.stackcache[order].list = list
   289  	c.stackcache[order].size = size
   290  }
   291  
   292  //go:systemstack
   293  func stackcacherelease(c *mcache, order uint8) {
   294  	if stackDebug >= 1 {
   295  		print("stackcacherelease order=", order, "\n")
   296  	}
   297  	x := c.stackcache[order].list
   298  	size := c.stackcache[order].size
   299  	lock(&stackpool[order].item.mu)
   300  	for size > _StackCacheSize/2 {
   301  		y := x.ptr().next
   302  		stackpoolfree(x, order)
   303  		x = y
   304  		size -= fixedStack << order
   305  	}
   306  	unlock(&stackpool[order].item.mu)
   307  	c.stackcache[order].list = x
   308  	c.stackcache[order].size = size
   309  }
   310  
   311  //go:systemstack
   312  func stackcache_clear(c *mcache) {
   313  	if stackDebug >= 1 {
   314  		print("stackcache clear\n")
   315  	}
   316  	for order := uint8(0); order < _NumStackOrders; order++ {
   317  		lock(&stackpool[order].item.mu)
   318  		x := c.stackcache[order].list
   319  		for x.ptr() != nil {
   320  			y := x.ptr().next
   321  			stackpoolfree(x, order)
   322  			x = y
   323  		}
   324  		c.stackcache[order].list = 0
   325  		c.stackcache[order].size = 0
   326  		unlock(&stackpool[order].item.mu)
   327  	}
   328  }
   329  
   330  // stackalloc allocates an n byte stack.
   331  //
   332  // stackalloc must run on the system stack because it uses per-P
   333  // resources and must not split the stack.
   334  //
   335  //go:systemstack
   336  func stackalloc(n uint32) stack {
   337  	// Stackalloc must be called on scheduler stack, so that we
   338  	// never try to grow the stack during the code that stackalloc runs.
   339  	// Doing so would cause a deadlock (issue 1547).
   340  	thisg := getg()
   341  	if thisg != thisg.m.g0 {
   342  		throw("stackalloc not on scheduler stack")
   343  	}
   344  	if n&(n-1) != 0 {
   345  		throw("stack size not a power of 2")
   346  	}
   347  	if stackDebug >= 1 {
   348  		print("stackalloc ", n, "\n")
   349  	}
   350  
   351  	if debug.efence != 0 || stackFromSystem != 0 {
   352  		n = uint32(alignUp(uintptr(n), physPageSize))
   353  		v := sysAlloc(uintptr(n), &memstats.stacks_sys)
   354  		if v == nil {
   355  			throw("out of memory (stackalloc)")
   356  		}
   357  		return stack{uintptr(v), uintptr(v) + uintptr(n)}
   358  	}
   359  
   360  	// Small stacks are allocated with a fixed-size free-list allocator.
   361  	// If we need a stack of a bigger size, we fall back on allocating
   362  	// a dedicated span.
   363  	var v unsafe.Pointer
   364  	if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
   365  		order := uint8(0)
   366  		n2 := n
   367  		for n2 > fixedStack {
   368  			order++
   369  			n2 >>= 1
   370  		}
   371  		var x gclinkptr
   372  		if stackNoCache != 0 || thisg.m.p == 0 || thisg.m.preemptoff != "" {
   373  			// thisg.m.p == 0 can happen in the guts of exitsyscall
   374  			// or procresize. Just get a stack from the global pool.
   375  			// Also don't touch stackcache during gc
   376  			// as it's flushed concurrently.
   377  			lock(&stackpool[order].item.mu)
   378  			x = stackpoolalloc(order)
   379  			unlock(&stackpool[order].item.mu)
   380  		} else {
   381  			c := thisg.m.p.ptr().mcache
   382  			x = c.stackcache[order].list
   383  			if x.ptr() == nil {
   384  				stackcacherefill(c, order)
   385  				x = c.stackcache[order].list
   386  			}
   387  			c.stackcache[order].list = x.ptr().next
   388  			c.stackcache[order].size -= uintptr(n)
   389  		}
   390  		v = unsafe.Pointer(x)
   391  	} else {
   392  		var s *mspan
   393  		npage := uintptr(n) >> _PageShift
   394  		log2npage := stacklog2(npage)
   395  
   396  		// Try to get a stack from the large stack cache.
   397  		lock(&stackLarge.lock)
   398  		if !stackLarge.free[log2npage].isEmpty() {
   399  			s = stackLarge.free[log2npage].first
   400  			stackLarge.free[log2npage].remove(s)
   401  		}
   402  		unlock(&stackLarge.lock)
   403  
   404  		lockWithRankMayAcquire(&mheap_.lock, lockRankMheap)
   405  
   406  		if s == nil {
   407  			// Allocate a new stack from the heap.
   408  			s = mheap_.allocManual(npage, spanAllocStack)
   409  			if s == nil {
   410  				throw("out of memory")
   411  			}
   412  			osStackAlloc(s)
   413  			s.elemsize = uintptr(n)
   414  		}
   415  		v = unsafe.Pointer(s.base())
   416  	}
   417  
   418  	if raceenabled {
   419  		racemalloc(v, uintptr(n))
   420  	}
   421  	if msanenabled {
   422  		msanmalloc(v, uintptr(n))
   423  	}
   424  	if asanenabled {
   425  		asanunpoison(v, uintptr(n))
   426  	}
   427  	if stackDebug >= 1 {
   428  		print("  allocated ", v, "\n")
   429  	}
   430  	return stack{uintptr(v), uintptr(v) + uintptr(n)}
   431  }
   432  
   433  // stackfree frees an n byte stack allocation at stk.
   434  //
   435  // stackfree must run on the system stack because it uses per-P
   436  // resources and must not split the stack.
   437  //
   438  //go:systemstack
   439  func stackfree(stk stack) {
   440  	gp := getg()
   441  	v := unsafe.Pointer(stk.lo)
   442  	n := stk.hi - stk.lo
   443  	if n&(n-1) != 0 {
   444  		throw("stack not a power of 2")
   445  	}
   446  	if stk.lo+n < stk.hi {
   447  		throw("bad stack size")
   448  	}
   449  	if stackDebug >= 1 {
   450  		println("stackfree", v, n)
   451  		memclrNoHeapPointers(v, n) // for testing, clobber stack data
   452  	}
   453  	if debug.efence != 0 || stackFromSystem != 0 {
   454  		if debug.efence != 0 || stackFaultOnFree != 0 {
   455  			sysFault(v, n)
   456  		} else {
   457  			sysFree(v, n, &memstats.stacks_sys)
   458  		}
   459  		return
   460  	}
   461  	if msanenabled {
   462  		msanfree(v, n)
   463  	}
   464  	if asanenabled {
   465  		asanpoison(v, n)
   466  	}
   467  	if n < fixedStack<<_NumStackOrders && n < _StackCacheSize {
   468  		order := uint8(0)
   469  		n2 := n
   470  		for n2 > fixedStack {
   471  			order++
   472  			n2 >>= 1
   473  		}
   474  		x := gclinkptr(v)
   475  		if stackNoCache != 0 || gp.m.p == 0 || gp.m.preemptoff != "" {
   476  			lock(&stackpool[order].item.mu)
   477  			stackpoolfree(x, order)
   478  			unlock(&stackpool[order].item.mu)
   479  		} else {
   480  			c := gp.m.p.ptr().mcache
   481  			if c.stackcache[order].size >= _StackCacheSize {
   482  				stackcacherelease(c, order)
   483  			}
   484  			x.ptr().next = c.stackcache[order].list
   485  			c.stackcache[order].list = x
   486  			c.stackcache[order].size += n
   487  		}
   488  	} else {
   489  		s := spanOfUnchecked(uintptr(v))
   490  		if s.state.get() != mSpanManual {
   491  			println(hex(s.base()), v)
   492  			throw("bad span state")
   493  		}
   494  		if gcphase == _GCoff {
   495  			// Free the stack immediately if we're
   496  			// sweeping.
   497  			osStackFree(s)
   498  			mheap_.freeManual(s, spanAllocStack)
   499  		} else {
   500  			// If the GC is running, we can't return a
   501  			// stack span to the heap because it could be
   502  			// reused as a heap span, and this state
   503  			// change would race with GC. Add it to the
   504  			// large stack cache instead.
   505  			log2npage := stacklog2(s.npages)
   506  			lock(&stackLarge.lock)
   507  			stackLarge.free[log2npage].insert(s)
   508  			unlock(&stackLarge.lock)
   509  		}
   510  	}
   511  }
   512  
   513  var maxstacksize uintptr = 1 << 20 // enough until runtime.main sets it for real
   514  
   515  var maxstackceiling = maxstacksize
   516  
   517  var ptrnames = []string{
   518  	0: "scalar",
   519  	1: "ptr",
   520  }
   521  
   522  // Stack frame layout
   523  //
   524  // (x86)
   525  // +------------------+
   526  // | args from caller |
   527  // +------------------+ <- frame->argp
   528  // |  return address  |
   529  // +------------------+
   530  // |  caller's BP (*) | (*) if framepointer_enabled && varp > sp
   531  // +------------------+ <- frame->varp
   532  // |     locals       |
   533  // +------------------+
   534  // |  args to callee  |
   535  // +------------------+ <- frame->sp
   536  //
   537  // (arm)
   538  // +------------------+
   539  // | args from caller |
   540  // +------------------+ <- frame->argp
   541  // | caller's retaddr |
   542  // +------------------+
   543  // |  caller's FP (*) | (*) on ARM64, if framepointer_enabled && varp > sp
   544  // +------------------+ <- frame->varp
   545  // |     locals       |
   546  // +------------------+
   547  // |  args to callee  |
   548  // +------------------+
   549  // |  return address  |
   550  // +------------------+ <- frame->sp
   551  //
   552  // varp > sp means that the function has a frame;
   553  // varp == sp means frameless function.
   554  
   555  type adjustinfo struct {
   556  	old   stack
   557  	delta uintptr // ptr distance from old to new stack (newbase - oldbase)
   558  
   559  	// sghi is the highest sudog.elem on the stack.
   560  	sghi uintptr
   561  }
   562  
   563  // adjustpointer checks whether *vpp is in the old stack described by adjinfo.
   564  // If so, it rewrites *vpp to point into the new stack.
   565  func adjustpointer(adjinfo *adjustinfo, vpp unsafe.Pointer) {
   566  	pp := (*uintptr)(vpp)
   567  	p := *pp
   568  	if stackDebug >= 4 {
   569  		print("        ", pp, ":", hex(p), "\n")
   570  	}
   571  	if adjinfo.old.lo <= p && p < adjinfo.old.hi {
   572  		*pp = p + adjinfo.delta
   573  		if stackDebug >= 3 {
   574  			print("        adjust ptr ", pp, ":", hex(p), " -> ", hex(*pp), "\n")
   575  		}
   576  	}
   577  }
   578  
   579  // Information from the compiler about the layout of stack frames.
   580  // Note: this type must agree with reflect.bitVector.
   581  type bitvector struct {
   582  	n        int32 // # of bits
   583  	bytedata *uint8
   584  }
   585  
   586  // ptrbit returns the i'th bit in bv.
   587  // ptrbit is less efficient than iterating directly over bitvector bits,
   588  // and should only be used in non-performance-critical code.
   589  // See adjustpointers for an example of a high-efficiency walk of a bitvector.
   590  func (bv *bitvector) ptrbit(i uintptr) uint8 {
   591  	b := *(addb(bv.bytedata, i/8))
   592  	return (b >> (i % 8)) & 1
   593  }
   594  
   595  // bv describes the memory starting at address scanp.
   596  // Adjust any pointers contained therein.
   597  func adjustpointers(scanp unsafe.Pointer, bv *bitvector, adjinfo *adjustinfo, f funcInfo) {
   598  	minp := adjinfo.old.lo
   599  	maxp := adjinfo.old.hi
   600  	delta := adjinfo.delta
   601  	num := uintptr(bv.n)
   602  	// If this frame might contain channel receive slots, use CAS
   603  	// to adjust pointers. If the slot hasn't been received into
   604  	// yet, it may contain stack pointers and a concurrent send
   605  	// could race with adjusting those pointers. (The sent value
   606  	// itself can never contain stack pointers.)
   607  	useCAS := uintptr(scanp) < adjinfo.sghi
   608  	for i := uintptr(0); i < num; i += 8 {
   609  		if stackDebug >= 4 {
   610  			for j := uintptr(0); j < 8; j++ {
   611  				print("        ", add(scanp, (i+j)*goarch.PtrSize), ":", ptrnames[bv.ptrbit(i+j)], ":", hex(*(*uintptr)(add(scanp, (i+j)*goarch.PtrSize))), " # ", i, " ", *addb(bv.bytedata, i/8), "\n")
   612  			}
   613  		}
   614  		b := *(addb(bv.bytedata, i/8))
   615  		for b != 0 {
   616  			j := uintptr(sys.TrailingZeros8(b))
   617  			b &= b - 1
   618  			pp := (*uintptr)(add(scanp, (i+j)*goarch.PtrSize))
   619  		retry:
   620  			p := *pp
   621  			if f.valid() && 0 < p && p < minLegalPointer && debug.invalidptr != 0 {
   622  				// Looks like a junk value in a pointer slot.
   623  				// Live analysis wrong?
   624  				getg().m.traceback = 2
   625  				print("runtime: bad pointer in frame ", funcname(f), " at ", pp, ": ", hex(p), "\n")
   626  				throw("invalid pointer found on stack")
   627  			}
   628  			if minp <= p && p < maxp {
   629  				if stackDebug >= 3 {
   630  					print("adjust ptr ", hex(p), " ", funcname(f), "\n")
   631  				}
   632  				if useCAS {
   633  					ppu := (*unsafe.Pointer)(unsafe.Pointer(pp))
   634  					if !atomic.Casp1(ppu, unsafe.Pointer(p), unsafe.Pointer(p+delta)) {
   635  						goto retry
   636  					}
   637  				} else {
   638  					*pp = p + delta
   639  				}
   640  			}
   641  		}
   642  	}
   643  }
   644  
   645  // Note: the argument/return area is adjusted by the callee.
   646  func adjustframe(frame *stkframe, adjinfo *adjustinfo) {
   647  	if frame.continpc == 0 {
   648  		// Frame is dead.
   649  		return
   650  	}
   651  	f := frame.fn
   652  	if stackDebug >= 2 {
   653  		print("    adjusting ", funcname(f), " frame=[", hex(frame.sp), ",", hex(frame.fp), "] pc=", hex(frame.pc), " continpc=", hex(frame.continpc), "\n")
   654  	}
   655  
   656  	// Adjust saved frame pointer if there is one.
   657  	if (goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.ARM64) && frame.argp-frame.varp == 2*goarch.PtrSize {
   658  		if stackDebug >= 3 {
   659  			print("      saved bp\n")
   660  		}
   661  		if debugCheckBP {
   662  			// Frame pointers should always point to the next higher frame on
   663  			// the Go stack (or be nil, for the top frame on the stack).
   664  			bp := *(*uintptr)(unsafe.Pointer(frame.varp))
   665  			if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
   666  				println("runtime: found invalid frame pointer")
   667  				print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
   668  				throw("bad frame pointer")
   669  			}
   670  		}
   671  		// On AMD64, this is the caller's frame pointer saved in the current
   672  		// frame.
   673  		// On ARM64, this is the frame pointer of the caller's caller saved
   674  		// by the caller in its frame (one word below its SP).
   675  		adjustpointer(adjinfo, unsafe.Pointer(frame.varp))
   676  	}
   677  
   678  	locals, args, objs := frame.getStackMap(true)
   679  
   680  	// Adjust local variables if stack frame has been allocated.
   681  	if locals.n > 0 {
   682  		size := uintptr(locals.n) * goarch.PtrSize
   683  		adjustpointers(unsafe.Pointer(frame.varp-size), &locals, adjinfo, f)
   684  	}
   685  
   686  	// Adjust arguments.
   687  	if args.n > 0 {
   688  		if stackDebug >= 3 {
   689  			print("      args\n")
   690  		}
   691  		adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{})
   692  	}
   693  
   694  	// Adjust pointers in all stack objects (whether they are live or not).
   695  	// See comments in mgcmark.go:scanframeworker.
   696  	if frame.varp != 0 {
   697  		for i := range objs {
   698  			obj := &objs[i]
   699  			off := obj.off
   700  			base := frame.varp // locals base pointer
   701  			if off >= 0 {
   702  				base = frame.argp // arguments and return values base pointer
   703  			}
   704  			p := base + uintptr(off)
   705  			if p < frame.sp {
   706  				// Object hasn't been allocated in the frame yet.
   707  				// (Happens when the stack bounds check fails and
   708  				// we call into morestack.)
   709  				continue
   710  			}
   711  			ptrdata := obj.ptrdata()
   712  			gcdata := obj.gcdata()
   713  			var s *mspan
   714  			if obj.useGCProg() {
   715  				// See comments in mgcmark.go:scanstack
   716  				s = materializeGCProg(ptrdata, gcdata)
   717  				gcdata = (*byte)(unsafe.Pointer(s.startAddr))
   718  			}
   719  			for i := uintptr(0); i < ptrdata; i += goarch.PtrSize {
   720  				if *addb(gcdata, i/(8*goarch.PtrSize))>>(i/goarch.PtrSize&7)&1 != 0 {
   721  					adjustpointer(adjinfo, unsafe.Pointer(p+i))
   722  				}
   723  			}
   724  			if s != nil {
   725  				dematerializeGCProg(s)
   726  			}
   727  		}
   728  	}
   729  }
   730  
   731  func adjustctxt(gp *g, adjinfo *adjustinfo) {
   732  	adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.ctxt))
   733  	if !framepointer_enabled {
   734  		return
   735  	}
   736  	if debugCheckBP {
   737  		bp := gp.sched.bp
   738  		if bp != 0 && (bp < adjinfo.old.lo || bp >= adjinfo.old.hi) {
   739  			println("runtime: found invalid top frame pointer")
   740  			print("bp=", hex(bp), " min=", hex(adjinfo.old.lo), " max=", hex(adjinfo.old.hi), "\n")
   741  			throw("bad top frame pointer")
   742  		}
   743  	}
   744  	oldfp := gp.sched.bp
   745  	adjustpointer(adjinfo, unsafe.Pointer(&gp.sched.bp))
   746  	if GOARCH == "arm64" {
   747  		// On ARM64, the frame pointer is saved one word *below* the SP,
   748  		// which is not copied or adjusted in any frame. Do it explicitly
   749  		// here.
   750  		if oldfp == gp.sched.sp-goarch.PtrSize {
   751  			memmove(unsafe.Pointer(gp.sched.bp), unsafe.Pointer(oldfp), goarch.PtrSize)
   752  			adjustpointer(adjinfo, unsafe.Pointer(gp.sched.bp))
   753  		}
   754  	}
   755  }
   756  
   757  func adjustdefers(gp *g, adjinfo *adjustinfo) {
   758  	// Adjust pointers in the Defer structs.
   759  	// We need to do this first because we need to adjust the
   760  	// defer.link fields so we always work on the new stack.
   761  	adjustpointer(adjinfo, unsafe.Pointer(&gp._defer))
   762  	for d := gp._defer; d != nil; d = d.link {
   763  		adjustpointer(adjinfo, unsafe.Pointer(&d.fn))
   764  		adjustpointer(adjinfo, unsafe.Pointer(&d.sp))
   765  		adjustpointer(adjinfo, unsafe.Pointer(&d.link))
   766  	}
   767  }
   768  
   769  func adjustpanics(gp *g, adjinfo *adjustinfo) {
   770  	// Panics are on stack and already adjusted.
   771  	// Update pointer to head of list in G.
   772  	adjustpointer(adjinfo, unsafe.Pointer(&gp._panic))
   773  }
   774  
   775  func adjustsudogs(gp *g, adjinfo *adjustinfo) {
   776  	// the data elements pointed to by a SudoG structure
   777  	// might be in the stack.
   778  	for s := gp.waiting; s != nil; s = s.waitlink {
   779  		adjustpointer(adjinfo, unsafe.Pointer(&s.elem))
   780  	}
   781  }
   782  
   783  func fillstack(stk stack, b byte) {
   784  	for p := stk.lo; p < stk.hi; p++ {
   785  		*(*byte)(unsafe.Pointer(p)) = b
   786  	}
   787  }
   788  
   789  func findsghi(gp *g, stk stack) uintptr {
   790  	var sghi uintptr
   791  	for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   792  		p := uintptr(sg.elem) + uintptr(sg.c.elemsize)
   793  		if stk.lo <= p && p < stk.hi && p > sghi {
   794  			sghi = p
   795  		}
   796  	}
   797  	return sghi
   798  }
   799  
   800  // syncadjustsudogs adjusts gp's sudogs and copies the part of gp's
   801  // stack they refer to while synchronizing with concurrent channel
   802  // operations. It returns the number of bytes of stack copied.
   803  func syncadjustsudogs(gp *g, used uintptr, adjinfo *adjustinfo) uintptr {
   804  	if gp.waiting == nil {
   805  		return 0
   806  	}
   807  
   808  	// Lock channels to prevent concurrent send/receive.
   809  	var lastc *hchan
   810  	for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   811  		if sg.c != lastc {
   812  			// There is a ranking cycle here between gscan bit and
   813  			// hchan locks. Normally, we only allow acquiring hchan
   814  			// locks and then getting a gscan bit. In this case, we
   815  			// already have the gscan bit. We allow acquiring hchan
   816  			// locks here as a special case, since a deadlock can't
   817  			// happen because the G involved must already be
   818  			// suspended. So, we get a special hchan lock rank here
   819  			// that is lower than gscan, but doesn't allow acquiring
   820  			// any other locks other than hchan.
   821  			lockWithRank(&sg.c.lock, lockRankHchanLeaf)
   822  		}
   823  		lastc = sg.c
   824  	}
   825  
   826  	// Adjust sudogs.
   827  	adjustsudogs(gp, adjinfo)
   828  
   829  	// Copy the part of the stack the sudogs point in to
   830  	// while holding the lock to prevent races on
   831  	// send/receive slots.
   832  	var sgsize uintptr
   833  	if adjinfo.sghi != 0 {
   834  		oldBot := adjinfo.old.hi - used
   835  		newBot := oldBot + adjinfo.delta
   836  		sgsize = adjinfo.sghi - oldBot
   837  		memmove(unsafe.Pointer(newBot), unsafe.Pointer(oldBot), sgsize)
   838  	}
   839  
   840  	// Unlock channels.
   841  	lastc = nil
   842  	for sg := gp.waiting; sg != nil; sg = sg.waitlink {
   843  		if sg.c != lastc {
   844  			unlock(&sg.c.lock)
   845  		}
   846  		lastc = sg.c
   847  	}
   848  
   849  	return sgsize
   850  }
   851  
   852  // Copies gp's stack to a new stack of a different size.
   853  // Caller must have changed gp status to Gcopystack.
   854  func copystack(gp *g, newsize uintptr) {
   855  	if gp.syscallsp != 0 {
   856  		throw("stack growth not allowed in system call")
   857  	}
   858  	old := gp.stack
   859  	if old.lo == 0 {
   860  		throw("nil stackbase")
   861  	}
   862  	used := old.hi - gp.sched.sp
   863  	// Add just the difference to gcController.addScannableStack.
   864  	// g0 stacks never move, so this will never account for them.
   865  	// It's also fine if we have no P, addScannableStack can deal with
   866  	// that case.
   867  	gcController.addScannableStack(getg().m.p.ptr(), int64(newsize)-int64(old.hi-old.lo))
   868  
   869  	// allocate new stack
   870  	new := stackalloc(uint32(newsize))
   871  	if stackPoisonCopy != 0 {
   872  		fillstack(new, 0xfd)
   873  	}
   874  	if stackDebug >= 1 {
   875  		print("copystack gp=", gp, " [", hex(old.lo), " ", hex(old.hi-used), " ", hex(old.hi), "]", " -> [", hex(new.lo), " ", hex(new.hi-used), " ", hex(new.hi), "]/", newsize, "\n")
   876  	}
   877  
   878  	// Compute adjustment.
   879  	var adjinfo adjustinfo
   880  	adjinfo.old = old
   881  	adjinfo.delta = new.hi - old.hi
   882  
   883  	// Adjust sudogs, synchronizing with channel ops if necessary.
   884  	ncopy := used
   885  	if !gp.activeStackChans {
   886  		if newsize < old.hi-old.lo && gp.parkingOnChan.Load() {
   887  			// It's not safe for someone to shrink this stack while we're actively
   888  			// parking on a channel, but it is safe to grow since we do that
   889  			// ourselves and explicitly don't want to synchronize with channels
   890  			// since we could self-deadlock.
   891  			throw("racy sudog adjustment due to parking on channel")
   892  		}
   893  		adjustsudogs(gp, &adjinfo)
   894  	} else {
   895  		// sudogs may be pointing in to the stack and gp has
   896  		// released channel locks, so other goroutines could
   897  		// be writing to gp's stack. Find the highest such
   898  		// pointer so we can handle everything there and below
   899  		// carefully. (This shouldn't be far from the bottom
   900  		// of the stack, so there's little cost in handling
   901  		// everything below it carefully.)
   902  		adjinfo.sghi = findsghi(gp, old)
   903  
   904  		// Synchronize with channel ops and copy the part of
   905  		// the stack they may interact with.
   906  		ncopy -= syncadjustsudogs(gp, used, &adjinfo)
   907  	}
   908  
   909  	// Copy the stack (or the rest of it) to the new location
   910  	memmove(unsafe.Pointer(new.hi-ncopy), unsafe.Pointer(old.hi-ncopy), ncopy)
   911  
   912  	// Adjust remaining structures that have pointers into stacks.
   913  	// We have to do most of these before we traceback the new
   914  	// stack because gentraceback uses them.
   915  	adjustctxt(gp, &adjinfo)
   916  	adjustdefers(gp, &adjinfo)
   917  	adjustpanics(gp, &adjinfo)
   918  	if adjinfo.sghi != 0 {
   919  		adjinfo.sghi += adjinfo.delta
   920  	}
   921  
   922  	// Swap out old stack for new one
   923  	gp.stack = new
   924  	gp.stackguard0 = new.lo + stackGuard // NOTE: might clobber a preempt request
   925  	gp.sched.sp = new.hi - used
   926  	gp.stktopsp += adjinfo.delta
   927  
   928  	// Adjust pointers in the new stack.
   929  	var u unwinder
   930  	for u.init(gp, 0); u.valid(); u.next() {
   931  		adjustframe(&u.frame, &adjinfo)
   932  	}
   933  
   934  	// free old stack
   935  	if stackPoisonCopy != 0 {
   936  		fillstack(old, 0xfc)
   937  	}
   938  	stackfree(old)
   939  }
   940  
   941  // round x up to a power of 2.
   942  func round2(x int32) int32 {
   943  	s := uint(0)
   944  	for 1<<s < x {
   945  		s++
   946  	}
   947  	return 1 << s
   948  }
   949  
   950  // Called from runtime·morestack when more stack is needed.
   951  // Allocate larger stack and relocate to new stack.
   952  // Stack growth is multiplicative, for constant amortized cost.
   953  //
   954  // g->atomicstatus will be Grunning or Gscanrunning upon entry.
   955  // If the scheduler is trying to stop this g, then it will set preemptStop.
   956  //
   957  // This must be nowritebarrierrec because it can be called as part of
   958  // stack growth from other nowritebarrierrec functions, but the
   959  // compiler doesn't check this.
   960  //
   961  //go:nowritebarrierrec
   962  func newstack() {
   963  	thisg := getg()
   964  	// TODO: double check all gp. shouldn't be getg().
   965  	if thisg.m.morebuf.g.ptr().stackguard0 == stackFork {
   966  		throw("stack growth after fork")
   967  	}
   968  	if thisg.m.morebuf.g.ptr() != thisg.m.curg {
   969  		print("runtime: newstack called from g=", hex(thisg.m.morebuf.g), "\n"+"\tm=", thisg.m, " m->curg=", thisg.m.curg, " m->g0=", thisg.m.g0, " m->gsignal=", thisg.m.gsignal, "\n")
   970  		morebuf := thisg.m.morebuf
   971  		traceback(morebuf.pc, morebuf.sp, morebuf.lr, morebuf.g.ptr())
   972  		throw("runtime: wrong goroutine in newstack")
   973  	}
   974  
   975  	gp := thisg.m.curg
   976  
   977  	if thisg.m.curg.throwsplit {
   978  		// Update syscallsp, syscallpc in case traceback uses them.
   979  		morebuf := thisg.m.morebuf
   980  		gp.syscallsp = morebuf.sp
   981  		gp.syscallpc = morebuf.pc
   982  		pcname, pcoff := "(unknown)", uintptr(0)
   983  		f := findfunc(gp.sched.pc)
   984  		if f.valid() {
   985  			pcname = funcname(f)
   986  			pcoff = gp.sched.pc - f.entry()
   987  		}
   988  		print("runtime: newstack at ", pcname, "+", hex(pcoff),
   989  			" sp=", hex(gp.sched.sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
   990  			"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
   991  			"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
   992  
   993  		thisg.m.traceback = 2 // Include runtime frames
   994  		traceback(morebuf.pc, morebuf.sp, morebuf.lr, gp)
   995  		throw("runtime: stack split at bad time")
   996  	}
   997  
   998  	morebuf := thisg.m.morebuf
   999  	thisg.m.morebuf.pc = 0
  1000  	thisg.m.morebuf.lr = 0
  1001  	thisg.m.morebuf.sp = 0
  1002  	thisg.m.morebuf.g = 0
  1003  
  1004  	// NOTE: stackguard0 may change underfoot, if another thread
  1005  	// is about to try to preempt gp. Read it just once and use that same
  1006  	// value now and below.
  1007  	stackguard0 := atomic.Loaduintptr(&gp.stackguard0)
  1008  
  1009  	// Be conservative about where we preempt.
  1010  	// We are interested in preempting user Go code, not runtime code.
  1011  	// If we're holding locks, mallocing, or preemption is disabled, don't
  1012  	// preempt.
  1013  	// This check is very early in newstack so that even the status change
  1014  	// from Grunning to Gwaiting and back doesn't happen in this case.
  1015  	// That status change by itself can be viewed as a small preemption,
  1016  	// because the GC might change Gwaiting to Gscanwaiting, and then
  1017  	// this goroutine has to wait for the GC to finish before continuing.
  1018  	// If the GC is in some way dependent on this goroutine (for example,
  1019  	// it needs a lock held by the goroutine), that small preemption turns
  1020  	// into a real deadlock.
  1021  	preempt := stackguard0 == stackPreempt
  1022  	if preempt {
  1023  		if !canPreemptM(thisg.m) {
  1024  			// Let the goroutine keep running for now.
  1025  			// gp->preempt is set, so it will be preempted next time.
  1026  			gp.stackguard0 = gp.stack.lo + stackGuard
  1027  			gogo(&gp.sched) // never return
  1028  		}
  1029  	}
  1030  
  1031  	if gp.stack.lo == 0 {
  1032  		throw("missing stack in newstack")
  1033  	}
  1034  	sp := gp.sched.sp
  1035  	if goarch.ArchFamily == goarch.AMD64 || goarch.ArchFamily == goarch.I386 || goarch.ArchFamily == goarch.WASM {
  1036  		// The call to morestack cost a word.
  1037  		sp -= goarch.PtrSize
  1038  	}
  1039  	if stackDebug >= 1 || sp < gp.stack.lo {
  1040  		print("runtime: newstack sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n",
  1041  			"\tmorebuf={pc:", hex(morebuf.pc), " sp:", hex(morebuf.sp), " lr:", hex(morebuf.lr), "}\n",
  1042  			"\tsched={pc:", hex(gp.sched.pc), " sp:", hex(gp.sched.sp), " lr:", hex(gp.sched.lr), " ctxt:", gp.sched.ctxt, "}\n")
  1043  	}
  1044  	if sp < gp.stack.lo {
  1045  		print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->status=", hex(readgstatus(gp)), "\n ")
  1046  		print("runtime: split stack overflow: ", hex(sp), " < ", hex(gp.stack.lo), "\n")
  1047  		throw("runtime: split stack overflow")
  1048  	}
  1049  
  1050  	if preempt {
  1051  		if gp == thisg.m.g0 {
  1052  			throw("runtime: preempt g0")
  1053  		}
  1054  		if thisg.m.p == 0 && thisg.m.locks == 0 {
  1055  			throw("runtime: g is running but p is not")
  1056  		}
  1057  
  1058  		if gp.preemptShrink {
  1059  			// We're at a synchronous safe point now, so
  1060  			// do the pending stack shrink.
  1061  			gp.preemptShrink = false
  1062  			shrinkstack(gp)
  1063  		}
  1064  
  1065  		if gp.preemptStop {
  1066  			preemptPark(gp) // never returns
  1067  		}
  1068  
  1069  		// Act like goroutine called runtime.Gosched.
  1070  		gopreempt_m(gp) // never return
  1071  	}
  1072  
  1073  	// Allocate a bigger segment and move the stack.
  1074  	oldsize := gp.stack.hi - gp.stack.lo
  1075  	newsize := oldsize * 2
  1076  
  1077  	// Make sure we grow at least as much as needed to fit the new frame.
  1078  	// (This is just an optimization - the caller of morestack will
  1079  	// recheck the bounds on return.)
  1080  	if f := findfunc(gp.sched.pc); f.valid() {
  1081  		max := uintptr(funcMaxSPDelta(f))
  1082  		needed := max + stackGuard
  1083  		used := gp.stack.hi - gp.sched.sp
  1084  		for newsize-used < needed {
  1085  			newsize *= 2
  1086  		}
  1087  	}
  1088  
  1089  	if stackguard0 == stackForceMove {
  1090  		// Forced stack movement used for debugging.
  1091  		// Don't double the stack (or we may quickly run out
  1092  		// if this is done repeatedly).
  1093  		newsize = oldsize
  1094  	}
  1095  
  1096  	if newsize > maxstacksize || newsize > maxstackceiling {
  1097  		if maxstacksize < maxstackceiling {
  1098  			print("runtime: goroutine stack exceeds ", maxstacksize, "-byte limit\n")
  1099  		} else {
  1100  			print("runtime: goroutine stack exceeds ", maxstackceiling, "-byte limit\n")
  1101  		}
  1102  		print("runtime: sp=", hex(sp), " stack=[", hex(gp.stack.lo), ", ", hex(gp.stack.hi), "]\n")
  1103  		throw("stack overflow")
  1104  	}
  1105  
  1106  	// The goroutine must be executing in order to call newstack,
  1107  	// so it must be Grunning (or Gscanrunning).
  1108  	casgstatus(gp, _Grunning, _Gcopystack)
  1109  
  1110  	// The concurrent GC will not scan the stack while we are doing the copy since
  1111  	// the gp is in a Gcopystack status.
  1112  	copystack(gp, newsize)
  1113  	if stackDebug >= 1 {
  1114  		print("stack grow done\n")
  1115  	}
  1116  	casgstatus(gp, _Gcopystack, _Grunning)
  1117  	gogo(&gp.sched)
  1118  }
  1119  
  1120  //go:nosplit
  1121  func nilfunc() {
  1122  	*(*uint8)(nil) = 0
  1123  }
  1124  
  1125  // adjust Gobuf as if it executed a call to fn
  1126  // and then stopped before the first instruction in fn.
  1127  func gostartcallfn(gobuf *gobuf, fv *funcval) {
  1128  	var fn unsafe.Pointer
  1129  	if fv != nil {
  1130  		fn = unsafe.Pointer(fv.fn)
  1131  	} else {
  1132  		fn = unsafe.Pointer(abi.FuncPCABIInternal(nilfunc))
  1133  	}
  1134  	gostartcall(gobuf, fn, unsafe.Pointer(fv))
  1135  }
  1136  
  1137  // isShrinkStackSafe returns whether it's safe to attempt to shrink
  1138  // gp's stack. Shrinking the stack is only safe when we have precise
  1139  // pointer maps for all frames on the stack.
  1140  func isShrinkStackSafe(gp *g) bool {
  1141  	// We can't copy the stack if we're in a syscall.
  1142  	// The syscall might have pointers into the stack and
  1143  	// often we don't have precise pointer maps for the innermost
  1144  	// frames.
  1145  	//
  1146  	// We also can't copy the stack if we're at an asynchronous
  1147  	// safe-point because we don't have precise pointer maps for
  1148  	// all frames.
  1149  	//
  1150  	// We also can't *shrink* the stack in the window between the
  1151  	// goroutine calling gopark to park on a channel and
  1152  	// gp.activeStackChans being set.
  1153  	return gp.syscallsp == 0 && !gp.asyncSafePoint && !gp.parkingOnChan.Load()
  1154  }
  1155  
  1156  // Maybe shrink the stack being used by gp.
  1157  //
  1158  // gp must be stopped and we must own its stack. It may be in
  1159  // _Grunning, but only if this is our own user G.
  1160  func shrinkstack(gp *g) {
  1161  	if gp.stack.lo == 0 {
  1162  		throw("missing stack in shrinkstack")
  1163  	}
  1164  	if s := readgstatus(gp); s&_Gscan == 0 {
  1165  		// We don't own the stack via _Gscan. We could still
  1166  		// own it if this is our own user G and we're on the
  1167  		// system stack.
  1168  		if !(gp == getg().m.curg && getg() != getg().m.curg && s == _Grunning) {
  1169  			// We don't own the stack.
  1170  			throw("bad status in shrinkstack")
  1171  		}
  1172  	}
  1173  	if !isShrinkStackSafe(gp) {
  1174  		throw("shrinkstack at bad time")
  1175  	}
  1176  	// Check for self-shrinks while in a libcall. These may have
  1177  	// pointers into the stack disguised as uintptrs, but these
  1178  	// code paths should all be nosplit.
  1179  	if gp == getg().m.curg && gp.m.libcallsp != 0 {
  1180  		throw("shrinking stack in libcall")
  1181  	}
  1182  
  1183  	if debug.gcshrinkstackoff > 0 {
  1184  		return
  1185  	}
  1186  	f := findfunc(gp.startpc)
  1187  	if f.valid() && f.funcID == abi.FuncID_gcBgMarkWorker {
  1188  		// We're not allowed to shrink the gcBgMarkWorker
  1189  		// stack (see gcBgMarkWorker for explanation).
  1190  		return
  1191  	}
  1192  
  1193  	oldsize := gp.stack.hi - gp.stack.lo
  1194  	newsize := oldsize / 2
  1195  	// Don't shrink the allocation below the minimum-sized stack
  1196  	// allocation.
  1197  	if newsize < fixedStack {
  1198  		return
  1199  	}
  1200  	// Compute how much of the stack is currently in use and only
  1201  	// shrink the stack if gp is using less than a quarter of its
  1202  	// current stack. The currently used stack includes everything
  1203  	// down to the SP plus the stack guard space that ensures
  1204  	// there's room for nosplit functions.
  1205  	avail := gp.stack.hi - gp.stack.lo
  1206  	if used := gp.stack.hi - gp.sched.sp + stackNosplit; used >= avail/4 {
  1207  		return
  1208  	}
  1209  
  1210  	if stackDebug > 0 {
  1211  		print("shrinking stack ", oldsize, "->", newsize, "\n")
  1212  	}
  1213  
  1214  	copystack(gp, newsize)
  1215  }
  1216  
  1217  // freeStackSpans frees unused stack spans at the end of GC.
  1218  func freeStackSpans() {
  1219  	// Scan stack pools for empty stack spans.
  1220  	for order := range stackpool {
  1221  		lock(&stackpool[order].item.mu)
  1222  		list := &stackpool[order].item.span
  1223  		for s := list.first; s != nil; {
  1224  			next := s.next
  1225  			if s.allocCount == 0 {
  1226  				list.remove(s)
  1227  				s.manualFreeList = 0
  1228  				osStackFree(s)
  1229  				mheap_.freeManual(s, spanAllocStack)
  1230  			}
  1231  			s = next
  1232  		}
  1233  		unlock(&stackpool[order].item.mu)
  1234  	}
  1235  
  1236  	// Free large stack spans.
  1237  	lock(&stackLarge.lock)
  1238  	for i := range stackLarge.free {
  1239  		for s := stackLarge.free[i].first; s != nil; {
  1240  			next := s.next
  1241  			stackLarge.free[i].remove(s)
  1242  			osStackFree(s)
  1243  			mheap_.freeManual(s, spanAllocStack)
  1244  			s = next
  1245  		}
  1246  	}
  1247  	unlock(&stackLarge.lock)
  1248  }
  1249  
  1250  // A stackObjectRecord is generated by the compiler for each stack object in a stack frame.
  1251  // This record must match the generator code in cmd/compile/internal/liveness/plive.go:emitStackObjects.
  1252  type stackObjectRecord struct {
  1253  	// offset in frame
  1254  	// if negative, offset from varp
  1255  	// if non-negative, offset from argp
  1256  	off       int32
  1257  	size      int32
  1258  	_ptrdata  int32  // ptrdata, or -ptrdata is GC prog is used
  1259  	gcdataoff uint32 // offset to gcdata from moduledata.rodata
  1260  }
  1261  
  1262  func (r *stackObjectRecord) useGCProg() bool {
  1263  	return r._ptrdata < 0
  1264  }
  1265  
  1266  func (r *stackObjectRecord) ptrdata() uintptr {
  1267  	x := r._ptrdata
  1268  	if x < 0 {
  1269  		return uintptr(-x)
  1270  	}
  1271  	return uintptr(x)
  1272  }
  1273  
  1274  // gcdata returns pointer map or GC prog of the type.
  1275  func (r *stackObjectRecord) gcdata() *byte {
  1276  	ptr := uintptr(unsafe.Pointer(r))
  1277  	var mod *moduledata
  1278  	for datap := &firstmoduledata; datap != nil; datap = datap.next {
  1279  		if datap.gofunc <= ptr && ptr < datap.end {
  1280  			mod = datap
  1281  			break
  1282  		}
  1283  	}
  1284  	// If you get a panic here due to a nil mod,
  1285  	// you may have made a copy of a stackObjectRecord.
  1286  	// You must use the original pointer.
  1287  	res := mod.rodata + uintptr(r.gcdataoff)
  1288  	return (*byte)(unsafe.Pointer(res))
  1289  }
  1290  
  1291  // This is exported as ABI0 via linkname so obj can call it.
  1292  //
  1293  //go:nosplit
  1294  //go:linkname morestackc
  1295  func morestackc() {
  1296  	throw("attempt to execute system stack code on user stack")
  1297  }
  1298  
  1299  // startingStackSize is the amount of stack that new goroutines start with.
  1300  // It is a power of 2, and between _FixedStack and maxstacksize, inclusive.
  1301  // startingStackSize is updated every GC by tracking the average size of
  1302  // stacks scanned during the GC.
  1303  var startingStackSize uint32 = fixedStack
  1304  
  1305  func gcComputeStartingStackSize() {
  1306  	if debug.adaptivestackstart == 0 {
  1307  		return
  1308  	}
  1309  	// For details, see the design doc at
  1310  	// https://docs.google.com/document/d/1YDlGIdVTPnmUiTAavlZxBI1d9pwGQgZT7IKFKlIXohQ/edit?usp=sharing
  1311  	// The basic algorithm is to track the average size of stacks
  1312  	// and start goroutines with stack equal to that average size.
  1313  	// Starting at the average size uses at most 2x the space that
  1314  	// an ideal algorithm would have used.
  1315  	// This is just a heuristic to avoid excessive stack growth work
  1316  	// early in a goroutine's lifetime. See issue 18138. Stacks that
  1317  	// are allocated too small can still grow, and stacks allocated
  1318  	// too large can still shrink.
  1319  	var scannedStackSize uint64
  1320  	var scannedStacks uint64
  1321  	for _, p := range allp {
  1322  		scannedStackSize += p.scannedStackSize
  1323  		scannedStacks += p.scannedStacks
  1324  		// Reset for next time
  1325  		p.scannedStackSize = 0
  1326  		p.scannedStacks = 0
  1327  	}
  1328  	if scannedStacks == 0 {
  1329  		startingStackSize = fixedStack
  1330  		return
  1331  	}
  1332  	avg := scannedStackSize/scannedStacks + stackGuard
  1333  	// Note: we add stackGuard to ensure that a goroutine that
  1334  	// uses the average space will not trigger a growth.
  1335  	if avg > uint64(maxstacksize) {
  1336  		avg = uint64(maxstacksize)
  1337  	}
  1338  	if avg < fixedStack {
  1339  		avg = fixedStack
  1340  	}
  1341  	// Note: maxstacksize fits in 30 bits, so avg also does.
  1342  	startingStackSize = uint32(round2(int32(avg)))
  1343  }
  1344  

View as plain text