1 // Copyright 2023 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build goexperiment.exectracer2 6 7 // Trace stack table and acquisition. 8 9 package runtime 10 11 import ( 12 "internal/abi" 13 "internal/goarch" 14 "unsafe" 15 ) 16 17 const ( 18 // Maximum number of PCs in a single stack trace. 19 // Since events contain only stack id rather than whole stack trace, 20 // we can allow quite large values here. 21 traceStackSize = 128 22 23 // logicalStackSentinel is a sentinel value at pcBuf[0] signifying that 24 // pcBuf[1:] holds a logical stack requiring no further processing. Any other 25 // value at pcBuf[0] represents a skip value to apply to the physical stack in 26 // pcBuf[1:] after inline expansion. 27 logicalStackSentinel = ^uintptr(0) 28 ) 29 30 // traceStack captures a stack trace and registers it in the trace stack table. 31 // It then returns its unique ID. 32 // 33 // skip controls the number of leaf frames to omit in order to hide tracer internals 34 // from stack traces, see CL 5523. 35 // 36 // Avoid calling this function directly. gen needs to be the current generation 37 // that this stack trace is being written out for, which needs to be synchronized with 38 // generations moving forward. Prefer traceEventWriter.stack. 39 func traceStack(skip int, mp *m, gen uintptr) uint64 { 40 var pcBuf [traceStackSize]uintptr 41 42 gp := getg() 43 curgp := gp.m.curg 44 nstk := 1 45 if tracefpunwindoff() || mp.hasCgoOnStack() { 46 // Slow path: Unwind using default unwinder. Used when frame pointer 47 // unwinding is unavailable or disabled (tracefpunwindoff), or might 48 // produce incomplete results or crashes (hasCgoOnStack). Note that no 49 // cgo callback related crashes have been observed yet. The main 50 // motivation is to take advantage of a potentially registered cgo 51 // symbolizer. 52 pcBuf[0] = logicalStackSentinel 53 if curgp == gp { 54 nstk += callers(skip+1, pcBuf[1:]) 55 } else if curgp != nil { 56 nstk += gcallers(curgp, skip, pcBuf[1:]) 57 } 58 } else { 59 // Fast path: Unwind using frame pointers. 60 pcBuf[0] = uintptr(skip) 61 if curgp == gp { 62 nstk += fpTracebackPCs(unsafe.Pointer(getfp()), pcBuf[1:]) 63 } else if curgp != nil { 64 // We're called on the g0 stack through mcall(fn) or systemstack(fn). To 65 // behave like gcallers above, we start unwinding from sched.bp, which 66 // points to the caller frame of the leaf frame on g's stack. The return 67 // address of the leaf frame is stored in sched.pc, which we manually 68 // capture here. 69 pcBuf[1] = curgp.sched.pc 70 nstk += 1 + fpTracebackPCs(unsafe.Pointer(curgp.sched.bp), pcBuf[2:]) 71 } 72 } 73 if nstk > 0 { 74 nstk-- // skip runtime.goexit 75 } 76 if nstk > 0 && curgp.goid == 1 { 77 nstk-- // skip runtime.main 78 } 79 id := trace.stackTab[gen%2].put(pcBuf[:nstk]) 80 return id 81 } 82 83 // traceStackTable maps stack traces (arrays of PC's) to unique uint32 ids. 84 // It is lock-free for reading. 85 type traceStackTable struct { 86 tab traceMap 87 } 88 89 // put returns a unique id for the stack trace pcs and caches it in the table, 90 // if it sees the trace for the first time. 91 func (t *traceStackTable) put(pcs []uintptr) uint64 { 92 if len(pcs) == 0 { 93 return 0 94 } 95 id, _ := t.tab.put(noescape(unsafe.Pointer(&pcs[0])), uintptr(len(pcs))*unsafe.Sizeof(uintptr(0))) 96 return id 97 } 98 99 // dump writes all previously cached stacks to trace buffers, 100 // releases all memory and resets state. It must only be called once the caller 101 // can guarantee that there are no more writers to the table. 102 // 103 // This must run on the system stack because it flushes buffers and thus 104 // may acquire trace.lock. 105 // 106 //go:systemstack 107 func (t *traceStackTable) dump(gen uintptr) { 108 w := unsafeTraceWriter(gen, nil) 109 110 // Iterate over the table. 111 // 112 // Do not acquire t.tab.lock. There's a conceptual lock cycle between acquiring this lock 113 // here and allocation-related locks. Specifically, this lock may be acquired when an event 114 // is emitted in allocation paths. Simultaneously, we might allocate here with the lock held, 115 // creating a cycle. In practice, this cycle is never exercised. Because the table is only 116 // dumped once there are no more writers, it's not possible for the cycle to occur. However 117 // the lockrank mode is not sophisticated enough to identify this, and if it's not possible 118 // for that cycle to happen, then it's also not possible for this to race with writers to 119 // the table. 120 for i := range t.tab.tab { 121 stk := t.tab.bucket(i) 122 for ; stk != nil; stk = stk.next() { 123 stack := unsafe.Slice((*uintptr)(unsafe.Pointer(&stk.data[0])), uintptr(len(stk.data))/unsafe.Sizeof(uintptr(0))) 124 125 // N.B. This might allocate, but that's OK because we're not writing to the M's buffer, 126 // but one we're about to create (with ensure). 127 frames := makeTraceFrames(gen, fpunwindExpand(stack)) 128 129 // Returns the maximum number of bytes required to hold the encoded stack, given that 130 // it contains N frames. 131 maxBytes := 1 + (2+4*len(frames))*traceBytesPerNumber 132 133 // Estimate the size of this record. This 134 // bound is pretty loose, but avoids counting 135 // lots of varint sizes. 136 // 137 // Add 1 because we might also write traceEvStacks. 138 var flushed bool 139 w, flushed = w.ensure(1 + maxBytes) 140 if flushed { 141 w.byte(byte(traceEvStacks)) 142 } 143 144 // Emit stack event. 145 w.byte(byte(traceEvStack)) 146 w.varint(uint64(stk.id)) 147 w.varint(uint64(len(frames))) 148 for _, frame := range frames { 149 w.varint(uint64(frame.PC)) 150 w.varint(frame.funcID) 151 w.varint(frame.fileID) 152 w.varint(frame.line) 153 } 154 } 155 } 156 // Still, hold the lock over reset. The callee expects it, even though it's 157 // not strictly necessary. 158 lock(&t.tab.lock) 159 t.tab.reset() 160 unlock(&t.tab.lock) 161 162 w.flush().end() 163 } 164 165 // makeTraceFrames returns the frames corresponding to pcs. It may 166 // allocate and may emit trace events. 167 func makeTraceFrames(gen uintptr, pcs []uintptr) []traceFrame { 168 frames := make([]traceFrame, 0, len(pcs)) 169 ci := CallersFrames(pcs) 170 for { 171 f, more := ci.Next() 172 frames = append(frames, makeTraceFrame(gen, f)) 173 if !more { 174 return frames 175 } 176 } 177 } 178 179 type traceFrame struct { 180 PC uintptr 181 funcID uint64 182 fileID uint64 183 line uint64 184 } 185 186 // makeTraceFrame sets up a traceFrame for a frame. 187 func makeTraceFrame(gen uintptr, f Frame) traceFrame { 188 var frame traceFrame 189 frame.PC = f.PC 190 191 fn := f.Function 192 const maxLen = 1 << 10 193 if len(fn) > maxLen { 194 fn = fn[len(fn)-maxLen:] 195 } 196 frame.funcID = trace.stringTab[gen%2].put(gen, fn) 197 frame.line = uint64(f.Line) 198 file := f.File 199 if len(file) > maxLen { 200 file = file[len(file)-maxLen:] 201 } 202 frame.fileID = trace.stringTab[gen%2].put(gen, file) 203 return frame 204 } 205 206 // tracefpunwindoff returns true if frame pointer unwinding for the tracer is 207 // disabled via GODEBUG or not supported by the architecture. 208 func tracefpunwindoff() bool { 209 return debug.tracefpunwindoff != 0 || (goarch.ArchFamily != goarch.AMD64 && goarch.ArchFamily != goarch.ARM64) 210 } 211 212 // fpTracebackPCs populates pcBuf with the return addresses for each frame and 213 // returns the number of PCs written to pcBuf. The returned PCs correspond to 214 // "physical frames" rather than "logical frames"; that is if A is inlined into 215 // B, this will return a PC for only B. 216 func fpTracebackPCs(fp unsafe.Pointer, pcBuf []uintptr) (i int) { 217 for i = 0; i < len(pcBuf) && fp != nil; i++ { 218 // return addr sits one word above the frame pointer 219 pcBuf[i] = *(*uintptr)(unsafe.Pointer(uintptr(fp) + goarch.PtrSize)) 220 // follow the frame pointer to the next one 221 fp = unsafe.Pointer(*(*uintptr)(fp)) 222 } 223 return i 224 } 225 226 // fpunwindExpand checks if pcBuf contains logical frames (which include inlined 227 // frames) or physical frames (produced by frame pointer unwinding) using a 228 // sentinel value in pcBuf[0]. Logical frames are simply returned without the 229 // sentinel. Physical frames are turned into logical frames via inline unwinding 230 // and by applying the skip value that's stored in pcBuf[0]. 231 func fpunwindExpand(pcBuf []uintptr) []uintptr { 232 if len(pcBuf) > 0 && pcBuf[0] == logicalStackSentinel { 233 // pcBuf contains logical rather than inlined frames, skip has already been 234 // applied, just return it without the sentinel value in pcBuf[0]. 235 return pcBuf[1:] 236 } 237 238 var ( 239 lastFuncID = abi.FuncIDNormal 240 newPCBuf = make([]uintptr, 0, traceStackSize) 241 skip = pcBuf[0] 242 // skipOrAdd skips or appends retPC to newPCBuf and returns true if more 243 // pcs can be added. 244 skipOrAdd = func(retPC uintptr) bool { 245 if skip > 0 { 246 skip-- 247 } else { 248 newPCBuf = append(newPCBuf, retPC) 249 } 250 return len(newPCBuf) < cap(newPCBuf) 251 } 252 ) 253 254 outer: 255 for _, retPC := range pcBuf[1:] { 256 callPC := retPC - 1 257 fi := findfunc(callPC) 258 if !fi.valid() { 259 // There is no funcInfo if callPC belongs to a C function. In this case 260 // we still keep the pc, but don't attempt to expand inlined frames. 261 if more := skipOrAdd(retPC); !more { 262 break outer 263 } 264 continue 265 } 266 267 u, uf := newInlineUnwinder(fi, callPC) 268 for ; uf.valid(); uf = u.next(uf) { 269 sf := u.srcFunc(uf) 270 if sf.funcID == abi.FuncIDWrapper && elideWrapperCalling(lastFuncID) { 271 // ignore wrappers 272 } else if more := skipOrAdd(uf.pc + 1); !more { 273 break outer 274 } 275 lastFuncID = sf.funcID 276 } 277 } 278 return newPCBuf 279 } 280 281 // startPCForTrace returns the start PC of a goroutine for tracing purposes. 282 // If pc is a wrapper, it returns the PC of the wrapped function. Otherwise it 283 // returns pc. 284 func startPCForTrace(pc uintptr) uintptr { 285 f := findfunc(pc) 286 if !f.valid() { 287 return pc // may happen for locked g in extra M since its pc is 0. 288 } 289 w := funcdata(f, abi.FUNCDATA_WrapInfo) 290 if w == nil { 291 return pc // not a wrapper 292 } 293 return f.datap.textAddr(*(*uint32)(w)) 294 } 295