1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "unsafe" 10 ) 11 12 // Should be a built-in for unsafe.Pointer? 13 // 14 //go:nosplit 15 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 16 return unsafe.Pointer(uintptr(p) + x) 17 } 18 19 // getg returns the pointer to the current g. 20 // The compiler rewrites calls to this function into instructions 21 // that fetch the g directly (from TLS or from the dedicated register). 22 func getg() *g 23 24 // mcall switches from the g to the g0 stack and invokes fn(g), 25 // where g is the goroutine that made the call. 26 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 27 // It is up to fn to arrange for that later execution, typically by recording 28 // g in a data structure, causing something to call ready(g) later. 29 // mcall returns to the original goroutine g later, when g has been rescheduled. 30 // fn must not return at all; typically it ends by calling schedule, to let the m 31 // run other goroutines. 32 // 33 // mcall can only be called from g stacks (not g0, not gsignal). 34 // 35 // This must NOT be go:noescape: if fn is a stack-allocated closure, 36 // fn puts g on a run queue, and g executes before fn returns, the 37 // closure will be invalidated while it is still executing. 38 func mcall(fn func(*g)) 39 40 // systemstack runs fn on a system stack. 41 // If systemstack is called from the per-OS-thread (g0) stack, or 42 // if systemstack is called from the signal handling (gsignal) stack, 43 // systemstack calls fn directly and returns. 44 // Otherwise, systemstack is being called from the limited stack 45 // of an ordinary goroutine. In this case, systemstack switches 46 // to the per-OS-thread stack, calls fn, and switches back. 47 // It is common to use a func literal as the argument, in order 48 // to share inputs and outputs with the code around the call 49 // to system stack: 50 // 51 // ... set up y ... 52 // systemstack(func() { 53 // x = bigcall(y) 54 // }) 55 // ... use x ... 56 // 57 //go:noescape 58 func systemstack(fn func()) 59 60 //go:nosplit 61 //go:nowritebarrierrec 62 func badsystemstack() { 63 writeErrStr("fatal: systemstack called from unexpected goroutine") 64 } 65 66 // memclrNoHeapPointers clears n bytes starting at ptr. 67 // 68 // Usually you should use typedmemclr. memclrNoHeapPointers should be 69 // used only when the caller knows that *ptr contains no heap pointers 70 // because either: 71 // 72 // *ptr is initialized memory and its type is pointer-free, or 73 // 74 // *ptr is uninitialized memory (e.g., memory that's being reused 75 // for a new allocation) and hence contains only "junk". 76 // 77 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 78 // is a multiple of the pointer size, then any pointer-aligned, 79 // pointer-sized portion is cleared atomically. Despite the function 80 // name, this is necessary because this function is the underlying 81 // implementation of typedmemclr and memclrHasPointers. See the doc of 82 // memmove for more details. 83 // 84 // The (CPU-specific) implementations of this function are in memclr_*.s. 85 // 86 //go:noescape 87 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 88 89 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 90 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 91 memclrNoHeapPointers(ptr, n) 92 } 93 94 // memmove copies n bytes from "from" to "to". 95 // 96 // memmove ensures that any pointer in "from" is written to "to" with 97 // an indivisible write, so that racy reads cannot observe a 98 // half-written pointer. This is necessary to prevent the garbage 99 // collector from observing invalid pointers, and differs from memmove 100 // in unmanaged languages. However, memmove is only required to do 101 // this if "from" and "to" may contain pointers, which can only be the 102 // case if "from", "to", and "n" are all be word-aligned. 103 // 104 // Implementations are in memmove_*.s. 105 // 106 //go:noescape 107 func memmove(to, from unsafe.Pointer, n uintptr) 108 109 // Outside assembly calls memmove. Make sure it has ABI wrappers. 110 // 111 //go:linkname memmove 112 113 //go:linkname reflect_memmove reflect.memmove 114 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 115 memmove(to, from, n) 116 } 117 118 // exported value for testing 119 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 120 121 // in internal/bytealg/equal_*.s 122 // 123 //go:noescape 124 func memequal(a, b unsafe.Pointer, size uintptr) bool 125 126 // noescape hides a pointer from escape analysis. noescape is 127 // the identity function but escape analysis doesn't think the 128 // output depends on the input. noescape is inlined and currently 129 // compiles down to zero instructions. 130 // USE CAREFULLY! 131 // 132 //go:nosplit 133 func noescape(p unsafe.Pointer) unsafe.Pointer { 134 x := uintptr(p) 135 return unsafe.Pointer(x ^ 0) 136 } 137 138 // noEscapePtr hides a pointer from escape analysis. See noescape. 139 // USE CAREFULLY! 140 // 141 //go:nosplit 142 func noEscapePtr[T any](p *T) *T { 143 x := uintptr(unsafe.Pointer(p)) 144 return (*T)(unsafe.Pointer(x ^ 0)) 145 } 146 147 // Not all cgocallback frames are actually cgocallback, 148 // so not all have these arguments. Mark them uintptr so that the GC 149 // does not misinterpret memory when the arguments are not present. 150 // cgocallback is not called from Go, only from crosscall2. 151 // This in turn calls cgocallbackg, which is where we'll find 152 // pointer-declared arguments. 153 // 154 // When fn is nil (frame is saved g), call dropm instead, 155 // this is used when the C thread is exiting. 156 func cgocallback(fn, frame, ctxt uintptr) 157 158 func gogo(buf *gobuf) 159 160 func asminit() 161 func setg(gg *g) 162 func breakpoint() 163 164 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 165 // frameSize, and regArgs. 166 // 167 // Arguments passed on the stack and space for return values passed on the stack 168 // must be laid out at the space pointed to by stackArgs (with total length 169 // stackArgsSize) according to the ABI. 170 // 171 // stackRetOffset must be some value <= stackArgsSize that indicates the 172 // offset within stackArgs where the return value space begins. 173 // 174 // frameSize is the total size of the argument frame at stackArgs and must 175 // therefore be >= stackArgsSize. It must include additional space for spilling 176 // register arguments for stack growth and preemption. 177 // 178 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 179 // since frameSize will be redundant with stackArgsSize. 180 // 181 // Arguments passed in registers must be laid out in regArgs according to the ABI. 182 // regArgs will hold any return values passed in registers after the call. 183 // 184 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 185 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 186 // in stackArgs once fn has completed. It also "unspills" argument registers from 187 // regArgs before calling fn, and spills them back into regArgs immediately 188 // following the call to fn. If there are results being returned on the stack, 189 // the caller should pass the argument frame type as stackArgsType so that 190 // reflectcall can execute appropriate write barriers during the copy. 191 // 192 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 193 // registers on the return path will contain Go pointers. It will then store 194 // these pointers in regArgs.Ptrs such that they are visible to the GC. 195 // 196 // Package reflect passes a frame type. In package runtime, there is only 197 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 198 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 199 // call site for justification. 200 // 201 // Package reflect accesses this symbol through a linkname. 202 // 203 // Arguments passed through to reflectcall do not escape. The type is used 204 // only in a very limited callee of reflectcall, the stackArgs are copied, and 205 // regArgs is only used in the reflectcall frame. 206 // 207 //go:noescape 208 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 209 210 func procyield(cycles uint32) 211 212 type neverCallThisFunction struct{} 213 214 // goexit is the return stub at the top of every goroutine call stack. 215 // Each goroutine stack is constructed as if goexit called the 216 // goroutine's entry point function, so that when the entry point 217 // function returns, it will return to goexit, which will call goexit1 218 // to perform the actual exit. 219 // 220 // This function must never be called directly. Call goexit1 instead. 221 // gentraceback assumes that goexit terminates the stack. A direct 222 // call on the stack will cause gentraceback to stop walking the stack 223 // prematurely and if there is leftover state it may panic. 224 func goexit(neverCallThisFunction) 225 226 // publicationBarrier performs a store/store barrier (a "publication" 227 // or "export" barrier). Some form of synchronization is required 228 // between initializing an object and making that object accessible to 229 // another processor. Without synchronization, the initialization 230 // writes and the "publication" write may be reordered, allowing the 231 // other processor to follow the pointer and observe an uninitialized 232 // object. In general, higher-level synchronization should be used, 233 // such as locking or an atomic pointer write. publicationBarrier is 234 // for when those aren't an option, such as in the implementation of 235 // the memory manager. 236 // 237 // There's no corresponding barrier for the read side because the read 238 // side naturally has a data dependency order. All architectures that 239 // Go supports or seems likely to ever support automatically enforce 240 // data dependency ordering. 241 func publicationBarrier() 242 243 // getcallerpc returns the program counter (PC) of its caller's caller. 244 // getcallersp returns the stack pointer (SP) of its caller's caller. 245 // The implementation may be a compiler intrinsic; there is not 246 // necessarily code implementing this on every platform. 247 // 248 // For example: 249 // 250 // func f(arg1, arg2, arg3 int) { 251 // pc := getcallerpc() 252 // sp := getcallersp() 253 // } 254 // 255 // These two lines find the PC and SP immediately following 256 // the call to f (where f will return). 257 // 258 // The call to getcallerpc and getcallersp must be done in the 259 // frame being asked about. 260 // 261 // The result of getcallersp is correct at the time of the return, 262 // but it may be invalidated by any subsequent call to a function 263 // that might relocate the stack in order to grow or shrink it. 264 // A general rule is that the result of getcallersp should be used 265 // immediately and can only be passed to nosplit functions. 266 267 //go:noescape 268 func getcallerpc() uintptr 269 270 //go:noescape 271 func getcallersp() uintptr // implemented as an intrinsic on all platforms 272 273 // getclosureptr returns the pointer to the current closure. 274 // getclosureptr can only be used in an assignment statement 275 // at the entry of a function. Moreover, go:nosplit directive 276 // must be specified at the declaration of caller function, 277 // so that the function prolog does not clobber the closure register. 278 // for example: 279 // 280 // //go:nosplit 281 // func f(arg1, arg2, arg3 int) { 282 // dx := getclosureptr() 283 // } 284 // 285 // The compiler rewrites calls to this function into instructions that fetch the 286 // pointer from a well-known register (DX on x86 architecture, etc.) directly. 287 // 288 // WARNING: PGO-based devirtualization cannot detect that caller of 289 // getclosureptr require closure context, and thus must maintain a list of 290 // these functions, which is in 291 // cmd/compile/internal/devirtualize/pgo.maybeDevirtualizeFunctionCall. 292 func getclosureptr() uintptr 293 294 //go:noescape 295 func asmcgocall(fn, arg unsafe.Pointer) int32 296 297 func morestack() 298 func morestack_noctxt() 299 func rt0_go() 300 301 // return0 is a stub used to return 0 from deferproc. 302 // It is called at the very end of deferproc to signal 303 // the calling Go function that it should not jump 304 // to deferreturn. 305 // in asm_*.s 306 func return0() 307 308 // in asm_*.s 309 // not called directly; definitions here supply type information for traceback. 310 // These must have the same signature (arg pointer map) as reflectcall. 311 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 312 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 313 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 314 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 315 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 316 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 317 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 318 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 319 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 320 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 321 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 322 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 323 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 324 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 325 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 326 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 327 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 328 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 329 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 330 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 331 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 332 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 333 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 334 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 335 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 336 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 337 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 338 339 func systemstack_switch() 340 341 // alignUp rounds n up to a multiple of a. a must be a power of 2. 342 // 343 //go:nosplit 344 func alignUp(n, a uintptr) uintptr { 345 return (n + a - 1) &^ (a - 1) 346 } 347 348 // alignDown rounds n down to a multiple of a. a must be a power of 2. 349 // 350 //go:nosplit 351 func alignDown(n, a uintptr) uintptr { 352 return n &^ (a - 1) 353 } 354 355 // divRoundUp returns ceil(n / a). 356 func divRoundUp(n, a uintptr) uintptr { 357 // a is generally a power of two. This will get inlined and 358 // the compiler will optimize the division. 359 return (n + a - 1) / a 360 } 361 362 // checkASM reports whether assembly runtime checks have passed. 363 func checkASM() bool 364 365 func memequal_varlen(a, b unsafe.Pointer) bool 366 367 // bool2int returns 0 if x is false or 1 if x is true. 368 func bool2int(x bool) int { 369 // Avoid branches. In the SSA compiler, this compiles to 370 // exactly what you would want it to. 371 return int(*(*uint8)(unsafe.Pointer(&x))) 372 } 373 374 // abort crashes the runtime in situations where even throw might not 375 // work. In general it should do something a debugger will recognize 376 // (e.g., an INT3 on x86). A crash in abort is recognized by the 377 // signal handler, which will attempt to tear down the runtime 378 // immediately. 379 func abort() 380 381 // Called from compiled code; declared for vet; do NOT call from Go. 382 func gcWriteBarrier1() 383 func gcWriteBarrier2() 384 func gcWriteBarrier3() 385 func gcWriteBarrier4() 386 func gcWriteBarrier5() 387 func gcWriteBarrier6() 388 func gcWriteBarrier7() 389 func gcWriteBarrier8() 390 func duffzero() 391 func duffcopy() 392 393 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 394 func addmoduledata() 395 396 // Injected by the signal handler for panicking signals. 397 // Initializes any registers that have fixed meaning at calls but 398 // are scratch in bodies and calls sigpanic. 399 // On many platforms it just jumps to sigpanic. 400 func sigpanic0() 401 402 // intArgRegs is used by the various register assignment 403 // algorithm implementations in the runtime. These include:. 404 // - Finalizers (mfinal.go) 405 // - Windows callbacks (syscall_windows.go) 406 // 407 // Both are stripped-down versions of the algorithm since they 408 // only have to deal with a subset of cases (finalizers only 409 // take a pointer or interface argument, Go Windows callbacks 410 // don't support floating point). 411 // 412 // It should be modified with care and are generally only 413 // modified when testing this package. 414 // 415 // It should never be set higher than its internal/abi 416 // constant counterparts, because the system relies on a 417 // structure that is at least large enough to hold the 418 // registers the system supports. 419 // 420 // Protected by finlock. 421 var intArgRegs = abi.IntArgRegs 422