1 // Copyright 2023 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 // A coro represents extra concurrency without extra parallelism, 10 // as would be needed for a coroutine implementation. 11 // The coro does not represent a specific coroutine, only the ability 12 // to do coroutine-style control transfers. 13 // It can be thought of as like a special channel that always has 14 // a goroutine blocked on it. If another goroutine calls coroswitch(c), 15 // the caller becomes the goroutine blocked in c, and the goroutine 16 // formerly blocked in c starts running. 17 // These switches continue until a call to coroexit(c), 18 // which ends the use of the coro by releasing the blocked 19 // goroutine in c and exiting the current goroutine. 20 // 21 // Coros are heap allocated and garbage collected, so that user code 22 // can hold a pointer to a coro without causing potential dangling 23 // pointer errors. 24 type coro struct { 25 gp guintptr 26 f func(*coro) 27 } 28 29 //go:linkname newcoro 30 31 // newcoro creates a new coro containing a 32 // goroutine blocked waiting to run f 33 // and returns that coro. 34 func newcoro(f func(*coro)) *coro { 35 c := new(coro) 36 c.f = f 37 pc := getcallerpc() 38 gp := getg() 39 systemstack(func() { 40 start := corostart 41 startfv := *(**funcval)(unsafe.Pointer(&start)) 42 gp = newproc1(startfv, gp, pc) 43 }) 44 gp.coroarg = c 45 gp.waitreason = waitReasonCoroutine 46 casgstatus(gp, _Grunnable, _Gwaiting) 47 c.gp.set(gp) 48 return c 49 } 50 51 //go:linkname corostart 52 53 // corostart is the entry func for a new coroutine. 54 // It runs the coroutine user function f passed to corostart 55 // and then calls coroexit to remove the extra concurrency. 56 func corostart() { 57 gp := getg() 58 c := gp.coroarg 59 gp.coroarg = nil 60 61 c.f(c) 62 coroexit(c) 63 } 64 65 // coroexit is like coroswitch but closes the coro 66 // and exits the current goroutine 67 func coroexit(c *coro) { 68 gp := getg() 69 gp.coroarg = c 70 gp.coroexit = true 71 mcall(coroswitch_m) 72 } 73 74 //go:linkname coroswitch 75 76 // coroswitch switches to the goroutine blocked on c 77 // and then blocks the current goroutine on c. 78 func coroswitch(c *coro) { 79 gp := getg() 80 gp.coroarg = c 81 mcall(coroswitch_m) 82 } 83 84 // coroswitch_m is the implementation of coroswitch 85 // that runs on the m stack. 86 // 87 // Note: Coroutine switches are expected to happen at 88 // an order of magnitude (or more) higher frequency 89 // than regular goroutine switches, so this path is heavily 90 // optimized to remove unnecessary work. 91 // The fast path here is three CAS: the one at the top on gp.atomicstatus, 92 // the one in the middle to choose the next g, 93 // and the one at the bottom on gnext.atomicstatus. 94 // It is important not to add more atomic operations or other 95 // expensive operations to the fast path. 96 func coroswitch_m(gp *g) { 97 // TODO(rsc,mknyszek): add tracing support in a lightweight manner. 98 // Probably the tracer will need a global bool (set and cleared during STW) 99 // that this code can check to decide whether to use trace.gen.Load(); 100 // we do not want to do the atomic load all the time, especially when 101 // tracer use is relatively rare. 102 c := gp.coroarg 103 gp.coroarg = nil 104 exit := gp.coroexit 105 gp.coroexit = false 106 mp := gp.m 107 108 if exit { 109 gdestroy(gp) 110 gp = nil 111 } else { 112 // If we can CAS ourselves directly from running to waiting, so do, 113 // keeping the control transfer as lightweight as possible. 114 gp.waitreason = waitReasonCoroutine 115 if !gp.atomicstatus.CompareAndSwap(_Grunning, _Gwaiting) { 116 // The CAS failed: use casgstatus, which will take care of 117 // coordinating with the garbage collector about the state change. 118 casgstatus(gp, _Grunning, _Gwaiting) 119 } 120 121 // Clear gp.m. 122 setMNoWB(&gp.m, nil) 123 } 124 125 // The goroutine stored in c is the one to run next. 126 // Swap it with ourselves. 127 var gnext *g 128 for { 129 // Note: this is a racy load, but it will eventually 130 // get the right value, and if it gets the wrong value, 131 // the c.gp.cas will fail, so no harm done other than 132 // a wasted loop iteration. 133 // The cas will also sync c.gp's 134 // memory enough that the next iteration of the racy load 135 // should see the correct value. 136 // We are avoiding the atomic load to keep this path 137 // as lightweight as absolutely possible. 138 // (The atomic load is free on x86 but not free elsewhere.) 139 next := c.gp 140 if next.ptr() == nil { 141 throw("coroswitch on exited coro") 142 } 143 var self guintptr 144 self.set(gp) 145 if c.gp.cas(next, self) { 146 gnext = next.ptr() 147 break 148 } 149 } 150 151 // Start running next, without heavy scheduling machinery. 152 // Set mp.curg and gnext.m and then update scheduling state 153 // directly if possible. 154 setGNoWB(&mp.curg, gnext) 155 setMNoWB(&gnext.m, mp) 156 if !gnext.atomicstatus.CompareAndSwap(_Gwaiting, _Grunning) { 157 // The CAS failed: use casgstatus, which will take care of 158 // coordinating with the garbage collector about the state change. 159 casgstatus(gnext, _Gwaiting, _Grunnable) 160 casgstatus(gnext, _Grunnable, _Grunning) 161 } 162 163 // Switch to gnext. Does not return. 164 gogo(&gnext.sched) 165 } 166