atomic_pointer.go

Documentation: runtime

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/goexperiment"
     9  	"runtime/internal/atomic"
    10  	"unsafe"
    11  )
    12  
    13  // These functions cannot have go:noescape annotations,
    14  // because while ptr does not escape, new does.
    15  // If new is marked as not escaping, the compiler will make incorrect
    16  // escape analysis decisions about the pointer value being stored.
    17  
    18  // atomicwb performs a write barrier before an atomic pointer write.
    19  // The caller should guard the call with "if writeBarrier.enabled".
    20  //
    21  //go:nosplit
    22  func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
    23  	slot := (*uintptr)(unsafe.Pointer(ptr))
    24  	buf := getg().m.p.ptr().wbBuf.get2()
    25  	buf[0] = *slot
    26  	buf[1] = uintptr(new)
    27  }
    28  
    29  // atomicstorep performs *ptr = new atomically and invokes a write barrier.
    30  //
    31  //go:nosplit
    32  func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
    33  	if writeBarrier.enabled {
    34  		atomicwb((*unsafe.Pointer)(ptr), new)
    35  	}
    36  	if goexperiment.CgoCheck2 {
    37  		cgoCheckPtrWrite((*unsafe.Pointer)(ptr), new)
    38  	}
    39  	atomic.StorepNoWB(noescape(ptr), new)
    40  }
    41  
    42  // atomic_storePointer is the implementation of runtime/internal/UnsafePointer.Store
    43  // (like StoreNoWB but with the write barrier).
    44  //
    45  //go:nosplit
    46  //go:linkname atomic_storePointer runtime/internal/atomic.storePointer
    47  func atomic_storePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    48  	atomicstorep(unsafe.Pointer(ptr), new)
    49  }
    50  
    51  // atomic_casPointer is the implementation of runtime/internal/UnsafePointer.CompareAndSwap
    52  // (like CompareAndSwapNoWB but with the write barrier).
    53  //
    54  //go:nosplit
    55  //go:linkname atomic_casPointer runtime/internal/atomic.casPointer
    56  func atomic_casPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
    57  	if writeBarrier.enabled {
    58  		atomicwb(ptr, new)
    59  	}
    60  	if goexperiment.CgoCheck2 {
    61  		cgoCheckPtrWrite(ptr, new)
    62  	}
    63  	return atomic.Casp1(ptr, old, new)
    64  }
    65  
    66  // Like above, but implement in terms of sync/atomic's uintptr operations.
    67  // We cannot just call the runtime routines, because the race detector expects
    68  // to be able to intercept the sync/atomic forms but not the runtime forms.
    69  
    70  //go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
    71  func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
    72  
    73  //go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
    74  //go:nosplit
    75  func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    76  	if writeBarrier.enabled {
    77  		atomicwb(ptr, new)
    78  	}
    79  	if goexperiment.CgoCheck2 {
    80  		cgoCheckPtrWrite(ptr, new)
    81  	}
    82  	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
    83  }
    84  
    85  //go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
    86  func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
    87  
    88  //go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
    89  //go:nosplit
    90  func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
    91  	if writeBarrier.enabled {
    92  		atomicwb(ptr, new)
    93  	}
    94  	if goexperiment.CgoCheck2 {
    95  		cgoCheckPtrWrite(ptr, new)
    96  	}
    97  	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
    98  	return old
    99  }
   100  
   101  //go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
   102  func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
   103  
   104  //go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
   105  //go:nosplit
   106  func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
   107  	if writeBarrier.enabled {
   108  		atomicwb(ptr, new)
   109  	}
   110  	if goexperiment.CgoCheck2 {
   111  		cgoCheckPtrWrite(ptr, new)
   112  	}
   113  	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
   114  }
   115
View as plain text