diff options
Diffstat (limited to 'src/runtime')
| -rw-r--r-- | src/runtime/atomic_386.go | 22 | ||||
| -rw-r--r-- | src/runtime/atomic_amd64x.go | 22 | ||||
| -rw-r--r-- | src/runtime/atomic_arm.go | 8 | ||||
| -rw-r--r-- | src/runtime/atomic_pointer.go | 96 | ||||
| -rw-r--r-- | src/runtime/atomic_ppc64x.go | 22 | ||||
| -rw-r--r-- | src/runtime/race_amd64.s | 15 | ||||
| -rw-r--r-- | src/runtime/stubs.go | 13 |
7 files changed, 107 insertions, 91 deletions
diff --git a/src/runtime/atomic_386.go b/src/runtime/atomic_386.go index 5563432eff..0171d907a3 100644 --- a/src/runtime/atomic_386.go +++ b/src/runtime/atomic_386.go @@ -48,18 +48,7 @@ func xadd(ptr *uint32, delta int32) uint32 //go:noescape func xchg(ptr *uint32, new uint32) uint32 -// xchgp cannot have a go:noescape annotation, because -// while ptr does not escape, new does. If new is marked as -// not escaping, the compiler will make incorrect escape analysis -// decisions about the value being xchg'ed. -// Instead, make xchgp a wrapper around the actual atomic. -// When calling the wrapper we mark ptr as noescape explicitly. - -//go:nosplit -func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { - return xchgp1(noescape(ptr), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func xchgp1(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer //go:noescape @@ -80,12 +69,5 @@ func atomicstore(ptr *uint32, val uint32) //go:noescape func atomicstore64(ptr *uint64, val uint64) -// atomicstorep cannot have a go:noescape annotation. -// See comment above for xchgp. - -//go:nosplit -func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { - atomicstorep1(noescape(ptr), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func atomicstorep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/atomic_amd64x.go b/src/runtime/atomic_amd64x.go index f2dd584118..c5355f6a4e 100644 --- a/src/runtime/atomic_amd64x.go +++ b/src/runtime/atomic_amd64x.go @@ -42,18 +42,7 @@ func xchg(ptr *uint32, new uint32) uint32 //go:noescape func xchg64(ptr *uint64, new uint64) uint64 -// xchgp cannot have a go:noescape annotation, because -// while ptr does not escape, new does. If new is marked as -// not escaping, the compiler will make incorrect escape analysis -// decisions about the value being xchg'ed. -// Instead, make xchgp a wrapper around the actual atomic. -// When calling the wrapper we mark ptr as noescape explicitly. - -//go:nosplit -func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { - return xchgp1(noescape(ptr), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func xchgp1(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer //go:noescape @@ -71,12 +60,5 @@ func atomicstore(ptr *uint32, val uint32) //go:noescape func atomicstore64(ptr *uint64, val uint64) -// atomicstorep cannot have a go:noescape annotation. -// See comment above for xchgp. - -//go:nosplit -func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { - atomicstorep1(noescape(ptr), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func atomicstorep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/atomic_arm.go b/src/runtime/atomic_arm.go index fd55a0aca8..7f0b929143 100644 --- a/src/runtime/atomic_arm.go +++ b/src/runtime/atomic_arm.go @@ -38,10 +38,10 @@ func xchg(addr *uint32, v uint32) uint32 { } //go:nosplit -func xchgp(addr *unsafe.Pointer, v unsafe.Pointer) unsafe.Pointer { +func xchgp1(addr *unsafe.Pointer, v unsafe.Pointer) unsafe.Pointer { for { old := *addr - if casp(addr, old, v) { + if casp1(addr, old, v) { return old } } @@ -63,10 +63,10 @@ func atomicloadp(addr unsafe.Pointer) unsafe.Pointer { } //go:nosplit -func atomicstorep(addr unsafe.Pointer, v unsafe.Pointer) { +func atomicstorep1(addr unsafe.Pointer, v unsafe.Pointer) { for { old := *(*unsafe.Pointer)(addr) - if casp((*unsafe.Pointer)(addr), old, v) { + if casp1((*unsafe.Pointer)(addr), old, v) { return } } diff --git a/src/runtime/atomic_pointer.go b/src/runtime/atomic_pointer.go new file mode 100644 index 0000000000..50a30242d9 --- /dev/null +++ b/src/runtime/atomic_pointer.go @@ -0,0 +1,96 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// These functions cannot have go:noescape annotations, +// because while ptr does not escape, new does. +// If new is marked as not escaping, the compiler will make incorrect +// escape analysis decisions about the pointer value being stored. +// Instead, these are wrappers around the actual atomics (xchgp1 and so on) +// that use noescape to convey which arguments do not escape. +// +// Additionally, these functions must update the shadow heap for +// write barrier checking. + +//go:nosplit +func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { + atomicstorep1(noescape(ptr), new) + writebarrierptr_nostore((*uintptr)(ptr), uintptr(new)) + if mheap_.shadow_enabled { + writebarrierptr_noshadow((*uintptr)(noescape(ptr))) + } +} + +//go:nosplit +func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { + old := xchgp1(noescape(ptr), new) + writebarrierptr_nostore((*uintptr)(ptr), uintptr(new)) + if mheap_.shadow_enabled { + writebarrierptr_noshadow((*uintptr)(noescape(ptr))) + } + return old +} + +//go:nosplit +func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { + if !casp1((*unsafe.Pointer)(noescape(unsafe.Pointer(ptr))), noescape(old), new) { + return false + } + writebarrierptr_nostore((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) + if mheap_.shadow_enabled { + writebarrierptr_noshadow((*uintptr)(noescape(unsafe.Pointer(ptr)))) + } + return true +} + +// Like above, but implement in terms of sync/atomic's uintptr operations. +// We cannot just call the runtime routines, because the race detector expects +// to be able to intercept the sync/atomic forms but not the runtime forms. + +//go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr +func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr) + +//go:linkname sync_atomic_StorePointer sync/atomic.StorePointer +//go:nosplit +func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) { + sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) + atomicstorep1(noescape(unsafe.Pointer(ptr)), new) + writebarrierptr_nostore((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) + if mheap_.shadow_enabled { + writebarrierptr_noshadow((*uintptr)(noescape(unsafe.Pointer(ptr)))) + } +} + +//go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr +func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr + +//go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer +//go:nosplit +func sync_atomic_SwapPointer(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { + old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(ptr)), uintptr(new))) + writebarrierptr_nostore((*uintptr)(ptr), uintptr(new)) + if mheap_.shadow_enabled { + writebarrierptr_noshadow((*uintptr)(noescape(ptr))) + } + return old +} + +//go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr +func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool + +//go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer +//go:nosplit +func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { + if !sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new)) { + return false + } + writebarrierptr_nostore((*uintptr)(unsafe.Pointer(ptr)), uintptr(new)) + if mheap_.shadow_enabled { + writebarrierptr_noshadow((*uintptr)(noescape(unsafe.Pointer(ptr)))) + } + return true +} diff --git a/src/runtime/atomic_ppc64x.go b/src/runtime/atomic_ppc64x.go index 4080af6a56..65dd9fc50c 100644 --- a/src/runtime/atomic_ppc64x.go +++ b/src/runtime/atomic_ppc64x.go @@ -20,18 +20,7 @@ func xchg(ptr *uint32, new uint32) uint32 //go:noescape func xchg64(ptr *uint64, new uint64) uint64 -// xchgp cannot have a go:noescape annotation, because -// while ptr does not escape, new does. If new is marked as -// not escaping, the compiler will make incorrect escape analysis -// decisions about the value being xchg'ed. -// Instead, make xchgp a wrapper around the actual atomic. -// When calling the wrapper we mark ptr as noescape explicitly. - -//go:nosplit -func xchgp(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer { - return xchgp1(noescape(ptr), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func xchgp1(ptr unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer //go:noescape @@ -58,12 +47,5 @@ func atomicstore(ptr *uint32, val uint32) //go:noescape func atomicstore64(ptr *uint64, val uint64) -// atomicstorep cannot have a go:noescape annotation. -// See comment above for xchgp. - -//go:nosplit -func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) { - atomicstorep1(noescape(ptr), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func atomicstorep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index d54d9798f0..adf557f8a2 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -202,9 +202,6 @@ TEXT sync∕atomic·LoadUint64(SB), NOSPLIT, $0-0 TEXT sync∕atomic·LoadUintptr(SB), NOSPLIT, $0-0 JMP sync∕atomic·LoadInt64(SB) -TEXT sync∕atomic·LoadPointer(SB), NOSPLIT, $0-0 - JMP sync∕atomic·LoadInt64(SB) - // Store TEXT sync∕atomic·StoreInt32(SB), NOSPLIT, $0-0 MOVQ $__tsan_go_atomic32_store(SB), AX @@ -225,9 +222,6 @@ TEXT sync∕atomic·StoreUint64(SB), NOSPLIT, $0-0 TEXT sync∕atomic·StoreUintptr(SB), NOSPLIT, $0-0 JMP sync∕atomic·StoreInt64(SB) -TEXT sync∕atomic·StorePointer(SB), NOSPLIT, $0-0 - JMP sync∕atomic·StoreInt64(SB) - // Swap TEXT sync∕atomic·SwapInt32(SB), NOSPLIT, $0-0 MOVQ $__tsan_go_atomic32_exchange(SB), AX @@ -248,9 +242,6 @@ TEXT sync∕atomic·SwapUint64(SB), NOSPLIT, $0-0 TEXT sync∕atomic·SwapUintptr(SB), NOSPLIT, $0-0 JMP sync∕atomic·SwapInt64(SB) -TEXT sync∕atomic·SwapPointer(SB), NOSPLIT, $0-0 - JMP sync∕atomic·SwapInt64(SB) - // Add TEXT sync∕atomic·AddInt32(SB), NOSPLIT, $0-0 MOVQ $__tsan_go_atomic32_fetch_add(SB), AX @@ -275,9 +266,6 @@ TEXT sync∕atomic·AddUint64(SB), NOSPLIT, $0-0 TEXT sync∕atomic·AddUintptr(SB), NOSPLIT, $0-0 JMP sync∕atomic·AddInt64(SB) -TEXT sync∕atomic·AddPointer(SB), NOSPLIT, $0-0 - JMP sync∕atomic·AddInt64(SB) - // CompareAndSwap TEXT sync∕atomic·CompareAndSwapInt32(SB), NOSPLIT, $0-0 MOVQ $__tsan_go_atomic32_compare_exchange(SB), AX @@ -298,9 +286,6 @@ TEXT sync∕atomic·CompareAndSwapUint64(SB), NOSPLIT, $0-0 TEXT sync∕atomic·CompareAndSwapUintptr(SB), NOSPLIT, $0-0 JMP sync∕atomic·CompareAndSwapInt64(SB) -TEXT sync∕atomic·CompareAndSwapPointer(SB), NOSPLIT, $0-0 - JMP sync∕atomic·CompareAndSwapInt64(SB) - // Generic atomic operation implementation. // AX already contains target function. TEXT racecallatomic<>(SB), NOSPLIT, $0-0 diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 591ece6b3a..67f78bdae5 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -118,18 +118,7 @@ func goexit() //go:noescape func cas(ptr *uint32, old, new uint32) bool -// casp cannot have a go:noescape annotation, because -// while ptr and old do not escape, new does. If new is marked as -// not escaping, the compiler will make incorrect escape analysis -// decisions about the value being xchg'ed. -// Instead, make casp a wrapper around the actual atomic. -// When calling the wrapper we mark ptr as noescape explicitly. - -//go:nosplit -func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool { - return casp1((*unsafe.Pointer)(noescape(unsafe.Pointer(ptr))), noescape(old), new) -} - +// NO go:noescape annotation; see atomic_pointer.go. func casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool func nop() // call to prevent inlining of function body |