aboutsummaryrefslogtreecommitdiff
path: root/src/runtime
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime')
-rw-r--r--src/runtime/malloc.go4
-rw-r--r--src/runtime/mbarrier.go6
-rw-r--r--src/runtime/netpoll.go22
-rw-r--r--src/runtime/netpoll_epoll.go12
-rw-r--r--src/runtime/netpoll_kqueue.go11
-rw-r--r--src/runtime/netpoll_solaris.go12
-rw-r--r--src/runtime/netpoll_windows.go11
-rw-r--r--src/runtime/os1_windows.go3
-rw-r--r--src/runtime/os3_plan9.go2
-rw-r--r--src/runtime/panic.go4
-rw-r--r--src/runtime/proc.go4
-rw-r--r--src/runtime/proc1.go315
-rw-r--r--src/runtime/runtime2.go101
-rw-r--r--src/runtime/signal_386.go2
-rw-r--r--src/runtime/signal_amd64x.go2
-rw-r--r--src/runtime/signal_arm.go2
-rw-r--r--src/runtime/signal_arm64.go2
-rw-r--r--src/runtime/signal_ppc64x.go2
-rw-r--r--src/runtime/stack1.go4
-rw-r--r--src/runtime/trace.go6
-rw-r--r--src/runtime/traceback.go2
21 files changed, 257 insertions, 272 deletions
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index e41c2736b1..5fe0b160e6 100644
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -817,8 +817,8 @@ func persistentalloc(size, align uintptr, stat *uint64) unsafe.Pointer {
mp := acquirem()
var persistent *persistentAlloc
- if mp != nil && mp.p != nil {
- persistent = &mp.p.palloc
+ if mp != nil && mp.p != 0 {
+ persistent = &mp.p.ptr().palloc
} else {
lock(&globalAlloc.mutex)
persistent = &globalAlloc.persistentAlloc
diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go
index 2e3e4d8041..9bf0e6b809 100644
--- a/src/runtime/mbarrier.go
+++ b/src/runtime/mbarrier.go
@@ -92,10 +92,6 @@ func needwb() bool {
// the p associated with an m. We use the fact that m.p == nil to indicate
// that we are in one these critical section and throw if the write is of
// a pointer to a heap object.
-// The p, m, and g pointers are the pointers that are used by the scheduler
-// and need to be operated on without write barriers. We use
-// the setPNoWriteBarrier, setMNoWriteBarrier and setGNowriteBarrier to
-// avoid having to do the write barrier.
//go:nosplit
func writebarrierptr_nostore1(dst *uintptr, src uintptr) {
mp := acquirem()
@@ -104,7 +100,7 @@ func writebarrierptr_nostore1(dst *uintptr, src uintptr) {
return
}
systemstack(func() {
- if mp.p == nil && memstats.enablegc && !mp.inwb && inheap(src) {
+ if mp.p == 0 && memstats.enablegc && !mp.inwb && inheap(src) {
throw("writebarrierptr_nostore1 called with mp.p == nil")
}
mp.inwb = true
diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go
index 4791e5eebe..c06722fb98 100644
--- a/src/runtime/netpoll.go
+++ b/src/runtime/netpoll.go
@@ -275,24 +275,22 @@ func net_runtime_pollUnblock(pd *pollDesc) {
// make pd ready, newly runnable goroutines (if any) are returned in rg/wg
// May run during STW, so write barriers are not allowed.
-// Eliminating WB calls using setGNoWriteBarrier are safe since the gs are
-// reachable through allg.
//go:nowritebarrier
-func netpollready(gpp **g, pd *pollDesc, mode int32) {
- var rg, wg *g
+func netpollready(gpp *guintptr, pd *pollDesc, mode int32) {
+ var rg, wg guintptr
if mode == 'r' || mode == 'r'+'w' {
- setGNoWriteBarrier(&rg, netpollunblock(pd, 'r', true))
+ rg.set(netpollunblock(pd, 'r', true))
}
if mode == 'w' || mode == 'r'+'w' {
- setGNoWriteBarrier(&wg, netpollunblock(pd, 'w', true))
+ wg.set(netpollunblock(pd, 'w', true))
}
- if rg != nil {
- setGNoWriteBarrier(&rg.schedlink, *gpp)
- setGNoWriteBarrier(gpp, rg)
+ if rg != 0 {
+ rg.ptr().schedlink = *gpp
+ *gpp = rg
}
- if wg != nil {
- setGNoWriteBarrier(&wg.schedlink, *gpp)
- setGNoWriteBarrier(gpp, wg)
+ if wg != 0 {
+ wg.ptr().schedlink = *gpp
+ *gpp = wg
}
}
diff --git a/src/runtime/netpoll_epoll.go b/src/runtime/netpoll_epoll.go
index 4f9ccaf477..7b4052a262 100644
--- a/src/runtime/netpoll_epoll.go
+++ b/src/runtime/netpoll_epoll.go
@@ -55,9 +55,9 @@ func netpollarm(pd *pollDesc, mode int) {
// polls for ready network connections
// returns list of goroutines that become runnable
-func netpoll(block bool) (gp *g) {
+func netpoll(block bool) *g {
if epfd == -1 {
- return
+ return nil
}
waitms := int32(-1)
if !block {
@@ -73,6 +73,7 @@ retry:
}
goto retry
}
+ var gp guintptr
for i := int32(0); i < n; i++ {
ev := &events[i]
if ev.events == 0 {
@@ -87,11 +88,12 @@ retry:
}
if mode != 0 {
pd := *(**pollDesc)(unsafe.Pointer(&ev.data))
- netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode)
+
+ netpollready(&gp, pd, mode)
}
}
- if block && gp == nil {
+ if block && gp == 0 {
goto retry
}
- return gp
+ return gp.ptr()
}
diff --git a/src/runtime/netpoll_kqueue.go b/src/runtime/netpoll_kqueue.go
index 080964bb72..01445dc231 100644
--- a/src/runtime/netpoll_kqueue.go
+++ b/src/runtime/netpoll_kqueue.go
@@ -62,9 +62,9 @@ func netpollarm(pd *pollDesc, mode int) {
// Polls for ready network connections.
// Returns list of goroutines that become runnable.
-func netpoll(block bool) (gp *g) {
+func netpoll(block bool) *g {
if kq == -1 {
- return
+ return nil
}
var tp *timespec
var ts timespec
@@ -81,6 +81,7 @@ retry:
}
goto retry
}
+ var gp guintptr
for i := 0; i < int(n); i++ {
ev := &events[i]
var mode int32
@@ -91,11 +92,11 @@ retry:
mode += 'w'
}
if mode != 0 {
- netpollready((**g)(noescape(unsafe.Pointer(&gp))), (*pollDesc)(unsafe.Pointer(ev.udata)), mode)
+ netpollready(&gp, (*pollDesc)(unsafe.Pointer(ev.udata)), mode)
}
}
- if block && gp == nil {
+ if block && gp == 0 {
goto retry
}
- return gp
+ return gp.ptr()
}
diff --git a/src/runtime/netpoll_solaris.go b/src/runtime/netpoll_solaris.go
index d8f050e1e2..abfe56d801 100644
--- a/src/runtime/netpoll_solaris.go
+++ b/src/runtime/netpoll_solaris.go
@@ -179,9 +179,9 @@ var netpolllasterr int32
// polls for ready network connections
// returns list of goroutines that become runnable
-func netpoll(block bool) (gp *g) {
+func netpoll(block bool) *g {
if portfd == -1 {
- return
+ return nil
}
var wait *timespec
@@ -201,7 +201,7 @@ retry:
goto retry
}
- gp = nil
+ var gp guintptr
for i := 0; i < int(n); i++ {
ev := &events[i]
@@ -232,12 +232,12 @@ retry:
}
if mode != 0 {
- netpollready((**g)(noescape(unsafe.Pointer(&gp))), pd, mode)
+ netpollready(&gp, pd, mode)
}
}
- if block && gp == nil {
+ if block && gp == 0 {
goto retry
}
- return gp
+ return gp.ptr()
}
diff --git a/src/runtime/netpoll_windows.go b/src/runtime/netpoll_windows.go
index 0861e20f0c..7e15cd231e 100644
--- a/src/runtime/netpoll_windows.go
+++ b/src/runtime/netpoll_windows.go
@@ -63,14 +63,13 @@ func netpoll(block bool) *g {
var wait, qty, key, flags, n, i uint32
var errno int32
var op *net_op
- var gp *g
+ var gp guintptr
mp := getg().m
if iocphandle == _INVALID_HANDLE_VALUE {
return nil
}
- gp = nil
wait = 0
if block {
wait = _INFINITE
@@ -125,13 +124,13 @@ retry:
mp.blocked = false
handlecompletion(&gp, op, errno, qty)
}
- if block && gp == nil {
+ if block && gp == 0 {
goto retry
}
- return gp
+ return gp.ptr()
}
-func handlecompletion(gpp **g, op *net_op, errno int32, qty uint32) {
+func handlecompletion(gpp *guintptr, op *net_op, errno int32, qty uint32) {
if op == nil {
throw("netpoll: GetQueuedCompletionStatus returned op == nil")
}
@@ -142,5 +141,5 @@ func handlecompletion(gpp **g, op *net_op, errno int32, qty uint32) {
}
op.errno = errno
op.qty = qty
- netpollready((**g)(noescape(unsafe.Pointer(gpp))), op.pd, mode)
+ netpollready(gpp, op.pd, mode)
}
diff --git a/src/runtime/os1_windows.go b/src/runtime/os1_windows.go
index 8df2968196..2b9739fca3 100644
--- a/src/runtime/os1_windows.go
+++ b/src/runtime/os1_windows.go
@@ -363,8 +363,7 @@ func stdcall(fn stdFunction) uintptr {
if mp.profilehz != 0 {
// leave pc/sp for cpu profiler
- // gp is on allg, so this WB can be eliminated.
- setGNoWriteBarrier(&mp.libcallg, gp)
+ mp.libcallg.set(gp)
mp.libcallpc = getcallerpc(unsafe.Pointer(&fn))
// sp must be the last, because once async cpu profiler finds
// all three values to be non-zero, it will use them
diff --git a/src/runtime/os3_plan9.go b/src/runtime/os3_plan9.go
index fa49ad681f..248576a102 100644
--- a/src/runtime/os3_plan9.go
+++ b/src/runtime/os3_plan9.go
@@ -81,7 +81,7 @@ func sighandler(_ureg *ureg, note *byte, gp *g) int {
}
Throw:
_g_.m.throwing = 1
- setGNoWriteBarrier(&_g_.m.caughtsig, gp)
+ _g_.m.caughtsig.set(gp)
startpanic()
print(notestr, "\n")
print("PC=", hex(c.pc()), "\n")
diff --git a/src/runtime/panic.go b/src/runtime/panic.go
index 9b937f5ad7..0e4086c7ef 100644
--- a/src/runtime/panic.go
+++ b/src/runtime/panic.go
@@ -165,7 +165,7 @@ func newdefer(siz int32) *_defer {
sc := deferclass(uintptr(siz))
mp := acquirem()
if sc < uintptr(len(p{}.deferpool)) {
- pp := mp.p
+ pp := mp.p.ptr()
if len(pp.deferpool[sc]) == 0 && sched.deferpool[sc] != nil {
lock(&sched.deferlock)
for len(pp.deferpool[sc]) < cap(pp.deferpool[sc])/2 && sched.deferpool[sc] != nil {
@@ -223,7 +223,7 @@ func freedefer(d *_defer) {
sc := deferclass(uintptr(d.siz))
if sc < uintptr(len(p{}.deferpool)) {
mp := acquirem()
- pp := mp.p
+ pp := mp.p.ptr()
if len(pp.deferpool[sc]) == cap(pp.deferpool[sc]) {
// Transfer half of local cache to the central cache.
var first, last *_defer
diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 88b2004c83..f725fc890b 100644
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -208,7 +208,7 @@ func acquireSudog() *sudog {
// The acquirem/releasem increments m.locks during new(sudog),
// which keeps the garbage collector from being invoked.
mp := acquirem()
- pp := mp.p
+ pp := mp.p.ptr()
if len(pp.sudogcache) == 0 {
lock(&sched.sudoglock)
// First, try to grab a batch from central cache.
@@ -257,7 +257,7 @@ func releaseSudog(s *sudog) {
throw("runtime: releaseSudog with non-nil gp.param")
}
mp := acquirem() // avoid rescheduling to another P
- pp := mp.p
+ pp := mp.p.ptr()
if len(pp.sudogcache) == cap(pp.sudogcache) {
// Transfer half of local cache to the central cache.
var first, last *sudog
diff --git a/src/runtime/proc1.go b/src/runtime/proc1.go
index a46ebc8cec..cf6e604ed0 100644
--- a/src/runtime/proc1.go
+++ b/src/runtime/proc1.go
@@ -145,7 +145,7 @@ func ready(gp *g, traceskip int) {
// status is Gwaiting or Gscanwaiting, make Grunnable and put on runq
casgstatus(gp, _Gwaiting, _Grunnable)
- runqput(_g_.m.p, gp)
+ runqput(_g_.m.p.ptr(), gp)
if atomicload(&sched.npidle) != 0 && atomicload(&sched.nmspinning) == 0 { // TODO: fast atomic
wakep()
}
@@ -185,7 +185,7 @@ func readyExecute(gp *g, traceskip int) {
// Preempt the current g
casgstatus(_g_, _Grunning, _Grunnable)
- runqput(_g_.m.p, _g_)
+ runqput(_g_.m.p.ptr(), _g_)
dropg()
// Ready gp and switch to it
@@ -239,7 +239,7 @@ func helpgc(nproc int32) {
throw("gcprocs inconsistency")
}
mp.helpgc = n
- mp.p = allp[pos]
+ mp.p.set(allp[pos])
mp.mcache = allp[pos].mcache
pos++
notewakeup(&mp.park)
@@ -603,7 +603,7 @@ func stoptheworld() {
atomicstore(&sched.gcwaiting, 1)
preemptall()
// stop current P
- _g_.m.p.status = _Pgcstop // Pgcstop is only diagnostic.
+ _g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic.
sched.stopwait--
// try to retake all P's in Psyscall status
for i := 0; i < int(gomaxprocs); i++ {
@@ -681,14 +681,14 @@ func starttheworld() {
for p1 != nil {
p := p1
- p1 = p1.link
- if p.m != nil {
- mp := p.m
- p.m = nil
- if mp.nextp != nil {
+ p1 = p1.link.ptr()
+ if p.m != 0 {
+ mp := p.m.ptr()
+ p.m = 0
+ if mp.nextp != 0 {
throw("starttheworld: inconsistent mp->nextp")
}
- mp.nextp = p
+ mp.nextp.set(p)
notewakeup(&mp.park)
} else {
// Start M to run P. Do not start another M below.
@@ -768,8 +768,7 @@ func mstart1() {
initsig()
}
- if _g_.m.mstartfn != 0 {
- fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn))
+ if fn := _g_.m.mstartfn; fn != nil {
fn()
}
@@ -777,8 +776,8 @@ func mstart1() {
_g_.m.helpgc = 0
stopm()
} else if _g_.m != &m0 {
- acquirep(_g_.m.nextp)
- _g_.m.nextp = nil
+ acquirep(_g_.m.nextp.ptr())
+ _g_.m.nextp = 0
}
schedule()
}
@@ -789,20 +788,22 @@ func mstart1() {
var cgoThreadStart unsafe.Pointer
type cgothreadstart struct {
- g *g
+ g guintptr
tls *uint64
fn unsafe.Pointer
}
// Allocate a new m unassociated with any thread.
// Can use p for allocation context if needed.
-func allocm(_p_ *p) *m {
+// fn is recorded as the new m's m.mstartfn.
+func allocm(_p_ *p, fn func()) *m {
_g_ := getg()
_g_.m.locks++ // disable GC because it can be called from sysmon
- if _g_.m.p == nil {
+ if _g_.m.p == 0 {
acquirep(_p_) // temporarily borrow p for mallocs in this function
}
mp := new(m)
+ mp.mstartfn = fn
mcommoninit(mp)
// In case of cgo or Solaris, pthread_create will make us a stack.
@@ -814,7 +815,7 @@ func allocm(_p_ *p) *m {
}
mp.g0.m = mp
- if _p_ == _g_.m.p {
+ if _p_ == _g_.m.p.ptr() {
releasep()
}
_g_.m.locks--
@@ -880,8 +881,8 @@ func needm(x byte) {
// after exitsyscall makes sure it is okay to be
// running at all (that is, there's no garbage collection
// running right now).
- mp.needextram = mp.schedlink == nil
- unlockextra(mp.schedlink)
+ mp.needextram = mp.schedlink == 0
+ unlockextra(mp.schedlink.ptr())
// Install g (= m->g0) and set the stack bounds
// to match the current stack. We don't actually know
@@ -910,7 +911,7 @@ func newextram() {
// The sched.pc will never be returned to, but setting it to
// goexit makes clear to the traceback routines where
// the goroutine stack ends.
- mp := allocm(nil)
+ mp := allocm(nil, nil)
gp := malg(4096)
gp.sched.pc = funcPC(goexit) + _PCQuantum
gp.sched.sp = gp.stack.hi
@@ -936,7 +937,7 @@ func newextram() {
// Add m to the extra list.
mnext := lockextra(true)
- mp.schedlink = mnext
+ mp.schedlink.set(mnext)
unlockextra(mp)
}
@@ -972,7 +973,7 @@ func dropm() {
// with no pointer manipulation.
mp := getg().m
mnext := lockextra(true)
- mp.schedlink = mnext
+ mp.schedlink.set(mnext)
setg(nil)
unlockextra(mp)
@@ -1019,18 +1020,14 @@ func unlockextra(mp *m) {
// May run with m.p==nil, so write barriers are not allowed.
//go:nowritebarrier
func newm(fn func(), _p_ *p) {
- mp := allocm(_p_)
- // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated
- setPNoWriteBarrier(&mp.nextp, _p_)
- // Store &fn as a uintptr since it is not heap allocated so the WB can be eliminated
- mp.mstartfn = *(*uintptr)(unsafe.Pointer(&fn))
+ mp := allocm(_p_, fn)
+ mp.nextp.set(_p_)
if iscgo {
var ts cgothreadstart
if _cgo_thread_start == nil {
throw("_cgo_thread_start missing")
}
- // mp is reachable via allm and mp.g0 never changes, so WB can be eliminated.
- setGNoWriteBarrier(&ts.g, mp.g0)
+ ts.g.set(mp.g0)
ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0]))
ts.fn = unsafe.Pointer(funcPC(mstart))
asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts))
@@ -1047,7 +1044,7 @@ func stopm() {
if _g_.m.locks != 0 {
throw("stopm holding locks")
}
- if _g_.m.p != nil {
+ if _g_.m.p != 0 {
throw("stopm holding p")
}
if _g_.m.spinning {
@@ -1065,11 +1062,11 @@ retry:
gchelper()
_g_.m.helpgc = 0
_g_.m.mcache = nil
- _g_.m.p = nil
+ _g_.m.p = 0
goto retry
}
- acquirep(_g_.m.nextp)
- _g_.m.nextp = nil
+ acquirep(_g_.m.nextp.ptr())
+ _g_.m.nextp = 0
}
func mspinning() {
@@ -1105,12 +1102,11 @@ func startm(_p_ *p, spinning bool) {
if mp.spinning {
throw("startm: m is spinning")
}
- if mp.nextp != nil {
+ if mp.nextp != 0 {
throw("startm: m has p")
}
mp.spinning = spinning
- // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated
- setPNoWriteBarrier(&mp.nextp, _p_)
+ mp.nextp.set(_p_)
notewakeup(&mp.park)
}
@@ -1173,7 +1169,7 @@ func stoplockedm() {
if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
throw("stoplockedm: inconsistent locking")
}
- if _g_.m.p != nil {
+ if _g_.m.p != 0 {
// Schedule another M to run this p.
_p_ := releasep()
handoffp(_p_)
@@ -1188,8 +1184,8 @@ func stoplockedm() {
dumpgstatus(_g_)
throw("stoplockedm: not runnable")
}
- acquirep(_g_.m.nextp)
- _g_.m.nextp = nil
+ acquirep(_g_.m.nextp.ptr())
+ _g_.m.nextp = 0
}
// Schedules the locked m to run the locked gp.
@@ -1202,14 +1198,13 @@ func startlockedm(gp *g) {
if mp == _g_.m {
throw("startlockedm: locked to me")
}
- if mp.nextp != nil {
+ if mp.nextp != 0 {
throw("startlockedm: m has p")
}
// directly handoff current P to the locked m
incidlelocked(-1)
_p_ := releasep()
- // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated
- setPNoWriteBarrier(&mp.nextp, _p_)
+ mp.nextp.set(_p_)
notewakeup(&mp.park)
stopm()
}
@@ -1246,7 +1241,7 @@ func execute(gp *g) {
gp.waitsince = 0
gp.preempt = false
gp.stackguard0 = gp.stack.lo + _StackGuard
- _g_.m.p.schedtick++
+ _g_.m.p.ptr().schedtick++
_g_.m.curg = gp
gp.m = _g_.m
@@ -1280,14 +1275,14 @@ top:
}
// local runq
- if gp := runqget(_g_.m.p); gp != nil {
+ if gp := runqget(_g_.m.p.ptr()); gp != nil {
return gp
}
// global runq
if sched.runqsize != 0 {
lock(&sched.lock)
- gp := globrunqget(_g_.m.p, 0)
+ gp := globrunqget(_g_.m.p.ptr(), 0)
unlock(&sched.lock)
if gp != nil {
return gp
@@ -1303,7 +1298,7 @@ top:
if netpollinited() && sched.lastpoll != 0 {
if gp := netpoll(false); gp != nil { // non-blocking
// netpoll returns list of goroutines linked by schedlink.
- injectglist(gp.schedlink)
+ injectglist(gp.schedlink.ptr())
casgstatus(gp, _Gwaiting, _Grunnable)
if trace.enabled {
traceGoUnpark(gp, 0)
@@ -1329,10 +1324,10 @@ top:
}
_p_ := allp[fastrand1()%uint32(gomaxprocs)]
var gp *g
- if _p_ == _g_.m.p {
+ if _p_ == _g_.m.p.ptr() {
gp = runqget(_p_)
} else {
- gp = runqsteal(_g_.m.p, _p_)
+ gp = runqsteal(_g_.m.p.ptr(), _p_)
}
if gp != nil {
return gp
@@ -1347,7 +1342,7 @@ stop:
goto top
}
if sched.runqsize != 0 {
- gp := globrunqget(_g_.m.p, 0)
+ gp := globrunqget(_g_.m.p.ptr(), 0)
unlock(&sched.lock)
return gp
}
@@ -1376,7 +1371,7 @@ stop:
// poll network
if netpollinited() && xchg64(&sched.lastpoll, 0) != 0 {
- if _g_.m.p != nil {
+ if _g_.m.p != 0 {
throw("findrunnable: netpoll with p")
}
if _g_.m.spinning {
@@ -1390,7 +1385,7 @@ stop:
unlock(&sched.lock)
if _p_ != nil {
acquirep(_p_)
- injectglist(gp.schedlink)
+ injectglist(gp.schedlink.ptr())
casgstatus(gp, _Gwaiting, _Grunnable)
if trace.enabled {
traceGoUnpark(gp, 0)
@@ -1432,7 +1427,7 @@ func injectglist(glist *g) {
return
}
if trace.enabled {
- for gp := glist; gp != nil; gp = gp.schedlink {
+ for gp := glist; gp != nil; gp = gp.schedlink.ptr() {
traceGoUnpark(gp, 0)
}
}
@@ -1440,7 +1435,7 @@ func injectglist(glist *g) {
var n int
for n = 0; glist != nil; n++ {
gp := glist
- glist = gp.schedlink
+ glist = gp.schedlink.ptr()
casgstatus(gp, _Gwaiting, _Grunnable)
globrunqput(gp)
}
@@ -1483,9 +1478,9 @@ top:
// Check the global runnable queue once in a while to ensure fairness.
// Otherwise two goroutines can completely occupy the local runqueue
// by constantly respawning each other.
- if _g_.m.p.schedtick%61 == 0 && sched.runqsize > 0 {
+ if _g_.m.p.ptr().schedtick%61 == 0 && sched.runqsize > 0 {
lock(&sched.lock)
- gp = globrunqget(_g_.m.p, 1)
+ gp = globrunqget(_g_.m.p.ptr(), 1)
unlock(&sched.lock)
if gp != nil {
resetspinning()
@@ -1493,7 +1488,7 @@ top:
}
}
if gp == nil {
- gp = runqget(_g_.m.p)
+ gp = runqget(_g_.m.p.ptr())
if gp != nil && _g_.m.spinning {
throw("schedule: spinning with local work")
}
@@ -1624,7 +1619,7 @@ func goexit0(gp *g) {
throw("internal lockOSThread error")
}
_g_.m.locked = 0
- gfput(_g_.m.p, gp)
+ gfput(_g_.m.p.ptr(), gp)
schedule()
}
@@ -1671,7 +1666,7 @@ func save(pc, sp uintptr) {
// when syscall returns we emit traceGoSysExit and when the goroutine starts running
// (potentially instantly, if exitsyscallfast returns true) we emit traceGoStart.
// To ensure that traceGoSysExit is emitted strictly after traceGoSysBlock,
-// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.syscalltick),
+// we remember current value of syscalltick in m (_g_.m.syscalltick = _g_.m.p.ptr().syscalltick),
// whoever emits traceGoSysBlock increments p.syscalltick afterwards;
// and we wait for the increment before emitting traceGoSysExit.
// Note that the increment is done even if tracing is not enabled,
@@ -1713,10 +1708,10 @@ func reentersyscall(pc, sp uintptr) {
save(pc, sp)
}
- _g_.m.syscalltick = _g_.m.p.syscalltick
+ _g_.m.syscalltick = _g_.m.p.ptr().syscalltick
_g_.m.mcache = nil
- _g_.m.p.m = nil
- atomicstore(&_g_.m.p.status, _Psyscall)
+ _g_.m.p.ptr().m = 0
+ atomicstore(&_g_.m.p.ptr().status, _Psyscall)
if sched.gcwaiting != 0 {
systemstack(entersyscall_gcwait)
save(pc, sp)
@@ -1746,7 +1741,7 @@ func entersyscall_sysmon() {
func entersyscall_gcwait() {
_g_ := getg()
- _p_ := _g_.m.p
+ _p_ := _g_.m.p.ptr()
lock(&sched.lock)
if sched.stopwait > 0 && cas(&_p_.status, _Psyscall, _Pgcstop) {
@@ -1770,8 +1765,8 @@ func entersyscallblock(dummy int32) {
_g_.m.locks++ // see comment in entersyscall
_g_.throwsplit = true
_g_.stackguard0 = stackPreempt // see comment in entersyscall
- _g_.m.syscalltick = _g_.m.p.syscalltick
- _g_.m.p.syscalltick++
+ _g_.m.syscalltick = _g_.m.p.ptr().syscalltick
+ _g_.m.p.ptr().syscalltick++
// Leave SP around for GC and traceback.
pc := getcallerpc(unsafe.Pointer(&dummy))
@@ -1807,7 +1802,7 @@ func entersyscallblock(dummy int32) {
func entersyscallblock_handoff() {
if trace.enabled {
traceGoSysCall()
- traceGoSysBlock(getg().m.p)
+ traceGoSysBlock(getg().m.p.ptr())
}
handoffp(releasep())
}
@@ -1826,18 +1821,18 @@ func exitsyscall(dummy int32) {
}
_g_.waitsince = 0
- oldp := _g_.m.p
+ oldp := _g_.m.p.ptr()
if exitsyscallfast() {
if _g_.m.mcache == nil {
throw("lost mcache")
}
if trace.enabled {
- if oldp != _g_.m.p || _g_.m.syscalltick != _g_.m.p.syscalltick {
+ if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
systemstack(traceGoStart)
}
}
// There's a cpu for us, so we can run.
- _g_.m.p.syscalltick++
+ _g_.m.p.ptr().syscalltick++
// We need to cas the status and scan before resuming...
casgstatus(_g_, _Gsyscall, _Grunning)
@@ -1891,7 +1886,7 @@ func exitsyscall(dummy int32) {
// we don't know for sure that the garbage collector
// is not running.
_g_.syscallsp = 0
- _g_.m.p.syscalltick++
+ _g_.m.p.ptr().syscalltick++
_g_.throwsplit = false
if exitTicks != 0 {
@@ -1909,37 +1904,37 @@ func exitsyscallfast() bool {
// Freezetheworld sets stopwait but does not retake P's.
if sched.stopwait == freezeStopWait {
_g_.m.mcache = nil
- _g_.m.p = nil
+ _g_.m.p = 0
return false
}
// Try to re-acquire the last P.
- if _g_.m.p != nil && _g_.m.p.status == _Psyscall && cas(&_g_.m.p.status, _Psyscall, _Prunning) {
+ if _g_.m.p != 0 && _g_.m.p.ptr().status == _Psyscall && cas(&_g_.m.p.ptr().status, _Psyscall, _Prunning) {
// There's a cpu for us, so we can run.
- _g_.m.mcache = _g_.m.p.mcache
- _g_.m.p.m = _g_.m
- if _g_.m.syscalltick != _g_.m.p.syscalltick {
+ _g_.m.mcache = _g_.m.p.ptr().mcache
+ _g_.m.p.ptr().m.set(_g_.m)
+ if _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
if trace.enabled {
// The p was retaken and then enter into syscall again (since _g_.m.syscalltick has changed).
// traceGoSysBlock for this syscall was already emitted,
// but here we effectively retake the p from the new syscall running on the same p.
systemstack(func() {
// Denote blocking of the new syscall.
- traceGoSysBlock(_g_.m.p)
+ traceGoSysBlock(_g_.m.p.ptr())
// Denote completion of the current syscall.
traceGoSysExit(0)
})
}
- _g_.m.p.syscalltick++
+ _g_.m.p.ptr().syscalltick++
}
return true
}
// Try to get any other idle P.
- oldp := _g_.m.p
+ oldp := _g_.m.p.ptr()
_g_.m.mcache = nil
- _g_.m.p = nil
- if sched.pidle != nil {
+ _g_.m.p = 0
+ if sched.pidle != 0 {
var ok bool
systemstack(func() {
ok = exitsyscallfast_pidle()
@@ -2101,7 +2096,7 @@ func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr
throw("newproc: function arguments too large for new goroutine")
}
- _p_ := _g_.m.p
+ _p_ := _g_.m.p.ptr()
newg := gfget(_p_)
if newg == nil {
newg = malg(_StackMin)
@@ -2184,7 +2179,7 @@ func gfput(_p_ *p, gp *g) {
gp.stackguard0 = 0
}
- gp.schedlink = _p_.gfree
+ gp.schedlink.set(_p_.gfree)
_p_.gfree = gp
_p_.gfreecnt++
if _p_.gfreecnt >= 64 {
@@ -2192,8 +2187,8 @@ func gfput(_p_ *p, gp *g) {
for _p_.gfreecnt >= 32 {
_p_.gfreecnt--
gp = _p_.gfree
- _p_.gfree = gp.schedlink
- gp.schedlink = sched.gfree
+ _p_.gfree = gp.schedlink.ptr()
+ gp.schedlink.set(sched.gfree)
sched.gfree = gp
sched.ngfree++
}
@@ -2211,16 +2206,16 @@ retry:
for _p_.gfreecnt < 32 && sched.gfree != nil {
_p_.gfreecnt++
gp = sched.gfree
- sched.gfree = gp.schedlink
+ sched.gfree = gp.schedlink.ptr()
sched.ngfree--
- gp.schedlink = _p_.gfree
+ gp.schedlink.set(_p_.gfree)
_p_.gfree = gp
}
unlock(&sched.gflock)
goto retry
}
if gp != nil {
- _p_.gfree = gp.schedlink
+ _p_.gfree = gp.schedlink.ptr()
_p_.gfreecnt--
if gp.stack.lo == 0 {
// Stack was deallocated in gfput. Allocate a new one.
@@ -2243,8 +2238,8 @@ func gfpurge(_p_ *p) {
for _p_.gfreecnt != 0 {
_p_.gfreecnt--
gp := _p_.gfree
- _p_.gfree = gp.schedlink
- gp.schedlink = sched.gfree
+ _p_.gfree = gp.schedlink.ptr()
+ gp.schedlink.set(sched.gfree)
sched.gfree = gp
sched.ngfree++
}
@@ -2453,10 +2448,10 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
// This is especially important on windows, since all syscalls are cgo calls.
n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[0], len(stk), nil, nil, 0)
}
- if GOOS == "windows" && n == 0 && mp.libcallg != nil && mp.libcallpc != 0 && mp.libcallsp != 0 {
+ if GOOS == "windows" && n == 0 && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 {
// Libcall, i.e. runtime syscall on windows.
// Collect Go stack that leads to the call.
- n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg, 0, &stk[0], len(stk), nil, nil, 0)
+ n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0)
}
if n == 0 {
// If all of the above has failed, account it against abstract "System" or "GC".
@@ -2570,7 +2565,7 @@ func procresize(nprocs int32) *p {
for i := nprocs; i < old; i++ {
p := allp[i]
if trace.enabled {
- if p == getg().m.p {
+ if p == getg().m.p.ptr() {
// moving to p[0], pretend that we were descheduled
// and then scheduled again to keep the trace sane.
traceGoSched()
@@ -2584,9 +2579,9 @@ func procresize(nprocs int32) *p {
gp := p.runq[p.runqtail%uint32(len(p.runq))]
// push onto head of global queue
gp.schedlink = sched.runqhead
- sched.runqhead = gp
- if sched.runqtail == nil {
- sched.runqtail = gp
+ sched.runqhead.set(gp)
+ if sched.runqtail == 0 {
+ sched.runqtail.set(gp)
}
sched.runqsize++
}
@@ -2609,18 +2604,18 @@ func procresize(nprocs int32) *p {
}
_g_ := getg()
- if _g_.m.p != nil && _g_.m.p.id < nprocs {
+ if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs {
// continue to use the current P
- _g_.m.p.status = _Prunning
+ _g_.m.p.ptr().status = _Prunning
} else {
// release the current P and acquire allp[0]
- if _g_.m.p != nil {
- _g_.m.p.m = nil
+ if _g_.m.p != 0 {
+ _g_.m.p.ptr().m = 0
}
- _g_.m.p = nil
+ _g_.m.p = 0
_g_.m.mcache = nil
p := allp[0]
- p.m = nil
+ p.m = 0
p.status = _Pidle
acquirep(p)
if trace.enabled {
@@ -2630,15 +2625,15 @@ func procresize(nprocs int32) *p {
var runnablePs *p
for i := nprocs - 1; i >= 0; i-- {
p := allp[i]
- if _g_.m.p == p {
+ if _g_.m.p.ptr() == p {
continue
}
p.status = _Pidle
if p.runqhead == p.runqtail {
pidleput(p)
} else {
- p.m = mget()
- p.link = runnablePs
+ p.m.set(mget())
+ p.link.set(runnablePs)
runnablePs = p
}
}
@@ -2648,53 +2643,57 @@ func procresize(nprocs int32) *p {
}
// Associate p and the current m.
+func acquirep(_p_ *p) {
+ acquirep1(_p_)
+
+ // have p; write barriers now allowed
+ _g_ := getg()
+ _g_.m.mcache = _p_.mcache
+
+ if trace.enabled {
+ traceProcStart()
+ }
+}
+
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
-func acquirep(_p_ *p) {
+func acquirep1(_p_ *p) {
_g_ := getg()
- if _g_.m.p != nil || _g_.m.mcache != nil {
+ if _g_.m.p != 0 || _g_.m.mcache != nil {
throw("acquirep: already in go")
}
- if _p_.m != nil || _p_.status != _Pidle {
+ if _p_.m != 0 || _p_.status != _Pidle {
id := int32(0)
- if _p_.m != nil {
- id = _p_.m.id
+ if _p_.m != 0 {
+ id = _p_.m.ptr().id
}
print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n")
throw("acquirep: invalid p state")
}
- // _p_.mcache holds the mcache and _p_ is in allp, so WB can be eliminated
- setMcacheNoWriteBarrier(&_g_.m.mcache, _p_.mcache)
- // _p_ is in allp so WB can be eliminated
- setPNoWriteBarrier(&_g_.m.p, _p_)
- // m is in _g_.m and is reachable through allg, so WB can be eliminated
- setMNoWriteBarrier(&_p_.m, _g_.m)
+ _g_.m.p.set(_p_)
+ _p_.m.set(_g_.m)
_p_.status = _Prunning
-
- if trace.enabled {
- traceProcStart()
- }
}
// Disassociate p and the current m.
func releasep() *p {
_g_ := getg()
- if _g_.m.p == nil || _g_.m.mcache == nil {
+ if _g_.m.p == 0 || _g_.m.mcache == nil {
throw("releasep: invalid arg")
}
- _p_ := _g_.m.p
- if _p_.m != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
- print("releasep: m=", _g_.m, " m->p=", _g_.m.p, " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
+ _p_ := _g_.m.p.ptr()
+ if _p_.m.ptr() != _g_.m || _p_.mcache != _g_.m.mcache || _p_.status != _Prunning {
+ print("releasep: m=", _g_.m, " m->p=", _g_.m.p.ptr(), " p->m=", _p_.m, " m->mcache=", _g_.m.mcache, " p->mcache=", _p_.mcache, " p->status=", _p_.status, "\n")
throw("releasep: invalid p state")
}
if trace.enabled {
- traceProcStop(_g_.m.p)
+ traceProcStop(_g_.m.p.ptr())
}
- _g_.m.p = nil
+ _g_.m.p = 0
_g_.m.mcache = nil
- _p_.m = nil
+ _p_.m = 0
_p_.status = _Pidle
return _p_
}
@@ -2773,7 +2772,7 @@ func checkdead() {
if mp == nil {
newm(nil, _p_)
} else {
- mp.nextp = _p_
+ mp.nextp.set(_p_)
notewakeup(&mp.park)
}
return
@@ -2865,7 +2864,7 @@ func sysmon() {
if lastgc != 0 && unixnow-lastgc > forcegcperiod && atomicload(&forcegc.idle) != 0 {
lock(&forcegc.lock)
forcegc.idle = 0
- forcegc.g.schedlink = nil
+ forcegc.g.schedlink = 0
injectglist(forcegc.g)
unlock(&forcegc.lock)
}
@@ -2978,7 +2977,7 @@ func preemptall() bool {
// and will be indicated by the gp->status no longer being
// Grunning
func preemptone(_p_ *p) bool {
- mp := _p_.m
+ mp := _p_.m.ptr()
if mp == nil || mp == getg().m {
return false
}
@@ -3018,7 +3017,7 @@ func schedtrace(detailed bool) {
if _p_ == nil {
continue
}
- mp := _p_.m
+ mp := _p_.m.ptr()
h := atomicload(&_p_.runqhead)
t := atomicload(&_p_.runqtail)
if detailed {
@@ -3047,7 +3046,7 @@ func schedtrace(detailed bool) {
}
for mp := allm; mp != nil; mp = mp.alllink {
- _p_ := mp.p
+ _p_ := mp.p.ptr()
gp := mp.curg
lockedg := mp.lockedg
id1 := int32(-1)
@@ -3089,10 +3088,8 @@ func schedtrace(detailed bool) {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func mput(mp *m) {
- // sched.midle is reachable via allm, so WB can be eliminated.
- setMNoWriteBarrier(&mp.schedlink, sched.midle)
- // mp is reachable via allm, so WB can be eliminated.
- setMNoWriteBarrier(&sched.midle, mp)
+ mp.schedlink = sched.midle
+ sched.midle.set(mp)
sched.nmidle++
checkdead()
}
@@ -3102,10 +3099,9 @@ func mput(mp *m) {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func mget() *m {
- mp := sched.midle
+ mp := sched.midle.ptr()
if mp != nil {
- // mp.schedlink is reachable via mp, which is on allm, so WB can be eliminated.
- setMNoWriteBarrier(&sched.midle, mp.schedlink)
+ sched.midle = mp.schedlink
sched.nmidle--
}
return mp
@@ -3116,27 +3112,26 @@ func mget() *m {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func globrunqput(gp *g) {
- gp.schedlink = nil
- if sched.runqtail != nil {
- // gp is on allg, so these three WBs can be eliminated.
- setGNoWriteBarrier(&sched.runqtail.schedlink, gp)
+ gp.schedlink = 0
+ if sched.runqtail != 0 {
+ sched.runqtail.ptr().schedlink.set(gp)
} else {
- setGNoWriteBarrier(&sched.runqhead, gp)
+ sched.runqhead.set(gp)
}
- setGNoWriteBarrier(&sched.runqtail, gp)
+ sched.runqtail.set(gp)
sched.runqsize++
}
// Put a batch of runnable goroutines on the global runnable queue.
// Sched must be locked.
func globrunqputbatch(ghead *g, gtail *g, n int32) {
- gtail.schedlink = nil
- if sched.runqtail != nil {
- sched.runqtail.schedlink = ghead
+ gtail.schedlink = 0
+ if sched.runqtail != 0 {
+ sched.runqtail.ptr().schedlink.set(ghead)
} else {
- sched.runqhead = ghead
+ sched.runqhead.set(ghead)
}
- sched.runqtail = gtail
+ sched.runqtail.set(gtail)
sched.runqsize += n
}
@@ -3160,14 +3155,14 @@ func globrunqget(_p_ *p, max int32) *g {
sched.runqsize -= n
if sched.runqsize == 0 {
- sched.runqtail = nil
+ sched.runqtail = 0
}
- gp := sched.runqhead
+ gp := sched.runqhead.ptr()
sched.runqhead = gp.schedlink
n--
for ; n > 0; n-- {
- gp1 := sched.runqhead
+ gp1 := sched.runqhead.ptr()
sched.runqhead = gp1.schedlink
runqput(_p_, gp1)
}
@@ -3179,9 +3174,8 @@ func globrunqget(_p_ *p, max int32) *g {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func pidleput(_p_ *p) {
- // sched.pidle, _p_.link and _p_ are reachable via allp, so WB can be eliminated.
- setPNoWriteBarrier(&_p_.link, sched.pidle)
- setPNoWriteBarrier(&sched.pidle, _p_)
+ _p_.link = sched.pidle
+ sched.pidle.set(_p_)
xadd(&sched.npidle, 1) // TODO: fast atomic
}
@@ -3190,10 +3184,9 @@ func pidleput(_p_ *p) {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrier
func pidleget() *p {
- _p_ := sched.pidle
+ _p_ := sched.pidle.ptr()
if _p_ != nil {
- // _p_.link is reachable via a _p_ in allp, so WB can be eliminated.
- setPNoWriteBarrier(&sched.pidle, _p_.link)
+ sched.pidle = _p_.link
xadd(&sched.npidle, -1) // TODO: fast atomic
}
return _p_
@@ -3239,7 +3232,7 @@ func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
// Link the goroutines.
for i := uint32(0); i < n; i++ {
- batch[i].schedlink = batch[i+1]
+ batch[i].schedlink.set(batch[i+1])
}
// Now put the batch on global queue.
@@ -3413,7 +3406,7 @@ func procPin() int {
mp := _g_.m
mp.locks++
- return int(mp.p.id)
+ return int(mp.p.ptr().id)
}
//go:nosplit
@@ -3458,7 +3451,7 @@ func sync_runtime_canSpin(i int) bool {
if i >= active_spin || ncpu <= 1 || gomaxprocs <= int32(sched.npidle+sched.nmspinning)+1 {
return false
}
- if p := getg().m.p; p.runqhead != p.runqtail {
+ if p := getg().m.p.ptr(); p.runqhead != p.runqtail {
return false
}
return true
diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index 3230d4e1a8..a59d77df85 100644
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -87,8 +87,28 @@ type eface struct {
data unsafe.Pointer
}
+// The guintptr, muintptr, and puintptr are all used to bypass write barriers.
+// It is particularly important to avoid write barriers when the current P has
+// been released, because the GC thinks the world is stopped, and an
+// unexpected write barrier would not be synchronized with the GC,
+// which can lead to a half-executed write barrier that has marked the object
+// but not queued it. If the GC skips the object and completes before the
+// queuing can occur, it will incorrectly free the object.
+//
+// We tried using special assignment functions invoked only when not
+// holding a running P, but then some updates to a particular memory
+// word went through write barriers and some did not. This breaks the
+// write barrier shadow checking mode, and it is also scary: better to have
+// a word that is completely ignored by the GC than to have one for which
+// only a few updates are ignored.
+//
+// Gs, Ms, and Ps are always reachable via true pointers in the
+// allgs, allm, and allp lists or (during allocation before they reach those lists)
+// from stack variables.
+
// A guintptr holds a goroutine pointer, but typed as a uintptr
-// to bypass write barriers. It is used in the Gobuf goroutine state.
+// to bypass write barriers. It is used in the Gobuf goroutine state
+// and in scheduling lists that are manipulated without a P.
//
// The Gobuf.g goroutine pointer is almost always updated by assembly code.
// In one of the few places it is updated by Go code - func save - it must be
@@ -107,41 +127,18 @@ type eface struct {
// alternate arena. Using guintptr doesn't make that problem any worse.
type guintptr uintptr
-func (gp guintptr) ptr() *g {
- return (*g)(unsafe.Pointer(gp))
-}
+func (gp guintptr) ptr() *g { return (*g)(unsafe.Pointer(gp)) }
+func (gp *guintptr) set(g *g) { *gp = guintptr(unsafe.Pointer(g)) }
-// ps, ms, gs, and mcache are structures that must be manipulated at a level
-// lower than that of the normal Go language. For example the routine that
-// stops the world removes the p from the m structure informing the GC that
-// this P is stopped and then it moves the g to the global runnable queue.
-// If write barriers were allowed to happen at this point not only does
-// the GC think the thread is stopped but the underlying structures
-// like a p or m are not in a state that is not coherent enough to
-// support the write barrier actions.
-// This is particularly painful since a partially executed write barrier
-// may mark the object but be delinquent in informing the GC that the
-// object needs to be scanned.
+type puintptr uintptr
-// setGNoWriteBarriers does *gdst = gval without a write barrier.
-func setGNoWriteBarrier(gdst **g, gval *g) {
- *(*uintptr)(unsafe.Pointer(gdst)) = uintptr(unsafe.Pointer(gval))
-}
+func (pp puintptr) ptr() *p { return (*p)(unsafe.Pointer(pp)) }
+func (pp *puintptr) set(p *p) { *pp = puintptr(unsafe.Pointer(p)) }
-// setMNoWriteBarriers does *mdst = mval without a write barrier.
-func setMNoWriteBarrier(mdst **m, mval *m) {
- *(*uintptr)(unsafe.Pointer(mdst)) = uintptr(unsafe.Pointer(mval))
-}
-
-// setPNoWriteBarriers does *pdst = pval without a write barrier.
-func setPNoWriteBarrier(pdst **p, pval *p) {
- *(*uintptr)(unsafe.Pointer(pdst)) = uintptr(unsafe.Pointer(pval))
-}
+type muintptr uintptr
-// setMcacheNoWriteBarriers does *mcachedst = mcacheval without a write barrier.
-func setMcacheNoWriteBarrier(mcachedst **mcache, mcacheval *mcache) {
- *(*uintptr)(unsafe.Pointer(mcachedst)) = uintptr(unsafe.Pointer(mcacheval))
-}
+func (mp muintptr) ptr() *m { return (*m)(unsafe.Pointer(mp)) }
+func (mp *muintptr) set(m *m) { *mp = muintptr(unsafe.Pointer(m)) }
type gobuf struct {
// The offsets of sp, pc, and g are known to (hard-coded in) libmach.
@@ -224,7 +221,7 @@ type g struct {
goid int64
waitsince int64 // approx time when the g become blocked
waitreason string // if status==gwaiting
- schedlink *g
+ schedlink guintptr
preempt bool // preemption signal, duplicates stackguard0 = stackpreempt
paniconfault bool // panic (instead of crash) on unexpected fault address
preemptscan bool // preempted g does scan for gc
@@ -263,11 +260,11 @@ type m struct {
procid uint64 // for debuggers, but offset not hard-coded
gsignal *g // signal-handling g
tls [4]uintptr // thread-local storage (for x86 extern register)
- mstartfn uintptr // TODO: type as func(); note: this is a non-heap allocated func()
- curg *g // current running goroutine
- caughtsig *g // goroutine running during fatal signal
- p *p // attached p for executing go code (nil if not executing go code)
- nextp *p
+ mstartfn func()
+ curg *g // current running goroutine
+ caughtsig guintptr // goroutine running during fatal signal
+ p puintptr // attached p for executing go code (nil if not executing go code)
+ nextp puintptr
id int32
mallocing int32
throwing int32
@@ -286,7 +283,7 @@ type m struct {
ncgo int32 // number of cgo calls currently in progress
park note
alllink *m // on allm
- schedlink *m
+ schedlink muintptr
machport uint32 // return address for mach ipc (os x)
mcache *mcache
lockedg *g
@@ -315,7 +312,7 @@ type m struct {
libcall libcall
libcallpc uintptr // for cpu profiler
libcallsp uintptr
- libcallg *g
+ libcallg guintptr
//#endif
//#ifdef GOOS_solaris
perrno *int32 // pointer to tls errno
@@ -336,10 +333,10 @@ type p struct {
id int32
status uint32 // one of pidle/prunning/...
- link *p
- schedtick uint32 // incremented on every scheduler call
- syscalltick uint32 // incremented on every system call
- m *m // back-link to associated m (nil if idle)
+ link puintptr
+ schedtick uint32 // incremented on every scheduler call
+ syscalltick uint32 // incremented on every system call
+ m muintptr // back-link to associated m (nil if idle)
mcache *mcache
deferpool [5][]*_defer // pool of available defer structs of different sizes (see panic.go)
@@ -379,19 +376,19 @@ type schedt struct {
goidgen uint64
- midle *m // idle m's waiting for work
- nmidle int32 // number of idle m's waiting for work
- nmidlelocked int32 // number of locked m's waiting for work
- mcount int32 // number of m's that have been created
- maxmcount int32 // maximum number of m's allowed (or die)
+ midle muintptr // idle m's waiting for work
+ nmidle int32 // number of idle m's waiting for work
+ nmidlelocked int32 // number of locked m's waiting for work
+ mcount int32 // number of m's that have been created
+ maxmcount int32 // maximum number of m's allowed (or die)
- pidle *p // idle p's
+ pidle puintptr // idle p's
npidle uint32
nmspinning uint32
// Global runnable queue.
- runqhead *g
- runqtail *g
+ runqhead guintptr
+ runqtail guintptr
runqsize int32
// Global cache of dead G's.
diff --git a/src/runtime/signal_386.go b/src/runtime/signal_386.go
index b63299511f..f3c36cb07a 100644
--- a/src/runtime/signal_386.go
+++ b/src/runtime/signal_386.go
@@ -100,7 +100,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
}
_g_.m.throwing = 1
- setGNoWriteBarrier(&_g_.m.caughtsig, gp)
+ _g_.m.caughtsig.set(gp)
startpanic()
if sig < uint32(len(sigtable)) {
diff --git a/src/runtime/signal_amd64x.go b/src/runtime/signal_amd64x.go
index 5dc9d8070a..182b16e5ec 100644
--- a/src/runtime/signal_amd64x.go
+++ b/src/runtime/signal_amd64x.go
@@ -136,7 +136,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
}
_g_.m.throwing = 1
- setGNoWriteBarrier(&_g_.m.caughtsig, gp)
+ _g_.m.caughtsig.set(gp)
if crashing == 0 {
startpanic()
diff --git a/src/runtime/signal_arm.go b/src/runtime/signal_arm.go
index 7d417fa2b3..38d7181b2a 100644
--- a/src/runtime/signal_arm.go
+++ b/src/runtime/signal_arm.go
@@ -95,7 +95,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
}
_g_.m.throwing = 1
- setGNoWriteBarrier(&_g_.m.caughtsig, gp)
+ _g_.m.caughtsig.set(gp)
startpanic()
if sig < uint32(len(sigtable)) {
diff --git a/src/runtime/signal_arm64.go b/src/runtime/signal_arm64.go
index d1904de215..dde3c7c43f 100644
--- a/src/runtime/signal_arm64.go
+++ b/src/runtime/signal_arm64.go
@@ -108,7 +108,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
}
_g_.m.throwing = 1
- setGNoWriteBarrier(&_g_.m.caughtsig, gp)
+ _g_.m.caughtsig.set(gp)
startpanic()
if sig < uint32(len(sigtable)) {
diff --git a/src/runtime/signal_ppc64x.go b/src/runtime/signal_ppc64x.go
index 018d7d62ef..04d8cfcec1 100644
--- a/src/runtime/signal_ppc64x.go
+++ b/src/runtime/signal_ppc64x.go
@@ -113,7 +113,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) {
}
_g_.m.throwing = 1
- setGNoWriteBarrier(&_g_.m.caughtsig, gp)
+ _g_.m.caughtsig.set(gp)
startpanic()
if sig < uint32(len(sigtable)) {
diff --git a/src/runtime/stack1.go b/src/runtime/stack1.go
index 5f28d28757..db7e3cbeca 100644
--- a/src/runtime/stack1.go
+++ b/src/runtime/stack1.go
@@ -680,7 +680,7 @@ func newstack() {
// it needs a lock held by the goroutine), that small preemption turns
// into a real deadlock.
if preempt {
- if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.status != _Prunning {
+ if thisg.m.locks != 0 || thisg.m.mallocing != 0 || thisg.m.preemptoff != "" || thisg.m.p.ptr().status != _Prunning {
// Let the goroutine keep running for now.
// gp->preempt is set, so it will be preempted next time.
gp.stackguard0 = gp.stack.lo + _StackGuard
@@ -724,7 +724,7 @@ func newstack() {
if gp == thisg.m.g0 {
throw("runtime: preempt g0")
}
- if thisg.m.p == nil && thisg.m.locks == 0 {
+ if thisg.m.p == 0 && thisg.m.locks == 0 {
throw("runtime: g is running but p is not")
}
if gp.preemptscan {
diff --git a/src/runtime/trace.go b/src/runtime/trace.go
index a149799527..e0eb7d82ce 100644
--- a/src/runtime/trace.go
+++ b/src/runtime/trace.go
@@ -506,7 +506,7 @@ func traceEvent(ev byte, skip int, args ...uint64) {
// traceAcquireBuffer returns trace buffer to use and, if necessary, locks it.
func traceAcquireBuffer() (mp *m, pid int32, bufp **traceBuf) {
mp = acquirem()
- if p := mp.p; p != nil {
+ if p := mp.p.ptr(); p != nil {
return mp, p.id, &p.tracebuf
}
lock(&trace.bufLock)
@@ -732,7 +732,7 @@ func traceProcStop(pp *p) {
// to handle this we temporary employ the P.
mp := acquirem()
oldp := mp.p
- mp.p = pp
+ mp.p.set(pp)
traceEvent(traceEvProcStop, -1)
mp.p = oldp
releasem(mp)
@@ -806,7 +806,7 @@ func traceGoSysBlock(pp *p) {
// to handle this we temporary employ the P.
mp := acquirem()
oldp := mp.p
- mp.p = pp
+ mp.p.set(pp)
traceEvent(traceEvGoSysBlock, -1)
mp.p = oldp
releasem(mp)
diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go
index b6f43747d6..91541fabce 100644
--- a/src/runtime/traceback.go
+++ b/src/runtime/traceback.go
@@ -528,7 +528,7 @@ func gcallers(gp *g, skip int, pcbuf []uintptr) int {
func showframe(f *_func, gp *g) bool {
g := getg()
- if g.m.throwing > 0 && gp != nil && (gp == g.m.curg || gp == g.m.caughtsig) {
+ if g.m.throwing > 0 && gp != nil && (gp == g.m.curg || gp == g.m.caughtsig.ptr()) {
return true
}
traceback := gotraceback(nil)
generated by cgit v1.3-5-g9baa (git 2.53.0) at 2026-04-17 15:15:16 +0000