diff options
Diffstat (limited to 'src/runtime/proc1.go')
| -rw-r--r-- | src/runtime/proc1.go | 76 |
1 files changed, 56 insertions, 20 deletions
diff --git a/src/runtime/proc1.go b/src/runtime/proc1.go index 5e5d0efa0b..0ff3bafbdb 100644 --- a/src/runtime/proc1.go +++ b/src/runtime/proc1.go @@ -720,7 +720,7 @@ func mstart1() { initsig() } - if _g_.m.mstartfn != nil { + if _g_.m.mstartfn != 0 { fn := *(*func())(unsafe.Pointer(&_g_.m.mstartfn)) fn() } @@ -971,17 +971,22 @@ func unlockextra(mp *m) { } // Create a new m. It will start off with a call to fn, or else the scheduler. +// fn needs to be static and not a heap allocated closure. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func newm(fn func(), _p_ *p) { mp := allocm(_p_) - mp.nextp = _p_ - mp.mstartfn = *(*unsafe.Pointer)(unsafe.Pointer(&fn)) - + // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated + setPNoWriteBarrier(&mp.nextp, _p_) + // Store &fn as a uintptr since it is not heap allocated so the WB can be eliminated + mp.mstartfn = *(*uintptr)(unsafe.Pointer(&fn)) if iscgo { var ts cgothreadstart if _cgo_thread_start == nil { throw("_cgo_thread_start missing") } - ts.g = mp.g0 + // mp is reachable via allm and mp.g0 never changes, so WB can be eliminated. + setGNoWriteBarrier(&ts.g, mp.g0) ts.tls = (*uint64)(unsafe.Pointer(&mp.tls[0])) ts.fn = unsafe.Pointer(funcPC(mstart)) asmcgocall(_cgo_thread_start, unsafe.Pointer(&ts)) @@ -1029,6 +1034,8 @@ func mspinning() { // Schedules some M to run the p (creates an M if necessary). // If p==nil, tries to get an idle P, if no idle P's does nothing. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func startm(_p_ *p, spinning bool) { lock(&sched.lock) if _p_ == nil { @@ -1058,7 +1065,8 @@ func startm(_p_ *p, spinning bool) { throw("startm: m has p") } mp.spinning = spinning - mp.nextp = _p_ + // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated + setPNoWriteBarrier(&mp.nextp, _p_) notewakeup(&mp.park) } @@ -1139,6 +1147,8 @@ func stoplockedm() { } // Schedules the locked m to run the locked gp. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func startlockedm(gp *g) { _g_ := getg() @@ -1152,7 +1162,8 @@ func startlockedm(gp *g) { // directly handoff current P to the locked m incidlelocked(-1) _p_ := releasep() - mp.nextp = _p_ + // procresize made _p_ reachable through allp, which doesn't change during GC, so WB can be eliminated + setPNoWriteBarrier(&mp.nextp, _p_) notewakeup(&mp.park) stopm() } @@ -1805,7 +1816,11 @@ func exitsyscall(dummy int32) { for oldp != nil && oldp.syscalltick == _g_.m.syscalltick { osyield() } - systemstack(traceGoSysExit) + // This can't be done since the GC may be running and this code + // will invoke write barriers. + // TODO: Figure out how to get traceGoSysExit into the trace log or + // it is likely not to work as expected. + // systemstack(traceGoSysExit) } _g_.m.locks-- @@ -2569,6 +2584,8 @@ func procresize(nprocs int32) *p { } // Associate p and the current m. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func acquirep(_p_ *p) { _g_ := getg() @@ -2583,9 +2600,12 @@ func acquirep(_p_ *p) { print("acquirep: p->m=", _p_.m, "(", id, ") p->status=", _p_.status, "\n") throw("acquirep: invalid p state") } - _g_.m.mcache = _p_.mcache - _g_.m.p = _p_ - _p_.m = _g_.m + // _p_.mcache holds the mcache and _p_ is in allp, so WB can be eliminated + setMcacheNoWriteBarrier(&_g_.m.mcache, _p_.mcache) + // _p_ is in allp so WB can be eliminated + setPNoWriteBarrier(&_g_.m.p, _p_) + // m is in _g_.m and is reachable through allg, so WB can be eliminated + setMNoWriteBarrier(&_p_.m, _g_.m) _p_.status = _Prunning if trace.enabled { @@ -2991,19 +3011,26 @@ func schedtrace(detailed bool) { // Put mp on midle list. // Sched must be locked. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func mput(mp *m) { - mp.schedlink = sched.midle - sched.midle = mp + // sched.midle is reachable via allm, so WB can be eliminated. + setMNoWriteBarrier(&mp.schedlink, sched.midle) + // mp is reachable via allm, so WB can be eliminated. + setMNoWriteBarrier(&sched.midle, mp) sched.nmidle++ checkdead() } // Try to get an m from midle list. // Sched must be locked. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func mget() *m { mp := sched.midle if mp != nil { - sched.midle = mp.schedlink + // mp.schedlink is reachable via mp, which is on allm, so WB can be eliminated. + setMNoWriteBarrier(&sched.midle, mp.schedlink) sched.nmidle-- } return mp @@ -3011,14 +3038,17 @@ func mget() *m { // Put gp on the global runnable queue. // Sched must be locked. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func globrunqput(gp *g) { gp.schedlink = nil if sched.runqtail != nil { - sched.runqtail.schedlink = gp + // gp is on allg, so these three WBs can be eliminated. + setGNoWriteBarrier(&sched.runqtail.schedlink, gp) } else { - sched.runqhead = gp + setGNoWriteBarrier(&sched.runqhead, gp) } - sched.runqtail = gp + setGNoWriteBarrier(&sched.runqtail, gp) sched.runqsize++ } @@ -3071,18 +3101,24 @@ func globrunqget(_p_ *p, max int32) *g { // Put p to on _Pidle list. // Sched must be locked. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func pidleput(_p_ *p) { - _p_.link = sched.pidle - sched.pidle = _p_ + // sched.pidle, _p_.link and _p_ are reachable via allp, so WB can be eliminated. + setPNoWriteBarrier(&_p_.link, sched.pidle) + setPNoWriteBarrier(&sched.pidle, _p_) xadd(&sched.npidle, 1) // TODO: fast atomic } // Try get a p from _Pidle list. // Sched must be locked. +// May run during STW, so write barriers are not allowed. +//go:nowritebarrier func pidleget() *p { _p_ := sched.pidle if _p_ != nil { - sched.pidle = _p_.link + // _p_.link is reachable via a _p_ in allp, so WB can be eliminated. + setPNoWriteBarrier(&sched.pidle, _p_.link) xadd(&sched.npidle, -1) // TODO: fast atomic } return _p_ |