aboutsummaryrefslogtreecommitdiff
path: root/src/runtime/proc.go
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime/proc.go')
-rw-r--r--src/runtime/proc.go673
1 files changed, 520 insertions, 153 deletions
diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 5787991f07..ff441badde 100644
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -176,6 +176,9 @@ func main() {
if _cgo_notify_runtime_init_done == nil {
throw("_cgo_notify_runtime_init_done missing")
}
+ // Start the template thread in case we enter Go from
+ // a C-created thread and need to create a new thread.
+ startTemplateThread()
cgocall(_cgo_notify_runtime_init_done, nil)
}
@@ -252,9 +255,10 @@ func forcegchelper() {
}
}
+//go:nosplit
+
// Gosched yields the processor, allowing other goroutines to run. It does not
// suspend the current goroutine, so execution resumes automatically.
-//go:nosplit
func Gosched() {
mcall(gosched_m)
}
@@ -342,8 +346,8 @@ func releaseSudog(s *sudog) {
if s.elem != nil {
throw("runtime: sudog with non-nil elem")
}
- if s.selectdone != nil {
- throw("runtime: sudog with non-nil selectdone")
+ if s.isSelect {
+ throw("runtime: sudog with non-false isSelect")
}
if s.next != nil {
throw("runtime: sudog with non-nil next")
@@ -432,7 +436,7 @@ func badctxt() {
func lockedOSThread() bool {
gp := getg()
- return gp.lockedm != nil && gp.m.lockedg != nil
+ return gp.lockedm != 0 && gp.m.lockedg != 0
}
var (
@@ -498,13 +502,21 @@ func schedinit() {
if n, ok := atoi32(gogetenv("GOMAXPROCS")); ok && n > 0 {
procs = n
}
- if procs > _MaxGomaxprocs {
- procs = _MaxGomaxprocs
- }
if procresize(procs) != nil {
throw("unknown runnable goroutine during bootstrap")
}
+ // For cgocheck > 1, we turn on the write barrier at all times
+ // and check all pointer writes. We can't do this until after
+ // procresize because the write barrier needs a P.
+ if debug.cgocheck > 1 {
+ writeBarrier.cgo = true
+ writeBarrier.enabled = true
+ for _, p := range allp {
+ p.wbBuf.reset()
+ }
+ }
+
if buildVersion == "" {
// Condition should never trigger. This code just serves
// to ensure runtime·buildVersion is kept in the resulting binary.
@@ -520,7 +532,7 @@ func dumpgstatus(gp *g) {
func checkmcount() {
// sched lock is held
- if sched.mcount > sched.maxmcount {
+ if mcount() > sched.maxmcount {
print("runtime: program exceeds ", sched.maxmcount, "-thread limit\n")
throw("thread exhaustion")
}
@@ -534,15 +546,20 @@ func mcommoninit(mp *m) {
callers(1, mp.createstack[:])
}
- mp.fastrand = 0x49f6428a + uint32(mp.id) + uint32(cputicks())
- if mp.fastrand == 0 {
- mp.fastrand = 0x49f6428a
- }
-
lock(&sched.lock)
- mp.id = sched.mcount
- sched.mcount++
+ if sched.mnext+1 < sched.mnext {
+ throw("runtime: thread ID overflow")
+ }
+ mp.id = sched.mnext
+ sched.mnext++
checkmcount()
+
+ mp.fastrand[0] = 1597334677 * uint32(mp.id)
+ mp.fastrand[1] = uint32(cputicks())
+ if mp.fastrand[0]|mp.fastrand[1] == 0 {
+ mp.fastrand[1] = 1
+ }
+
mpreinit(mp)
if mp.gsignal != nil {
mp.gsignal.stackguard1 = mp.gsignal.stack.lo + _StackGuard
@@ -765,8 +782,10 @@ func casgstatus(gp *g, oldval, newval uint32) {
// _Grunning or _Grunning|_Gscan; either way,
// we own gp.gcscanvalid, so it's safe to read.
// gp.gcscanvalid must not be true when we are running.
- print("runtime: casgstatus ", hex(oldval), "->", hex(newval), " gp.status=", hex(gp.atomicstatus), " gp.gcscanvalid=true\n")
- throw("casgstatus")
+ systemstack(func() {
+ print("runtime: casgstatus ", hex(oldval), "->", hex(newval), " gp.status=", hex(gp.atomicstatus), " gp.gcscanvalid=true\n")
+ throw("casgstatus")
+ })
}
// See http://golang.org/cl/21503 for justification of the yield delay.
@@ -951,7 +970,7 @@ func stopTheWorld(reason string) {
// startTheWorld undoes the effects of stopTheWorld.
func startTheWorld() {
- systemstack(startTheWorldWithSema)
+ systemstack(func() { startTheWorldWithSema(false) })
// worldsema must be held over startTheWorldWithSema to ensure
// gomaxprocs cannot change while worldsema is held.
semrelease(&worldsema)
@@ -1001,8 +1020,7 @@ func stopTheWorldWithSema() {
_g_.m.p.ptr().status = _Pgcstop // Pgcstop is only diagnostic.
sched.stopwait--
// try to retake all P's in Psyscall status
- for i := 0; i < int(gomaxprocs); i++ {
- p := allp[i]
+ for _, p := range allp {
s := p.status
if s == _Psyscall && atomic.Cas(&p.status, s, _Pgcstop) {
if trace.enabled {
@@ -1042,8 +1060,7 @@ func stopTheWorldWithSema() {
if sched.stopwait != 0 {
bad = "stopTheWorld: not stopped (stopwait != 0)"
} else {
- for i := 0; i < int(gomaxprocs); i++ {
- p := allp[i]
+ for _, p := range allp {
if p.status != _Pgcstop {
bad = "stopTheWorld: not stopped (status != _Pgcstop)"
}
@@ -1067,12 +1084,14 @@ func mhelpgc() {
_g_.m.helpgc = -1
}
-func startTheWorldWithSema() {
+func startTheWorldWithSema(emitTraceEvent bool) int64 {
_g_ := getg()
- _g_.m.locks++ // disable preemption because it can be holding p in a local var
- gp := netpoll(false) // non-blocking
- injectglist(gp)
+ _g_.m.locks++ // disable preemption because it can be holding p in a local var
+ if netpollinited() {
+ gp := netpoll(false) // non-blocking
+ injectglist(gp)
+ }
add := needaddgcproc()
lock(&sched.lock)
@@ -1107,6 +1126,12 @@ func startTheWorldWithSema() {
}
}
+ // Capture start-the-world time before doing clean-up tasks.
+ startTime := nanotime()
+ if emitTraceEvent {
+ traceGCSTWDone()
+ }
+
// Wakeup an additional proc in case we have excessive runnable goroutines
// in local queues or in the global queue. If we don't, the proc will park itself.
// If we have lots of excessive work, resetspinning will unpark additional procs as necessary.
@@ -1128,14 +1153,25 @@ func startTheWorldWithSema() {
if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
_g_.stackguard0 = stackPreempt
}
+
+ return startTime
}
// Called to start an M.
+//
+// This must not split the stack because we may not even have stack
+// bounds set up yet.
+//
+// May run during STW (because it doesn't have a P yet), so write
+// barriers are not allowed.
+//
//go:nosplit
+//go:nowritebarrierrec
func mstart() {
_g_ := getg()
- if _g_.stack.lo == 0 {
+ osStack := _g_.stack.lo == 0
+ if osStack {
// Initialize stack bounds from system stack.
// Cgo may have left stack size in stack.hi.
size := _g_.stack.hi
@@ -1149,33 +1185,37 @@ func mstart() {
// both Go and C functions with stack growth prologues.
_g_.stackguard0 = _g_.stack.lo + _StackGuard
_g_.stackguard1 = _g_.stackguard0
- mstart1()
+ mstart1(0)
+
+ // Exit this thread.
+ if GOOS == "windows" || GOOS == "solaris" || GOOS == "plan9" {
+ // Window, Solaris and Plan 9 always system-allocate
+ // the stack, but put it in _g_.stack before mstart,
+ // so the logic above hasn't set osStack yet.
+ osStack = true
+ }
+ mexit(osStack)
}
-func mstart1() {
+func mstart1(dummy int32) {
_g_ := getg()
if _g_ != _g_.m.g0 {
throw("bad runtime·mstart")
}
- // Record top of stack for use by mcall.
- // Once we call schedule we're never coming back,
- // so other calls can reuse this stack space.
- gosave(&_g_.m.g0.sched)
- _g_.m.g0.sched.pc = ^uintptr(0) // make sure it is never used
+ // Record the caller for use as the top of stack in mcall and
+ // for terminating the thread.
+ // We're never coming back to mstart1 after we call schedule,
+ // so other calls can reuse the current frame.
+ save(getcallerpc(), getcallersp(unsafe.Pointer(&dummy)))
asminit()
minit()
// Install signal handlers; after minit so that minit can
// prepare the thread to be able to handle the signals.
if _g_.m == &m0 {
- // Create an extra M for callbacks on threads not created by Go.
- if iscgo && !cgoHasExtraM {
- cgoHasExtraM = true
- newextram()
- }
- initsig(false)
+ mstartm0()
}
if fn := _g_.m.mstartfn; fn != nil {
@@ -1192,6 +1232,114 @@ func mstart1() {
schedule()
}
+// mstartm0 implements part of mstart1 that only runs on the m0.
+//
+// Write barriers are allowed here because we know the GC can't be
+// running yet, so they'll be no-ops.
+//
+//go:yeswritebarrierrec
+func mstartm0() {
+ // Create an extra M for callbacks on threads not created by Go.
+ if iscgo && !cgoHasExtraM {
+ cgoHasExtraM = true
+ newextram()
+ }
+ initsig(false)
+}
+
+// mexit tears down and exits the current thread.
+//
+// Don't call this directly to exit the thread, since it must run at
+// the top of the thread stack. Instead, use gogo(&_g_.m.g0.sched) to
+// unwind the stack to the point that exits the thread.
+//
+// It is entered with m.p != nil, so write barriers are allowed. It
+// will release the P before exiting.
+//
+//go:yeswritebarrierrec
+func mexit(osStack bool) {
+ g := getg()
+ m := g.m
+
+ if m == &m0 {
+ // This is the main thread. Just wedge it.
+ //
+ // On Linux, exiting the main thread puts the process
+ // into a non-waitable zombie state. On Plan 9,
+ // exiting the main thread unblocks wait even though
+ // other threads are still running. On Solaris we can
+ // neither exitThread nor return from mstart. Other
+ // bad things probably happen on other platforms.
+ //
+ // We could try to clean up this M more before wedging
+ // it, but that complicates signal handling.
+ handoffp(releasep())
+ lock(&sched.lock)
+ sched.nmfreed++
+ checkdead()
+ unlock(&sched.lock)
+ notesleep(&m.park)
+ throw("locked m0 woke up")
+ }
+
+ sigblock()
+ unminit()
+
+ // Free the gsignal stack.
+ if m.gsignal != nil {
+ stackfree(m.gsignal.stack)
+ }
+
+ // Remove m from allm.
+ lock(&sched.lock)
+ for pprev := &allm; *pprev != nil; pprev = &(*pprev).alllink {
+ if *pprev == m {
+ *pprev = m.alllink
+ goto found
+ }
+ }
+ throw("m not found in allm")
+found:
+ if !osStack {
+ // Delay reaping m until it's done with the stack.
+ //
+ // If this is using an OS stack, the OS will free it
+ // so there's no need for reaping.
+ atomic.Store(&m.freeWait, 1)
+ // Put m on the free list, though it will not be reaped until
+ // freeWait is 0. Note that the free list must not be linked
+ // through alllink because some functions walk allm without
+ // locking, so may be using alllink.
+ m.freelink = sched.freem
+ sched.freem = m
+ }
+ unlock(&sched.lock)
+
+ // Release the P.
+ handoffp(releasep())
+ // After this point we must not have write barriers.
+
+ // Invoke the deadlock detector. This must happen after
+ // handoffp because it may have started a new M to take our
+ // P's work.
+ lock(&sched.lock)
+ sched.nmfreed++
+ checkdead()
+ unlock(&sched.lock)
+
+ if osStack {
+ // Return from mstart and let the system thread
+ // library free the g0 stack and terminate the thread.
+ return
+ }
+
+ // mstart is the thread's entry point, so there's nothing to
+ // return to. Exit the thread directly. exitThread will clear
+ // m.freeWait when it's done with the stack and the m can be
+ // reaped.
+ exitThread(&m.freeWait)
+}
+
// forEachP calls fn(p) for every P p when p reaches a GC safe point.
// If a P is currently executing code, this will bring the P to a GC
// safe point and execute fn on that P. If the P is not executing code
@@ -1215,7 +1363,7 @@ func forEachP(fn func(*p)) {
sched.safePointFn = fn
// Ask all Ps to run the safe point function.
- for _, p := range allp[:gomaxprocs] {
+ for _, p := range allp {
if p != _p_ {
atomic.Store(&p.runSafePointFn, 1)
}
@@ -1243,8 +1391,7 @@ func forEachP(fn func(*p)) {
// Force Ps currently in _Psyscall into _Pidle and hand them
// off to induce safe point function execution.
- for i := 0; i < int(gomaxprocs); i++ {
- p := allp[i]
+ for _, p := range allp {
s := p.status
if s == _Psyscall && p.runSafePointFn == 1 && atomic.Cas(&p.status, s, _Pidle) {
if trace.enabled {
@@ -1273,8 +1420,7 @@ func forEachP(fn func(*p)) {
if sched.safePointWait != 0 {
throw("forEachP: not done")
}
- for i := 0; i < int(gomaxprocs); i++ {
- p := allp[i]
+ for _, p := range allp {
if p.runSafePointFn != 0 {
throw("forEachP: P did not run fn")
}
@@ -1339,6 +1485,27 @@ func allocm(_p_ *p, fn func()) *m {
if _g_.m.p == 0 {
acquirep(_p_) // temporarily borrow p for mallocs in this function
}
+
+ // Release the free M list. We need to do this somewhere and
+ // this may free up a stack we can use.
+ if sched.freem != nil {
+ lock(&sched.lock)
+ var newList *m
+ for freem := sched.freem; freem != nil; {
+ if freem.freeWait != 0 {
+ next := freem.freelink
+ freem.freelink = newList
+ newList = freem
+ freem = next
+ continue
+ }
+ stackfree(freem.g0.stack)
+ freem = freem.freelink
+ }
+ sched.freem = newList
+ unlock(&sched.lock)
+ }
+
mp := new(m)
mp.mstartfn = fn
mcommoninit(mp)
@@ -1498,9 +1665,9 @@ func oneNewExtraM() {
casgstatus(gp, _Gidle, _Gdead)
gp.m = mp
mp.curg = gp
- mp.locked = _LockInternal
- mp.lockedg = gp
- gp.lockedm = mp
+ mp.lockedInt++
+ mp.lockedg.set(gp)
+ gp.lockedm.set(mp)
gp.goid = int64(atomic.Xadd64(&sched.goidgen, 1))
if raceenabled {
gp.racectx = racegostart(funcPC(newextram) + sys.PCQuantum)
@@ -1629,6 +1796,27 @@ func unlockextra(mp *m) {
// around exec'ing while creating/destroying threads. See issue #19546.
var execLock rwmutex
+// newmHandoff contains a list of m structures that need new OS threads.
+// This is used by newm in situations where newm itself can't safely
+// start an OS thread.
+var newmHandoff struct {
+ lock mutex
+
+ // newm points to a list of M structures that need new OS
+ // threads. The list is linked through m.schedlink.
+ newm muintptr
+
+ // waiting indicates that wake needs to be notified when an m
+ // is put on the list.
+ waiting bool
+ wake note
+
+ // haveTemplateThread indicates that the templateThread has
+ // been started. This is not protected by lock. Use cas to set
+ // to 1.
+ haveTemplateThread uint32
+}
+
// Create a new m. It will start off with a call to fn, or else the scheduler.
// fn needs to be static and not a heap allocated closure.
// May run with m.p==nil, so write barriers are not allowed.
@@ -1637,6 +1825,35 @@ func newm(fn func(), _p_ *p) {
mp := allocm(_p_, fn)
mp.nextp.set(_p_)
mp.sigmask = initSigmask
+ if gp := getg(); gp != nil && gp.m != nil && (gp.m.lockedExt != 0 || gp.m.incgo) && GOOS != "plan9" {
+ // We're on a locked M or a thread that may have been
+ // started by C. The kernel state of this thread may
+ // be strange (the user may have locked it for that
+ // purpose). We don't want to clone that into another
+ // thread. Instead, ask a known-good thread to create
+ // the thread for us.
+ //
+ // This is disabled on Plan 9. See golang.org/issue/22227.
+ //
+ // TODO: This may be unnecessary on Windows, which
+ // doesn't model thread creation off fork.
+ lock(&newmHandoff.lock)
+ if newmHandoff.haveTemplateThread == 0 {
+ throw("on a locked thread with no template thread")
+ }
+ mp.schedlink = newmHandoff.newm
+ newmHandoff.newm.set(mp)
+ if newmHandoff.waiting {
+ newmHandoff.waiting = false
+ notewakeup(&newmHandoff.wake)
+ }
+ unlock(&newmHandoff.lock)
+ return
+ }
+ newm1(mp)
+}
+
+func newm1(mp *m) {
if iscgo {
var ts cgothreadstart
if _cgo_thread_start == nil {
@@ -1658,6 +1875,56 @@ func newm(fn func(), _p_ *p) {
execLock.runlock()
}
+// startTemplateThread starts the template thread if it is not already
+// running.
+//
+// The calling thread must itself be in a known-good state.
+func startTemplateThread() {
+ if !atomic.Cas(&newmHandoff.haveTemplateThread, 0, 1) {
+ return
+ }
+ newm(templateThread, nil)
+}
+
+// tmeplateThread is a thread in a known-good state that exists solely
+// to start new threads in known-good states when the calling thread
+// may not be a a good state.
+//
+// Many programs never need this, so templateThread is started lazily
+// when we first enter a state that might lead to running on a thread
+// in an unknown state.
+//
+// templateThread runs on an M without a P, so it must not have write
+// barriers.
+//
+//go:nowritebarrierrec
+func templateThread() {
+ lock(&sched.lock)
+ sched.nmsys++
+ checkdead()
+ unlock(&sched.lock)
+
+ for {
+ lock(&newmHandoff.lock)
+ for newmHandoff.newm != 0 {
+ newm := newmHandoff.newm.ptr()
+ newmHandoff.newm = 0
+ unlock(&newmHandoff.lock)
+ for newm != nil {
+ next := newm.schedlink.ptr()
+ newm.schedlink = 0
+ newm1(newm)
+ newm = next
+ }
+ lock(&newmHandoff.lock)
+ }
+ newmHandoff.waiting = true
+ noteclear(&newmHandoff.wake)
+ unlock(&newmHandoff.lock)
+ notesleep(&newmHandoff.wake)
+ }
+}
+
// Stops execution of the current m until new work is available.
// Returns with acquired P.
func stopm() {
@@ -1680,7 +1947,9 @@ retry:
notesleep(&_g_.m.park)
noteclear(&_g_.m.park)
if _g_.m.helpgc != 0 {
+ // helpgc() set _g_.m.p and _g_.m.mcache, so we have a P.
gchelper()
+ // Undo the effects of helpgc().
_g_.m.helpgc = 0
_g_.m.mcache = nil
_g_.m.p = 0
@@ -1814,7 +2083,7 @@ func wakep() {
func stoplockedm() {
_g_ := getg()
- if _g_.m.lockedg == nil || _g_.m.lockedg.lockedm != _g_.m {
+ if _g_.m.lockedg == 0 || _g_.m.lockedg.ptr().lockedm.ptr() != _g_.m {
throw("stoplockedm: inconsistent locking")
}
if _g_.m.p != 0 {
@@ -1826,7 +2095,7 @@ func stoplockedm() {
// Wait until another thread schedules lockedg again.
notesleep(&_g_.m.park)
noteclear(&_g_.m.park)
- status := readgstatus(_g_.m.lockedg)
+ status := readgstatus(_g_.m.lockedg.ptr())
if status&^_Gscan != _Grunnable {
print("runtime:stoplockedm: g is not Grunnable or Gscanrunnable\n")
dumpgstatus(_g_)
@@ -1842,7 +2111,7 @@ func stoplockedm() {
func startlockedm(gp *g) {
_g_ := getg()
- mp := gp.lockedm
+ mp := gp.lockedm.ptr()
if mp == _g_.m {
throw("startlockedm: locked to me")
}
@@ -1968,11 +2237,12 @@ top:
// Poll network.
// This netpoll is only an optimization before we resort to stealing.
- // We can safely skip it if there a thread blocked in netpoll already.
- // If there is any kind of logical race with that blocked thread
- // (e.g. it has already returned from netpoll, but does not set lastpoll yet),
- // this thread will do blocking netpoll below anyway.
- if netpollinited() && sched.lastpoll != 0 {
+ // We can safely skip it if there are no waiters or a thread is blocked
+ // in netpoll already. If there is any kind of logical race with that
+ // blocked thread (e.g. it has already returned from netpoll, but does
+ // not set lastpoll yet), this thread will do blocking netpoll below
+ // anyway.
+ if netpollinited() && atomic.Load(&netpollWaiters) > 0 && atomic.Load64(&sched.lastpoll) != 0 {
if gp := netpoll(false); gp != nil { // non-blocking
// netpoll returns list of goroutines linked by schedlink.
injectglist(gp.schedlink.ptr())
@@ -2068,9 +2338,8 @@ stop:
}
// check all runqueues once again
- for i := 0; i < int(gomaxprocs); i++ {
- _p_ := allp[i]
- if _p_ != nil && !runqempty(_p_) {
+ for _, _p_ := range allp {
+ if !runqempty(_p_) {
lock(&sched.lock)
_p_ = pidleget()
unlock(&sched.lock)
@@ -2209,9 +2478,15 @@ func schedule() {
throw("schedule: holding locks")
}
- if _g_.m.lockedg != nil {
+ if _g_.m.lockedg != 0 {
stoplockedm()
- execute(_g_.m.lockedg, false) // Never returns.
+ execute(_g_.m.lockedg.ptr(), false) // Never returns.
+ }
+
+ // We should not schedule away from a g that is executing a cgo call,
+ // since the cgo call is using the m's g0 stack.
+ if _g_.m.incgo {
+ throw("schedule: in cgo")
}
top:
@@ -2262,7 +2537,7 @@ top:
resetspinning()
}
- if gp.lockedm != nil {
+ if gp.lockedm != 0 {
// Hands off own p to the locked m,
// then blocks waiting for a new p.
startlockedm(gp)
@@ -2381,8 +2656,9 @@ func goexit0(gp *g) {
atomic.Xadd(&sched.ngsys, -1)
}
gp.m = nil
- gp.lockedm = nil
- _g_.m.lockedg = nil
+ locked := gp.lockedm != 0
+ gp.lockedm = 0
+ _g_.m.lockedg = 0
gp.paniconfault = false
gp._defer = nil // should be true already but just in case.
gp._panic = nil // non-nil for Goexit during panic. points at stack-allocated data.
@@ -2392,17 +2668,37 @@ func goexit0(gp *g) {
gp.labels = nil
gp.timer = nil
+ if gcBlackenEnabled != 0 && gp.gcAssistBytes > 0 {
+ // Flush assist credit to the global pool. This gives
+ // better information to pacing if the application is
+ // rapidly creating an exiting goroutines.
+ scanCredit := int64(gcController.assistWorkPerByte * float64(gp.gcAssistBytes))
+ atomic.Xaddint64(&gcController.bgScanCredit, scanCredit)
+ gp.gcAssistBytes = 0
+ }
+
// Note that gp's stack scan is now "valid" because it has no
// stack.
gp.gcscanvalid = true
dropg()
- if _g_.m.locked&^_LockExternal != 0 {
- print("invalid m->locked = ", _g_.m.locked, "\n")
+ if _g_.m.lockedInt != 0 {
+ print("invalid m->lockedInt = ", _g_.m.lockedInt, "\n")
throw("internal lockOSThread error")
}
- _g_.m.locked = 0
+ _g_.m.lockedExt = 0
gfput(_g_.m.p.ptr(), gp)
+ if locked {
+ // The goroutine may have locked this thread because
+ // it put it in an unusual kernel state. Kill it
+ // rather than returning it to the thread pool.
+
+ // Return to mstart, which will release the P and exit
+ // the thread.
+ if GOOS != "plan9" { // See golang.org/issue/22227.
+ gogo(&_g_.m.g0.sched)
+ }
+ }
schedule()
}
@@ -2532,7 +2828,7 @@ func reentersyscall(pc, sp uintptr) {
// Standard syscall entry used by the go syscall library and normal cgo calls.
//go:nosplit
func entersyscall(dummy int32) {
- reentersyscall(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
+ reentersyscall(getcallerpc(), getcallersp(unsafe.Pointer(&dummy)))
}
func entersyscall_sysmon() {
@@ -2575,7 +2871,7 @@ func entersyscallblock(dummy int32) {
_g_.m.p.ptr().syscalltick++
// Leave SP around for GC and traceback.
- pc := getcallerpc(unsafe.Pointer(&dummy))
+ pc := getcallerpc()
sp := getcallersp(unsafe.Pointer(&dummy))
save(pc, sp)
_g_.syscallsp = _g_.sched.sp
@@ -2600,7 +2896,7 @@ func entersyscallblock(dummy int32) {
systemstack(entersyscallblock_handoff)
// Resave for traceback during blocked call.
- save(getcallerpc(unsafe.Pointer(&dummy)), getcallersp(unsafe.Pointer(&dummy)))
+ save(getcallerpc(), getcallersp(unsafe.Pointer(&dummy)))
_g_.m.locks--
}
@@ -2639,7 +2935,9 @@ func exitsyscall(dummy int32) {
oldp := _g_.m.p.ptr()
if exitsyscallfast() {
if _g_.m.mcache == nil {
- throw("lost mcache")
+ systemstack(func() {
+ throw("lost mcache")
+ })
}
if trace.enabled {
if oldp != _g_.m.p.ptr() || _g_.m.syscalltick != _g_.m.p.ptr().syscalltick {
@@ -2686,7 +2984,9 @@ func exitsyscall(dummy int32) {
mcall(exitsyscall0)
if _g_.m.mcache == nil {
- throw("lost mcache")
+ systemstack(func() {
+ throw("lost mcache")
+ })
}
// Scheduler returned, so we're allowed to run now.
@@ -2810,7 +3110,7 @@ func exitsyscall0(gp *g) {
acquirep(_p_)
execute(gp, false) // Never returns.
}
- if _g_.m.lockedg != nil {
+ if _g_.m.lockedg != 0 {
// Wait until another thread schedules gp and so m again.
stoplockedm()
execute(gp, false) // Never returns.
@@ -2928,17 +3228,16 @@ func malg(stacksize int32) *g {
//go:nosplit
func newproc(siz int32, fn *funcval) {
argp := add(unsafe.Pointer(&fn), sys.PtrSize)
- pc := getcallerpc(unsafe.Pointer(&siz))
+ pc := getcallerpc()
systemstack(func() {
- newproc1(fn, (*uint8)(argp), siz, 0, pc)
+ newproc1(fn, (*uint8)(argp), siz, pc)
})
}
// Create a new g running fn with narg bytes of arguments starting
-// at argp and returning nret bytes of results. callerpc is the
-// address of the go statement that created this. The new g is put
-// on the queue of g's waiting to run.
-func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr) *g {
+// at argp. callerpc is the address of the go statement that created
+// this. The new g is put on the queue of g's waiting to run.
+func newproc1(fn *funcval, argp *uint8, narg int32, callerpc uintptr) {
_g_ := getg()
if fn == nil {
@@ -2946,7 +3245,7 @@ func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr
throw("go of nil func value")
}
_g_.m.locks++ // disable preemption because it can be holding p in a local var
- siz := narg + nret
+ siz := narg
siz = (siz + 7) &^ 7
// We could allocate a larger initial stack if necessary.
@@ -3041,7 +3340,6 @@ func newproc1(fn *funcval, argp *uint8, narg int32, nret int32, callerpc uintptr
if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack
_g_.stackguard0 = stackPreempt
}
- return newg
}
// Put on gfree list.
@@ -3160,23 +3458,41 @@ func Breakpoint() {
//go:nosplit
func dolockOSThread() {
_g_ := getg()
- _g_.m.lockedg = _g_
- _g_.lockedm = _g_.m
+ _g_.m.lockedg.set(_g_)
+ _g_.lockedm.set(_g_.m)
}
//go:nosplit
// LockOSThread wires the calling goroutine to its current operating system thread.
-// Until the calling goroutine exits or calls UnlockOSThread, it will always
-// execute in that thread, and no other goroutine can.
+// The calling goroutine will always execute in that thread,
+// and no other goroutine will execute in it,
+// until the calling goroutine has made as many calls to
+// UnlockOSThread as to LockOSThread.
+// If the calling goroutine exits without unlocking the thread,
+// the thread will be terminated.
+//
+// A goroutine should call LockOSThread before calling OS services or
+// non-Go library functions that depend on per-thread state.
func LockOSThread() {
- getg().m.locked |= _LockExternal
+ if atomic.Load(&newmHandoff.haveTemplateThread) == 0 && GOOS != "plan9" {
+ // If we need to start a new thread from the locked
+ // thread, we need the template thread. Start it now
+ // while we're in a known-good state.
+ startTemplateThread()
+ }
+ _g_ := getg()
+ _g_.m.lockedExt++
+ if _g_.m.lockedExt == 0 {
+ _g_.m.lockedExt--
+ panic("LockOSThread nesting overflow")
+ }
dolockOSThread()
}
//go:nosplit
func lockOSThread() {
- getg().m.locked += _LockInternal
+ getg().m.lockedInt++
dolockOSThread()
}
@@ -3186,29 +3502,43 @@ func lockOSThread() {
//go:nosplit
func dounlockOSThread() {
_g_ := getg()
- if _g_.m.locked != 0 {
+ if _g_.m.lockedInt != 0 || _g_.m.lockedExt != 0 {
return
}
- _g_.m.lockedg = nil
- _g_.lockedm = nil
+ _g_.m.lockedg = 0
+ _g_.lockedm = 0
}
//go:nosplit
-// UnlockOSThread unwires the calling goroutine from its fixed operating system thread.
-// If the calling goroutine has not called LockOSThread, UnlockOSThread is a no-op.
+// UnlockOSThread undoes an earlier call to LockOSThread.
+// If this drops the number of active LockOSThread calls on the
+// calling goroutine to zero, it unwires the calling goroutine from
+// its fixed operating system thread.
+// If there are no active LockOSThread calls, this is a no-op.
+//
+// Before calling UnlockOSThread, the caller must ensure that the OS
+// thread is suitable for running other goroutines. If the caller made
+// any permanent changes to the state of the thread that would affect
+// other goroutines, it should not call this function and thus leave
+// the goroutine locked to the OS thread until the goroutine (and
+// hence the thread) exits.
func UnlockOSThread() {
- getg().m.locked &^= _LockExternal
+ _g_ := getg()
+ if _g_.m.lockedExt == 0 {
+ return
+ }
+ _g_.m.lockedExt--
dounlockOSThread()
}
//go:nosplit
func unlockOSThread() {
_g_ := getg()
- if _g_.m.locked < _LockInternal {
+ if _g_.m.lockedInt == 0 {
systemstack(badunlockosthread)
}
- _g_.m.locked -= _LockInternal
+ _g_.m.lockedInt--
dounlockOSThread()
}
@@ -3218,10 +3548,7 @@ func badunlockosthread() {
func gcount() int32 {
n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys))
- for _, _p_ := range &allp {
- if _p_ == nil {
- break
- }
+ for _, _p_ := range allp {
n -= _p_.gfreecnt
}
@@ -3234,7 +3561,7 @@ func gcount() int32 {
}
func mcount() int32 {
- return sched.mcount
+ return int32(sched.mnext - sched.nmfreed)
}
var prof struct {
@@ -3516,7 +3843,7 @@ func setcpuprofilerate(hz int32) {
// Returns list of Ps with local work, they need to be scheduled by the caller.
func procresize(nprocs int32) *p {
old := gomaxprocs
- if old < 0 || old > _MaxGomaxprocs || nprocs <= 0 || nprocs > _MaxGomaxprocs {
+ if old < 0 || nprocs <= 0 {
throw("procresize: invalid arg")
}
if trace.enabled {
@@ -3530,6 +3857,23 @@ func procresize(nprocs int32) *p {
}
sched.procresizetime = now
+ // Grow allp if necessary.
+ if nprocs > int32(len(allp)) {
+ // Synchronize with retake, which could be running
+ // concurrently since it doesn't run on a P.
+ lock(&allpLock)
+ if nprocs <= int32(cap(allp)) {
+ allp = allp[:nprocs]
+ } else {
+ nallp := make([]*p, nprocs)
+ // Copy everything up to allp's cap so we
+ // never lose old allocated Ps.
+ copy(nallp, allp[:cap(allp)])
+ allp = nallp
+ }
+ unlock(&allpLock)
+ }
+
// initialize new P's
for i := int32(0); i < nprocs; i++ {
pp := allp[i]
@@ -3541,6 +3885,7 @@ func procresize(nprocs int32) *p {
for i := range pp.deferpool {
pp.deferpool[i] = pp.deferpoolbuf[i][:0]
}
+ pp.wbBuf.reset()
atomicstorep(unsafe.Pointer(&allp[i]), unsafe.Pointer(pp))
}
if pp.mcache == nil {
@@ -3566,13 +3911,11 @@ func procresize(nprocs int32) *p {
// free unused P's
for i := nprocs; i < old; i++ {
p := allp[i]
- if trace.enabled {
- if p == getg().m.p.ptr() {
- // moving to p[0], pretend that we were descheduled
- // and then scheduled again to keep the trace sane.
- traceGoSched()
- traceProcStop(p)
- }
+ if trace.enabled && p == getg().m.p.ptr() {
+ // moving to p[0], pretend that we were descheduled
+ // and then scheduled again to keep the trace sane.
+ traceGoSched()
+ traceProcStop(p)
}
// move all runnable goroutines to the global queue
for p.runqhead != p.runqtail {
@@ -3598,6 +3941,11 @@ func procresize(nprocs int32) *p {
// world is stopped.
p.gcBgMarkWorker.set(nil)
}
+ // Flush p's write barrier buffer.
+ if gcphase != _GCoff {
+ wbBufFlush1(p)
+ p.gcw.dispose()
+ }
for i := range p.sudogbuf {
p.sudogbuf[i] = nil
}
@@ -3616,10 +3964,18 @@ func procresize(nprocs int32) *p {
raceprocdestroy(p.racectx)
p.racectx = 0
}
+ p.gcAssistTime = 0
p.status = _Pdead
// can't free P itself because it can be referenced by an M in syscall
}
+ // Trim allp.
+ if int32(len(allp)) != nprocs {
+ lock(&allpLock)
+ allp = allp[:nprocs]
+ unlock(&allpLock)
+ }
+
_g_ := getg()
if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs {
// continue to use the current P
@@ -3691,7 +4047,7 @@ func acquirep1(_p_ *p) {
throw("acquirep: already in go")
}
if _p_.m != 0 || _p_.status != _Pidle {
- id := int32(0)
+ id := int64(0)
if _p_.m != 0 {
id = _p_.m.ptr().id
}
@@ -3736,6 +4092,7 @@ func incidlelocked(v int32) {
// Check for deadlock situation.
// The check is based on number of running M's, if 0 -> deadlock.
+// sched.lock must be held.
func checkdead() {
// For -buildmode=c-shared or -buildmode=c-archive it's OK if
// there are no running goroutines. The calling program is
@@ -3752,13 +4109,12 @@ func checkdead() {
return
}
- // -1 for sysmon
- run := sched.mcount - sched.nmidle - sched.nmidlelocked - 1
+ run := mcount() - sched.nmidle - sched.nmidlelocked - sched.nmsys
if run > 0 {
return
}
if run < 0 {
- print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", sched.mcount, "\n")
+ print("runtime: checkdead: nmidle=", sched.nmidle, " nmidlelocked=", sched.nmidlelocked, " mcount=", mcount(), " nmsys=", sched.nmsys, "\n")
throw("checkdead: inconsistent counts")
}
@@ -3821,6 +4177,11 @@ var forcegcperiod int64 = 2 * 60 * 1e9
//
//go:nowritebarrierrec
func sysmon() {
+ lock(&sched.lock)
+ sched.nmsys++
+ checkdead()
+ unlock(&sched.lock)
+
// If a heap span goes unused for 5 minutes after a garbage collection,
// we hand it back to the operating system.
scavengelimit := int64(5 * 60 * 1e9)
@@ -3860,15 +4221,11 @@ func sysmon() {
}
shouldRelax := true
if osRelaxMinNS > 0 {
- lock(&timers.lock)
- if timers.sleeping {
- now := nanotime()
- next := timers.sleepUntil
- if next-now < osRelaxMinNS {
- shouldRelax = false
- }
+ next := timeSleepUntil()
+ now := nanotime()
+ if next-now < osRelaxMinNS {
+ shouldRelax = false
}
- unlock(&timers.lock)
}
if shouldRelax {
osRelax(true)
@@ -3892,7 +4249,7 @@ func sysmon() {
// poll network if not polled for more than 10ms
lastpoll := int64(atomic.Load64(&sched.lastpoll))
now := nanotime()
- if lastpoll != 0 && lastpoll+10*1000*1000 < now {
+ if netpollinited() && lastpoll != 0 && lastpoll+10*1000*1000 < now {
atomic.Cas64(&sched.lastpoll, uint64(lastpoll), uint64(now))
gp := netpoll(false) // non-blocking - returns list of goroutines
if gp != nil {
@@ -3949,9 +4306,17 @@ const forcePreemptNS = 10 * 1000 * 1000 // 10ms
func retake(now int64) uint32 {
n := 0
- for i := int32(0); i < gomaxprocs; i++ {
+ // Prevent allp slice changes. This lock will be completely
+ // uncontended unless we're already stopping the world.
+ lock(&allpLock)
+ // We can't use a range loop over allp because we may
+ // temporarily drop the allpLock. Hence, we need to re-fetch
+ // allp each time around the loop.
+ for i := 0; i < len(allp); i++ {
_p_ := allp[i]
if _p_ == nil {
+ // This can happen if procresize has grown
+ // allp but not yet created new Ps.
continue
}
pd := &_p_.sysmontick
@@ -3970,6 +4335,8 @@ func retake(now int64) uint32 {
if runqempty(_p_) && atomic.Load(&sched.nmspinning)+atomic.Load(&sched.npidle) > 0 && pd.syscallwhen+10*1000*1000 > now {
continue
}
+ // Drop allpLock so we can take sched.lock.
+ unlock(&allpLock)
// Need to decrement number of idle locked M's
// (pretending that one more is running) before the CAS.
// Otherwise the M from which we retake can exit the syscall,
@@ -3985,6 +4352,7 @@ func retake(now int64) uint32 {
handoffp(_p_)
}
incidlelocked(1)
+ lock(&allpLock)
} else if s == _Prunning {
// Preempt G if it's running for too long.
t := int64(_p_.schedtick)
@@ -3999,6 +4367,7 @@ func retake(now int64) uint32 {
preemptone(_p_)
}
}
+ unlock(&allpLock)
return uint32(n)
}
@@ -4009,9 +4378,8 @@ func retake(now int64) uint32 {
// Returns true if preemption request was issued to at least one goroutine.
func preemptall() bool {
res := false
- for i := int32(0); i < gomaxprocs; i++ {
- _p_ := allp[i]
- if _p_ == nil || _p_.status != _Prunning {
+ for _, _p_ := range allp {
+ if _p_.status != _Prunning {
continue
}
if preemptone(_p_) {
@@ -4060,23 +4428,19 @@ func schedtrace(detailed bool) {
}
lock(&sched.lock)
- print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", sched.mcount, " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize)
+ print("SCHED ", (now-starttime)/1e6, "ms: gomaxprocs=", gomaxprocs, " idleprocs=", sched.npidle, " threads=", mcount(), " spinningthreads=", sched.nmspinning, " idlethreads=", sched.nmidle, " runqueue=", sched.runqsize)
if detailed {
print(" gcwaiting=", sched.gcwaiting, " nmidlelocked=", sched.nmidlelocked, " stopwait=", sched.stopwait, " sysmonwait=", sched.sysmonwait, "\n")
}
// We must be careful while reading data from P's, M's and G's.
// Even if we hold schedlock, most data can be changed concurrently.
// E.g. (p->m ? p->m->id : -1) can crash if p->m changes from non-nil to nil.
- for i := int32(0); i < gomaxprocs; i++ {
- _p_ := allp[i]
- if _p_ == nil {
- continue
- }
+ for i, _p_ := range allp {
mp := _p_.m.ptr()
h := atomic.Load(&_p_.runqhead)
t := atomic.Load(&_p_.runqtail)
if detailed {
- id := int32(-1)
+ id := int64(-1)
if mp != nil {
id = mp.id
}
@@ -4089,7 +4453,7 @@ func schedtrace(detailed bool) {
print("[")
}
print(t - h)
- if i == gomaxprocs-1 {
+ if i == len(allp)-1 {
print("]\n")
}
}
@@ -4103,7 +4467,7 @@ func schedtrace(detailed bool) {
for mp := allm; mp != nil; mp = mp.alllink {
_p_ := mp.p.ptr()
gp := mp.curg
- lockedg := mp.lockedg
+ lockedg := mp.lockedg.ptr()
id1 := int32(-1)
if _p_ != nil {
id1 = _p_.id
@@ -4123,12 +4487,12 @@ func schedtrace(detailed bool) {
for gi := 0; gi < len(allgs); gi++ {
gp := allgs[gi]
mp := gp.m
- lockedm := gp.lockedm
- id1 := int32(-1)
+ lockedm := gp.lockedm.ptr()
+ id1 := int64(-1)
if mp != nil {
id1 = mp.id
}
- id2 := int32(-1)
+ id2 := int64(-1)
if lockedm != nil {
id2 = lockedm.id
}
@@ -4410,22 +4774,25 @@ func runqgrab(_p_ *p, batch *[256]guintptr, batchHead uint32, stealRunNextG bool
if stealRunNextG {
// Try to steal from _p_.runnext.
if next := _p_.runnext; next != 0 {
- // Sleep to ensure that _p_ isn't about to run the g we
- // are about to steal.
- // The important use case here is when the g running on _p_
- // ready()s another g and then almost immediately blocks.
- // Instead of stealing runnext in this window, back off
- // to give _p_ a chance to schedule runnext. This will avoid
- // thrashing gs between different Ps.
- // A sync chan send/recv takes ~50ns as of time of writing,
- // so 3us gives ~50x overshoot.
- if GOOS != "windows" {
- usleep(3)
- } else {
- // On windows system timer granularity is 1-15ms,
- // which is way too much for this optimization.
- // So just yield.
- osyield()
+ if _p_.status == _Prunning {
+ // Sleep to ensure that _p_ isn't about to run the g
+ // we are about to steal.
+ // The important use case here is when the g running
+ // on _p_ ready()s another g and then almost
+ // immediately blocks. Instead of stealing runnext
+ // in this window, back off to give _p_ a chance to
+ // schedule runnext. This will avoid thrashing gs
+ // between different Ps.
+ // A sync chan send/recv takes ~50ns as of time of
+ // writing, so 3us gives ~50x overshoot.
+ if GOOS != "windows" {
+ usleep(3)
+ } else {
+ // On windows system timer granularity is
+ // 1-15ms, which is way too much for this
+ // optimization. So just yield.
+ osyield()
+ }
}
if !_p_.runnext.cas(next, 0) {
continue
generated by cgit v1.3-6-g1900 (git 2.53.0) at 2026-04-17 21:37:32 +0000