aboutsummaryrefslogtreecommitdiff
path: root/src/runtime/proc.go
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime/proc.go')
-rw-r--r--src/runtime/proc.go329
1 files changed, 201 insertions, 128 deletions
diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index b5486321ed..73b4a1d9d6 100644
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -477,20 +477,14 @@ const (
_GoidCacheBatch = 16
)
-//go:linkname internal_cpu_initialize internal/cpu.initialize
-func internal_cpu_initialize(env string)
-
-//go:linkname internal_cpu_debugOptions internal/cpu.debugOptions
-var internal_cpu_debugOptions bool
-
// cpuinit extracts the environment variable GODEBUGCPU from the environment on
-// Linux and Darwin if the GOEXPERIMENT debugcpu was set and calls internal/cpu.initialize.
+// Linux and Darwin if the GOEXPERIMENT debugcpu was set and calls internal/cpu.Initialize.
func cpuinit() {
const prefix = "GODEBUGCPU="
var env string
if haveexperiment("debugcpu") && (GOOS == "linux" || GOOS == "darwin") {
- internal_cpu_debugOptions = true
+ cpu.DebugOptions = true
// Similar to goenv_unix but extracts the environment value for
// GODEBUGCPU directly.
@@ -504,18 +498,18 @@ func cpuinit() {
p := argv_index(argv, argc+1+i)
s := *(*string)(unsafe.Pointer(&stringStruct{unsafe.Pointer(p), findnull(p)}))
- if hasprefix(s, prefix) {
+ if hasPrefix(s, prefix) {
env = gostring(p)[len(prefix):]
break
}
}
}
- internal_cpu_initialize(env)
+ cpu.Initialize(env)
- support_erms = cpu.X86.HasERMS
+ // Support cpu feature variables are used in code generated by the compiler
+ // to guard execution of instructions that can not be assumed to be always supported.
support_popcnt = cpu.X86.HasPOPCNT
- support_sse2 = cpu.X86.HasSSE2
support_sse41 = cpu.X86.HasSSE41
arm64_support_atomics = cpu.ARM64.HasATOMICS
@@ -1148,8 +1142,8 @@ func startTheWorldWithSema(emitTraceEvent bool) int64 {
_g_.m.locks++ // disable preemption because it can be holding p in a local var
if netpollinited() {
- gp := netpoll(false) // non-blocking
- injectglist(gp)
+ list := netpoll(false) // non-blocking
+ injectglist(&list)
}
add := needaddgcproc()
lock(&sched.lock)
@@ -1233,6 +1227,7 @@ func mstart() {
if osStack {
// Initialize stack bounds from system stack.
// Cgo may have left stack size in stack.hi.
+ // minit may update the stack bounds.
size := _g_.stack.hi
if size == 0 {
size = 8192 * sys.StackGuardMultiplier
@@ -1604,7 +1599,7 @@ func allocm(_p_ *p, fn func()) *m {
// the following strategy: there is a stack of available m's
// that can be stolen. Using compare-and-swap
// to pop from the stack has ABA races, so we simulate
-// a lock by doing an exchange (via casp) to steal the stack
+// a lock by doing an exchange (via Casuintptr) to steal the stack
// head and replace the top pointer with MLOCKED (1).
// This serves as a simple spin lock that we can use even
// without an m. The thread that locks the stack in this way
@@ -2311,9 +2306,9 @@ top:
// not set lastpoll yet), this thread will do blocking netpoll below
// anyway.
if netpollinited() && atomic.Load(&netpollWaiters) > 0 && atomic.Load64(&sched.lastpoll) != 0 {
- if gp := netpoll(false); gp != nil { // non-blocking
- // netpoll returns list of goroutines linked by schedlink.
- injectglist(gp.schedlink.ptr())
+ if list := netpoll(false); !list.empty() { // non-blocking
+ gp := list.pop()
+ injectglist(&list)
casgstatus(gp, _Gwaiting, _Grunnable)
if trace.enabled {
traceGoUnpark(gp, 0)
@@ -2465,22 +2460,23 @@ stop:
if _g_.m.spinning {
throw("findrunnable: netpoll with spinning")
}
- gp := netpoll(true) // block until new work is available
+ list := netpoll(true) // block until new work is available
atomic.Store64(&sched.lastpoll, uint64(nanotime()))
- if gp != nil {
+ if !list.empty() {
lock(&sched.lock)
_p_ = pidleget()
unlock(&sched.lock)
if _p_ != nil {
acquirep(_p_)
- injectglist(gp.schedlink.ptr())
+ gp := list.pop()
+ injectglist(&list)
casgstatus(gp, _Gwaiting, _Grunnable)
if trace.enabled {
traceGoUnpark(gp, 0)
}
return gp, false
}
- injectglist(gp)
+ injectglist(&list)
}
}
stopm()
@@ -2500,8 +2496,8 @@ func pollWork() bool {
return true
}
if netpollinited() && atomic.Load(&netpollWaiters) > 0 && sched.lastpoll != 0 {
- if gp := netpoll(false); gp != nil {
- injectglist(gp)
+ if list := netpoll(false); !list.empty() {
+ injectglist(&list)
return true
}
}
@@ -2526,22 +2522,21 @@ func resetspinning() {
}
}
-// Injects the list of runnable G's into the scheduler.
+// Injects the list of runnable G's into the scheduler and clears glist.
// Can run concurrently with GC.
-func injectglist(glist *g) {
- if glist == nil {
+func injectglist(glist *gList) {
+ if glist.empty() {
return
}
if trace.enabled {
- for gp := glist; gp != nil; gp = gp.schedlink.ptr() {
+ for gp := glist.head.ptr(); gp != nil; gp = gp.schedlink.ptr() {
traceGoUnpark(gp, 0)
}
}
lock(&sched.lock)
var n int
- for n = 0; glist != nil; n++ {
- gp := glist
- glist = gp.schedlink.ptr()
+ for n = 0; !glist.empty(); n++ {
+ gp := glist.pop()
casgstatus(gp, _Gwaiting, _Grunnable)
globrunqput(gp)
}
@@ -2549,6 +2544,7 @@ func injectglist(glist *g) {
for ; n != 0 && sched.npidle != 0; n-- {
startm(nil, false)
}
+ *glist = gList{}
}
// One round of scheduler: find a runnable goroutine and execute it.
@@ -3469,25 +3465,21 @@ func gfput(_p_ *p, gp *g) {
gp.stackguard0 = 0
}
- gp.schedlink.set(_p_.gfree)
- _p_.gfree = gp
- _p_.gfreecnt++
- if _p_.gfreecnt >= 64 {
- lock(&sched.gflock)
- for _p_.gfreecnt >= 32 {
- _p_.gfreecnt--
- gp = _p_.gfree
- _p_.gfree = gp.schedlink.ptr()
+ _p_.gFree.push(gp)
+ _p_.gFree.n++
+ if _p_.gFree.n >= 64 {
+ lock(&sched.gFree.lock)
+ for _p_.gFree.n >= 32 {
+ _p_.gFree.n--
+ gp = _p_.gFree.pop()
if gp.stack.lo == 0 {
- gp.schedlink.set(sched.gfreeNoStack)
- sched.gfreeNoStack = gp
+ sched.gFree.noStack.push(gp)
} else {
- gp.schedlink.set(sched.gfreeStack)
- sched.gfreeStack = gp
+ sched.gFree.stack.push(gp)
}
- sched.ngfree++
+ sched.gFree.n++
}
- unlock(&sched.gflock)
+ unlock(&sched.gFree.lock)
}
}
@@ -3495,44 +3487,42 @@ func gfput(_p_ *p, gp *g) {
// If local list is empty, grab a batch from global list.
func gfget(_p_ *p) *g {
retry:
- gp := _p_.gfree
- if gp == nil && (sched.gfreeStack != nil || sched.gfreeNoStack != nil) {
- lock(&sched.gflock)
- for _p_.gfreecnt < 32 {
- if sched.gfreeStack != nil {
- // Prefer Gs with stacks.
- gp = sched.gfreeStack
- sched.gfreeStack = gp.schedlink.ptr()
- } else if sched.gfreeNoStack != nil {
- gp = sched.gfreeNoStack
- sched.gfreeNoStack = gp.schedlink.ptr()
- } else {
- break
+ if _p_.gFree.empty() && (!sched.gFree.stack.empty() || !sched.gFree.noStack.empty()) {
+ lock(&sched.gFree.lock)
+ // Move a batch of free Gs to the P.
+ for _p_.gFree.n < 32 {
+ // Prefer Gs with stacks.
+ gp := sched.gFree.stack.pop()
+ if gp == nil {
+ gp = sched.gFree.noStack.pop()
+ if gp == nil {
+ break
+ }
}
- _p_.gfreecnt++
- sched.ngfree--
- gp.schedlink.set(_p_.gfree)
- _p_.gfree = gp
+ sched.gFree.n--
+ _p_.gFree.push(gp)
+ _p_.gFree.n++
}
- unlock(&sched.gflock)
+ unlock(&sched.gFree.lock)
goto retry
}
- if gp != nil {
- _p_.gfree = gp.schedlink.ptr()
- _p_.gfreecnt--
- if gp.stack.lo == 0 {
- // Stack was deallocated in gfput. Allocate a new one.
- systemstack(func() {
- gp.stack = stackalloc(_FixedStack)
- })
- gp.stackguard0 = gp.stack.lo + _StackGuard
- } else {
- if raceenabled {
- racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
- }
- if msanenabled {
- msanmalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
- }
+ gp := _p_.gFree.pop()
+ if gp == nil {
+ return nil
+ }
+ _p_.gFree.n--
+ if gp.stack.lo == 0 {
+ // Stack was deallocated in gfput. Allocate a new one.
+ systemstack(func() {
+ gp.stack = stackalloc(_FixedStack)
+ })
+ gp.stackguard0 = gp.stack.lo + _StackGuard
+ } else {
+ if raceenabled {
+ racemalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
+ }
+ if msanenabled {
+ msanmalloc(unsafe.Pointer(gp.stack.lo), gp.stack.hi-gp.stack.lo)
}
}
return gp
@@ -3540,21 +3530,18 @@ retry:
// Purge all cached G's from gfree list to the global list.
func gfpurge(_p_ *p) {
- lock(&sched.gflock)
- for _p_.gfreecnt != 0 {
- _p_.gfreecnt--
- gp := _p_.gfree
- _p_.gfree = gp.schedlink.ptr()
+ lock(&sched.gFree.lock)
+ for !_p_.gFree.empty() {
+ gp := _p_.gFree.pop()
+ _p_.gFree.n--
if gp.stack.lo == 0 {
- gp.schedlink.set(sched.gfreeNoStack)
- sched.gfreeNoStack = gp
+ sched.gFree.noStack.push(gp)
} else {
- gp.schedlink.set(sched.gfreeStack)
- sched.gfreeStack = gp
+ sched.gFree.stack.push(gp)
}
- sched.ngfree++
+ sched.gFree.n++
}
- unlock(&sched.gflock)
+ unlock(&sched.gFree.lock)
}
// Breakpoint executes a breakpoint trap.
@@ -3667,9 +3654,9 @@ func badunlockosthread() {
}
func gcount() int32 {
- n := int32(allglen) - sched.ngfree - int32(atomic.Load(&sched.ngsys))
+ n := int32(allglen) - sched.gFree.n - int32(atomic.Load(&sched.ngsys))
for _, _p_ := range allp {
- n -= _p_.gfreecnt
+ n -= _p_.gFree.n
}
// All these variables can be changed concurrently, so the result can be inconsistent.
@@ -3715,7 +3702,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) {
// received from somewhere else (with _LostSIGPROFDuringAtomic64 as pc).
if GOARCH == "mips" || GOARCH == "mipsle" || GOARCH == "arm" {
if f := findfunc(pc); f.valid() {
- if hasprefix(funcname(f), "runtime/internal/atomic") {
+ if hasPrefix(funcname(f), "runtime/internal/atomic") {
lostAtomic64Count++
return
}
@@ -4386,8 +4373,8 @@ func sysmon() {
now := nanotime()
if netpollinited() && lastpoll != 0 && lastpoll+10*1000*1000 < now {
atomic.Cas64(&sched.lastpoll, uint64(lastpoll), uint64(now))
- gp := netpoll(false) // non-blocking - returns list of goroutines
- if gp != nil {
+ list := netpoll(false) // non-blocking - returns list of goroutines
+ if !list.empty() {
// Need to decrement number of idle locked M's
// (pretending that one more is running) before injectglist.
// Otherwise it can lead to the following situation:
@@ -4396,7 +4383,7 @@ func sysmon() {
// observes that there is no work to do and no other running M's
// and reports deadlock.
incidlelocked(-1)
- injectglist(gp)
+ injectglist(&list)
incidlelocked(1)
}
}
@@ -4411,8 +4398,9 @@ func sysmon() {
if t := (gcTrigger{kind: gcTriggerTime, now: now}); t.test() && atomic.Load(&forcegc.idle) != 0 {
lock(&forcegc.lock)
forcegc.idle = 0
- forcegc.g.schedlink = 0
- injectglist(forcegc.g)
+ var list gList
+ list.push(forcegc.g)
+ injectglist(&list)
unlock(&forcegc.lock)
}
// scavenge heap once in a while
@@ -4579,7 +4567,7 @@ func schedtrace(detailed bool) {
if mp != nil {
id = mp.id
}
- print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gfreecnt, "\n")
+ print(" P", i, ": status=", _p_.status, " schedtick=", _p_.schedtick, " syscalltick=", _p_.syscalltick, " m=", id, " runqsize=", t-h, " gfreecnt=", _p_.gFree.n, "\n")
} else {
// In non-detailed mode format lengths of per-P run queues as:
// [len1 len2 len3 len4]
@@ -4666,13 +4654,7 @@ func mget() *m {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrierrec
func globrunqput(gp *g) {
- gp.schedlink = 0
- if sched.runqtail != 0 {
- sched.runqtail.ptr().schedlink.set(gp)
- } else {
- sched.runqhead.set(gp)
- }
- sched.runqtail.set(gp)
+ sched.runq.pushBack(gp)
sched.runqsize++
}
@@ -4681,25 +4663,17 @@ func globrunqput(gp *g) {
// May run during STW, so write barriers are not allowed.
//go:nowritebarrierrec
func globrunqputhead(gp *g) {
- gp.schedlink = sched.runqhead
- sched.runqhead.set(gp)
- if sched.runqtail == 0 {
- sched.runqtail.set(gp)
- }
+ sched.runq.push(gp)
sched.runqsize++
}
// Put a batch of runnable goroutines on the global runnable queue.
+// This clears *batch.
// Sched must be locked.
-func globrunqputbatch(ghead *g, gtail *g, n int32) {
- gtail.schedlink = 0
- if sched.runqtail != 0 {
- sched.runqtail.ptr().schedlink.set(ghead)
- } else {
- sched.runqhead.set(ghead)
- }
- sched.runqtail.set(gtail)
+func globrunqputbatch(batch *gQueue, n int32) {
+ sched.runq.pushBackAll(*batch)
sched.runqsize += n
+ *batch = gQueue{}
}
// Try get a batch of G's from the global runnable queue.
@@ -4721,16 +4695,11 @@ func globrunqget(_p_ *p, max int32) *g {
}
sched.runqsize -= n
- if sched.runqsize == 0 {
- sched.runqtail = 0
- }
- gp := sched.runqhead.ptr()
- sched.runqhead = gp.schedlink
+ gp := sched.runq.pop()
n--
for ; n > 0; n-- {
- gp1 := sched.runqhead.ptr()
- sched.runqhead = gp1.schedlink
+ gp1 := sched.runq.pop()
runqput(_p_, gp1, false)
}
return gp
@@ -4858,10 +4827,13 @@ func runqputslow(_p_ *p, gp *g, h, t uint32) bool {
for i := uint32(0); i < n; i++ {
batch[i].schedlink.set(batch[i+1])
}
+ var q gQueue
+ q.head.set(batch[0])
+ q.tail.set(batch[n])
// Now put the batch on global queue.
lock(&sched.lock)
- globrunqputbatch(batch[0], batch[n], int32(n+1))
+ globrunqputbatch(&q, int32(n+1))
unlock(&sched.lock)
return true
}
@@ -4973,6 +4945,107 @@ func runqsteal(_p_, p2 *p, stealRunNextG bool) *g {
return gp
}
+// A gQueue is a dequeue of Gs linked through g.schedlink. A G can only
+// be on one gQueue or gList at a time.
+type gQueue struct {
+ head guintptr
+ tail guintptr
+}
+
+// empty returns true if q is empty.
+func (q *gQueue) empty() bool {
+ return q.head == 0
+}
+
+// push adds gp to the head of q.
+func (q *gQueue) push(gp *g) {
+ gp.schedlink = q.head
+ q.head.set(gp)
+ if q.tail == 0 {
+ q.tail.set(gp)
+ }
+}
+
+// pushBack adds gp to the tail of q.
+func (q *gQueue) pushBack(gp *g) {
+ gp.schedlink = 0
+ if q.tail != 0 {
+ q.tail.ptr().schedlink.set(gp)
+ } else {
+ q.head.set(gp)
+ }
+ q.tail.set(gp)
+}
+
+// pushBackAll adds all Gs in l2 to the tail of q. After this q2 must
+// not be used.
+func (q *gQueue) pushBackAll(q2 gQueue) {
+ if q2.tail == 0 {
+ return
+ }
+ q2.tail.ptr().schedlink = 0
+ if q.tail != 0 {
+ q.tail.ptr().schedlink = q2.head
+ } else {
+ q.head = q2.head
+ }
+ q.tail = q2.tail
+}
+
+// pop removes and returns the head of queue q. It returns nil if
+// q is empty.
+func (q *gQueue) pop() *g {
+ gp := q.head.ptr()
+ if gp != nil {
+ q.head = gp.schedlink
+ if q.head == 0 {
+ q.tail = 0
+ }
+ }
+ return gp
+}
+
+// popList takes all Gs in q and returns them as a gList.
+func (q *gQueue) popList() gList {
+ stack := gList{q.head}
+ *q = gQueue{}
+ return stack
+}
+
+// A gList is a list of Gs linked through g.schedlink. A G can only be
+// on one gQueue or gList at a time.
+type gList struct {
+ head guintptr
+}
+
+// empty returns true if l is empty.
+func (l *gList) empty() bool {
+ return l.head == 0
+}
+
+// push adds gp to the head of l.
+func (l *gList) push(gp *g) {
+ gp.schedlink = l.head
+ l.head.set(gp)
+}
+
+// pushAll prepends all Gs in q to l.
+func (l *gList) pushAll(q gQueue) {
+ if !q.empty() {
+ q.tail.ptr().schedlink = l.head
+ l.head = q.head
+ }
+}
+
+// pop removes and returns the head of l. If l is empty, it returns nil.
+func (l *gList) pop() *g {
+ gp := l.head.ptr()
+ if gp != nil {
+ l.head = gp.schedlink
+ }
+ return gp
+}
+
//go:linkname setMaxThreads runtime/debug.setMaxThreads
func setMaxThreads(in int) (out int) {
lock(&sched.lock)
generated by cgit v1.3-6-g1900 (git 2.53.0) at 2026-04-17 19:10:39 +0000