diff options
Diffstat (limited to 'src/runtime/mgcwork.go')
| -rw-r--r-- | src/runtime/mgcwork.go | 103 |
1 files changed, 13 insertions, 90 deletions
diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index 99771e2e57..c32c5eddd7 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -46,10 +46,7 @@ func init() { // // (preemption must be disabled) // gcw := &getg().m.p.ptr().gcw -// .. call gcw.put() to produce and gcw.get() to consume .. -// if gcBlackenPromptly { -// gcw.dispose() -// } +// .. call gcw.put() to produce and gcw.tryGet() to consume .. // // It's important that any use of gcWork during the mark phase prevent // the garbage collector from transitioning to mark termination since @@ -83,6 +80,12 @@ type gcWork struct { // Scan work performed on this gcWork. This is aggregated into // gcController by dispose and may also be flushed by callers. scanWork int64 + + // flushedWork indicates that a non-empty work buffer was + // flushed to the global work list since the last gcMarkDone + // termination check. Specifically, this indicates that this + // gcWork may have communicated work to another gcWork. + flushedWork bool } // Most of the methods of gcWork are go:nowritebarrierrec because the @@ -116,6 +119,7 @@ func (w *gcWork) put(obj uintptr) { wbuf = w.wbuf1 if wbuf.nobj == len(wbuf.obj) { putfull(wbuf) + w.flushedWork = true wbuf = getempty() w.wbuf1 = wbuf flushed = true @@ -169,6 +173,7 @@ func (w *gcWork) putBatch(obj []uintptr) { for len(obj) > 0 { for wbuf.nobj == len(wbuf.obj) { putfull(wbuf) + w.flushedWork = true w.wbuf1, w.wbuf2 = w.wbuf2, getempty() wbuf = w.wbuf1 flushed = true @@ -231,37 +236,6 @@ func (w *gcWork) tryGetFast() uintptr { return wbuf.obj[wbuf.nobj] } -// get dequeues a pointer for the garbage collector to trace, blocking -// if necessary to ensure all pointers from all queues and caches have -// been retrieved. get returns 0 if there are no pointers remaining. -//go:nowritebarrierrec -func (w *gcWork) get() uintptr { - wbuf := w.wbuf1 - if wbuf == nil { - w.init() - wbuf = w.wbuf1 - // wbuf is empty at this point. - } - if wbuf.nobj == 0 { - w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 - wbuf = w.wbuf1 - if wbuf.nobj == 0 { - owbuf := wbuf - wbuf = getfull() - if wbuf == nil { - return 0 - } - putempty(owbuf) - w.wbuf1 = wbuf - } - } - - // TODO: This might be a good place to add prefetch code - - wbuf.nobj-- - return wbuf.obj[wbuf.nobj] -} - // dispose returns any cached pointers to the global queue. // The buffers are being put on the full queue so that the // write barriers will not simply reacquire them before the @@ -275,6 +249,7 @@ func (w *gcWork) dispose() { putempty(wbuf) } else { putfull(wbuf) + w.flushedWork = true } w.wbuf1 = nil @@ -283,6 +258,7 @@ func (w *gcWork) dispose() { putempty(wbuf) } else { putfull(wbuf) + w.flushedWork = true } w.wbuf2 = nil } @@ -309,9 +285,11 @@ func (w *gcWork) balance() { } if wbuf := w.wbuf2; wbuf.nobj != 0 { putfull(wbuf) + w.flushedWork = true w.wbuf2 = getempty() } else if wbuf := w.wbuf1; wbuf.nobj > 4 { w.wbuf1 = handoff(wbuf) + w.flushedWork = true // handoff did putfull } else { return } @@ -440,61 +418,6 @@ func trygetfull() *workbuf { return b } -// Get a full work buffer off the work.full list. -// If nothing is available wait until all the other gc helpers have -// finished and then return nil. -// getfull acts as a barrier for work.nproc helpers. As long as one -// gchelper is actively marking objects it -// may create a workbuffer that the other helpers can work on. -// The for loop either exits when a work buffer is found -// or when _all_ of the work.nproc GC helpers are in the loop -// looking for work and thus not capable of creating new work. -// This is in fact the termination condition for the STW mark -// phase. -//go:nowritebarrier -func getfull() *workbuf { - b := (*workbuf)(work.full.pop()) - if b != nil { - b.checknonempty() - return b - } - - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait > work.nproc { - println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - for i := 0; ; i++ { - if work.full != 0 { - decnwait := atomic.Xadd(&work.nwait, -1) - if decnwait == work.nproc { - println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - b = (*workbuf)(work.full.pop()) - if b != nil { - b.checknonempty() - return b - } - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait > work.nproc { - println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - } - if work.nwait == work.nproc && work.markrootNext >= work.markrootJobs { - return nil - } - if i < 10 { - procyield(20) - } else if i < 20 { - osyield() - } else { - usleep(100) - } - } -} - //go:nowritebarrier func handoff(b *workbuf) *workbuf { // Make new buffer with half of b's pointers. |