From e46c8e0558d287fcffde75bb458419288e71db62 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 19 Feb 2025 16:33:21 +0000 Subject: runtime: schedule cleanups across multiple goroutines This change splits the finalizer and cleanup queues and implements a new lock-free blocking queue for cleanups. The basic design is as follows: The cleanup queue is organized in fixed-sized blocks. Individual cleanup functions are queued, but only whole blocks are dequeued. Enqueuing cleanups places them in P-local cleanup blocks. These are flushed to the full list as they get full. Cleanups can only be enqueued by an active sweeper. Dequeuing cleanups always dequeues entire blocks from the full list. Cleanup blocks can be dequeued and executed at any time. The very last active sweeper in the sweep phase is responsible for flushing all local cleanup blocks to the full list. It can do this without any synchronization because the next GC can't start yet, so we can be very certain that nobody else will be accessing the local blocks. Cleanup blocks are stored off-heap because the need to be allocated by the sweeper, which is called from heap allocation paths. As a result, the GC treats cleanup blocks as roots, just like finalizer blocks. Flushes to the full list signal to the scheduler that cleanup goroutines should be awoken. Every time the scheduler goes to wake up a cleanup goroutine and there were more signals than goroutines to wake, it then forwards this signal to runtime.AddCleanup, so that it creates another goroutine the next time it is called, up to gomaxprocs goroutines. The signals here are a little convoluted, but exist because the sweeper and the scheduler cannot safely create new goroutines. For #71772. For #71825. Change-Id: Ie839fde2b67e1b79ac1426be0ea29a8d923a62cc Reviewed-on: https://go-review.googlesource.com/c/go/+/650697 Reviewed-by: Michael Pratt LUCI-TryBot-Result: Go LUCI Auto-Submit: Michael Knyszek --- src/runtime/traceback.go | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'src/runtime/traceback.go') diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index d6aa022674..276e601f7c 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -1131,9 +1131,9 @@ func showfuncinfo(sf srcFunc, firstFrame bool, calleeID abi.FuncID) bool { return false } - // Always show runtime.runFinalizersAndCleanups as context that this - // goroutine is running finalizers, otherwise there is no obvious - // indicator. + // Always show runtime.runFinalizers and runtime.runCleanups as + // context that this goroutine is running finalizers or cleanups, + // otherwise there is no obvious indicator. // // TODO(prattmic): A more general approach would be to always show the // outermost frame (besides runtime.goexit), even if it is a runtime. @@ -1142,8 +1142,8 @@ func showfuncinfo(sf srcFunc, firstFrame bool, calleeID abi.FuncID) bool { // // Unfortunately, implementing this requires looking ahead at the next // frame, which goes against traceback's incremental approach (see big - // coment in traceback1). - if sf.funcID == abi.FuncID_runFinalizersAndCleanups { + // comment in traceback1). + if sf.funcID == abi.FuncID_runFinalizers || sf.funcID == abi.FuncID_runCleanups { return true } @@ -1352,7 +1352,7 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { // in stack dumps and deadlock detector. This is any goroutine that // starts at a runtime.* entry point, except for runtime.main, // runtime.handleAsyncEvent (wasm only) and sometimes -// runtime.runFinalizersAndCleanups. +// runtime.runFinalizers/runtime.runCleanups. // // If fixed is true, any goroutine that can vary between user and // system (that is, the finalizer goroutine) is considered a user @@ -1366,7 +1366,7 @@ func isSystemGoroutine(gp *g, fixed bool) bool { if f.funcID == abi.FuncID_runtime_main || f.funcID == abi.FuncID_corostart || f.funcID == abi.FuncID_handleAsyncEvent { return false } - if f.funcID == abi.FuncID_runFinalizersAndCleanups { + if f.funcID == abi.FuncID_runFinalizers { // We include the finalizer goroutine if it's calling // back into user code. if fixed { @@ -1376,6 +1376,16 @@ func isSystemGoroutine(gp *g, fixed bool) bool { } return fingStatus.Load()&fingRunningFinalizer == 0 } + if f.funcID == abi.FuncID_runCleanups { + // We include the cleanup goroutines if they're calling + // back into user code. + if fixed { + // This goroutine can vary. In fixed mode, + // always consider it a user goroutine. + return false + } + return !gp.runningCleanups.Load() + } return stringslite.HasPrefix(funcname(f), "runtime.") } -- cgit v1.3