runtime: block sweep completion on all sweep paths

The runtime currently has two different notions of sweep completion:

1. All spans are either swept or have begun sweeping.

2. The sweeper has *finished* sweeping all spans.

Most things depend on condition 1. Notably, GC correctness depends on
condition 1, but since all sweep operations a non-preemptible, the STW
at the beginning of GC forces condition 1 to become condition 2.

runtime.GC(), however, depends on condition 2, since the intent is to
complete a complete GC cycle, and also update the heap profile (which
can only be done after sweeping is complete).

However, the way we compute condition 2 is racy right now and may in
fact only indicate condition 1. Specifically, sweepone blocks
condition 2 until all sweepone calls are done, but there are many
other ways to enter the sweeper that don't block this. Hence, sweepone
may see that there are no more spans in the sweep list and see that
it's the last sweepone and declare sweeping done, while there's some
other sweeper still working on a span.

Fix this by making sure every entry to the sweeper participates in the
protocol that blocks condition 2. To make sure we get this right, this
CL introduces a type to track sweep blocking and (lightly) enforces
span sweep ownership via the type system. This has the nice
side-effect of abstracting the pattern of acquiring sweep ownership
that's currently repeated in many different places.

Fixes #45315.

Change-Id: I7fab30170c5ae14c8b2f10998628735b8be6d901
Reviewed-on: https://go-review.googlesource.com/c/go/+/307915
Trust: Austin Clements <austin@google.com>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

1 files changed, 15 insertions, 6 deletions

diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go
index 8664ed48ab..4eeac3be88 100644
--- a/src/runtime/mcentral.go
+++ b/src/runtime/mcentral.go
@@ -81,8 +81,6 @@ func (c *mcentral) cacheSpan() *mspan {
 	spanBytes := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) * _PageSize
 	deductSweepCredit(spanBytes, 0)
 
-	sg := mheap_.sweepgen
-
 	traceDone := false
 	if trace.enabled {
 		traceGCSweepStart()
@@ -104,6 +102,8 @@ func (c *mcentral) cacheSpan() *mspan {
 	spanBudget := 100
 
 	var s *mspan
+	sl := newSweepLocker()
+	sg := sl.sweepGen
 
 	// Try partial swept spans first.
 	if s = c.partialSwept(sg).pop(); s != nil {
@@ -116,9 +116,10 @@ func (c *mcentral) cacheSpan() *mspan {
 		if s == nil {
 			break
 		}
-		if atomic.Load(&s.sweepgen) == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) {
+		if s, ok := sl.tryAcquire(s); ok {
 			// We got ownership of the span, so let's sweep it and use it.
 			s.sweep(true)
+			sl.dispose()
 			goto havespan
 		}
 		// We failed to get ownership of the span, which means it's being or
@@ -135,20 +136,22 @@ func (c *mcentral) cacheSpan() *mspan {
 		if s == nil {
 			break
 		}
-		if atomic.Load(&s.sweepgen) == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) {
+		if s, ok := sl.tryAcquire(s); ok {
 			// We got ownership of the span, so let's sweep it.
 			s.sweep(true)
 			// Check if there's any free space.
 			freeIndex := s.nextFreeIndex()
 			if freeIndex != s.nelems {
 				s.freeindex = freeIndex
+				sl.dispose()
 				goto havespan
 			}
 			// Add it to the swept list, because sweeping didn't give us any free space.
-			c.fullSwept(sg).push(s)
+			c.fullSwept(sg).push(s.mspan)
 		}
 		// See comment for partial unswept spans.
 	}
+	sl.dispose()
 	if trace.enabled {
 		traceGCSweepDone()
 		traceDone = true
@@ -211,7 +214,13 @@ func (c *mcentral) uncacheSpan(s *mspan) {
 	if stale {
 		// It's stale, so just sweep it. Sweeping will put it on
 		// the right list.
-		s.sweep(false)
+		//
+		// We don't use a sweepLocker here. Stale cached spans
+		// aren't in the global sweep lists, so mark termination
+		// itself holds up sweep completion until all mcaches
+		// have been swept.
+		ss := sweepLocked{s}
+		ss.sweep(false)
 	} else {
 		if int(s.nelems)-int(s.allocCount) > 0 {
 			// Put it back on the partial swept list.