runtime: make work.spanSPMCs.all doubly-linked

Making this a doubly-linked list allows spanQueue.destroy to immediately
remove and free rings rather than simply marking them as dead and
waiting for the sweeper to deal with them.

For #75771.

Change-Id: I6a6a636c0fb6be08ee967cb6d8f0577511a33c13
Reviewed-on: https://go-review.googlesource.com/c/go/+/709657
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

1 files changed, 42 insertions, 19 deletions

diff --git a/src/runtime/mgcmark_greenteagc.go b/src/runtime/mgcmark_greenteagc.go
index 6ebd7ced81..7b78611cf7 100644
--- a/src/runtime/mgcmark_greenteagc.go
+++ b/src/runtime/mgcmark_greenteagc.go
@@ -635,14 +635,22 @@ func (q *spanQueue) destroy() {
 
 	lock(&work.spanSPMCs.lock)
 
-	// Mark each ring as dead. The sweeper will actually free them.
-	//
-	// N.B., we could free directly here, but work.spanSPMCs.all is a
-	// singly-linked list, so we'd need to walk the entire list to find the
-	// previous node. If the list becomes doubly-linked, we can free
-	// directly.
+	// Remove and free each ring.
 	for r := (*spanSPMC)(q.chain.tail.Load()); r != nil; r = (*spanSPMC)(r.prev.Load()) {
-		r.dead.Store(true)
+		prev := r.allprev
+		next := r.allnext
+		if prev != nil {
+			prev.allnext = next
+		}
+		if next != nil {
+			next.allprev = prev
+		}
+		if work.spanSPMCs.all == r {
+			work.spanSPMCs.all = next
+		}
+
+		r.deinit()
+		mheap_.spanSPMCAlloc.free(unsafe.Pointer(r))
 	}
 
 	q.chain.head = nil
@@ -685,6 +693,11 @@ type spanSPMC struct {
 	// work.spanSPMCs.lock.
 	allnext *spanSPMC
 
+	// allprev is the link to the previous spanSPMC on the work.spanSPMCs
+	// list. This is used to find and free dead spanSPMCs. Protected by
+	// work.spanSPMCs.lock.
+	allprev *spanSPMC
+
 	// dead indicates whether the spanSPMC is no longer in use.
 	// Protected by the CAS to the prev field of the spanSPMC pointing
 	// to this spanSPMC. That is, whoever wins that CAS takes ownership
@@ -711,7 +724,11 @@ type spanSPMC struct {
 func newSpanSPMC(cap uint32) *spanSPMC {
 	lock(&work.spanSPMCs.lock)
 	r := (*spanSPMC)(mheap_.spanSPMCAlloc.alloc())
-	r.allnext = work.spanSPMCs.all
+	next := work.spanSPMCs.all
+	r.allnext = next
+	if next != nil {
+		next.allprev = r
+	}
 	work.spanSPMCs.all = r
 	unlock(&work.spanSPMCs.lock)
 
@@ -748,6 +765,8 @@ func (r *spanSPMC) deinit() {
 	r.head.Store(0)
 	r.tail.Store(0)
 	r.cap = 0
+	r.allnext = nil
+	r.allprev = nil
 }
 
 // slot returns a pointer to slot i%r.cap.
@@ -780,22 +799,26 @@ func freeDeadSpanSPMCs() {
 		unlock(&work.spanSPMCs.lock)
 		return
 	}
-	rp := &work.spanSPMCs.all
-	for {
-		r := *rp
-		if r == nil {
-			break
-		}
+	r := work.spanSPMCs.all
+	for r != nil {
+		next := r.allnext
 		if r.dead.Load() {
 			// It's dead. Deinitialize and free it.
-			*rp = r.allnext
+			prev := r.allprev
+			if prev != nil {
+				prev.allnext = next
+			}
+			if next != nil {
+				next.allprev = prev
+			}
+			if work.spanSPMCs.all == r {
+				work.spanSPMCs.all = next
+			}
+
 			r.deinit()
 			mheap_.spanSPMCAlloc.free(unsafe.Pointer(r))
-		} else {
-			// Still alive, likely in some P's chain.
-			// Skip it.
-			rp = &r.allnext
 		}
+		r = next
 	}
 	unlock(&work.spanSPMCs.lock)
 }