runtime: use type-based write barrier for remote stack write during chansend

A send on an unbuffered channel to a blocked receiver is the only
case in the runtime where one goroutine writes directly to the stack
of another. The garbage collector assumes that if a goroutine is
blocked, its stack contains no new pointers since the last time it ran.
The send on an unbuffered channel violates this, so it needs an
explicit write barrier. It has an explicit write barrier, but not one that
can handle a write to another stack. Use one that can (based on type bitmap
instead of heap bitmap).

To make this work, raise the limit for type bitmaps so that they are
used for all types up to 64 kB in size (256 bytes of bitmap).
(The runtime already imposes a limit of 64 kB for a channel element size.)

I have been unable to reproduce this problem in a simple test program.

Could help #11035.

Change-Id: I06ad994032d8cff3438c9b3eaa8d853915128af5
Reviewed-on: https://go-review.googlesource.com/10815
Reviewed-by: Austin Clements <austin@google.com>

2 files changed, 55 insertions, 1 deletions

diff --git a/src/runtime/chan.go b/src/runtime/chan.go
index 87e05bdf6c..a9eb83aeb3 100644
--- a/src/runtime/chan.go
+++ b/src/runtime/chan.go
@@ -165,7 +165,16 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
 
 			recvg := sg.g
 			if sg.elem != nil {
-				typedmemmove(c.elemtype, unsafe.Pointer(sg.elem), ep)
+				// This is the only place in the entire runtime where one goroutine
+				// writes to the stack of another goroutine. The GC assumes that
+				// stack writes only happen when the goroutine is running and are
+				// only done by that goroutine. Using a write barrier is sufficient to
+				// make up for violating that assumption, but the write barrier has to work.
+				// typedmemmove will call heapBitsBulkBarrier, but the target bytes
+				// are not in the heap, so that will not help. We arrange to call
+				// memmove and typeBitsBulkBarrier instead.
+				memmove(sg.elem, ep, c.elemtype.size)
+				typeBitsBulkBarrier(c.elemtype, uintptr(sg.elem), c.elemtype.size)
 				sg.elem = nil
 			}
 			recvg.param = unsafe.Pointer(sg)
diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index c97bf0a450..146ffbfcb6 100644
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -413,6 +413,51 @@ func heapBitsBulkBarrier(p, size uintptr) {
 	}
 }
 
+// typeBitsBulkBarrier executes writebarrierptr_nostore
+// for every pointer slot in the memory range [p, p+size),
+// using the type bitmap to locate those pointer slots.
+// The type typ must correspond exactly to [p, p+size).
+// This executes the write barriers necessary after a copy.
+// Both p and size must be pointer-aligned.
+// The type typ must have a plain bitmap, not a GC program.
+// The only use of this function is in channel sends, and the
+// 64 kB channel element limit takes care of this for us.
+//
+// Must not be preempted because it typically runs right after memmove,
+// and the GC must not complete between those two.
+//
+//go:nosplit
+func typeBitsBulkBarrier(typ *_type, p, size uintptr) {
+	if typ == nil {
+		throw("runtime: typeBitsBulkBarrier without type")
+	}
+	if typ.size != size {
+		println("runtime: typeBitsBulkBarrier with type ", *typ._string, " of size ", typ.size, " but memory size", size)
+		throw("runtime: invalid typeBitsBulkBarrier")
+	}
+	if typ.kind&kindGCProg != 0 {
+		println("runtime: typeBitsBulkBarrier with type ", *typ._string, " with GC prog")
+		throw("runtime: invalid typeBitsBulkBarrier")
+	}
+	if !writeBarrierEnabled {
+		return
+	}
+	ptrmask := typ.gcdata
+	var bits uint32
+	for i := uintptr(0); i < typ.ptrdata; i += ptrSize {
+		if i&(ptrSize*8-1) == 0 {
+			bits = uint32(*ptrmask)
+			ptrmask = addb(ptrmask, 1)
+		} else {
+			bits = bits >> 1
+		}
+		if bits&1 != 0 {
+			x := (*uintptr)(unsafe.Pointer(p + i))
+			writebarrierptr_nostore(x, *x)
+		}
+	}
+}
+
 // The methods operating on spans all require that h has been returned
 // by heapBitsForSpan and that size, n, total are the span layout description
 // returned by the mspan's layout method.