runtime: break atomics out into package runtime/internal/atomic

This change breaks out most of the atomics functions in the runtime
into package runtime/internal/atomic. It adds some basic support
in the toolchain for runtime packages, and also modifies linux/arm
atomics to remove the dependency on the runtime's mutex. The mutexes
have been replaced with spinlocks.

all trybots are happy!
In addition to the trybots, I've tested on the darwin/arm64 builder,
on the darwin/arm builder, and on a ppc64le machine.

Change-Id: I6698c8e3cf3834f55ce5824059f44d00dc8e3c2f
Reviewed-on: https://go-review.googlesource.com/14204
Run-TryBot: Michael Matloob <matloob@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

1 files changed, 13 insertions, 10 deletions

diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index 42afdf4390..11cb6e2c6c 100644
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -66,7 +66,10 @@
 
 package runtime
 
-import "unsafe"
+import (
+	"runtime/internal/atomic"
+	"unsafe"
+)
 
 const (
 	bitPointer = 1 << 0
@@ -302,7 +305,7 @@ func (h heapBits) setMarked() {
 	// Might be racing with other updates, so use atomic update always.
 	// We used to be clever here and use a non-atomic update in certain
 	// cases, but it's not worth the risk.
-	atomicor8(h.bitp, bitMarked<<h.shift)
+	atomic.Or8(h.bitp, bitMarked<<h.shift)
 }
 
 // setMarkedNonAtomic sets the marked bit in the heap bits, non-atomically.
@@ -367,10 +370,10 @@ func (h heapBits) isCheckmarked(size uintptr) bool {
 // h must describe the initial word of the object.
 func (h heapBits) setCheckmarked(size uintptr) {
 	if size == ptrSize {
-		atomicor8(h.bitp, bitPointer<<h.shift)
+		atomic.Or8(h.bitp, bitPointer<<h.shift)
 		return
 	}
-	atomicor8(h.bitp, bitMarked<<(heapBitsShift+h.shift))
+	atomic.Or8(h.bitp, bitMarked<<(heapBitsShift+h.shift))
 }
 
 // heapBitsBulkBarrier executes writebarrierptr_nostore
@@ -724,14 +727,14 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
 				if gcphase == _GCoff {
 					*h.bitp |= bitPointer << h.shift
 				} else {
-					atomicor8(h.bitp, bitPointer<<h.shift)
+					atomic.Or8(h.bitp, bitPointer<<h.shift)
 				}
 			} else {
 				// 2-element slice of pointer.
 				if gcphase == _GCoff {
 					*h.bitp |= (bitPointer | bitPointer<<heapBitsShift) << h.shift
 				} else {
-					atomicor8(h.bitp, (bitPointer|bitPointer<<heapBitsShift)<<h.shift)
+					atomic.Or8(h.bitp, (bitPointer|bitPointer<<heapBitsShift)<<h.shift)
 				}
 			}
 			return
@@ -748,7 +751,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
 		if gcphase == _GCoff {
 			*h.bitp |= uint8(hb << h.shift)
 		} else {
-			atomicor8(h.bitp, uint8(hb<<h.shift))
+			atomic.Or8(h.bitp, uint8(hb<<h.shift))
 		}
 		return
 	}
@@ -960,7 +963,7 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) {
 		if gcphase == _GCoff {
 			*hbitp |= uint8(hb)
 		} else {
-			atomicor8(hbitp, uint8(hb))
+			atomic.Or8(hbitp, uint8(hb))
 		}
 		hbitp = subtract1(hbitp)
 		if w += 2; w >= nw {
@@ -1080,8 +1083,8 @@ Phase3:
 		if gcphase == _GCoff {
 			*hbitp = *hbitp&^(bitPointer|bitMarked|(bitPointer|bitMarked)<<heapBitsShift) | uint8(hb)
 		} else {
-			atomicand8(hbitp, ^uint8(bitPointer|bitMarked|(bitPointer|bitMarked)<<heapBitsShift))
-			atomicor8(hbitp, uint8(hb))
+			atomic.And8(hbitp, ^uint8(bitPointer|bitMarked|(bitPointer|bitMarked)<<heapBitsShift))
+			atomic.Or8(hbitp, uint8(hb))
 		}
 	}