runtime: break atomics out into package runtime/internal/atomic

This change breaks out most of the atomics functions in the runtime
into package runtime/internal/atomic. It adds some basic support
in the toolchain for runtime packages, and also modifies linux/arm
atomics to remove the dependency on the runtime's mutex. The mutexes
have been replaced with spinlocks.

all trybots are happy!
In addition to the trybots, I've tested on the darwin/arm64 builder,
on the darwin/arm builder, and on a ppc64le machine.

Change-Id: I6698c8e3cf3834f55ce5824059f44d00dc8e3c2f
Reviewed-on: https://go-review.googlesource.com/14204
Run-TryBot: Michael Matloob <matloob@golang.org>
Reviewed-by: Russ Cox <rsc@golang.org>

1 files changed, 17 insertions, 16 deletions

diff --git a/src/runtime/hashmap_fast.go b/src/runtime/hashmap_fast.go
index de9b267fde..9f310f8bf3 100644
--- a/src/runtime/hashmap_fast.go
+++ b/src/runtime/hashmap_fast.go
@@ -5,6 +5,7 @@
 package runtime
 
 import (
+	"runtime/internal/atomic"
 	"unsafe"
 )
 
@@ -14,7 +15,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
 		racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess1_fast32))
 	}
 	if h == nil || h.count == 0 {
-		return atomicloadp(unsafe.Pointer(&zeroptr))
+		return atomic.Loadp(unsafe.Pointer(&zeroptr))
 	}
 	var b *bmap
 	if h.B == 0 {
@@ -45,7 +46,7 @@ func mapaccess1_fast32(t *maptype, h *hmap, key uint32) unsafe.Pointer {
 		}
 		b = b.overflow(t)
 		if b == nil {
-			return atomicloadp(unsafe.Pointer(&zeroptr))
+			return atomic.Loadp(unsafe.Pointer(&zeroptr))
 		}
 	}
 }
@@ -56,7 +57,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
 		racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess2_fast32))
 	}
 	if h == nil || h.count == 0 {
-		return atomicloadp(unsafe.Pointer(&zeroptr)), false
+		return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 	}
 	var b *bmap
 	if h.B == 0 {
@@ -87,7 +88,7 @@ func mapaccess2_fast32(t *maptype, h *hmap, key uint32) (unsafe.Pointer, bool) {
 		}
 		b = b.overflow(t)
 		if b == nil {
-			return atomicloadp(unsafe.Pointer(&zeroptr)), false
+			return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 		}
 	}
 }
@@ -98,7 +99,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
 		racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess1_fast64))
 	}
 	if h == nil || h.count == 0 {
-		return atomicloadp(unsafe.Pointer(&zeroptr))
+		return atomic.Loadp(unsafe.Pointer(&zeroptr))
 	}
 	var b *bmap
 	if h.B == 0 {
@@ -129,7 +130,7 @@ func mapaccess1_fast64(t *maptype, h *hmap, key uint64) unsafe.Pointer {
 		}
 		b = b.overflow(t)
 		if b == nil {
-			return atomicloadp(unsafe.Pointer(&zeroptr))
+			return atomic.Loadp(unsafe.Pointer(&zeroptr))
 		}
 	}
 }
@@ -140,7 +141,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
 		racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess2_fast64))
 	}
 	if h == nil || h.count == 0 {
-		return atomicloadp(unsafe.Pointer(&zeroptr)), false
+		return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 	}
 	var b *bmap
 	if h.B == 0 {
@@ -171,7 +172,7 @@ func mapaccess2_fast64(t *maptype, h *hmap, key uint64) (unsafe.Pointer, bool) {
 		}
 		b = b.overflow(t)
 		if b == nil {
-			return atomicloadp(unsafe.Pointer(&zeroptr)), false
+			return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 		}
 	}
 }
@@ -182,7 +183,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
 		racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess1_faststr))
 	}
 	if h == nil || h.count == 0 {
-		return atomicloadp(unsafe.Pointer(&zeroptr))
+		return atomic.Loadp(unsafe.Pointer(&zeroptr))
 	}
 	key := stringStructOf(&ky)
 	if h.B == 0 {
@@ -203,7 +204,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
 					return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(t.valuesize))
 				}
 			}
-			return atomicloadp(unsafe.Pointer(&zeroptr))
+			return atomic.Loadp(unsafe.Pointer(&zeroptr))
 		}
 		// long key, try not to do more comparisons than necessary
 		keymaybe := uintptr(bucketCnt)
@@ -241,7 +242,7 @@ func mapaccess1_faststr(t *maptype, h *hmap, ky string) unsafe.Pointer {
 				return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+keymaybe*uintptr(t.valuesize))
 			}
 		}
-		return atomicloadp(unsafe.Pointer(&zeroptr))
+		return atomic.Loadp(unsafe.Pointer(&zeroptr))
 	}
 dohash:
 	hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
@@ -273,7 +274,7 @@ dohash:
 		}
 		b = b.overflow(t)
 		if b == nil {
-			return atomicloadp(unsafe.Pointer(&zeroptr))
+			return atomic.Loadp(unsafe.Pointer(&zeroptr))
 		}
 	}
 }
@@ -284,7 +285,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
 		racereadpc(unsafe.Pointer(h), callerpc, funcPC(mapaccess2_faststr))
 	}
 	if h == nil || h.count == 0 {
-		return atomicloadp(unsafe.Pointer(&zeroptr)), false
+		return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 	}
 	key := stringStructOf(&ky)
 	if h.B == 0 {
@@ -305,7 +306,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
 					return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+i*uintptr(t.valuesize)), true
 				}
 			}
-			return atomicloadp(unsafe.Pointer(&zeroptr)), false
+			return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 		}
 		// long key, try not to do more comparisons than necessary
 		keymaybe := uintptr(bucketCnt)
@@ -341,7 +342,7 @@ func mapaccess2_faststr(t *maptype, h *hmap, ky string) (unsafe.Pointer, bool) {
 				return add(unsafe.Pointer(b), dataOffset+bucketCnt*2*ptrSize+keymaybe*uintptr(t.valuesize)), true
 			}
 		}
-		return atomicloadp(unsafe.Pointer(&zeroptr)), false
+		return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 	}
 dohash:
 	hash := t.key.alg.hash(noescape(unsafe.Pointer(&ky)), uintptr(h.hash0))
@@ -373,7 +374,7 @@ dohash:
 		}
 		b = b.overflow(t)
 		if b == nil {
-			return atomicloadp(unsafe.Pointer(&zeroptr)), false
+			return atomic.Loadp(unsafe.Pointer(&zeroptr)), false
 		}
 	}
 }