runtime, sync/atomic: add write barrier for atomic write of pointer

Add write barrier to atomic operations manipulating pointers.

In general an atomic write of a pointer word may indicate racy accesses,
so there is no strictly safe way to attempt to keep the shadow copy
in sync with the real one. Instead, mark the shadow copy as not used.

Redirect sync/atomic pointer routines back to the runtime ones,
so that there is only one copy of the write barrier and shadow logic.
In time we might consider doing this for most of the sync/atomic
functions, but for now only the pointer routines need that treatment.

Found with GODEBUG=wbshadow=1 mode.
Eventually that will run automatically, but right now
it still detects other missing write barriers.

Change-Id: I852936b9a111a6cb9079cfaf6bd78b43016c0242
Reviewed-on: https://go-review.googlesource.com/2066
Reviewed-by: Rick Hudson <rlh@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

8 files changed, 2 insertions, 80 deletions

diff --git a/src/sync/atomic/asm_386.s b/src/sync/atomic/asm_386.s
index 740dfe76ba..383d759ae7 100644
--- a/src/sync/atomic/asm_386.s
+++ b/src/sync/atomic/asm_386.s
@@ -50,9 +50,6 @@ swaploop:
 TEXT ·SwapUintptr(SB),NOSPLIT,$0-12
 	JMP	·SwapUint32(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0-12
-	JMP	·SwapUint32(SB)
-
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-13
 	JMP	·CompareAndSwapUint32(SB)
 
@@ -69,9 +66,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-13
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0-13
 	JMP	·CompareAndSwapUint32(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0-13
-	JMP	·CompareAndSwapUint32(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0-21
 	JMP	·CompareAndSwapUint64(SB)
 
@@ -209,6 +203,3 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0-12
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0-8
 	JMP	·StoreUint32(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0-8
-	JMP	·StoreUint32(SB)
diff --git a/src/sync/atomic/asm_amd64.s b/src/sync/atomic/asm_amd64.s
index 6e53ebedd2..551c002674 100644
--- a/src/sync/atomic/asm_amd64.s
+++ b/src/sync/atomic/asm_amd64.s
@@ -29,9 +29,6 @@ TEXT ·SwapUint64(SB),NOSPLIT,$0-24
 TEXT ·SwapUintptr(SB),NOSPLIT,$0-24
 	JMP	·SwapUint64(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0-24
-	JMP	·SwapUint64(SB)
-
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-17
 	JMP	·CompareAndSwapUint32(SB)
 
@@ -47,9 +44,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-17
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0-25
 	JMP	·CompareAndSwapUint64(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0-25
-	JMP	·CompareAndSwapUint64(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0-25
 	JMP	·CompareAndSwapUint64(SB)
 
@@ -137,10 +131,4 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0-16
 	RET
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0-16
-	JMP	·StorePointer(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0-16
-	MOVQ	addr+0(FP), BP
-	MOVQ	val+8(FP), AX
-	XCHGQ	AX, 0(BP)
-	RET
+	JMP	·StoreUint64(SB)
diff --git a/src/sync/atomic/asm_amd64p32.s b/src/sync/atomic/asm_amd64p32.s
index d77cc2c088..b4e19ee257 100644
--- a/src/sync/atomic/asm_amd64p32.s
+++ b/src/sync/atomic/asm_amd64p32.s
@@ -30,9 +30,6 @@ TEXT ·SwapUint64(SB),NOSPLIT,$0-24
 TEXT ·SwapUintptr(SB),NOSPLIT,$0-12
 	JMP	·SwapUint32(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0-12
-	JMP	·SwapUint32(SB)
-
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-17
 	JMP	·CompareAndSwapUint32(SB)
 
@@ -48,9 +45,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-17
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0-17
 	JMP	·CompareAndSwapUint32(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0-17
-	JMP	·CompareAndSwapUint32(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0-25
 	JMP	·CompareAndSwapUint64(SB)
 
@@ -150,10 +144,4 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0-16
 	RET
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0-8
-	JMP	·StorePointer(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0-8
-	MOVL	addr+0(FP), BX
-	MOVL	val+4(FP), AX
-	XCHGL	AX, 0(BX)
-	RET
+	JMP	·StoreUint32(SB)
diff --git a/src/sync/atomic/asm_freebsd_arm.s b/src/sync/atomic/asm_freebsd_arm.s
index 06b975e897..46710eab01 100644
--- a/src/sync/atomic/asm_freebsd_arm.s
+++ b/src/sync/atomic/asm_freebsd_arm.s
@@ -16,9 +16,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint32(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0
-	B ·CompareAndSwapUint32(SB)
-
 TEXT ·AddInt32(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
@@ -37,9 +34,6 @@ TEXT ·SwapUint32(SB),NOSPLIT,$0
 TEXT ·SwapUintptr(SB),NOSPLIT,$0
 	B ·SwapUint32(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0
-	B ·SwapUint32(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -104,6 +98,3 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0
 	B ·StoreUint32(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0
-	B ·StoreUint32(SB)
diff --git a/src/sync/atomic/asm_linux_arm.s b/src/sync/atomic/asm_linux_arm.s
index 944758441a..b388e4c550 100644
--- a/src/sync/atomic/asm_linux_arm.s
+++ b/src/sync/atomic/asm_linux_arm.s
@@ -57,9 +57,6 @@ cascheck:
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0
 	B	·CompareAndSwapUint32(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0
-	B	·CompareAndSwapUint32(SB)
-
 TEXT ·AddInt32(SB),NOSPLIT,$0
 	B	·AddUint32(SB)
 
@@ -97,9 +94,6 @@ swaploop1:
 TEXT ·SwapUintptr(SB),NOSPLIT,$0
 	B	·SwapUint32(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0
-	B	·SwapUint32(SB)
-
 TEXT cas64<>(SB),NOSPLIT,$0
 	MOVW	$0xffff0f60, PC // __kuser_cmpxchg64: Linux-3.1 and above
 
@@ -211,6 +205,3 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0
 	B	·StoreUint32(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0
-	B	·StoreUint32(SB)
diff --git a/src/sync/atomic/asm_nacl_arm.s b/src/sync/atomic/asm_nacl_arm.s
index 76f6233364..8b4b6872a6 100644
--- a/src/sync/atomic/asm_nacl_arm.s
+++ b/src/sync/atomic/asm_nacl_arm.s
@@ -16,9 +16,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint32(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0
-	B ·CompareAndSwapUint32(SB)
-
 TEXT ·AddInt32(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
@@ -37,9 +34,6 @@ TEXT ·SwapUint32(SB),NOSPLIT,$0
 TEXT ·SwapUintptr(SB),NOSPLIT,$0
 	B ·SwapUint32(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0
-	B ·SwapUint32(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -104,6 +98,3 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0
 	B ·StoreUint32(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0
-	B ·StoreUint32(SB)
diff --git a/src/sync/atomic/asm_netbsd_arm.s b/src/sync/atomic/asm_netbsd_arm.s
index dbe80898fd..5c98de33c6 100644
--- a/src/sync/atomic/asm_netbsd_arm.s
+++ b/src/sync/atomic/asm_netbsd_arm.s
@@ -16,9 +16,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint32(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0
-	B ·CompareAndSwapUint32(SB)
-
 TEXT ·AddInt32(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
@@ -37,9 +34,6 @@ TEXT ·SwapUint32(SB),NOSPLIT,$0
 TEXT ·SwapUintptr(SB),NOSPLIT,$0
 	B ·SwapUint32(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0
-	B ·SwapUint32(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -104,6 +98,3 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0
 	B ·StoreUint32(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0
-	B ·StoreUint32(SB)
diff --git a/src/sync/atomic/asm_ppc64x.s b/src/sync/atomic/asm_ppc64x.s
index bcd46c5cff..00e1afb78f 100644
--- a/src/sync/atomic/asm_ppc64x.s
+++ b/src/sync/atomic/asm_ppc64x.s
@@ -39,9 +39,6 @@ TEXT ·SwapUint64(SB),NOSPLIT,$0-24
 TEXT ·SwapUintptr(SB),NOSPLIT,$0-24
 	BR	·SwapUint64(SB)
 
-TEXT ·SwapPointer(SB),NOSPLIT,$0-24
-	BR	·SwapUint64(SB)
-
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-17
 	BR	·CompareAndSwapUint32(SB)
 
@@ -66,9 +63,6 @@ TEXT ·CompareAndSwapUint32(SB),NOSPLIT,$0-17
 TEXT ·CompareAndSwapUintptr(SB),NOSPLIT,$0-25
 	BR	·CompareAndSwapUint64(SB)
 
-TEXT ·CompareAndSwapPointer(SB),NOSPLIT,$0-25
-	BR	·CompareAndSwapUint64(SB)
-
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0-25
 	BR	·CompareAndSwapUint64(SB)
 
@@ -178,7 +172,4 @@ TEXT ·StoreUint64(SB),NOSPLIT,$0-16
 	RETURN
 
 TEXT ·StoreUintptr(SB),NOSPLIT,$0-16
-	BR	·StorePointer(SB)
-
-TEXT ·StorePointer(SB),NOSPLIT,$0-16
 	BR	·StoreUint64(SB)