From a3bc52b7862621ea2ca246d6637d0490cae91988 Mon Sep 17 00:00:00 2001 From: ZhangYunHao Date: Wed, 14 Oct 2020 03:34:57 +0000 Subject: sync: fix typo in pooldequeue . Change-Id: I26fa26d67d01bcd583a1efaaf9a38398cbf793f7 GitHub-Last-Rev: ded020d02ca2e429f7c31065e5a27dae6eb7a611 GitHub-Pull-Request: golang/go#41932 Reviewed-on: https://go-review.googlesource.com/c/go/+/261477 Trust: Alberto Donizetti Reviewed-by: Austin Clements --- src/sync/poolqueue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/sync') diff --git a/src/sync/poolqueue.go b/src/sync/poolqueue.go index 22f74969d9..9be83e9a43 100644 --- a/src/sync/poolqueue.go +++ b/src/sync/poolqueue.go @@ -57,7 +57,7 @@ const dequeueBits = 32 // the index. We divide by 4 so this fits in an int on 32-bit. const dequeueLimit = (1 << dequeueBits) / 4 -// dequeueNil is used in poolDeqeue to represent interface{}(nil). +// dequeueNil is used in poolDequeue to represent interface{}(nil). // Since we use nil to represent empty slots, we need a sentinel value // to represent nil. type dequeueNil *struct{} -- cgit v1.3 From 83317d9e3cb0674f71d1118d8814aefb31ac1239 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 15 Oct 2020 15:52:58 -0400 Subject: runtime/internal/atomic: panic nicely on unaligned 64-bit atomics On 386 and arm, unaligned 64-bit atomics aren't safe, so we check for this and panic. Currently, we panic by dereferencing nil, which may be expedient but is pretty user-hostile since it gives no hint of what the actual problem was. This CL replaces this with an actual panic. The only subtlety here is now the atomic assembly implementations are calling back into Go, so they have to play nicely with stack maps and stack scanning. On 386, this just requires declaring NO_LOCAL_POINTERS. On arm, this is somewhat more complicated: first, we have to move the alignment check into the functions that have Go signatures. Then we have to support both the tail call from these functions to the underlying implementation (which requires that they have no frame) and the call into Go to panic (which requires that they have a frame). We resolve this by forcing them to have no frame and setting up the frame manually just before the panic call. Change-Id: I19f1e860045df64088013db37a18acea47342c69 Reviewed-on: https://go-review.googlesource.com/c/go/+/262778 Trust: Austin Clements Run-TryBot: Austin Clements TryBot-Result: Go Bot Reviewed-by: Cherry Zhang Reviewed-by: Michael Knyszek --- src/runtime/internal/atomic/asm_386.s | 17 ++++-- src/runtime/internal/atomic/asm_arm.s | 89 +++++++++++++++++++---------- src/runtime/internal/atomic/atomic_mipsx.go | 2 +- src/runtime/internal/atomic/atomic_test.go | 9 ++- src/runtime/internal/atomic/unaligned.go | 9 +++ src/sync/atomic/atomic_test.go | 9 ++- 6 files changed, 96 insertions(+), 39 deletions(-) create mode 100644 src/runtime/internal/atomic/unaligned.go (limited to 'src/sync') diff --git a/src/runtime/internal/atomic/asm_386.s b/src/runtime/internal/atomic/asm_386.s index 357ca95625..bcefff373f 100644 --- a/src/runtime/internal/atomic/asm_386.s +++ b/src/runtime/internal/atomic/asm_386.s @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. #include "textflag.h" +#include "funcdata.h" // bool Cas(int32 *val, int32 old, int32 new) // Atomically: @@ -44,7 +45,6 @@ TEXT ·Loadint64(SB), NOSPLIT, $0-12 TEXT ·Xaddint64(SB), NOSPLIT, $0-20 JMP ·Xadd64(SB) - // bool ·Cas64(uint64 *val, uint64 old, uint64 new) // Atomically: // if(*val == *old){ @@ -54,10 +54,11 @@ TEXT ·Xaddint64(SB), NOSPLIT, $0-20 // return 0; // } TEXT ·Cas64(SB), NOSPLIT, $0-21 + NO_LOCAL_POINTERS MOVL ptr+0(FP), BP TESTL $7, BP JZ 2(PC) - MOVL 0, BP // crash with nil ptr deref + CALL ·panicUnaligned(SB) MOVL old_lo+4(FP), AX MOVL old_hi+8(FP), DX MOVL new_lo+12(FP), BX @@ -98,11 +99,12 @@ TEXT ·Xadd(SB), NOSPLIT, $0-12 RET TEXT ·Xadd64(SB), NOSPLIT, $0-20 + NO_LOCAL_POINTERS // no XADDQ so use CMPXCHG8B loop MOVL ptr+0(FP), BP TESTL $7, BP JZ 2(PC) - MOVL 0, AX // crash when unaligned + CALL ·panicUnaligned(SB) // DI:SI = delta MOVL delta_lo+4(FP), SI MOVL delta_hi+8(FP), DI @@ -144,11 +146,12 @@ TEXT ·Xchguintptr(SB), NOSPLIT, $0-12 JMP ·Xchg(SB) TEXT ·Xchg64(SB),NOSPLIT,$0-20 + NO_LOCAL_POINTERS // no XCHGQ so use CMPXCHG8B loop MOVL ptr+0(FP), BP TESTL $7, BP JZ 2(PC) - MOVL 0, AX // crash when unaligned + CALL ·panicUnaligned(SB) // CX:BX = new MOVL new_lo+4(FP), BX MOVL new_hi+8(FP), CX @@ -188,10 +191,11 @@ TEXT ·StoreRel(SB), NOSPLIT, $0-8 // uint64 atomicload64(uint64 volatile* addr); TEXT ·Load64(SB), NOSPLIT, $0-12 + NO_LOCAL_POINTERS MOVL ptr+0(FP), AX TESTL $7, AX JZ 2(PC) - MOVL 0, AX // crash with nil ptr deref + CALL ·panicUnaligned(SB) MOVQ (AX), M0 MOVQ M0, ret+4(FP) EMMS @@ -199,10 +203,11 @@ TEXT ·Load64(SB), NOSPLIT, $0-12 // void ·Store64(uint64 volatile* addr, uint64 v); TEXT ·Store64(SB), NOSPLIT, $0-12 + NO_LOCAL_POINTERS MOVL ptr+0(FP), AX TESTL $7, AX JZ 2(PC) - MOVL 0, AX // crash with nil ptr deref + CALL ·panicUnaligned(SB) // MOVQ and EMMS were introduced on the Pentium MMX. MOVQ val+4(FP), M0 MOVQ M0, (AX) diff --git a/src/runtime/internal/atomic/asm_arm.s b/src/runtime/internal/atomic/asm_arm.s index db1267423d..c3d1d9025d 100644 --- a/src/runtime/internal/atomic/asm_arm.s +++ b/src/runtime/internal/atomic/asm_arm.s @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. #include "textflag.h" +#include "funcdata.h" // bool armcas(int32 *val, int32 old, int32 new) // Atomically: @@ -96,11 +97,7 @@ TEXT ·Xaddint64(SB),NOSPLIT,$0-20 // atomics with locks. TEXT armCas64<>(SB),NOSPLIT,$0-21 - MOVW addr+0(FP), R1 - // make unaligned atomic access panic - AND.S $7, R1, R2 - BEQ 2(PC) - MOVW R2, (R2) // crash. AND.S above left only low 3 bits in R2. + // addr is already in R1 MOVW old_lo+4(FP), R2 MOVW old_hi+8(FP), R3 MOVW new_lo+12(FP), R4 @@ -129,11 +126,7 @@ cas64fail: RET TEXT armXadd64<>(SB),NOSPLIT,$0-20 - MOVW addr+0(FP), R1 - // make unaligned atomic access panic - AND.S $7, R1, R2 - BEQ 2(PC) - MOVW R2, (R2) // crash. AND.S above left only low 3 bits in R2. + // addr is already in R1 MOVW delta_lo+4(FP), R2 MOVW delta_hi+8(FP), R3 @@ -155,11 +148,7 @@ add64loop: RET TEXT armXchg64<>(SB),NOSPLIT,$0-20 - MOVW addr+0(FP), R1 - // make unaligned atomic access panic - AND.S $7, R1, R2 - BEQ 2(PC) - MOVW R2, (R2) // crash. AND.S above left only low 3 bits in R2. + // addr is already in R1 MOVW new_lo+4(FP), R2 MOVW new_hi+8(FP), R3 @@ -179,11 +168,7 @@ swap64loop: RET TEXT armLoad64<>(SB),NOSPLIT,$0-12 - MOVW addr+0(FP), R1 - // make unaligned atomic access panic - AND.S $7, R1, R2 - BEQ 2(PC) - MOVW R2, (R2) // crash. AND.S above left only low 3 bits in R2. + // addr is already in R1 LDREXD (R1), R2 // loads R2 and R3 DMB MB_ISH @@ -193,11 +178,7 @@ TEXT armLoad64<>(SB),NOSPLIT,$0-12 RET TEXT armStore64<>(SB),NOSPLIT,$0-12 - MOVW addr+0(FP), R1 - // make unaligned atomic access panic - AND.S $7, R1, R2 - BEQ 2(PC) - MOVW R2, (R2) // crash. AND.S above left only low 3 bits in R2. + // addr is already in R1 MOVW val_lo+4(FP), R2 MOVW val_hi+8(FP), R3 @@ -213,35 +194,83 @@ store64loop: DMB MB_ISH RET -TEXT ·Cas64(SB),NOSPLIT,$0-21 +// The following functions all panic if their address argument isn't +// 8-byte aligned. Since we're calling back into Go code to do this, +// we have to cooperate with stack unwinding. In the normal case, the +// functions tail-call into the appropriate implementation, which +// means they must not open a frame. Hence, when they go down the +// panic path, at that point they push the LR to create a real frame +// (they don't need to pop it because panic won't return). + +TEXT ·Cas64(SB),NOSPLIT,$-4-21 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + // make unaligned atomic access panic + AND.S $7, R1, R2 + BEQ 3(PC) + MOVW.W R14, -4(R13) // prepare a real frame + BL ·panicUnaligned(SB) + MOVB runtime·goarm(SB), R11 CMP $7, R11 BLT 2(PC) JMP armCas64<>(SB) JMP ·goCas64(SB) -TEXT ·Xadd64(SB),NOSPLIT,$0-20 +TEXT ·Xadd64(SB),NOSPLIT,$-4-20 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + // make unaligned atomic access panic + AND.S $7, R1, R2 + BEQ 3(PC) + MOVW.W R14, -4(R13) // prepare a real frame + BL ·panicUnaligned(SB) + MOVB runtime·goarm(SB), R11 CMP $7, R11 BLT 2(PC) JMP armXadd64<>(SB) JMP ·goXadd64(SB) -TEXT ·Xchg64(SB),NOSPLIT,$0-20 +TEXT ·Xchg64(SB),NOSPLIT,$-4-20 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + // make unaligned atomic access panic + AND.S $7, R1, R2 + BEQ 3(PC) + MOVW.W R14, -4(R13) // prepare a real frame + BL ·panicUnaligned(SB) + MOVB runtime·goarm(SB), R11 CMP $7, R11 BLT 2(PC) JMP armXchg64<>(SB) JMP ·goXchg64(SB) -TEXT ·Load64(SB),NOSPLIT,$0-12 +TEXT ·Load64(SB),NOSPLIT,$-4-12 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + // make unaligned atomic access panic + AND.S $7, R1, R2 + BEQ 3(PC) + MOVW.W R14, -4(R13) // prepare a real frame + BL ·panicUnaligned(SB) + MOVB runtime·goarm(SB), R11 CMP $7, R11 BLT 2(PC) JMP armLoad64<>(SB) JMP ·goLoad64(SB) -TEXT ·Store64(SB),NOSPLIT,$0-12 +TEXT ·Store64(SB),NOSPLIT,$-4-12 + NO_LOCAL_POINTERS + MOVW addr+0(FP), R1 + // make unaligned atomic access panic + AND.S $7, R1, R2 + BEQ 3(PC) + MOVW.W R14, -4(R13) // prepare a real frame + BL ·panicUnaligned(SB) + MOVB runtime·goarm(SB), R11 CMP $7, R11 BLT 2(PC) diff --git a/src/runtime/internal/atomic/atomic_mipsx.go b/src/runtime/internal/atomic/atomic_mipsx.go index 0e2d77ade1..b99bfe7dbf 100644 --- a/src/runtime/internal/atomic/atomic_mipsx.go +++ b/src/runtime/internal/atomic/atomic_mipsx.go @@ -34,7 +34,7 @@ func spinUnlock(state *uint32) func lockAndCheck(addr *uint64) { // ensure 8-byte alignment if uintptr(unsafe.Pointer(addr))&7 != 0 { - addr = nil + panicUnaligned() } // force dereference before taking lock _ = *addr diff --git a/src/runtime/internal/atomic/atomic_test.go b/src/runtime/internal/atomic/atomic_test.go index b0a8fa0610..a9f95077c0 100644 --- a/src/runtime/internal/atomic/atomic_test.go +++ b/src/runtime/internal/atomic/atomic_test.go @@ -73,8 +73,15 @@ func TestXadduintptrOnUint64(t *testing.T) { func shouldPanic(t *testing.T, name string, f func()) { defer func() { - if recover() == nil { + // Check that all GC maps are sane. + runtime.GC() + + err := recover() + want := "unaligned 64-bit atomic operation" + if err == nil { t.Errorf("%s did not panic", name) + } else if s, _ := err.(string); s != want { + t.Errorf("%s: wanted panic %q, got %q", name, want, err) } }() f() diff --git a/src/runtime/internal/atomic/unaligned.go b/src/runtime/internal/atomic/unaligned.go new file mode 100644 index 0000000000..a859de4144 --- /dev/null +++ b/src/runtime/internal/atomic/unaligned.go @@ -0,0 +1,9 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +func panicUnaligned() { + panic("unaligned 64-bit atomic operation") +} diff --git a/src/sync/atomic/atomic_test.go b/src/sync/atomic/atomic_test.go index 83e7c8d763..eadc962f70 100644 --- a/src/sync/atomic/atomic_test.go +++ b/src/sync/atomic/atomic_test.go @@ -1397,8 +1397,15 @@ func TestStoreLoadRelAcq64(t *testing.T) { func shouldPanic(t *testing.T, name string, f func()) { defer func() { - if recover() == nil { + // Check that all GC maps are sane. + runtime.GC() + + err := recover() + want := "unaligned 64-bit atomic operation" + if err == nil { t.Errorf("%s did not panic", name) + } else if s, _ := err.(string); s != want { + t.Errorf("%s: wanted panic %q, got %q", name, want, err) } }() f() -- cgit v1.3 From 15ead857dbc638b9d83a7686acf0dc746fc45918 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Wed, 9 Sep 2020 17:24:23 -0500 Subject: cmd/compiler,cmd/go,sync: add internal {LoadAcq,StoreRel}64 on ppc64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an internal atomic intrinsic for load with acquire semantics (extending LoadAcq to 64b) and add LoadAcquintptr for internal use within the sync package. For other arches, this remaps to the appropriate atomic.Load{,64} intrinsic which should not alter code generation. Similarly, add StoreRel{uintptr,64} for consistency, and inline. Finally, add an exception to allow sync to directly use the runtime/internal/atomic package which avoids more convoluted workarounds (contributed by Lynn Boger). In an extreme example, sync.(*Pool).pin consumes 20% of wall time during fmt tests. This is reduced to 5% on ppc64le/power9. From the fmt benchmarks on ppc64le: name old time/op new time/op delta SprintfPadding 468ns ± 0% 451ns ± 0% -3.63% SprintfEmpty 73.3ns ± 0% 51.9ns ± 0% -29.20% SprintfString 135ns ± 0% 122ns ± 0% -9.63% SprintfTruncateString 232ns ± 0% 214ns ± 0% -7.76% SprintfTruncateBytes 216ns ± 0% 202ns ± 0% -6.48% SprintfSlowParsingPath 162ns ± 0% 142ns ± 0% -12.35% SprintfQuoteString 1.00µs ± 0% 0.99µs ± 0% -1.39% SprintfInt 117ns ± 0% 104ns ± 0% -11.11% SprintfIntInt 190ns ± 0% 175ns ± 0% -7.89% SprintfPrefixedInt 232ns ± 0% 212ns ± 0% -8.62% SprintfFloat 270ns ± 0% 255ns ± 0% -5.56% SprintfComplex 1.01µs ± 0% 0.99µs ± 0% -1.68% SprintfBoolean 127ns ± 0% 111ns ± 0% -12.60% SprintfHexString 220ns ± 0% 198ns ± 0% -10.00% SprintfHexBytes 261ns ± 0% 252ns ± 0% -3.45% SprintfBytes 600ns ± 0% 590ns ± 0% -1.67% SprintfStringer 684ns ± 0% 658ns ± 0% -3.80% SprintfStructure 2.57µs ± 0% 2.57µs ± 0% -0.12% ManyArgs 669ns ± 0% 646ns ± 0% -3.44% FprintInt 140ns ± 0% 136ns ± 0% -2.86% FprintfBytes 184ns ± 0% 181ns ± 0% -1.63% FprintIntNoAlloc 140ns ± 0% 136ns ± 0% -2.86% ScanInts 929µs ± 0% 921µs ± 0% -0.79% ScanRecursiveInt 122ms ± 0% 121ms ± 0% -0.11% ScanRecursiveIntReaderWrapper 122ms ± 0% 122ms ± 0% -0.18% Change-Id: I4d66780261b57b06ef600229e475462e7313f0d6 Reviewed-on: https://go-review.googlesource.com/c/go/+/253748 Run-TryBot: Lynn Boger Reviewed-by: Lynn Boger Reviewed-by: Keith Randall Trust: Lynn Boger TryBot-Result: Go Bot --- src/cmd/compile/internal/gc/ssa.go | 19 ++++++++++++++ src/cmd/compile/internal/ssa/branchelim.go | 2 +- src/cmd/compile/internal/ssa/gen/PPC64.rules | 4 +-- src/cmd/compile/internal/ssa/gen/genericOps.go | 2 ++ src/cmd/compile/internal/ssa/opGen.go | 13 ++++++++++ src/cmd/compile/internal/ssa/rewritePPC64.go | 34 ++++++++++++++++++++++++++ src/cmd/go/internal/load/pkg.go | 5 ++++ src/runtime/internal/atomic/asm_386.s | 3 +++ src/runtime/internal/atomic/asm_amd64.s | 6 +++++ src/runtime/internal/atomic/asm_arm.s | 6 +++++ src/runtime/internal/atomic/asm_mips64x.s | 6 +++++ src/runtime/internal/atomic/asm_mipsx.s | 3 +++ src/runtime/internal/atomic/asm_ppc64x.s | 13 ++++++++++ src/runtime/internal/atomic/atomic_386.go | 9 +++++++ src/runtime/internal/atomic/atomic_amd64.go | 18 ++++++++++++++ src/runtime/internal/atomic/atomic_arm.go | 6 +++++ src/runtime/internal/atomic/atomic_arm64.go | 12 +++++++++ src/runtime/internal/atomic/atomic_arm64.s | 14 +++++++++++ src/runtime/internal/atomic/atomic_mips64x.go | 12 +++++++++ src/runtime/internal/atomic/atomic_mips64x.s | 8 ++++++ src/runtime/internal/atomic/atomic_mipsx.go | 6 +++++ src/runtime/internal/atomic/atomic_ppc64x.go | 12 +++++++++ src/runtime/internal/atomic/atomic_ppc64x.s | 20 +++++++++++++++ src/runtime/internal/atomic/atomic_riscv64.go | 12 +++++++++ src/runtime/internal/atomic/atomic_riscv64.s | 12 +++++++++ src/runtime/internal/atomic/atomic_s390x.go | 24 ++++++++++++++++++ src/runtime/internal/atomic/atomic_wasm.go | 24 ++++++++++++++++++ src/sync/pool.go | 15 ++++++------ 28 files changed, 310 insertions(+), 10 deletions(-) (limited to 'src/sync') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index d8f627c213..e1455d2c3f 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -3389,6 +3389,13 @@ func init() { return s.newValue1(ssa.OpSelect0, types.Types[TUINT32], v) }, sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "LoadAcq64", + func(s *state, n *Node, args []*ssa.Value) *ssa.Value { + v := s.newValue2(ssa.OpAtomicLoadAcq64, types.NewTuple(types.Types[TUINT64], types.TypeMem), args[0], s.mem()) + s.vars[&memVar] = s.newValue1(ssa.OpSelect1, types.TypeMem, v) + return s.newValue1(ssa.OpSelect0, types.Types[TUINT64], v) + }, + sys.PPC64) addF("runtime/internal/atomic", "Loadp", func(s *state, n *Node, args []*ssa.Value) *ssa.Value { v := s.newValue2(ssa.OpAtomicLoadPtr, types.NewTuple(s.f.Config.Types.BytePtr, types.TypeMem), args[0], s.mem()) @@ -3427,6 +3434,12 @@ func init() { return nil }, sys.PPC64, sys.S390X) + addF("runtime/internal/atomic", "StoreRel64", + func(s *state, n *Node, args []*ssa.Value) *ssa.Value { + s.vars[&memVar] = s.newValue3(ssa.OpAtomicStoreRel64, types.TypeMem, args[0], args[1], s.mem()) + return nil + }, + sys.PPC64) addF("runtime/internal/atomic", "Xchg", func(s *state, n *Node, args []*ssa.Value) *ssa.Value { @@ -3542,9 +3555,15 @@ func init() { alias("runtime/internal/atomic", "Loaduintptr", "runtime/internal/atomic", "Load", p4...) alias("runtime/internal/atomic", "Loaduintptr", "runtime/internal/atomic", "Load64", p8...) alias("runtime/internal/atomic", "LoadAcq", "runtime/internal/atomic", "Load", lwatomics...) + alias("runtime/internal/atomic", "LoadAcq64", "runtime/internal/atomic", "Load64", lwatomics...) + alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) + alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store", p4...) alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store64", p8...) alias("runtime/internal/atomic", "StoreRel", "runtime/internal/atomic", "Store", lwatomics...) + alias("runtime/internal/atomic", "StoreRel64", "runtime/internal/atomic", "Store64", lwatomics...) + alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) + alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg", p4...) alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg64", p8...) alias("runtime/internal/atomic", "Xadduintptr", "runtime/internal/atomic", "Xadd", p4...) diff --git a/src/cmd/compile/internal/ssa/branchelim.go b/src/cmd/compile/internal/ssa/branchelim.go index 4f9fd8e22e..1d34f8160b 100644 --- a/src/cmd/compile/internal/ssa/branchelim.go +++ b/src/cmd/compile/internal/ssa/branchelim.go @@ -35,7 +35,7 @@ func branchelim(f *Func) { for _, b := range f.Blocks { for _, v := range b.Values { switch v.Op { - case OpLoad, OpAtomicLoad8, OpAtomicLoad32, OpAtomicLoad64, OpAtomicLoadPtr, OpAtomicLoadAcq32: + case OpLoad, OpAtomicLoad8, OpAtomicLoad32, OpAtomicLoad64, OpAtomicLoadPtr, OpAtomicLoadAcq32, OpAtomicLoadAcq64: loadAddr.add(v.Args[0].ID) case OpMove: loadAddr.add(v.Args[1].ID) diff --git a/src/cmd/compile/internal/ssa/gen/PPC64.rules b/src/cmd/compile/internal/ssa/gen/PPC64.rules index a05cfee654..11b1a318fe 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64.rules +++ b/src/cmd/compile/internal/ssa/gen/PPC64.rules @@ -967,10 +967,10 @@ // atomic intrinsics (AtomicLoad(8|32|64|Ptr) ptr mem) => (LoweredAtomicLoad(8|32|64|Ptr) [1] ptr mem) -(AtomicLoadAcq32 ptr mem) => (LoweredAtomicLoad32 [0] ptr mem) +(AtomicLoadAcq(32|64) ptr mem) => (LoweredAtomicLoad(32|64) [0] ptr mem) (AtomicStore(8|32|64) ptr val mem) => (LoweredAtomicStore(8|32|64) [1] ptr val mem) -(AtomicStoreRel32 ptr val mem) => (LoweredAtomicStore32 [0] ptr val mem) +(AtomicStoreRel(32|64) ptr val mem) => (LoweredAtomicStore(32|64) [0] ptr val mem) //(AtomicStorePtrNoWB ptr val mem) => (STLR ptr val mem) (AtomicExchange(32|64) ...) => (LoweredAtomicExchange(32|64) ...) diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go index 85839303c5..12ba9f1fc9 100644 --- a/src/cmd/compile/internal/ssa/gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/gen/genericOps.go @@ -550,11 +550,13 @@ var genericOps = []opData{ {name: "AtomicLoad64", argLength: 2, typ: "(UInt64,Mem)"}, // Load from arg0. arg1=memory. Returns loaded value and new memory. {name: "AtomicLoadPtr", argLength: 2, typ: "(BytePtr,Mem)"}, // Load from arg0. arg1=memory. Returns loaded value and new memory. {name: "AtomicLoadAcq32", argLength: 2, typ: "(UInt32,Mem)"}, // Load from arg0. arg1=memory. Lock acquisition, returns loaded value and new memory. + {name: "AtomicLoadAcq64", argLength: 2, typ: "(UInt64,Mem)"}, // Load from arg0. arg1=memory. Lock acquisition, returns loaded value and new memory. {name: "AtomicStore8", argLength: 3, typ: "Mem", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Returns memory. {name: "AtomicStore32", argLength: 3, typ: "Mem", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Returns memory. {name: "AtomicStore64", argLength: 3, typ: "Mem", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Returns memory. {name: "AtomicStorePtrNoWB", argLength: 3, typ: "Mem", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Returns memory. {name: "AtomicStoreRel32", argLength: 3, typ: "Mem", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Lock release, returns memory. + {name: "AtomicStoreRel64", argLength: 3, typ: "Mem", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Lock release, returns memory. {name: "AtomicExchange32", argLength: 3, typ: "(UInt32,Mem)", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Returns old contents of *arg0 and new memory. {name: "AtomicExchange64", argLength: 3, typ: "(UInt64,Mem)", hasSideEffects: true}, // Store arg1 to *arg0. arg2=memory. Returns old contents of *arg0 and new memory. {name: "AtomicAdd32", argLength: 3, typ: "(UInt32,Mem)", hasSideEffects: true}, // Do *arg0 += arg1. arg2=memory. Returns sum and new memory. diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 051550fb17..eae30c79ba 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -2839,11 +2839,13 @@ const ( OpAtomicLoad64 OpAtomicLoadPtr OpAtomicLoadAcq32 + OpAtomicLoadAcq64 OpAtomicStore8 OpAtomicStore32 OpAtomicStore64 OpAtomicStorePtrNoWB OpAtomicStoreRel32 + OpAtomicStoreRel64 OpAtomicExchange32 OpAtomicExchange64 OpAtomicAdd32 @@ -35429,6 +35431,11 @@ var opcodeTable = [...]opInfo{ argLen: 2, generic: true, }, + { + name: "AtomicLoadAcq64", + argLen: 2, + generic: true, + }, { name: "AtomicStore8", argLen: 3, @@ -35459,6 +35466,12 @@ var opcodeTable = [...]opInfo{ hasSideEffects: true, generic: true, }, + { + name: "AtomicStoreRel64", + argLen: 3, + hasSideEffects: true, + generic: true, + }, { name: "AtomicExchange32", argLen: 3, diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go index 1b8a5a78ca..a820bc0c4e 100644 --- a/src/cmd/compile/internal/ssa/rewritePPC64.go +++ b/src/cmd/compile/internal/ssa/rewritePPC64.go @@ -82,6 +82,8 @@ func rewriteValuePPC64(v *Value) bool { return rewriteValuePPC64_OpAtomicLoad8(v) case OpAtomicLoadAcq32: return rewriteValuePPC64_OpAtomicLoadAcq32(v) + case OpAtomicLoadAcq64: + return rewriteValuePPC64_OpAtomicLoadAcq64(v) case OpAtomicLoadPtr: return rewriteValuePPC64_OpAtomicLoadPtr(v) case OpAtomicOr8: @@ -95,6 +97,8 @@ func rewriteValuePPC64(v *Value) bool { return rewriteValuePPC64_OpAtomicStore8(v) case OpAtomicStoreRel32: return rewriteValuePPC64_OpAtomicStoreRel32(v) + case OpAtomicStoreRel64: + return rewriteValuePPC64_OpAtomicStoreRel64(v) case OpAvg64u: return rewriteValuePPC64_OpAvg64u(v) case OpBitLen32: @@ -930,6 +934,20 @@ func rewriteValuePPC64_OpAtomicLoadAcq32(v *Value) bool { return true } } +func rewriteValuePPC64_OpAtomicLoadAcq64(v *Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (AtomicLoadAcq64 ptr mem) + // result: (LoweredAtomicLoad64 [0] ptr mem) + for { + ptr := v_0 + mem := v_1 + v.reset(OpPPC64LoweredAtomicLoad64) + v.AuxInt = int64ToAuxInt(0) + v.AddArg2(ptr, mem) + return true + } +} func rewriteValuePPC64_OpAtomicLoadPtr(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -1008,6 +1026,22 @@ func rewriteValuePPC64_OpAtomicStoreRel32(v *Value) bool { return true } } +func rewriteValuePPC64_OpAtomicStoreRel64(v *Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (AtomicStoreRel64 ptr val mem) + // result: (LoweredAtomicStore64 [0] ptr val mem) + for { + ptr := v_0 + val := v_1 + mem := v_2 + v.reset(OpPPC64LoweredAtomicStore64) + v.AuxInt = int64ToAuxInt(0) + v.AddArg3(ptr, val, mem) + return true + } +} func rewriteValuePPC64_OpAvg64u(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 2bdc08ba36..c9665265e9 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1337,6 +1337,11 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * return p } + // Allow sync package to access lightweight atomic functions limited to the runtime. + if p.Standard && strings.HasPrefix(importerPath, "sync") && p.ImportPath == "runtime/internal/atomic" { + return p + } + // Internal is present. // Map import path back to directory corresponding to parent of internal. if i > 0 { diff --git a/src/runtime/internal/atomic/asm_386.s b/src/runtime/internal/atomic/asm_386.s index bcefff373f..7ebf675ac5 100644 --- a/src/runtime/internal/atomic/asm_386.s +++ b/src/runtime/internal/atomic/asm_386.s @@ -189,6 +189,9 @@ TEXT ·Store(SB), NOSPLIT, $0-8 TEXT ·StoreRel(SB), NOSPLIT, $0-8 JMP ·Store(SB) +TEXT runtime∕internal∕atomic·StoreReluintptr(SB), NOSPLIT, $0-8 + JMP runtime∕internal∕atomic·Store(SB) + // uint64 atomicload64(uint64 volatile* addr); TEXT ·Load64(SB), NOSPLIT, $0-12 NO_LOCAL_POINTERS diff --git a/src/runtime/internal/atomic/asm_amd64.s b/src/runtime/internal/atomic/asm_amd64.s index 90c56424c9..80fb31285d 100644 --- a/src/runtime/internal/atomic/asm_amd64.s +++ b/src/runtime/internal/atomic/asm_amd64.s @@ -136,6 +136,12 @@ TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 TEXT runtime∕internal∕atomic·StoreRel(SB), NOSPLIT, $0-12 JMP runtime∕internal∕atomic·Store(SB) +TEXT runtime∕internal∕atomic·StoreRel64(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·StoreReluintptr(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Store64(SB) + TEXT runtime∕internal∕atomic·Store8(SB), NOSPLIT, $0-9 MOVQ ptr+0(FP), BX MOVB val+8(FP), AX diff --git a/src/runtime/internal/atomic/asm_arm.s b/src/runtime/internal/atomic/asm_arm.s index c3d1d9025d..274925ed60 100644 --- a/src/runtime/internal/atomic/asm_arm.s +++ b/src/runtime/internal/atomic/asm_arm.s @@ -57,6 +57,9 @@ TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8 B ·Load(SB) +TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8 + B ·Load(SB) + TEXT ·Casuintptr(SB),NOSPLIT,$0-13 B ·Cas(SB) @@ -81,6 +84,9 @@ TEXT ·StorepNoWB(SB),NOSPLIT,$0-8 TEXT ·StoreRel(SB),NOSPLIT,$0-8 B ·Store(SB) +TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8 + B ·Store(SB) + TEXT ·Xadduintptr(SB),NOSPLIT,$0-12 B ·Xadd(SB) diff --git a/src/runtime/internal/atomic/asm_mips64x.s b/src/runtime/internal/atomic/asm_mips64x.s index 3290fb726a..03fb822929 100644 --- a/src/runtime/internal/atomic/asm_mips64x.s +++ b/src/runtime/internal/atomic/asm_mips64x.s @@ -158,6 +158,12 @@ TEXT ·StorepNoWB(SB), NOSPLIT, $0-16 TEXT ·StoreRel(SB), NOSPLIT, $0-12 JMP ·Store(SB) +TEXT ·StoreRel64(SB), NOSPLIT, $0-16 + JMP ·Store64(SB) + +TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16 + JMP ·Store64(SB) + TEXT ·Store(SB), NOSPLIT, $0-12 MOVV ptr+0(FP), R1 MOVW val+8(FP), R2 diff --git a/src/runtime/internal/atomic/asm_mipsx.s b/src/runtime/internal/atomic/asm_mipsx.s index 62811a6599..63bb548825 100644 --- a/src/runtime/internal/atomic/asm_mipsx.s +++ b/src/runtime/internal/atomic/asm_mipsx.s @@ -122,6 +122,9 @@ TEXT ·StorepNoWB(SB),NOSPLIT,$0-8 TEXT ·StoreRel(SB),NOSPLIT,$0-8 JMP ·Store(SB) +TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8 + JMP ·Store(SB) + // void Or8(byte volatile*, byte); TEXT ·Or8(SB),NOSPLIT,$0-5 MOVW ptr+0(FP), R1 diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s index 06dc931bf4..c0237de4d0 100644 --- a/src/runtime/internal/atomic/asm_ppc64x.s +++ b/src/runtime/internal/atomic/asm_ppc64x.s @@ -83,12 +83,18 @@ TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-25 TEXT runtime∕internal∕atomic·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16 BR runtime∕internal∕atomic·Load64(SB) +TEXT runtime∕internal∕atomic·LoadAcquintptr(SB), NOSPLIT|NOFRAME, $0-16 + BR runtime∕internal∕atomic·LoadAcq64(SB) + TEXT runtime∕internal∕atomic·Loaduint(SB), NOSPLIT|NOFRAME, $0-16 BR runtime∕internal∕atomic·Load64(SB) TEXT runtime∕internal∕atomic·Storeuintptr(SB), NOSPLIT, $0-16 BR runtime∕internal∕atomic·Store64(SB) +TEXT runtime∕internal∕atomic·StoreReluintptr(SB), NOSPLIT, $0-16 + BR runtime∕internal∕atomic·StoreRel64(SB) + TEXT runtime∕internal∕atomic·Xadduintptr(SB), NOSPLIT, $0-24 BR runtime∕internal∕atomic·Xadd64(SB) @@ -191,6 +197,13 @@ TEXT runtime∕internal∕atomic·StoreRel(SB), NOSPLIT, $0-12 MOVW R4, 0(R3) RET +TEXT runtime∕internal∕atomic·StoreRel64(SB), NOSPLIT, $0-16 + MOVD ptr+0(FP), R3 + MOVD val+8(FP), R4 + LWSYNC + MOVD R4, 0(R3) + RET + // void runtime∕internal∕atomic·Or8(byte volatile*, byte); TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9 MOVD ptr+0(FP), R3 diff --git a/src/runtime/internal/atomic/atomic_386.go b/src/runtime/internal/atomic/atomic_386.go index 8d002ebfe3..06ce6a5356 100644 --- a/src/runtime/internal/atomic/atomic_386.go +++ b/src/runtime/internal/atomic/atomic_386.go @@ -30,6 +30,12 @@ func LoadAcq(ptr *uint32) uint32 { return *ptr } +//go:nosplit +//go:noinline +func LoadAcquintptr(ptr *uintptr) uintptr { + return *ptr +} + //go:noescape func Xadd64(ptr *uint64, delta int64) uint64 @@ -83,5 +89,8 @@ func Store64(ptr *uint64, val uint64) //go:noescape func StoreRel(ptr *uint32, val uint32) +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) + // NO go:noescape annotation; see atomic_pointer.go. func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_amd64.go b/src/runtime/internal/atomic/atomic_amd64.go index 14b8101720..1b71a16d94 100644 --- a/src/runtime/internal/atomic/atomic_amd64.go +++ b/src/runtime/internal/atomic/atomic_amd64.go @@ -35,6 +35,18 @@ func LoadAcq(ptr *uint32) uint32 { return *ptr } +//go:nosplit +//go:noinline +func LoadAcq64(ptr *uint64) uint64 { + return *ptr +} + +//go:nosplit +//go:noinline +func LoadAcquintptr(ptr *uintptr) uintptr { + return *ptr +} + //go:noescape func Xadd(ptr *uint32, delta int32) uint32 @@ -85,6 +97,12 @@ func Store64(ptr *uint64, val uint64) //go:noescape func StoreRel(ptr *uint32, val uint32) +//go:noescape +func StoreRel64(ptr *uint64, val uint64) + +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) + // StorepNoWB performs *ptr = val atomically and without a write // barrier. // diff --git a/src/runtime/internal/atomic/atomic_arm.go b/src/runtime/internal/atomic/atomic_arm.go index 95713afcc1..67d529c1cb 100644 --- a/src/runtime/internal/atomic/atomic_arm.go +++ b/src/runtime/internal/atomic/atomic_arm.go @@ -81,6 +81,9 @@ func Store(addr *uint32, v uint32) //go:noescape func StoreRel(addr *uint32, v uint32) +//go:noescape +func StoreReluintptr(addr *uintptr, v uintptr) + //go:nosplit func goCas64(addr *uint64, old, new uint64) bool { if uintptr(unsafe.Pointer(addr))&7 != 0 { @@ -194,6 +197,9 @@ func Load8(addr *uint8) uint8 //go:noescape func LoadAcq(addr *uint32) uint32 +//go:noescape +func LoadAcquintptr(ptr *uintptr) uintptr + //go:noescape func Cas64(addr *uint64, old, new uint64) bool diff --git a/src/runtime/internal/atomic/atomic_arm64.go b/src/runtime/internal/atomic/atomic_arm64.go index 26ca94d54c..c9b4322fe9 100644 --- a/src/runtime/internal/atomic/atomic_arm64.go +++ b/src/runtime/internal/atomic/atomic_arm64.go @@ -41,6 +41,12 @@ func Loadp(ptr unsafe.Pointer) unsafe.Pointer //go:noescape func LoadAcq(addr *uint32) uint32 +//go:noescape +func LoadAcq64(ptr *uint64) uint64 + +//go:noescape +func LoadAcquintptr(ptr *uintptr) uintptr + //go:noescape func Or8(ptr *uint8, val uint8) @@ -67,3 +73,9 @@ func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) //go:noescape func StoreRel(ptr *uint32, val uint32) + +//go:noescape +func StoreRel64(ptr *uint64, val uint64) + +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) diff --git a/src/runtime/internal/atomic/atomic_arm64.s b/src/runtime/internal/atomic/atomic_arm64.s index a2eb7568d2..36c7698b18 100644 --- a/src/runtime/internal/atomic/atomic_arm64.s +++ b/src/runtime/internal/atomic/atomic_arm64.s @@ -36,12 +36,26 @@ TEXT ·Loadp(SB),NOSPLIT,$0-16 TEXT ·LoadAcq(SB),NOSPLIT,$0-12 B ·Load(SB) +// uint64 runtime∕internal∕atomic·LoadAcquintptr(uint64 volatile* addr) +TEXT ·LoadAcq64(SB),NOSPLIT,$0-16 + B ·Load64(SB) + +// uintptr runtime∕internal∕atomic·LoadAcq64(uintptr volatile* addr) +TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16 + B ·Load64(SB) + TEXT runtime∕internal∕atomic·StorepNoWB(SB), NOSPLIT, $0-16 B runtime∕internal∕atomic·Store64(SB) TEXT runtime∕internal∕atomic·StoreRel(SB), NOSPLIT, $0-12 B runtime∕internal∕atomic·Store(SB) +TEXT runtime∕internal∕atomic·StoreRel64(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·StoreReluintptr(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Store64(SB) + TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 MOVD ptr+0(FP), R0 MOVW val+8(FP), R1 diff --git a/src/runtime/internal/atomic/atomic_mips64x.go b/src/runtime/internal/atomic/atomic_mips64x.go index 1d9977850b..fca2242514 100644 --- a/src/runtime/internal/atomic/atomic_mips64x.go +++ b/src/runtime/internal/atomic/atomic_mips64x.go @@ -41,6 +41,12 @@ func Loadp(ptr unsafe.Pointer) unsafe.Pointer //go:noescape func LoadAcq(ptr *uint32) uint32 +//go:noescape +func LoadAcq64(ptr *uint64) uint64 + +//go:noescape +func LoadAcquintptr(ptr *uintptr) uintptr + //go:noescape func And8(ptr *uint8, val uint8) @@ -69,3 +75,9 @@ func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) //go:noescape func StoreRel(ptr *uint32, val uint32) + +//go:noescape +func StoreRel64(ptr *uint64, val uint64) + +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) diff --git a/src/runtime/internal/atomic/atomic_mips64x.s b/src/runtime/internal/atomic/atomic_mips64x.s index 1ed90937c9..125c0c221c 100644 --- a/src/runtime/internal/atomic/atomic_mips64x.s +++ b/src/runtime/internal/atomic/atomic_mips64x.s @@ -47,3 +47,11 @@ TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-16 // uint32 runtime∕internal∕atomic·LoadAcq(uint32 volatile* ptr) TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12 JMP atomic·Load(SB) + +// uint64 runtime∕internal∕atomic·LoadAcq64(uint64 volatile* ptr) +TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16 + JMP atomic·Load64(SB) + +// uintptr runtime∕internal∕atomic·LoadAcquintptr(uintptr volatile* ptr) +TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16 + JMP atomic·Load64(SB) diff --git a/src/runtime/internal/atomic/atomic_mipsx.go b/src/runtime/internal/atomic/atomic_mipsx.go index b99bfe7dbf..be1e6a038b 100644 --- a/src/runtime/internal/atomic/atomic_mipsx.go +++ b/src/runtime/internal/atomic/atomic_mipsx.go @@ -132,6 +132,9 @@ func Loadp(ptr unsafe.Pointer) unsafe.Pointer //go:noescape func LoadAcq(ptr *uint32) uint32 +//go:noescape +func LoadAcquintptr(ptr *uintptr) uintptr + //go:noescape func And8(ptr *uint8, val uint8) @@ -150,5 +153,8 @@ func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) //go:noescape func StoreRel(ptr *uint32, val uint32) +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) + //go:noescape func CasRel(addr *uint32, old, new uint32) bool diff --git a/src/runtime/internal/atomic/atomic_ppc64x.go b/src/runtime/internal/atomic/atomic_ppc64x.go index a48ecf5ee8..e759bb27a2 100644 --- a/src/runtime/internal/atomic/atomic_ppc64x.go +++ b/src/runtime/internal/atomic/atomic_ppc64x.go @@ -41,6 +41,12 @@ func Loadp(ptr unsafe.Pointer) unsafe.Pointer //go:noescape func LoadAcq(ptr *uint32) uint32 +//go:noescape +func LoadAcq64(ptr *uint64) uint64 + +//go:noescape +func LoadAcquintptr(ptr *uintptr) uintptr + //go:noescape func And8(ptr *uint8, val uint8) @@ -67,5 +73,11 @@ func Store64(ptr *uint64, val uint64) //go:noescape func StoreRel(ptr *uint32, val uint32) +//go:noescape +func StoreRel64(ptr *uint64, val uint64) + +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) + // NO go:noescape annotation; see atomic_pointer.go. func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_ppc64x.s b/src/runtime/internal/atomic/atomic_ppc64x.s index c2f696fb34..b79cdbca34 100644 --- a/src/runtime/internal/atomic/atomic_ppc64x.s +++ b/src/runtime/internal/atomic/atomic_ppc64x.s @@ -6,6 +6,15 @@ #include "textflag.h" + +// For more details about how various memory models are +// enforced on POWER, the following paper provides more +// details about how they enforce C/C++ like models. This +// gives context about why the strange looking code +// sequences below work. +// +// http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html + // uint32 runtime∕internal∕atomic·Load(uint32 volatile* ptr) TEXT ·Load(SB),NOSPLIT|NOFRAME,$-8-12 MOVD ptr+0(FP), R3 @@ -56,5 +65,16 @@ TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$-8-12 MOVWZ 0(R3), R3 CMPW R3, R3, CR7 BC 4, 30, 1(PC) // bne- cr7, 0x4 + ISYNC MOVW R3, ret+8(FP) RET + +// uint64 runtime∕internal∕atomic·LoadAcq64(uint64 volatile* ptr) +TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$-8-16 + MOVD ptr+0(FP), R3 + MOVD 0(R3), R3 + CMP R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7, 0x4 + ISYNC + MOVD R3, ret+8(FP) + RET diff --git a/src/runtime/internal/atomic/atomic_riscv64.go b/src/runtime/internal/atomic/atomic_riscv64.go index d52512369e..617bc1a3eb 100644 --- a/src/runtime/internal/atomic/atomic_riscv64.go +++ b/src/runtime/internal/atomic/atomic_riscv64.go @@ -39,6 +39,12 @@ func Loadp(ptr unsafe.Pointer) unsafe.Pointer //go:noescape func LoadAcq(ptr *uint32) uint32 +//go:noescape +func LoadAcq64(ptr *uint64) uint64 + +//go:noescape +func LoadAcquintptr(ptr *uintptr) uintptr + //go:noescape func Or8(ptr *uint8, val uint8) @@ -65,3 +71,9 @@ func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer) //go:noescape func StoreRel(ptr *uint32, val uint32) + +//go:noescape +func StoreRel64(ptr *uint64, val uint64) + +//go:noescape +func StoreReluintptr(ptr *uintptr, val uintptr) diff --git a/src/runtime/internal/atomic/atomic_riscv64.s b/src/runtime/internal/atomic/atomic_riscv64.s index d005325ca3..db139d690a 100644 --- a/src/runtime/internal/atomic/atomic_riscv64.s +++ b/src/runtime/internal/atomic/atomic_riscv64.s @@ -150,6 +150,12 @@ TEXT ·Xaddint64(SB),NOSPLIT,$0-24 TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12 JMP ·Load(SB) +TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16 + JMP ·Load64(SB) + +TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16 + JMP ·Load64(SB) + // func Loadp(ptr unsafe.Pointer) unsafe.Pointer TEXT ·Loadp(SB),NOSPLIT,$0-16 JMP ·Load64(SB) @@ -161,6 +167,12 @@ TEXT ·StorepNoWB(SB), NOSPLIT, $0-16 TEXT ·StoreRel(SB), NOSPLIT, $0-12 JMP ·Store(SB) +TEXT ·StoreRel64(SB), NOSPLIT, $0-16 + JMP ·Store64(SB) + +TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16 + JMP ·Store64(SB) + // func Xchg(ptr *uint32, new uint32) uint32 TEXT ·Xchg(SB), NOSPLIT, $0-20 MOV ptr+0(FP), A0 diff --git a/src/runtime/internal/atomic/atomic_s390x.go b/src/runtime/internal/atomic/atomic_s390x.go index 4d73b39baf..b649caa39f 100644 --- a/src/runtime/internal/atomic/atomic_s390x.go +++ b/src/runtime/internal/atomic/atomic_s390x.go @@ -41,6 +41,18 @@ func LoadAcq(ptr *uint32) uint32 { return *ptr } +//go:nosplit +//go:noinline +func LoadAcq64(ptr *uint64) uint64 { + return *ptr +} + +//go:nosplit +//go:noinline +func LoadAcquintptr(ptr *uintptr) uintptr { + return *ptr +} + //go:noescape func Store(ptr *uint32, val uint32) @@ -59,6 +71,18 @@ func StoreRel(ptr *uint32, val uint32) { *ptr = val } +//go:nosplit +//go:noinline +func StoreRel64(ptr *uint64, val uint64) { + *ptr = val +} + +//go:nosplit +//go:noinline +func StoreReluintptr(ptr *uintptr, val uintptr) { + *ptr = val +} + //go:noescape func And8(ptr *uint8, val uint8) diff --git a/src/runtime/internal/atomic/atomic_wasm.go b/src/runtime/internal/atomic/atomic_wasm.go index 2c0c3a8174..60a4942884 100644 --- a/src/runtime/internal/atomic/atomic_wasm.go +++ b/src/runtime/internal/atomic/atomic_wasm.go @@ -45,6 +45,18 @@ func LoadAcq(ptr *uint32) uint32 { return *ptr } +//go:nosplit +//go:noinline +func LoadAcq64(ptr *uint64) uint64 { + return *ptr +} + +//go:nosplit +//go:noinline +func LoadAcquintptr(ptr *uintptr) uintptr { + return *ptr +} + //go:nosplit //go:noinline func Load8(ptr *uint8) uint8 { @@ -141,6 +153,18 @@ func StoreRel(ptr *uint32, val uint32) { *ptr = val } +//go:nosplit +//go:noinline +func StoreRel64(ptr *uint64, val uint64) { + *ptr = val +} + +//go:nosplit +//go:noinline +func StoreReluintptr(ptr *uintptr, val uintptr) { + *ptr = val +} + //go:nosplit //go:noinline func Store8(ptr *uint8, val uint8) { diff --git a/src/sync/pool.go b/src/sync/pool.go index ca7afdb12f..137413fdc4 100644 --- a/src/sync/pool.go +++ b/src/sync/pool.go @@ -7,6 +7,7 @@ package sync import ( "internal/race" "runtime" + runtimeatomic "runtime/internal/atomic" "sync/atomic" "unsafe" ) @@ -152,8 +153,8 @@ func (p *Pool) Get() interface{} { func (p *Pool) getSlow(pid int) interface{} { // See the comment in pin regarding ordering of the loads. - size := atomic.LoadUintptr(&p.localSize) // load-acquire - locals := p.local // load-consume + size := runtimeatomic.LoadAcquintptr(&p.localSize) // load-acquire + locals := p.local // load-consume // Try to steal one element from other procs. for i := 0; i < int(size); i++ { l := indexLocal(locals, (pid+i+1)%int(size)) @@ -165,7 +166,7 @@ func (p *Pool) getSlow(pid int) interface{} { // Try the victim cache. We do this after attempting to steal // from all primary caches because we want objects in the // victim cache to age out if at all possible. - size = atomic.LoadUintptr(&p.victimSize) + size = runtimeatomic.Loaduintptr(&p.victimSize) if uintptr(pid) >= size { return nil } @@ -198,8 +199,8 @@ func (p *Pool) pin() (*poolLocal, int) { // Since we've disabled preemption, GC cannot happen in between. // Thus here we must observe local at least as large localSize. // We can observe a newer/larger local, it is fine (we must observe its zero-initialized-ness). - s := atomic.LoadUintptr(&p.localSize) // load-acquire - l := p.local // load-consume + s := runtimeatomic.LoadAcquintptr(&p.localSize) // load-acquire + l := p.local // load-consume if uintptr(pid) < s { return indexLocal(l, pid), pid } @@ -225,8 +226,8 @@ func (p *Pool) pinSlow() (*poolLocal, int) { // If GOMAXPROCS changes between GCs, we re-allocate the array and lose the old one. size := runtime.GOMAXPROCS(0) local := make([]poolLocal, size) - atomic.StorePointer(&p.local, unsafe.Pointer(&local[0])) // store-release - atomic.StoreUintptr(&p.localSize, uintptr(size)) // store-release + atomic.StorePointer(&p.local, unsafe.Pointer(&local[0])) // store-release + runtimeatomic.StoreReluintptr(&p.localSize, uintptr(size)) // store-release return &local[pid], pid } -- cgit v1.3 From f8376a55b06304bd24612d10812cf8ada29a5c84 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Fri, 23 Oct 2020 03:46:19 +0000 Subject: sync: document that Once must not be copied Fixes #42160. Change-Id: I9bf8b6f0bf1eccd3ab32cbd94c812f768746d291 Reviewed-on: https://go-review.googlesource.com/c/go/+/264557 Trust: Dmitri Shuralyov Run-TryBot: Dmitri Shuralyov TryBot-Result: Go Bot Reviewed-by: Bryan C. Mills --- src/sync/once.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/sync') diff --git a/src/sync/once.go b/src/sync/once.go index ca04408224..bf4b80c867 100644 --- a/src/sync/once.go +++ b/src/sync/once.go @@ -9,6 +9,8 @@ import ( ) // Once is an object that will perform exactly one action. +// +// A Once must not be copied after first use. type Once struct { // done indicates whether the action has been performed. // It is first in the struct because it is used in the hot path. -- cgit v1.3 From c305e49e96deafe54a8e43010ea76fead6da0a98 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 26 Oct 2020 14:32:13 -0400 Subject: cmd/go,cmd/compile,sync: remove special import case in cmd/go CL 253748 introduced a special case in cmd/go to allow sync to import runtime/internal/atomic. Besides introducing unnecessary complexity into cmd/go, this breaks other packages (like gopls) that understand how imports work, but don't understand this special case. Fix this by using the more standard linkname-based approach to pull the necessary functions from runtime/internal/atomic into sync. Since these are compiler intrinsics, we also have to tell the compiler that the linknamed symbols are intrinsics to get this optimization in sync. Fixes #42196. Change-Id: I1f91498c255c91583950886a89c3c9adc39a32f0 Reviewed-on: https://go-review.googlesource.com/c/go/+/265124 Trust: Austin Clements Run-TryBot: Austin Clements Reviewed-by: Bryan C. Mills Reviewed-by: Keith Randall Reviewed-by: Paul Murphy TryBot-Result: Go Bot --- src/cmd/compile/internal/gc/ssa.go | 4 ++++ src/cmd/go/internal/load/pkg.go | 5 ----- src/sync/pool.go | 25 +++++++++++++++++-------- 3 files changed, 21 insertions(+), 13 deletions(-) (limited to 'src/sync') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index f840ef4066..a1b5a03687 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -3569,13 +3569,17 @@ func init() { alias("runtime/internal/atomic", "LoadAcq", "runtime/internal/atomic", "Load", lwatomics...) alias("runtime/internal/atomic", "LoadAcq64", "runtime/internal/atomic", "Load64", lwatomics...) alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) + alias("sync", "runtime_LoadAcquintptr", "runtime/internal/atomic", "LoadAcq", p4...) // linknamed alias("runtime/internal/atomic", "LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) + alias("sync", "runtime_LoadAcquintptr", "runtime/internal/atomic", "LoadAcq64", p8...) // linknamed alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store", p4...) alias("runtime/internal/atomic", "Storeuintptr", "runtime/internal/atomic", "Store64", p8...) alias("runtime/internal/atomic", "StoreRel", "runtime/internal/atomic", "Store", lwatomics...) alias("runtime/internal/atomic", "StoreRel64", "runtime/internal/atomic", "Store64", lwatomics...) alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) + alias("sync", "runtime_StoreReluintptr", "runtime/internal/atomic", "StoreRel", p4...) // linknamed alias("runtime/internal/atomic", "StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) + alias("sync", "runtime_StoreReluintptr", "runtime/internal/atomic", "StoreRel64", p8...) // linknamed alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg", p4...) alias("runtime/internal/atomic", "Xchguintptr", "runtime/internal/atomic", "Xchg64", p8...) alias("runtime/internal/atomic", "Xadduintptr", "runtime/internal/atomic", "Xadd", p4...) diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index fcd7728c7b..615b5ef769 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -1338,11 +1338,6 @@ func disallowInternal(srcDir string, importer *Package, importerPath string, p * return p } - // Allow sync package to access lightweight atomic functions limited to the runtime. - if p.Standard && strings.HasPrefix(importerPath, "sync") && p.ImportPath == "runtime/internal/atomic" { - return p - } - // Internal is present. // Map import path back to directory corresponding to parent of internal. if i > 0 { diff --git a/src/sync/pool.go b/src/sync/pool.go index 137413fdc4..1ae70127ac 100644 --- a/src/sync/pool.go +++ b/src/sync/pool.go @@ -7,7 +7,6 @@ package sync import ( "internal/race" "runtime" - runtimeatomic "runtime/internal/atomic" "sync/atomic" "unsafe" ) @@ -153,8 +152,8 @@ func (p *Pool) Get() interface{} { func (p *Pool) getSlow(pid int) interface{} { // See the comment in pin regarding ordering of the loads. - size := runtimeatomic.LoadAcquintptr(&p.localSize) // load-acquire - locals := p.local // load-consume + size := runtime_LoadAcquintptr(&p.localSize) // load-acquire + locals := p.local // load-consume // Try to steal one element from other procs. for i := 0; i < int(size); i++ { l := indexLocal(locals, (pid+i+1)%int(size)) @@ -166,7 +165,7 @@ func (p *Pool) getSlow(pid int) interface{} { // Try the victim cache. We do this after attempting to steal // from all primary caches because we want objects in the // victim cache to age out if at all possible. - size = runtimeatomic.Loaduintptr(&p.victimSize) + size = atomic.LoadUintptr(&p.victimSize) if uintptr(pid) >= size { return nil } @@ -199,8 +198,8 @@ func (p *Pool) pin() (*poolLocal, int) { // Since we've disabled preemption, GC cannot happen in between. // Thus here we must observe local at least as large localSize. // We can observe a newer/larger local, it is fine (we must observe its zero-initialized-ness). - s := runtimeatomic.LoadAcquintptr(&p.localSize) // load-acquire - l := p.local // load-consume + s := runtime_LoadAcquintptr(&p.localSize) // load-acquire + l := p.local // load-consume if uintptr(pid) < s { return indexLocal(l, pid), pid } @@ -226,8 +225,8 @@ func (p *Pool) pinSlow() (*poolLocal, int) { // If GOMAXPROCS changes between GCs, we re-allocate the array and lose the old one. size := runtime.GOMAXPROCS(0) local := make([]poolLocal, size) - atomic.StorePointer(&p.local, unsafe.Pointer(&local[0])) // store-release - runtimeatomic.StoreReluintptr(&p.localSize, uintptr(size)) // store-release + atomic.StorePointer(&p.local, unsafe.Pointer(&local[0])) // store-release + runtime_StoreReluintptr(&p.localSize, uintptr(size)) // store-release return &local[pid], pid } @@ -283,3 +282,13 @@ func indexLocal(l unsafe.Pointer, i int) *poolLocal { func runtime_registerPoolCleanup(cleanup func()) func runtime_procPin() int func runtime_procUnpin() + +// The below are implemented in runtime/internal/atomic and the +// compiler also knows to intrinsify the symbol we linkname into this +// package. + +//go:linkname runtime_LoadAcquintptr runtime/internal/atomic.LoadAcquintptr +func runtime_LoadAcquintptr(ptr *uintptr) uintptr + +//go:linkname runtime_StoreReluintptr runtime/internal/atomic.StoreReluintptr +func runtime_StoreReluintptr(ptr *uintptr, val uintptr) uintptr -- cgit v1.3 From b4f3d52f6a90aa520799f836e5951d5cf65f7fe4 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Tue, 17 Nov 2020 16:47:08 -0500 Subject: sync: document RWMutex race semantics RWMutex provides explicit acquire/release synchronization events to the race detector to model the mutex. It disables sync events within the methods to avoid e.g., the atomics from adding false synchronization events, which could cause false negatives in the race detector. Change-Id: I5126ce2efaab151811ac264864aab1fa025a4aaf Reviewed-on: https://go-review.googlesource.com/c/go/+/270865 Run-TryBot: Michael Pratt TryBot-Result: Go Bot Reviewed-by: Austin Clements Trust: Michael Pratt --- src/sync/rwmutex.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/sync') diff --git a/src/sync/rwmutex.go b/src/sync/rwmutex.go index dc0faf6a60..3012b5548e 100644 --- a/src/sync/rwmutex.go +++ b/src/sync/rwmutex.go @@ -35,6 +35,19 @@ type RWMutex struct { const rwmutexMaxReaders = 1 << 30 +// Happens-before relationships are indicated to the race detector via: +// - Unlock -> Lock: readerSem +// - Unlock -> RLock: readerSem +// - RUnlock -> Lock: writerSem +// +// The methods below temporarily disable handling of race synchronization +// events in order to provide the more precise model above to the race +// detector. +// +// For example, atomic.AddInt32 in RLock should not appear to provide +// acquire-release semantics, which would incorrectly synchronize racing +// readers, thus potentially missing races. + // RLock locks rw for reading. // // It should not be used for recursive read locking; a blocked Lock -- cgit v1.3 From d902791b509b641683d4ec58b282180c56918aec Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Sat, 21 Nov 2020 15:53:18 +0100 Subject: sync: use 386 instead of x86-32 to refer to the 32 bit x86 architecture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This aligns the naming with GOARCH using 386 as a build target for this architecture and makes it more easily found when searching for documentation related to the build target. Change-Id: I393bb89dd2f71e568124107b13e1b288fbd0c76a Reviewed-on: https://go-review.googlesource.com/c/go/+/271988 Trust: Martin Möhrmann Run-TryBot: Martin Möhrmann TryBot-Result: Go Bot Reviewed-by: Ian Lance Taylor --- src/sync/atomic/doc.go | 11 +++++------ src/sync/once.go | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) (limited to 'src/sync') diff --git a/src/sync/atomic/doc.go b/src/sync/atomic/doc.go index ff4ad80049..805ef956d5 100644 --- a/src/sync/atomic/doc.go +++ b/src/sync/atomic/doc.go @@ -43,15 +43,14 @@ import ( "unsafe" ) -// BUG(rsc): On x86-32, the 64-bit functions use instructions unavailable before the Pentium MMX. +// BUG(rsc): On 386, the 64-bit functions use instructions unavailable before the Pentium MMX. // // On non-Linux ARM, the 64-bit functions use instructions unavailable before the ARMv6k core. // -// On ARM, x86-32, and 32-bit MIPS, -// it is the caller's responsibility to arrange for 64-bit -// alignment of 64-bit words accessed atomically. The first word in a -// variable or in an allocated struct, array, or slice can be relied upon to be -// 64-bit aligned. +// On ARM, 386, and 32-bit MIPS, it is the caller's responsibility +// to arrange for 64-bit alignment of 64-bit words accessed atomically. +// The first word in a variable or in an allocated struct, array, or slice can +// be relied upon to be 64-bit aligned. // SwapInt32 atomically stores new into *addr and returns the previous *addr value. func SwapInt32(addr *int32, new int32) (old int32) diff --git a/src/sync/once.go b/src/sync/once.go index bf4b80c867..8844314e7e 100644 --- a/src/sync/once.go +++ b/src/sync/once.go @@ -15,7 +15,7 @@ type Once struct { // done indicates whether the action has been performed. // It is first in the struct because it is used in the hot path. // The hot path is inlined at every call site. - // Placing done first allows more compact instructions on some architectures (amd64/x86), + // Placing done first allows more compact instructions on some architectures (amd64/386), // and fewer instructions (to calculate offset) on other architectures. done uint32 m Mutex -- cgit v1.3