diff options
Diffstat (limited to 'src/runtime/internal/atomic')
30 files changed, 1766 insertions, 0 deletions
diff --git a/src/runtime/internal/atomic/arch1_386.go b/src/runtime/internal/atomic/arch1_386.go new file mode 100644 index 0000000000..76a9e2ecc4 --- /dev/null +++ b/src/runtime/internal/atomic/arch1_386.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + _CacheLineSize = 64 +) diff --git a/src/runtime/internal/atomic/arch1_amd64.go b/src/runtime/internal/atomic/arch1_amd64.go new file mode 100644 index 0000000000..76a9e2ecc4 --- /dev/null +++ b/src/runtime/internal/atomic/arch1_amd64.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + _CacheLineSize = 64 +) diff --git a/src/runtime/internal/atomic/arch1_amd64p32.go b/src/runtime/internal/atomic/arch1_amd64p32.go new file mode 100644 index 0000000000..7bb6e1aaf9 --- /dev/null +++ b/src/runtime/internal/atomic/arch1_amd64p32.go @@ -0,0 +1,14 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + thechar = '6' + _BigEndian = 0 + _CacheLineSize = 64 + _PCQuantum = 1 + _Int64Align = 8 + hugePageSize = 1 << 21 +) diff --git a/src/runtime/internal/atomic/arch1_arm.go b/src/runtime/internal/atomic/arch1_arm.go new file mode 100644 index 0000000000..d9b997c5ca --- /dev/null +++ b/src/runtime/internal/atomic/arch1_arm.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + _CacheLineSize = 32 +) diff --git a/src/runtime/internal/atomic/arch1_arm64.go b/src/runtime/internal/atomic/arch1_arm64.go new file mode 100644 index 0000000000..d9b997c5ca --- /dev/null +++ b/src/runtime/internal/atomic/arch1_arm64.go @@ -0,0 +1,9 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + _CacheLineSize = 32 +) diff --git a/src/runtime/internal/atomic/arch1_ppc64.go b/src/runtime/internal/atomic/arch1_ppc64.go new file mode 100644 index 0000000000..815a5840f7 --- /dev/null +++ b/src/runtime/internal/atomic/arch1_ppc64.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + _CacheLineSize = 64 +) diff --git a/src/runtime/internal/atomic/arch1_ppc64le.go b/src/runtime/internal/atomic/arch1_ppc64le.go new file mode 100644 index 0000000000..815a5840f7 --- /dev/null +++ b/src/runtime/internal/atomic/arch1_ppc64le.go @@ -0,0 +1,9 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +const ( + _CacheLineSize = 64 +) diff --git a/src/runtime/internal/atomic/asm.s b/src/runtime/internal/atomic/asm.s new file mode 100644 index 0000000000..b5d0211803 --- /dev/null +++ b/src/runtime/internal/atomic/asm.s @@ -0,0 +1,8 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT runtime∕internal∕atomic·nop(SB),NOSPLIT,$0-0 + RET diff --git a/src/runtime/internal/atomic/asm_386.s b/src/runtime/internal/atomic/asm_386.s new file mode 100644 index 0000000000..ce84fd83d1 --- /dev/null +++ b/src/runtime/internal/atomic/asm_386.s @@ -0,0 +1,166 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// bool Cas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// }else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-13 + MOVL ptr+0(FP), BX + MOVL old+4(FP), AX + MOVL new+8(FP), CX + LOCK + CMPXCHGL CX, 0(BX) + SETEQ ret+12(FP) + RET + +TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-13 + JMP runtime∕internal∕atomic·Cas(SB) + +TEXT runtime∕internal∕atomic·Loaduintptr(SB), NOSPLIT, $0-8 + JMP runtime∕internal∕atomic·Load(SB) + +TEXT runtime∕internal∕atomic·Loaduint(SB), NOSPLIT, $0-8 + JMP runtime∕internal∕atomic·Load(SB) + +TEXT runtime∕internal∕atomic·Storeuintptr(SB), NOSPLIT, $0-8 + JMP runtime∕internal∕atomic·Store(SB) + +TEXT runtime∕internal∕atomic·Xadduintptr(SB), NOSPLIT, $0-8 + JMP runtime∕internal∕atomic·Xadd(SB) + +TEXT runtime∕internal∕atomic·Loadint64(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Xaddint64(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Xadd64(SB) + + +// bool runtime∕internal∕atomic·Cas64(uint64 *val, uint64 old, uint64 new) +// Atomically: +// if(*val == *old){ +// *val = new; +// return 1; +// } else { +// return 0; +// } +TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-21 + MOVL ptr+0(FP), BP + MOVL old_lo+4(FP), AX + MOVL old_hi+8(FP), DX + MOVL new_lo+12(FP), BX + MOVL new_hi+16(FP), CX + LOCK + CMPXCHG8B 0(BP) + SETEQ ret+20(FP) + RET + +// bool Casp(void **p, void *old, void *new) +// Atomically: +// if(*p == old){ +// *p = new; +// return 1; +// }else +// return 0; +TEXT runtime∕internal∕atomic·Casp1(SB), NOSPLIT, $0-13 + MOVL ptr+0(FP), BX + MOVL old+4(FP), AX + MOVL new+8(FP), CX + LOCK + CMPXCHGL CX, 0(BX) + SETEQ ret+12(FP) + RET + +// uint32 Xadd(uint32 volatile *val, int32 delta) +// Atomically: +// *val += delta; +// return *val; +TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), BX + MOVL delta+4(FP), AX + MOVL AX, CX + LOCK + XADDL AX, 0(BX) + ADDL CX, AX + MOVL AX, ret+8(FP) + RET + +TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), BX + MOVL new+4(FP), AX + XCHGL AX, 0(BX) + MOVL AX, ret+8(FP) + RET + +TEXT runtime∕internal∕atomic·Xchguintptr(SB), NOSPLIT, $0-12 + JMP runtime∕internal∕atomic·Xchg(SB) + + +TEXT runtime∕internal∕atomic·Storep1(SB), NOSPLIT, $0-8 + MOVL ptr+0(FP), BX + MOVL val+4(FP), AX + XCHGL AX, 0(BX) + RET + +TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-8 + MOVL ptr+0(FP), BX + MOVL val+4(FP), AX + XCHGL AX, 0(BX) + RET + +// uint64 atomicload64(uint64 volatile* addr); +TEXT runtime∕internal∕atomic·Load64(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), AX + TESTL $7, AX + JZ 2(PC) + MOVL 0, AX // crash with nil ptr deref + LEAL ret_lo+4(FP), BX + // MOVQ (%EAX), %MM0 + BYTE $0x0f; BYTE $0x6f; BYTE $0x00 + // MOVQ %MM0, 0(%EBX) + BYTE $0x0f; BYTE $0x7f; BYTE $0x03 + // EMMS + BYTE $0x0F; BYTE $0x77 + RET + +// void runtime∕internal∕atomic·Store64(uint64 volatile* addr, uint64 v); +TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), AX + TESTL $7, AX + JZ 2(PC) + MOVL 0, AX // crash with nil ptr deref + // MOVQ and EMMS were introduced on the Pentium MMX. + // MOVQ 0x8(%ESP), %MM0 + BYTE $0x0f; BYTE $0x6f; BYTE $0x44; BYTE $0x24; BYTE $0x08 + // MOVQ %MM0, (%EAX) + BYTE $0x0f; BYTE $0x7f; BYTE $0x00 + // EMMS + BYTE $0x0F; BYTE $0x77 + // This is essentially a no-op, but it provides required memory fencing. + // It can be replaced with MFENCE, but MFENCE was introduced only on the Pentium4 (SSE2). + MOVL $0, AX + LOCK + XADDL AX, (SP) + RET + +// void runtime∕internal∕atomic·Or8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-5 + MOVL ptr+0(FP), AX + MOVB val+4(FP), BX + LOCK + ORB BX, (AX) + RET + +// void runtime∕internal∕atomic·And8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-5 + MOVL ptr+0(FP), AX + MOVB val+4(FP), BX + LOCK + ANDB BX, (AX) + RET diff --git a/src/runtime/internal/atomic/asm_amd64.s b/src/runtime/internal/atomic/asm_amd64.s new file mode 100644 index 0000000000..7463fec4a1 --- /dev/null +++ b/src/runtime/internal/atomic/asm_amd64.s @@ -0,0 +1,150 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// bool Cas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0-17 + MOVQ ptr+0(FP), BX + MOVL old+8(FP), AX + MOVL new+12(FP), CX + LOCK + CMPXCHGL CX, 0(BX) + SETEQ ret+16(FP) + RET + +// bool runtime∕internal∕atomic·Cas64(uint64 *val, uint64 old, uint64 new) +// Atomically: +// if(*val == *old){ +// *val = new; +// return 1; +// } else { +// return 0; +// } +TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25 + MOVQ ptr+0(FP), BX + MOVQ old+8(FP), AX + MOVQ new+16(FP), CX + LOCK + CMPXCHGQ CX, 0(BX) + SETEQ ret+24(FP) + RET + +TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-25 + JMP runtime∕internal∕atomic·Cas64(SB) + +TEXT runtime∕internal∕atomic·Loaduintptr(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Loaduint(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Storeuintptr(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·Loadint64(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Xaddint64(SB), NOSPLIT, $0-16 + JMP runtime∕internal∕atomic·Xadd64(SB) + +// bool Casp(void **val, void *old, void *new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Casp1(SB), NOSPLIT, $0-25 + MOVQ ptr+0(FP), BX + MOVQ old+8(FP), AX + MOVQ new+16(FP), CX + LOCK + CMPXCHGQ CX, 0(BX) + SETEQ ret+24(FP) + RET + +// uint32 Xadd(uint32 volatile *val, int32 delta) +// Atomically: +// *val += delta; +// return *val; +TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-20 + MOVQ ptr+0(FP), BX + MOVL delta+8(FP), AX + MOVL AX, CX + LOCK + XADDL AX, 0(BX) + ADDL CX, AX + MOVL AX, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24 + MOVQ ptr+0(FP), BX + MOVQ delta+8(FP), AX + MOVQ AX, CX + LOCK + XADDQ AX, 0(BX) + ADDQ CX, AX + MOVQ AX, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xadduintptr(SB), NOSPLIT, $0-24 + JMP runtime∕internal∕atomic·Xadd64(SB) + +TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20 + MOVQ ptr+0(FP), BX + MOVL new+8(FP), AX + XCHGL AX, 0(BX) + MOVL AX, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24 + MOVQ ptr+0(FP), BX + MOVQ new+8(FP), AX + XCHGQ AX, 0(BX) + MOVQ AX, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchguintptr(SB), NOSPLIT, $0-24 + JMP runtime∕internal∕atomic·Xchg64(SB) + +TEXT runtime∕internal∕atomic·Storep1(SB), NOSPLIT, $0-16 + MOVQ ptr+0(FP), BX + MOVQ val+8(FP), AX + XCHGQ AX, 0(BX) + RET + +TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 + MOVQ ptr+0(FP), BX + MOVL val+8(FP), AX + XCHGL AX, 0(BX) + RET + +TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 + MOVQ ptr+0(FP), BX + MOVQ val+8(FP), AX + XCHGQ AX, 0(BX) + RET + +// void runtime∕internal∕atomic·Or8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9 + MOVQ ptr+0(FP), AX + MOVB val+8(FP), BX + LOCK + ORB BX, (AX) + RET + +// void runtime∕internal∕atomic·And8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9 + MOVQ ptr+0(FP), AX + MOVB val+8(FP), BX + LOCK + ANDB BX, (AX) + RET diff --git a/src/runtime/internal/atomic/asm_amd64p32.s b/src/runtime/internal/atomic/asm_amd64p32.s new file mode 100644 index 0000000000..f1e2c3aca6 --- /dev/null +++ b/src/runtime/internal/atomic/asm_amd64p32.s @@ -0,0 +1,150 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// bool Cas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-17 + MOVL ptr+0(FP), BX + MOVL old+4(FP), AX + MOVL new+8(FP), CX + LOCK + CMPXCHGL CX, 0(BX) + SETEQ ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-17 + JMP runtime∕internal∕atomic·Cas(SB) + +TEXT runtime∕internal∕atomic·Loaduintptr(SB), NOSPLIT, $0-12 + JMP runtime∕internal∕atomic·Load(SB) + +TEXT runtime∕internal∕atomic·Loaduint(SB), NOSPLIT, $0-12 + JMP runtime∕internal∕atomic·Load(SB) + +TEXT runtime∕internal∕atomic·Storeuintptr(SB), NOSPLIT, $0-12 + JMP runtime∕internal∕atomic·Store(SB) + +TEXT runtime∕internal∕atomic·Loadint64(SB), NOSPLIT, $0-24 + JMP runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Xaddint64(SB), NOSPLIT, $0-24 + JMP runtime∕internal∕atomic·Xadd64(SB) + +// bool runtime∕internal∕atomic·cas64(uint64 *val, uint64 old, uint64 new) +// Atomically: +// if(*val == *old){ +// *val = new; +// return 1; +// } else { +// return 0; +// } +TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25 + MOVL ptr+0(FP), BX + MOVQ old+8(FP), AX + MOVQ new+16(FP), CX + LOCK + CMPXCHGQ CX, 0(BX) + SETEQ ret+24(FP) + RET + +// bool Casp(void **val, void *old, void *new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Casp1(SB), NOSPLIT, $0-17 + MOVL ptr+0(FP), BX + MOVL old+4(FP), AX + MOVL new+8(FP), CX + LOCK + CMPXCHGL CX, 0(BX) + SETEQ ret+16(FP) + RET + +// uint32 Xadd(uint32 volatile *val, int32 delta) +// Atomically: +// *val += delta; +// return *val; +TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), BX + MOVL delta+4(FP), AX + MOVL AX, CX + LOCK + XADDL AX, 0(BX) + ADDL CX, AX + MOVL AX, ret+8(FP) + RET + +TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24 + MOVL ptr+0(FP), BX + MOVQ delta+8(FP), AX + MOVQ AX, CX + LOCK + XADDQ AX, 0(BX) + ADDQ CX, AX + MOVQ AX, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xadduintptr(SB), NOSPLIT, $0-12 + JMP runtime∕internal∕atomic·Xadd(SB) + +TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-12 + MOVL ptr+0(FP), BX + MOVL new+4(FP), AX + XCHGL AX, 0(BX) + MOVL AX, ret+8(FP) + RET + +TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24 + MOVL ptr+0(FP), BX + MOVQ new+8(FP), AX + XCHGQ AX, 0(BX) + MOVQ AX, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchguintptr(SB), NOSPLIT, $0-12 + JMP runtime∕internal∕atomic·Xchg(SB) + +TEXT runtime∕internal∕atomic·Storep1(SB), NOSPLIT, $0-8 + MOVL ptr+0(FP), BX + MOVL val+4(FP), AX + XCHGL AX, 0(BX) + RET + +TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-8 + MOVL ptr+0(FP), BX + MOVL val+4(FP), AX + XCHGL AX, 0(BX) + RET + +TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 + MOVL ptr+0(FP), BX + MOVQ val+8(FP), AX + XCHGQ AX, 0(BX) + RET + +// void runtime∕internal∕atomic·Or8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-5 + MOVL ptr+0(FP), BX + MOVB val+4(FP), AX + LOCK + ORB AX, 0(BX) + RET + +// void runtime∕internal∕atomic·And8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-5 + MOVL ptr+0(FP), BX + MOVB val+4(FP), AX + LOCK + ANDB AX, 0(BX) + RET diff --git a/src/runtime/internal/atomic/asm_arm.s b/src/runtime/internal/atomic/asm_arm.s new file mode 100644 index 0000000000..235e8bfd20 --- /dev/null +++ b/src/runtime/internal/atomic/asm_arm.s @@ -0,0 +1,71 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// bool armcas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// }else +// return 0; +// +// To implement runtime∕internal∕atomic·cas in sys_$GOOS_arm.s +// using the native instructions, use: +// +// TEXT runtime∕internal∕atomic·cas(SB),NOSPLIT,$0 +// B runtime∕internal∕atomic·armcas(SB) +// +TEXT runtime∕internal∕atomic·armcas(SB),NOSPLIT,$0-13 + MOVW valptr+0(FP), R1 + MOVW old+4(FP), R2 + MOVW new+8(FP), R3 +casl: + LDREX (R1), R0 + CMP R0, R2 + BNE casfail + + MOVB runtime·goarm(SB), R11 + CMP $7, R11 + BLT 2(PC) + WORD $0xf57ff05a // dmb ishst + + STREX R3, (R1), R0 + CMP $0, R0 + BNE casl + MOVW $1, R0 + + MOVB runtime·goarm(SB), R11 + CMP $7, R11 + BLT 2(PC) + WORD $0xf57ff05b // dmb ish + + MOVB R0, ret+12(FP) + RET +casfail: + MOVW $0, R0 + MOVB R0, ret+12(FP) + RET + +TEXT runtime∕internal∕atomic·Casuintptr(SB),NOSPLIT,$0-13 + B runtime∕internal∕atomic·Cas(SB) + +TEXT runtime∕internal∕atomic·Loaduintptr(SB),NOSPLIT,$0-8 + B runtime∕internal∕atomic·Load(SB) + +TEXT runtime∕internal∕atomic·Loaduint(SB),NOSPLIT,$0-8 + B runtime∕internal∕atomic·Load(SB) + +TEXT runtime∕internal∕atomic·Storeuintptr(SB),NOSPLIT,$0-8 + B runtime∕internal∕atomic·Store(SB) + +TEXT runtime∕internal∕atomic·Xadduintptr(SB),NOSPLIT,$0-8 + B runtime∕internal∕atomic·Xadd(SB) + +TEXT runtime∕internal∕atomic·Loadint64(SB),NOSPLIT,$0-16 + B runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Xaddint64(SB),NOSPLIT,$0-16 + B runtime∕internal∕atomic·Xadd64(SB) diff --git a/src/runtime/internal/atomic/asm_arm64.s b/src/runtime/internal/atomic/asm_arm64.s new file mode 100644 index 0000000000..c255677f78 --- /dev/null +++ b/src/runtime/internal/atomic/asm_arm64.s @@ -0,0 +1,58 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// bool Cas(uint32 *ptr, uint32 old, uint32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R0 + MOVW old+8(FP), R1 + MOVW new+12(FP), R2 +again: + LDAXRW (R0), R3 + CMPW R1, R3 + BNE ok + STLXRW R2, (R0), R3 + CBNZ R3, again +ok: + CSET EQ, R0 + MOVB R0, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-25 + B runtime∕internal∕atomic·Cas64(SB) + +TEXT runtime∕internal∕atomic·Loaduintptr(SB), NOSPLIT, $-8-16 + B runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Loaduint(SB), NOSPLIT, $-8-16 + B runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Storeuintptr(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·Xadduintptr(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Xadd64(SB) + +TEXT runtime∕internal∕atomic·Loadint64(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Xaddint64(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Xadd64(SB) + +// bool Casp(void **val, void *old, void *new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Casp1(SB), NOSPLIT, $0-25 + B runtime∕internal∕atomic·Cas64(SB) diff --git a/src/runtime/internal/atomic/asm_ppc64x.s b/src/runtime/internal/atomic/asm_ppc64x.s new file mode 100644 index 0000000000..87f7f5d892 --- /dev/null +++ b/src/runtime/internal/atomic/asm_ppc64x.s @@ -0,0 +1,225 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +#include "textflag.h" + +// bool cas(uint32 *ptr, uint32 old, uint32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB), NOSPLIT, $0-17 + MOVD ptr+0(FP), R3 + MOVWZ old+8(FP), R4 + MOVWZ new+12(FP), R5 +cas_again: + SYNC + LWAR (R3), R6 + CMPW R6, R4 + BNE cas_fail + STWCCC R5, (R3) + BNE cas_again + MOVD $1, R3 + SYNC + ISYNC + MOVB R3, ret+16(FP) + RET +cas_fail: + MOVD $0, R3 + BR -5(PC) + +// bool runtime∕internal∕atomic·Cas64(uint64 *ptr, uint64 old, uint64 new) +// Atomically: +// if(*val == *old){ +// *val = new; +// return 1; +// } else { +// return 0; +// } +TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25 + MOVD ptr+0(FP), R3 + MOVD old+8(FP), R4 + MOVD new+16(FP), R5 +cas64_again: + SYNC + LDAR (R3), R6 + CMP R6, R4 + BNE cas64_fail + STDCCC R5, (R3) + BNE cas64_again + MOVD $1, R3 + SYNC + ISYNC + MOVB R3, ret+24(FP) + RET +cas64_fail: + MOVD $0, R3 + BR -5(PC) + +TEXT runtime∕internal∕atomic·Casuintptr(SB), NOSPLIT, $0-25 + BR runtime∕internal∕atomic·Cas64(SB) + +TEXT runtime∕internal∕atomic·Loaduintptr(SB), NOSPLIT|NOFRAME, $0-16 + BR runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Loaduint(SB), NOSPLIT|NOFRAME, $0-16 + BR runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Storeuintptr(SB), NOSPLIT, $0-16 + BR runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·Xadduintptr(SB), NOSPLIT, $0-24 + BR runtime∕internal∕atomic·Xadd64(SB) + +TEXT runtime∕internal∕atomic·Loadint64(SB), NOSPLIT, $0-16 + BR runtime∕internal∕atomic·Load64(SB) + +TEXT runtime∕internal∕atomic·Xaddint64(SB), NOSPLIT, $0-16 + BR runtime∕internal∕atomic·Xadd64(SB) + +// bool casp(void **val, void *old, void *new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// } else +// return 0; +TEXT runtime∕internal∕atomic·Casp1(SB), NOSPLIT, $0-25 + BR runtime∕internal∕atomic·Cas64(SB) + +// uint32 xadd(uint32 volatile *ptr, int32 delta) +// Atomically: +// *val += delta; +// return *val; +TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-20 + MOVD ptr+0(FP), R4 + MOVW delta+8(FP), R5 + SYNC + LWAR (R4), R3 + ADD R5, R3 + STWCCC R3, (R4) + BNE -4(PC) + SYNC + ISYNC + MOVW R3, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24 + MOVD ptr+0(FP), R4 + MOVD delta+8(FP), R5 + SYNC + LDAR (R4), R3 + ADD R5, R3 + STDCCC R3, (R4) + BNE -4(PC) + SYNC + ISYNC + MOVD R3, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20 + MOVD ptr+0(FP), R4 + MOVW new+8(FP), R5 + SYNC + LWAR (R4), R3 + STWCCC R5, (R4) + BNE -3(PC) + SYNC + ISYNC + MOVW R3, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24 + MOVD ptr+0(FP), R4 + MOVD new+8(FP), R5 + SYNC + LDAR (R4), R3 + STDCCC R5, (R4) + BNE -3(PC) + SYNC + ISYNC + MOVD R3, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchguintptr(SB), NOSPLIT, $0-24 + BR runtime∕internal∕atomic·Xchg64(SB) + + +TEXT runtime∕internal∕atomic·Storep1(SB), NOSPLIT, $0-16 + BR runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 + MOVD ptr+0(FP), R3 + MOVW val+8(FP), R4 + SYNC + MOVW R4, 0(R3) + RET + +TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 + MOVD ptr+0(FP), R3 + MOVD val+8(FP), R4 + SYNC + MOVD R4, 0(R3) + RET + +// void runtime∕internal∕atomic·Or8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·Or8(SB), NOSPLIT, $0-9 + MOVD ptr+0(FP), R3 + MOVBZ val+8(FP), R4 + // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R5 = (R3 << 0) & ~3 + RLDCR $0, R3, $~3, R5 + // Compute val shift. +#ifdef GOARCH_ppc64 + // Big endian. ptr = ptr ^ 3 + XOR $3, R3 +#endif + // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) + RLDC $3, R3, $(3*8), R6 + // Shift val for aligned ptr. R4 = val << R6 + SLD R6, R4, R4 + +again: + SYNC + LWAR (R5), R6 + OR R4, R6 + STWCCC R6, (R5) + BNE again + SYNC + ISYNC + RET + +// void runtime∕internal∕atomic·And8(byte volatile*, byte); +TEXT runtime∕internal∕atomic·And8(SB), NOSPLIT, $0-9 + MOVD ptr+0(FP), R3 + MOVBZ val+8(FP), R4 + // Align ptr down to 4 bytes so we can use 32-bit load/store. + // R5 = (R3 << 0) & ~3 + RLDCR $0, R3, $~3, R5 + // Compute val shift. +#ifdef GOARCH_ppc64 + // Big endian. ptr = ptr ^ 3 + XOR $3, R3 +#endif + // R6 = ((ptr & 3) * 8) = (ptr << 3) & (3*8) + RLDC $3, R3, $(3*8), R6 + // Shift val for aligned ptr. R4 = val << R6 | ^(0xFF << R6) + MOVD $0xFF, R7 + SLD R6, R4 + SLD R6, R7 + XOR $-1, R7 + OR R7, R4 +again: + SYNC + LWAR (R5), R6 + AND R4, R6 + STWCCC R6, (R5) + BNE again + SYNC + ISYNC + RET diff --git a/src/runtime/internal/atomic/atomic_386.go b/src/runtime/internal/atomic/atomic_386.go new file mode 100644 index 0000000000..ae7b58325a --- /dev/null +++ b/src/runtime/internal/atomic/atomic_386.go @@ -0,0 +1,80 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 + +package atomic + +import "unsafe" + +// The calls to nop are to keep these functions from being inlined. +// If they are inlined we have no guarantee that later rewrites of the +// code by optimizers will preserve the relative order of memory accesses. + +//go:nosplit +func Load(ptr *uint32) uint32 { + nop() + return *ptr +} + +//go:nosplit +func Loadp(ptr unsafe.Pointer) unsafe.Pointer { + nop() + return *(*unsafe.Pointer)(ptr) +} + +//go:nosplit +func Xadd64(ptr *uint64, delta int64) uint64 { + for { + old := *ptr + if Cas64(ptr, old, old+uint64(delta)) { + return old + uint64(delta) + } + } +} + +//go:noescape +func Xadduintptr(ptr *uintptr, delta uintptr) uintptr + +//go:nosplit +func Xchg64(ptr *uint64, new uint64) uint64 { + for { + old := *ptr + if Cas64(ptr, old, new) { + return old + } + } +} + +//go:noescape +func Xadd(ptr *uint32, delta int32) uint32 + +//go:noescape +func Xchg(ptr *uint32, new uint32) uint32 + +//go:noescape +func Xchguintptr(ptr *uintptr, new uintptr) uintptr + +//go:noescape +func Load64(ptr *uint64) uint64 + +//go:noescape +func And8(ptr *uint8, val uint8) + +//go:noescape +func Or8(ptr *uint8, val uint8) + +// NOTE: Do not add atomicxor8 (XOR is not idempotent). + +//go:noescape +func Cas64(ptr *uint64, old, new uint64) bool + +//go:noescape +func Store(ptr *uint32, val uint32) + +//go:noescape +func Store64(ptr *uint64, val uint64) + +// NO go:noescape annotation; see atomic_pointer.go. +func Storep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_amd64x.go b/src/runtime/internal/atomic/atomic_amd64x.go new file mode 100644 index 0000000000..7f6c892364 --- /dev/null +++ b/src/runtime/internal/atomic/atomic_amd64x.go @@ -0,0 +1,69 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build amd64 amd64p32 + +package atomic + +import "unsafe" + +// The calls to nop are to keep these functions from being inlined. +// If they are inlined we have no guarantee that later rewrites of the +// code by optimizers will preserve the relative order of memory accesses. + +//go:nosplit +func Load(ptr *uint32) uint32 { + nop() + return *ptr +} + +//go:nosplit +func Loadp(ptr unsafe.Pointer) unsafe.Pointer { + nop() + return *(*unsafe.Pointer)(ptr) +} + +//go:nosplit +func Load64(ptr *uint64) uint64 { + nop() + return *ptr +} + +//go:noescape +func Xadd(ptr *uint32, delta int32) uint32 + +//go:noescape +func Xadd64(ptr *uint64, delta int64) uint64 + +//go:noescape +func Xadduintptr(ptr *uintptr, delta uintptr) uintptr + +//go:noescape +func Xchg(ptr *uint32, new uint32) uint32 + +//go:noescape +func Xchg64(ptr *uint64, new uint64) uint64 + +//go:noescape +func Xchguintptr(ptr *uintptr, new uintptr) uintptr + +//go:noescape +func And8(ptr *uint8, val uint8) + +//go:noescape +func Or8(ptr *uint8, val uint8) + +// NOTE: Do not add atomicxor8 (XOR is not idempotent). + +//go:noescape +func Cas64(ptr *uint64, old, new uint64) bool + +//go:noescape +func Store(ptr *uint32, val uint32) + +//go:noescape +func Store64(ptr *uint64, val uint64) + +// NO go:noescape annotation; see atomic_pointer.go. +func Storep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_arm.go b/src/runtime/internal/atomic/atomic_arm.go new file mode 100644 index 0000000000..1f33eef3c4 --- /dev/null +++ b/src/runtime/internal/atomic/atomic_arm.go @@ -0,0 +1,180 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm + +package atomic + +import "unsafe" + +type spinlock struct { + v uint32 +} + +//go:nosplit +func (l *spinlock) lock() { + for { + if Cas(&l.v, 0, 1) { + return + } + } +} + +//go:nosplit +func (l *spinlock) unlock() { + Store(&l.v, 0) +} + +var locktab [57]struct { + l spinlock + pad [_CacheLineSize - unsafe.Sizeof(spinlock{})]byte +} + +func addrLock(addr *uint64) *spinlock { + return &locktab[(uintptr(unsafe.Pointer(addr))>>3)%uintptr(len(locktab))].l +} + +// Atomic add and return new value. +//go:nosplit +func Xadd(val *uint32, delta int32) uint32 { + for { + oval := *val + nval := oval + uint32(delta) + if Cas(val, oval, nval) { + return nval + } + } +} + +//go:noescape +func Xadduintptr(ptr *uintptr, delta uintptr) uintptr + +//go:nosplit +func Xchg(addr *uint32, v uint32) uint32 { + for { + old := *addr + if Cas(addr, old, v) { + return old + } + } +} + +//go:nosplit +func Xchguintptr(addr *uintptr, v uintptr) uintptr { + return uintptr(Xchg((*uint32)(unsafe.Pointer(addr)), uint32(v))) +} + +//go:nosplit +func Load(addr *uint32) uint32 { + return Xadd(addr, 0) +} + +// Should be a built-in for unsafe.Pointer? +//go:nosplit +func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { + return unsafe.Pointer(uintptr(p) + x) +} + +//go:nosplit +func Loadp(addr unsafe.Pointer) unsafe.Pointer { + return unsafe.Pointer(uintptr(Xadd((*uint32)(addr), 0))) +} + +//go:nosplit +func Storep1(addr unsafe.Pointer, v unsafe.Pointer) { + for { + old := *(*unsafe.Pointer)(addr) + if Casp1((*unsafe.Pointer)(addr), old, v) { + return + } + } +} + +//go:nosplit +func Store(addr *uint32, v uint32) { + for { + old := *addr + if Cas(addr, old, v) { + return + } + } +} + +//go:nosplit +func Cas64(addr *uint64, old, new uint64) bool { + var ok bool + addrLock(addr).lock() + if *addr == old { + *addr = new + ok = true + } + addrLock(addr).unlock() + return ok +} + +//go:nosplit +func Xadd64(addr *uint64, delta int64) uint64 { + var r uint64 + addrLock(addr).lock() + r = *addr + uint64(delta) + *addr = r + addrLock(addr).unlock() + return r +} + +//go:nosplit +func Xchg64(addr *uint64, v uint64) uint64 { + var r uint64 + addrLock(addr).lock() + r = *addr + *addr = v + addrLock(addr).unlock() + return r +} + +//go:nosplit +func Load64(addr *uint64) uint64 { + var r uint64 + addrLock(addr).lock() + r = *addr + addrLock(addr).unlock() + return r +} + +//go:nosplit +func Store64(addr *uint64, v uint64) { + addrLock(addr).lock() + *addr = v + addrLock(addr).unlock() +} + +//go:nosplit +func Or8(addr *uint8, v uint8) { + // Align down to 4 bytes and use 32-bit CAS. + uaddr := uintptr(unsafe.Pointer(addr)) + addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) + word := uint32(v) << ((uaddr & 3) * 8) // little endian + for { + old := *addr32 + if Cas(addr32, old, old|word) { + return + } + } +} + +//go:nosplit +func And8(addr *uint8, v uint8) { + // Align down to 4 bytes and use 32-bit CAS. + uaddr := uintptr(unsafe.Pointer(addr)) + addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) + word := uint32(v) << ((uaddr & 3) * 8) // little endian + mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian + word |= ^mask + for { + old := *addr32 + if Cas(addr32, old, old&word) { + return + } + } +} diff --git a/src/runtime/internal/atomic/atomic_arm64.go b/src/runtime/internal/atomic/atomic_arm64.go new file mode 100644 index 0000000000..6b32346656 --- /dev/null +++ b/src/runtime/internal/atomic/atomic_arm64.go @@ -0,0 +1,80 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build arm64 + +package atomic + +import "unsafe" + +//go:noescape +func Xadd(ptr *uint32, delta int32) uint32 + +//go:noescape +func Xadd64(ptr *uint64, delta int64) uint64 + +//go:noescape +func Xadduintptr(ptr *uintptr, delta uintptr) uintptr + +//go:noescape +func Xchg(ptr *uint32, new uint32) uint32 + +//go:noescape +func Xchg64(ptr *uint64, new uint64) uint64 + +//go:noescape +func Xchguintptr(ptr *uintptr, new uintptr) uintptr + +//go:noescape +func Load(ptr *uint32) uint32 + +//go:noescape +func Load64(ptr *uint64) uint64 + +//go:noescape +func Loadp(ptr unsafe.Pointer) unsafe.Pointer + +//go:nosplit +func Or8(addr *uint8, v uint8) { + // TODO(dfc) implement this in asm. + // Align down to 4 bytes and use 32-bit CAS. + uaddr := uintptr(unsafe.Pointer(addr)) + addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) + word := uint32(v) << ((uaddr & 3) * 8) // little endian + for { + old := *addr32 + if Cas(addr32, old, old|word) { + return + } + } +} + +//go:nosplit +func And8(addr *uint8, v uint8) { + // TODO(dfc) implement this in asm. + // Align down to 4 bytes and use 32-bit CAS. + uaddr := uintptr(unsafe.Pointer(addr)) + addr32 := (*uint32)(unsafe.Pointer(uaddr &^ 3)) + word := uint32(v) << ((uaddr & 3) * 8) // little endian + mask := uint32(0xFF) << ((uaddr & 3) * 8) // little endian + word |= ^mask + for { + old := *addr32 + if Cas(addr32, old, old&word) { + return + } + } +} + +//go:noescape +func Cas64(ptr *uint64, old, new uint64) bool + +//go:noescape +func Store(ptr *uint32, val uint32) + +//go:noescape +func Store64(ptr *uint64, val uint64) + +// NO go:noescape annotation; see atomic_pointer.go. +func Storep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_arm64.s b/src/runtime/internal/atomic/atomic_arm64.s new file mode 100644 index 0000000000..7b1b0efaf6 --- /dev/null +++ b/src/runtime/internal/atomic/atomic_arm64.s @@ -0,0 +1,113 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// uint32 runtime∕internal∕atomic·Load(uint32 volatile* addr) +TEXT ·Load(SB),NOSPLIT,$-8-12 + MOVD ptr+0(FP), R0 + LDARW (R0), R0 + MOVW R0, ret+8(FP) + RET + +// uint64 runtime∕internal∕atomic·Load64(uint64 volatile* addr) +TEXT ·Load64(SB),NOSPLIT,$-8-16 + MOVD ptr+0(FP), R0 + LDAR (R0), R0 + MOVD R0, ret+8(FP) + RET + +// void *runtime∕internal∕atomic·Loadp(void *volatile *addr) +TEXT ·Loadp(SB),NOSPLIT,$-8-16 + MOVD ptr+0(FP), R0 + LDAR (R0), R0 + MOVD R0, ret+8(FP) + RET + +TEXT runtime∕internal∕atomic·Storep1(SB), NOSPLIT, $0-16 + B runtime∕internal∕atomic·Store64(SB) + +TEXT runtime∕internal∕atomic·Store(SB), NOSPLIT, $0-12 + MOVD ptr+0(FP), R0 + MOVW val+8(FP), R1 + STLRW R1, (R0) + RET + +TEXT runtime∕internal∕atomic·Store64(SB), NOSPLIT, $0-16 + MOVD ptr+0(FP), R0 + MOVD val+8(FP), R1 + STLR R1, (R0) + RET + +TEXT runtime∕internal∕atomic·Xchg(SB), NOSPLIT, $0-20 +again: + MOVD ptr+0(FP), R0 + MOVW new+8(FP), R1 + LDAXRW (R0), R2 + STLXRW R1, (R0), R3 + CBNZ R3, again + MOVW R2, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchg64(SB), NOSPLIT, $0-24 +again: + MOVD ptr+0(FP), R0 + MOVD new+8(FP), R1 + LDAXR (R0), R2 + STLXR R1, (R0), R3 + CBNZ R3, again + MOVD R2, ret+16(FP) + RET + +// bool runtime∕internal∕atomic·Cas64(uint64 *ptr, uint64 old, uint64 new) +// Atomically: +// if(*val == *old){ +// *val = new; +// return 1; +// } else { +// return 0; +// } +TEXT runtime∕internal∕atomic·Cas64(SB), NOSPLIT, $0-25 + MOVD ptr+0(FP), R0 + MOVD old+8(FP), R1 + MOVD new+16(FP), R2 +again: + LDAXR (R0), R3 + CMP R1, R3 + BNE ok + STLXR R2, (R0), R3 + CBNZ R3, again +ok: + CSET EQ, R0 + MOVB R0, ret+24(FP) + RET + +// uint32 xadd(uint32 volatile *ptr, int32 delta) +// Atomically: +// *val += delta; +// return *val; +TEXT runtime∕internal∕atomic·Xadd(SB), NOSPLIT, $0-20 +again: + MOVD ptr+0(FP), R0 + MOVW delta+8(FP), R1 + LDAXRW (R0), R2 + ADDW R2, R1, R2 + STLXRW R2, (R0), R3 + CBNZ R3, again + MOVW R2, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xadd64(SB), NOSPLIT, $0-24 +again: + MOVD ptr+0(FP), R0 + MOVD delta+8(FP), R1 + LDAXR (R0), R2 + ADD R2, R1, R2 + STLXR R2, (R0), R3 + CBNZ R3, again + MOVD R2, ret+16(FP) + RET + +TEXT runtime∕internal∕atomic·Xchguintptr(SB), NOSPLIT, $0-24 + B runtime∕internal∕atomic·Xchg64(SB) diff --git a/src/runtime/internal/atomic/atomic_ppc64x.go b/src/runtime/internal/atomic/atomic_ppc64x.go new file mode 100644 index 0000000000..bf82b82643 --- /dev/null +++ b/src/runtime/internal/atomic/atomic_ppc64x.go @@ -0,0 +1,56 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package atomic + +import "unsafe" + +//go:noescape +func Xadd(ptr *uint32, delta int32) uint32 + +//go:noescape +func Xadd64(ptr *uint64, delta int64) uint64 + +//go:noescape +func Xadduintptr(ptr *uintptr, delta uintptr) uintptr + +//go:noescape +func Xchg(ptr *uint32, new uint32) uint32 + +//go:noescape +func Xchg64(ptr *uint64, new uint64) uint64 + +//go:noescape +func Xchguintptr(ptr *uintptr, new uintptr) uintptr + +//go:noescape +func Load(ptr *uint32) uint32 + +//go:noescape +func Load64(ptr *uint64) uint64 + +//go:noescape +func Loadp(ptr unsafe.Pointer) unsafe.Pointer + +//go:noescape +func And8(ptr *uint8, val uint8) + +//go:noescape +func Or8(ptr *uint8, val uint8) + +// NOTE: Do not add atomicxor8 (XOR is not idempotent). + +//go:noescape +func Cas64(ptr *uint64, old, new uint64) bool + +//go:noescape +func Store(ptr *uint32, val uint32) + +//go:noescape +func Store64(ptr *uint64, val uint64) + +// NO go:noescape annotation; see atomic_pointer.go. +func Storep1(ptr unsafe.Pointer, val unsafe.Pointer) diff --git a/src/runtime/internal/atomic/atomic_ppc64x.s b/src/runtime/internal/atomic/atomic_ppc64x.s new file mode 100644 index 0000000000..1a7537ed33 --- /dev/null +++ b/src/runtime/internal/atomic/atomic_ppc64x.s @@ -0,0 +1,40 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +#include "textflag.h" + +// uint32 runtime∕internal∕atomic·Load(uint32 volatile* addr) +TEXT ·Load(SB),NOSPLIT|NOFRAME,$-8-12 + MOVD addr+0(FP), R3 + SYNC + MOVWZ 0(R3), R3 + CMPW R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7,0x4 + ISYNC + MOVW R3, ret+8(FP) + RET + +// uint64 runtime∕internal∕atomic·Load64(uint64 volatile* addr) +TEXT ·Load64(SB),NOSPLIT|NOFRAME,$-8-16 + MOVD addr+0(FP), R3 + SYNC + MOVD 0(R3), R3 + CMP R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7,0x4 + ISYNC + MOVD R3, ret+8(FP) + RET + +// void *runtime∕internal∕atomic·Loadp(void *volatile *addr) +TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$-8-16 + MOVD addr+0(FP), R3 + SYNC + MOVD 0(R3), R3 + CMP R3, R3, CR7 + BC 4, 30, 1(PC) // bne- cr7,0x4 + ISYNC + MOVD R3, ret+8(FP) + RET diff --git a/src/runtime/internal/atomic/atomic_test.go b/src/runtime/internal/atomic/atomic_test.go new file mode 100644 index 0000000000..e8ec788d6a --- /dev/null +++ b/src/runtime/internal/atomic/atomic_test.go @@ -0,0 +1,67 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic_test + +import ( + "runtime" + "runtime/internal/atomic" + "testing" + "unsafe" +) + +func runParallel(N, iter int, f func()) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(int(N))) + done := make(chan bool) + for i := 0; i < N; i++ { + go func() { + for j := 0; j < iter; j++ { + f() + } + done <- true + }() + } + for i := 0; i < N; i++ { + <-done + } +} + +func TestXadduintptr(t *testing.T) { + const N = 20 + const iter = 100000 + inc := uintptr(100) + total := uintptr(0) + runParallel(N, iter, func() { + atomic.Xadduintptr(&total, inc) + }) + if want := uintptr(N * iter * inc); want != total { + t.Fatalf("xadduintpr error, want %d, got %d", want, total) + } + total = 0 + runParallel(N, iter, func() { + atomic.Xadduintptr(&total, inc) + atomic.Xadduintptr(&total, uintptr(-int64(inc))) + }) + if total != 0 { + t.Fatalf("xadduintpr total error, want %d, got %d", 0, total) + } +} + +// Tests that xadduintptr correctly updates 64-bit values. The place where +// we actually do so is mstats.go, functions mSysStat{Inc,Dec}. +func TestXadduintptrOnUint64(t *testing.T) { + /* if runtime.BigEndian != 0 { + // On big endian architectures, we never use xadduintptr to update + // 64-bit values and hence we skip the test. (Note that functions + // mSysStat{Inc,Dec} in mstats.go have explicit checks for + // big-endianness.) + return + }*/ + const inc = 100 + val := uint64(0) + atomic.Xadduintptr((*uintptr)(unsafe.Pointer(&val)), inc) + if inc != val { + t.Fatalf("xadduintptr should increase lower-order bits, want %d, got %d", inc, val) + } +} diff --git a/src/runtime/internal/atomic/stubs.go b/src/runtime/internal/atomic/stubs.go new file mode 100644 index 0000000000..826829938d --- /dev/null +++ b/src/runtime/internal/atomic/stubs.go @@ -0,0 +1,35 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package atomic + +import "unsafe" + +//go:noescape +func Cas(ptr *uint32, old, new uint32) bool + +// NO go:noescape annotation; see atomic_pointer.go. +func Casp1(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool + +func nop() // call to prevent inlining of function body + +//go:noescape +func Casuintptr(ptr *uintptr, old, new uintptr) bool + +//go:noescape +func Storeuintptr(ptr *uintptr, new uintptr) + +//go:noescape +func Loaduintptr(ptr *uintptr) uintptr + +//go:noescape +func Loaduint(ptr *uint) uint + +// TODO(matloob): Should these functions have the go:noescape annotation? + +//go:noescape +func Loadint64(ptr *int64) int64 + +//go:noescape +func Xaddint64(ptr *int64, delta int64) int64 diff --git a/src/runtime/internal/atomic/sys_darwin_arm.s b/src/runtime/internal/atomic/sys_darwin_arm.s new file mode 100644 index 0000000000..01b7aefd1c --- /dev/null +++ b/src/runtime/internal/atomic/sys_darwin_arm.s @@ -0,0 +1,11 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·armcas(SB) + +TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·Cas(SB) diff --git a/src/runtime/internal/atomic/sys_freebsd_arm.s b/src/runtime/internal/atomic/sys_freebsd_arm.s new file mode 100644 index 0000000000..30d49b8d39 --- /dev/null +++ b/src/runtime/internal/atomic/sys_freebsd_arm.s @@ -0,0 +1,19 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// TODO(minux): this is only valid for ARMv6+ +// bool armcas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// }else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·armcas(SB) + +TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·Cas(SB) diff --git a/src/runtime/internal/atomic/sys_linux_arm.s b/src/runtime/internal/atomic/sys_linux_arm.s new file mode 100644 index 0000000000..f8de2a2a41 --- /dev/null +++ b/src/runtime/internal/atomic/sys_linux_arm.s @@ -0,0 +1,42 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// Use kernel version instead of native armcas in asm_arm.s. +// See ../../../sync/atomic/asm_linux_arm.s for details. +TEXT cas<>(SB),NOSPLIT,$0 + MOVW $0xffff0fc0, R15 // R15 is hardware PC. + +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0 + MOVW ptr+0(FP), R2 + MOVW old+4(FP), R0 +loop: + MOVW new+8(FP), R1 + BL cas<>(SB) + BCC check + MOVW $1, R0 + MOVB R0, ret+12(FP) + RET +check: + // Kernel lies; double-check. + MOVW ptr+0(FP), R2 + MOVW old+4(FP), R0 + MOVW 0(R2), R3 + CMP R0, R3 + BEQ loop + MOVW $0, R0 + MOVB R0, ret+12(FP) + RET + +TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·Cas(SB) + +// As for cas, memory barriers are complicated on ARM, but the kernel +// provides a user helper. ARMv5 does not support SMP and has no +// memory barrier instruction at all. ARMv6 added SMP support and has +// a memory barrier, but it requires writing to a coprocessor +// register. ARMv7 introduced the DMB instruction, but it's expensive +// even on single-core devices. The kernel helper takes care of all of +// this for us.
\ No newline at end of file diff --git a/src/runtime/internal/atomic/sys_nacl_arm.s b/src/runtime/internal/atomic/sys_nacl_arm.s new file mode 100644 index 0000000000..efa960474d --- /dev/null +++ b/src/runtime/internal/atomic/sys_nacl_arm.s @@ -0,0 +1,16 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT runtime∕internal∕atomic·Casp(SB),NOSPLIT,$0 + B runtime·cas(SB) + +// This is only valid for ARMv6+, however, NaCl/ARM is only defined +// for ARMv7A anyway. +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·armcas(SB) + +TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·Cas(SB) diff --git a/src/runtime/internal/atomic/sys_netbsd_arm.s b/src/runtime/internal/atomic/sys_netbsd_arm.s new file mode 100644 index 0000000000..3277d94af6 --- /dev/null +++ b/src/runtime/internal/atomic/sys_netbsd_arm.s @@ -0,0 +1,21 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// TODO(minux): this is only valid for ARMv6+ +// bool Armcas(int32 *val, int32 old, int32 new) +// Atomically: +// if(*val == old){ +// *val = new; +// return 1; +// }else +// return 0; +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·armcas(SB) + +TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·Cas(SB) + + diff --git a/src/runtime/internal/atomic/sys_openbsd_arm.s b/src/runtime/internal/atomic/sys_openbsd_arm.s new file mode 100644 index 0000000000..01b7aefd1c --- /dev/null +++ b/src/runtime/internal/atomic/sys_openbsd_arm.s @@ -0,0 +1,11 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +TEXT runtime∕internal∕atomic·Cas(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·armcas(SB) + +TEXT runtime∕internal∕atomic·Casp1(SB),NOSPLIT,$0 + B runtime∕internal∕atomic·Cas(SB) diff --git a/src/runtime/internal/atomic/textflag.h b/src/runtime/internal/atomic/textflag.h new file mode 100644 index 0000000000..dbf3d9977c --- /dev/null +++ b/src/runtime/internal/atomic/textflag.h @@ -0,0 +1,30 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file defines flags attached to various functions +// and data objects. The compilers, assemblers, and linker must +// all agree on these values. + +// Don't profile the marked routine. This flag is deprecated. +#define NOPROF 1 +// It is ok for the linker to get multiple of these symbols. It will +// pick one of the duplicates to use. +#define DUPOK 2 +// Don't insert stack check preamble. +#define NOSPLIT 4 +// Put this data in a read-only section. +#define RODATA 8 +// This data contains no pointers. +#define NOPTR 16 +// This is a wrapper function and should not count as disabling 'recover'. +#define WRAPPER 32 +// This function uses its incoming context register. +#define NEEDCTXT 64 +// Allocate a word of thread local storage and store the offset from the +// thread local base to the thread local storage in this variable. +#define TLSBSS 256 +// Do not insert instructions to allocate a stack frame for this function. +// Only valid on functions that declare a frame size of 0. +// TODO(mwhudson): only implemented for ppc64x at present. +#define NOFRAME 512 |