aboutsummaryrefslogtreecommitdiff
path: root/src/runtime
diff options
context:
space:
mode:
authorMichael Anthony Knyszek <mknyszek@google.com>2020-10-22 16:29:04 +0000
committerMichael Knyszek <mknyszek@google.com>2021-02-16 15:51:50 +0000
commite0215315f51c62f6d2c5ea5ed7008b7e7963dd5d (patch)
tree1ea397eaca06f15c08b4fd79a6a95af6f215ed26 /src/runtime
parentb81efb7ec4348951211058cf4fdfc045c75255d6 (diff)
downloadgo-e0215315f51c62f6d2c5ea5ed7008b7e7963dd5d.tar.xz
[dev.regabi] reflect: support for register ABI on amd64 for reflect.(Value).Call
This change adds support for the new register ABI on amd64 to reflect.(Value).Call. If internal/abi's register counts are non-zero, reflect will try to set up arguments in registers on the Call path. Note that because the register ABI becomes ABI0 with zero registers available, this should keep working as it did before. This change does not add any tests for the register ABI case because there's no way to do so at the moment. For #40724. Change-Id: I8aa089a5aa5a31b72e56b3d9388dd3f82203985b Reviewed-on: https://go-review.googlesource.com/c/go/+/272568 Trust: Michael Knyszek <mknyszek@google.com> Run-TryBot: Michael Knyszek <mknyszek@google.com> TryBot-Result: Go Bot <gobot@golang.org> Reviewed-by: Cherry Zhang <cherryyz@google.com> Reviewed-by: Than McIntosh <thanm@google.com>
Diffstat (limited to 'src/runtime')
-rw-r--r--src/runtime/asm_386.s23
-rw-r--r--src/runtime/asm_amd64.s100
-rw-r--r--src/runtime/asm_arm.s24
-rw-r--r--src/runtime/asm_arm64.s23
-rw-r--r--src/runtime/asm_mips64x.s23
-rw-r--r--src/runtime/asm_mipsx.s23
-rw-r--r--src/runtime/asm_ppc64x.s23
-rw-r--r--src/runtime/asm_riscv64.s27
-rw-r--r--src/runtime/asm_s390x.s23
-rw-r--r--src/runtime/asm_wasm.s23
-rw-r--r--src/runtime/mbarrier.go10
-rw-r--r--src/runtime/mfinal.go7
-rw-r--r--src/runtime/panic.go13
-rw-r--r--src/runtime/stubs.go56
-rw-r--r--src/runtime/syscall_windows.go7
15 files changed, 274 insertions, 131 deletions
diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s
index 429f3fef82..471451df28 100644
--- a/src/runtime/asm_386.s
+++ b/src/runtime/asm_386.s
@@ -458,7 +458,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
JMP runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -470,8 +470,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
JMP AX
// Note: can't just "JMP NAME(SB)" - bad inlining results.
-TEXT ·reflectcall(SB), NOSPLIT, $0-20
- MOVL argsize+12(FP), CX
+TEXT ·reflectcall(SB), NOSPLIT, $0-28
+ MOVL frameSize+20(FP), CX
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -503,11 +503,11 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-20
JMP AX
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVL argptr+8(FP), SI; \
- MOVL argsize+12(FP), CX; \
+ MOVL stackArgs+8(FP), SI; \
+ MOVL stackArgsSize+12(FP), CX; \
MOVL SP, DI; \
REP;MOVSB; \
/* call function */ \
@@ -516,10 +516,10 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
PCDATA $PCDATA_StackMapIndex, $0; \
CALL AX; \
/* copy return values back */ \
- MOVL argtype+0(FP), DX; \
- MOVL argptr+8(FP), DI; \
- MOVL argsize+12(FP), CX; \
- MOVL retoffset+16(FP), BX; \
+ MOVL stackArgsType+0(FP), DX; \
+ MOVL stackArgs+8(FP), DI; \
+ MOVL stackArgsSize+12(FP), CX; \
+ MOVL stackRetOffset+16(FP), BX; \
MOVL SP, SI; \
ADDL BX, DI; \
ADDL BX, SI; \
@@ -531,11 +531,12 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $16-0
+TEXT callRet<>(SB), NOSPLIT, $20-0
MOVL DX, 0(SP)
MOVL DI, 4(SP)
MOVL SI, 8(SP)
MOVL CX, 12(SP)
+ MOVL $0, 16(SP)
CALL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index 93280eee4a..5e1ed9b2ad 100644
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -445,8 +445,74 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
MOVL $0, DX
JMP runtime·morestack(SB)
+#ifdef GOEXPERIMENT_REGABI
+// spillArgs stores return values from registers to a *internal/abi.RegArgs in R12.
+TEXT spillArgs<>(SB),NOSPLIT,$0-0
+ MOVQ AX, 0(R12)
+ MOVQ BX, 8(R12)
+ MOVQ CX, 16(R12)
+ MOVQ DI, 24(R12)
+ MOVQ SI, 32(R12)
+ MOVQ R8, 40(R12)
+ MOVQ R9, 48(R12)
+ MOVQ R10, 56(R12)
+ MOVQ R11, 64(R12)
+ MOVQ X0, 72(R12)
+ MOVQ X1, 80(R12)
+ MOVQ X2, 88(R12)
+ MOVQ X3, 96(R12)
+ MOVQ X4, 104(R12)
+ MOVQ X5, 112(R12)
+ MOVQ X6, 120(R12)
+ MOVQ X7, 128(R12)
+ MOVQ X8, 136(R12)
+ MOVQ X9, 144(R12)
+ MOVQ X10, 152(R12)
+ MOVQ X11, 160(R12)
+ MOVQ X12, 168(R12)
+ MOVQ X13, 176(R12)
+ MOVQ X14, 184(R12)
+ RET
+
+// unspillArgs loads args into registers from a *internal/abi.RegArgs in R12.
+TEXT unspillArgs<>(SB),NOSPLIT,$0-0
+ MOVQ 0(R12), AX
+ MOVQ 8(R12), BX
+ MOVQ 16(R12), CX
+ MOVQ 24(R12), DI
+ MOVQ 32(R12), SI
+ MOVQ 40(R12), R8
+ MOVQ 48(R12), R9
+ MOVQ 56(R12), R10
+ MOVQ 64(R12), R11
+ MOVQ 72(R12), X0
+ MOVQ 80(R12), X1
+ MOVQ 88(R12), X2
+ MOVQ 96(R12), X3
+ MOVQ 104(R12), X4
+ MOVQ 112(R12), X5
+ MOVQ 120(R12), X6
+ MOVQ 128(R12), X7
+ MOVQ 136(R12), X8
+ MOVQ 144(R12), X9
+ MOVQ 152(R12), X10
+ MOVQ 160(R12), X11
+ MOVQ 168(R12), X12
+ MOVQ 176(R12), X13
+ MOVQ 184(R12), X14
+ RET
+#else
+// spillArgs stores return values from registers to a pointer in R12.
+TEXT spillArgs<>(SB),NOSPLIT,$0-0
+ RET
+
+// unspillArgs loads args into registers from a pointer in R12.
+TEXT unspillArgs<>(SB),NOSPLIT,$0-0
+ RET
+#endif
+
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -458,8 +524,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
JMP AX
// Note: can't just "JMP NAME(SB)" - bad inlining results.
-TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
- MOVLQZX argsize+24(FP), CX
+TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-48
+ MOVLQZX frameSize+32(FP), CX
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -491,23 +557,28 @@ TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
JMP AX
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVQ argptr+16(FP), SI; \
- MOVLQZX argsize+24(FP), CX; \
+ MOVQ stackArgs+16(FP), SI; \
+ MOVLQZX stackArgsSize+24(FP), CX; \
MOVQ SP, DI; \
REP;MOVSB; \
+ /* set up argument registers */ \
+ MOVQ regArgs+40(FP), R12; \
+ CALL unspillArgs<>(SB); \
/* call function */ \
MOVQ f+8(FP), DX; \
PCDATA $PCDATA_StackMapIndex, $0; \
- MOVQ (DX), AX; \
- CALL AX; \
- /* copy return values back */ \
- MOVQ argtype+0(FP), DX; \
- MOVQ argptr+16(FP), DI; \
- MOVLQZX argsize+24(FP), CX; \
- MOVLQZX retoffset+28(FP), BX; \
+ MOVQ (DX), R12; \
+ CALL R12; \
+ /* copy register return values back */ \
+ MOVQ regArgs+40(FP), R12; \
+ CALL spillArgs<>(SB); \
+ MOVLQZX stackArgsSize+24(FP), CX; \
+ MOVLQZX stackRetOffset+28(FP), BX; \
+ MOVQ stackArgs+16(FP), DI; \
+ MOVQ stackArgsType+0(FP), DX; \
MOVQ SP, SI; \
ADDQ BX, DI; \
ADDQ BX, SI; \
@@ -519,12 +590,13 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
NO_LOCAL_POINTERS
MOVQ DX, 0(SP)
MOVQ DI, 8(SP)
MOVQ SI, 16(SP)
MOVQ CX, 24(SP)
+ MOVQ R12, 32(SP)
CALL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s
index 8eec84d3f2..23619b1408 100644
--- a/src/runtime/asm_arm.s
+++ b/src/runtime/asm_arm.s
@@ -404,7 +404,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
B runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -415,8 +415,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
MOVW $NAME(SB), R1; \
B (R1)
-TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-20
- MOVW argsize+12(FP), R0
+TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
+ MOVW frameSize+20(FP), R0
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -448,11 +448,11 @@ TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-20
B (R1)
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-28; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVW argptr+8(FP), R0; \
- MOVW argsize+12(FP), R2; \
+ MOVW stackArgs+8(FP), R0; \
+ MOVW stackArgsSize+12(FP), R2; \
ADD $4, R13, R1; \
CMP $0, R2; \
B.EQ 5(PC); \
@@ -466,10 +466,10 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
PCDATA $PCDATA_StackMapIndex, $0; \
BL (R0); \
/* copy return values back */ \
- MOVW argtype+0(FP), R4; \
- MOVW argptr+8(FP), R0; \
- MOVW argsize+12(FP), R2; \
- MOVW retoffset+16(FP), R3; \
+ MOVW stackArgsType+0(FP), R4; \
+ MOVW stackArgs+8(FP), R0; \
+ MOVW stackArgsSize+12(FP), R2; \
+ MOVW stackArgsRetOffset+16(FP), R3; \
ADD $4, R13, R1; \
ADD R3, R1; \
ADD R3, R0; \
@@ -481,11 +481,13 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $16-0
+TEXT callRet<>(SB), NOSPLIT, $20-0
MOVW R4, 4(R13)
MOVW R0, 8(R13)
MOVW R1, 12(R13)
MOVW R2, 16(R13)
+ MOVW $0, R7
+ MOVW R7, 20(R13)
BL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s
index 8e4a1f74f9..0ab92be1e4 100644
--- a/src/runtime/asm_arm64.s
+++ b/src/runtime/asm_arm64.s
@@ -312,7 +312,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
B runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -325,8 +325,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
B (R27)
// Note: can't just "B NAME(SB)" - bad inlining results.
-TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
- MOVWU argsize+24(FP), R16
+TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
+ MOVWU frameSize+32(FP), R16
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -358,11 +358,11 @@ TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
B (R0)
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVD arg+16(FP), R3; \
- MOVWU argsize+24(FP), R4; \
+ MOVD stackArgs+16(FP), R3; \
+ MOVWU stackArgsSize+24(FP), R4; \
ADD $8, RSP, R5; \
BIC $0xf, R4, R6; \
CBZ R6, 6(PC); \
@@ -388,10 +388,10 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
PCDATA $PCDATA_StackMapIndex, $0; \
BL (R0); \
/* copy return values back */ \
- MOVD argtype+0(FP), R7; \
- MOVD arg+16(FP), R3; \
- MOVWU n+24(FP), R4; \
- MOVWU retoffset+28(FP), R6; \
+ MOVD stackArgsType+0(FP), R7; \
+ MOVD stackArgs+16(FP), R3; \
+ MOVWU stackArgsSize+24(FP), R4; \
+ MOVWU stackRetOffset+28(FP), R6; \
ADD $8, RSP, R5; \
ADD R6, R5; \
ADD R6, R3; \
@@ -403,11 +403,12 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $40-0
+TEXT callRet<>(SB), NOSPLIT, $48-0
MOVD R7, 8(RSP)
MOVD R3, 16(RSP)
MOVD R5, 24(RSP)
MOVD R4, 32(RSP)
+ MOVD $0, 40(RSP)
BL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s
index 054a89dc37..694950663a 100644
--- a/src/runtime/asm_mips64x.s
+++ b/src/runtime/asm_mips64x.s
@@ -264,7 +264,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
JMP runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -277,8 +277,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
JMP (R4)
// Note: can't just "BR NAME(SB)" - bad inlining results.
-TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
- MOVWU argsize+24(FP), R1
+TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
+ MOVWU frameSize+32(FP), R1
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -310,11 +310,11 @@ TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
JMP (R4)
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVV arg+16(FP), R1; \
- MOVWU argsize+24(FP), R2; \
+ MOVV stackArgs+16(FP), R1; \
+ MOVWU stackArgsSize+24(FP), R2; \
MOVV R29, R3; \
ADDV $8, R3; \
ADDV R3, R2; \
@@ -330,10 +330,10 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
PCDATA $PCDATA_StackMapIndex, $0; \
JAL (R4); \
/* copy return values back */ \
- MOVV argtype+0(FP), R5; \
- MOVV arg+16(FP), R1; \
- MOVWU n+24(FP), R2; \
- MOVWU retoffset+28(FP), R4; \
+ MOVV stackArgsType+0(FP), R5; \
+ MOVV stackArgs+16(FP), R1; \
+ MOVWU stackArgsSize+24(FP), R2; \
+ MOVWU stackRetOffset+28(FP), R4; \
ADDV $8, R29, R3; \
ADDV R4, R3; \
ADDV R4, R1; \
@@ -345,11 +345,12 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
MOVV R5, 8(R29)
MOVV R1, 16(R29)
MOVV R3, 24(R29)
MOVV R2, 32(R29)
+ MOVV $0, 40(R29)
JAL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s
index f57437d590..8e5753d255 100644
--- a/src/runtime/asm_mipsx.s
+++ b/src/runtime/asm_mipsx.s
@@ -265,7 +265,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
JMP runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
@@ -276,8 +276,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0-0
MOVW $NAME(SB), R4; \
JMP (R4)
-TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-20
- MOVW argsize+12(FP), R1
+TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-28
+ MOVW frameSize+20(FP), R1
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
@@ -310,11 +310,11 @@ TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-20
JMP (R4)
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB),WRAPPER,$MAXSIZE-20; \
+TEXT NAME(SB),WRAPPER,$MAXSIZE-28; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVW arg+8(FP), R1; \
- MOVW argsize+12(FP), R2; \
+ MOVW stackArgs+8(FP), R1; \
+ MOVW stackArgsSize+12(FP), R2; \
MOVW R29, R3; \
ADDU $4, R3; \
ADDU R3, R2; \
@@ -330,10 +330,10 @@ TEXT NAME(SB),WRAPPER,$MAXSIZE-20; \
PCDATA $PCDATA_StackMapIndex, $0; \
JAL (R4); \
/* copy return values back */ \
- MOVW argtype+0(FP), R5; \
- MOVW arg+8(FP), R1; \
- MOVW n+12(FP), R2; \
- MOVW retoffset+16(FP), R4; \
+ MOVW stackArgsType+0(FP), R5; \
+ MOVW stackArgs+8(FP), R1; \
+ MOVW stackArgsSize+12(FP), R2; \
+ MOVW stackRetOffset+16(FP), R4; \
ADDU $4, R29, R3; \
ADDU R4, R3; \
ADDU R4, R1; \
@@ -345,11 +345,12 @@ TEXT NAME(SB),WRAPPER,$MAXSIZE-20; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $16-0
+TEXT callRet<>(SB), NOSPLIT, $20-0
MOVW R5, 4(R29)
MOVW R1, 8(R29)
MOVW R3, 12(R29)
MOVW R2, 16(R29)
+ MOVW $0, 20(R29)
JAL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s
index 763a92adf1..834023cce1 100644
--- a/src/runtime/asm_ppc64x.s
+++ b/src/runtime/asm_ppc64x.s
@@ -339,7 +339,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
BR runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -353,8 +353,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
BR (CTR)
// Note: can't just "BR NAME(SB)" - bad inlining results.
-TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
- MOVWZ argsize+24(FP), R3
+TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
+ MOVWZ frameSize+32(FP), R3
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -387,11 +387,11 @@ TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
BR (CTR)
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVD arg+16(FP), R3; \
- MOVWZ argsize+24(FP), R4; \
+ MOVD stackArgs+16(FP), R3; \
+ MOVWZ stackArgsSize+24(FP), R4; \
MOVD R1, R5; \
CMP R4, $8; \
BLT tailsetup; \
@@ -439,10 +439,10 @@ callfn: \
MOVD 24(R1), R2; \
#endif \
/* copy return values back */ \
- MOVD argtype+0(FP), R7; \
- MOVD arg+16(FP), R3; \
- MOVWZ n+24(FP), R4; \
- MOVWZ retoffset+28(FP), R6; \
+ MOVD stackArgsType+0(FP), R7; \
+ MOVD stackArgs+16(FP), R3; \
+ MOVWZ stackArgsSize+24(FP), R4; \
+ MOVWZ stackRetOffset+28(FP), R6; \
ADD $FIXED_FRAME, R1, R5; \
ADD R6, R5; \
ADD R6, R3; \
@@ -454,11 +454,12 @@ callfn: \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
MOVD R7, FIXED_FRAME+0(R1)
MOVD R3, FIXED_FRAME+8(R1)
MOVD R5, FIXED_FRAME+16(R1)
MOVD R4, FIXED_FRAME+24(R1)
+ MOVD $0, FIXED_FRAME+32(R1)
BL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s
index cf460d1586..31e324d677 100644
--- a/src/runtime/asm_riscv64.s
+++ b/src/runtime/asm_riscv64.s
@@ -359,7 +359,7 @@ TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
RET
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -371,13 +371,13 @@ TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
JALR ZERO, T2
// Note: can't just "BR NAME(SB)" - bad inlining results.
-// func call(argtype *rtype, fn, arg unsafe.Pointer, n uint32, retoffset uint32)
+// func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
TEXT reflect·call(SB), NOSPLIT, $0-0
JMP ·reflectcall(SB)
-// func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32)
-TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
- MOVWU argsize+24(FP), T0
+// func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
+TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
+ MOVWU frameSize+32(FP), T0
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -409,11 +409,11 @@ TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
JALR ZERO, T2
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOV arg+16(FP), A1; \
- MOVWU argsize+24(FP), A2; \
+ MOV stackArgs+16(FP), A1; \
+ MOVWU stackArgsSize+24(FP), A2; \
MOV X2, A3; \
ADD $8, A3; \
ADD A3, A2; \
@@ -429,10 +429,10 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
PCDATA $PCDATA_StackMapIndex, $0; \
JALR RA, A4; \
/* copy return values back */ \
- MOV argtype+0(FP), A5; \
- MOV arg+16(FP), A1; \
- MOVWU n+24(FP), A2; \
- MOVWU retoffset+28(FP), A4; \
+ MOV stackArgsType+0(FP), A5; \
+ MOV stackArgs+16(FP), A1; \
+ MOVWU stackArgsSize+24(FP), A2; \
+ MOVWU stackRetOffset+28(FP), A4; \
ADD $8, X2, A3; \
ADD A4, A3; \
ADD A4, A1; \
@@ -444,11 +444,12 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
MOV A5, 8(X2)
MOV A1, 16(X2)
MOV A3, 24(X2)
MOV A2, 32(X2)
+ MOV $0, 40(X2)
CALL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s
index 1cd5eca06f..fbd185c353 100644
--- a/src/runtime/asm_s390x.s
+++ b/src/runtime/asm_s390x.s
@@ -353,7 +353,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
BR runtime·morestack(SB)
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -366,8 +366,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
BR (R5)
// Note: can't just "BR NAME(SB)" - bad inlining results.
-TEXT ·reflectcall(SB), NOSPLIT, $-8-32
- MOVWZ argsize+24(FP), R3
+TEXT ·reflectcall(SB), NOSPLIT, $-8-48
+ MOVWZ frameSize+32(FP), R3
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -399,11 +399,11 @@ TEXT ·reflectcall(SB), NOSPLIT, $-8-32
BR (R5)
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVD arg+16(FP), R4; \
- MOVWZ argsize+24(FP), R5; \
+ MOVD stackArgs+16(FP), R4; \
+ MOVWZ stackArgsSize+24(FP), R5; \
MOVD $stack-MAXSIZE(SP), R6; \
loopArgs: /* copy 256 bytes at a time */ \
CMP R5, $256; \
@@ -424,11 +424,11 @@ callFunction: \
PCDATA $PCDATA_StackMapIndex, $0; \
BL (R8); \
/* copy return values back */ \
- MOVD argtype+0(FP), R7; \
- MOVD arg+16(FP), R6; \
- MOVWZ n+24(FP), R5; \
+ MOVD stackArgsType+0(FP), R7; \
+ MOVD stackArgs+16(FP), R6; \
+ MOVWZ stackArgsSize+24(FP), R5; \
MOVD $stack-MAXSIZE(SP), R4; \
- MOVWZ retoffset+28(FP), R1; \
+ MOVWZ stackRetOffset+28(FP), R1; \
ADD R1, R4; \
ADD R1, R6; \
SUB R1, R5; \
@@ -439,11 +439,12 @@ callFunction: \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
MOVD R7, 8(R15)
MOVD R6, 16(R15)
MOVD R4, 24(R15)
MOVD R5, 32(R15)
+ MOVD $0, 40(R15)
BL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s
index fcb780f1dc..cf3d961b74 100644
--- a/src/runtime/asm_wasm.s
+++ b/src/runtime/asm_wasm.s
@@ -296,14 +296,14 @@ TEXT ·asmcgocall(SB), NOSPLIT, $0-0
JMP NAME(SB); \
End
-TEXT ·reflectcall(SB), NOSPLIT, $0-32
+TEXT ·reflectcall(SB), NOSPLIT, $0-48
I64Load fn+8(FP)
I64Eqz
If
CALLNORESUME runtime·sigpanic<ABIInternal>(SB)
End
- MOVW argsize+24(FP), R0
+ MOVW frameSize+32(FP), R0
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
@@ -335,18 +335,18 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-32
JMP runtime·badreflectcall(SB)
#define CALLFN(NAME, MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
- MOVW argsize+24(FP), R0; \
+ MOVW stackArgsSize+24(FP), R0; \
\
Get R0; \
I64Eqz; \
Not; \
If; \
Get SP; \
- I64Load argptr+16(FP); \
+ I64Load stackArgs+16(FP); \
I32WrapI64; \
- I64Load argsize+24(FP); \
+ I64Load stackArgsSize+24(FP); \
I64Const $3; \
I64ShrU; \
I32WrapI64; \
@@ -359,12 +359,12 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
I64Load $0; \
CALL; \
\
- I64Load32U retoffset+28(FP); \
+ I64Load32U stackRetOffset+28(FP); \
Set R0; \
\
- MOVD argtype+0(FP), RET0; \
+ MOVD stackArgsType+0(FP), RET0; \
\
- I64Load argptr+16(FP); \
+ I64Load stackArgs+16(FP); \
Get R0; \
I64Add; \
Set RET1; \
@@ -375,7 +375,7 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
I64Add; \
Set RET2; \
\
- I64Load32U argsize+24(FP); \
+ I64Load32U stackArgsSize+24(FP); \
Get R0; \
I64Sub; \
Set RET3; \
@@ -387,12 +387,13 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
NO_LOCAL_POINTERS
MOVD RET0, 0(SP)
MOVD RET1, 8(SP)
MOVD RET2, 16(SP)
MOVD RET3, 24(SP)
+ MOVD $0, 32(SP)
CALL runtime·reflectcallmove(SB)
RET
diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go
index 2b5affce52..4994347bde 100644
--- a/src/runtime/mbarrier.go
+++ b/src/runtime/mbarrier.go
@@ -14,6 +14,7 @@
package runtime
import (
+ "internal/abi"
"runtime/internal/sys"
"unsafe"
)
@@ -223,11 +224,18 @@ func reflect_typedmemmovepartial(typ *_type, dst, src unsafe.Pointer, off, size
// stack map of reflectcall is wrong.
//
//go:nosplit
-func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr) {
+func reflectcallmove(typ *_type, dst, src unsafe.Pointer, size uintptr, regs *abi.RegArgs) {
if writeBarrier.needed && typ != nil && typ.ptrdata != 0 && size >= sys.PtrSize {
bulkBarrierPreWrite(uintptr(dst), uintptr(src), size)
}
memmove(dst, src, size)
+
+ // Move pointers returned in registers to a place where the GC can see them.
+ for i := range regs.Ints {
+ if regs.ReturnIsPtr.Get(i) {
+ regs.Ptrs[i] = unsafe.Pointer(regs.Ints[i])
+ }
+ }
}
//go:nosplit
diff --git a/src/runtime/mfinal.go b/src/runtime/mfinal.go
index f4dbd77252..7d0313be12 100644
--- a/src/runtime/mfinal.go
+++ b/src/runtime/mfinal.go
@@ -7,6 +7,7 @@
package runtime
import (
+ "internal/abi"
"runtime/internal/atomic"
"runtime/internal/sys"
"unsafe"
@@ -219,7 +220,11 @@ func runfinq() {
throw("bad kind in runfinq")
}
fingRunning = true
- reflectcall(nil, unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz))
+ // Pass a dummy RegArgs for now.
+ //
+ // TODO(mknyszek): Pass arguments in registers.
+ var regs abi.RegArgs
+ reflectcall(nil, unsafe.Pointer(f.fn), frame, uint32(framesz), uint32(framesz), uint32(framesz), &regs)
fingRunning = false
// Drop finalizer queue heap references
diff --git a/src/runtime/panic.go b/src/runtime/panic.go
index 5b2ccdd874..e320eaa596 100644
--- a/src/runtime/panic.go
+++ b/src/runtime/panic.go
@@ -5,6 +5,7 @@
package runtime
import (
+ "internal/abi"
"runtime/internal/atomic"
"runtime/internal/sys"
"unsafe"
@@ -874,7 +875,13 @@ func reflectcallSave(p *_panic, fn, arg unsafe.Pointer, argsize uint32) {
p.pc = getcallerpc()
p.sp = unsafe.Pointer(getcallersp())
}
- reflectcall(nil, fn, arg, argsize, argsize)
+ // Pass a dummy RegArgs for now since no function actually implements
+ // the register-based ABI.
+ //
+ // TODO(mknyszek): Implement this properly, setting up arguments in
+ // registers as necessary in the caller.
+ var regs abi.RegArgs
+ reflectcall(nil, fn, arg, argsize, argsize, argsize, &regs)
if p != nil {
p.pc = 0
p.sp = unsafe.Pointer(nil)
@@ -968,7 +975,9 @@ func gopanic(e interface{}) {
}
} else {
p.argp = unsafe.Pointer(getargp(0))
- reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz))
+
+ var regs abi.RegArgs
+ reflectcall(nil, unsafe.Pointer(d.fn), deferArgs(d), uint32(d.siz), uint32(d.siz), uint32(d.siz), &regs)
}
p.argp = nil
diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go
index 3d1e0c0bb4..c0cc95ec65 100644
--- a/src/runtime/stubs.go
+++ b/src/runtime/stubs.go
@@ -4,7 +4,10 @@
package runtime
-import "unsafe"
+import (
+ "internal/abi"
+ "unsafe"
+)
// Should be a built-in for unsafe.Pointer?
//go:nosplit
@@ -174,19 +177,50 @@ func asminit()
func setg(gg *g)
func breakpoint()
-// reflectcall calls fn with a copy of the n argument bytes pointed at by arg.
-// After fn returns, reflectcall copies n-retoffset result bytes
-// back into arg+retoffset before returning. If copying result bytes back,
-// the caller should pass the argument frame type as argtype, so that
-// call can execute appropriate write barriers during the copy.
+// reflectcall calls fn with arguments described by stackArgs, stackArgsSize,
+// frameSize, and regArgs.
//
-// Package reflect always passes a frame type. In package runtime,
-// Windows callbacks are the only use of this that copies results
-// back, and those cannot have pointers in their results, so runtime
-// passes nil for the frame type.
+// Arguments passed on the stack and space for return values passed on the stack
+// must be laid out at the space pointed to by stackArgs (with total length
+// stackArgsSize) according to the ABI.
+//
+// stackRetOffset must be some value <= stackArgsSize that indicates the
+// offset within stackArgs where the return value space begins.
+//
+// frameSize is the total size of the argument frame at stackArgs and must
+// therefore be >= stackArgsSize. It must include additional space for spilling
+// register arguments for stack growth and preemption.
+//
+// TODO(mknyszek): Once we don't need the additional spill space, remove frameSize,
+// since frameSize will be redundant with stackArgsSize.
+//
+// Arguments passed in registers must be laid out in regArgs according to the ABI.
+// regArgs will hold any return values passed in registers after the call.
+//
+// reflectcall copies stack arguments from stackArgs to the goroutine stack, and
+// then copies back stackArgsSize-stackRetOffset bytes back to the return space
+// in stackArgs once fn has completed. It also "unspills" argument registers from
+// regArgs before calling fn, and spills them back into regArgs immediately
+// following the call to fn. If there are results being returned on the stack,
+// the caller should pass the argument frame type as stackArgsType so that
+// reflectcall can execute appropriate write barriers during the copy.
+//
+// reflectcall expects regArgs.ReturnIsPtr to be populated indicating which
+// registers on the return path will contain Go pointers. It will then store
+// these pointers in regArgs.Ptrs such that they are visible to the GC.
+//
+// Package reflect passes a frame type. In package runtime, there is only
+// one call that copies results back, in callbackWrap in syscall_windows.go, and it
+// does NOT pass a frame type, meaning there are no write barriers invoked. See that
+// call site for justification.
//
// Package reflect accesses this symbol through a linkname.
-func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32)
+//
+// Arguments passed through to reflectcall do not escape. The type is used
+// only in a very limited callee of reflectcall, the stackArgs are copied, and
+// regArgs is only used in the reflectcall frame.
+//go:noescape
+func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
func procyield(cycles uint32)
diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go
index 7835b492f7..add40bb0b3 100644
--- a/src/runtime/syscall_windows.go
+++ b/src/runtime/syscall_windows.go
@@ -5,6 +5,7 @@
package runtime
import (
+ "internal/abi"
"runtime/internal/sys"
"unsafe"
)
@@ -242,7 +243,11 @@ func callbackWrap(a *callbackArgs) {
// Even though this is copying back results, we can pass a nil
// type because those results must not require write barriers.
- reflectcall(nil, unsafe.Pointer(c.fn), noescape(goArgs), uint32(c.retOffset)+sys.PtrSize, uint32(c.retOffset))
+ //
+ // Pass a dummy RegArgs for now.
+ // TODO(mknyszek): Pass arguments in registers.
+ var regs abi.RegArgs
+ reflectcall(nil, unsafe.Pointer(c.fn), noescape(goArgs), uint32(c.retOffset)+sys.PtrSize, uint32(c.retOffset), uint32(c.retOffset)+sys.PtrSize, &regs)
// Extract the result.
a.result = *(*uintptr)(unsafe.Pointer(&frame[c.retOffset]))