aboutsummaryrefslogtreecommitdiff
path: root/src/runtime/asm_amd64.s
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime/asm_amd64.s')
-rw-r--r--src/runtime/asm_amd64.s161
1 files changed, 83 insertions, 78 deletions
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index fa25c55b96..196252e1dd 100644
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -84,7 +84,9 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
DATA _rt0_amd64_lib_argv<>(SB)/8, $0
GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
-TEXT runtime·rt0_go(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stack-based Go ABI (and
+// in addition there are no calls to this entry point from Go code).
+TEXT runtime·rt0_go<ABIInternal>(SB),NOSPLIT,$0
// copy arguments forward on an even stack
MOVQ DI, AX // argc
MOVQ SI, BX // argv
@@ -229,10 +231,13 @@ ok:
// Prevent dead-code elimination of debugCallV1, which is
// intended to be called by debuggers.
- MOVQ $runtime·debugCallV1(SB), AX
+ MOVQ $runtime·debugCallV1<ABIInternal>(SB), AX
RET
-DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
+// mainPC is a function value for runtime.main, to be passed to newproc.
+// The reference to runtime.main is made via ABIInternal, since the
+// actual function (not the ABI0 wrapper) is needed by newproc.
+DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
GLOBL runtime·mainPC(SB),RODATA,$8
TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
@@ -468,8 +473,9 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
JMP AX
// Note: can't just "JMP NAME(SB)" - bad inlining results.
-TEXT ·reflectcall(SB), NOSPLIT, $0-32
+TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
MOVLQZX argsize+24(FP), CX
+ DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
DISPATCH(runtime·call128, 128)
@@ -537,6 +543,7 @@ TEXT callRet<>(SB), NOSPLIT, $32-0
CALL runtime·reflectcallmove(SB)
RET
+CALLFN(·call16, 16)
CALLFN(·call32, 32)
CALLFN(·call64, 64)
CALLFN(·call128, 128)
@@ -689,25 +696,9 @@ nosave:
MOVL AX, ret+16(FP)
RET
-// func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr)
-// Turn the fn into a Go func (by taking its address) and call
-// cgocallback_gofunc.
-TEXT runtime·cgocallback(SB),NOSPLIT,$32-32
- LEAQ fn+0(FP), AX
- MOVQ AX, 0(SP)
- MOVQ frame+8(FP), AX
- MOVQ AX, 8(SP)
- MOVQ framesize+16(FP), AX
- MOVQ AX, 16(SP)
- MOVQ ctxt+24(FP), AX
- MOVQ AX, 24(SP)
- MOVQ $runtime·cgocallback_gofunc(SB), AX
- CALL AX
- RET
-
-// func cgocallback_gofunc(fn, frame, framesize, ctxt uintptr)
+// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
// See cgocall.go for more details.
-TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32
+TEXT ·cgocallback(SB),NOSPLIT,$24-24
NO_LOCAL_POINTERS
// If g is nil, Go did not create the current thread.
@@ -725,13 +716,12 @@ TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32
CMPQ BX, $0
JEQ needm
MOVQ g_m(BX), BX
- MOVQ BX, R8 // holds oldm until end of function
+ MOVQ BX, savedm-8(SP) // saved copy of oldm
JMP havem
needm:
- MOVQ $0, 0(SP)
- MOVQ $runtime·needm(SB), AX
+ MOVQ $runtime·needm(SB), AX
CALL AX
- MOVQ 0(SP), R8
+ MOVQ $0, savedm-8(SP) // dropm on return
get_tls(CX)
MOVQ g(CX), BX
MOVQ g_m(BX), BX
@@ -767,37 +757,36 @@ havem:
// save that information (m->curg->sched) so we can restore it.
// We can restore m->curg->sched.sp easily, because calling
// runtime.cgocallbackg leaves SP unchanged upon return.
- // To save m->curg->sched.pc, we push it onto the stack.
- // This has the added benefit that it looks to the traceback
- // routine like cgocallbackg is going to return to that
- // PC (because the frame we allocate below has the same
- // size as cgocallback_gofunc's frame declared above)
- // so that the traceback will seamlessly trace back into
- // the earlier calls.
- //
- // In the new goroutine, 8(SP) holds the saved R8.
+ // To save m->curg->sched.pc, we push it onto the curg stack and
+ // open a frame the same size as cgocallback's g0 frame.
+ // Once we switch to the curg stack, the pushed PC will appear
+ // to be the return PC of cgocallback, so that the traceback
+ // will seamlessly trace back into the earlier calls.
MOVQ m_curg(BX), SI
MOVQ SI, g(CX)
MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI
MOVQ (g_sched+gobuf_pc)(SI), BX
- MOVQ BX, -8(DI)
+ MOVQ BX, -8(DI) // "push" return PC on the g stack
+ // Gather our arguments into registers.
+ MOVQ fn+0(FP), BX
+ MOVQ frame+8(FP), CX
+ MOVQ ctxt+16(FP), DX
// Compute the size of the frame, including return PC and, if
// GOEXPERIMENT=framepointer, the saved base pointer
- MOVQ ctxt+24(FP), BX
- LEAQ fv+0(FP), AX
- SUBQ SP, AX
- SUBQ AX, DI
+ LEAQ fn+0(FP), AX
+ SUBQ SP, AX // AX is our actual frame size
+ SUBQ AX, DI // Allocate the same frame size on the g stack
MOVQ DI, SP
- MOVQ R8, 8(SP)
MOVQ BX, 0(SP)
+ MOVQ CX, 8(SP)
+ MOVQ DX, 16(SP)
CALL runtime·cgocallbackg(SB)
- MOVQ 8(SP), R8
// Compute the size of the frame again. FP and SP have
// completely different values here than they did above,
// but only their difference matters.
- LEAQ fv+0(FP), AX
+ LEAQ fn+0(FP), AX
SUBQ SP, AX
// Restore g->sched (== m->curg->sched) from saved values.
@@ -822,7 +811,8 @@ havem:
// If the m on entry was nil, we called needm above to borrow an m
// for the duration of the call. Since the call is over, return it with dropm.
- CMPQ R8, $0
+ MOVQ savedm-8(SP), BX
+ CMPQ BX, $0
JNE 3(PC)
MOVQ $runtime·dropm(SB), AX
CALL AX
@@ -1369,8 +1359,11 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0
RET
// The top-most function running on a goroutine
-// returns to goexit+PCQuantum.
-TEXT runtime·goexit(SB),NOSPLIT,$0-0
+// returns to goexit+PCQuantum. Defined as ABIInternal
+// so as to make it identifiable to traceback (this
+// function it used as a sentinel; traceback wants to
+// see the func PC, not a wrapper PC).
+TEXT runtime·goexit<ABIInternal>(SB),NOSPLIT,$0-0
BYTE $0x90 // NOP
CALL runtime·goexit1(SB) // does not return
// traceback from goexit1 must hit code range of goexit
@@ -1392,7 +1385,8 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
// - AX is the value being written at DI
// It clobbers FLAGS. It does not clobber any general-purpose registers,
// but may clobber others (e.g., SSE registers).
-TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120
+// Defined as ABIInternal since it does not use the stack-based Go ABI.
+TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$120
// Save the registers clobbered by the fast path. This is slightly
// faster than having the caller spill these.
MOVQ R14, 104(SP)
@@ -1476,51 +1470,58 @@ flush:
JMP ret
// gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX.
-TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierCX<ABIInternal>(SB),NOSPLIT,$0
XCHGQ CX, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ CX, AX
RET
// gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX.
-TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierDX<ABIInternal>(SB),NOSPLIT,$0
XCHGQ DX, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ DX, AX
RET
// gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX.
-TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierBX<ABIInternal>(SB),NOSPLIT,$0
XCHGQ BX, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ BX, AX
RET
// gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP.
-TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierBP<ABIInternal>(SB),NOSPLIT,$0
XCHGQ BP, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ BP, AX
RET
// gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI.
-TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierSI<ABIInternal>(SB),NOSPLIT,$0
XCHGQ SI, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ SI, AX
RET
// gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8.
-TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierR8<ABIInternal>(SB),NOSPLIT,$0
XCHGQ R8, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ R8, AX
RET
// gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9.
-TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0
+// Defined as ABIInternal since it does not use the stable Go ABI.
+TEXT runtime·gcWriteBarrierR9<ABIInternal>(SB),NOSPLIT,$0
XCHGQ R9, AX
- CALL runtime·gcWriteBarrier(SB)
+ CALL runtime·gcWriteBarrier<ABIInternal>(SB)
XCHGQ R9, AX
RET
@@ -1559,7 +1560,10 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
// obey escape analysis requirements. Specifically, it must not pass
// a stack pointer to an escaping argument. debugCallV1 cannot check
// this invariant.
-TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0
+//
+// This is ABIInternal because Go code injects its PC directly into new
+// goroutine stacks.
+TEXT runtime·debugCallV1<ABIInternal>(SB),NOSPLIT,$152-0
// Save all registers that may contain pointers so they can be
// conservatively scanned.
//
@@ -1720,67 +1724,68 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
// in the caller's stack frame. These stubs write the args into that stack space and
// then tail call to the corresponding runtime handler.
// The tail call makes these stubs disappear in backtraces.
-TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
+// Defined as ABIInternal since they do not use the stack-based Go ABI.
+TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP)
JMP runtime·goPanicIndex(SB)
-TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
+TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP)
JMP runtime·goPanicIndexU(SB)
-TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
+TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAlen(SB)
-TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAlenU(SB)
-TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
+TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAcap(SB)
-TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP)
JMP runtime·goPanicSliceAcapU(SB)
-TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
+TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP)
JMP runtime·goPanicSliceB(SB)
-TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP)
JMP runtime·goPanicSliceBU(SB)
-TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3Alen(SB)
-TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3AlenU(SB)
-TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3Acap(SB)
-TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ DX, x+0(FP)
MOVQ BX, y+8(FP)
JMP runtime·goPanicSlice3AcapU(SB)
-TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP)
JMP runtime·goPanicSlice3B(SB)
-TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ CX, x+0(FP)
MOVQ DX, y+8(FP)
JMP runtime·goPanicSlice3BU(SB)
-TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP)
JMP runtime·goPanicSlice3C(SB)
-TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
+TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
MOVQ AX, x+0(FP)
MOVQ CX, y+8(FP)
JMP runtime·goPanicSlice3CU(SB)
generated by cgit v1.3 (git 2.53.0) at 2026-04-15 16:04:11 +0000