From 40bff82885b8de850f909f38357c53670562f815 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sat, 3 Oct 2020 20:40:49 -0400 Subject: runtime: define and use call16 everywhere Currently, runtime.call16 is defined and used only on 32-bit architectures, while 64-bit architectures all start at call32 and go up from there. This led to unnecessary complexity because call16's prototype needed to be in a different file, separate from all of the other call* prototypes, which in turn led to it getting out of sync with the other call* prototypes. This CL adds call16 on 64-bit architectures, bringing them all into sync, and moves the call16 prototype to live with the others. Prior to CL 31655 (in 2016), call16 couldn't be implemented on 64-bit architectures because it needed at least four words of argument space to invoke "callwritebarrier" after copying back the results. CL 31655 changed the way call* invoked the write barrier in preparation for the hybrid barrier; since the hybrid barrier had to be invoked prior to copying back results, it needed a different solution that didn't reuse call*'s stack space. At this point, call16 was no longer a problem on 64-bit, but we never added it. Until now. Change-Id: Id10ade0e4f75c6ea76afa6229ddaee2b994c27dd Reviewed-on: https://go-review.googlesource.com/c/go/+/259339 Trust: Austin Clements Reviewed-by: Cherry Zhang --- src/runtime/asm_amd64.s | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/runtime/asm_amd64.s') diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index fa25c55b96..256f4112cd 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -470,6 +470,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 TEXT ·reflectcall(SB), NOSPLIT, $0-32 MOVLQZX argsize+24(FP), CX + DISPATCH(runtime·call16, 16) DISPATCH(runtime·call32, 32) DISPATCH(runtime·call64, 64) DISPATCH(runtime·call128, 128) @@ -537,6 +538,7 @@ TEXT callRet<>(SB), NOSPLIT, $32-0 CALL runtime·reflectcallmove(SB) RET +CALLFN(·call16, 16) CALLFN(·call32, 32) CALLFN(·call64, 64) CALLFN(·call128, 128) -- cgit v1.3 From 30c18878730434027dbefd343aad74963a1fdc48 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 1 Oct 2020 17:22:38 -0400 Subject: runtime,cmd/cgo: simplify C -> Go call path This redesigns the way calls work from C to exported Go functions. It removes several steps from the call path, makes cmd/cgo no longer sensitive to the Go calling convention, and eliminates the use of reflectcall from cgo. In order to avoid generating a large amount of FFI glue between the C and Go ABIs, the cgo tool has long depended on generating a C function that marshals the arguments into a struct, and then the actual ABI switch happens in functions with fixed signatures that simply take a pointer to this struct. In a way, this CL simply pushes this idea further. Currently, the cgo tool generates this argument struct in the exact layout of the Go stack frame and depends on reflectcall to unpack it into the appropriate Go call (even though it's actually reflectcall'ing a function generated by cgo). In this CL, we decouple this struct from the Go stack layout. Instead, cgo generates a Go function that takes the struct, unpacks it, and calls the exported function. Since this generated function has a generic signature (like the rest of the call path), we don't need reflectcall and can instead depend on the Go compiler itself to implement the call to the exported Go function. One complication is that syscall.NewCallback on Windows, which converts a Go function into a C function pointer, depends on cgocallback's current dynamic calling approach since the signatures of the callbacks aren't known statically. For this specific case, we continue to depend on reflectcall. Really, the current approach makes some overly simplistic assumptions about translating the C ABI to the Go ABI. Now we're at least in a much better position to do a proper ABI translation. For comparison, the current cgo call path looks like: GoF (generated C function) -> crosscall2 (in cgo/asm_*.s) -> _cgoexp_GoF (generated Go function) -> cgocallback (in asm_*.s) -> cgocallback_gofunc (in asm_*.s) -> cgocallbackg (in cgocall.go) -> cgocallbackg1 (in cgocall.go) -> reflectcall (in asm_*.s) -> _cgoexpwrap_GoF (generated Go function) -> p.GoF Now the call path looks like: GoF (generated C function) -> crosscall2 (in cgo/asm_*.s) -> cgocallback (in asm_*.s) -> cgocallbackg (in cgocall.go) -> cgocallbackg1 (in cgocall.go) -> _cgoexp_GoF (generated Go function) -> p.GoF Notably: 1. We combine _cgoexp_GoF and _cgoexpwrap_GoF and move the combined operation to the end of the sequence. This combined function also handles reflectcall's previous role. 2. We combined cgocallback and cgocallback_gofunc since the only purpose of having both was to convert a raw PC into a Go function value. We instead construct the Go function value in cgocallbackg1. 3. cgocallbackg1 no longer reaches backwards through the stack to get the arguments to cgocallback_gofunc. Instead, we just pass the arguments down. 4. Currently, we need an explicit msanwrite to mark the results struct as written because reflectcall doesn't do this. Now, the results are written by regular Go assignments, so the Go compiler generates the necessary MSAN annotations. This also means we no longer need to track the size of the arguments frame. Updates #40724, since now we don't need to teach cgo about the register ABI or change how it uses reflectcall. Change-Id: I7840489a2597962aeb670e0c1798a16a7359c94f Reviewed-on: https://go-review.googlesource.com/c/go/+/258938 Trust: Austin Clements Run-TryBot: Austin Clements TryBot-Result: Go Bot Reviewed-by: Cherry Zhang --- misc/cgo/test/callback.go | 2 +- src/cmd/cgo/doc.go | 2 +- src/cmd/cgo/out.go | 162 +++++++++++++------------------------- src/cmd/internal/objabi/funcid.go | 6 +- src/runtime/asm_386.s | 56 +++++-------- src/runtime/asm_amd64.s | 55 +++++-------- src/runtime/asm_arm.s | 58 +++++--------- src/runtime/asm_arm64.s | 48 ++++------- src/runtime/asm_mips64x.s | 54 +++++-------- src/runtime/asm_mipsx.s | 54 +++++-------- src/runtime/asm_ppc64x.s | 55 +++++-------- src/runtime/asm_riscv64.s | 5 +- src/runtime/asm_s390x.s | 54 +++++-------- src/runtime/asm_wasm.s | 5 +- src/runtime/cgo/asm_386.s | 12 +-- src/runtime/cgo/asm_amd64.s | 20 +++-- src/runtime/cgo/asm_arm.s | 61 +++++++------- src/runtime/cgo/asm_arm64.s | 19 ++--- src/runtime/cgo/asm_mips64x.s | 19 ++--- src/runtime/cgo/asm_mipsx.s | 19 ++--- src/runtime/cgo/asm_ppc64x.s | 20 ++--- src/runtime/cgo/asm_s390x.s | 14 ++-- src/runtime/cgo/callbacks.go | 29 +++---- src/runtime/cgocall.go | 140 +++++++++----------------------- src/runtime/proc.go | 2 +- src/runtime/race/output_test.go | 2 +- src/runtime/stubs.go | 25 +++--- src/runtime/symtab.go | 2 +- src/runtime/sys_windows_386.s | 67 ++++++---------- src/runtime/sys_windows_amd64.s | 53 ++++++------- src/runtime/sys_windows_arm.s | 3 + src/runtime/syscall_windows.go | 42 +++++++++- src/runtime/traceback.go | 2 +- 33 files changed, 471 insertions(+), 696 deletions(-) (limited to 'src/runtime/asm_amd64.s') diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index e749650293..814888e3ac 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -181,7 +181,7 @@ func testCallbackCallers(t *testing.T) { name := []string{ "runtime.cgocallbackg1", "runtime.cgocallbackg", - "runtime.cgocallback_gofunc", + "runtime.cgocallback", "runtime.asmcgocall", "runtime.cgocall", "test._Cfunc_callback", diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index b3f371b08c..e782c866ac 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -721,7 +721,7 @@ linkage to the desired libraries. The main function is provided by _cgo_main.c: int main() { return 0; } - void crosscall2(void(*fn)(void*, int, uintptr_t), void *a, int c, uintptr_t ctxt) { } + void crosscall2(void(*fn)(void*), void *a, int c, uintptr_t ctxt) { } uintptr_t _cgo_wait_runtime_init_done(void) { return 0; } void _cgo_release_context(uintptr_t ctxt) { } char* _cgo_topofstack(void) { return (char*)0; } diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 82316a300b..eef54f2d0f 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -59,14 +59,14 @@ func (p *Package) writeDefs() { // Write C main file for using gcc to resolve imports. fmt.Fprintf(fm, "int main() { return 0; }\n") if *importRuntimeCgo { - fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int, __SIZE_TYPE__), void *a, int c, __SIZE_TYPE__ ctxt) { }\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*), void *a, int c, __SIZE_TYPE__ ctxt) { }\n") fmt.Fprintf(fm, "__SIZE_TYPE__ _cgo_wait_runtime_init_done(void) { return 0; }\n") fmt.Fprintf(fm, "void _cgo_release_context(__SIZE_TYPE__ ctxt) { }\n") fmt.Fprintf(fm, "char* _cgo_topofstack(void) { return (char*)0; }\n") } else { // If we're not importing runtime/cgo, we *are* runtime/cgo, // which provides these functions. We just need a prototype. - fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int, __SIZE_TYPE__), void *a, int c, __SIZE_TYPE__ ctxt);\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*), void *a, int c, __SIZE_TYPE__ ctxt);\n") fmt.Fprintf(fm, "__SIZE_TYPE__ _cgo_wait_runtime_init_done(void);\n") fmt.Fprintf(fm, "void _cgo_release_context(__SIZE_TYPE__);\n") } @@ -852,7 +852,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "#pragma GCC diagnostic ignored \"-Wpragmas\"\n") fmt.Fprintf(fgcc, "#pragma GCC diagnostic ignored \"-Waddress-of-packed-member\"\n") - fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *, int, __SIZE_TYPE__), void *, int, __SIZE_TYPE__);\n") + fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *), void *, int, __SIZE_TYPE__);\n") fmt.Fprintf(fgcc, "extern __SIZE_TYPE__ _cgo_wait_runtime_init_done(void);\n") fmt.Fprintf(fgcc, "extern void _cgo_release_context(__SIZE_TYPE__);\n\n") fmt.Fprintf(fgcc, "extern char* _cgo_topofstack(void);") @@ -862,59 +862,48 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { for _, exp := range p.ExpFunc { fn := exp.Func - // Construct a gcc struct matching the gc argument and - // result frame. The gcc struct will be compiled with - // __attribute__((packed)) so all padding must be accounted - // for explicitly. + // Construct a struct that will be used to communicate + // arguments from C to Go. The C and Go definitions + // just have to agree. The gcc struct will be compiled + // with __attribute__((packed)) so all padding must be + // accounted for explicitly. ctype := "struct {\n" + gotype := new(bytes.Buffer) + fmt.Fprintf(gotype, "struct {\n") off := int64(0) npad := 0 - if fn.Recv != nil { - t := p.cgoType(fn.Recv.List[0].Type) - ctype += fmt.Sprintf("\t\t%s recv;\n", t.C) + argField := func(typ ast.Expr, namePat string, args ...interface{}) { + name := fmt.Sprintf(namePat, args...) + t := p.cgoType(typ) + if off%t.Align != 0 { + pad := t.Align - off%t.Align + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + ctype += fmt.Sprintf("\t\t%s %s;\n", t.C, name) + fmt.Fprintf(gotype, "\t\t%s ", name) + noSourceConf.Fprint(gotype, fset, typ) + fmt.Fprintf(gotype, "\n") off += t.Size } + if fn.Recv != nil { + argField(fn.Recv.List[0].Type, "recv") + } fntype := fn.Type forFieldList(fntype.Params, func(i int, aname string, atype ast.Expr) { - t := p.cgoType(atype) - if off%t.Align != 0 { - pad := t.Align - off%t.Align - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } - ctype += fmt.Sprintf("\t\t%s p%d;\n", t.C, i) - off += t.Size + argField(atype, "p%d", i) }) - if off%p.PtrSize != 0 { - pad := p.PtrSize - off%p.PtrSize - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } forFieldList(fntype.Results, func(i int, aname string, atype ast.Expr) { - t := p.cgoType(atype) - if off%t.Align != 0 { - pad := t.Align - off%t.Align - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } - ctype += fmt.Sprintf("\t\t%s r%d;\n", t.C, i) - off += t.Size + argField(atype, "r%d", i) }) - if off%p.PtrSize != 0 { - pad := p.PtrSize - off%p.PtrSize - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } if ctype == "struct {\n" { ctype += "\t\tchar unused;\n" // avoid empty struct } ctype += "\t}" + fmt.Fprintf(gotype, "\t}") // Get the return type of the wrapper function // compiled by gcc. @@ -965,12 +954,15 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { } fmt.Fprintf(fgcch, "extern %s;\n", s) - fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *, int, __SIZE_TYPE__);\n", cPrefix, exp.ExpName) + fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *);\n", cPrefix, exp.ExpName) fmt.Fprintf(fgcc, "\nCGO_NO_SANITIZE_THREAD") fmt.Fprintf(fgcc, "\n%s\n", s) fmt.Fprintf(fgcc, "{\n") fmt.Fprintf(fgcc, "\t__SIZE_TYPE__ _cgo_ctxt = _cgo_wait_runtime_init_done();\n") - fmt.Fprintf(fgcc, "\t%s %v _cgo_a;\n", ctype, p.packedAttribute()) + // The results part of the argument structure must be + // initialized to 0 so the write barriers generated by + // the assignments to these fields in Go are safe. + fmt.Fprintf(fgcc, "\t%s %v _cgo_a = {0};\n", ctype, p.packedAttribute()) if gccResult != "void" && (len(fntype.Results.List) > 1 || len(fntype.Results.List[0].Names) > 1) { fmt.Fprintf(fgcc, "\t%s r;\n", gccResult) } @@ -999,82 +991,28 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "}\n") // Build the wrapper function compiled by cmd/compile. - goname := "_cgoexpwrap" + cPrefix + "_" - if fn.Recv != nil { - goname += fn.Recv.List[0].Names[0].Name + "_" - } - goname += exp.Func.Name.Name + // This unpacks the argument struct above and calls the Go function. fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName) fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName) fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName) - fmt.Fprintf(fgo2, "//go:nosplit\n") // no split stack, so no use of m or g - fmt.Fprintf(fgo2, "//go:norace\n") // must not have race detector calls inserted - fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32, ctxt uintptr) {\n", cPrefix, exp.ExpName) - fmt.Fprintf(fgo2, "\tfn := %s\n", goname) - // The indirect here is converting from a Go function pointer to a C function pointer. - fmt.Fprintf(fgo2, "\t_cgo_runtime_cgocallback(**(**unsafe.Pointer)(unsafe.Pointer(&fn)), a, uintptr(n), ctxt);\n") - fmt.Fprintf(fgo2, "}\n") + fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a *%s) {\n", cPrefix, exp.ExpName, gotype) fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName) - // This code uses printer.Fprint, not conf.Fprint, - // because we don't want //line comments in the middle - // of the function types. - fmt.Fprintf(fgo2, "\n") - fmt.Fprintf(fgo2, "func %s(", goname) - comma := false - if fn.Recv != nil { - fmt.Fprintf(fgo2, "recv ") - printer.Fprint(fgo2, fset, fn.Recv.List[0].Type) - comma = true - } - forFieldList(fntype.Params, - func(i int, aname string, atype ast.Expr) { - if comma { - fmt.Fprintf(fgo2, ", ") - } - fmt.Fprintf(fgo2, "p%d ", i) - printer.Fprint(fgo2, fset, atype) - comma = true - }) - fmt.Fprintf(fgo2, ")") if gccResult != "void" { - fmt.Fprint(fgo2, " (") + // Write results back to frame. + fmt.Fprintf(fgo2, "\t") forFieldList(fntype.Results, func(i int, aname string, atype ast.Expr) { if i > 0 { - fmt.Fprint(fgo2, ", ") + fmt.Fprintf(fgo2, ", ") } - fmt.Fprintf(fgo2, "r%d ", i) - printer.Fprint(fgo2, fset, atype) + fmt.Fprintf(fgo2, "a.r%d", i) }) - fmt.Fprint(fgo2, ")") - } - fmt.Fprint(fgo2, " {\n") - if gccResult == "void" { - fmt.Fprint(fgo2, "\t") - } else { - // Verify that any results don't contain any - // Go pointers. - addedDefer := false - forFieldList(fntype.Results, - func(i int, aname string, atype ast.Expr) { - if !p.hasPointer(nil, atype, false) { - return - } - if !addedDefer { - fmt.Fprint(fgo2, "\tdefer func() {\n") - addedDefer = true - } - fmt.Fprintf(fgo2, "\t\t_cgoCheckResult(r%d)\n", i) - }) - if addedDefer { - fmt.Fprint(fgo2, "\t}()\n") - } - fmt.Fprint(fgo2, "\treturn ") + fmt.Fprintf(fgo2, " = ") } if fn.Recv != nil { - fmt.Fprintf(fgo2, "recv.") + fmt.Fprintf(fgo2, "a.recv.") } fmt.Fprintf(fgo2, "%s(", exp.Func.Name) forFieldList(fntype.Params, @@ -1082,9 +1020,20 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { if i > 0 { fmt.Fprint(fgo2, ", ") } - fmt.Fprintf(fgo2, "p%d", i) + fmt.Fprintf(fgo2, "a.p%d", i) }) fmt.Fprint(fgo2, ")\n") + if gccResult != "void" { + // Verify that any results don't contain any + // Go pointers. + forFieldList(fntype.Results, + func(i int, aname string, atype ast.Expr) { + if !p.hasPointer(nil, atype, false) { + return + } + fmt.Fprintf(fgo2, "\t_cgoCheckResult(a.r%d)\n", i) + }) + } fmt.Fprint(fgo2, "}\n") } @@ -1582,9 +1531,6 @@ const goProlog = ` //go:linkname _cgo_runtime_cgocall runtime.cgocall func _cgo_runtime_cgocall(unsafe.Pointer, uintptr) int32 -//go:linkname _cgo_runtime_cgocallback runtime.cgocallback -func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) - //go:linkname _cgoCheckPointer runtime.cgoCheckPointer func _cgoCheckPointer(interface{}, interface{}) diff --git a/src/cmd/internal/objabi/funcid.go b/src/cmd/internal/objabi/funcid.go index 6c9336f31c..1d098ee172 100644 --- a/src/cmd/internal/objabi/funcid.go +++ b/src/cmd/internal/objabi/funcid.go @@ -26,7 +26,7 @@ const ( FuncID_gcBgMarkWorker FuncID_systemstack_switch FuncID_systemstack - FuncID_cgocallback_gofunc + FuncID_cgocallback FuncID_gogo FuncID_externalthreadhandler FuncID_debugCallV1 @@ -70,8 +70,8 @@ func GetFuncID(name string, isWrapper bool) FuncID { return FuncID_systemstack_switch case "runtime.systemstack": return FuncID_systemstack - case "runtime.cgocallback_gofunc": - return FuncID_cgocallback_gofunc + case "runtime.cgocallback": + return FuncID_cgocallback case "runtime.gogo": return FuncID_gogo case "runtime.externalthreadhandler": diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 11863fba39..a54b68e03d 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -702,25 +702,9 @@ nosave: MOVL AX, ret+8(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 - LEAL fn+0(FP), AX - MOVL AX, 0(SP) - MOVL frame+4(FP), AX - MOVL AX, 4(SP) - MOVL framesize+8(FP), AX - MOVL AX, 8(SP) - MOVL ctxt+12(FP), AX - MOVL AX, 12(SP) - MOVL $runtime·cgocallback_gofunc(SB), AX - CALL AX - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$12-16 +TEXT ·cgocallback(SB),NOSPLIT,$16-12 // Frame size must match commented places below NO_LOCAL_POINTERS // If g is nil, Go did not create the current thread. @@ -780,34 +764,36 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. // - // In the new goroutine, 4(SP) holds the saved oldm (DX) register. - // 8(SP) is unused. + // In the new goroutine, 12(SP) holds the saved oldm (DX) register. MOVL m_curg(BP), SI MOVL SI, g(CX) MOVL (g_sched+gobuf_sp)(SI), DI // prepare stack as DI MOVL (g_sched+gobuf_pc)(SI), BP - MOVL BP, -4(DI) - MOVL ctxt+12(FP), CX - LEAL -(4+12)(DI), SP - MOVL DX, 4(SP) - MOVL CX, 0(SP) + MOVL BP, -4(DI) // "push" return PC on the g stack + // Gather our arguments into registers. + MOVL fn+0(FP), AX + MOVL frame+4(FP), BX + MOVL ctxt+8(FP), CX + LEAL -(4+16)(DI), SP // Must match declared frame size + MOVL DX, 12(SP) + MOVL AX, 0(SP) + MOVL BX, 4(SP) + MOVL CX, 8(SP) CALL runtime·cgocallbackg(SB) - MOVL 4(SP), DX + MOVL 12(SP), DX // Restore g->sched (== m->curg->sched) from saved values. get_tls(CX) MOVL g(CX), SI - MOVL 12(SP), BP + MOVL 16(SP), BP // Must match declared frame size MOVL BP, (g_sched+gobuf_pc)(SI) - LEAL (12+4)(SP), DI + LEAL (16+4)(SP), DI // Must match declared frame size MOVL DI, (g_sched+gobuf_sp)(SI) // Switch back to m->g0's stack and restore m->g0->sched.sp. diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 256f4112cd..3d5d9c4d58 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -691,25 +691,9 @@ nosave: MOVL AX, ret+16(FP) RET -// func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - LEAQ fn+0(FP), AX - MOVQ AX, 0(SP) - MOVQ frame+8(FP), AX - MOVQ AX, 8(SP) - MOVQ framesize+16(FP), AX - MOVQ AX, 16(SP) - MOVQ ctxt+24(FP), AX - MOVQ AX, 24(SP) - MOVQ $runtime·cgocallback_gofunc(SB), AX - CALL AX - RET - -// func cgocallback_gofunc(fn, frame, framesize, ctxt uintptr) +// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$32-24 NO_LOCAL_POINTERS // If g is nil, Go did not create the current thread. @@ -769,37 +753,40 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. // - // In the new goroutine, 8(SP) holds the saved R8. + // In the new goroutine, 24(SP) holds the saved R8. MOVQ m_curg(BX), SI MOVQ SI, g(CX) MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI MOVQ (g_sched+gobuf_pc)(SI), BX - MOVQ BX, -8(DI) + MOVQ BX, -8(DI) // "push" return PC on the g stack + // Gather our arguments into registers. + MOVQ fn+0(FP), BX + MOVQ frame+8(FP), CX + MOVQ ctxt+16(FP), DX // Compute the size of the frame, including return PC and, if // GOEXPERIMENT=framepointer, the saved base pointer - MOVQ ctxt+24(FP), BX - LEAQ fv+0(FP), AX - SUBQ SP, AX - SUBQ AX, DI + LEAQ fn+0(FP), AX + SUBQ SP, AX // AX is our actual frame size + SUBQ AX, DI // Allocate the same frame size on the g stack MOVQ DI, SP - MOVQ R8, 8(SP) + MOVQ R8, 24(SP) MOVQ BX, 0(SP) + MOVQ CX, 8(SP) + MOVQ DX, 16(SP) CALL runtime·cgocallbackg(SB) - MOVQ 8(SP), R8 + MOVQ 24(SP), R8 // Compute the size of the frame again. FP and SP have // completely different values here than they did above, // but only their difference matters. - LEAQ fv+0(FP), AX + LEAQ fn+0(FP), AX SUBQ SP, AX // Restore g->sched (== m->curg->sched) from saved values. diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 51a50c604c..c54b4eb006 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -643,25 +643,9 @@ nosave: MOVW R0, ret+8(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 - MOVW $fn+0(FP), R0 - MOVW R0, 4(R13) - MOVW frame+4(FP), R0 - MOVW R0, 8(R13) - MOVW framesize+8(FP), R0 - MOVW R0, 12(R13) - MOVW ctxt+12(FP), R0 - MOVW R0, 16(R13) - MOVW $runtime·cgocallback_gofunc(SB), R0 - BL (R0) - RET - -// cgocallback_gofunc(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16 +TEXT ·cgocallback(SB),NOSPLIT,$12-12 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -686,7 +670,7 @@ needm: MOVW $runtime·needm(SB), R0 BL (R0) - // Set m->sched.sp = SP, so that if a panic happens + // Set m->g0->sched.sp = SP, so that if a panic happens // during the function we are about to execute, it will // have a valid SP to run on the g0 stack. // The next few lines (after the havem label) @@ -706,10 +690,10 @@ havem: // Save current m->g0->sched.sp on stack and then set it to SP. // Save current sp in m->g0->sched.sp in preparation for // switch back to m->curg stack. - // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-8(SP). + // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP). MOVW m_g0(R8), R3 MOVW (g_sched+gobuf_sp)(R3), R4 - MOVW R4, savedsp-8(SP) + MOVW R4, savedsp-12(SP) // must match frame size MOVW R13, (g_sched+gobuf_sp)(R3) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -718,30 +702,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -4(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVW m_curg(R8), R0 BL setg<>(SB) MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 MOVW (g_sched+gobuf_pc)(g), R5 - MOVW R5, -12(R4) - MOVW ctxt+12(FP), R0 - MOVW R0, -8(R4) - MOVW $-12(R4), R13 + MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVW fn+0(FP), R1 + MOVW frame+4(FP), R2 + MOVW ctxt+8(FP), R3 + MOVW $-(12+4)(R4), R13 // switch stack; must match frame size + MOVW R1, 4(R13) + MOVW R2, 8(R13) + MOVW R3, 12(R13) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVW 0(R13), R5 MOVW R5, (g_sched+gobuf_pc)(g) - MOVW $12(R13), R4 + MOVW $(12+4)(R13), R4 // must match frame size MOVW R4, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -751,7 +735,7 @@ havem: MOVW m_g0(R8), R0 BL setg<>(SB) MOVW (g_sched+gobuf_sp)(g), R13 - MOVW savedsp-8(SP), R4 + MOVW savedsp-12(SP), R4 // must match frame size MOVW R4, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 1f46d1962c..a45e342478 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -958,25 +958,9 @@ nosave: MOVD R0, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$40-32 - MOVD $fn+0(FP), R0 - MOVD R0, 8(RSP) - MOVD frame+8(FP), R0 - MOVD R0, 16(RSP) - MOVD framesize+16(FP), R0 - MOVD R0, 24(RSP) - MOVD ctxt+24(FP), R0 - MOVD R0, 32(RSP) - MOVD $runtime·cgocallback_gofunc(SB), R0 - BL (R0) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load g from thread-local storage. @@ -1001,7 +985,7 @@ needm: MOVD $runtime·needm(SB), R0 BL (R0) - // Set m->sched.sp = SP, so that if a panic happens + // Set m->g0->sched.sp = SP, so that if a panic happens // during the function we are about to execute, it will // have a valid SP to run on the g0 stack. // The next few lines (after the havem label) @@ -1037,16 +1021,11 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVD m_curg(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 @@ -1054,10 +1033,15 @@ havem: MOVD R5, -48(R4) MOVD (g_sched+gobuf_bp)(g), R5 MOVD R5, -56(R4) - MOVD ctxt+24(FP), R0 - MOVD R0, -40(R4) + // Gather our arguments into registers. + MOVD fn+0(FP), R1 + MOVD frame+8(FP), R2 + MOVD ctxt+16(FP), R3 MOVD $-48(R4), R0 // maintain 16-byte SP alignment - MOVD R0, RSP + MOVD R0, RSP // switch stack + MOVD R1, 8(RSP) + MOVD R2, 16(RSP) + MOVD R3, 24(RSP) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 0ff1b24225..19781f7885 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -471,25 +471,9 @@ g0: MOVW R2, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - MOVV $fn+0(FP), R1 - MOVV R1, 8(R29) - MOVV frame+8(FP), R1 - MOVV R1, 16(R29) - MOVV framesize+16(FP), R1 - MOVV R1, 24(R29) - MOVV ctxt+24(FP), R1 - MOVV R1, 32(R29) - MOVV $runtime·cgocallback_gofunc(SB), R1 - JAL (R1) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -537,7 +521,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 8(R29) aka savedsp-16(SP). MOVV m_g0(R3), R1 MOVV (g_sched+gobuf_sp)(R1), R2 - MOVV R2, savedsp-16(SP) + MOVV R2, savedsp-24(SP) // must match frame size MOVV R29, (g_sched+gobuf_sp)(R1) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -546,30 +530,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVV m_curg(R3), g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R2 // prepare stack as R2 MOVV (g_sched+gobuf_pc)(g), R4 - MOVV R4, -24(R2) - MOVV ctxt+24(FP), R1 - MOVV R1, -16(R2) - MOVV $-24(R2), R29 + MOVV R4, -(24+8)(R2) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVV fn+0(FP), R5 + MOVV frame+8(FP), R6 + MOVV ctxt+16(FP), R7 + MOVV $-(24+8)(R2), R29 // switch stack; must match frame size + MOVV R5, 8(R29) + MOVV R6, 16(R29) + MOVV R7, 24(R29) JAL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVV 0(R29), R4 MOVV R4, (g_sched+gobuf_pc)(g) - MOVV $24(R29), R2 + MOVV $(24+8)(R29), R2 // must match frame size MOVV R2, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -579,7 +563,7 @@ havem: MOVV m_g0(R3), g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R29 - MOVV savedsp-16(SP), R2 + MOVV savedsp-24(SP), R2 // must match frame size MOVV R2, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index aca0510b69..ee87d81436 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -472,25 +472,9 @@ g0: MOVW R2, ret+8(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 - MOVW $fn+0(FP), R1 - MOVW R1, 4(R29) - MOVW frame+4(FP), R1 - MOVW R1, 8(R29) - MOVW framesize+8(FP), R1 - MOVW R1, 12(R29) - MOVW ctxt+12(FP), R1 - MOVW R1, 16(R29) - MOVW $runtime·cgocallback_gofunc(SB), R1 - JAL (R1) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16 +TEXT ·cgocallback(SB),NOSPLIT,$12-12 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -538,7 +522,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 4(R29) aka savedsp-8(SP). MOVW m_g0(R3), R1 MOVW (g_sched+gobuf_sp)(R1), R2 - MOVW R2, savedsp-8(SP) + MOVW R2, savedsp-12(SP) // must match frame size MOVW R29, (g_sched+gobuf_sp)(R1) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -547,30 +531,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -4(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVW m_curg(R3), g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R2 // prepare stack as R2 MOVW (g_sched+gobuf_pc)(g), R4 - MOVW R4, -12(R2) - MOVW ctxt+12(FP), R1 - MOVW R1, -8(R2) - MOVW $-12(R2), R29 + MOVW R4, -(12+4)(R2) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVW fn+0(FP), R5 + MOVW frame+4(FP), R6 + MOVW ctxt+8(FP), R7 + MOVW $-(12+4)(R2), R29 // switch stack; must match frame size + MOVW R5, 4(R29) + MOVW R6, 8(R29) + MOVW R7, 12(R29) JAL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVW 0(R29), R4 MOVW R4, (g_sched+gobuf_pc)(g) - MOVW $12(R29), R2 + MOVW $(12+4)(R29), R2 // must match frame size MOVW R2, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -580,7 +564,7 @@ havem: MOVW m_g0(R3), g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R29 - MOVW savedsp-8(SP), R2 + MOVW savedsp-12(SP), R2 // must match frame size MOVW R2, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 603058a61b..dc34c0e4c8 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -651,26 +651,9 @@ g0: MOVW R3, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - MOVD $fn+0(FP), R3 - MOVD R3, FIXED_FRAME+0(R1) - MOVD frame+8(FP), R3 - MOVD R3, FIXED_FRAME+8(R1) - MOVD framesize+16(FP), R3 - MOVD R3, FIXED_FRAME+16(R1) - MOVD ctxt+24(FP), R3 - MOVD R3, FIXED_FRAME+24(R1) - MOVD $runtime·cgocallback_gofunc(SB), R12 - MOVD R12, CTR - BL (CTR) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -721,7 +704,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). MOVD m_g0(R8), R3 MOVD (g_sched+gobuf_sp)(R3), R4 - MOVD R4, savedsp-16(SP) + MOVD R4, savedsp-24(SP) // must match frame size MOVD R1, (g_sched+gobuf_sp)(R3) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -730,30 +713,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVD m_curg(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 MOVD (g_sched+gobuf_pc)(g), R5 - MOVD R5, -(FIXED_FRAME+16)(R4) - MOVD ctxt+24(FP), R3 - MOVD R3, -16(R4) - MOVD $-(FIXED_FRAME+16)(R4), R1 + MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVD fn+0(FP), R5 + MOVD frame+8(FP), R6 + MOVD ctxt+16(FP), R7 + MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size + MOVD R5, FIXED_FRAME+0(R1) + MOVD R6, FIXED_FRAME+8(R1) + MOVD R7, FIXED_FRAME+16(R1) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVD 0(R1), R5 MOVD R5, (g_sched+gobuf_pc)(g) - MOVD $(FIXED_FRAME+16)(R1), R4 + MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -763,7 +746,7 @@ havem: MOVD m_g0(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R1 - MOVD savedsp-16(SP), R4 + MOVD savedsp-24(SP), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index a136085084..fd01fd6f07 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -453,8 +453,9 @@ TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 // traceback from goexit1 must hit code range of goexit MOV ZERO, ZERO // NOP -// func cgocallback_gofunc(fv uintptr, frame uintptr, framesize, ctxt uintptr) -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) +// See cgocall.go for more details. +TEXT ·cgocallback(SB),NOSPLIT,$0-24 // TODO(jsing): Add support for cgo - issue #36641. WORD $0 // crash diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index 46a434119b..7baef37324 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -575,25 +575,9 @@ g0: MOVW R2, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - MOVD $fn+0(FP), R3 - MOVD R3, 8(R15) - MOVD frame+8(FP), R3 - MOVD R3, 16(R15) - MOVD framesize+16(FP), R3 - MOVD R3, 24(R15) - MOVD ctxt+24(FP), R3 - MOVD R3, 32(R15) - MOVD $runtime·cgocallback_gofunc(SB), R3 - BL (R3) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -641,7 +625,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). MOVD m_g0(R8), R3 MOVD (g_sched+gobuf_sp)(R3), R4 - MOVD R4, savedsp-16(SP) + MOVD R4, savedsp-24(SP) // must match frame size MOVD R15, (g_sched+gobuf_sp)(R3) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -650,30 +634,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVD m_curg(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 MOVD (g_sched+gobuf_pc)(g), R5 - MOVD R5, -24(R4) - MOVD ctxt+24(FP), R5 - MOVD R5, -16(R4) - MOVD $-24(R4), R15 + MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVD fn+0(FP), R1 + MOVD frame+8(FP), R2 + MOVD ctxt+16(FP), R3 + MOVD $-(24+8)(R4), R15 // switch stack; must match frame size + MOVD R1, 8(R15) + MOVD R2, 16(R15) + MOVD R3, 24(R15) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVD 0(R15), R5 MOVD R5, (g_sched+gobuf_pc)(g) - MOVD $24(R15), R4 + MOVD $(24+8)(R15), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -683,7 +667,7 @@ havem: MOVD m_g0(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R15 - MOVD savedsp-16(SP), R4 + MOVD savedsp-24(SP), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index 1275af136b..67e81adf0b 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -288,9 +288,6 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 TEXT ·asmcgocall(SB), NOSPLIT, $0-0 UNDEF -TEXT ·cgocallback_gofunc(SB), NOSPLIT, $16-32 - UNDEF - #define DISPATCH(NAME, MAXSIZE) \ Get R0; \ I64Const $MAXSIZE; \ @@ -432,7 +429,7 @@ TEXT runtime·goexit(SB), NOSPLIT, $0-0 CALL runtime·goexit1(SB) // does not return UNDEF -TEXT runtime·cgocallback(SB), NOSPLIT, $32-32 +TEXT runtime·cgocallback(SB), NOSPLIT, $0-24 UNDEF // gcWriteBarrier performs a heap pointer write and informs the GC. diff --git a/src/runtime/cgo/asm_386.s b/src/runtime/cgo/asm_386.s index 7293c20bf8..2e7e9512e2 100644 --- a/src/runtime/cgo/asm_386.s +++ b/src/runtime/cgo/asm_386.s @@ -5,8 +5,9 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT,$28-16 MOVL BP, 24(SP) MOVL BX, 20(SP) @@ -15,12 +16,11 @@ TEXT crosscall2(SB),NOSPLIT,$28-16 MOVL ctxt+12(FP), AX MOVL AX, 8(SP) - MOVL n+8(FP), AX - MOVL AX, 4(SP) MOVL a+4(FP), AX - MOVL AX, 0(SP) + MOVL AX, 4(SP) MOVL fn+0(FP), AX - CALL AX + MOVL AX, 0(SP) + CALL runtime·cgocallback(SB) MOVL 12(SP), DI MOVL 16(SP), SI diff --git a/src/runtime/cgo/asm_amd64.s b/src/runtime/cgo/asm_amd64.s index 06c538b9bc..5dc8e2d235 100644 --- a/src/runtime/cgo/asm_amd64.s +++ b/src/runtime/cgo/asm_amd64.s @@ -5,8 +5,10 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +// This signature is known to SWIG, so we can't change it. #ifndef GOOS_windows TEXT crosscall2(SB),NOSPLIT,$0x50-0 /* keeps stack pointer 32-byte aligned */ #else @@ -33,11 +35,12 @@ TEXT crosscall2(SB),NOSPLIT,$0x110-0 /* also need to save xmm6 - xmm15 */ MOVUPS X14, 0xe0(SP) MOVUPS X15, 0xf0(SP) - MOVQ DX, 0x0(SP) /* arg */ - MOVQ R8, 0x8(SP) /* argsize (includes padding) */ + MOVQ CX, 0x0(SP) /* fn */ + MOVQ DX, 0x8(SP) /* arg */ + // Skip n in R8. MOVQ R9, 0x10(SP) /* ctxt */ - CALL CX /* fn */ + CALL runtime·cgocallback(SB) MOVQ 0x48(SP), DI MOVQ 0x50(SP), SI @@ -52,11 +55,12 @@ TEXT crosscall2(SB),NOSPLIT,$0x110-0 /* also need to save xmm6 - xmm15 */ MOVUPS 0xe0(SP), X14 MOVUPS 0xf0(SP), X15 #else - MOVQ SI, 0x0(SP) /* arg */ - MOVQ DX, 0x8(SP) /* argsize (includes padding) */ + MOVQ DI, 0x0(SP) /* fn */ + MOVQ SI, 0x8(SP) /* arg */ + // Skip n in DX. MOVQ CX, 0x10(SP) /* ctxt */ - CALL DI /* fn */ + CALL runtime·cgocallback(SB) #endif MOVQ 0x18(SP), BX diff --git a/src/runtime/cgo/asm_arm.s b/src/runtime/cgo/asm_arm.s index 60132c14a8..ea55e173c1 100644 --- a/src/runtime/cgo/asm_arm.s +++ b/src/runtime/cgo/asm_arm.s @@ -5,51 +5,52 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 - /* - * We still need to save all callee save register as before, and then - * push 3 args for fn (R1, R2, R3). - * Also note that at procedure entry in gc world, 4(R13) will be the - * first arg, so we must push another dummy reg (R0) for 0(R13). - * Additionally, runtime·load_g will clobber R0, so we need to save R0 - * nevertheless. - */ SUB $(8*9), R13 // Reserve space for the floating point registers. - MOVM.WP [R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, g, R11, R12, R14], (R13) + // The C arguments arrive in R0, R1, R2, and R3. We want to + // pass R0, R1, and R3 to Go, so we push those on the stack. + // Also, save C callee-save registers R4-R12. + MOVM.WP [R0, R1, R3, R4, R5, R6, R7, R8, R9, g, R11, R12], (R13) + // Finally, save the link register R14. This also puts the + // arguments we pushed for cgocallback where they need to be, + // starting at 4(R13). + MOVW.W R14, -4(R13) // Skip floating point registers on GOARM < 6. MOVB runtime·goarm(SB), R11 CMP $6, R11 BLT skipfpsave - MOVD F8, (14*4+8*1)(R13) - MOVD F9, (14*4+8*2)(R13) - MOVD F10, (14*4+8*3)(R13) - MOVD F11, (14*4+8*4)(R13) - MOVD F12, (14*4+8*5)(R13) - MOVD F13, (14*4+8*6)(R13) - MOVD F14, (14*4+8*7)(R13) - MOVD F15, (14*4+8*8)(R13) + MOVD F8, (13*4+8*1)(R13) + MOVD F9, (13*4+8*2)(R13) + MOVD F10, (13*4+8*3)(R13) + MOVD F11, (13*4+8*4)(R13) + MOVD F12, (13*4+8*5)(R13) + MOVD F13, (13*4+8*6)(R13) + MOVD F14, (13*4+8*7)(R13) + MOVD F15, (13*4+8*8)(R13) skipfpsave: BL runtime·load_g(SB) - MOVW R15, R14 // R15 is PC. - MOVW 0(R13), R15 + // We set up the arguments to cgocallback when saving registers above. + BL runtime·cgocallback(SB) MOVB runtime·goarm(SB), R11 CMP $6, R11 BLT skipfprest - MOVD (14*4+8*1)(R13), F8 - MOVD (14*4+8*2)(R13), F9 - MOVD (14*4+8*3)(R13), F10 - MOVD (14*4+8*4)(R13), F11 - MOVD (14*4+8*5)(R13), F12 - MOVD (14*4+8*6)(R13), F13 - MOVD (14*4+8*7)(R13), F14 - MOVD (14*4+8*8)(R13), F15 + MOVD (13*4+8*1)(R13), F8 + MOVD (13*4+8*2)(R13), F9 + MOVD (13*4+8*3)(R13), F10 + MOVD (13*4+8*4)(R13), F11 + MOVD (13*4+8*5)(R13), F12 + MOVD (13*4+8*6)(R13), F13 + MOVD (13*4+8*7)(R13), F14 + MOVD (13*4+8*8)(R13), F15 skipfprest: - MOVM.IAW (R13), [R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, g, R11, R12, R14] + MOVW.P 4(R13), R14 + MOVM.IAW (R13), [R0, R1, R3, R4, R5, R6, R7, R8, R9, g, R11, R12] ADD $(8*9), R13 MOVW R14, R15 diff --git a/src/runtime/cgo/asm_arm64.s b/src/runtime/cgo/asm_arm64.s index ce56f9b1c7..1cb25cf89e 100644 --- a/src/runtime/cgo/asm_arm64.s +++ b/src/runtime/cgo/asm_arm64.s @@ -5,19 +5,20 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 /* * We still need to save all callee save register as before, and then - * push 3 args for fn (R1, R2, R3). + * push 3 args for fn (R0, R1, R3), skipping R2. * Also note that at procedure entry in gc world, 8(RSP) will be the * first arg. * TODO(minux): use LDP/STP here if it matters. */ SUB $(8*24), RSP - MOVD R1, (8*1)(RSP) - MOVD R2, (8*2)(RSP) + MOVD R0, (8*1)(RSP) + MOVD R1, (8*2)(RSP) MOVD R3, (8*3)(RSP) MOVD R19, (8*4)(RSP) MOVD R20, (8*5)(RSP) @@ -40,15 +41,11 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 FMOVD F14, (8*22)(RSP) FMOVD F15, (8*23)(RSP) - MOVD R0, R19 - // Initialize Go ABI environment BL runtime·load_g(SB) - BL (R19) - MOVD (8*1)(RSP), R1 - MOVD (8*2)(RSP), R2 - MOVD (8*3)(RSP), R3 + BL runtime·cgocallback(SB) + MOVD (8*4)(RSP), R19 MOVD (8*5)(RSP), R20 MOVD (8*6)(RSP), R21 diff --git a/src/runtime/cgo/asm_mips64x.s b/src/runtime/cgo/asm_mips64x.s index 1235852dbe..e51cdf3d12 100644 --- a/src/runtime/cgo/asm_mips64x.s +++ b/src/runtime/cgo/asm_mips64x.s @@ -6,14 +6,14 @@ #include "textflag.h" -/* - * void crosscall2(void (*fn)(void*, int32, uintptr), void*, int32, uintptr) - * Save registers and call fn with two arguments. - */ +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 /* * We still need to save all callee save register as before, and then - * push 3 args for fn (R5, R6, R7). + * push 3 args for fn (R4, R5, R7), skipping R6. * Also note that at procedure entry in gc world, 8(R29) will be the * first arg. */ @@ -22,9 +22,9 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 #else ADDV $(-8*15), R29 #endif - MOVV R5, (8*1)(R29) // void* - MOVW R6, (8*2)(R29) // int32 - MOVV R7, (8*3)(R29) // uintptr + MOVV R4, (8*1)(R29) // fn unsafe.Pointer + MOVV R5, (8*2)(R29) // a unsafe.Pointer + MOVV R7, (8*3)(R29) // ctxt uintptr MOVV R16, (8*4)(R29) MOVV R17, (8*5)(R29) MOVV R18, (8*6)(R29) @@ -52,7 +52,8 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 SRLV $32, R31, RSB SLLV $32, RSB JAL runtime·load_g(SB) - JAL (R4) + + JAL runtime·cgocallback(SB) MOVV (8*4)(R29), R16 MOVV (8*5)(R29), R17 diff --git a/src/runtime/cgo/asm_mipsx.s b/src/runtime/cgo/asm_mipsx.s index e3090da223..1127c8beb4 100644 --- a/src/runtime/cgo/asm_mipsx.s +++ b/src/runtime/cgo/asm_mipsx.s @@ -6,14 +6,14 @@ #include "textflag.h" -/* - * void crosscall2(void (*fn)(void*, int32, uintptr), void*, int32, uintptr) - * Save registers and call fn with two arguments. - */ +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 /* * We still need to save all callee save register as before, and then - * push 3 args for fn (R5, R6, R7). + * push 3 args for fn (R4, R5, R7), skipping R6. * Also note that at procedure entry in gc world, 4(R29) will be the * first arg. */ @@ -25,9 +25,9 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 #else SUBU $(4*14-16), R29 // For soft-float, no FPR. #endif - MOVW R5, (4*1)(R29) - MOVW R6, (4*2)(R29) - MOVW R7, (4*3)(R29) + MOVW R4, (4*1)(R29) // fn unsafe.Pointer + MOVW R5, (4*2)(R29) // a unsafe.Pointer + MOVW R7, (4*3)(R29) // ctxt uintptr MOVW R16, (4*4)(R29) MOVW R17, (4*5)(R29) MOVW R18, (4*6)(R29) @@ -47,7 +47,8 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 MOVD F30, (4*14+8*5)(R29) #endif JAL runtime·load_g(SB) - JAL (R4) + + JAL runtime·cgocallback(SB) MOVW (4*4)(R29), R16 MOVW (4*5)(R29), R17 diff --git a/src/runtime/cgo/asm_ppc64x.s b/src/runtime/cgo/asm_ppc64x.s index 3876f9389c..f4efc1e67d 100644 --- a/src/runtime/cgo/asm_ppc64x.s +++ b/src/runtime/cgo/asm_ppc64x.s @@ -8,8 +8,9 @@ #include "asm_ppc64x.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 // Start with standard C stack frame layout and linkage MOVD LR, R0 @@ -26,19 +27,18 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 BL runtime·reginit(SB) BL runtime·load_g(SB) - MOVD R3, R12 #ifdef GOARCH_ppc64 // ppc64 use elf ABI v1. we must get the real entry address from // first slot of the function descriptor before call. // Same for AIX. - MOVD 8(R12), R2 - MOVD (R12), R12 + MOVD 8(R3), R2 + MOVD (R3), R3 #endif - MOVD R12, CTR - MOVD R4, FIXED_FRAME+0(R1) - MOVW R5, FIXED_FRAME+8(R1) - MOVD R6, FIXED_FRAME+16(R1) - BL (CTR) + MOVD R3, FIXED_FRAME+0(R1) // fn unsafe.Pointer + MOVD R4, FIXED_FRAME+8(R1) // a unsafe.Pointer + // Skip R5 = n uint32 + MOVD R6, FIXED_FRAME+16(R1) // ctxt uintptr + BL runtime·cgocallback(SB) ADD $(288+3*8+FIXED_FRAME), R1 diff --git a/src/runtime/cgo/asm_s390x.s b/src/runtime/cgo/asm_s390x.s index 7eab8f652a..8bf16e75e2 100644 --- a/src/runtime/cgo/asm_s390x.s +++ b/src/runtime/cgo/asm_s390x.s @@ -5,8 +5,9 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 // Start with standard C stack frame layout and linkage. @@ -29,10 +30,11 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 // Initialize Go ABI environment. BL runtime·load_g(SB) - MOVD R3, 8(R15) // arg1 - MOVW R4, 16(R15) // arg2 - MOVD R5, 24(R15) // arg3 - BL (R2) // fn(arg1, arg2, arg3) + MOVD R2, 8(R15) // fn unsafe.Pointer + MOVD R3, 16(R15) // a unsafe.Pointer + // Skip R4 = n uint32 + MOVD R5, 24(R15) // ctxt uintptr + BL runtime·cgocallback(SB) FMOVD 32(R15), F8 FMOVD 40(R15), F9 diff --git a/src/runtime/cgo/callbacks.go b/src/runtime/cgo/callbacks.go index 14a218ec92..cd8b795387 100644 --- a/src/runtime/cgo/callbacks.go +++ b/src/runtime/cgo/callbacks.go @@ -9,20 +9,18 @@ import "unsafe" // These utility functions are available to be called from code // compiled with gcc via crosscall2. -// cgocallback is defined in runtime -//go:linkname _runtime_cgocallback runtime.cgocallback -func _runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) - // The declaration of crosscall2 is: -// void crosscall2(void (*fn)(void *, int), void *, int); +// void crosscall2(void (*fn)(void *), void *, int); // // We need to export the symbol crosscall2 in order to support // callbacks from shared libraries. This applies regardless of // linking mode. // -// Compatibility note: crosscall2 actually takes four arguments, but -// it works to call it with three arguments when calling _cgo_panic. -// That is supported for backward compatibility. +// Compatibility note: SWIG uses crosscall2 in exactly one situation: +// to call _cgo_panic using the pattern shown below. We need to keep +// that pattern working. In particular, crosscall2 actually takes four +// arguments, but it works to call it with three arguments when +// calling _cgo_panic. //go:cgo_export_static crosscall2 //go:cgo_export_dynamic crosscall2 @@ -34,21 +32,18 @@ func _runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) // crosscall2(_cgo_panic, &a, sizeof a); // /* The function call will not return. */ +// TODO: We should export a regular C function to panic, change SWIG +// to use that instead of the above pattern, and then we can drop +// backwards-compatibility from crosscall2 and stop exporting it. + //go:linkname _runtime_cgo_panic_internal runtime._cgo_panic_internal func _runtime_cgo_panic_internal(p *byte) //go:linkname _cgo_panic _cgo_panic //go:cgo_export_static _cgo_panic //go:cgo_export_dynamic _cgo_panic -//go:nosplit -//go:norace -func _cgo_panic(a unsafe.Pointer, n int32) { - f := _runtime_cgo_panic_internal - type funcval struct { - pc unsafe.Pointer - } - fv := *(**funcval)(unsafe.Pointer(&f)) - _runtime_cgocallback(fv.pc, a, uintptr(n), 0) +func _cgo_panic(a *struct{ cstr *byte }) { + _runtime_cgo_panic_internal(a.cstr) } //go:cgo_import_static x_cgo_init diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 7ab42a0ed0..9bca279318 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -35,31 +35,25 @@ // cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't // know about packages). The gcc-compiled C function f calls GoF. // -// GoF calls crosscall2(_cgoexp_GoF, frame, framesize, ctxt). -// Crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from -// the gcc function call ABI to the gc function call ABI. -// It is called from gcc to call gc functions. In this case it calls -// _cgoexp_GoF(frame, framesize), still running on m.g0's stack -// and outside the $GOMAXPROCS limit. Thus, this code cannot yet -// call arbitrary Go code directly and must be careful not to allocate -// memory or use up m.g0's stack. +// GoF initializes "frame", a structure containing all of its +// arguments and slots for p.GoF's results. It calls +// crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI. // -// _cgoexp_GoF (generated by cmd/cgo) calls -// runtime.cgocallback(funcPC(p.GoF), frame, framesize, ctxt). -// (The reason for having _cgoexp_GoF instead of writing a crosscall3 -// to make this call directly is that _cgoexp_GoF, because it is compiled -// with gc instead of gcc, can refer to dotted names like -// runtime.cgocallback and p.GoF.) +// crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from +// the gcc function call ABI to the gc function call ABI. At this +// point we're in the Go runtime, but we're still running on m.g0's +// stack and outside the $GOMAXPROCS limit. crosscall2 calls +// runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI. +// (crosscall2's framesize argument is no longer used, but there's one +// case where SWIG calls crosscall2 directly and expects to pass this +// argument. See _cgo_panic.) // -// runtime.cgocallback (in asm_$GOARCH.s) turns the raw PC of p.GoF -// into a Go function value and calls runtime.cgocallback_gofunc. -// -// runtime.cgocallback_gofunc (in asm_$GOARCH.s) switches from m.g0's -// stack to the original g (m.curg)'s stack, on which it calls -// runtime.cgocallbackg(p.GoF, frame, framesize). -// As part of the stack switch, runtime.cgocallback saves the current -// SP as m.g0.sched.sp, so that any use of m.g0's stack during the -// execution of the callback will be done below the existing stack frames. +// runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack +// to the original g (m.curg)'s stack, on which it calls +// runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the +// stack switch, runtime.cgocallback saves the current SP as +// m.g0.sched.sp, so that any use of m.g0's stack during the execution +// of the callback will be done below the existing stack frames. // Before overwriting m.g0.sched.sp, it pushes the old value on the // m.g0 stack, so that it can be restored later. // @@ -67,19 +61,26 @@ // stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will // block until the $GOMAXPROCS limit allows running this goroutine. // Once exitsyscall has returned, it is safe to do things like call the memory -// allocator or invoke the Go callback function p.GoF. runtime.cgocallbackg +// allocator or invoke the Go callback function. runtime.cgocallbackg // first defers a function to unwind m.g0.sched.sp, so that if p.GoF // panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack // and the m.curg stack will be unwound in lock step. -// Then it calls p.GoF. Finally it pops but does not execute the deferred -// function, calls runtime.entersyscall, and returns to runtime.cgocallback. +// Then it calls _cgoexp_GoF(frame). +// +// _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments +// from frame, calls p.GoF, writes the results back to frame, and +// returns. Now we start unwinding this whole process. +// +// runtime.cgocallbackg pops but does not execute the deferred +// function to unwind m.g0.sched.sp, calls runtime.entersyscall, and +// returns to runtime.cgocallback. // // After it regains control, runtime.cgocallback switches back to // m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old -// m.g0.sched.sp value from the stack, and returns to _cgoexp_GoF. +// m.g0.sched.sp value from the stack, and returns to crosscall2. // -// _cgoexp_GoF immediately returns to crosscall2, which restores the -// callee-save registers for gcc and returns to GoF, which returns to f. +// crosscall2 restores the callee-save registers for gcc and returns +// to GoF, which unpacks any result values and returns to f. package runtime @@ -196,7 +197,7 @@ func cgocall(fn, arg unsafe.Pointer) int32 { // Call from C back to Go. //go:nosplit -func cgocallbackg(ctxt uintptr) { +func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { gp := getg() if gp != gp.m.curg { println("runtime: bad g in cgocallback") @@ -224,7 +225,7 @@ func cgocallbackg(ctxt uintptr) { osPreemptExtExit(gp.m) - cgocallbackg1(ctxt) + cgocallbackg1(fn, frame, ctxt) // At this point unlockOSThread has been called. // The following code must not change to a different m. @@ -239,7 +240,7 @@ func cgocallbackg(ctxt uintptr) { gp.m.syscall = syscall } -func cgocallbackg1(ctxt uintptr) { +func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { gp := getg() if gp.m.needextram || atomic.Load(&extraMWaiters) > 0 { gp.m.needextram = false @@ -283,79 +284,16 @@ func cgocallbackg1(ctxt uintptr) { raceacquire(unsafe.Pointer(&racecgosync)) } - type args struct { - fn *funcval - arg unsafe.Pointer - argsize uintptr - } - var cb *args - - // Location of callback arguments depends on stack frame layout - // and size of stack frame of cgocallback_gofunc. - sp := gp.m.g0.sched.sp - switch GOARCH { - default: - throw("cgocallbackg is unimplemented on arch") - case "arm": - // On arm, stack frame is two words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "arm64": - // On arm64, stack frame is four words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - // Additional two words (16-byte alignment) are for saving FP. - cb = (*args)(unsafe.Pointer(sp + 7*sys.PtrSize)) - case "amd64": - // On amd64, stack frame is two words, plus caller PC and BP. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "386": - // On 386, stack frame is three words, plus caller PC. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "ppc64", "ppc64le", "s390x": - // On ppc64 and s390x, the callback arguments are in the arguments area of - // cgocallback's stack frame. The stack looks like this: - // +--------------------+------------------------------+ - // | | ... | - // | cgoexp_$fn +------------------------------+ - // | | fixed frame area | - // +--------------------+------------------------------+ - // | | arguments area | - // | cgocallback +------------------------------+ <- sp + 2*minFrameSize + 2*ptrSize - // | | fixed frame area | - // +--------------------+------------------------------+ <- sp + minFrameSize + 2*ptrSize - // | | local variables (2 pointers) | - // | cgocallback_gofunc +------------------------------+ <- sp + minFrameSize - // | | fixed frame area | - // +--------------------+------------------------------+ <- sp - cb = (*args)(unsafe.Pointer(sp + 2*sys.MinFrameSize + 2*sys.PtrSize)) - case "mips64", "mips64le": - // On mips64x, stack frame is two words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "mips", "mipsle": - // On mipsx, stack frame is two words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - } - - // Invoke callback. - // NOTE(rsc): passing nil for argtype means that the copying of the - // results back into cb.arg happens without any corresponding write barriers. - // For cgo, cb.arg points into a C stack frame and therefore doesn't - // hold any pointers that the GC can find anyway - the write barrier - // would be a no-op. - reflectcall(nil, unsafe.Pointer(cb.fn), cb.arg, uint32(cb.argsize), 0) + // Invoke callback. This function is generated by cmd/cgo and + // will unpack the argument frame and call the Go function. + var cb func(frame unsafe.Pointer) + cbFV := funcval{uintptr(fn)} + *(*unsafe.Pointer)(unsafe.Pointer(&cb)) = noescape(unsafe.Pointer(&cbFV)) + cb(frame) if raceenabled { racereleasemerge(unsafe.Pointer(&racecgosync)) } - if msanenabled { - // Tell msan that we wrote to the entire argument block. - // This tells msan that we set the results. - // Since we have already called the function it doesn't - // matter that we are writing to the non-result parameters. - msanwrite(cb.arg, cb.argsize) - } // Do not unwind m->g0->sched.sp. // Our caller, cgocallback, will do that. diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 83d2a524e0..c629fd45f0 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -4243,7 +4243,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { // First, it may be that the g switch has no PC update, because the SP // either corresponds to a user g throughout (as in asmcgocall) // or because it has been arranged to look like a user g frame - // (as in cgocallback_gofunc). In this case, since the entire + // (as in cgocallback). In this case, since the entire // transition is a g+SP update, a partial transition updating just one of // those will be detected by the stack bounds check. // diff --git a/src/runtime/race/output_test.go b/src/runtime/race/output_test.go index d3e7762175..b4b8936c7c 100644 --- a/src/runtime/race/output_test.go +++ b/src/runtime/race/output_test.go @@ -309,7 +309,7 @@ Read at 0x[0-9,a-f]+ by main goroutine: Previous write at 0x[0-9,a-f]+ by goroutine [0-9]: main\.goCallback\(\) .*/main\.go:27 \+0x[0-9,a-f]+ - main._cgoexpwrap_[0-9a-z]+_goCallback\(\) + _cgoexp_[0-9a-z]+_goCallback\(\) .*_cgo_gotypes\.go:[0-9]+ \+0x[0-9,a-f]+ Goroutine [0-9] \(running\) created at: diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 6290142a41..d77cb4d460 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -148,7 +148,13 @@ func noescape(p unsafe.Pointer) unsafe.Pointer { return unsafe.Pointer(x ^ 0) } -func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr) +// Not all cgocallback frames are actually cgocallback, +// so not all have these arguments. Mark them uintptr so that the GC +// does not misinterpret memory when the arguments are not present. +// cgocallback is not called from Go, only from crosscall2. +// This in turn calls cgocallbackg, which is where we'll find +// pointer-declared arguments. +func cgocallback(fn, frame, ctxt uintptr) func gogo(buf *gobuf) func gosave(buf *gobuf) @@ -163,10 +169,11 @@ func breakpoint() // back into arg+retoffset before returning. If copying result bytes back, // the caller should pass the argument frame type as argtype, so that // call can execute appropriate write barriers during the copy. -// Package reflect passes a frame type. In package runtime, there is only -// one call that copies results back, in cgocallbackg1, and it does NOT pass a -// frame type, meaning there are no write barriers invoked. See that call -// site for justification. +// +// Package reflect always passes a frame type. In package runtime, +// Windows callbacks are the only use of this that copies results +// back, and those cannot have pointers in their results, so runtime +// passes nil for the frame type. // // Package reflect accesses this symbol through a linkname. func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32) @@ -187,14 +194,6 @@ type neverCallThisFunction struct{} // prematurely and if there is leftover state it may panic. func goexit(neverCallThisFunction) -// Not all cgocallback_gofunc frames are actually cgocallback_gofunc, -// so not all have these arguments. Mark them uintptr so that the GC -// does not misinterpret memory when the arguments are not present. -// cgocallback_gofunc is not called from go, only from cgocallback, -// so the arguments will be found via cgocallback's pointer-declared arguments. -// See the assembly implementations for more details. -func cgocallback_gofunc(fv, frame, framesize, ctxt uintptr) - // publicationBarrier performs a store/store barrier (a "publication" // or "export" barrier). Some form of synchronization is required // between initializing an object and making that object accessible to diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 84637376bf..932fba3de0 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -326,7 +326,7 @@ const ( funcID_gcBgMarkWorker funcID_systemstack_switch funcID_systemstack - funcID_cgocallback_gofunc + funcID_cgocallback funcID_gogo funcID_externalthreadhandler funcID_debugCallV1 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index 4ac1527ab1..2e5e82879c 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -239,7 +239,7 @@ GLOBL runtime·cbctxts(SB), NOPTR, $4 TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVL 0(SP), AX // will use to find our callback context - // remove return address from stack, we are not returning there + // remove return address from stack, we are not returning to callbackasm, but to its caller. ADDL $4, SP // address to callback parameters into CX @@ -251,50 +251,35 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 PUSHL BP PUSHL BX - // determine index into runtime·cbctxts table + // Go ABI requires DF flag to be cleared. + CLD + + // determine index into runtime·cbs table SUBL $runtime·callbackasm(SB), AX MOVL $0, DX MOVL $5, BX // divide by 5 because each call instruction in runtime·callbacks is 5 bytes long DIVL BX - - // find correspondent runtime·cbctxts table entry - MOVL runtime·cbctxts(SB), BX - MOVL -4(BX)(AX*4), BX - - // extract callback context - MOVL wincallbackcontext_gobody(BX), AX - MOVL wincallbackcontext_argsize(BX), DX - - // preserve whatever's at the memory location that - // the callback will use to store the return value - PUSHL 0(CX)(DX*1) - - // extend argsize by size of return value - ADDL $4, DX - - // remember how to restore stack on return - MOVL wincallbackcontext_restorestack(BX), BX - PUSHL BX - - // call target Go function - PUSHL DX // argsize (including return value) - PUSHL CX // callback parameters - PUSHL AX // address of target Go function - CLD - CALL runtime·cgocallback_gofunc(SB) - POPL AX - POPL CX - POPL DX - - // how to restore stack on return - POPL BX - - // return value into AX (as per Windows spec) - // and restore previously preserved value - MOVL -4(CX)(DX*1), AX - POPL -4(CX)(DX*1) - - MOVL BX, CX // cannot use BX anymore + SUBL $1, AX // subtract 1 because return PC is to the next slot + + // Create a struct callbackArgs on our stack. + SUBL $(12+callbackArgs__size), SP + MOVL AX, (12+callbackArgs_index)(SP) // callback index + MOVL CX, (12+callbackArgs_args)(SP) // address of args vector + MOVL $0, (12+callbackArgs_result)(SP) // result + LEAL 12(SP), AX // AX = &callbackArgs{...} + + // Call cgocallback, which will call callbackWrap(frame). + MOVL $0, 8(SP) // context + MOVL AX, 4(SP) // frame (address of callbackArgs) + LEAL ·callbackWrap(SB), AX + MOVL AX, 0(SP) // PC of function to call + CALL runtime·cgocallback(SB) + + // Get callback result. + MOVL (12+callbackArgs_result)(SP), AX + // Get popRet. + MOVL (12+callbackArgs_retPop)(SP), CX // Can't use a callee-save register + ADDL $(12+callbackArgs__size), SP // restore registers as required for windows callback POPL BX diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 847542592b..e9ec99a51d 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -291,31 +291,20 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVQ DX, (16+8)(SP) MOVQ R8, (16+16)(SP) MOVQ R9, (16+24)(SP) + // R8 = address of args vector + LEAQ (16+0)(SP), R8 - // remove return address from stack, we are not returning there + // remove return address from stack, we are not returning to callbackasm, but to its caller. MOVQ 0(SP), AX ADDQ $8, SP - // determine index into runtime·cbctxts table + // determine index into runtime·cbs table MOVQ $runtime·callbackasm(SB), DX SUBQ DX, AX MOVQ $0, DX MOVQ $5, CX // divide by 5 because each call instruction in runtime·callbacks is 5 bytes long DIVL CX - - // find correspondent runtime·cbctxts table entry - MOVQ runtime·cbctxts(SB), CX - MOVQ -8(CX)(AX*8), AX - - // extract callback context - MOVQ wincallbackcontext_argsize(AX), DX - MOVQ wincallbackcontext_gobody(AX), AX - - // preserve whatever's at the memory location that - // the callback will use to store the return value - LEAQ 8(SP), CX // args vector, skip return address - PUSHQ 0(CX)(DX*1) // store 8 bytes from just after the args array - ADDQ $8, DX // extend argsize by size of return value + SUBQ $1, AX // subtract 1 because return PC is to the next slot // DI SI BP BX R12 R13 R14 R15 registers and DF flag are preserved // as required by windows callback convention. @@ -330,18 +319,25 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVQ R14, 8(SP) MOVQ R15, 0(SP) - // prepare call stack. use SUBQ to hide from stack frame checks - // cgocallback(Go func, void *frame, uintptr framesize) - SUBQ $24, SP - MOVQ DX, 16(SP) // argsize (including return value) - MOVQ CX, 8(SP) // callback parameters - MOVQ AX, 0(SP) // address of target Go function + // Go ABI requires DF flag to be cleared. CLD - CALL runtime·cgocallback_gofunc(SB) - MOVQ 0(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), DX - ADDQ $24, SP + + // Create a struct callbackArgs on our stack to be passed as + // the "frame" to cgocallback and on to callbackWrap. + SUBQ $(24+callbackArgs__size), SP + MOVQ AX, (24+callbackArgs_index)(SP) // callback index + MOVQ R8, (24+callbackArgs_args)(SP) // address of args vector + MOVQ $0, (24+callbackArgs_result)(SP) // result + LEAQ 24(SP), AX + // Call cgocallback, which will call callbackWrap(frame). + MOVQ $0, 16(SP) // context + MOVQ AX, 8(SP) // frame (address of callbackArgs) + LEAQ ·callbackWrap(SB), BX + MOVQ BX, 0(SP) // PC of function value to call (callbackWrap) + CALL ·cgocallback(SB) + // Get callback result. + MOVQ (24+callbackArgs_result)(SP), AX + ADDQ $(24+callbackArgs__size), SP // restore registers as required for windows callback MOVQ 0(SP), R15 @@ -355,8 +351,7 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 ADDQ $64, SP POPFQ - MOVQ -8(CX)(DX*1), AX // return value - POPQ -8(CX)(DX*1) // restore bytes just after the args + // The return value was placed in AX above. RET // uint32 tstart_stdcall(M *newm); diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 57415e1306..3fc6d27cb0 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -314,6 +314,9 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME,$0 GLOBL runtime·cbctxts(SB), NOPTR, $4 TEXT runtime·callbackasm1(SB),NOSPLIT|NOFRAME,$0 + // TODO(austin): This needs to be converted to match changes + // in cgocallback, but I have no way to test. See CL 258938, + // and callbackasm1 on amd64 and 386. MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} SUB $36, R13 // space for locals diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 0e2fcfb02d..ff43e7cbed 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -5,6 +5,7 @@ package runtime import ( + "runtime/internal/sys" "unsafe" ) @@ -22,10 +23,7 @@ func (c *wincallbackcontext) setCleanstack(cleanstack bool) { c.cleanstack = cleanstack } -var ( - cbs callbacks - cbctxts **wincallbackcontext = &cbs.ctxt[0] // to simplify access to cbs.ctxt in sys_windows_*.s -) +var cbs callbacks func callbackasm() @@ -53,6 +51,8 @@ func callbackasmAddr(i int) uintptr { return funcPC(callbackasm) + uintptr(i*entrySize) } +const callbackMaxArgs = 64 + //go:linkname compileCallback syscall.compileCallback func compileCallback(fn eface, cleanstack bool) (code uintptr) { if fn._type == nil || (fn._type.kind&kindMask) != kindFunc { @@ -66,6 +66,9 @@ func compileCallback(fn eface, cleanstack bool) (code uintptr) { if ft.out()[0].size != uintptrSize { panic("compileCallback: expected function with one uintptr-sized result") } + if len(ft.in()) > callbackMaxArgs { + panic("compileCallback: too many function arguments") + } argsize := uintptr(0) for _, t := range ft.in() { if t.size > uintptrSize { @@ -106,6 +109,37 @@ func compileCallback(fn eface, cleanstack bool) (code uintptr) { return r } +type callbackArgs struct { + index uintptr + args *uintptr // Arguments in stdcall/cdecl convention, with registers spilled + // Below are out-args from callbackWrap + result uintptr + retPop uintptr // For 386 cdecl, how many bytes to pop on return +} + +// callbackWrap is called by callbackasm to invoke a registered C callback. +func callbackWrap(a *callbackArgs) { + c := cbs.ctxt[a.index] + a.retPop = c.restorestack + + // Convert from stdcall to Go ABI. We assume the stack layout + // is the same, and we just need to make room for the result. + // + // TODO: This isn't a good assumption. For example, a function + // that takes two uint16 arguments will be laid out + // differently by the stdcall and Go ABIs. We should implement + // proper ABI conversion. + var frame [callbackMaxArgs + 1]uintptr + memmove(unsafe.Pointer(&frame), unsafe.Pointer(a.args), c.argsize) + + // Even though this is copying back results, we can pass a nil + // type because those results must not require write barriers. + reflectcall(nil, c.gobody, noescape(unsafe.Pointer(&frame)), sys.PtrSize+uint32(c.argsize), uint32(c.argsize)) + + // Extract the result. + a.result = frame[c.argsize/sys.PtrSize] +} + const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 // When available, this function will use LoadLibraryEx with the filename diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 94f4a44976..f3df152535 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -450,7 +450,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } n++ - if f.funcID == funcID_cgocallback_gofunc && len(cgoCtxt) > 0 { + if f.funcID == funcID_cgocallback && len(cgoCtxt) > 0 { ctxt := cgoCtxt[len(cgoCtxt)-1] cgoCtxt = cgoCtxt[:len(cgoCtxt)-1] -- cgit v1.3 From c91dffbc9aeaacd087eb0c0c3f718739bc5f8c4a Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 7 Oct 2020 22:53:52 -0400 Subject: runtime: tidy cgocallback On amd64 and 386, we have a very roundabout way of remembering that we need to dropm on return that currently involves saving a zero to needm's argument slot and later bringing it back. Just store the zero. This also makes amd64 and 386 more consistent with cgocallback on all other platforms: rather than saving the old M to the G stack, they now save it to a named slot on the G0 stack. The needm function no longer needs a dummy argument to get the SP, so we drop that. Change-Id: I7e84bb4a5ff9552de70dcf41d8accf02310535e7 Reviewed-on: https://go-review.googlesource.com/c/go/+/263268 Trust: Austin Clements Run-TryBot: Austin Clements TryBot-Result: Go Bot Reviewed-by: Cherry Zhang --- src/runtime/asm_386.s | 18 +++++++----------- src/runtime/asm_amd64.s | 16 ++++++---------- src/runtime/proc.go | 6 +++--- src/runtime/signal_unix.go | 6 +++--- 4 files changed, 19 insertions(+), 27 deletions(-) (limited to 'src/runtime/asm_amd64.s') diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index a54b68e03d..fa3b1be339 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -704,7 +704,7 @@ nosave: // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback(SB),NOSPLIT,$16-12 // Frame size must match commented places below +TEXT ·cgocallback(SB),NOSPLIT,$12-12 // Frame size must match commented places below NO_LOCAL_POINTERS // If g is nil, Go did not create the current thread. @@ -722,13 +722,12 @@ TEXT ·cgocallback(SB),NOSPLIT,$16-12 // Frame size must match commented places CMPL BP, $0 JEQ needm MOVL g_m(BP), BP - MOVL BP, DX // saved copy of oldm + MOVL BP, savedm-4(SP) // saved copy of oldm JMP havem needm: - MOVL $0, 0(SP) MOVL $runtime·needm(SB), AX CALL AX - MOVL 0(SP), DX + MOVL $0, savedm-4(SP) // dropm on return get_tls(CX) MOVL g(CX), BP MOVL g_m(BP), BP @@ -769,8 +768,6 @@ havem: // Once we switch to the curg stack, the pushed PC will appear // to be the return PC of cgocallback, so that the traceback // will seamlessly trace back into the earlier calls. - // - // In the new goroutine, 12(SP) holds the saved oldm (DX) register. MOVL m_curg(BP), SI MOVL SI, g(CX) MOVL (g_sched+gobuf_sp)(SI), DI // prepare stack as DI @@ -780,20 +777,18 @@ havem: MOVL fn+0(FP), AX MOVL frame+4(FP), BX MOVL ctxt+8(FP), CX - LEAL -(4+16)(DI), SP // Must match declared frame size - MOVL DX, 12(SP) + LEAL -(4+12)(DI), SP // Must match declared frame size MOVL AX, 0(SP) MOVL BX, 4(SP) MOVL CX, 8(SP) CALL runtime·cgocallbackg(SB) - MOVL 12(SP), DX // Restore g->sched (== m->curg->sched) from saved values. get_tls(CX) MOVL g(CX), SI - MOVL 16(SP), BP // Must match declared frame size + MOVL 12(SP), BP // Must match declared frame size MOVL BP, (g_sched+gobuf_pc)(SI) - LEAL (16+4)(SP), DI // Must match declared frame size + LEAL (12+4)(SP), DI // Must match declared frame size MOVL DI, (g_sched+gobuf_sp)(SI) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -809,6 +804,7 @@ havem: // If the m on entry was nil, we called needm above to borrow an m // for the duration of the call. Since the call is over, return it with dropm. + MOVL savedm-4(SP), DX CMPL DX, $0 JNE 3(PC) MOVL $runtime·dropm(SB), AX diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 3d5d9c4d58..19a3bb2d7d 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -693,7 +693,7 @@ nosave: // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback(SB),NOSPLIT,$32-24 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // If g is nil, Go did not create the current thread. @@ -711,13 +711,12 @@ TEXT ·cgocallback(SB),NOSPLIT,$32-24 CMPQ BX, $0 JEQ needm MOVQ g_m(BX), BX - MOVQ BX, R8 // holds oldm until end of function + MOVQ BX, savedm-8(SP) // saved copy of oldm JMP havem needm: - MOVQ $0, 0(SP) - MOVQ $runtime·needm(SB), AX + MOVQ $runtime·needm(SB), AX CALL AX - MOVQ 0(SP), R8 + MOVQ $0, savedm-8(SP) // dropm on return get_tls(CX) MOVQ g(CX), BX MOVQ g_m(BX), BX @@ -758,8 +757,6 @@ havem: // Once we switch to the curg stack, the pushed PC will appear // to be the return PC of cgocallback, so that the traceback // will seamlessly trace back into the earlier calls. - // - // In the new goroutine, 24(SP) holds the saved R8. MOVQ m_curg(BX), SI MOVQ SI, g(CX) MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI @@ -776,12 +773,10 @@ havem: SUBQ AX, DI // Allocate the same frame size on the g stack MOVQ DI, SP - MOVQ R8, 24(SP) MOVQ BX, 0(SP) MOVQ CX, 8(SP) MOVQ DX, 16(SP) CALL runtime·cgocallbackg(SB) - MOVQ 24(SP), R8 // Compute the size of the frame again. FP and SP have // completely different values here than they did above, @@ -811,7 +806,8 @@ havem: // If the m on entry was nil, we called needm above to borrow an m // for the duration of the call. Since the call is over, return it with dropm. - CMPQ R8, $0 + MOVQ savedm-8(SP), BX + CMPQ BX, $0 JNE 3(PC) MOVQ $runtime·dropm(SB), AX CALL AX diff --git a/src/runtime/proc.go b/src/runtime/proc.go index c629fd45f0..ec4e6d8751 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1695,7 +1695,7 @@ func allocm(_p_ *p, fn func(), id int64) *m { // When the callback is done with the m, it calls dropm to // put the m back on the list. //go:nosplit -func needm(x byte) { +func needm() { if (iscgo || GOOS == "windows") && !cgoHasExtraM { // Can happen if C/C++ code calls Go from a global ctor. // Can also happen on Windows if a global ctor uses a @@ -1740,8 +1740,8 @@ func needm(x byte) { // which is more than enough for us. setg(mp.g0) _g_ := getg() - _g_.stack.hi = uintptr(noescape(unsafe.Pointer(&x))) + 1024 - _g_.stack.lo = uintptr(noescape(unsafe.Pointer(&x))) - 32*1024 + _g_.stack.hi = getcallersp() + 1024 + _g_.stack.lo = getcallersp() - 32*1024 _g_.stackguard0 = _g_.stack.lo + _StackGuard // Initialize this thread to use the m. diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index e8b6f95d8f..9318a9b8bc 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -504,14 +504,14 @@ func adjustSignalStack(sig uint32, mp *m, gsigStack *gsignalStack) bool { sigaltstack(nil, &st) if st.ss_flags&_SS_DISABLE != 0 { setg(nil) - needm(0) + needm() noSignalStack(sig) dropm() } stsp := uintptr(unsafe.Pointer(st.ss_sp)) if sp < stsp || sp >= stsp+st.ss_size { setg(nil) - needm(0) + needm() sigNotOnStack(sig) dropm() } @@ -951,7 +951,7 @@ func badsignal(sig uintptr, c *sigctxt) { exit(2) *(*uintptr)(unsafe.Pointer(uintptr(123))) = 2 } - needm(0) + needm() if !sigsend(uint32(sig)) { // A foreign thread received the signal sig, and the // Go code does not want to handle it. -- cgit v1.3 From 50af50d136551e2009b2b52e829570536271cdaa Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 14 Oct 2020 08:36:11 -0400 Subject: reflect,runtime: use internal ABI for selected ASM routines Change the definitions of selected runtime assembly routines from ABI0 (the default) to ABIInternal. The ABIInternal def is intended to indicate that these functions don't follow the existing Go runtime ABI. In addition, convert the assembly reference to runtime.main (from runtime.mainPC) to ABIInternal. Finally, for functions such as "runtime.duffzero" that are called directly from generated code, make sure that the compiler looks up the correct ABI version. This is intended to support the register abi work, however these changes should not have any issues even when GOEXPERIMENT=regabi is not in effect. Updates #27539, #40724. Change-Id: I9846f8dcaccc95718cf2e61a18b7e924a0677e4c Reviewed-on: https://go-review.googlesource.com/c/go/+/262319 Run-TryBot: Than McIntosh TryBot-Result: Go Bot Trust: Than McIntosh Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/gc/ssa.go | 86 ++++++++++++++++---------------- src/cmd/internal/obj/wasm/wasmobj.go | 6 +-- src/reflect/asm_amd64.s | 12 +++-- src/runtime/asm_amd64.s | 96 ++++++++++++++++++++++-------------- src/runtime/asm_wasm.s | 4 +- src/runtime/duff_amd64.s | 4 +- src/runtime/mkpreempt.go | 3 +- src/runtime/preempt_386.s | 3 +- src/runtime/preempt_amd64.s | 3 +- src/runtime/preempt_arm.s | 3 +- src/runtime/preempt_arm64.s | 3 +- src/runtime/preempt_mips64x.s | 3 +- src/runtime/preempt_mipsx.s | 3 +- src/runtime/preempt_ppc64x.s | 3 +- src/runtime/preempt_riscv64.s | 3 +- src/runtime/preempt_s390x.s | 3 +- src/runtime/preempt_wasm.s | 3 +- src/runtime/race_amd64.s | 12 +++-- src/runtime/sys_linux_amd64.s | 13 +++-- 19 files changed, 154 insertions(+), 112 deletions(-) (limited to 'src/runtime/asm_amd64.s') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 45d628cc5e..7388e4e3e8 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -72,9 +72,9 @@ func initssaconfig() { deferproc = sysfunc("deferproc") deferprocStack = sysfunc("deferprocStack") Deferreturn = sysfunc("deferreturn") - Duffcopy = sysvar("duffcopy") // asm func with special ABI - Duffzero = sysvar("duffzero") // asm func with special ABI - gcWriteBarrier = sysvar("gcWriteBarrier") // asm func with special ABI + Duffcopy = sysfunc("duffcopy") + Duffzero = sysfunc("duffzero") + gcWriteBarrier = sysfunc("gcWriteBarrier") goschedguarded = sysfunc("goschedguarded") growslice = sysfunc("growslice") msanread = sysfunc("msanread") @@ -105,51 +105,51 @@ func initssaconfig() { // asm funcs with special ABI if thearch.LinkArch.Name == "amd64" { GCWriteBarrierReg = map[int16]*obj.LSym{ - x86.REG_AX: sysvar("gcWriteBarrier"), - x86.REG_CX: sysvar("gcWriteBarrierCX"), - x86.REG_DX: sysvar("gcWriteBarrierDX"), - x86.REG_BX: sysvar("gcWriteBarrierBX"), - x86.REG_BP: sysvar("gcWriteBarrierBP"), - x86.REG_SI: sysvar("gcWriteBarrierSI"), - x86.REG_R8: sysvar("gcWriteBarrierR8"), - x86.REG_R9: sysvar("gcWriteBarrierR9"), + x86.REG_AX: sysfunc("gcWriteBarrier"), + x86.REG_CX: sysfunc("gcWriteBarrierCX"), + x86.REG_DX: sysfunc("gcWriteBarrierDX"), + x86.REG_BX: sysfunc("gcWriteBarrierBX"), + x86.REG_BP: sysfunc("gcWriteBarrierBP"), + x86.REG_SI: sysfunc("gcWriteBarrierSI"), + x86.REG_R8: sysfunc("gcWriteBarrierR8"), + x86.REG_R9: sysfunc("gcWriteBarrierR9"), } } if thearch.LinkArch.Family == sys.Wasm { - BoundsCheckFunc[ssa.BoundsIndex] = sysvar("goPanicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("goPanicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("goPanicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("goPanicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("goPanicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("goPanicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("goPanicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("goPanicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("goPanicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("goPanicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("goPanicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("goPanicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("goPanicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("goPanicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("goPanicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("goPanicSlice3CU") + BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("goPanicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("goPanicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("goPanicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("goPanicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("goPanicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("goPanicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("goPanicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("goPanicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("goPanicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("goPanicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("goPanicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("goPanicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("goPanicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("goPanicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("goPanicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("goPanicSlice3CU") } else { - BoundsCheckFunc[ssa.BoundsIndex] = sysvar("panicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("panicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("panicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("panicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("panicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("panicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("panicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("panicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("panicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("panicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("panicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("panicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("panicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("panicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("panicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("panicSlice3CU") + BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("panicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("panicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("panicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("panicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("panicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("panicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("panicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("panicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("panicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("panicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("panicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("panicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("panicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("panicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("panicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("panicSlice3CU") } if thearch.LinkArch.PtrSize == 4 { ExtendCheckFunc[ssa.BoundsIndex] = sysvar("panicExtendIndex") diff --git a/src/cmd/internal/obj/wasm/wasmobj.go b/src/cmd/internal/obj/wasm/wasmobj.go index f7f66a1255..2e9890d86c 100644 --- a/src/cmd/internal/obj/wasm/wasmobj.go +++ b/src/cmd/internal/obj/wasm/wasmobj.go @@ -129,7 +129,6 @@ var ( morestackNoCtxt *obj.LSym gcWriteBarrier *obj.LSym sigpanic *obj.LSym - sigpanic0 *obj.LSym deferreturn *obj.LSym jmpdefer *obj.LSym ) @@ -142,9 +141,8 @@ const ( func instinit(ctxt *obj.Link) { morestack = ctxt.Lookup("runtime.morestack") morestackNoCtxt = ctxt.Lookup("runtime.morestack_noctxt") - gcWriteBarrier = ctxt.Lookup("runtime.gcWriteBarrier") + gcWriteBarrier = ctxt.LookupABI("runtime.gcWriteBarrier", obj.ABIInternal) sigpanic = ctxt.LookupABI("runtime.sigpanic", obj.ABIInternal) - sigpanic0 = ctxt.LookupABI("runtime.sigpanic", 0) // sigpanic called from assembly, which has ABI0 deferreturn = ctxt.LookupABI("runtime.deferreturn", obj.ABIInternal) // jmpdefer is defined in assembly as ABI0, but what we're // looking for is the *call* to jmpdefer from the Go function @@ -493,7 +491,7 @@ func preprocess(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) { } // return value of call is on the top of the stack, indicating whether to unwind the WebAssembly stack - if call.As == ACALLNORESUME && call.To.Sym != sigpanic && call.To.Sym != sigpanic0 { // sigpanic unwinds the stack, but it never resumes + if call.As == ACALLNORESUME && call.To.Sym != sigpanic { // sigpanic unwinds the stack, but it never resumes // trying to unwind WebAssembly stack but call has no resume point, terminate with error p = appendp(p, AIf) p = appendp(p, obj.AUNDEF) diff --git a/src/reflect/asm_amd64.s b/src/reflect/asm_amd64.s index fb28ab87f1..5c8e56558c 100644 --- a/src/reflect/asm_amd64.s +++ b/src/reflect/asm_amd64.s @@ -9,7 +9,9 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 +// makeFuncStub must be ABIInternal because it is placed directly +// in function values. +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX @@ -17,14 +19,16 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 MOVB $0, 24(SP) LEAQ 24(SP), AX MOVQ AX, 16(SP) - CALL ·callReflect(SB) + CALL ·callReflect(SB) RET // methodValueCall is the code half of the function returned by makeMethodValue. // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 +// methodValueCall must be ABIInternal because it is placed directly +// in function values. +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX @@ -32,5 +36,5 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 MOVB $0, 24(SP) LEAQ 24(SP), AX MOVQ AX, 16(SP) - CALL ·callMethod(SB) + CALL ·callMethod(SB) RET diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 19a3bb2d7d..196252e1dd 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -84,7 +84,9 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8 DATA _rt0_amd64_lib_argv<>(SB)/8, $0 GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8 -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stack-based Go ABI (and +// in addition there are no calls to this entry point from Go code). +TEXT runtime·rt0_go(SB),NOSPLIT,$0 // copy arguments forward on an even stack MOVQ DI, AX // argc MOVQ SI, BX // argv @@ -229,10 +231,13 @@ ok: // Prevent dead-code elimination of debugCallV1, which is // intended to be called by debuggers. - MOVQ $runtime·debugCallV1(SB), AX + MOVQ $runtime·debugCallV1(SB), AX RET -DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) +// mainPC is a function value for runtime.main, to be passed to newproc. +// The reference to runtime.main is made via ABIInternal, since the +// actual function (not the ABI0 wrapper) is needed by newproc. +DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) GLOBL runtime·mainPC(SB),RODATA,$8 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 @@ -468,7 +473,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 JMP AX // Note: can't just "JMP NAME(SB)" - bad inlining results. -TEXT ·reflectcall(SB), NOSPLIT, $0-32 +TEXT ·reflectcall(SB), NOSPLIT, $0-32 MOVLQZX argsize+24(FP), CX DISPATCH(runtime·call16, 16) DISPATCH(runtime·call32, 32) @@ -1354,8 +1359,11 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 RET // The top-most function running on a goroutine -// returns to goexit+PCQuantum. -TEXT runtime·goexit(SB),NOSPLIT,$0-0 +// returns to goexit+PCQuantum. Defined as ABIInternal +// so as to make it identifiable to traceback (this +// function it used as a sentinel; traceback wants to +// see the func PC, not a wrapper PC). +TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return // traceback from goexit1 must hit code range of goexit @@ -1377,7 +1385,8 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 // - AX is the value being written at DI // It clobbers FLAGS. It does not clobber any general-purpose registers, // but may clobber others (e.g., SSE registers). -TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 // Save the registers clobbered by the fast path. This is slightly // faster than having the caller spill these. MOVQ R14, 104(SP) @@ -1461,51 +1470,58 @@ flush: JMP ret // gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX. -TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 XCHGQ CX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ CX, AX RET // gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX. -TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 XCHGQ DX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ DX, AX RET // gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX. -TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 XCHGQ BX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ BX, AX RET // gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP. -TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 XCHGQ BP, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ BP, AX RET // gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI. -TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 XCHGQ SI, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ SI, AX RET // gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8. -TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 XCHGQ R8, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ R8, AX RET // gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9. -TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 XCHGQ R9, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ R9, AX RET @@ -1544,7 +1560,10 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below // obey escape analysis requirements. Specifically, it must not pass // a stack pointer to an escaping argument. debugCallV1 cannot check // this invariant. -TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 +// +// This is ABIInternal because Go code injects its PC directly into new +// goroutine stacks. +TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 // Save all registers that may contain pointers so they can be // conservatively scanned. // @@ -1705,67 +1724,68 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 // in the caller's stack frame. These stubs write the args into that stack space and // then tail call to the corresponding runtime handler. // The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 +// Defined as ABIInternal since they do not use the stack-based Go ABI. +TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 +TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSlice3CU(SB) diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index 67e81adf0b..fcb780f1dc 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -196,7 +196,7 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16 Get CTXT I64Eqz If - CALLNORESUME runtime·sigpanic(SB) + CALLNORESUME runtime·sigpanic(SB) End // caller sp after CALL @@ -300,7 +300,7 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-32 I64Load fn+8(FP) I64Eqz If - CALLNORESUME runtime·sigpanic(SB) + CALLNORESUME runtime·sigpanic(SB) End MOVW argsize+24(FP), R0 diff --git a/src/runtime/duff_amd64.s b/src/runtime/duff_amd64.s index 44dc75d297..2ff5bf6dbc 100644 --- a/src/runtime/duff_amd64.s +++ b/src/runtime/duff_amd64.s @@ -4,7 +4,7 @@ #include "textflag.h" -TEXT runtime·duffzero(SB), NOSPLIT, $0-0 +TEXT runtime·duffzero(SB), NOSPLIT, $0-0 MOVUPS X0,(DI) MOVUPS X0,16(DI) MOVUPS X0,32(DI) @@ -103,7 +103,7 @@ TEXT runtime·duffzero(SB), NOSPLIT, $0-0 RET -TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 +TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 MOVUPS (SI), X0 ADDQ $16, SI MOVUPS X0, (DI) diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 286f81489a..1d614dd003 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -126,7 +126,8 @@ func header(arch string) { } fmt.Fprintf(out, "#include \"go_asm.h\"\n") fmt.Fprintf(out, "#include \"textflag.h\"\n\n") - fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") + fmt.Fprintf(out, "// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.\n") + fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") } func p(f string, args ...interface{}) { diff --git a/src/runtime/preempt_386.s b/src/runtime/preempt_386.s index c3a5fa1f36..a803b24dc6 100644 --- a/src/runtime/preempt_386.s +++ b/src/runtime/preempt_386.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 PUSHFL ADJSP $156 NOP SP diff --git a/src/runtime/preempt_amd64.s b/src/runtime/preempt_amd64.s index 4765e9f448..92c664d79a 100644 --- a/src/runtime/preempt_amd64.s +++ b/src/runtime/preempt_amd64.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 PUSHQ BP MOVQ SP, BP // Save flags before clobbering them diff --git a/src/runtime/preempt_arm.s b/src/runtime/preempt_arm.s index 8f243c0dcd..bbc9fbb1ea 100644 --- a/src/runtime/preempt_arm.s +++ b/src/runtime/preempt_arm.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVW.W R14, -188(R13) MOVW R0, 4(R13) MOVW R1, 8(R13) diff --git a/src/runtime/preempt_arm64.s b/src/runtime/preempt_arm64.s index 36ee13282c..2b70a28479 100644 --- a/src/runtime/preempt_arm64.s +++ b/src/runtime/preempt_arm64.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R30, -496(RSP) SUB $496, RSP #ifdef GOOS_linux diff --git a/src/runtime/preempt_mips64x.s b/src/runtime/preempt_mips64x.s index 1e123e8077..0d0c157c36 100644 --- a/src/runtime/preempt_mips64x.s +++ b/src/runtime/preempt_mips64x.s @@ -5,7 +5,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVV R31, -488(R29) SUBV $488, R29 MOVV R1, 8(R29) diff --git a/src/runtime/preempt_mipsx.s b/src/runtime/preempt_mipsx.s index afac33e0a0..86d3a918d3 100644 --- a/src/runtime/preempt_mipsx.s +++ b/src/runtime/preempt_mipsx.s @@ -5,7 +5,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVW R31, -244(R29) SUB $244, R29 MOVW R1, 4(R29) diff --git a/src/runtime/preempt_ppc64x.s b/src/runtime/preempt_ppc64x.s index b2d7e30ec7..90634386db 100644 --- a/src/runtime/preempt_ppc64x.s +++ b/src/runtime/preempt_ppc64x.s @@ -5,7 +5,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R31, -488(R1) MOVD LR, R31 MOVDU R31, -520(R1) diff --git a/src/runtime/preempt_riscv64.s b/src/runtime/preempt_riscv64.s index eb68dcba2b..d4f9cc277f 100644 --- a/src/runtime/preempt_riscv64.s +++ b/src/runtime/preempt_riscv64.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOV X1, -472(X2) ADD $-472, X2 MOV X3, 8(X2) diff --git a/src/runtime/preempt_s390x.s b/src/runtime/preempt_s390x.s index ca9e47cde1..c6f11571df 100644 --- a/src/runtime/preempt_s390x.s +++ b/src/runtime/preempt_s390x.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 IPM R10 MOVD R14, -248(R15) ADD $-248, R15 diff --git a/src/runtime/preempt_wasm.s b/src/runtime/preempt_wasm.s index 0cf57d3d22..da90e8aa6d 100644 --- a/src/runtime/preempt_wasm.s +++ b/src/runtime/preempt_wasm.s @@ -3,6 +3,7 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // No async preemption on wasm UNDEF diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index 758d543203..4a86b3371a 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -41,7 +41,9 @@ // func runtime·raceread(addr uintptr) // Called from instrumented code. -TEXT runtime·raceread(SB), NOSPLIT, $0-8 +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would render runtime.getcallerpc ineffective. +TEXT runtime·raceread(SB), NOSPLIT, $0-8 MOVQ addr+0(FP), RARG1 MOVQ (SP), RARG2 // void __tsan_read(ThreadState *thr, void *addr, void *pc); @@ -65,7 +67,9 @@ TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 // func runtime·racewrite(addr uintptr) // Called from instrumented code. -TEXT runtime·racewrite(SB), NOSPLIT, $0-8 +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would render runtime.getcallerpc ineffective. +TEXT runtime·racewrite(SB), NOSPLIT, $0-8 MOVQ addr+0(FP), RARG1 MOVQ (SP), RARG2 // void __tsan_write(ThreadState *thr, void *addr, void *pc); @@ -114,7 +118,9 @@ TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 // func runtime·racewriterange(addr, size uintptr) // Called from instrumented code. -TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would render runtime.getcallerpc ineffective. +TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 MOVQ addr+0(FP), RARG1 MOVQ size+8(FP), RARG2 MOVQ (SP), RARG3 diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 681cd20274..37cb8dad03 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -380,7 +380,8 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$72 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Save callee-saved C registers, since the caller may be a C signal handler. MOVQ BX, bx-8(SP) MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set @@ -407,7 +408,8 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Used instead of sigtramp in programs that use cgo. // Arguments from kernel are in DI, SI, DX. -TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // If no traceback function, do usual sigtramp. MOVQ runtime·cgoTraceback(SB), AX TESTQ AX, AX @@ -450,12 +452,12 @@ TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // The first three arguments, and the fifth, are already in registers. // Set the two remaining arguments now. MOVQ runtime·cgoTraceback(SB), CX - MOVQ $runtime·sigtramp(SB), R9 + MOVQ $runtime·sigtramp(SB), R9 MOVQ _cgo_callers(SB), AX JMP AX sigtramp: - JMP runtime·sigtramp(SB) + JMP runtime·sigtramp(SB) sigtrampnog: // Signal arrived on a non-Go thread. If this is SIGPROF, get a @@ -486,7 +488,8 @@ sigtrampnog: // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/unix/sysv/linux/x86_64/sigaction.c // The code that cares about the precise instructions used is: // https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup -TEXT runtime·sigreturn(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·sigreturn(SB),NOSPLIT,$0 MOVQ $SYS_rt_sigreturn, AX SYSCALL INT $3 // not reached -- cgit v1.3 From ddc7e1d16f58c73a2587bba130a4a49ffac8b0d1 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Thu, 29 Oct 2020 15:37:35 +0000 Subject: Revert "reflect,runtime: use internal ABI for selected ASM routines" This reverts commit 50af50d136551e2009b2b52e829570536271cdaa. Reason for revert: Causes failures in the runtime package test on Darwin, apparently. Change-Id: I006bc1b3443fa7207e92fb4a93e3fb438d4d3de3 Reviewed-on: https://go-review.googlesource.com/c/go/+/266257 Trust: Than McIntosh Run-TryBot: Than McIntosh TryBot-Result: Go Bot Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/gc/ssa.go | 86 ++++++++++++++++---------------- src/cmd/internal/obj/wasm/wasmobj.go | 6 ++- src/reflect/asm_amd64.s | 12 ++--- src/runtime/asm_amd64.s | 96 ++++++++++++++---------------------- src/runtime/asm_wasm.s | 4 +- src/runtime/duff_amd64.s | 4 +- src/runtime/mkpreempt.go | 3 +- src/runtime/preempt_386.s | 3 +- src/runtime/preempt_amd64.s | 3 +- src/runtime/preempt_arm.s | 3 +- src/runtime/preempt_arm64.s | 3 +- src/runtime/preempt_mips64x.s | 3 +- src/runtime/preempt_mipsx.s | 3 +- src/runtime/preempt_ppc64x.s | 3 +- src/runtime/preempt_riscv64.s | 3 +- src/runtime/preempt_s390x.s | 3 +- src/runtime/preempt_wasm.s | 3 +- src/runtime/race_amd64.s | 12 ++--- src/runtime/sys_linux_amd64.s | 13 ++--- 19 files changed, 112 insertions(+), 154 deletions(-) (limited to 'src/runtime/asm_amd64.s') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 7388e4e3e8..45d628cc5e 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -72,9 +72,9 @@ func initssaconfig() { deferproc = sysfunc("deferproc") deferprocStack = sysfunc("deferprocStack") Deferreturn = sysfunc("deferreturn") - Duffcopy = sysfunc("duffcopy") - Duffzero = sysfunc("duffzero") - gcWriteBarrier = sysfunc("gcWriteBarrier") + Duffcopy = sysvar("duffcopy") // asm func with special ABI + Duffzero = sysvar("duffzero") // asm func with special ABI + gcWriteBarrier = sysvar("gcWriteBarrier") // asm func with special ABI goschedguarded = sysfunc("goschedguarded") growslice = sysfunc("growslice") msanread = sysfunc("msanread") @@ -105,51 +105,51 @@ func initssaconfig() { // asm funcs with special ABI if thearch.LinkArch.Name == "amd64" { GCWriteBarrierReg = map[int16]*obj.LSym{ - x86.REG_AX: sysfunc("gcWriteBarrier"), - x86.REG_CX: sysfunc("gcWriteBarrierCX"), - x86.REG_DX: sysfunc("gcWriteBarrierDX"), - x86.REG_BX: sysfunc("gcWriteBarrierBX"), - x86.REG_BP: sysfunc("gcWriteBarrierBP"), - x86.REG_SI: sysfunc("gcWriteBarrierSI"), - x86.REG_R8: sysfunc("gcWriteBarrierR8"), - x86.REG_R9: sysfunc("gcWriteBarrierR9"), + x86.REG_AX: sysvar("gcWriteBarrier"), + x86.REG_CX: sysvar("gcWriteBarrierCX"), + x86.REG_DX: sysvar("gcWriteBarrierDX"), + x86.REG_BX: sysvar("gcWriteBarrierBX"), + x86.REG_BP: sysvar("gcWriteBarrierBP"), + x86.REG_SI: sysvar("gcWriteBarrierSI"), + x86.REG_R8: sysvar("gcWriteBarrierR8"), + x86.REG_R9: sysvar("gcWriteBarrierR9"), } } if thearch.LinkArch.Family == sys.Wasm { - BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("goPanicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("goPanicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("goPanicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("goPanicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("goPanicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("goPanicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("goPanicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("goPanicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("goPanicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("goPanicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("goPanicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("goPanicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("goPanicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("goPanicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("goPanicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("goPanicSlice3CU") + BoundsCheckFunc[ssa.BoundsIndex] = sysvar("goPanicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("goPanicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("goPanicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("goPanicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("goPanicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("goPanicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("goPanicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("goPanicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("goPanicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("goPanicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("goPanicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("goPanicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("goPanicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("goPanicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("goPanicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("goPanicSlice3CU") } else { - BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("panicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("panicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("panicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("panicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("panicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("panicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("panicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("panicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("panicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("panicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("panicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("panicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("panicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("panicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("panicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("panicSlice3CU") + BoundsCheckFunc[ssa.BoundsIndex] = sysvar("panicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("panicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("panicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("panicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("panicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("panicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("panicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("panicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("panicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("panicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("panicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("panicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("panicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("panicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("panicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("panicSlice3CU") } if thearch.LinkArch.PtrSize == 4 { ExtendCheckFunc[ssa.BoundsIndex] = sysvar("panicExtendIndex") diff --git a/src/cmd/internal/obj/wasm/wasmobj.go b/src/cmd/internal/obj/wasm/wasmobj.go index 2e9890d86c..f7f66a1255 100644 --- a/src/cmd/internal/obj/wasm/wasmobj.go +++ b/src/cmd/internal/obj/wasm/wasmobj.go @@ -129,6 +129,7 @@ var ( morestackNoCtxt *obj.LSym gcWriteBarrier *obj.LSym sigpanic *obj.LSym + sigpanic0 *obj.LSym deferreturn *obj.LSym jmpdefer *obj.LSym ) @@ -141,8 +142,9 @@ const ( func instinit(ctxt *obj.Link) { morestack = ctxt.Lookup("runtime.morestack") morestackNoCtxt = ctxt.Lookup("runtime.morestack_noctxt") - gcWriteBarrier = ctxt.LookupABI("runtime.gcWriteBarrier", obj.ABIInternal) + gcWriteBarrier = ctxt.Lookup("runtime.gcWriteBarrier") sigpanic = ctxt.LookupABI("runtime.sigpanic", obj.ABIInternal) + sigpanic0 = ctxt.LookupABI("runtime.sigpanic", 0) // sigpanic called from assembly, which has ABI0 deferreturn = ctxt.LookupABI("runtime.deferreturn", obj.ABIInternal) // jmpdefer is defined in assembly as ABI0, but what we're // looking for is the *call* to jmpdefer from the Go function @@ -491,7 +493,7 @@ func preprocess(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) { } // return value of call is on the top of the stack, indicating whether to unwind the WebAssembly stack - if call.As == ACALLNORESUME && call.To.Sym != sigpanic { // sigpanic unwinds the stack, but it never resumes + if call.As == ACALLNORESUME && call.To.Sym != sigpanic && call.To.Sym != sigpanic0 { // sigpanic unwinds the stack, but it never resumes // trying to unwind WebAssembly stack but call has no resume point, terminate with error p = appendp(p, AIf) p = appendp(p, obj.AUNDEF) diff --git a/src/reflect/asm_amd64.s b/src/reflect/asm_amd64.s index 5c8e56558c..fb28ab87f1 100644 --- a/src/reflect/asm_amd64.s +++ b/src/reflect/asm_amd64.s @@ -9,9 +9,7 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -// makeFuncStub must be ABIInternal because it is placed directly -// in function values. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX @@ -19,16 +17,14 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 MOVB $0, 24(SP) LEAQ 24(SP), AX MOVQ AX, 16(SP) - CALL ·callReflect(SB) + CALL ·callReflect(SB) RET // methodValueCall is the code half of the function returned by makeMethodValue. // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -// methodValueCall must be ABIInternal because it is placed directly -// in function values. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX @@ -36,5 +32,5 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 MOVB $0, 24(SP) LEAQ 24(SP), AX MOVQ AX, 16(SP) - CALL ·callMethod(SB) + CALL ·callMethod(SB) RET diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 196252e1dd..19a3bb2d7d 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -84,9 +84,7 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8 DATA _rt0_amd64_lib_argv<>(SB)/8, $0 GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8 -// Defined as ABIInternal since it does not use the stack-based Go ABI (and -// in addition there are no calls to this entry point from Go code). -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT,$0 // copy arguments forward on an even stack MOVQ DI, AX // argc MOVQ SI, BX // argv @@ -231,13 +229,10 @@ ok: // Prevent dead-code elimination of debugCallV1, which is // intended to be called by debuggers. - MOVQ $runtime·debugCallV1(SB), AX + MOVQ $runtime·debugCallV1(SB), AX RET -// mainPC is a function value for runtime.main, to be passed to newproc. -// The reference to runtime.main is made via ABIInternal, since the -// actual function (not the ABI0 wrapper) is needed by newproc. -DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) +DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) GLOBL runtime·mainPC(SB),RODATA,$8 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 @@ -473,7 +468,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 JMP AX // Note: can't just "JMP NAME(SB)" - bad inlining results. -TEXT ·reflectcall(SB), NOSPLIT, $0-32 +TEXT ·reflectcall(SB), NOSPLIT, $0-32 MOVLQZX argsize+24(FP), CX DISPATCH(runtime·call16, 16) DISPATCH(runtime·call32, 32) @@ -1359,11 +1354,8 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 RET // The top-most function running on a goroutine -// returns to goexit+PCQuantum. Defined as ABIInternal -// so as to make it identifiable to traceback (this -// function it used as a sentinel; traceback wants to -// see the func PC, not a wrapper PC). -TEXT runtime·goexit(SB),NOSPLIT,$0-0 +// returns to goexit+PCQuantum. +TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return // traceback from goexit1 must hit code range of goexit @@ -1385,8 +1377,7 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 // - AX is the value being written at DI // It clobbers FLAGS. It does not clobber any general-purpose registers, // but may clobber others (e.g., SSE registers). -// Defined as ABIInternal since it does not use the stack-based Go ABI. -TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 +TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 // Save the registers clobbered by the fast path. This is slightly // faster than having the caller spill these. MOVQ R14, 104(SP) @@ -1470,58 +1461,51 @@ flush: JMP ret // gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 XCHGQ CX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ CX, AX RET // gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 XCHGQ DX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ DX, AX RET // gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 XCHGQ BX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ BX, AX RET // gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 XCHGQ BP, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ BP, AX RET // gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 XCHGQ SI, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ SI, AX RET // gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 XCHGQ R8, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ R8, AX RET // gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 +TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 XCHGQ R9, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ R9, AX RET @@ -1560,10 +1544,7 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below // obey escape analysis requirements. Specifically, it must not pass // a stack pointer to an escaping argument. debugCallV1 cannot check // this invariant. -// -// This is ABIInternal because Go code injects its PC directly into new -// goroutine stacks. -TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 +TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 // Save all registers that may contain pointers so they can be // conservatively scanned. // @@ -1724,68 +1705,67 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 // in the caller's stack frame. These stubs write the args into that stack space and // then tail call to the corresponding runtime handler. // The tail call makes these stubs disappear in backtraces. -// Defined as ABIInternal since they do not use the stack-based Go ABI. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 +TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 +TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSlice3CU(SB) diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index fcb780f1dc..67e81adf0b 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -196,7 +196,7 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16 Get CTXT I64Eqz If - CALLNORESUME runtime·sigpanic(SB) + CALLNORESUME runtime·sigpanic(SB) End // caller sp after CALL @@ -300,7 +300,7 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-32 I64Load fn+8(FP) I64Eqz If - CALLNORESUME runtime·sigpanic(SB) + CALLNORESUME runtime·sigpanic(SB) End MOVW argsize+24(FP), R0 diff --git a/src/runtime/duff_amd64.s b/src/runtime/duff_amd64.s index 2ff5bf6dbc..44dc75d297 100644 --- a/src/runtime/duff_amd64.s +++ b/src/runtime/duff_amd64.s @@ -4,7 +4,7 @@ #include "textflag.h" -TEXT runtime·duffzero(SB), NOSPLIT, $0-0 +TEXT runtime·duffzero(SB), NOSPLIT, $0-0 MOVUPS X0,(DI) MOVUPS X0,16(DI) MOVUPS X0,32(DI) @@ -103,7 +103,7 @@ TEXT runtime·duffzero(SB), NOSPLIT, $0-0 RET -TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 +TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 MOVUPS (SI), X0 ADDQ $16, SI MOVUPS X0, (DI) diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 1d614dd003..286f81489a 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -126,8 +126,7 @@ func header(arch string) { } fmt.Fprintf(out, "#include \"go_asm.h\"\n") fmt.Fprintf(out, "#include \"textflag.h\"\n\n") - fmt.Fprintf(out, "// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.\n") - fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") + fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") } func p(f string, args ...interface{}) { diff --git a/src/runtime/preempt_386.s b/src/runtime/preempt_386.s index a803b24dc6..c3a5fa1f36 100644 --- a/src/runtime/preempt_386.s +++ b/src/runtime/preempt_386.s @@ -3,8 +3,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 PUSHFL ADJSP $156 NOP SP diff --git a/src/runtime/preempt_amd64.s b/src/runtime/preempt_amd64.s index 92c664d79a..4765e9f448 100644 --- a/src/runtime/preempt_amd64.s +++ b/src/runtime/preempt_amd64.s @@ -3,8 +3,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 PUSHQ BP MOVQ SP, BP // Save flags before clobbering them diff --git a/src/runtime/preempt_arm.s b/src/runtime/preempt_arm.s index bbc9fbb1ea..8f243c0dcd 100644 --- a/src/runtime/preempt_arm.s +++ b/src/runtime/preempt_arm.s @@ -3,8 +3,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVW.W R14, -188(R13) MOVW R0, 4(R13) MOVW R1, 8(R13) diff --git a/src/runtime/preempt_arm64.s b/src/runtime/preempt_arm64.s index 2b70a28479..36ee13282c 100644 --- a/src/runtime/preempt_arm64.s +++ b/src/runtime/preempt_arm64.s @@ -3,8 +3,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R30, -496(RSP) SUB $496, RSP #ifdef GOOS_linux diff --git a/src/runtime/preempt_mips64x.s b/src/runtime/preempt_mips64x.s index 0d0c157c36..1e123e8077 100644 --- a/src/runtime/preempt_mips64x.s +++ b/src/runtime/preempt_mips64x.s @@ -5,8 +5,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVV R31, -488(R29) SUBV $488, R29 MOVV R1, 8(R29) diff --git a/src/runtime/preempt_mipsx.s b/src/runtime/preempt_mipsx.s index 86d3a918d3..afac33e0a0 100644 --- a/src/runtime/preempt_mipsx.s +++ b/src/runtime/preempt_mipsx.s @@ -5,8 +5,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVW R31, -244(R29) SUB $244, R29 MOVW R1, 4(R29) diff --git a/src/runtime/preempt_ppc64x.s b/src/runtime/preempt_ppc64x.s index 90634386db..b2d7e30ec7 100644 --- a/src/runtime/preempt_ppc64x.s +++ b/src/runtime/preempt_ppc64x.s @@ -5,8 +5,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R31, -488(R1) MOVD LR, R31 MOVDU R31, -520(R1) diff --git a/src/runtime/preempt_riscv64.s b/src/runtime/preempt_riscv64.s index d4f9cc277f..eb68dcba2b 100644 --- a/src/runtime/preempt_riscv64.s +++ b/src/runtime/preempt_riscv64.s @@ -3,8 +3,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOV X1, -472(X2) ADD $-472, X2 MOV X3, 8(X2) diff --git a/src/runtime/preempt_s390x.s b/src/runtime/preempt_s390x.s index c6f11571df..ca9e47cde1 100644 --- a/src/runtime/preempt_s390x.s +++ b/src/runtime/preempt_s390x.s @@ -3,8 +3,7 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 IPM R10 MOVD R14, -248(R15) ADD $-248, R15 diff --git a/src/runtime/preempt_wasm.s b/src/runtime/preempt_wasm.s index da90e8aa6d..0cf57d3d22 100644 --- a/src/runtime/preempt_wasm.s +++ b/src/runtime/preempt_wasm.s @@ -3,7 +3,6 @@ #include "go_asm.h" #include "textflag.h" -// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // No async preemption on wasm UNDEF diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index 4a86b3371a..758d543203 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -41,9 +41,7 @@ // func runtime·raceread(addr uintptr) // Called from instrumented code. -// Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. -TEXT runtime·raceread(SB), NOSPLIT, $0-8 +TEXT runtime·raceread(SB), NOSPLIT, $0-8 MOVQ addr+0(FP), RARG1 MOVQ (SP), RARG2 // void __tsan_read(ThreadState *thr, void *addr, void *pc); @@ -67,9 +65,7 @@ TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 // func runtime·racewrite(addr uintptr) // Called from instrumented code. -// Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. -TEXT runtime·racewrite(SB), NOSPLIT, $0-8 +TEXT runtime·racewrite(SB), NOSPLIT, $0-8 MOVQ addr+0(FP), RARG1 MOVQ (SP), RARG2 // void __tsan_write(ThreadState *thr, void *addr, void *pc); @@ -118,9 +114,7 @@ TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 // func runtime·racewriterange(addr, size uintptr) // Called from instrumented code. -// Defined as ABIInternal so as to avoid introducing a wrapper, -// which would render runtime.getcallerpc ineffective. -TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 +TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 MOVQ addr+0(FP), RARG1 MOVQ size+8(FP), RARG2 MOVQ (SP), RARG3 diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 37cb8dad03..681cd20274 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -380,8 +380,7 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -// Defined as ABIInternal since it does not use the stack-based Go ABI. -TEXT runtime·sigtramp(SB),NOSPLIT,$72 +TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Save callee-saved C registers, since the caller may be a C signal handler. MOVQ BX, bx-8(SP) MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set @@ -408,8 +407,7 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Used instead of sigtramp in programs that use cgo. // Arguments from kernel are in DI, SI, DX. -// Defined as ABIInternal since it does not use the stack-based Go ABI. -TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 +TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // If no traceback function, do usual sigtramp. MOVQ runtime·cgoTraceback(SB), AX TESTQ AX, AX @@ -452,12 +450,12 @@ TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // The first three arguments, and the fifth, are already in registers. // Set the two remaining arguments now. MOVQ runtime·cgoTraceback(SB), CX - MOVQ $runtime·sigtramp(SB), R9 + MOVQ $runtime·sigtramp(SB), R9 MOVQ _cgo_callers(SB), AX JMP AX sigtramp: - JMP runtime·sigtramp(SB) + JMP runtime·sigtramp(SB) sigtrampnog: // Signal arrived on a non-Go thread. If this is SIGPROF, get a @@ -488,8 +486,7 @@ sigtrampnog: // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/unix/sysv/linux/x86_64/sigaction.c // The code that cares about the precise instructions used is: // https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup -// Defined as ABIInternal since it does not use the stack-based Go ABI. -TEXT runtime·sigreturn(SB),NOSPLIT,$0 +TEXT runtime·sigreturn(SB),NOSPLIT,$0 MOVQ $SYS_rt_sigreturn, AX SYSCALL INT $3 // not reached -- cgit v1.3 From a313eec3869c609c0da2402a4cac2d32113d599c Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 14 Oct 2020 08:36:11 -0400 Subject: reflect,runtime: use internal ABI for selected ASM routines, attempt 2 [This is a roll-forward of CL 262319, with a fix for some Darwin test failures]. Change the definitions of selected runtime assembly routines from ABI0 (the default) to ABIInternal. The ABIInternal def is intended to indicate that these functions don't follow the existing Go runtime ABI. In addition, convert the assembly reference to runtime.main (from runtime.mainPC) to ABIInternal. Finally, for functions such as "runtime.duffzero" that are called directly from generated code, make sure that the compiler looks up the correct ABI version. This is intended to support the register abi work, however these changes should not have any issues even when GOEXPERIMENT=regabi is not in effect. Updates #27539, #40724. Change-Id: Idf507f1c06176073563845239e1a54dad51a9ea9 Reviewed-on: https://go-review.googlesource.com/c/go/+/266638 Trust: Than McIntosh Run-TryBot: Than McIntosh Reviewed-by: Cherry Zhang TryBot-Result: Go Bot --- src/cmd/compile/internal/gc/ssa.go | 86 ++++++++++++++++---------------- src/cmd/internal/obj/wasm/wasmobj.go | 6 +-- src/cmd/internal/obj/x86/obj6.go | 4 +- src/reflect/asm_amd64.s | 12 +++-- src/runtime/asm_amd64.s | 96 ++++++++++++++++++++++-------------- src/runtime/asm_wasm.s | 4 +- src/runtime/duff_amd64.s | 4 +- src/runtime/mkpreempt.go | 3 +- src/runtime/preempt_386.s | 3 +- src/runtime/preempt_amd64.s | 3 +- src/runtime/preempt_arm.s | 3 +- src/runtime/preempt_arm64.s | 3 +- src/runtime/preempt_mips64x.s | 3 +- src/runtime/preempt_mipsx.s | 3 +- src/runtime/preempt_ppc64x.s | 3 +- src/runtime/preempt_riscv64.s | 3 +- src/runtime/preempt_s390x.s | 3 +- src/runtime/preempt_wasm.s | 3 +- src/runtime/race_amd64.s | 12 +++-- src/runtime/sys_linux_amd64.s | 13 +++-- 20 files changed, 156 insertions(+), 114 deletions(-) (limited to 'src/runtime/asm_amd64.s') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 45d628cc5e..7388e4e3e8 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -72,9 +72,9 @@ func initssaconfig() { deferproc = sysfunc("deferproc") deferprocStack = sysfunc("deferprocStack") Deferreturn = sysfunc("deferreturn") - Duffcopy = sysvar("duffcopy") // asm func with special ABI - Duffzero = sysvar("duffzero") // asm func with special ABI - gcWriteBarrier = sysvar("gcWriteBarrier") // asm func with special ABI + Duffcopy = sysfunc("duffcopy") + Duffzero = sysfunc("duffzero") + gcWriteBarrier = sysfunc("gcWriteBarrier") goschedguarded = sysfunc("goschedguarded") growslice = sysfunc("growslice") msanread = sysfunc("msanread") @@ -105,51 +105,51 @@ func initssaconfig() { // asm funcs with special ABI if thearch.LinkArch.Name == "amd64" { GCWriteBarrierReg = map[int16]*obj.LSym{ - x86.REG_AX: sysvar("gcWriteBarrier"), - x86.REG_CX: sysvar("gcWriteBarrierCX"), - x86.REG_DX: sysvar("gcWriteBarrierDX"), - x86.REG_BX: sysvar("gcWriteBarrierBX"), - x86.REG_BP: sysvar("gcWriteBarrierBP"), - x86.REG_SI: sysvar("gcWriteBarrierSI"), - x86.REG_R8: sysvar("gcWriteBarrierR8"), - x86.REG_R9: sysvar("gcWriteBarrierR9"), + x86.REG_AX: sysfunc("gcWriteBarrier"), + x86.REG_CX: sysfunc("gcWriteBarrierCX"), + x86.REG_DX: sysfunc("gcWriteBarrierDX"), + x86.REG_BX: sysfunc("gcWriteBarrierBX"), + x86.REG_BP: sysfunc("gcWriteBarrierBP"), + x86.REG_SI: sysfunc("gcWriteBarrierSI"), + x86.REG_R8: sysfunc("gcWriteBarrierR8"), + x86.REG_R9: sysfunc("gcWriteBarrierR9"), } } if thearch.LinkArch.Family == sys.Wasm { - BoundsCheckFunc[ssa.BoundsIndex] = sysvar("goPanicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("goPanicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("goPanicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("goPanicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("goPanicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("goPanicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("goPanicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("goPanicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("goPanicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("goPanicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("goPanicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("goPanicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("goPanicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("goPanicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("goPanicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("goPanicSlice3CU") + BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("goPanicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("goPanicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("goPanicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("goPanicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("goPanicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("goPanicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("goPanicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("goPanicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("goPanicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("goPanicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("goPanicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("goPanicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("goPanicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("goPanicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("goPanicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("goPanicSlice3CU") } else { - BoundsCheckFunc[ssa.BoundsIndex] = sysvar("panicIndex") - BoundsCheckFunc[ssa.BoundsIndexU] = sysvar("panicIndexU") - BoundsCheckFunc[ssa.BoundsSliceAlen] = sysvar("panicSliceAlen") - BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysvar("panicSliceAlenU") - BoundsCheckFunc[ssa.BoundsSliceAcap] = sysvar("panicSliceAcap") - BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysvar("panicSliceAcapU") - BoundsCheckFunc[ssa.BoundsSliceB] = sysvar("panicSliceB") - BoundsCheckFunc[ssa.BoundsSliceBU] = sysvar("panicSliceBU") - BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysvar("panicSlice3Alen") - BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysvar("panicSlice3AlenU") - BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysvar("panicSlice3Acap") - BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysvar("panicSlice3AcapU") - BoundsCheckFunc[ssa.BoundsSlice3B] = sysvar("panicSlice3B") - BoundsCheckFunc[ssa.BoundsSlice3BU] = sysvar("panicSlice3BU") - BoundsCheckFunc[ssa.BoundsSlice3C] = sysvar("panicSlice3C") - BoundsCheckFunc[ssa.BoundsSlice3CU] = sysvar("panicSlice3CU") + BoundsCheckFunc[ssa.BoundsIndex] = sysfunc("panicIndex") + BoundsCheckFunc[ssa.BoundsIndexU] = sysfunc("panicIndexU") + BoundsCheckFunc[ssa.BoundsSliceAlen] = sysfunc("panicSliceAlen") + BoundsCheckFunc[ssa.BoundsSliceAlenU] = sysfunc("panicSliceAlenU") + BoundsCheckFunc[ssa.BoundsSliceAcap] = sysfunc("panicSliceAcap") + BoundsCheckFunc[ssa.BoundsSliceAcapU] = sysfunc("panicSliceAcapU") + BoundsCheckFunc[ssa.BoundsSliceB] = sysfunc("panicSliceB") + BoundsCheckFunc[ssa.BoundsSliceBU] = sysfunc("panicSliceBU") + BoundsCheckFunc[ssa.BoundsSlice3Alen] = sysfunc("panicSlice3Alen") + BoundsCheckFunc[ssa.BoundsSlice3AlenU] = sysfunc("panicSlice3AlenU") + BoundsCheckFunc[ssa.BoundsSlice3Acap] = sysfunc("panicSlice3Acap") + BoundsCheckFunc[ssa.BoundsSlice3AcapU] = sysfunc("panicSlice3AcapU") + BoundsCheckFunc[ssa.BoundsSlice3B] = sysfunc("panicSlice3B") + BoundsCheckFunc[ssa.BoundsSlice3BU] = sysfunc("panicSlice3BU") + BoundsCheckFunc[ssa.BoundsSlice3C] = sysfunc("panicSlice3C") + BoundsCheckFunc[ssa.BoundsSlice3CU] = sysfunc("panicSlice3CU") } if thearch.LinkArch.PtrSize == 4 { ExtendCheckFunc[ssa.BoundsIndex] = sysvar("panicExtendIndex") diff --git a/src/cmd/internal/obj/wasm/wasmobj.go b/src/cmd/internal/obj/wasm/wasmobj.go index f7f66a1255..2e9890d86c 100644 --- a/src/cmd/internal/obj/wasm/wasmobj.go +++ b/src/cmd/internal/obj/wasm/wasmobj.go @@ -129,7 +129,6 @@ var ( morestackNoCtxt *obj.LSym gcWriteBarrier *obj.LSym sigpanic *obj.LSym - sigpanic0 *obj.LSym deferreturn *obj.LSym jmpdefer *obj.LSym ) @@ -142,9 +141,8 @@ const ( func instinit(ctxt *obj.Link) { morestack = ctxt.Lookup("runtime.morestack") morestackNoCtxt = ctxt.Lookup("runtime.morestack_noctxt") - gcWriteBarrier = ctxt.Lookup("runtime.gcWriteBarrier") + gcWriteBarrier = ctxt.LookupABI("runtime.gcWriteBarrier", obj.ABIInternal) sigpanic = ctxt.LookupABI("runtime.sigpanic", obj.ABIInternal) - sigpanic0 = ctxt.LookupABI("runtime.sigpanic", 0) // sigpanic called from assembly, which has ABI0 deferreturn = ctxt.LookupABI("runtime.deferreturn", obj.ABIInternal) // jmpdefer is defined in assembly as ABI0, but what we're // looking for is the *call* to jmpdefer from the Go function @@ -493,7 +491,7 @@ func preprocess(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) { } // return value of call is on the top of the stack, indicating whether to unwind the WebAssembly stack - if call.As == ACALLNORESUME && call.To.Sym != sigpanic && call.To.Sym != sigpanic0 { // sigpanic unwinds the stack, but it never resumes + if call.As == ACALLNORESUME && call.To.Sym != sigpanic { // sigpanic unwinds the stack, but it never resumes // trying to unwind WebAssembly stack but call has no resume point, terminate with error p = appendp(p, AIf) p = appendp(p, obj.AUNDEF) diff --git a/src/cmd/internal/obj/x86/obj6.go b/src/cmd/internal/obj/x86/obj6.go index e11fa13f65..184fb4308b 100644 --- a/src/cmd/internal/obj/x86/obj6.go +++ b/src/cmd/internal/obj/x86/obj6.go @@ -324,9 +324,9 @@ func rewriteToUseGot(ctxt *obj.Link, p *obj.Prog, newprog obj.ProgAlloc) { // flags and duffzero on 386 does not otherwise do so). var sym *obj.LSym if p.As == obj.ADUFFZERO { - sym = ctxt.Lookup("runtime.duffzero") + sym = ctxt.LookupABI("runtime.duffzero", obj.ABIInternal) } else { - sym = ctxt.Lookup("runtime.duffcopy") + sym = ctxt.LookupABI("runtime.duffcopy", obj.ABIInternal) } offset := p.To.Offset p.As = mov diff --git a/src/reflect/asm_amd64.s b/src/reflect/asm_amd64.s index fb28ab87f1..5c8e56558c 100644 --- a/src/reflect/asm_amd64.s +++ b/src/reflect/asm_amd64.s @@ -9,7 +9,9 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 +// makeFuncStub must be ABIInternal because it is placed directly +// in function values. +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX @@ -17,14 +19,16 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 MOVB $0, 24(SP) LEAQ 24(SP), AX MOVQ AX, 16(SP) - CALL ·callReflect(SB) + CALL ·callReflect(SB) RET // methodValueCall is the code half of the function returned by makeMethodValue. // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 +// methodValueCall must be ABIInternal because it is placed directly +// in function values. +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX @@ -32,5 +36,5 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 MOVB $0, 24(SP) LEAQ 24(SP), AX MOVQ AX, 16(SP) - CALL ·callMethod(SB) + CALL ·callMethod(SB) RET diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 19a3bb2d7d..196252e1dd 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -84,7 +84,9 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8 DATA _rt0_amd64_lib_argv<>(SB)/8, $0 GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8 -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stack-based Go ABI (and +// in addition there are no calls to this entry point from Go code). +TEXT runtime·rt0_go(SB),NOSPLIT,$0 // copy arguments forward on an even stack MOVQ DI, AX // argc MOVQ SI, BX // argv @@ -229,10 +231,13 @@ ok: // Prevent dead-code elimination of debugCallV1, which is // intended to be called by debuggers. - MOVQ $runtime·debugCallV1(SB), AX + MOVQ $runtime·debugCallV1(SB), AX RET -DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) +// mainPC is a function value for runtime.main, to be passed to newproc. +// The reference to runtime.main is made via ABIInternal, since the +// actual function (not the ABI0 wrapper) is needed by newproc. +DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) GLOBL runtime·mainPC(SB),RODATA,$8 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 @@ -468,7 +473,7 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 JMP AX // Note: can't just "JMP NAME(SB)" - bad inlining results. -TEXT ·reflectcall(SB), NOSPLIT, $0-32 +TEXT ·reflectcall(SB), NOSPLIT, $0-32 MOVLQZX argsize+24(FP), CX DISPATCH(runtime·call16, 16) DISPATCH(runtime·call32, 32) @@ -1354,8 +1359,11 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 RET // The top-most function running on a goroutine -// returns to goexit+PCQuantum. -TEXT runtime·goexit(SB),NOSPLIT,$0-0 +// returns to goexit+PCQuantum. Defined as ABIInternal +// so as to make it identifiable to traceback (this +// function it used as a sentinel; traceback wants to +// see the func PC, not a wrapper PC). +TEXT runtime·goexit(SB),NOSPLIT,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return // traceback from goexit1 must hit code range of goexit @@ -1377,7 +1385,8 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 // - AX is the value being written at DI // It clobbers FLAGS. It does not clobber any general-purpose registers, // but may clobber others (e.g., SSE registers). -TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$120 // Save the registers clobbered by the fast path. This is slightly // faster than having the caller spill these. MOVQ R14, 104(SP) @@ -1461,51 +1470,58 @@ flush: JMP ret // gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX. -TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT,$0 XCHGQ CX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ CX, AX RET // gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX. -TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT,$0 XCHGQ DX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ DX, AX RET // gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX. -TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT,$0 XCHGQ BX, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ BX, AX RET // gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP. -TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT,$0 XCHGQ BP, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ BP, AX RET // gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI. -TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT,$0 XCHGQ SI, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ SI, AX RET // gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8. -TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT,$0 XCHGQ R8, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ R8, AX RET // gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9. -TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stable Go ABI. +TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT,$0 XCHGQ R9, AX - CALL runtime·gcWriteBarrier(SB) + CALL runtime·gcWriteBarrier(SB) XCHGQ R9, AX RET @@ -1544,7 +1560,10 @@ GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below // obey escape analysis requirements. Specifically, it must not pass // a stack pointer to an escaping argument. debugCallV1 cannot check // this invariant. -TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 +// +// This is ABIInternal because Go code injects its PC directly into new +// goroutine stacks. +TEXT runtime·debugCallV1(SB),NOSPLIT,$152-0 // Save all registers that may contain pointers so they can be // conservatively scanned. // @@ -1705,67 +1724,68 @@ TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16 // in the caller's stack frame. These stubs write the args into that stack space and // then tail call to the corresponding runtime handler. // The tail call makes these stubs disappear in backtraces. -TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 +// Defined as ABIInternal since they do not use the stack-based Go ABI. +TEXT runtime·panicIndex(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicIndex(SB) -TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 +TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicIndexU(SB) -TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAlen(SB) -TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAlenU(SB) -TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAcap(SB) -TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSliceAcapU(SB) -TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSliceB(SB) -TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSliceBU(SB) -TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3Alen(SB) -TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3AlenU(SB) -TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3Acap(SB) -TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16 MOVQ DX, x+0(FP) MOVQ BX, y+8(FP) JMP runtime·goPanicSlice3AcapU(SB) -TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSlice3B(SB) -TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16 MOVQ CX, x+0(FP) MOVQ DX, y+8(FP) JMP runtime·goPanicSlice3BU(SB) -TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSlice3C(SB) -TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 +TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16 MOVQ AX, x+0(FP) MOVQ CX, y+8(FP) JMP runtime·goPanicSlice3CU(SB) diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index 67e81adf0b..fcb780f1dc 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -196,7 +196,7 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16 Get CTXT I64Eqz If - CALLNORESUME runtime·sigpanic(SB) + CALLNORESUME runtime·sigpanic(SB) End // caller sp after CALL @@ -300,7 +300,7 @@ TEXT ·reflectcall(SB), NOSPLIT, $0-32 I64Load fn+8(FP) I64Eqz If - CALLNORESUME runtime·sigpanic(SB) + CALLNORESUME runtime·sigpanic(SB) End MOVW argsize+24(FP), R0 diff --git a/src/runtime/duff_amd64.s b/src/runtime/duff_amd64.s index 44dc75d297..2ff5bf6dbc 100644 --- a/src/runtime/duff_amd64.s +++ b/src/runtime/duff_amd64.s @@ -4,7 +4,7 @@ #include "textflag.h" -TEXT runtime·duffzero(SB), NOSPLIT, $0-0 +TEXT runtime·duffzero(SB), NOSPLIT, $0-0 MOVUPS X0,(DI) MOVUPS X0,16(DI) MOVUPS X0,32(DI) @@ -103,7 +103,7 @@ TEXT runtime·duffzero(SB), NOSPLIT, $0-0 RET -TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 +TEXT runtime·duffcopy(SB), NOSPLIT, $0-0 MOVUPS (SI), X0 ADDQ $16, SI MOVUPS X0, (DI) diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 286f81489a..1d614dd003 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -126,7 +126,8 @@ func header(arch string) { } fmt.Fprintf(out, "#include \"go_asm.h\"\n") fmt.Fprintf(out, "#include \"textflag.h\"\n\n") - fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") + fmt.Fprintf(out, "// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally.\n") + fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") } func p(f string, args ...interface{}) { diff --git a/src/runtime/preempt_386.s b/src/runtime/preempt_386.s index c3a5fa1f36..a803b24dc6 100644 --- a/src/runtime/preempt_386.s +++ b/src/runtime/preempt_386.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 PUSHFL ADJSP $156 NOP SP diff --git a/src/runtime/preempt_amd64.s b/src/runtime/preempt_amd64.s index 4765e9f448..92c664d79a 100644 --- a/src/runtime/preempt_amd64.s +++ b/src/runtime/preempt_amd64.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 PUSHQ BP MOVQ SP, BP // Save flags before clobbering them diff --git a/src/runtime/preempt_arm.s b/src/runtime/preempt_arm.s index 8f243c0dcd..bbc9fbb1ea 100644 --- a/src/runtime/preempt_arm.s +++ b/src/runtime/preempt_arm.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVW.W R14, -188(R13) MOVW R0, 4(R13) MOVW R1, 8(R13) diff --git a/src/runtime/preempt_arm64.s b/src/runtime/preempt_arm64.s index 36ee13282c..2b70a28479 100644 --- a/src/runtime/preempt_arm64.s +++ b/src/runtime/preempt_arm64.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R30, -496(RSP) SUB $496, RSP #ifdef GOOS_linux diff --git a/src/runtime/preempt_mips64x.s b/src/runtime/preempt_mips64x.s index 1e123e8077..0d0c157c36 100644 --- a/src/runtime/preempt_mips64x.s +++ b/src/runtime/preempt_mips64x.s @@ -5,7 +5,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVV R31, -488(R29) SUBV $488, R29 MOVV R1, 8(R29) diff --git a/src/runtime/preempt_mipsx.s b/src/runtime/preempt_mipsx.s index afac33e0a0..86d3a918d3 100644 --- a/src/runtime/preempt_mipsx.s +++ b/src/runtime/preempt_mipsx.s @@ -5,7 +5,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVW R31, -244(R29) SUB $244, R29 MOVW R1, 4(R29) diff --git a/src/runtime/preempt_ppc64x.s b/src/runtime/preempt_ppc64x.s index b2d7e30ec7..90634386db 100644 --- a/src/runtime/preempt_ppc64x.s +++ b/src/runtime/preempt_ppc64x.s @@ -5,7 +5,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOVD R31, -488(R1) MOVD LR, R31 MOVDU R31, -520(R1) diff --git a/src/runtime/preempt_riscv64.s b/src/runtime/preempt_riscv64.s index eb68dcba2b..d4f9cc277f 100644 --- a/src/runtime/preempt_riscv64.s +++ b/src/runtime/preempt_riscv64.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 MOV X1, -472(X2) ADD $-472, X2 MOV X3, 8(X2) diff --git a/src/runtime/preempt_s390x.s b/src/runtime/preempt_s390x.s index ca9e47cde1..c6f11571df 100644 --- a/src/runtime/preempt_s390x.s +++ b/src/runtime/preempt_s390x.s @@ -3,7 +3,8 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 IPM R10 MOVD R14, -248(R15) ADD $-248, R15 diff --git a/src/runtime/preempt_wasm.s b/src/runtime/preempt_wasm.s index 0cf57d3d22..da90e8aa6d 100644 --- a/src/runtime/preempt_wasm.s +++ b/src/runtime/preempt_wasm.s @@ -3,6 +3,7 @@ #include "go_asm.h" #include "textflag.h" -TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 +// Note: asyncPreempt doesn't use the internal ABI, but we must be able to inject calls to it from the signal handler, so Go code has to see the PC of this function literally. +TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0 // No async preemption on wasm UNDEF diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index 758d543203..4a86b3371a 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -41,7 +41,9 @@ // func runtime·raceread(addr uintptr) // Called from instrumented code. -TEXT runtime·raceread(SB), NOSPLIT, $0-8 +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would render runtime.getcallerpc ineffective. +TEXT runtime·raceread(SB), NOSPLIT, $0-8 MOVQ addr+0(FP), RARG1 MOVQ (SP), RARG2 // void __tsan_read(ThreadState *thr, void *addr, void *pc); @@ -65,7 +67,9 @@ TEXT runtime·racereadpc(SB), NOSPLIT, $0-24 // func runtime·racewrite(addr uintptr) // Called from instrumented code. -TEXT runtime·racewrite(SB), NOSPLIT, $0-8 +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would render runtime.getcallerpc ineffective. +TEXT runtime·racewrite(SB), NOSPLIT, $0-8 MOVQ addr+0(FP), RARG1 MOVQ (SP), RARG2 // void __tsan_write(ThreadState *thr, void *addr, void *pc); @@ -114,7 +118,9 @@ TEXT runtime·racereadrangepc1(SB), NOSPLIT, $0-24 // func runtime·racewriterange(addr, size uintptr) // Called from instrumented code. -TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 +// Defined as ABIInternal so as to avoid introducing a wrapper, +// which would render runtime.getcallerpc ineffective. +TEXT runtime·racewriterange(SB), NOSPLIT, $0-16 MOVQ addr+0(FP), RARG1 MOVQ size+8(FP), RARG2 MOVQ (SP), RARG3 diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 681cd20274..37cb8dad03 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -380,7 +380,8 @@ TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$72 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Save callee-saved C registers, since the caller may be a C signal handler. MOVQ BX, bx-8(SP) MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set @@ -407,7 +408,8 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$72 // Used instead of sigtramp in programs that use cgo. // Arguments from kernel are in DI, SI, DX. -TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // If no traceback function, do usual sigtramp. MOVQ runtime·cgoTraceback(SB), AX TESTQ AX, AX @@ -450,12 +452,12 @@ TEXT runtime·cgoSigtramp(SB),NOSPLIT,$0 // The first three arguments, and the fifth, are already in registers. // Set the two remaining arguments now. MOVQ runtime·cgoTraceback(SB), CX - MOVQ $runtime·sigtramp(SB), R9 + MOVQ $runtime·sigtramp(SB), R9 MOVQ _cgo_callers(SB), AX JMP AX sigtramp: - JMP runtime·sigtramp(SB) + JMP runtime·sigtramp(SB) sigtrampnog: // Signal arrived on a non-Go thread. If this is SIGPROF, get a @@ -486,7 +488,8 @@ sigtrampnog: // https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/unix/sysv/linux/x86_64/sigaction.c // The code that cares about the precise instructions used is: // https://gcc.gnu.org/viewcvs/gcc/trunk/libgcc/config/i386/linux-unwind.h?revision=219188&view=markup -TEXT runtime·sigreturn(SB),NOSPLIT,$0 +// Defined as ABIInternal since it does not use the stack-based Go ABI. +TEXT runtime·sigreturn(SB),NOSPLIT,$0 MOVQ $SYS_rt_sigreturn, AX SYSCALL INT $3 // not reached -- cgit v1.3