From 0d94f989d12a52ddc3869dbaa02255873f7a8196 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sat, 30 Jan 2021 07:07:42 -0500 Subject: runtime: clean up system calls during cgo callback init During a cgocallback, the runtime calls needm to get an m. The calls made during needm cannot themselves assume that there is an m or a g (which is attached to the m). In the old days of making direct system calls, the only thing you had to do for such functions was mark them //go:nosplit, to avoid the use of g in the stack split prologue. But now, on operating systems that make system calls through shared libraries and use code that saves state in the g or m before doing so, it's not safe to assume g exists. In fact, it is not even safe to call getg(), because it might fault deferencing the TLS storage to find the g pointer (that storage may not be initialized yet, at least on Windows, and perhaps on other systems in the future). The specific routines that are problematic are usleep and osyield, which are called during lock contention in lockextra, called from needm. All this is rather subtle and hidden, so in addition to fixing the problem on Windows, this CL makes the fact of not running on a g much clearer by introducing variants usleep_no_g and osyield_no_g whose names should make clear that there is no g. And then we can remove the various sketchy getg() == nil checks in the existing routines. As part of this cleanup, this CL also deletes onosstack on Windows. onosstack is from back when the runtime was implemented in C. It predates systemstack but does essentially the same thing. Instead of having two different copies of this code, we can use systemstack consistently. This way we need not port onosstack to each architecture. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I3352de1fd0a3c26267c6e209063e6e86abd26187 Reviewed-on: https://go-review.googlesource.com/c/go/+/288793 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/runtime/asm_386.s | 16 ++++++++ src/runtime/asm_amd64.s | 17 ++++++++ src/runtime/asm_arm.s | 15 +++++++ src/runtime/asm_arm64.s | 13 +++++- src/runtime/asm_mips64x.s | 9 ++++ src/runtime/os2_aix.go | 42 ++++++++----------- src/runtime/os3_solaris.go | 20 +++++---- src/runtime/os_darwin.go | 5 +++ src/runtime/os_dragonfly.go | 5 +++ src/runtime/os_freebsd.go | 5 +++ src/runtime/os_js.go | 10 +++++ src/runtime/os_linux.go | 5 +++ src/runtime/os_netbsd.go | 5 +++ src/runtime/os_openbsd_syscall1.go | 5 +++ src/runtime/os_openbsd_syscall2.go | 5 +++ src/runtime/os_plan9.go | 10 +++++ src/runtime/os_windows.go | 38 ++++++++++------- src/runtime/proc.go | 6 +-- src/runtime/stubs2.go | 5 +++ src/runtime/stubs_386.go | 3 ++ src/runtime/stubs_amd64.go | 5 +++ src/runtime/stubs_arm.go | 5 +++ src/runtime/stubs_arm64.go | 5 +++ src/runtime/stubs_mips64x.go | 5 +++ src/runtime/sys_darwin.go | 6 +++ src/runtime/sys_openbsd1.go | 5 +++ src/runtime/sys_openbsd2.go | 6 +++ src/runtime/sys_windows_386.s | 79 +++++------------------------------ src/runtime/sys_windows_amd64.s | 71 ++++---------------------------- src/runtime/sys_windows_arm.s | 84 ++++---------------------------------- 30 files changed, 254 insertions(+), 256 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 471451df28..3030101f03 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -621,6 +621,22 @@ TEXT gosave<>(SB),NOSPLIT,$0 POPL AX RET +// func asmcgocall_no_g(fn, arg unsafe.Pointer) +// Call fn(arg) aligned appropriately for the gcc ABI. +// Called on a system stack, and there may be no g yet (during needm). +TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8 + MOVL fn+0(FP), AX + MOVL arg+4(FP), BX + MOVL SP, DX + SUBL $32, SP + ANDL $~15, SP // alignment, perhaps unnecessary + MOVL DX, 8(SP) // save old SP + MOVL BX, 0(SP) // first argument in x86-32 ABI + CALL AX + MOVL 8(SP), DX + MOVL DX, SP + RET + // func asmcgocall(fn, arg unsafe.Pointer) int32 // Call fn(arg) on the scheduler stack, // aligned appropriately for the gcc ABI. diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 05422c9699..9362ce1213 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -679,6 +679,23 @@ TEXT gosave<>(SB),NOSPLIT,$0 CALL runtime·badctxt(SB) RET +// func asmcgocall_no_g(fn, arg unsafe.Pointer) +// Call fn(arg) aligned appropriately for the gcc ABI. +// Called on a system stack, and there may be no g yet (during needm). +TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16 + MOVQ fn+0(FP), AX + MOVQ arg+8(FP), BX + MOVQ SP, DX + SUBQ $32, SP + ANDQ $~15, SP // alignment + MOVQ DX, 8(SP) + MOVQ BX, DI // DI = first argument in AMD64 ABI + MOVQ BX, CX // CX = first argument in Win64 + CALL AX + MOVQ 8(SP), DX + MOVQ DX, SP + RET + // func asmcgocall(fn, arg unsafe.Pointer) int32 // Call fn(arg) on the scheduler stack, // aligned appropriately for the gcc ABI. diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 23619b1408..109030aada 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -552,6 +552,21 @@ TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 CALL runtime·badctxt(SB) RET +// func asmcgocall_no_g(fn, arg unsafe.Pointer) +// Call fn(arg) aligned appropriately for the gcc ABI. +// Called on a system stack, and there may be no g yet (during needm). +TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-8 + MOVW fn+0(FP), R1 + MOVW arg+4(FP), R0 + MOVW R13, R2 + SUB $32, R13 + BIC $0x7, R13 // alignment for gcc ABI + MOVW R2, 8(R13) + BL (R1) + MOVW 8(R13), R2 + MOVW R2, R13 + RET + // func asmcgocall(fn, arg unsafe.Pointer) int32 // Call fn(arg) on the scheduler stack, // aligned appropriately for the gcc ABI. diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 0ab92be1e4..79efd4cb17 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -873,6 +873,17 @@ TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 CALL runtime·badctxt(SB) RET +// func asmcgocall_no_g(fn, arg unsafe.Pointer) +// Call fn(arg) aligned appropriately for the gcc ABI. +// Called on a system stack, and there may be no g yet (during needm). +TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16 + MOVD fn+0(FP), R1 + MOVD arg+8(FP), R0 + SUB $16, RSP // skip over saved frame pointer below RSP + BL (R1) + ADD $16, RSP // skip over saved frame pointer below RSP + RET + // func asmcgocall(fn, arg unsafe.Pointer) int32 // Call fn(arg) on the scheduler stack, // aligned appropriately for the gcc ABI. @@ -951,7 +962,7 @@ nosave: BL (R1) // Restore stack pointer. MOVD 8(RSP), R2 - MOVD R2, RSP + MOVD R2, RSP MOVD R0, ret+16(FP) RET diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 694950663a..6e1d25cd90 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -413,6 +413,15 @@ TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 JAL runtime·badctxt(SB) RET +// func asmcgocall_no_g(fn, arg unsafe.Pointer) +// Call fn(arg) aligned appropriately for the gcc ABI. +// Called on a system stack, and there may be no g yet (during needm). +TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16 + MOVV fn+0(FP), R25 + MOVV arg+8(FP), R4 + JAL (R25) + RET + // func asmcgocall(fn, arg unsafe.Pointer) int32 // Call fn(arg) on the scheduler stack, // aligned appropriately for the gcc ABI. diff --git a/src/runtime/os2_aix.go b/src/runtime/os2_aix.go index abd1010be9..4d77f0de6d 100644 --- a/src/runtime/os2_aix.go +++ b/src/runtime/os2_aix.go @@ -527,20 +527,17 @@ func internal_cpu_getsystemcfg(label uint) uint { func usleep1(us uint32) //go:nosplit -func usleep(us uint32) { - _g_ := getg() +func usleep_no_g(us uint32) { + usleep1(us) +} - // Check the validity of m because we might be called in cgo callback - // path early enough where there isn't a g or a m available yet. - if _g_ != nil && _g_.m != nil { - r, err := syscall1(&libc_usleep, uintptr(us)) - if int32(r) == -1 { - println("syscall usleep failed: ", hex(err)) - throw("syscall usleep") - } - return +//go:nosplit +func usleep(us uint32) { + r, err := syscall1(&libc_usleep, uintptr(us)) + if int32(r) == -1 { + println("syscall usleep failed: ", hex(err)) + throw("syscall usleep") } - usleep1(us) } //go:nosplit @@ -611,20 +608,17 @@ func raiseproc(sig uint32) { func osyield1() //go:nosplit -func osyield() { - _g_ := getg() +func osyield_no_g() { + osyield1() +} - // Check the validity of m because it might be called during a cgo - // callback early enough where m isn't available yet. - if _g_ != nil && _g_.m != nil { - r, err := syscall0(&libc_sched_yield) - if int32(r) == -1 { - println("syscall osyield failed: ", hex(err)) - throw("syscall osyield") - } - return +//go:nosplit +func osyield() { + r, err := syscall0(&libc_sched_yield) + if int32(r) == -1 { + println("syscall osyield failed: ", hex(err)) + throw("syscall osyield") } - osyield1() } //go:nosplit diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go index 6ba11afd93..4b65139eb8 100644 --- a/src/runtime/os3_solaris.go +++ b/src/runtime/os3_solaris.go @@ -521,6 +521,11 @@ func sysconf(name int32) int64 { func usleep1(usec uint32) +//go:nosplit +func usleep_no_g(µs uint32) { + usleep1(µs) +} + //go:nosplit func usleep(µs uint32) { usleep1(µs) @@ -569,18 +574,15 @@ func setNonblock(fd int32) { func osyield1() //go:nosplit -func osyield() { - _g_ := getg() - - // Check the validity of m because we might be called in cgo callback - // path early enough where there isn't a m available yet. - if _g_ != nil && _g_.m != nil { - sysvicall0(&libc_sched_yield) - return - } +func osyield_no_g() { osyield1() } +//go:nosplit +func osyield() { + sysvicall0(&libc_sched_yield) +} + //go:linkname executablePath os.executablePath var executablePath string diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index 9ca17c20df..470698d0a3 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -330,6 +330,11 @@ func unminit() { func mdestroy(mp *m) { } +//go:nosplit +func osyield_no_g() { + usleep_no_g(1) +} + //go:nosplit func osyield() { usleep(1) diff --git a/src/runtime/os_dragonfly.go b/src/runtime/os_dragonfly.go index 383df54bd4..b786c8ab5f 100644 --- a/src/runtime/os_dragonfly.go +++ b/src/runtime/os_dragonfly.go @@ -51,6 +51,11 @@ func sys_umtx_wakeup(addr *uint32, val int32) int32 func osyield() +//go:nosplit +func osyield_no_g() { + osyield() +} + func kqueue() int32 //go:noescape diff --git a/src/runtime/os_freebsd.go b/src/runtime/os_freebsd.go index 09065ccb68..09dd50ce59 100644 --- a/src/runtime/os_freebsd.go +++ b/src/runtime/os_freebsd.go @@ -36,6 +36,11 @@ func sys_umtx_op(addr *uint32, mode int32, val uint32, uaddr1 uintptr, ut *umtx_ func osyield() +//go:nosplit +func osyield_no_g() { + osyield() +} + func kqueue() int32 //go:noescape diff --git a/src/runtime/os_js.go b/src/runtime/os_js.go index 24261e88a2..5b2c53795a 100644 --- a/src/runtime/os_js.go +++ b/src/runtime/os_js.go @@ -30,12 +30,22 @@ func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) func usleep(usec uint32) +//go:nosplit +func usleep_no_g(usec uint32) { + usleep(usec) +} + func exitThread(wait *uint32) type mOS struct{} func osyield() +//go:nosplit +func osyield_no_g() { + osyield() +} + const _SIGSEGV = 0xb func sigpanic() { diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 058c7daf9c..21d3ae653e 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -410,6 +410,11 @@ func raiseproc(sig uint32) func sched_getaffinity(pid, len uintptr, buf *byte) int32 func osyield() +//go:nosplit +func osyield_no_g() { + osyield() +} + func pipe() (r, w int32, errno int32) func pipe2(flags int32) (r, w int32, errno int32) func setNonblock(fd int32) diff --git a/src/runtime/os_netbsd.go b/src/runtime/os_netbsd.go index 2b742a3711..0328fa57ae 100644 --- a/src/runtime/os_netbsd.go +++ b/src/runtime/os_netbsd.go @@ -67,6 +67,11 @@ func lwp_self() int32 func osyield() +//go:nosplit +func osyield_no_g() { + osyield() +} + func kqueue() int32 //go:noescape diff --git a/src/runtime/os_openbsd_syscall1.go b/src/runtime/os_openbsd_syscall1.go index b0bef4c504..f37da04194 100644 --- a/src/runtime/os_openbsd_syscall1.go +++ b/src/runtime/os_openbsd_syscall1.go @@ -13,3 +13,8 @@ func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort func thrwakeup(ident uintptr, n int32) int32 func osyield() + +//go:nosplit +func osyield_no_g() { + osyield() +} diff --git a/src/runtime/os_openbsd_syscall2.go b/src/runtime/os_openbsd_syscall2.go index ab940510af..81cfb085aa 100644 --- a/src/runtime/os_openbsd_syscall2.go +++ b/src/runtime/os_openbsd_syscall2.go @@ -32,6 +32,11 @@ func closefd(fd int32) int32 func exit(code int32) func usleep(usec uint32) +//go:nosplit +func usleep_no_g(usec uint32) { + usleep(usec) +} + // write calls the write system call. // It returns a non-negative number of bytes written or a negative errno value. //go:noescape diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index 2a84a73716..77665f461a 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -339,6 +339,11 @@ func osyield() { sleep(0) } +//go:nosplit +func osyield_no_g() { + osyield() +} + //go:nosplit func usleep(µs uint32) { ms := int32(µs / 1000) @@ -348,6 +353,11 @@ func usleep(µs uint32) { sleep(ms) } +//go:nosplit +func usleep_no_g(usec uint32) { + usleep(usec) +} + //go:nosplit func nanotime1() int64 { var scratch int64 diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index e6b22e3167..1bf3309dfd 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -461,15 +461,12 @@ func initHighResTimer() { h := createHighResTimer() if h != 0 { haveHighResTimer = true - usleep2Addr = unsafe.Pointer(funcPC(usleep2HighRes)) stdcall1(_CloseHandle, h) } } func osinit() { asmstdcallAddr = unsafe.Pointer(funcPC(asmstdcall)) - usleep2Addr = unsafe.Pointer(funcPC(usleep2)) - switchtothreadAddr = unsafe.Pointer(funcPC(switchtothread)) setBadSignalMsg() @@ -1061,26 +1058,39 @@ func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { return stdcall(fn) } -// In sys_windows_386.s and sys_windows_amd64.s. -func onosstack(fn unsafe.Pointer, arg uint32) - -// These are not callable functions. They should only be called via onosstack. -func usleep2(usec uint32) -func usleep2HighRes(usec uint32) +// These must run on the system stack only. +func usleep2(dt int32) +func usleep2HighRes(dt int32) func switchtothread() -var usleep2Addr unsafe.Pointer -var switchtothreadAddr unsafe.Pointer +//go:nosplit +func osyield_no_g() { + switchtothread() +} //go:nosplit func osyield() { - onosstack(switchtothreadAddr, 0) + systemstack(switchtothread) +} + +//go:nosplit +func usleep_no_g(us uint32) { + dt := -10 * int32(us) // relative sleep (negative), 100ns units + usleep2(dt) } //go:nosplit func usleep(us uint32) { - // Have 1us units; want 100ns units. - onosstack(usleep2Addr, 10*us) + systemstack(func() { + dt := -10 * int32(us) // relative sleep (negative), 100ns units + // If the high-res timer is available and its handle has been allocated for this m, use it. + // Otherwise fall back to the low-res one, which doesn't need a handle. + if haveHighResTimer && getg().m.highResTimer != 0 { + usleep2HighRes(dt) + } else { + usleep2(dt) + } + }) } func ctrlhandler1(_type uint32) uint32 { diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 73a789c189..4092dd55cb 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2012,7 +2012,7 @@ func lockextra(nilokay bool) *m { for { old := atomic.Loaduintptr(&extram) if old == locked { - osyield() + osyield_no_g() continue } if old == 0 && !nilokay { @@ -2023,13 +2023,13 @@ func lockextra(nilokay bool) *m { atomic.Xadd(&extraMWaiters, 1) incr = true } - usleep(1) + usleep_no_g(1) continue } if atomic.Casuintptr(&extram, old, locked) { return (*m)(unsafe.Pointer(old)) } - osyield() + osyield_no_g() continue } } diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index 85088b3ab9..96096d236b 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -23,6 +23,11 @@ func closefd(fd int32) int32 func exit(code int32) func usleep(usec uint32) +//go:nosplit +func usleep_no_g(usec uint32) { + usleep(usec) +} + // write calls the write system call. // It returns a non-negative number of bytes written or a negative errno value. //go:noescape diff --git a/src/runtime/stubs_386.go b/src/runtime/stubs_386.go index 5108294d83..300f167fff 100644 --- a/src/runtime/stubs_386.go +++ b/src/runtime/stubs_386.go @@ -15,3 +15,6 @@ func stackcheck() // Called from assembly only; declared for go vet. func setldt(slot uintptr, base unsafe.Pointer, size uintptr) func emptyfunc() + +//go:noescape +func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/stubs_amd64.go b/src/runtime/stubs_amd64.go index 8c14bc2271..bf98493e9d 100644 --- a/src/runtime/stubs_amd64.go +++ b/src/runtime/stubs_amd64.go @@ -4,6 +4,8 @@ package runtime +import "unsafe" + // Called from compiled code; declared for vet; do NOT call from Go. func gcWriteBarrierCX() func gcWriteBarrierDX() @@ -35,3 +37,6 @@ func retpolineR12() func retpolineR13() func retpolineR14() func retpolineR15() + +//go:noescape +func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/stubs_arm.go b/src/runtime/stubs_arm.go index c13bf16de2..52c32937ae 100644 --- a/src/runtime/stubs_arm.go +++ b/src/runtime/stubs_arm.go @@ -4,6 +4,8 @@ package runtime +import "unsafe" + // Called from compiler-generated code; declared for go vet. func udiv() func _div() @@ -18,3 +20,6 @@ func save_g() func emptyfunc() func _initcgo() func read_tls_fallback() + +//go:noescape +func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/stubs_arm64.go b/src/runtime/stubs_arm64.go index 44c566e602..6e6e7df6b8 100644 --- a/src/runtime/stubs_arm64.go +++ b/src/runtime/stubs_arm64.go @@ -4,6 +4,11 @@ package runtime +import "unsafe" + // Called from assembly only; declared for go vet. func load_g() func save_g() + +//go:noescape +func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/stubs_mips64x.go b/src/runtime/stubs_mips64x.go index 4e62c1ce90..652e7a9e34 100644 --- a/src/runtime/stubs_mips64x.go +++ b/src/runtime/stubs_mips64x.go @@ -6,6 +6,11 @@ package runtime +import "unsafe" + // Called from assembly only; declared for go vet. func load_g() func save_g() + +//go:noescape +func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 4a3f2fc453..dacce2ee1a 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -227,6 +227,12 @@ func usleep(usec uint32) { } func usleep_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func usleep_no_g(usec uint32) { + asmcgocall_no_g(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) +} + //go:nosplit //go:cgo_unsafe_args func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { diff --git a/src/runtime/sys_openbsd1.go b/src/runtime/sys_openbsd1.go index e2886218db..44c7871ceb 100644 --- a/src/runtime/sys_openbsd1.go +++ b/src/runtime/sys_openbsd1.go @@ -27,6 +27,11 @@ func osyield() { } func sched_yield_trampoline() +//go:nosplit +func osyield_no_g() { + asmcgocall_no_g(unsafe.Pointer(funcPC(sched_yield_trampoline)), unsafe.Pointer(nil)) +} + //go:cgo_import_dynamic libc_thrsleep __thrsleep "libc.so" //go:cgo_import_dynamic libc_thrwakeup __thrwakeup "libc.so" //go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go index 474e7145e7..33032596c3 100644 --- a/src/runtime/sys_openbsd2.go +++ b/src/runtime/sys_openbsd2.go @@ -128,6 +128,12 @@ func usleep(usec uint32) { } func usleep_trampoline() +//go:nosplit +//go:cgo_unsafe_args +func usleep_no_g(usec uint32) { + asmcgocall_no_g(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) +} + //go:nosplit //go:cgo_unsafe_args func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index ef8a3dd3c2..b3972ac78d 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -347,60 +347,11 @@ TEXT runtime·setldt(SB),NOSPLIT,$0 MOVL CX, 0x14(FS) RET -// onosstack calls fn on OS stack. -// func onosstack(fn unsafe.Pointer, arg uint32) -TEXT runtime·onosstack(SB),NOSPLIT,$0 - MOVL fn+0(FP), AX // to hide from 8l - MOVL arg+4(FP), BX - - // Execute call on m->g0 stack, in case we are not actually - // calling a system call wrapper, like when running under WINE. - get_tls(CX) - CMPL CX, $0 - JNE 3(PC) - // Not a Go-managed thread. Do not switch stack. - CALL AX - RET - - MOVL g(CX), BP - MOVL g_m(BP), BP - - // leave pc/sp for cpu profiler - MOVL (SP), SI - MOVL SI, m_libcallpc(BP) - MOVL g(CX), SI - MOVL SI, m_libcallg(BP) - // sp must be the last, because once async cpu profiler finds - // all three values to be non-zero, it will use them - LEAL fn+0(FP), SI - MOVL SI, m_libcallsp(BP) - - MOVL m_g0(BP), SI - CMPL g(CX), SI - JNE switch - // executing on m->g0 already - CALL AX - JMP ret - -switch: - // Switch to m->g0 stack and back. - MOVL (g_sched+gobuf_sp)(SI), SI - MOVL SP, -4(SI) - LEAL -4(SI), SP - CALL AX - MOVL 0(SP), SP - -ret: - get_tls(CX) - MOVL g(CX), BP - MOVL g_m(BP), BP - MOVL $0, m_libcallsp(BP) - RET - -// Runs on OS stack. duration (in 100ns units) is in BX. -TEXT runtime·usleep2(SB),NOSPLIT,$20 - // Want negative 100ns units. - NEGL BX +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g may be nil. +TEXT runtime·usleep2(SB),NOSPLIT,$20-4 + MOVL dt+0(FP), BX MOVL $-1, hi-4(SP) MOVL BX, lo-8(SP) LEAL lo-8(SP), BX @@ -413,17 +364,15 @@ TEXT runtime·usleep2(SB),NOSPLIT,$20 MOVL BP, SP RET -// Runs on OS stack. duration (in 100ns units) is in BX. -TEXT runtime·usleep2HighRes(SB),NOSPLIT,$36 - get_tls(CX) - CMPL CX, $0 - JE gisnotset - - // Want negative 100ns units. - NEGL BX +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g is valid. +TEXT runtime·usleep2HighRes(SB),NOSPLIT,$36-4 + MOVL dt+0(FP), BX MOVL $-1, hi-4(SP) MOVL BX, lo-8(SP) + get_tls(CX) MOVL g(CX), CX MOVL g_m(CX), CX MOVL (m_mOS+mOS_highResTimer)(CX), CX @@ -452,12 +401,6 @@ TEXT runtime·usleep2HighRes(SB),NOSPLIT,$36 RET -gisnotset: - // TLS is not configured. Call usleep2 instead. - MOVL $runtime·usleep2(SB), AX - CALL AX - RET - // Runs on OS stack. TEXT runtime·switchtothread(SB),NOSPLIT,$0 MOVL SP, BP diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index d1690cad58..2bd7b74848 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -388,61 +388,16 @@ TEXT runtime·settls(SB),NOSPLIT,$0 MOVQ DI, 0x28(GS) RET -// func onosstack(fn unsafe.Pointer, arg uint32) -TEXT runtime·onosstack(SB),NOSPLIT,$0 - MOVQ fn+0(FP), AX // to hide from 6l - MOVL arg+8(FP), BX - - // Execute call on m->g0 stack, in case we are not actually - // calling a system call wrapper, like when running under WINE. - get_tls(R15) - CMPQ R15, $0 - JNE 3(PC) - // Not a Go-managed thread. Do not switch stack. - CALL AX - RET - - MOVQ g(R15), R13 - MOVQ g_m(R13), R13 - - // leave pc/sp for cpu profiler - MOVQ (SP), R12 - MOVQ R12, m_libcallpc(R13) - MOVQ g(R15), R12 - MOVQ R12, m_libcallg(R13) - // sp must be the last, because once async cpu profiler finds - // all three values to be non-zero, it will use them - LEAQ fn+0(FP), R12 - MOVQ R12, m_libcallsp(R13) - - MOVQ m_g0(R13), R14 - CMPQ g(R15), R14 - JNE switch - // executing on m->g0 already - CALL AX - JMP ret - -switch: - // Switch to m->g0 stack and back. - MOVQ (g_sched+gobuf_sp)(R14), R14 - MOVQ SP, -8(R14) - LEAQ -8(R14), SP - CALL AX - MOVQ 0(SP), SP - -ret: - MOVQ $0, m_libcallsp(R13) - RET - -// Runs on OS stack. duration (in 100ns units) is in BX. +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g may be nil. // The function leaves room for 4 syscall parameters // (as per windows amd64 calling convention). -TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$48 +TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$48-4 + MOVLQSX dt+0(FP), BX MOVQ SP, AX ANDQ $~15, SP // alignment as per Windows requirement MOVQ AX, 40(SP) - // Want negative 100ns units. - NEGQ BX LEAQ 32(SP), R8 // ptime MOVQ BX, (R8) MOVQ $-1, CX // handle @@ -452,11 +407,11 @@ TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$48 MOVQ 40(SP), SP RET -// Runs on OS stack. duration (in 100ns units) is in BX. -TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$72 +// Runs on OS stack. duration (in -100ns units) is in dt+0(FP). +// g is valid. +TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$72-4 + MOVLQSX dt+0(FP), BX get_tls(CX) - CMPQ CX, $0 - JE gisnotset MOVQ SP, AX ANDQ $~15, SP // alignment as per Windows requirement @@ -466,8 +421,6 @@ TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$72 MOVQ g_m(CX), CX MOVQ (m_mOS+mOS_highResTimer)(CX), CX // hTimer MOVQ CX, 48(SP) // save hTimer for later - // Want negative 100ns units. - NEGQ BX LEAQ 56(SP), DX // lpDueTime MOVQ BX, (DX) MOVQ $0, R8 // lPeriod @@ -487,12 +440,6 @@ TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$72 MOVQ 64(SP), SP RET -gisnotset: - // TLS is not configured. Call usleep2 instead. - MOVQ $runtime·usleep2(SB), AX - CALL AX - RET - // Runs on OS stack. TEXT runtime·switchtothread(SB),NOSPLIT|NOFRAME,$0 MOVQ SP, AX diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index fe267080cc..1d928a4f7d 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -377,79 +377,11 @@ TEXT runtime·tstart_stdcall(SB),NOSPLIT|NOFRAME,$0 MOVW $0, R0 MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} -// onosstack calls fn on OS stack. -// adapted from asm_arm.s : systemstack -// func onosstack(fn unsafe.Pointer, arg uint32) -TEXT runtime·onosstack(SB),NOSPLIT,$0 - MOVW fn+0(FP), R5 // R5 = fn - MOVW arg+4(FP), R6 // R6 = arg - - // This function can be called when there is no g, - // for example, when we are handling a callback on a non-go thread. - // In this case we're already on the system stack. - CMP $0, g - BEQ noswitch - - MOVW g_m(g), R1 // R1 = m - - MOVW m_gsignal(R1), R2 // R2 = gsignal - CMP g, R2 - B.EQ noswitch - - MOVW m_g0(R1), R2 // R2 = g0 - CMP g, R2 - B.EQ noswitch - - MOVW m_curg(R1), R3 - CMP g, R3 - B.EQ switch - - // Bad: g is not gsignal, not g0, not curg. What is it? - // Hide call from linker nosplit analysis. - MOVW $runtime·badsystemstack(SB), R0 - BL (R0) - B runtime·abort(SB) - -switch: - // save our state in g->sched. Pretend to - // be systemstack_switch if the G stack is scanned. - MOVW $runtime·systemstack_switch(SB), R3 - ADD $4, R3, R3 // get past push {lr} - MOVW R3, (g_sched+gobuf_pc)(g) - MOVW R13, (g_sched+gobuf_sp)(g) - MOVW LR, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) - - // switch to g0 - MOVW R2, g - MOVW (g_sched+gobuf_sp)(R2), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $4, R3, R3 - MOVW $runtime·mstart(SB), R4 - MOVW R4, 0(R3) - MOVW R3, R13 - - // call target function - MOVW R6, R0 // arg - BL (R5) - - // switch back to g - MOVW g_m(g), R1 - MOVW m_curg(R1), g - MOVW (g_sched+gobuf_sp)(g), R13 - MOVW $0, R3 - MOVW R3, (g_sched+gobuf_sp)(g) - RET - -noswitch: - // Using a tail call here cleans up tracebacks since we won't stop - // at an intermediate systemstack. - MOVW.P 4(R13), R14 // restore LR - MOVW R6, R0 // arg - B (R5) - -// Runs on OS stack. Duration (in 100ns units) is in R0. -TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$0 +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g may be nil. +TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$0-4 + MOVW dt+0(FP), R0 MOVM.DB.W [R4, R14], (R13) // push {r4, lr} MOVW R13, R4 // Save SP SUB $8, R13 // R13 = R13 - 8 @@ -465,9 +397,11 @@ TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$0 MOVW R4, R13 // Restore SP MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} -// Runs on OS stack. Duration (in 100ns units) is in R0. +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g is valid. // TODO: neeeds to be implemented properly. -TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$0-4 B runtime·abort(SB) // Runs on OS stack. -- cgit v1.3-5-g9baa From 678568a5cfe1806c16bf478234d6dac283c3474d Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sat, 30 Jan 2021 16:18:51 -0500 Subject: runtime: delete windows setlasterror (unused) This is dead code and need not be ported to each architecture. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I2d0072b377f73e49d7158ea304670c26f5486c59 Reviewed-on: https://go-review.googlesource.com/c/go/+/288794 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Alex Brainman Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/runtime/os_windows.go | 1 - src/runtime/sys_windows_386.s | 5 ----- src/runtime/sys_windows_amd64.s | 6 ------ src/runtime/sys_windows_arm.s | 5 ----- 4 files changed, 17 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 1bf3309dfd..a2a124cd9d 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -379,7 +379,6 @@ const ( // in sys_windows_386.s and sys_windows_amd64.s: func externalthreadhandler() func getlasterror() uint32 -func setlasterror(err uint32) // When loading DLLs, we prefer to use LoadLibraryEx with // LOAD_LIBRARY_SEARCH_* flags, if available. LoadLibraryEx is not diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index b3972ac78d..c556e3a3c2 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -66,11 +66,6 @@ TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVL AX, ret+0(FP) RET -TEXT runtime·setlasterror(SB),NOSPLIT,$0 - MOVL err+0(FP), AX - MOVL AX, 0x34(FS) - RET - // Called by Windows as a Vectored Exception Handler (VEH). // First argument is pointer to struct containing // exception record and context pointers. diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 2bd7b74848..9cd14016b0 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -103,12 +103,6 @@ TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVL AX, ret+0(FP) RET -TEXT runtime·setlasterror(SB),NOSPLIT,$0 - MOVL err+0(FP), AX - MOVQ 0x30(GS), CX - MOVL AX, 0x68(CX) - RET - // Called by Windows as a Vectored Exception Handler (VEH). // First argument is pointer to struct containing // exception record and context pointers. diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 1d928a4f7d..d2bdc50e3b 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -103,11 +103,6 @@ TEXT runtime·getlasterror(SB),NOSPLIT,$0 MOVW R0, ret+0(FP) RET -TEXT runtime·setlasterror(SB),NOSPLIT|NOFRAME,$0 - MRC 15, 0, R1, C13, C0, 2 - MOVW R0, 0x34(R1) - RET - // Called by Windows as a Vectored Exception Handler (VEH). // First argument is pointer to struct containing // exception record and context pointers. -- cgit v1.3-5-g9baa From 8ac23a1f151a9b1842797652ed7761f397055b5b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 01:06:52 -0500 Subject: runtime: document, clean up internal/sys Document what the values in internal/sys mean. Remove various special cases for arm64 in the code using StackAlign. Delete Uintreg - it was for GOARCH=amd64p32, which was specific to GOOS=nacl and has been retired. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I40e8fa07b4e192298b6536b98a72a751951a4383 Reviewed-on: https://go-review.googlesource.com/c/go/+/288795 Trust: Russ Cox Reviewed-by: Cherry Zhang Reviewed-by: Ian Lance Taylor --- src/runtime/cgocall.go | 9 +------- src/runtime/export_test.go | 2 -- src/runtime/internal/sys/arch.go | 38 +++++++++++++++++++++++++++++++ src/runtime/internal/sys/arch_386.go | 13 ++++------- src/runtime/internal/sys/arch_amd64.go | 13 ++++------- src/runtime/internal/sys/arch_arm.go | 13 ++++------- src/runtime/internal/sys/arch_arm64.go | 13 ++++------- src/runtime/internal/sys/arch_mips.go | 13 ++++------- src/runtime/internal/sys/arch_mips64.go | 13 ++++------- src/runtime/internal/sys/arch_mips64le.go | 13 ++++------- src/runtime/internal/sys/arch_mipsle.go | 13 ++++------- src/runtime/internal/sys/arch_ppc64.go | 13 ++++------- src/runtime/internal/sys/arch_ppc64le.go | 13 ++++------- src/runtime/internal/sys/arch_riscv64.go | 15 ++++-------- src/runtime/internal/sys/arch_s390x.go | 13 ++++------- src/runtime/internal/sys/arch_wasm.go | 13 ++++------- src/runtime/internal/sys/stubs.go | 16 ------------- src/runtime/os3_plan9.go | 4 ---- src/runtime/proc.go | 12 +++++----- src/runtime/runtime2.go | 2 +- src/runtime/runtime_test.go | 4 ++-- src/runtime/signal_arm64.go | 2 +- src/runtime/stack.go | 4 ++-- src/runtime/sys_wasm.go | 4 ---- src/runtime/sys_x86.go | 4 ---- src/runtime/traceback.go | 20 +++++++--------- 26 files changed, 124 insertions(+), 168 deletions(-) delete mode 100644 src/runtime/internal/sys/stubs.go (limited to 'src/runtime') diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 20cacd6043..534a2c4295 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -306,14 +306,7 @@ func unwindm(restore *bool) { // unwind of g's stack (see comment at top of file). mp := acquirem() sched := &mp.g0.sched - switch GOARCH { - default: - throw("unwindm not implemented") - case "386", "amd64", "arm", "ppc64", "ppc64le", "mips64", "mips64le", "s390x", "mips", "mipsle", "riscv64": - sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + sys.MinFrameSize)) - case "arm64": - sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + 16)) - } + sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + alignUp(sys.MinFrameSize, sys.StackAlign))) // Do the accounting that cgocall will not have a chance to do // during an unwind. diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 22fef3134f..a48bb2636f 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -200,8 +200,6 @@ func GostringW(w []uint16) (s string) { return } -type Uintreg sys.Uintreg - var Open = open var Close = closefd var Read = read diff --git a/src/runtime/internal/sys/arch.go b/src/runtime/internal/sys/arch.go index 13c00cf639..69278bf2d5 100644 --- a/src/runtime/internal/sys/arch.go +++ b/src/runtime/internal/sys/arch.go @@ -18,3 +18,41 @@ const ( S390X WASM ) + +// PtrSize is the size of a pointer in bytes - unsafe.Sizeof(uintptr(0)) but as an ideal constant. +// It is also the size of the machine's native word size (that is, 4 on 32-bit systems, 8 on 64-bit). +const PtrSize = 4 << (^uintptr(0) >> 63) + +// AIX requires a larger stack for syscalls. +const StackGuardMultiplier = StackGuardMultiplierDefault*(1-GoosAix) + 2*GoosAix + +// ArchFamily is the architecture family (AMD64, ARM, ...) +const ArchFamily ArchFamilyType = _ArchFamily + +// BigEndian reports whether the architecture is big-endian. +const BigEndian = GoarchArmbe|GoarchArm64be|GoarchMips|GoarchMips64|GoarchPpc|GoarchPpc64|GoarchS390|GoarchS390x|GoarchSparc|GoarchSparc64 == 1 + +// DefaultPhysPageSize is the default physical page size. +const DefaultPhysPageSize = _DefaultPhysPageSize + +// PCQuantum is the minimal unit for a program counter (1 on x86, 4 on most other systems). +// The various PC tables record PC deltas pre-divided by PCQuantum. +const PCQuantum = _PCQuantum + +// Int64Align is the required alignment for a 64-bit integer (4 on 32-bit systems, 8 on 64-bit). +const Int64Align = PtrSize + +// MinFrameSize is the size of the system-reserved words at the bottom +// of a frame (just above the architectural stack pointer). +// It is zero on x86 and PtrSize on most non-x86 (LR-based) systems. +// On PowerPC it is larger, to cover three more reserved words: +// the compiler word, the link editor word, and the TOC save word. +const MinFrameSize = _MinFrameSize + +// StackAlign is the required alignment of the SP register. +// The stack must be at least word aligned, but some architectures require more. +const StackAlign = _StackAlign + +// DefaultGoroot is set by the linker for use by package runtime. +// It doesn't really belong in this file or this package. +var DefaultGoroot string diff --git a/src/runtime/internal/sys/arch_386.go b/src/runtime/internal/sys/arch_386.go index b51f70a512..1ebce3435e 100644 --- a/src/runtime/internal/sys/arch_386.go +++ b/src/runtime/internal/sys/arch_386.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = I386 - BigEndian = false - DefaultPhysPageSize = 4096 - PCQuantum = 1 - Int64Align = 4 - MinFrameSize = 0 + _ArchFamily = I386 + _DefaultPhysPageSize = 4096 + _PCQuantum = 1 + _MinFrameSize = 0 + _StackAlign = PtrSize ) - -type Uintreg uint32 diff --git a/src/runtime/internal/sys/arch_amd64.go b/src/runtime/internal/sys/arch_amd64.go index 3d6776e71e..7f003d0f1d 100644 --- a/src/runtime/internal/sys/arch_amd64.go +++ b/src/runtime/internal/sys/arch_amd64.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = AMD64 - BigEndian = false - DefaultPhysPageSize = 4096 - PCQuantum = 1 - Int64Align = 8 - MinFrameSize = 0 + _ArchFamily = AMD64 + _DefaultPhysPageSize = 4096 + _PCQuantum = 1 + _MinFrameSize = 0 + _StackAlign = PtrSize ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_arm.go b/src/runtime/internal/sys/arch_arm.go index 97960d6f83..ef2048bb71 100644 --- a/src/runtime/internal/sys/arch_arm.go +++ b/src/runtime/internal/sys/arch_arm.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = ARM - BigEndian = false - DefaultPhysPageSize = 65536 - PCQuantum = 4 - Int64Align = 4 - MinFrameSize = 4 + _ArchFamily = ARM + _DefaultPhysPageSize = 65536 + _PCQuantum = 4 + _MinFrameSize = 4 + _StackAlign = PtrSize ) - -type Uintreg uint32 diff --git a/src/runtime/internal/sys/arch_arm64.go b/src/runtime/internal/sys/arch_arm64.go index 911a9485e1..b9f2f7b1fe 100644 --- a/src/runtime/internal/sys/arch_arm64.go +++ b/src/runtime/internal/sys/arch_arm64.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = ARM64 - BigEndian = false - DefaultPhysPageSize = 65536 - PCQuantum = 4 - Int64Align = 8 - MinFrameSize = 8 + _ArchFamily = ARM64 + _DefaultPhysPageSize = 65536 + _PCQuantum = 4 + _MinFrameSize = 8 + _StackAlign = 16 ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_mips.go b/src/runtime/internal/sys/arch_mips.go index 75cdb2e07f..4cb0eebea7 100644 --- a/src/runtime/internal/sys/arch_mips.go +++ b/src/runtime/internal/sys/arch_mips.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = MIPS - BigEndian = true - DefaultPhysPageSize = 65536 - PCQuantum = 4 - Int64Align = 4 - MinFrameSize = 4 + _ArchFamily = MIPS + _DefaultPhysPageSize = 65536 + _PCQuantum = 4 + _MinFrameSize = 4 + _StackAlign = PtrSize ) - -type Uintreg uint32 diff --git a/src/runtime/internal/sys/arch_mips64.go b/src/runtime/internal/sys/arch_mips64.go index 494291a802..57636ac4a4 100644 --- a/src/runtime/internal/sys/arch_mips64.go +++ b/src/runtime/internal/sys/arch_mips64.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = MIPS64 - BigEndian = true - DefaultPhysPageSize = 16384 - PCQuantum = 4 - Int64Align = 8 - MinFrameSize = 8 + _ArchFamily = MIPS64 + _DefaultPhysPageSize = 16384 + _PCQuantum = 4 + _MinFrameSize = 8 + _StackAlign = PtrSize ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_mips64le.go b/src/runtime/internal/sys/arch_mips64le.go index d36d1202f6..57636ac4a4 100644 --- a/src/runtime/internal/sys/arch_mips64le.go +++ b/src/runtime/internal/sys/arch_mips64le.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = MIPS64 - BigEndian = false - DefaultPhysPageSize = 16384 - PCQuantum = 4 - Int64Align = 8 - MinFrameSize = 8 + _ArchFamily = MIPS64 + _DefaultPhysPageSize = 16384 + _PCQuantum = 4 + _MinFrameSize = 8 + _StackAlign = PtrSize ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_mipsle.go b/src/runtime/internal/sys/arch_mipsle.go index 323bf82059..4240f5ce47 100644 --- a/src/runtime/internal/sys/arch_mipsle.go +++ b/src/runtime/internal/sys/arch_mipsle.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = MIPS - BigEndian = false - DefaultPhysPageSize = 65536 - PCQuantum = 4 - Int64Align = 4 - MinFrameSize = 4 + _ArchFamily = MIPS + _DefaultPhysPageSize = 65536 + _PCQuantum = 4 + _MinFrameSize = 4 + _StackAlign = PtrSize ) - -type Uintreg uint32 diff --git a/src/runtime/internal/sys/arch_ppc64.go b/src/runtime/internal/sys/arch_ppc64.go index da1fe3d596..1869213ce2 100644 --- a/src/runtime/internal/sys/arch_ppc64.go +++ b/src/runtime/internal/sys/arch_ppc64.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = PPC64 - BigEndian = true - DefaultPhysPageSize = 65536 - PCQuantum = 4 - Int64Align = 8 - MinFrameSize = 32 + _ArchFamily = PPC64 + _DefaultPhysPageSize = 65536 + _PCQuantum = 4 + _MinFrameSize = 32 + _StackAlign = 16 ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_ppc64le.go b/src/runtime/internal/sys/arch_ppc64le.go index 605979903a..1869213ce2 100644 --- a/src/runtime/internal/sys/arch_ppc64le.go +++ b/src/runtime/internal/sys/arch_ppc64le.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = PPC64 - BigEndian = false - DefaultPhysPageSize = 65536 - PCQuantum = 4 - Int64Align = 8 - MinFrameSize = 32 + _ArchFamily = PPC64 + _DefaultPhysPageSize = 65536 + _PCQuantum = 4 + _MinFrameSize = 32 + _StackAlign = 16 ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_riscv64.go b/src/runtime/internal/sys/arch_riscv64.go index 7cdcc8fcbd..360d236e32 100644 --- a/src/runtime/internal/sys/arch_riscv64.go +++ b/src/runtime/internal/sys/arch_riscv64.go @@ -5,14 +5,9 @@ package sys const ( - ArchFamily = RISCV64 - BigEndian = false - CacheLineSize = 64 - DefaultPhysPageSize = 4096 - PCQuantum = 4 - Int64Align = 8 - HugePageSize = 1 << 21 - MinFrameSize = 8 + _ArchFamily = RISCV64 + _DefaultPhysPageSize = 4096 + _PCQuantum = 4 + _MinFrameSize = 8 + _StackAlign = PtrSize ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_s390x.go b/src/runtime/internal/sys/arch_s390x.go index 12cb8a0fcb..e33e0b7f2b 100644 --- a/src/runtime/internal/sys/arch_s390x.go +++ b/src/runtime/internal/sys/arch_s390x.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = S390X - BigEndian = true - DefaultPhysPageSize = 4096 - PCQuantum = 2 - Int64Align = 8 - MinFrameSize = 8 + _ArchFamily = S390X + _DefaultPhysPageSize = 4096 + _PCQuantum = 2 + _MinFrameSize = 8 + _StackAlign = PtrSize ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/arch_wasm.go b/src/runtime/internal/sys/arch_wasm.go index eb825df626..ee919ff9e6 100644 --- a/src/runtime/internal/sys/arch_wasm.go +++ b/src/runtime/internal/sys/arch_wasm.go @@ -5,12 +5,9 @@ package sys const ( - ArchFamily = WASM - BigEndian = false - DefaultPhysPageSize = 65536 - PCQuantum = 1 - Int64Align = 8 - MinFrameSize = 0 + _ArchFamily = WASM + _DefaultPhysPageSize = 65536 + _PCQuantum = 1 + _MinFrameSize = 0 + _StackAlign = PtrSize ) - -type Uintreg uint64 diff --git a/src/runtime/internal/sys/stubs.go b/src/runtime/internal/sys/stubs.go deleted file mode 100644 index 10b0173f60..0000000000 --- a/src/runtime/internal/sys/stubs.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package sys - -// Declarations for runtime services implemented in C or assembly. - -const PtrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const -const RegSize = 4 << (^Uintreg(0) >> 63) // unsafe.Sizeof(uintreg(0)) but an ideal const -const SpAlign = 1*(1-GoarchArm64) + 16*GoarchArm64 // SP alignment: 1 normally, 16 for ARM64 - -var DefaultGoroot string // set at link time - -// AIX requires a larger stack for syscalls. -const StackGuardMultiplier = StackGuardMultiplierDefault*(1-GoosAix) + 2*GoosAix diff --git a/src/runtime/os3_plan9.go b/src/runtime/os3_plan9.go index 15ca3359d2..b6ee98cab6 100644 --- a/src/runtime/os3_plan9.go +++ b/src/runtime/os3_plan9.go @@ -92,10 +92,6 @@ func sighandler(_ureg *ureg, note *byte, gp *g) int { if usesLR { c.setlr(pc) } else { - if sys.RegSize > sys.PtrSize { - sp -= sys.PtrSize - *(*uintptr)(unsafe.Pointer(sp)) = 0 - } sp -= sys.PtrSize *(*uintptr)(unsafe.Pointer(sp)) = pc c.setsp(sp) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 4092dd55cb..1dbd01ed40 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1900,7 +1900,7 @@ func oneNewExtraM() { gp := malg(4096) gp.sched.pc = funcPC(goexit) + sys.PCQuantum gp.sched.sp = gp.stack.hi - gp.sched.sp -= 4 * sys.RegSize // extra space in case of reads slightly beyond frame + gp.sched.sp -= 4 * sys.PtrSize // extra space in case of reads slightly beyond frame gp.sched.lr = 0 gp.sched.g = guintptr(unsafe.Pointer(gp)) gp.syscallpc = gp.sched.pc @@ -4009,9 +4009,9 @@ func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerp // We could allocate a larger initial stack if necessary. // Not worth it: this is almost always an error. - // 4*sizeof(uintreg): extra space added below - // sizeof(uintreg): caller's LR (arm) or return address (x86, in gostartcall). - if siz >= _StackMin-4*sys.RegSize-sys.RegSize { + // 4*PtrSize: extra space added below + // PtrSize: caller's LR (arm) or return address (x86, in gostartcall). + if siz >= _StackMin-4*sys.PtrSize-sys.PtrSize { throw("newproc: function arguments too large for new goroutine") } @@ -4030,8 +4030,8 @@ func newproc1(fn *funcval, argp unsafe.Pointer, narg int32, callergp *g, callerp throw("newproc1: new g is not Gdead") } - totalSize := 4*sys.RegSize + uintptr(siz) + sys.MinFrameSize // extra space in case of reads slightly beyond frame - totalSize += -totalSize & (sys.SpAlign - 1) // align to spAlign + totalSize := 4*sys.PtrSize + uintptr(siz) + sys.MinFrameSize // extra space in case of reads slightly beyond frame + totalSize += -totalSize & (sys.StackAlign - 1) // align to StackAlign sp := newg.stack.hi - totalSize spArg := sp if usesLR { diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index b7c7b4cff2..675c613b6e 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -327,7 +327,7 @@ type gobuf struct { pc uintptr g guintptr ctxt unsafe.Pointer - ret sys.Uintreg + ret uintptr lr uintptr bp uintptr // for framepointer-enabled architectures } diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go index e5d2d97d05..4572a25195 100644 --- a/src/runtime/runtime_test.go +++ b/src/runtime/runtime_test.go @@ -266,8 +266,8 @@ func TestTrailingZero(t *testing.T) { n int64 z struct{} } - if unsafe.Sizeof(T2{}) != 8+unsafe.Sizeof(Uintreg(0)) { - t.Errorf("sizeof(%#v)==%d, want %d", T2{}, unsafe.Sizeof(T2{}), 8+unsafe.Sizeof(Uintreg(0))) + if unsafe.Sizeof(T2{}) != 8+unsafe.Sizeof(uintptr(0)) { + t.Errorf("sizeof(%#v)==%d, want %d", T2{}, unsafe.Sizeof(T2{}), 8+unsafe.Sizeof(uintptr(0))) } type T3 struct { n byte diff --git a/src/runtime/signal_arm64.go b/src/runtime/signal_arm64.go index 3c20139c99..b559b93938 100644 --- a/src/runtime/signal_arm64.go +++ b/src/runtime/signal_arm64.go @@ -63,7 +63,7 @@ func (c *sigctxt) preparePanic(sig uint32, gp *g) { // functions are correctly handled. This smashes // the stack frame but we're not going back there // anyway. - sp := c.sp() - sys.SpAlign // needs only sizeof uint64, but must align the stack + sp := c.sp() - sys.StackAlign // needs only sizeof uint64, but must align the stack c.set_sp(sp) *(*uint64)(unsafe.Pointer(uintptr(sp))) = c.lr() diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 7b9dce5393..8c90e7b46f 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -651,7 +651,7 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { // Adjust saved base pointer if there is one. // TODO what about arm64 frame pointer adjustment? - if sys.ArchFamily == sys.AMD64 && frame.argp-frame.varp == 2*sys.RegSize { + if sys.ArchFamily == sys.AMD64 && frame.argp-frame.varp == 2*sys.PtrSize { if stackDebug >= 3 { print(" saved bp\n") } @@ -1245,7 +1245,7 @@ func getStackMap(frame *stkframe, cache *pcvalueCache, debug bool) (locals, args var minsize uintptr switch sys.ArchFamily { case sys.ARM64: - minsize = sys.SpAlign + minsize = sys.StackAlign default: minsize = sys.MinFrameSize } diff --git a/src/runtime/sys_wasm.go b/src/runtime/sys_wasm.go index 9bf710ba0e..3ed621f92e 100644 --- a/src/runtime/sys_wasm.go +++ b/src/runtime/sys_wasm.go @@ -30,10 +30,6 @@ func wasmExit(code int32) // and then did an immediate gosave. func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { sp := buf.sp - if sys.RegSize > sys.PtrSize { - sp -= sys.PtrSize - *(*uintptr)(unsafe.Pointer(sp)) = 0 - } sp -= sys.PtrSize *(*uintptr)(unsafe.Pointer(sp)) = buf.pc buf.sp = sp diff --git a/src/runtime/sys_x86.go b/src/runtime/sys_x86.go index f917cb8bd7..5b7a666679 100644 --- a/src/runtime/sys_x86.go +++ b/src/runtime/sys_x86.go @@ -15,10 +15,6 @@ import ( // and then did an immediate gosave. func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { sp := buf.sp - if sys.RegSize > sys.PtrSize { - sp -= sys.PtrSize - *(*uintptr)(unsafe.Pointer(sp)) = 0 - } sp -= sys.PtrSize *(*uintptr)(unsafe.Pointer(sp)) = buf.pc buf.sp = sp diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 2601cd697f..127f54e42e 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -144,8 +144,8 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in frame.pc = *(*uintptr)(unsafe.Pointer(frame.sp)) frame.lr = 0 } else { - frame.pc = uintptr(*(*sys.Uintreg)(unsafe.Pointer(frame.sp))) - frame.sp += sys.RegSize + frame.pc = uintptr(*(*uintptr)(unsafe.Pointer(frame.sp))) + frame.sp += sys.PtrSize } } @@ -208,7 +208,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in frame.fp = frame.sp + uintptr(funcspdelta(f, frame.pc, &cache)) if !usesLR { // On x86, call instruction pushes return PC before entering new function. - frame.fp += sys.RegSize + frame.fp += sys.PtrSize } } var flr funcInfo @@ -235,8 +235,8 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } } else { if frame.lr == 0 { - lrPtr = frame.fp - sys.RegSize - frame.lr = uintptr(*(*sys.Uintreg)(unsafe.Pointer(lrPtr))) + lrPtr = frame.fp - sys.PtrSize + frame.lr = uintptr(*(*uintptr)(unsafe.Pointer(lrPtr))) } } flr = findfunc(frame.lr) @@ -266,13 +266,13 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in frame.varp = frame.fp if !usesLR { // On x86, call instruction pushes return PC before entering new function. - frame.varp -= sys.RegSize + frame.varp -= sys.PtrSize } // For architectures with frame pointers, if there's // a frame, then there's a saved frame pointer here. if frame.varp > frame.sp && (GOARCH == "amd64" || GOARCH == "arm64") { - frame.varp -= sys.RegSize + frame.varp -= sys.PtrSize } // Derive size of arguments. @@ -490,11 +490,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // before faking a call. if usesLR && injectedCall { x := *(*uintptr)(unsafe.Pointer(frame.sp)) - frame.sp += sys.MinFrameSize - if GOARCH == "arm64" { - // arm64 needs 16-byte aligned SP, always - frame.sp += sys.PtrSize - } + frame.sp += alignUp(sys.MinFrameSize, sys.StackAlign) f = findfunc(frame.pc) frame.fn = f if !f.valid() { -- cgit v1.3-5-g9baa From a78879ac67d62c4919492fcb5e05c8b21058217d Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 10:47:38 -0500 Subject: runtime: move sys.DefaultGoroot to runtime.defaultGOROOT The default GOROOT has nothing to do with system details. Move it next to its one use in package runtime. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I1a601fad6335336b4616b834bb21bd8437ee1313 Reviewed-on: https://go-review.googlesource.com/c/go/+/288796 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/cmd/link/internal/ld/main.go | 2 +- src/runtime/extern.go | 4 +++- src/runtime/internal/sys/arch.go | 4 ---- 3 files changed, 4 insertions(+), 6 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go index cbd811846b..68dee18598 100644 --- a/src/cmd/link/internal/ld/main.go +++ b/src/cmd/link/internal/ld/main.go @@ -116,7 +116,7 @@ func Main(arch *sys.Arch, theArch Arch) { } final := gorootFinal() - addstrdata1(ctxt, "runtime/internal/sys.DefaultGoroot="+final) + addstrdata1(ctxt, "runtime.defaultGOROOT="+final) addstrdata1(ctxt, "cmd/internal/objabi.defaultGOROOT="+final) // TODO(matloob): define these above and then check flag values here diff --git a/src/runtime/extern.go b/src/runtime/extern.go index dacdf4f383..bbe41dd0d4 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -229,6 +229,8 @@ func Callers(skip int, pc []uintptr) int { return callers(skip, pc) } +var defaultGOROOT string // set by cmd/link + // GOROOT returns the root of the Go tree. It uses the // GOROOT environment variable, if set at process start, // or else the root used during the Go build. @@ -237,7 +239,7 @@ func GOROOT() string { if s != "" { return s } - return sys.DefaultGoroot + return defaultGOROOT } // Version returns the Go tree's version string. diff --git a/src/runtime/internal/sys/arch.go b/src/runtime/internal/sys/arch.go index 69278bf2d5..3c99a2f7da 100644 --- a/src/runtime/internal/sys/arch.go +++ b/src/runtime/internal/sys/arch.go @@ -52,7 +52,3 @@ const MinFrameSize = _MinFrameSize // StackAlign is the required alignment of the SP register. // The stack must be at least word aligned, but some architectures require more. const StackAlign = _StackAlign - -// DefaultGoroot is set by the linker for use by package runtime. -// It doesn't really belong in this file or this package. -var DefaultGoroot string -- cgit v1.3-5-g9baa From c80da0a33a240469892a0b0713f09607efb28752 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:00:21 -0500 Subject: runtime: handle nil gp in cpuprof This can happen on Windows when recording profile samples for system threads. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I5a7ba32b1900a69f3b7acada9cb6cf8396d8a03f Reviewed-on: https://go-review.googlesource.com/c/go/+/288797 Trust: Russ Cox Reviewed-by: Cherry Zhang Reviewed-by: Ian Lance Taylor --- src/runtime/cpuprof.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/cpuprof.go b/src/runtime/cpuprof.go index 9bfdfe7c74..e5d0193b9c 100644 --- a/src/runtime/cpuprof.go +++ b/src/runtime/cpuprof.go @@ -103,7 +103,16 @@ func (p *cpuProfile) add(gp *g, stk []uintptr) { // because otherwise its write barrier behavior may not // be correct. See the long comment there before // changing the argument here. - cpuprof.log.write(&gp.labels, nanotime(), hdr[:], stk) + // + // Note: it can happen on Windows, where we are calling + // p.add with a gp that is not the current g, that gp is nil, + // meaning we interrupted a system thread with no g. + // Avoid faulting in that case. + var tagPtr *unsafe.Pointer + if gp != nil { + tagPtr = &gp.labels + } + cpuprof.log.write(tagPtr, nanotime(), hdr[:], stk) } atomic.Store(&prof.signalLock, 0) -- cgit v1.3-5-g9baa From 229695a2833ead7bbee53071f52f34e2ce1c2802 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 28 Jan 2021 09:32:55 -0500 Subject: runtime: clean up funcID assignment Large enum sets should be sorted by name when the values don't matter, as they don't here. Do that. Also replace the large switch with a map lookup. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: Ibe727b5d8866bf4c40c96020e1f4632bde7efd59 Reviewed-on: https://go-review.googlesource.com/c/go/+/288798 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld Reviewed-by: Ian Lance Taylor --- src/cmd/internal/objabi/funcid.go | 113 ++++++++++++++++---------------------- src/runtime/symtab.go | 26 ++++----- 2 files changed, 61 insertions(+), 78 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/internal/objabi/funcid.go b/src/cmd/internal/objabi/funcid.go index 1d098ee172..e921a82c0c 100644 --- a/src/cmd/internal/objabi/funcid.go +++ b/src/cmd/internal/objabi/funcid.go @@ -4,6 +4,8 @@ package objabi +import "strings" + // A FuncID identifies particular functions that need to be treated // specially by the runtime. // Note that in some situations involving plugins, there may be multiple @@ -13,88 +15,69 @@ type FuncID uint8 const ( FuncID_normal FuncID = iota // not a special function - FuncID_runtime_main + FuncID_asmcgocall + FuncID_asyncPreempt + FuncID_cgocallback + FuncID_debugCallV1 + FuncID_externalthreadhandler + FuncID_gcBgMarkWorker FuncID_goexit + FuncID_gogo + FuncID_gopanic + FuncID_handleAsyncEvent FuncID_jmpdefer FuncID_mcall FuncID_morestack FuncID_mstart + FuncID_panicwrap FuncID_rt0_go - FuncID_asmcgocall - FuncID_sigpanic FuncID_runfinq - FuncID_gcBgMarkWorker - FuncID_systemstack_switch + FuncID_runtime_main + FuncID_sigpanic FuncID_systemstack - FuncID_cgocallback - FuncID_gogo - FuncID_externalthreadhandler - FuncID_debugCallV1 - FuncID_gopanic - FuncID_panicwrap - FuncID_handleAsyncEvent - FuncID_asyncPreempt + FuncID_systemstack_switch FuncID_wrapper // any autogenerated code (hash/eq algorithms, method wrappers, etc.) ) +var funcIDs = map[string]FuncID{ + "asmcgocall": FuncID_asmcgocall, + "asyncPreempt": FuncID_asyncPreempt, + "cgocallback": FuncID_cgocallback, + "debugCallV1": FuncID_debugCallV1, + "externalthreadhandler": FuncID_externalthreadhandler, + "gcBgMarkWorker": FuncID_gcBgMarkWorker, + "go": FuncID_rt0_go, + "goexit": FuncID_goexit, + "gogo": FuncID_gogo, + "gopanic": FuncID_gopanic, + "handleAsyncEvent": FuncID_handleAsyncEvent, + "jmpdefer": FuncID_jmpdefer, + "main": FuncID_runtime_main, + "mcall": FuncID_mcall, + "morestack": FuncID_morestack, + "mstart": FuncID_mstart, + "panicwrap": FuncID_panicwrap, + "runfinq": FuncID_runfinq, + "sigpanic": FuncID_sigpanic, + "switch": FuncID_systemstack_switch, + "systemstack": FuncID_systemstack, + + // Don't show in call stack but otherwise not special. + "deferreturn": FuncID_wrapper, + "runOpenDeferFrame": FuncID_wrapper, + "reflectcallSave": FuncID_wrapper, +} + // Get the function ID for the named function in the named file. // The function should be package-qualified. func GetFuncID(name string, isWrapper bool) FuncID { if isWrapper { return FuncID_wrapper } - switch name { - case "runtime.main": - return FuncID_runtime_main - case "runtime.goexit": - return FuncID_goexit - case "runtime.jmpdefer": - return FuncID_jmpdefer - case "runtime.mcall": - return FuncID_mcall - case "runtime.morestack": - return FuncID_morestack - case "runtime.mstart": - return FuncID_mstart - case "runtime.rt0_go": - return FuncID_rt0_go - case "runtime.asmcgocall": - return FuncID_asmcgocall - case "runtime.sigpanic": - return FuncID_sigpanic - case "runtime.runfinq": - return FuncID_runfinq - case "runtime.gcBgMarkWorker": - return FuncID_gcBgMarkWorker - case "runtime.systemstack_switch": - return FuncID_systemstack_switch - case "runtime.systemstack": - return FuncID_systemstack - case "runtime.cgocallback": - return FuncID_cgocallback - case "runtime.gogo": - return FuncID_gogo - case "runtime.externalthreadhandler": - return FuncID_externalthreadhandler - case "runtime.debugCallV1": - return FuncID_debugCallV1 - case "runtime.gopanic": - return FuncID_gopanic - case "runtime.panicwrap": - return FuncID_panicwrap - case "runtime.handleAsyncEvent": - return FuncID_handleAsyncEvent - case "runtime.asyncPreempt": - return FuncID_asyncPreempt - case "runtime.deferreturn": - // Don't show in the call stack (used when invoking defer functions) - return FuncID_wrapper - case "runtime.runOpenDeferFrame": - // Don't show in the call stack (used when invoking defer functions) - return FuncID_wrapper - case "runtime.reflectcallSave": - // Don't show in the call stack (used when invoking defer functions) - return FuncID_wrapper + if strings.HasPrefix(name, "runtime.") { + if id, ok := funcIDs[name[len("runtime."):]]; ok { + return id + } } return FuncID_normal } diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 7667f23f1d..fc93c00c2d 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -308,27 +308,27 @@ type funcID uint8 const ( funcID_normal funcID = iota // not a special function - funcID_runtime_main + funcID_asmcgocall + funcID_asyncPreempt + funcID_cgocallback + funcID_debugCallV1 + funcID_externalthreadhandler + funcID_gcBgMarkWorker funcID_goexit + funcID_gogo + funcID_gopanic + funcID_handleAsyncEvent funcID_jmpdefer funcID_mcall funcID_morestack funcID_mstart + funcID_panicwrap funcID_rt0_go - funcID_asmcgocall - funcID_sigpanic funcID_runfinq - funcID_gcBgMarkWorker - funcID_systemstack_switch + funcID_runtime_main + funcID_sigpanic funcID_systemstack - funcID_cgocallback - funcID_gogo - funcID_externalthreadhandler - funcID_debugCallV1 - funcID_gopanic - funcID_panicwrap - funcID_handleAsyncEvent - funcID_asyncPreempt + funcID_systemstack_switch funcID_wrapper // any autogenerated code (hash/eq algorithms, method wrappers, etc.) ) -- cgit v1.3-5-g9baa From 01f05d8ff1a88c4a63bdaaff5a095a92ce476f58 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 1 Feb 2021 22:58:28 -0500 Subject: runtime: unify asmcgocall and systemstack traceback setup Both asmcgocall and systemstack need to save the calling Go code's context for use by traceback, but they do it differently. Systemstack's appraoch is better, because it doesn't require a special case in traceback. So make them both use that. While we are here, the fake mstart caller in systemstack is no longer needed and can be removed. (traceback knows to stop in systemstack because of the writes to SP.) Also remove the fake mstarts in sys_windows_*.s. And while we are there, fix the control flow guard code in sys_windows_arm.s. The current code is using pointers to a stack frame that technically is gone once we hit the RET instruction. Clearly it's working OK, but better not to depend on data below SP being preserved, even for just a few instructions. Store the value we need in other registers instead. (This code is only used for pushing a sigpanic call, which does not actually return to the site of the fault and therefore doesn't need to preserve any of the registers.) This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: Id1e3ef5e54f7ad786e4b87043f2626eba7c3bbd9 Reviewed-on: https://go-review.googlesource.com/c/go/+/288799 Trust: Russ Cox Reviewed-by: Cherry Zhang --- misc/cgo/test/callback.go | 2 +- src/runtime/asm_386.s | 19 +++++++--------- src/runtime/asm_amd64.s | 22 ++++++++---------- src/runtime/asm_arm.s | 34 +++++++++++++--------------- src/runtime/asm_arm64.s | 31 +++++++++++-------------- src/runtime/asm_mips64x.s | 26 ++++++++++----------- src/runtime/asm_mipsx.s | 26 ++++++++++----------- src/runtime/asm_ppc64x.s | 28 ++++++++++------------- src/runtime/asm_riscv64.s | 26 ++++++++++----------- src/runtime/asm_s390x.s | 29 +++++++++++------------- src/runtime/sys_windows_386.s | 12 ++++------ src/runtime/sys_windows_amd64.s | 12 +++------- src/runtime/sys_windows_arm.s | 50 +++++++++++++++++++---------------------- 13 files changed, 137 insertions(+), 180 deletions(-) (limited to 'src/runtime') diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index 814888e3ac..08dd9b39d8 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -182,7 +182,7 @@ func testCallbackCallers(t *testing.T) { "runtime.cgocallbackg1", "runtime.cgocallbackg", "runtime.cgocallback", - "runtime.asmcgocall", + "runtime.systemstack_switch", "runtime.cgocall", "test._Cfunc_callback", "test.nestedCall.func1", diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 3030101f03..a59054226c 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -352,18 +352,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-4 // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVL $runtime·systemstack_switch(SB), (g_sched+gobuf_pc)(AX) - MOVL SP, (g_sched+gobuf_sp)(AX) - MOVL AX, (g_sched+gobuf_g)(AX) + CALL gosave_systemstack_switch<>(SB) // switch to g0 get_tls(CX) MOVL DX, g(CX) MOVL (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called systemstack on g0, to stop traceback - SUBL $4, BX - MOVL $runtime·mstart(SB), DX - MOVL DX, 0(BX) MOVL BX, SP // call target function @@ -601,15 +595,18 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-8 MOVL 0(DX), BX JMP BX // but first run the deferred function -// Save state of caller into g->sched. -TEXT gosave<>(SB),NOSPLIT,$0 +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 PUSHL AX PUSHL BX get_tls(BX) MOVL g(BX), BX LEAL arg+0(FP), AX MOVL AX, (g_sched+gobuf_sp)(BX) - MOVL -4(AX), AX + MOVL $runtime·systemstack_switch(SB), AX MOVL AX, (g_sched+gobuf_pc)(BX) MOVL $0, (g_sched+gobuf_ret)(BX) // Assert ctxt is zero. See func save. @@ -661,7 +658,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12 JEQ noswitch CMPL DI, m_gsignal(BP) JEQ noswitch - CALL gosave<>(SB) + CALL gosave_systemstack_switch<>(SB) get_tls(CX) MOVL SI, g(CX) MOVL (g_sched+gobuf_sp)(SI), SP diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 9362ce1213..cd5ce3effb 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -338,20 +338,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 // switch stacks // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVQ $runtime·systemstack_switch(SB), SI - MOVQ SI, (g_sched+gobuf_pc)(AX) - MOVQ SP, (g_sched+gobuf_sp)(AX) - MOVQ AX, (g_sched+gobuf_g)(AX) - MOVQ BP, (g_sched+gobuf_bp)(AX) + CALL gosave_systemstack_switch<>(SB) // switch to g0 MOVQ DX, g(CX) MOVQ DX, R14 // set the g register MOVQ (g_sched+gobuf_sp)(DX), BX - // make it look like mstart called systemstack on g0, to stop traceback - SUBQ $8, BX - MOVQ $runtime·mstart(SB), DX - MOVQ DX, 0(BX) MOVQ BX, SP // call target function @@ -660,13 +652,17 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16 MOVQ 0(DX), BX JMP BX // but first run the deferred function -// Save state of caller into g->sched. Smashes R9. -TEXT gosave<>(SB),NOSPLIT,$0 +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R9. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 #ifndef GOEXPERIMENT_REGABI get_tls(R14) MOVQ g(R14), R14 #endif - MOVQ 0(SP), R9 + MOVQ $runtime·systemstack_switch(SB), R9 MOVQ R9, (g_sched+gobuf_pc)(R14) LEAQ 8(SP), R9 MOVQ R9, (g_sched+gobuf_sp)(R14) @@ -724,7 +720,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 // Switch to system stack. MOVQ m_g0(R8), SI - CALL gosave<>(SB) + CALL gosave_systemstack_switch<>(SB) MOVQ SI, g(CX) MOVQ (g_sched+gobuf_sp)(SI), SP diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 109030aada..c8c53e70db 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -305,24 +305,14 @@ TEXT runtime·systemstack(SB),NOSPLIT,$0-4 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVW $runtime·systemstack_switch(SB), R3 - ADD $4, R3, R3 // get past push {lr} - MOVW R3, (g_sched+gobuf_pc)(g) - MOVW R13, (g_sched+gobuf_sp)(g) - MOVW LR, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVW R0, R5 MOVW R2, R0 BL setg<>(SB) MOVW R5, R0 - MOVW (g_sched+gobuf_sp)(R2), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $4, R3, R3 - MOVW $runtime·mstart(SB), R4 - MOVW R4, 0(R3) - MOVW R3, R13 + MOVW (g_sched+gobuf_sp)(R2), R13 // call target function MOVW R0, R7 @@ -537,19 +527,25 @@ TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 MOVW 0(R7), R1 B (R1) -// Save state of caller into g->sched. Smashes R11. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVW LR, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R11. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·systemstack_switch(SB), R11 + ADD $4, R11 // get past push {lr} + MOVW R11, (g_sched+gobuf_pc)(g) MOVW R13, (g_sched+gobuf_sp)(g) + MOVW g, (g_sched+gobuf_g)(g) MOVW $0, R11 MOVW R11, (g_sched+gobuf_lr)(g) MOVW R11, (g_sched+gobuf_ret)(g) - MOVW R11, (g_sched+gobuf_ctxt)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R11 - CMP $0, R11 + TST R11, R11 B.EQ 2(PC) - CALL runtime·badctxt(SB) + BL runtime·badctxt(SB) RET // func asmcgocall_no_g(fn, arg unsafe.Pointer) @@ -590,7 +586,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12 MOVW m_g0(R8), R3 CMP R3, g BEQ nosave - BL gosave<>(SB) + BL gosave_systemstack_switch<>(SB) MOVW R0, R5 MOVW R3, R0 BL setg<>(SB) diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 79efd4cb17..31a6fe57b9 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -205,24 +205,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVD $runtime·systemstack_switch(SB), R6 - ADD $8, R6 // get past prologue - MOVD R6, (g_sched+gobuf_pc)(g) - MOVD RSP, R0 - MOVD R0, (g_sched+gobuf_sp)(g) - MOVD R29, (g_sched+gobuf_bp)(g) - MOVD $0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVD R5, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $16, R3 - AND $~15, R3 - MOVD $runtime·mstart(SB), R4 - MOVD R4, 0(R3) MOVD R3, RSP MOVD (g_sched+gobuf_bp)(g), R29 @@ -859,14 +847,21 @@ TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 MOVD 0(R26), R3 B (R3) -// Save state of caller into g->sched. Smashes R0. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVD LR, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R0. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVD $runtime·systemstack_switch(SB), R0 + ADD $8, R0 // get past prologue + MOVD R0, (g_sched+gobuf_pc)(g) MOVD RSP, R0 MOVD R0, (g_sched+gobuf_sp)(g) MOVD R29, (g_sched+gobuf_bp)(g) MOVD $0, (g_sched+gobuf_lr)(g) MOVD $0, (g_sched+gobuf_ret)(g) + MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R0 CBZ R0, 2(PC) @@ -908,8 +903,8 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 BEQ nosave // Switch to system stack. - MOVD R0, R9 // gosave<> and save_g might clobber R0 - BL gosave<>(SB) + MOVD R0, R9 // gosave_systemstack_switch<> and save_g might clobber R0 + BL gosave_systemstack_switch<>(SB) MOVD R3, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R0 diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 6e1d25cd90..75bb223066 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -169,21 +169,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVV $runtime·systemstack_switch(SB), R4 - ADDV $8, R4 // get past prologue - MOVV R4, (g_sched+gobuf_pc)(g) - MOVV R29, (g_sched+gobuf_sp)(g) - MOVV R0, (g_sched+gobuf_lr)(g) - MOVV g, (g_sched+gobuf_g)(g) + JAL gosave_systemstack_switch<>(SB) // switch to g0 MOVV R3, g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R1 - // make it look like mstart called systemstack on g0, to stop traceback - ADDV $-8, R1 - MOVV $runtime·mstart(SB), R2 - MOVV R2, 0(R1) MOVV R1, R29 // call target function @@ -401,12 +392,19 @@ TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 MOVV 0(REGCTXT), R4 JMP (R4) -// Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVV R31, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R1. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVV $runtime·systemstack_switch(SB), R1 + ADDV $8, R1 // get past prologue + MOVV R1, (g_sched+gobuf_pc)(g) MOVV R29, (g_sched+gobuf_sp)(g) MOVV R0, (g_sched+gobuf_lr)(g) MOVV R0, (g_sched+gobuf_ret)(g) + MOVV g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVV (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) @@ -440,7 +438,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVV m_g0(R5), R6 BEQ R6, g, g0 - JAL gosave<>(SB) + JAL gosave_systemstack_switch<>(SB) MOVV R6, g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R29 diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index 8e5753d255..341da8e8d7 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -170,21 +170,12 @@ TEXT runtime·systemstack(SB),NOSPLIT,$0-4 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVW $runtime·systemstack_switch(SB), R4 - ADDU $8, R4 // get past prologue - MOVW R4, (g_sched+gobuf_pc)(g) - MOVW R29, (g_sched+gobuf_sp)(g) - MOVW R0, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) + JAL gosave_systemstack_switch<>(SB) // switch to g0 MOVW R3, g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R1 - // make it look like mstart called systemstack on g0, to stop traceback - ADDU $-4, R1 - MOVW $runtime·mstart(SB), R2 - MOVW R2, 0(R1) MOVW R1, R29 // call target function @@ -401,12 +392,19 @@ TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 MOVW 0(REGCTXT), R4 JMP (R4) -// Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVW R31, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R1. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·systemstack_switch(SB), R1 + ADDU $8, R1 // get past prologue + MOVW R1, (g_sched+gobuf_pc)(g) MOVW R29, (g_sched+gobuf_sp)(g) MOVW R0, (g_sched+gobuf_lr)(g) MOVW R0, (g_sched+gobuf_ret)(g) + MOVW g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) @@ -431,7 +429,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-12 MOVW m_g0(R5), R6 BEQ R6, g, g0 - JAL gosave<>(SB) + JAL gosave_systemstack_switch<>(SB) MOVW R6, g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R29 diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 834023cce1..a99a61fd88 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -229,22 +229,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVD $runtime·systemstack_switch(SB), R6 - ADD $16, R6 // get past prologue (including r2-setting instructions when they're there) - MOVD R6, (g_sched+gobuf_pc)(g) - MOVD R1, (g_sched+gobuf_sp)(g) - MOVD R0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVD R5, g BL runtime·save_g(SB) - MOVD (g_sched+gobuf_sp)(g), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $FIXED_FRAME, R3 - MOVD $runtime·mstart(SB), R4 - MOVD R4, 0(R3) - MOVD R3, R1 + MOVD (g_sched+gobuf_sp)(g), R1 // call target function MOVD 0(R11), R12 // code pointer @@ -534,13 +524,19 @@ TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16 MOVD R12, CTR BR (CTR) -// Save state of caller into g->sched. Smashes R31. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVD LR, R31 +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R31. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVD $runtime·systemstack_switch(SB), R31 + ADD $16, R31 // get past prologue (including r2-setting instructions when they're there) MOVD R31, (g_sched+gobuf_pc)(g) MOVD R1, (g_sched+gobuf_sp)(g) MOVD R0, (g_sched+gobuf_lr)(g) MOVD R0, (g_sched+gobuf_ret)(g) + MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R31 CMP R0, R31 @@ -577,7 +573,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVD m_g0(R8), R6 CMP R6, g BEQ g0 - BL gosave<>(SB) + BL gosave_systemstack_switch<>(SB) MOVD R6, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R1 diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 3d0349471a..fb7c6530dc 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -114,21 +114,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOV $runtime·systemstack_switch(SB), T2 - ADD $8, T2 // get past prologue - MOV T2, (g_sched+gobuf_pc)(g) - MOV X2, (g_sched+gobuf_sp)(g) - MOV ZERO, (g_sched+gobuf_lr)(g) - MOV g, (g_sched+gobuf_g)(g) + CALL gosave_systemstack_switch<>(SB) // switch to g0 MOV T1, g CALL runtime·save_g(SB) MOV (g_sched+gobuf_sp)(g), T0 - // make it look like mstart called systemstack on g0, to stop traceback - ADD $-8, T0 - MOV $runtime·mstart(SB), T1 - MOV T1, 0(T0) MOV T0, X2 // call target function @@ -297,12 +288,19 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 JALR RA, T1 JMP runtime·badmcall2(SB) -// Save state of caller into g->sched. Smashes X31. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOV X1, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes X31. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOV $runtime·systemstack_switch(SB), X31 + ADD $8, X31 // get past prologue + MOV X31, (g_sched+gobuf_pc)(g) MOV X2, (g_sched+gobuf_sp)(g) MOV ZERO, (g_sched+gobuf_lr)(g) MOV ZERO, (g_sched+gobuf_ret)(g) + MOV g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOV (g_sched+gobuf_ctxt)(g), X31 BEQ ZERO, X31, 2(PC) @@ -327,7 +325,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOV m_g0(X6), X7 BEQ X7, g, g0 - CALL gosave<>(SB) + CALL gosave_systemstack_switch<>(SB) MOV X7, g CALL runtime·save_g(SB) MOV (g_sched+gobuf_sp)(g), X2 diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index fbd185c353..43244d961f 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -256,22 +256,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 switch: // save our state in g->sched. Pretend to // be systemstack_switch if the G stack is scanned. - MOVD $runtime·systemstack_switch(SB), R6 - ADD $16, R6 // get past prologue - MOVD R6, (g_sched+gobuf_pc)(g) - MOVD R15, (g_sched+gobuf_sp)(g) - MOVD $0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) + BL gosave_systemstack_switch<>(SB) // switch to g0 MOVD R5, g BL runtime·save_g(SB) - MOVD (g_sched+gobuf_sp)(g), R3 - // make it look like mstart called systemstack on g0, to stop traceback - SUB $8, R3 - MOVD $runtime·mstart(SB), R4 - MOVD R4, 0(R3) - MOVD R3, R15 + MOVD (g_sched+gobuf_sp)(g), R15 // call target function MOVD 0(R12), R3 // code pointer @@ -498,12 +488,19 @@ TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16 MOVD 0(R12), R3 BR (R3) -// Save state of caller into g->sched. Smashes R1. -TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0 - MOVD LR, (g_sched+gobuf_pc)(g) +// Save state of caller into g->sched, +// but using fake PC from systemstack_switch. +// Must only be called from functions with no locals ($0) +// or else unwinding from systemstack_switch is incorrect. +// Smashes R1. +TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 + MOVD $runtime·systemstack_switch(SB), R1 + ADD $16, R1 // get past prologue + MOVD R1, (g_sched+gobuf_pc)(g) MOVD R15, (g_sched+gobuf_sp)(g) MOVD $0, (g_sched+gobuf_lr)(g) MOVD $0, (g_sched+gobuf_ret)(g) + MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R1 CMPBEQ R1, $0, 2(PC) @@ -529,7 +526,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVD g_m(g), R6 MOVD m_g0(R6), R6 CMPBEQ R6, g, g0 - BL gosave<>(SB) + BL gosave_systemstack_switch<>(SB) MOVD R6, g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R15 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index c556e3a3c2..576dd28771 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -94,7 +94,7 @@ TEXT sigtramp<>(SB),NOSPLIT,$0-0 JNE 2(PC) CALL runtime·badsignal2(SB) - // save g and SP in case of stack switch + // save g in case of stack switch MOVL DX, 32(SP) // g MOVL SP, 36(SP) @@ -108,13 +108,9 @@ TEXT sigtramp<>(SB),NOSPLIT,$0-0 get_tls(BP) MOVL BX, g(BP) MOVL (g_sched+gobuf_sp)(BX), DI - // make it look like mstart called us on g0, to stop traceback - SUBL $4, DI - MOVL $runtime·mstart(SB), 0(DI) - // traceback will think that we've done SUBL - // on this stack, so subtract them here to match. - // (we need room for sighandler arguments anyway). + // make room for sighandler arguments // and re-save old SP for restoring later. + // (note that the 36(DI) here must match the 36(SP) above.) SUBL $40, DI MOVL SP, 36(DI) MOVL DI, SP @@ -132,7 +128,7 @@ g0: // switch back to original stack and g // no-op if we never left. MOVL 36(SP), SP - MOVL 32(SP), DX + MOVL 32(SP), DX // note: different SP get_tls(BP) MOVL DX, g(BP) diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 9cd14016b0..6f2abb5444 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -151,16 +151,10 @@ TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0-0 get_tls(BP) MOVQ BX, g(BP) MOVQ (g_sched+gobuf_sp)(BX), DI - // make it look like mstart called us on g0, to stop traceback - SUBQ $8, DI - MOVQ $runtime·mstart(SB), SI - MOVQ SI, 0(DI) - // traceback will think that we've done PUSHFQ and SUBQ - // on this stack, so subtract them here to match. - // (we need room for sighandler arguments anyway). + // make room for sighandler arguments // and re-save old SP for restoring later. - SUBQ $(112+8), DI - // save g, save old stack pointer. + // (note that the 104(DI) here must match the 104(SP) above.) + SUBQ $120, DI MOVQ SP, 104(DI) MOVQ DI, SP diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index d2bdc50e3b..7d134dd3f4 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -6,6 +6,8 @@ #include "go_tls.h" #include "textflag.h" +// Note: For system ABI, R0-R3 are args, R4-R11 are callee-save. + // void runtime·asmstdcall(void *c); TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} @@ -139,11 +141,10 @@ TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0 MOVW (g_sched+gobuf_sp)(g), R3 // R3 = g->gobuf.sp BL runtime·save_g(SB) - // traceback will think that we've done PUSH and SUB - // on this stack, so subtract them here to match. - // (we need room for sighandler arguments anyway). + // make room for sighandler arguments // and re-save old SP for restoring later. - SUB $(40+8+20), R3 + // (note that the 24(R3) here must match the 24(R13) above.) + SUB $40, R3 MOVW R13, 24(R3) // save old stack pointer MOVW R3, R13 // switch stack @@ -151,22 +152,14 @@ g0: MOVW 0(R6), R2 // R2 = ExceptionPointers->ExceptionRecord MOVW 4(R6), R3 // R3 = ExceptionPointers->ContextRecord - // make it look like mstart called us on g0, to stop traceback - MOVW $runtime·mstart(SB), R4 - - MOVW R4, 0(R13) // Save link register for traceback + MOVW $0, R4 + MOVW R4, 0(R13) // No saved link register. MOVW R2, 4(R13) // Move arg0 (ExceptionRecord) into position MOVW R3, 8(R13) // Move arg1 (ContextRecord) into position MOVW R5, 12(R13) // Move arg2 (original g) into position BL (R7) // Call the go routine MOVW 16(R13), R4 // Fetch return value from stack - // Compute the value of the g0 stack pointer after deallocating - // this frame, then allocating 8 bytes. We may need to store - // the resume SP and PC on the g0 stack to work around - // control flow guard when we resume from the exception. - ADD $(40+20), R13, R12 - // switch back to original stack and g MOVW 24(R13), R13 MOVW 20(R13), g @@ -192,33 +185,36 @@ done: // If returntramp has already been set up by a previous exception // handler, don't clobber the stored SP and PC on the stack. MOVW 4(R3), R3 // PEXCEPTION_POINTERS->Context - MOVW 0x40(R3), R2 // load PC from context record + MOVW context_pc(R3), R2 // load PC from context record MOVW $returntramp<>(SB), R1 CMP R1, R2 B.EQ return // do not clobber saved SP/PC - // Save resume SP and PC on g0 stack - MOVW 0x38(R3), R2 // load SP from context record - MOVW R2, 0(R12) // Store resume SP on g0 stack - MOVW 0x40(R3), R2 // load PC from context record - MOVW R2, 4(R12) // Store resume PC on g0 stack + // Save resume SP and PC into R0, R1. + MOVW context_spr(R3), R2 + MOVW R2, context_r0(R3) + MOVW context_pc(R3), R2 + MOVW R2, context_r1(R3) // Set up context record to return to returntramp on g0 stack - MOVW R12, 0x38(R3) // save g0 stack pointer - // in context record - MOVW $returntramp<>(SB), R2 // save resume address - MOVW R2, 0x40(R3) // in context record + MOVW R12, context_spr(R3) + MOVW $returntramp<>(SB), R2 + MOVW R2, context_pc(R3) return: B (R14) // return -// // Trampoline to resume execution from exception handler. // This is part of the control flow guard workaround. // It switches stacks and jumps to the continuation address. -// +// R0 and R1 are set above at the end of sigtramp<> +// in the context that starts executing at returntramp<>. TEXT returntramp<>(SB),NOSPLIT|NOFRAME,$0 - MOVM.IA (R13), [R13, R15] // ldm sp, [sp, pc] + // Important: do not smash LR, + // which is set to a live value when handling + // a signal by pushing a call to sigpanic onto the stack. + MOVW R0, R13 + B (R1) TEXT runtime·exceptiontramp(SB),NOSPLIT|NOFRAME,$0 MOVW $runtime·exceptionhandler(SB), R1 -- cgit v1.3-5-g9baa From aa0388f2ed937669e9f938da8a65c75ea144ebfd Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 15 Feb 2021 09:25:55 -0500 Subject: runtime: remove unnecessary writes to gp.sched.g A g's sched.g is set in newproc1: newg.sched.g = guintptr(unsafe.Pointer(newg)) After that, it never changes. Yet lots of assembly code does "g.sched.g = g" unnecessarily. Remove all those lines to avoid confusion about whether it ever changes. Also, split gogo into two functions, one that does the nil g check and a second that does the actual switch. This way, if the nil g check fails, we get a stack trace showing the call stack that led to the failure. (The SP write would otherwise cause the stack trace to abort.) Also restore the proper nil g check in a handful of assembly functions. (There is little point in checking for nil g *after* installing it as the real g.) Change-Id: I22866b093f901f765de1d074e36eeec10366abfb Reviewed-on: https://go-review.googlesource.com/c/go/+/292109 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/asm_386.s | 7 ++++--- src/runtime/asm_amd64.s | 7 ++++--- src/runtime/asm_arm.s | 19 +++++-------------- src/runtime/asm_arm64.s | 12 +++++++----- src/runtime/asm_mips64x.s | 11 +++++++---- src/runtime/asm_mipsx.s | 13 +++++++------ src/runtime/asm_ppc64x.s | 12 +++++++----- src/runtime/asm_riscv64.s | 12 +++++++----- src/runtime/asm_s390x.s | 11 +++++++---- src/runtime/asm_wasm.s | 7 +++---- src/runtime/proc.go | 24 ++++++++++++++++++++---- 11 files changed, 78 insertions(+), 57 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index a59054226c..fcf74a03cf 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -275,10 +275,13 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB), NOSPLIT, $8-4 +TEXT runtime·gogo(SB), NOSPLIT, $0-4 MOVL buf+0(FP), BX // gobuf MOVL gobuf_g(BX), DX MOVL 0(DX), CX // make sure g != nil + JMP gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT, $0 get_tls(CX) MOVL DX, g(CX) MOVL gobuf_sp(BX), SP // restore SP @@ -303,7 +306,6 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-4 MOVL BX, (g_sched+gobuf_pc)(AX) LEAL fn+0(FP), BX // caller's SP MOVL BX, (g_sched+gobuf_sp)(AX) - MOVL AX, (g_sched+gobuf_g)(AX) // switch to m->g0 & its stack, call fn MOVL g(DX), BX @@ -432,7 +434,6 @@ TEXT runtime·morestack(SB),NOSPLIT,$0-0 // Set g->sched to context in f. MOVL 0(SP), AX // f's PC MOVL AX, (g_sched+gobuf_pc)(SI) - MOVL SI, (g_sched+gobuf_g)(SI) LEAL 4(SP), AX // f's SP MOVL AX, (g_sched+gobuf_sp)(SI) MOVL DX, (g_sched+gobuf_ctxt)(SI) diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index cd5ce3effb..8ee2ac2123 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -256,10 +256,13 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 // func gogo(buf *gobuf) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB), NOSPLIT, $16-8 +TEXT runtime·gogo(SB), NOSPLIT, $0-8 MOVQ buf+0(FP), BX // gobuf MOVQ gobuf_g(BX), DX MOVQ 0(DX), CX // make sure g != nil + JMP gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT, $0 get_tls(CX) MOVQ DX, g(CX) MOVQ DX, R14 // set the g register @@ -287,7 +290,6 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8 MOVQ BX, (g_sched+gobuf_pc)(AX) LEAQ fn+0(FP), BX // caller's SP MOVQ BX, (g_sched+gobuf_sp)(AX) - MOVQ AX, (g_sched+gobuf_g)(AX) MOVQ BP, (g_sched+gobuf_bp)(AX) // switch to m->g0 & its stack, call fn @@ -418,7 +420,6 @@ TEXT runtime·morestack(SB),NOSPLIT,$0-0 // Set g->sched to context in f. MOVQ 0(SP), AX // f's PC MOVQ AX, (g_sched+gobuf_pc)(SI) - MOVQ SI, (g_sched+gobuf_g)(SI) LEAQ 8(SP), AX // f's SP MOVQ AX, (g_sched+gobuf_sp)(SI) MOVQ BP, (g_sched+gobuf_bp)(SI) diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index c8c53e70db..92d7854306 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -208,21 +208,14 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB),NOSPLIT,$8-4 +TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4 MOVW buf+0(FP), R1 MOVW gobuf_g(R1), R0 - BL setg<>(SB) + MOVW 0(R0), R2 // make sure g != nil + B gogo<>(SB) - // NOTE: We updated g above, and we are about to update SP. - // Until LR and PC are also updated, the g/SP/LR/PC quadruple - // are out of sync and must not be used as the basis of a traceback. - // Sigprof skips the traceback when SP is not within g's bounds, - // and when the PC is inside this function, runtime.gogo. - // Since we are about to update SP, until we complete runtime.gogo - // we must not leave this function. In particular, no calls - // after this point: it must be straight-line code until the - // final B instruction. - // See large comment in sigprof for more details. +TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0 + BL setg<>(SB) MOVW gobuf_sp(R1), R13 // restore SP==R13 MOVW gobuf_lr(R1), LR MOVW gobuf_ret(R1), R0 @@ -246,7 +239,6 @@ TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4 MOVW LR, (g_sched+gobuf_pc)(g) MOVW $0, R11 MOVW R11, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOVW g, R1 @@ -537,7 +529,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 ADD $4, R11 // get past push {lr} MOVW R11, (g_sched+gobuf_pc)(g) MOVW R13, (g_sched+gobuf_sp)(g) - MOVW g, (g_sched+gobuf_g)(g) MOVW $0, R11 MOVW R11, (g_sched+gobuf_lr)(g) MOVW R11, (g_sched+gobuf_ret)(g) diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 31a6fe57b9..4f0a680fa4 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -115,12 +115,16 @@ TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB), NOSPLIT, $24-8 +TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 MOVD buf+0(FP), R5 - MOVD gobuf_g(R5), g + MOVD gobuf_g(R5), R6 + MOVD 0(R6), R4 // make sure g != nil + B gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 + MOVD R6, g BL runtime·save_g(SB) - MOVD 0(g), R4 // make sure g is not nil MOVD gobuf_sp(R5), R0 MOVD R0, RSP MOVD gobuf_bp(R5), R29 @@ -147,7 +151,6 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 MOVD R29, (g_sched+gobuf_bp)(g) MOVD LR, (g_sched+gobuf_pc)(g) MOVD $0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOVD g, R3 @@ -861,7 +864,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD R29, (g_sched+gobuf_bp)(g) MOVD $0, (g_sched+gobuf_lr)(g) MOVD $0, (g_sched+gobuf_ret)(g) - MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R0 CBZ R0, 2(PC) diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 75bb223066..f6d8931a15 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -91,9 +91,14 @@ TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB), NOSPLIT, $16-8 +TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 MOVV buf+0(FP), R3 - MOVV gobuf_g(R3), g // make sure g is not nil + MOVV gobuf_g(R3), R4 + MOVV 0(R4), R5 // make sure g != nil + JMP gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 + MOVV R5, g JAL runtime·save_g(SB) MOVV 0(g), R2 @@ -117,7 +122,6 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 MOVV R29, (g_sched+gobuf_sp)(g) MOVV R31, (g_sched+gobuf_pc)(g) MOVV R0, (g_sched+gobuf_lr)(g) - MOVV g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOVV g, R1 @@ -404,7 +408,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVV R29, (g_sched+gobuf_sp)(g) MOVV R0, (g_sched+gobuf_lr)(g) MOVV R0, (g_sched+gobuf_ret)(g) - MOVV g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVV (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index 341da8e8d7..cf4b1b42cc 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -92,12 +92,15 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB),NOSPLIT,$8-4 +TEXT runtime·gogo(SB),NOSPLIT|NOFRAME,$0-4 MOVW buf+0(FP), R3 - MOVW gobuf_g(R3), g // make sure g is not nil - JAL runtime·save_g(SB) + MOVW gobuf_g(R3), R4 + MOVW 0(R4), R5 // make sure g != nil + JMP gogo<>(SB) - MOVW 0(g), R2 +TEXT gogo<>(SB),NOSPLIT|NOFRAME,$0 + MOVW R4, g + JAL runtime·save_g(SB) MOVW gobuf_sp(R3), R29 MOVW gobuf_lr(R3), R31 MOVW gobuf_ret(R3), R1 @@ -118,7 +121,6 @@ TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4 MOVW R29, (g_sched+gobuf_sp)(g) MOVW R31, (g_sched+gobuf_pc)(g) MOVW R0, (g_sched+gobuf_lr)(g) - MOVW g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOVW g, R1 @@ -404,7 +406,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVW R29, (g_sched+gobuf_sp)(g) MOVW R0, (g_sched+gobuf_lr)(g) MOVW R0, (g_sched+gobuf_ret)(g) - MOVW g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index a99a61fd88..90f14d8e54 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -130,12 +130,16 @@ TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB), NOSPLIT, $16-8 +TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 MOVD buf+0(FP), R5 - MOVD gobuf_g(R5), g // make sure g is not nil + MOVD gobuf_g(R5), R6 + MOVD 0(R6), R4 // make sure g != nil + BR gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 + MOVD R6, g BL runtime·save_g(SB) - MOVD 0(g), R4 MOVD gobuf_sp(R5), R1 MOVD gobuf_lr(R5), R31 #ifndef GOOS_aix @@ -163,7 +167,6 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 MOVD LR, R31 MOVD R31, (g_sched+gobuf_pc)(g) MOVD R0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOVD g, R3 @@ -536,7 +539,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD R1, (g_sched+gobuf_sp)(g) MOVD R0, (g_sched+gobuf_lr)(g) MOVD R0, (g_sched+gobuf_ret)(g) - MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R31 CMP R0, R31 diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index fb7c6530dc..d06c77b948 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -224,12 +224,16 @@ TEXT runtime·return0(SB), NOSPLIT, $0 // restore state from Gobuf; longjmp // func gogo(buf *gobuf) -TEXT runtime·gogo(SB), NOSPLIT, $16-8 +TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 MOV buf+0(FP), T0 - MOV gobuf_g(T0), g // make sure g is not nil + MOV gobuf_g(T0), T1 + MOV 0(T1), ZERO // make sure g != nil + JMP gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 + MOV T1, g CALL runtime·save_g(SB) - MOV (g), ZERO // make sure g is not nil MOV gobuf_sp(T0), X2 MOV gobuf_lr(T0), RA MOV gobuf_ret(T0), A0 @@ -270,7 +274,6 @@ TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8 MOV X2, (g_sched+gobuf_sp)(g) MOV RA, (g_sched+gobuf_pc)(g) MOV ZERO, (g_sched+gobuf_lr)(g) - MOV g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOV g, T0 @@ -300,7 +303,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOV X2, (g_sched+gobuf_sp)(g) MOV ZERO, (g_sched+gobuf_lr)(g) MOV ZERO, (g_sched+gobuf_ret)(g) - MOV g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOV (g_sched+gobuf_ctxt)(g), X31 BEQ ZERO, X31, 2(PC) diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index 43244d961f..203754f32c 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -176,9 +176,14 @@ TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 // void gogo(Gobuf*) // restore state from Gobuf; longjmp -TEXT runtime·gogo(SB), NOSPLIT, $16-8 +TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 MOVD buf+0(FP), R5 - MOVD gobuf_g(R5), g // make sure g is not nil + MOVD gobuf_g(R5), R6 + MOVD 0(R6), R7 // make sure g != nil + BR gogo<>(SB) + +TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 + MOVD R6, g BL runtime·save_g(SB) MOVD 0(g), R4 @@ -203,7 +208,6 @@ TEXT runtime·mcall(SB), NOSPLIT, $-8-8 MOVD R15, (g_sched+gobuf_sp)(g) MOVD LR, (g_sched+gobuf_pc)(g) MOVD $0, (g_sched+gobuf_lr)(g) - MOVD g, (g_sched+gobuf_g)(g) // Switch to m->g0 & its stack, call fn. MOVD g, R3 @@ -500,7 +504,6 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD R15, (g_sched+gobuf_sp)(g) MOVD $0, (g_sched+gobuf_lr)(g) MOVD $0, (g_sched+gobuf_ret)(g) - MOVD g, (g_sched+gobuf_g)(g) // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R1 CMPBEQ R1, $0, 2(PC) diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index cf3d961b74..3765c756b3 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -34,7 +34,9 @@ TEXT ·checkASM(SB), NOSPLIT, $0-1 TEXT runtime·gogo(SB), NOSPLIT, $0-8 MOVD buf+0(FP), R0 - MOVD gobuf_g(R0), g + MOVD gobuf_g(R0), R1 + MOVD 0(R1), R2 // make sure g != nil + MOVD R1, g MOVD gobuf_sp(R0), SP // Put target PC at -8(SP), wasm_pc_f_loop will pick it up @@ -69,7 +71,6 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8 // save state in g->sched MOVD 0(SP), g_sched+gobuf_pc(g) // caller's PC MOVD $fn+0(FP), g_sched+gobuf_sp(g) // caller's SP - MOVD g, g_sched+gobuf_g(g) // if g == g0 call badmcall Get g @@ -143,7 +144,6 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8 MOVD $runtime·systemstack_switch(SB), g_sched+gobuf_pc(g) MOVD SP, g_sched+gobuf_sp(g) - MOVD g, g_sched+gobuf_g(g) // switch to g0 MOVD R2, g @@ -270,7 +270,6 @@ TEXT runtime·morestack(SB), NOSPLIT, $0-0 // Set g->sched to context in f. MOVD 0(SP), g_sched+gobuf_pc(g) - MOVD g, g_sched+gobuf_g(g) MOVD $8(SP), g_sched+gobuf_sp(g) // f's SP MOVD CTXT, g_sched+gobuf_ctxt(g) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 1dbd01ed40..388d843004 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1281,6 +1281,9 @@ func mstart() { mexit(osStack) } +// The go:noinline is to guarantee the getcallerpc/getcallersp below are safe, +// so that we can set up g0.sched to return to the call of mstart1 above. +//go:noinline func mstart1() { _g_ := getg() @@ -1288,11 +1291,16 @@ func mstart1() { throw("bad runtime·mstart") } - // Record the caller for use as the top of stack in mcall and - // for terminating the thread. + // Set up m.g0.sched as a label returning returning to just + // after the mstart1 call in mstart0 above, for use by goexit0 and mcall. // We're never coming back to mstart1 after we call schedule, // so other calls can reuse the current frame. - save(getcallerpc(), getcallersp()) + // And goexit0 does a gogo that needs to return from mstart1 + // and let mstart0 exit the thread. + _g_.sched.g = guintptr(unsafe.Pointer(_g_)) + _g_.sched.pc = getcallerpc() + _g_.sched.sp = getcallersp() + asminit() minit() @@ -3445,11 +3453,19 @@ func goexit0(gp *g) { func save(pc, sp uintptr) { _g_ := getg() + if _g_ == _g_.m.g0 || _g_ == _g_.m.gsignal { + // m.g0.sched is special and must describe the context + // for exiting the thread. mstart1 writes to it directly. + // m.gsignal.sched should not be used at all. + // This check makes sure save calls do not accidentally + // run in contexts where they'd write to system g's. + throw("save on system g not allowed") + } + _g_.sched.pc = pc _g_.sched.sp = sp _g_.sched.lr = 0 _g_.sched.ret = 0 - _g_.sched.g = guintptr(unsafe.Pointer(_g_)) // We need to ensure ctxt is zero, but can't have a write // barrier here. However, it should always already be zero. // Assert that. -- cgit v1.3-5-g9baa From 4dd77bdc910494adcd57fe9d87cd46f72d8d8985 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 28 Jan 2021 15:21:33 -0500 Subject: cmd/asm, cmd/link, runtime: introduce FuncInfo flag bits The runtime traceback code has its own definition of which functions mark the top frame of a stack, separate from the TOPFRAME bits that exist in the assembly and are passed along in DWARF information. It's error-prone and redundant to have two different sources of truth. This CL provides the actual TOPFRAME bits to the runtime, so that the runtime can use those bits instead of reinventing its own category. This CL also adds a new bit, SPWRITE, which marks functions that write directly to SP (anything but adding and subtracting constants). Such functions must stop a traceback, because the traceback has no way to rederive the SP on entry. Again, the runtime has its own definition which is mostly correct, but also missing some functions. During ordinary goroutine context switches, such functions do not appear on the stack, so the incompleteness in the runtime usually doesn't matter. But profiling signals can arrive at any moment, and the runtime may crash during traceback if it attempts to unwind an SP-writing frame and gets out-of-sync with the actual stack. The runtime contains code to try to detect likely candidates but again it is incomplete. Deriving the SPWRITE bit automatically from the actual assembly code provides the complete truth, and passing it to the runtime lets the runtime use it. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I227f53b23ac5b3dabfcc5e8ee3f00df4e113cf58 Reviewed-on: https://go-review.googlesource.com/c/go/+/288800 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/cmd/asm/internal/asm/endtoend_test.go | 2 + src/cmd/asm/internal/flags/flags.go | 2 + src/cmd/asm/main.go | 1 + src/cmd/compile/internal/test/fixedbugs_test.go | 2 +- src/cmd/compile/internal/test/global_test.go | 4 +- src/cmd/internal/goobj/funcinfo.go | 60 ++++++------------------- src/cmd/internal/goobj/objfile.go | 2 - src/cmd/internal/obj/arm/obj5.go | 16 +++++++ src/cmd/internal/obj/arm64/obj7.go | 16 +++++++ src/cmd/internal/obj/link.go | 23 ++++++---- src/cmd/internal/obj/mips/obj0.go | 16 +++++++ src/cmd/internal/obj/objfile.go | 12 +++-- src/cmd/internal/obj/plist.go | 10 ++++- src/cmd/internal/obj/ppc64/obj9.go | 16 +++++++ src/cmd/internal/obj/riscv/obj.go | 16 +++++++ src/cmd/internal/obj/s390x/objz.go | 16 +++++++ src/cmd/internal/obj/util.go | 2 +- src/cmd/internal/obj/x86/obj6.go | 15 +++++++ src/cmd/internal/objabi/funcid.go | 11 ++++- src/cmd/link/internal/ld/dwarf.go | 4 +- src/cmd/link/internal/ld/pcln.go | 9 +++- src/cmd/link/internal/loader/loader.go | 37 +++++---------- src/runtime/runtime2.go | 9 ++-- src/runtime/symtab.go | 9 ++++ 24 files changed, 209 insertions(+), 101 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/asm/internal/asm/endtoend_test.go b/src/cmd/asm/internal/asm/endtoend_test.go index 7472507caf..a4153f3af1 100644 --- a/src/cmd/asm/internal/asm/endtoend_test.go +++ b/src/cmd/asm/internal/asm/endtoend_test.go @@ -36,6 +36,7 @@ func testEndToEnd(t *testing.T, goarch, file string) { var ok bool testOut = new(bytes.Buffer) // The assembler writes test output to this buffer. ctxt.Bso = bufio.NewWriter(os.Stdout) + ctxt.IsAsm = true defer ctxt.Bso.Flush() failed := false ctxt.DiagFunc = func(format string, args ...interface{}) { @@ -278,6 +279,7 @@ func testErrors(t *testing.T, goarch, file string) { var ok bool testOut = new(bytes.Buffer) // The assembler writes test output to this buffer. ctxt.Bso = bufio.NewWriter(os.Stdout) + ctxt.IsAsm = true defer ctxt.Bso.Flush() failed := false var errBuf bytes.Buffer diff --git a/src/cmd/asm/internal/flags/flags.go b/src/cmd/asm/internal/flags/flags.go index 1335860315..dd947c7b5b 100644 --- a/src/cmd/asm/internal/flags/flags.go +++ b/src/cmd/asm/internal/flags/flags.go @@ -32,11 +32,13 @@ var ( D MultiFlag I MultiFlag PrintOut int + DebugV bool ) func init() { flag.Var(&D, "D", "predefined symbol with optional simple value -D=identifier=value; can be set multiple times") flag.Var(&I, "I", "include directory; can be set multiple times") + flag.BoolVar(&DebugV, "v", false, "print debug output") objabi.AddVersionFlag() // -V objabi.Flagcount("S", "print assembly and machine code", &PrintOut) } diff --git a/src/cmd/asm/main.go b/src/cmd/asm/main.go index 31636e3045..98618a67ef 100644 --- a/src/cmd/asm/main.go +++ b/src/cmd/asm/main.go @@ -36,6 +36,7 @@ func main() { ctxt := obj.Linknew(architecture.LinkArch) ctxt.Debugasm = flags.PrintOut + ctxt.Debugvlog = flags.DebugV ctxt.Flag_dynlink = *flags.Dynlink ctxt.Flag_linkshared = *flags.Linkshared ctxt.Flag_shared = *flags.Shared || *flags.Dynlink diff --git a/src/cmd/compile/internal/test/fixedbugs_test.go b/src/cmd/compile/internal/test/fixedbugs_test.go index e7e2f7e58e..376b45edfc 100644 --- a/src/cmd/compile/internal/test/fixedbugs_test.go +++ b/src/cmd/compile/internal/test/fixedbugs_test.go @@ -75,7 +75,7 @@ func TestIssue16214(t *testing.T) { cmd := exec.Command(testenv.GoToolPath(t), "tool", "compile", "-S", "-o", filepath.Join(dir, "out.o"), src) out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("fail to run go tool compile: %v", err) + t.Fatalf("go tool compile: %v\n%s", err, out) } if strings.Contains(string(out), "unknown line number") { diff --git a/src/cmd/compile/internal/test/global_test.go b/src/cmd/compile/internal/test/global_test.go index 5f5f7d6198..93de894f37 100644 --- a/src/cmd/compile/internal/test/global_test.go +++ b/src/cmd/compile/internal/test/global_test.go @@ -50,7 +50,7 @@ func main() { cmd := exec.Command(testenv.GoToolPath(t), "build", "-o", dst, src) out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("could not build target: %v", err) + t.Fatalf("could not build target: %v\n%s", err, out) } // Check destination to see if scanf code was included. @@ -95,7 +95,7 @@ func main() { cmd := exec.Command(testenv.GoToolPath(t), "build", "-gcflags", "-S", "-o", filepath.Join(dir, "test"), src) out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("could not build target: %v", err) + t.Fatalf("could not build target: %v\n%s", err, out) } patterns := []string{ diff --git a/src/cmd/internal/goobj/funcinfo.go b/src/cmd/internal/goobj/funcinfo.go index 2cca8f6c4e..6d33a10a51 100644 --- a/src/cmd/internal/goobj/funcinfo.go +++ b/src/cmd/internal/goobj/funcinfo.go @@ -19,9 +19,10 @@ type CUFileIndex uint32 // // TODO: make each pcdata a separate symbol? type FuncInfo struct { - Args uint32 - Locals uint32 - FuncID objabi.FuncID + Args uint32 + Locals uint32 + FuncID objabi.FuncID + FuncFlag objabi.FuncFlag Pcsp SymRef Pcfile SymRef @@ -35,6 +36,9 @@ type FuncInfo struct { } func (a *FuncInfo) Write(w *bytes.Buffer) { + writeUint8 := func(x uint8) { + w.WriteByte(x) + } var b [4]byte writeUint32 := func(x uint32) { binary.LittleEndian.PutUint32(b[:], x) @@ -47,8 +51,10 @@ func (a *FuncInfo) Write(w *bytes.Buffer) { writeUint32(a.Args) writeUint32(a.Locals) - writeUint32(uint32(a.FuncID)) - + writeUint8(uint8(a.FuncID)) + writeUint8(uint8(a.FuncFlag)) + writeUint8(0) // pad to uint32 boundary + writeUint8(0) writeSymRef(a.Pcsp) writeSymRef(a.Pcfile) writeSymRef(a.Pcline) @@ -72,46 +78,6 @@ func (a *FuncInfo) Write(w *bytes.Buffer) { } } -func (a *FuncInfo) Read(b []byte) { - readUint32 := func() uint32 { - x := binary.LittleEndian.Uint32(b) - b = b[4:] - return x - } - readSymIdx := func() SymRef { - return SymRef{readUint32(), readUint32()} - } - - a.Args = readUint32() - a.Locals = readUint32() - a.FuncID = objabi.FuncID(readUint32()) - - a.Pcsp = readSymIdx() - a.Pcfile = readSymIdx() - a.Pcline = readSymIdx() - a.Pcinline = readSymIdx() - a.Pcdata = make([]SymRef, readUint32()) - for i := range a.Pcdata { - a.Pcdata[i] = readSymIdx() - } - - funcdataofflen := readUint32() - a.Funcdataoff = make([]uint32, funcdataofflen) - for i := range a.Funcdataoff { - a.Funcdataoff[i] = readUint32() - } - filelen := readUint32() - a.File = make([]CUFileIndex, filelen) - for i := range a.File { - a.File[i] = CUFileIndex(readUint32()) - } - inltreelen := readUint32() - a.InlTree = make([]InlTreeNode, inltreelen) - for i := range a.InlTree { - b = a.InlTree[i].Read(b) - } -} - // FuncInfoLengths is a cache containing a roadmap of offsets and // lengths for things within a serialized FuncInfo. Each length field // stores the number of items (e.g. files, inltree nodes, etc), and the @@ -159,7 +125,9 @@ func (*FuncInfo) ReadArgs(b []byte) uint32 { return binary.LittleEndian.Uint32(b func (*FuncInfo) ReadLocals(b []byte) uint32 { return binary.LittleEndian.Uint32(b[4:]) } -func (*FuncInfo) ReadFuncID(b []byte) uint32 { return binary.LittleEndian.Uint32(b[8:]) } +func (*FuncInfo) ReadFuncID(b []byte) objabi.FuncID { return objabi.FuncID(b[8]) } + +func (*FuncInfo) ReadFuncFlag(b []byte) objabi.FuncFlag { return objabi.FuncFlag(b[9]) } func (*FuncInfo) ReadPcsp(b []byte) SymRef { return SymRef{binary.LittleEndian.Uint32(b[12:]), binary.LittleEndian.Uint32(b[16:])} diff --git a/src/cmd/internal/goobj/objfile.go b/src/cmd/internal/goobj/objfile.go index e6447e455d..d1b838f676 100644 --- a/src/cmd/internal/goobj/objfile.go +++ b/src/cmd/internal/goobj/objfile.go @@ -298,7 +298,6 @@ const ( SymFlagNoSplit SymFlagReflectMethod SymFlagGoType - SymFlagTopFrame ) // Sym.Flag2 @@ -332,7 +331,6 @@ func (s *Sym) Leaf() bool { return s.Flag()&SymFlagLeaf != 0 } func (s *Sym) NoSplit() bool { return s.Flag()&SymFlagNoSplit != 0 } func (s *Sym) ReflectMethod() bool { return s.Flag()&SymFlagReflectMethod != 0 } func (s *Sym) IsGoType() bool { return s.Flag()&SymFlagGoType != 0 } -func (s *Sym) TopFrame() bool { return s.Flag()&SymFlagTopFrame != 0 } func (s *Sym) UsedInIface() bool { return s.Flag2()&SymFlagUsedInIface != 0 } func (s *Sym) IsItab() bool { return s.Flag2()&SymFlagItab != 0 } diff --git a/src/cmd/internal/obj/arm/obj5.go b/src/cmd/internal/obj/arm/obj5.go index 29d3a5867d..7de04302d9 100644 --- a/src/cmd/internal/obj/arm/obj5.go +++ b/src/cmd/internal/obj/arm/obj5.go @@ -34,6 +34,7 @@ import ( "cmd/internal/obj" "cmd/internal/objabi" "cmd/internal/sys" + "log" ) var progedit_tlsfallback *obj.LSym @@ -613,6 +614,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.From.Reg = REGSP } } + + if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 { + f := c.cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + c.cursym.Func().FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s %v\n", c.cursym.Name, p) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } } } diff --git a/src/cmd/internal/obj/arm64/obj7.go b/src/cmd/internal/obj/arm64/obj7.go index 0baf51973a..3b88543852 100644 --- a/src/cmd/internal/obj/arm64/obj7.go +++ b/src/cmd/internal/obj/arm64/obj7.go @@ -35,6 +35,7 @@ import ( "cmd/internal/objabi" "cmd/internal/src" "cmd/internal/sys" + "log" "math" ) @@ -970,6 +971,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p = q5 } } + + if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 { + f := c.cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + c.cursym.Func().FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s %v\n", c.cursym.Name, p) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } } } diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index 8206902328..a48db3bdc8 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -454,6 +454,7 @@ type FuncInfo struct { Locals int32 Align int32 FuncID objabi.FuncID + FuncFlag objabi.FuncFlag Text *Prog Autot map[*LSym]struct{} Pcln Pcln @@ -619,10 +620,6 @@ const ( // target of an inline during compilation AttrWasInlined - // TopFrame means that this function is an entry point and unwinders should not - // keep unwinding beyond this frame. - AttrTopFrame - // Indexed indicates this symbol has been assigned with an index (when using the // new object file format). AttrIndexed @@ -663,7 +660,6 @@ func (a *Attribute) NeedCtxt() bool { return a.load()&AttrNeedCtxt != func (a *Attribute) NoFrame() bool { return a.load()&AttrNoFrame != 0 } func (a *Attribute) Static() bool { return a.load()&AttrStatic != 0 } func (a *Attribute) WasInlined() bool { return a.load()&AttrWasInlined != 0 } -func (a *Attribute) TopFrame() bool { return a.load()&AttrTopFrame != 0 } func (a *Attribute) Indexed() bool { return a.load()&AttrIndexed != 0 } func (a *Attribute) UsedInIface() bool { return a.load()&AttrUsedInIface != 0 } func (a *Attribute) ContentAddressable() bool { return a.load()&AttrContentAddressable != 0 } @@ -713,14 +709,13 @@ var textAttrStrings = [...]struct { {bit: AttrNoFrame, s: "NOFRAME"}, {bit: AttrStatic, s: "STATIC"}, {bit: AttrWasInlined, s: ""}, - {bit: AttrTopFrame, s: "TOPFRAME"}, {bit: AttrIndexed, s: ""}, {bit: AttrContentAddressable, s: ""}, {bit: AttrABIWrapper, s: "ABIWRAPPER"}, } -// TextAttrString formats a for printing in as part of a TEXT prog. -func (a Attribute) TextAttrString() string { +// String formats a for printing in as part of a TEXT prog. +func (a Attribute) String() string { var s string for _, x := range textAttrStrings { if a&x.bit != 0 { @@ -746,6 +741,18 @@ func (a Attribute) TextAttrString() string { return s } +// TextAttrString formats the symbol attributes for printing in as part of a TEXT prog. +func (s *LSym) TextAttrString() string { + attr := s.Attribute.String() + if s.Func().FuncFlag&objabi.FuncFlag_TOPFRAME != 0 { + if attr != "" { + attr += "|" + } + attr += "TOPFRAME" + } + return attr +} + func (s *LSym) String() string { return s.Name } diff --git a/src/cmd/internal/obj/mips/obj0.go b/src/cmd/internal/obj/mips/obj0.go index 135a8df3aa..91bba90d41 100644 --- a/src/cmd/internal/obj/mips/obj0.go +++ b/src/cmd/internal/obj/mips/obj0.go @@ -35,6 +35,7 @@ import ( "cmd/internal/sys" "encoding/binary" "fmt" + "log" "math" ) @@ -536,6 +537,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.From.Reg = REGSP } } + + if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 { + f := c.cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + c.cursym.Func().FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s %v\n", c.cursym.Name, p) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } } if c.ctxt.Arch.Family == sys.MIPS { diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go index bb58b4f0c2..85f0570e5d 100644 --- a/src/cmd/internal/obj/objfile.go +++ b/src/cmd/internal/obj/objfile.go @@ -330,9 +330,6 @@ func (w *writer) Sym(s *LSym) { if s.ReflectMethod() { flag |= goobj.SymFlagReflectMethod } - if s.TopFrame() { - flag |= goobj.SymFlagTopFrame - } if strings.HasPrefix(s.Name, "type.") && s.Name[5] != '.' && s.Type == objabi.SRODATA { flag |= goobj.SymFlagGoType } @@ -673,9 +670,10 @@ func genFuncInfoSyms(ctxt *Link) { continue } o := goobj.FuncInfo{ - Args: uint32(fn.Args), - Locals: uint32(fn.Locals), - FuncID: objabi.FuncID(fn.FuncID), + Args: uint32(fn.Args), + Locals: uint32(fn.Locals), + FuncID: fn.FuncID, + FuncFlag: fn.FuncFlag, } pc := &fn.Pcln o.Pcsp = makeSymRef(preparePcSym(pc.Pcsp)) @@ -788,7 +786,7 @@ func (ctxt *Link) writeSymDebugNamed(s *LSym, name string) { if s.NoSplit() { fmt.Fprintf(ctxt.Bso, "nosplit ") } - if s.TopFrame() { + if s.Func() != nil && s.Func().FuncFlag&objabi.FuncFlag_TOPFRAME != 0 { fmt.Fprintf(ctxt.Bso, "topframe ") } fmt.Fprintf(ctxt.Bso, "size=%d", s.Size) diff --git a/src/cmd/internal/obj/plist.go b/src/cmd/internal/obj/plist.go index 679ce7eb8f..177083261c 100644 --- a/src/cmd/internal/obj/plist.go +++ b/src/cmd/internal/obj/plist.go @@ -134,6 +134,7 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) { } name := strings.Replace(s.Name, "\"\"", ctxt.Pkgpath, -1) s.Func().FuncID = objabi.GetFuncID(name, flag&WRAPPER != 0) + s.Func().FuncFlag = toFuncFlag(flag) s.Set(AttrOnList, true) s.Set(AttrDuplicateOK, flag&DUPOK != 0) s.Set(AttrNoSplit, flag&NOSPLIT != 0) @@ -142,7 +143,6 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) { s.Set(AttrABIWrapper, flag&ABIWRAPPER != 0) s.Set(AttrNeedCtxt, flag&NEEDCTXT != 0) s.Set(AttrNoFrame, flag&NOFRAME != 0) - s.Set(AttrTopFrame, flag&TOPFRAME != 0) s.Type = objabi.STEXT ctxt.Text = append(ctxt.Text, s) @@ -150,6 +150,14 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) { ctxt.dwarfSym(s) } +func toFuncFlag(flag int) objabi.FuncFlag { + var out objabi.FuncFlag + if flag&TOPFRAME != 0 { + out |= objabi.FuncFlag_TOPFRAME + } + return out +} + func (ctxt *Link) Globl(s *LSym, size int64, flag int) { if s.OnList() { ctxt.Diag("symbol %s listed multiple times", s.Name) diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go index fddf552156..a77be29cf0 100644 --- a/src/cmd/internal/obj/ppc64/obj9.go +++ b/src/cmd/internal/obj/ppc64/obj9.go @@ -34,6 +34,7 @@ import ( "cmd/internal/objabi" "cmd/internal/src" "cmd/internal/sys" + "log" ) func progedit(ctxt *obj.Link, p *obj.Prog, newprog obj.ProgAlloc) { @@ -984,6 +985,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.From.Reg = REGSP } } + + if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 && p.As != ACMPU { + f := c.cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + c.cursym.Func().FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s %v\n", c.cursym.Name, p) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } } } diff --git a/src/cmd/internal/obj/riscv/obj.go b/src/cmd/internal/obj/riscv/obj.go index 5ef334dd6a..d104f1cfa5 100644 --- a/src/cmd/internal/obj/riscv/obj.go +++ b/src/cmd/internal/obj/riscv/obj.go @@ -25,6 +25,7 @@ import ( "cmd/internal/objabi" "cmd/internal/sys" "fmt" + "log" ) func buildop(ctxt *obj.Link) {} @@ -716,6 +717,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.Spadj = int32(-p.From.Offset) } } + + if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 { + f := cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + f.FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s %v\n", cursym.Name, p) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } } // Rewrite MOV pseudo-instructions. This cannot be done in diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index 970cf827d6..a02c4fc17f 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -33,6 +33,7 @@ import ( "cmd/internal/obj" "cmd/internal/objabi" "cmd/internal/sys" + "log" "math" ) @@ -545,6 +546,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.From.Reg = REGSP } } + + if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 { + f := c.cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + c.cursym.Func().FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s\n", c.cursym.Name) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } } if wasSplit { c.stacksplitPost(pLast, pPre, pPreempt, autosize) // emit post part of split check diff --git a/src/cmd/internal/obj/util.go b/src/cmd/internal/obj/util.go index b9bacb7a22..1c34b4e833 100644 --- a/src/cmd/internal/obj/util.go +++ b/src/cmd/internal/obj/util.go @@ -187,7 +187,7 @@ func (p *Prog) WriteInstructionString(w io.Writer) { // In short, print one of these two: // TEXT foo(SB), DUPOK|NOSPLIT, $0 // TEXT foo(SB), $0 - s := p.From.Sym.Attribute.TextAttrString() + s := p.From.Sym.TextAttrString() if s != "" { fmt.Fprintf(w, "%s%s", sep, s) sep = ", " diff --git a/src/cmd/internal/obj/x86/obj6.go b/src/cmd/internal/obj/x86/obj6.go index 84de58a4c4..bc3a3b4bbe 100644 --- a/src/cmd/internal/obj/x86/obj6.go +++ b/src/cmd/internal/obj/x86/obj6.go @@ -35,6 +35,7 @@ import ( "cmd/internal/objabi" "cmd/internal/src" "cmd/internal/sys" + "log" "math" "strings" ) @@ -839,6 +840,20 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { switch p.As { default: + if p.To.Type == obj.TYPE_REG && p.To.Reg == REG_SP && p.As != ACMPL && p.As != ACMPQ { + f := cursym.Func() + if f.FuncFlag&objabi.FuncFlag_SPWRITE == 0 { + f.FuncFlag |= objabi.FuncFlag_SPWRITE + if ctxt.Debugvlog || !ctxt.IsAsm { + ctxt.Logf("auto-SPWRITE: %s %v\n", cursym.Name, p) + if !ctxt.IsAsm { + ctxt.Diag("invalid auto-SPWRITE in non-assembly") + ctxt.DiagFlush() + log.Fatalf("bad SPWRITE") + } + } + } + } continue case APUSHL, APUSHFL: diff --git a/src/cmd/internal/objabi/funcid.go b/src/cmd/internal/objabi/funcid.go index e921a82c0c..6e188e31bb 100644 --- a/src/cmd/internal/objabi/funcid.go +++ b/src/cmd/internal/objabi/funcid.go @@ -6,13 +6,22 @@ package objabi import "strings" +// A FuncFlag records bits about a function, passed to the runtime. +type FuncFlag uint8 + +// Note: This list must match the list in runtime/symtab.go. +const ( + FuncFlag_TOPFRAME = 1 << iota + FuncFlag_SPWRITE +) + // A FuncID identifies particular functions that need to be treated // specially by the runtime. // Note that in some situations involving plugins, there may be multiple // copies of a particular special runtime function. -// Note: this list must match the list in runtime/symtab.go. type FuncID uint8 +// Note: this list must match the list in runtime/symtab.go. const ( FuncID_normal FuncID = iota // not a special function FuncID_asmcgocall diff --git a/src/cmd/link/internal/ld/dwarf.go b/src/cmd/link/internal/ld/dwarf.go index 2ab9a55e96..561f6f1475 100644 --- a/src/cmd/link/internal/ld/dwarf.go +++ b/src/cmd/link/internal/ld/dwarf.go @@ -1454,7 +1454,7 @@ func (d *dwctxt) writeframes(fs loader.Sym) dwarfSecInfo { // Emit a FDE, Section 6.4.1. // First build the section contents into a byte buffer. deltaBuf = deltaBuf[:0] - if haslr && d.ldr.AttrTopFrame(fn) { + if haslr && fi.TopFrame() { // Mark the link register as having an undefined value. // This stops call stack unwinders progressing any further. // TODO: similar mark on non-LR architectures. @@ -1480,7 +1480,7 @@ func (d *dwctxt) writeframes(fs loader.Sym) dwarfSecInfo { spdelta += int64(d.arch.PtrSize) } - if haslr && !d.ldr.AttrTopFrame(fn) { + if haslr && !fi.TopFrame() { // TODO(bryanpkc): This is imprecise. In general, the instruction // that stores the return address to the stack frame is not the // same one that allocates the frame. diff --git a/src/cmd/link/internal/ld/pcln.go b/src/cmd/link/internal/ld/pcln.go index 72bf33e611..fb733117be 100644 --- a/src/cmd/link/internal/ld/pcln.go +++ b/src/cmd/link/internal/ld/pcln.go @@ -796,7 +796,14 @@ func writeFuncs(ctxt *Link, sb *loader.SymbolBuilder, funcs []loader.Sym, inlSym } off = uint32(sb.SetUint8(ctxt.Arch, int64(off), uint8(funcID))) - off += 2 // pad + // flag uint8 + var flag objabi.FuncFlag + if fi.Valid() { + flag = fi.FuncFlag() + } + off = uint32(sb.SetUint8(ctxt.Arch, int64(off), uint8(flag))) + + off += 1 // pad // nfuncdata must be the final entry. funcdata, funcdataoff = funcData(fi, 0, funcdata, funcdataoff) diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go index 98c2131c2b..68dc3de273 100644 --- a/src/cmd/link/internal/loader/loader.go +++ b/src/cmd/link/internal/loader/loader.go @@ -241,7 +241,6 @@ type Loader struct { attrExternal Bitmap // external symbols, indexed by ext sym index attrReadOnly map[Sym]bool // readonly data for this sym - attrTopFrame map[Sym]struct{} // top frame symbols attrSpecial map[Sym]struct{} // "special" frame symbols attrCgoExportDynamic map[Sym]struct{} // "cgo_export_dynamic" symbols attrCgoExportStatic map[Sym]struct{} // "cgo_export_static" symbols @@ -349,7 +348,6 @@ func NewLoader(flags uint32, elfsetstring elfsetstringFunc, reporter *ErrorRepor plt: make(map[Sym]int32), got: make(map[Sym]int32), dynid: make(map[Sym]int32), - attrTopFrame: make(map[Sym]struct{}), attrSpecial: make(map[Sym]struct{}), attrCgoExportDynamic: make(map[Sym]struct{}), attrCgoExportStatic: make(map[Sym]struct{}), @@ -1009,24 +1007,6 @@ func (l *Loader) SetAttrExternal(i Sym, v bool) { } } -// AttrTopFrame returns true for a function symbol that is an entry -// point, meaning that unwinders should stop when they hit this -// function. -func (l *Loader) AttrTopFrame(i Sym) bool { - _, ok := l.attrTopFrame[i] - return ok -} - -// SetAttrTopFrame sets the "top frame" property for a symbol (see -// AttrTopFrame). -func (l *Loader) SetAttrTopFrame(i Sym, v bool) { - if v { - l.attrTopFrame[i] = struct{}{} - } else { - delete(l.attrTopFrame, i) - } -} - // AttrSpecial returns true for a symbols that do not have their // address (i.e. Value) computed by the usual mechanism of // data.go:dodata() & data.go:address(). @@ -1905,7 +1885,11 @@ func (fi *FuncInfo) Locals() int { } func (fi *FuncInfo) FuncID() objabi.FuncID { - return objabi.FuncID((*goobj.FuncInfo)(nil).ReadFuncID(fi.data)) + return (*goobj.FuncInfo)(nil).ReadFuncID(fi.data) +} + +func (fi *FuncInfo) FuncFlag() objabi.FuncFlag { + return (*goobj.FuncInfo)(nil).ReadFuncFlag(fi.data) } func (fi *FuncInfo) Pcsp() Sym { @@ -1992,6 +1976,13 @@ func (fi *FuncInfo) File(k int) goobj.CUFileIndex { return (*goobj.FuncInfo)(nil).ReadFile(fi.data, fi.lengths.FileOff, uint32(k)) } +// TopFrame returns true if the function associated with this FuncInfo +// is an entry point, meaning that unwinders should stop when they hit +// this function. +func (fi *FuncInfo) TopFrame() bool { + return (fi.FuncFlag() & objabi.FuncFlag_TOPFRAME) != 0 +} + type InlTreeNode struct { Parent int32 File goobj.CUFileIndex @@ -2151,9 +2142,6 @@ func (st *loadState) preloadSyms(r *oReader, kind int) { } gi := st.addSym(name, v, r, i, kind, osym) r.syms[i] = gi - if osym.TopFrame() { - l.SetAttrTopFrame(gi, true) - } if osym.Local() { l.SetAttrLocal(gi, true) } @@ -2411,7 +2399,6 @@ func (l *Loader) CopyAttributes(src Sym, dst Sym) { // when copying attributes from a dupOK ABI wrapper symbol to // the real target symbol (which may not be marked dupOK). } - l.SetAttrTopFrame(dst, l.AttrTopFrame(src)) l.SetAttrSpecial(dst, l.AttrSpecial(src)) l.SetAttrCgoExportDynamic(dst, l.AttrCgoExportDynamic(src)) l.SetAttrCgoExportStatic(dst, l.AttrCgoExportStatic(src)) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 675c613b6e..05520d07b2 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -833,10 +833,11 @@ type _func struct { pcfile uint32 pcln uint32 npcdata uint32 - cuOffset uint32 // runtime.cutab offset of this function's CU - funcID funcID // set for certain special runtime functions - _ [2]byte // pad - nfuncdata uint8 // must be last + cuOffset uint32 // runtime.cutab offset of this function's CU + funcID funcID // set for certain special runtime functions + flag funcFlag + _ [1]byte // pad + nfuncdata uint8 // must be last, must end on a uint32-aligned boundary } // Pseudo-Func that is returned for PCs that occur in inlined code. diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index fc93c00c2d..d7da255e43 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -332,6 +332,15 @@ const ( funcID_wrapper // any autogenerated code (hash/eq algorithms, method wrappers, etc.) ) +// A FuncFlag holds bits about a function. +// This list must match the list in cmd/internal/objabi/funcid.go. +type funcFlag uint8 + +const ( + funcFlag_TOPFRAME funcFlag = 1 << iota + funcFlag_SPWRITE +) + // pcHeader holds data used by the pclntab lookups. type pcHeader struct { magic uint32 // 0xFFFFFFFA -- cgit v1.3-5-g9baa From fbe74dbf4263841819368a2a3c90e599392e0808 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 28 Jan 2021 16:10:58 -0500 Subject: runtime: use FuncInfo SPWRITE flag to identify untraceable profile samples The old code was very clever about predicting whether a traceback was safe. That cleverness has not aged well. In particular, the setsSP function is missing a bunch of functions that write to SP and will confuse traceback. And one such function - jmpdefer - was handled as a special case in gentraceback instead of simply listing it in setsSP. Throw away all the clever prediction about whether traceback will crash. Instead, make traceback NOT crash, by checking whether the function being walked writes to SP. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I3d55fe257a22745e4919ac4dc9a9378c984ba0da Reviewed-on: https://go-review.googlesource.com/c/go/+/288801 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/asm_arm.s | 4 -- src/runtime/pprof/pprof_test.go | 18 +++++--- src/runtime/proc.go | 95 +---------------------------------------- src/runtime/traceback.go | 52 +++++++++++----------- 4 files changed, 42 insertions(+), 127 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 92d7854306..4620f19074 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -506,10 +506,6 @@ CALLFN(·call1073741824, 1073741824) // 1. grab stored LR for caller // 2. sub 4 bytes to get back to BL deferreturn // 3. B to fn -// TODO(rsc): Push things on stack and then use pop -// to load all registers simultaneously, so that a profiling -// interrupt can never see mismatched SP/LR/PC. -// (And double-check that pop is atomic in that way.) TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8 MOVW 0(R13), LR MOVW $-4(LR), LR // BL deferreturn diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index b6ee160e84..37f12de0d9 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -514,8 +514,10 @@ func TestGoroutineSwitch(t *testing.T) { } StopCPUProfile() - // Read profile to look for entries for runtime.gogo with an attempt at a traceback. - // The special entry + // Read profile to look for entries for gogo with an attempt at a traceback. + // "runtime.gogo" is OK, because that's the part of the context switch + // before the actual switch begins. But we should not see "gogo", + // aka "gogo<>(SB)", which does the actual switch and is marked SPWRITE. parseProfile(t, prof.Bytes(), func(count uintptr, stk []*profile.Location, _ map[string][]string) { // An entry with two frames with 'System' in its top frame // exists to record a PC without a traceback. Those are okay. @@ -526,13 +528,19 @@ func TestGoroutineSwitch(t *testing.T) { } } - // Otherwise, should not see runtime.gogo. + // An entry with just one frame is OK too: + // it knew to stop at gogo. + if len(stk) == 1 { + return + } + + // Otherwise, should not see gogo. // The place we'd see it would be the inner most frame. name := stk[0].Line[0].Function.Name - if name == "runtime.gogo" { + if name == "gogo" { var buf bytes.Buffer fprintStack(&buf, stk) - t.Fatalf("found profile entry for runtime.gogo:\n%s", buf.String()) + t.Fatalf("found profile entry for gogo:\n%s", buf.String()) } }) } diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 388d843004..e6670135db 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -4420,75 +4420,6 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { // See golang.org/issue/17165. getg().m.mallocing++ - // Define that a "user g" is a user-created goroutine, and a "system g" - // is one that is m->g0 or m->gsignal. - // - // We might be interrupted for profiling halfway through a - // goroutine switch. The switch involves updating three (or four) values: - // g, PC, SP, and (on arm) LR. The PC must be the last to be updated, - // because once it gets updated the new g is running. - // - // When switching from a user g to a system g, LR is not considered live, - // so the update only affects g, SP, and PC. Since PC must be last, there - // the possible partial transitions in ordinary execution are (1) g alone is updated, - // (2) both g and SP are updated, and (3) SP alone is updated. - // If SP or g alone is updated, we can detect the partial transition by checking - // whether the SP is within g's stack bounds. (We could also require that SP - // be changed only after g, but the stack bounds check is needed by other - // cases, so there is no need to impose an additional requirement.) - // - // There is one exceptional transition to a system g, not in ordinary execution. - // When a signal arrives, the operating system starts the signal handler running - // with an updated PC and SP. The g is updated last, at the beginning of the - // handler. There are two reasons this is okay. First, until g is updated the - // g and SP do not match, so the stack bounds check detects the partial transition. - // Second, signal handlers currently run with signals disabled, so a profiling - // signal cannot arrive during the handler. - // - // When switching from a system g to a user g, there are three possibilities. - // - // First, it may be that the g switch has no PC update, because the SP - // either corresponds to a user g throughout (as in asmcgocall) - // or because it has been arranged to look like a user g frame - // (as in cgocallback). In this case, since the entire - // transition is a g+SP update, a partial transition updating just one of - // those will be detected by the stack bounds check. - // - // Second, when returning from a signal handler, the PC and SP updates - // are performed by the operating system in an atomic update, so the g - // update must be done before them. The stack bounds check detects - // the partial transition here, and (again) signal handlers run with signals - // disabled, so a profiling signal cannot arrive then anyway. - // - // Third, the common case: it may be that the switch updates g, SP, and PC - // separately. If the PC is within any of the functions that does this, - // we don't ask for a traceback. C.F. the function setsSP for more about this. - // - // There is another apparently viable approach, recorded here in case - // the "PC within setsSP function" check turns out not to be usable. - // It would be possible to delay the update of either g or SP until immediately - // before the PC update instruction. Then, because of the stack bounds check, - // the only problematic interrupt point is just before that PC update instruction, - // and the sigprof handler can detect that instruction and simulate stepping past - // it in order to reach a consistent state. On ARM, the update of g must be made - // in two places (in R10 and also in a TLS slot), so the delayed update would - // need to be the SP update. The sigprof handler must read the instruction at - // the current PC and if it was the known instruction (for example, JMP BX or - // MOV R2, PC), use that other register in place of the PC value. - // The biggest drawback to this solution is that it requires that we can tell - // whether it's safe to read from the memory pointed at by PC. - // In a correct program, we can test PC == nil and otherwise read, - // but if a profiling signal happens at the instant that a program executes - // a bad jump (before the program manages to handle the resulting fault) - // the profiling handler could fault trying to read nonexistent memory. - // - // To recap, there are no constraints on the assembly being used for the - // transition. We simply require that g and SP match and that the PC is not - // in gogo. - traceback := true - if gp == nil || sp < gp.stack.lo || gp.stack.hi < sp || setsSP(pc) || (mp != nil && mp.vdsoSP != 0) { - traceback = false - } var stk [maxCPUProfStack]uintptr n := 0 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 { @@ -4511,7 +4442,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { if n > 0 { n += cgoOff } - } else if traceback { + } else { n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap|_TraceJumpStack) } @@ -4590,30 +4521,6 @@ func sigprofNonGoPC(pc uintptr) { } } -// Reports whether a function will set the SP -// to an absolute value. Important that -// we don't traceback when these are at the bottom -// of the stack since we can't be sure that we will -// find the caller. -// -// If the function is not on the bottom of the stack -// we assume that it will have set it up so that traceback will be consistent, -// either by being a traceback terminating function -// or putting one on the stack at the right offset. -func setsSP(pc uintptr) bool { - f := findfunc(pc) - if !f.valid() { - // couldn't find the function for this PC, - // so assume the worst and stop traceback - return true - } - switch f.funcID { - case funcID_gogo, funcID_systemstack, funcID_mcall, funcID_morestack: - return true - } - return false -} - // setcpuprofilerate sets the CPU profiling rate to hz times per second. // If hz <= 0, setcpuprofilerate turns off CPU profiling. func setcpuprofilerate(hz int32) { diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 127f54e42e..18d8a42854 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -15,24 +15,9 @@ import ( // The most important fact about a given architecture is whether it uses a link register. // On systems with link registers, the prologue for a non-leaf function stores the // incoming value of LR at the bottom of the newly allocated stack frame. -// On systems without link registers, the architecture pushes a return PC during +// On systems without link registers (x86), the architecture pushes a return PC during // the call instruction, so the return PC ends up above the stack frame. // In this file, the return PC is always called LR, no matter how it was found. -// -// To date, the opposite of a link register architecture is an x86 architecture. -// This code may need to change if some other kind of non-link-register -// architecture comes along. -// -// The other important fact is the size of a pointer: on 32-bit systems the LR -// takes up only 4 bytes on the stack, while on 64-bit systems it takes up 8 bytes. -// Typically this is ptrSize. -// -// As an exception, amd64p32 had ptrSize == 4 but the CALL instruction still -// stored an 8-byte return PC onto the stack. To accommodate this, we used regSize -// as the size of the architecture-pushed return PC. -// -// usesLR is defined below in terms of minFrameSize, which is defined in -// arch_$GOARCH.go. ptrSize and regSize are defined in stubs.go. const usesLR = sys.MinFrameSize > 0 @@ -180,6 +165,16 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in break } + // Compute function info flags. + flag := f.flag + if f.funcID == funcID_cgocallback { + // cgocallback does write SP to switch from the g0 to the curg stack, + // but it carefully arranges that during the transition BOTH stacks + // have cgocallback frame valid for unwinding through. + // So we don't need to exclude it with the other SP-writing functions. + flag &^= funcFlag_SPWRITE + } + // Found an actual function. // Derive frame pointer and link register. if frame.fp == 0 { @@ -196,6 +191,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in frame.pc = gp.m.curg.sched.pc frame.fn = findfunc(frame.pc) f = frame.fn + flag = f.flag frame.sp = gp.m.curg.sched.sp cgoCtxt = gp.m.curg.cgoCtxt case funcID_systemstack: @@ -203,6 +199,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // stack transition. frame.sp = gp.m.curg.sched.sp cgoCtxt = gp.m.curg.cgoCtxt + flag &^= funcFlag_SPWRITE } } frame.fp = frame.sp + uintptr(funcspdelta(f, frame.pc, &cache)) @@ -213,19 +210,26 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } var flr funcInfo if topofstack(f, gp.m != nil && gp == gp.m.g0) { + // This function marks the top of the stack. Stop the traceback. frame.lr = 0 flr = funcInfo{} - } else if usesLR && f.funcID == funcID_jmpdefer { - // jmpdefer modifies SP/LR/PC non-atomically. - // If a profiling interrupt arrives during jmpdefer, - // the stack unwind may see a mismatched register set - // and get confused. Stop if we see PC within jmpdefer - // to avoid that confusion. - // See golang.org/issue/8153. + } else if flag&funcFlag_SPWRITE != 0 { + // The function we are in does a write to SP that we don't know + // how to encode in the spdelta table. Examples include context + // switch routines like runtime.gogo but also any code that switches + // to the g0 stack to run host C code. Since we can't reliably unwind + // the SP (we might not even be on the stack we think we are), + // we stop the traceback here. if callback != nil { - throw("traceback_arm: found jmpdefer when tracing with callback") + // Finding an SPWRITE should only happen for a profiling signal, which can + // arrive at any time. For a GC stack traversal (callback != nil), + // we shouldn't see this case, and we must be sure to walk the + // entire stack or the GC is invalid. So crash. + println("traceback: unexpected SPWRITE function", funcname(f)) + throw("traceback") } frame.lr = 0 + flr = funcInfo{} } else { var lrPtr uintptr if usesLR { -- cgit v1.3-5-g9baa From 54da3ab385686dec5554164ba9558214445deec9 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 28 Jan 2021 16:22:52 -0500 Subject: runtime: use TOPFRAME to identify top-of-frame functions No change to actual runtime, but helps reduce the laundry list of functions. mcall, morestack, and asmcgocall are not actually top-of-frame, so those need more attention in follow-up CLs. mstart moved to assembly so that it can be marked TOPFRAME. Since TOPFRAME also tells DWARF consumers not to unwind this way, this change should also improve debuggers a marginal amount. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: If1e0d46ca973de5e46b62948d076f675f285b5d9 Reviewed-on: https://go-review.googlesource.com/c/go/+/288802 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/asm_386.s | 8 ++++++-- src/runtime/asm_amd64.s | 8 ++++++-- src/runtime/asm_arm.s | 6 +++++- src/runtime/asm_arm64.s | 6 +++++- src/runtime/asm_mips64x.s | 6 +++++- src/runtime/asm_mipsx.s | 6 +++++- src/runtime/asm_ppc64x.s | 6 +++++- src/runtime/asm_riscv64.s | 6 +++++- src/runtime/asm_s390x.s | 6 +++++- src/runtime/asm_wasm.s | 8 ++++++-- src/runtime/proc.go | 7 +++++-- src/runtime/sys_windows_386.s | 2 +- src/runtime/sys_windows_amd64.s | 2 +- src/runtime/sys_windows_arm.s | 2 +- src/runtime/traceback.go | 5 +---- 15 files changed, 62 insertions(+), 22 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index fcf74a03cf..5b0852f780 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -89,7 +89,7 @@ GLOBL _rt0_386_lib_argc<>(SB),NOPTR, $4 DATA _rt0_386_lib_argv<>(SB)/4, $0 GLOBL _rt0_386_lib_argv<>(SB),NOPTR, $4 -TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 // Copy arguments forward on an even stack. // Users of this function jump to it, they don't call it. MOVL 0(SP), AX @@ -269,6 +269,10 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 FLDCW runtime·controlWord64(SB) RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + CALL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ @@ -1307,7 +1311,7 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 // The top-most function running on a goroutine // returns to goexit+PCQuantum. -TEXT runtime·goexit(SB),NOSPLIT,$0-0 +TEXT runtime·goexit(SB),NOSPLIT|TOPFRAME,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return // traceback from goexit1 must hit code range of goexit diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 8ee2ac2123..a68dc72ae5 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -84,7 +84,7 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8 DATA _rt0_amd64_lib_argv<>(SB)/8, $0 GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8 -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // copy arguments forward on an even stack MOVQ DI, AX // argc MOVQ SI, BX // argv @@ -250,6 +250,10 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 // No per-thread init. RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + CALL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ @@ -1440,7 +1444,7 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0 // so as to make it identifiable to traceback (this // function it used as a sentinel; traceback wants to // see the func PC, not a wrapper PC). -TEXT runtime·goexit(SB),NOSPLIT,$0-0 +TEXT runtime·goexit(SB),NOSPLIT|TOPFRAME,$0-0 BYTE $0x90 // NOP CALL runtime·goexit1(SB) // does not return // traceback from goexit1 must hit code range of goexit diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 4620f19074..f9535bb1bc 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -112,7 +112,7 @@ GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4 // using NOFRAME means do not save LR on stack. // argc is in R0, argv is in R1. -TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVW $0xcafebabe, R12 // copy arguments forward on an even stack @@ -202,6 +202,10 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0 WORD $0xeee1ba10 // vmsr fpscr, r11 RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + BL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 4f0a680fa4..d81759537e 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -8,7 +8,7 @@ #include "funcdata.h" #include "textflag.h" -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // SP = stack; R0 = argc; R1 = argv SUB $32, RSP @@ -109,6 +109,10 @@ TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + BL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index f6d8931a15..af27b9b555 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -11,7 +11,7 @@ #define REGCTXT R22 -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // R29 = stack; R4 = argc; R5 = argv ADDV $-24, R29 @@ -85,6 +85,10 @@ TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + JAL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index cf4b1b42cc..0c7d28dcf7 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -11,7 +11,7 @@ #define REGCTXT R22 -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // R29 = stack; R4 = argc; R5 = argv ADDU $-12, R29 @@ -86,6 +86,10 @@ TEXT runtime·breakpoint(SB),NOSPLIT,$0-0 TEXT runtime·asminit(SB),NOSPLIT,$0-0 RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + JAL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 90f14d8e54..56e73742ea 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -16,7 +16,7 @@ #define cgoCalleeStackSize 32 #endif -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer // initialize essential registers @@ -124,6 +124,10 @@ TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0 XOR R0, R0 RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + BL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index d06c77b948..30f2bd2e4a 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -7,7 +7,7 @@ #include "textflag.h" // func rt0_go() -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // X2 = stack; A0 = argc; A1 = argv ADD $-24, X2 MOV A0, 8(X2) // argc @@ -70,6 +70,10 @@ nocgo: WORD $0 // crash if reached RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + CALL runtime·mstart0(SB) + RET // not reached + // void setg_gcc(G*); set g called from gcc with g in A0 TEXT setg_gcc<>(SB),NOSPLIT,$0-0 MOV A0, g diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index 203754f32c..f9fb1a4c55 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -84,7 +84,7 @@ GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8 DATA _rt0_s90x_lib_argv<>(SB)/8, $0 GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8 -TEXT runtime·rt0_go(SB),NOSPLIT,$0 +TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer // C TLS base pointer in AR0:AR1 @@ -170,6 +170,10 @@ TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 RET +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + CALL runtime·mstart0(SB) + RET // not reached + /* * go-routine */ diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index 3765c756b3..33c335ba5a 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -7,7 +7,7 @@ #include "funcdata.h" #include "textflag.h" -TEXT runtime·rt0_go(SB), NOSPLIT|NOFRAME, $0 +TEXT runtime·rt0_go(SB), NOSPLIT|NOFRAME|TOPFRAME, $0 // save m->g0 = g0 MOVD $runtime·g0(SB), runtime·m0+m_g0(SB) // save m0 to g0->m @@ -24,6 +24,10 @@ TEXT runtime·rt0_go(SB), NOSPLIT|NOFRAME, $0 CALL runtime·mstart(SB) // WebAssembly stack will unwind when switching to another goroutine UNDEF +TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 + CALL runtime·mstart0(SB) + RET // not reached + DATA runtime·mainPC+0(SB)/8,$runtime·main(SB) GLOBL runtime·mainPC(SB),RODATA,$8 @@ -424,7 +428,7 @@ CALLFN(·call268435456, 268435456) CALLFN(·call536870912, 536870912) CALLFN(·call1073741824, 1073741824) -TEXT runtime·goexit(SB), NOSPLIT, $0-0 +TEXT runtime·goexit(SB), NOSPLIT|TOPFRAME, $0-0 NOP // first PC of goexit is skipped CALL runtime·goexit1(SB) // does not return UNDEF diff --git a/src/runtime/proc.go b/src/runtime/proc.go index e6670135db..ccfe085691 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1234,7 +1234,10 @@ func mStackIsSystemAllocated() bool { } // mstart is the entry-point for new Ms. -// +// It is written in assembly, marked TOPFRAME, and calls mstart0. +func mstart() + +// mstart0 is the Go entry-point for new Ms. // This must not split the stack because we may not even have stack // bounds set up yet. // @@ -1243,7 +1246,7 @@ func mStackIsSystemAllocated() bool { // //go:nosplit //go:nowritebarrierrec -func mstart() { +func mstart0() { _g_ := getg() osStack := _g_.stack.lo == 0 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index 576dd28771..4f00c58c16 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -174,7 +174,7 @@ TEXT runtime·profileloop(SB),NOSPLIT,$0 ADDL $12, SP JMP CX -TEXT runtime·externalthreadhandler(SB),NOSPLIT,$0 +TEXT runtime·externalthreadhandler(SB),NOSPLIT|TOPFRAME,$0 PUSHL BP MOVL SP, BP PUSHL BX diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 6f2abb5444..aba2811e59 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -215,7 +215,7 @@ TEXT runtime·profileloop(SB),NOSPLIT|NOFRAME,$8 CALL runtime·externalthreadhandler(SB) RET -TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 PUSHQ BP MOVQ SP, BP PUSHQ BX diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 7d134dd3f4..a55f474d39 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -257,7 +257,7 @@ TEXT runtime·profileloop(SB),NOSPLIT|NOFRAME,$0 // 0 | retval | // +----------------+ // -TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME,$0 +TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} SUB $(m__size + g__size + 20), R13 // space for locals MOVW R0, 12(R13) diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 18d8a42854..e2bd968919 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -1002,12 +1002,9 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { // Does f mark the top of a goroutine stack? func topofstack(f funcInfo, g0 bool) bool { - return f.funcID == funcID_goexit || - f.funcID == funcID_mstart || + return f.flag&funcFlag_TOPFRAME != 0 || f.funcID == funcID_mcall || f.funcID == funcID_morestack || - f.funcID == funcID_rt0_go || - f.funcID == funcID_externalthreadhandler || // asmcgocall is TOS on the system stack because it // switches to the system stack, but in this case we // can come back to the regular stack and still want -- cgit v1.3-5-g9baa From 5ecd9e34dfe0491f1d76372e272d782578ad5bdb Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 28 Jan 2021 17:17:38 -0500 Subject: runtime: do not treat mcall as a topofstack I added mcall to this list in 2013 without explaining why. (https://codereview.appspot.com/11085043/diff/61001/src/pkg/runtime/traceback_x86.c) I suspect I was stopping crashes during profiling where the unwind tried to walk up past mcall and got confused. mcall is not something you can unwind past, because it switches stacks, but it's also not something you should expect as a standard top-of-stack frame. So if you do see it during say a garbage collection stack walk, it would be important to crash instead of silently stopping the walk prematurely. This CL removes it from the topofstack list to avoid the silent stop. Now that mcall is detected as SPWRITE, that will stop the unwind (with a crash if encountered during GC, which we want). This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I666487ce24efd72292f2bc3eac7fe0477e16bddd Reviewed-on: https://go-review.googlesource.com/c/go/+/288803 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/symtab.go | 14 ++++++++++++++ src/runtime/traceback.go | 1 - 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index d7da255e43..00f802aaa7 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -337,7 +337,21 @@ const ( type funcFlag uint8 const ( + // TOPFRAME indicates a function that appears at the top of its stack. + // The traceback routine stop at such a function and consider that a + // successful, complete traversal of the stack. + // Examples of TOPFRAME functions include goexit, which appears + // at the top of a user goroutine stack, and mstart, which appears + // at the top of a system goroutine stack. funcFlag_TOPFRAME funcFlag = 1 << iota + + // SPWRITE indicates a function that writes an arbitrary value to SP + // (any write other than adding or subtracting a constant amount). + // The traceback routines cannot encode such changes into the + // pcsp tables, so the function traceback cannot safely unwind past + // SPWRITE functions. Stopping at an SPWRITE function is considered + // to be an incomplete unwinding of the stack. In certain contexts + // (in particular garbage collector stack scans) that is a fatal error. funcFlag_SPWRITE ) diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index e2bd968919..c89a8913ae 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -1003,7 +1003,6 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { // Does f mark the top of a goroutine stack? func topofstack(f funcInfo, g0 bool) bool { return f.flag&funcFlag_TOPFRAME != 0 || - f.funcID == funcID_mcall || f.funcID == funcID_morestack || // asmcgocall is TOS on the system stack because it // switches to the system stack, but in this case we -- cgit v1.3-5-g9baa From 776ee4079a1d5fabd855a05b300aebdc3ea53efb Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 29 Jan 2021 09:05:10 -0500 Subject: runtime: do not treat morestack as a topofstack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I added morestack to this list in 2013 with an explanation that they were needed if we “start a garbage collection on g0 during a stack split or unsplit”. (https://golang.org/cl/11533043) This explanation no longer applies for a handful of reasons, most importantly that if we did stop a stack scan in the middle of a call to morestack, we'd ignore pointers above the split, which would lead to memory corruption. But we don't scan goroutine stacks during morestack now, so that can't happen. If we did see morestack during a GC, that would be a good time to crash the program. The real problem with morestack is during profiling, as noted in the code review conversation during 2013. And in profiling we just need to know to stop and not unwind further, which the new SPWRITE bit will do for us. So remove from topofstack and let the program crash if GC sees morestack and otherwise let the SPWRITE stop morestack unwinding during profiling. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I06d95920b18c599c7c46f64c21028104978215d7 Reviewed-on: https://go-review.googlesource.com/c/go/+/288804 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/traceback.go | 1 - 1 file changed, 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index c89a8913ae..dbc2bddf42 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -1003,7 +1003,6 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { // Does f mark the top of a goroutine stack? func topofstack(f funcInfo, g0 bool) bool { return f.flag&funcFlag_TOPFRAME != 0 || - f.funcID == funcID_morestack || // asmcgocall is TOS on the system stack because it // switches to the system stack, but in this case we // can come back to the regular stack and still want -- cgit v1.3-5-g9baa From a54f7fc0fde79e8edc696de002fe8a73604f077f Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 29 Jan 2021 09:55:03 -0500 Subject: runtime: do not treat asmcgocall as a topofstack on g0 This was added in 2018 to fix a runtime crash during unwind during a unhandled-panic-induced crash. (See https://golang.org/cl/90895 and #23576.) Clearly we cannot unwind past this function, and the change did stop the unwind. But it's not a top-of-stack function, and the real issue is that SP is changed. The new SPWRITE bit takes care of this instead, so we can drop it from the topofstack function. At this point the topofstack function is only checking the TOPFRAME bit, so we can inline that into the one call site. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I856552298032770e48e06c95a20823a1dbd5e38c Reviewed-on: https://go-review.googlesource.com/c/go/+/288805 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/traceback.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index dbc2bddf42..7321790b78 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -209,7 +209,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } } var flr funcInfo - if topofstack(f, gp.m != nil && gp == gp.m.g0) { + if flag&funcFlag_TOPFRAME != 0 { // This function marks the top of the stack. Stop the traceback. frame.lr = 0 flr = funcInfo{} @@ -1000,17 +1000,6 @@ func tracebackHexdump(stk stack, frame *stkframe, bad uintptr) { }) } -// Does f mark the top of a goroutine stack? -func topofstack(f funcInfo, g0 bool) bool { - return f.flag&funcFlag_TOPFRAME != 0 || - // asmcgocall is TOS on the system stack because it - // switches to the system stack, but in this case we - // can come back to the regular stack and still want - // to be able to unwind through the call that appeared - // on the regular stack. - (g0 && f.funcID == funcID_asmcgocall) -} - // isSystemGoroutine reports whether the goroutine g must be omitted // in stack dumps and deadlock detector. This is any goroutine that // starts at a runtime.* entry point, except for runtime.main, -- cgit v1.3-5-g9baa From ece954d8b8e13a76de891c8078c27c5e7f884f9f Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:06:04 -0500 Subject: runtime: find g in Windows profiler using SP The architecture-specific interpretation of m->tls[0] is unnecessary and fragile. Delete it. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I927345e52fa2f1741d4914478a29d1fb8acb0dc3 Reviewed-on: https://go-review.googlesource.com/c/go/+/288806 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Jason A. Donenfeld Reviewed-by: Cherry Zhang --- src/runtime/os_windows.go | 22 +++++++++++----------- src/runtime/sys_windows_arm.s | 34 ---------------------------------- 2 files changed, 11 insertions(+), 45 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index a2a124cd9d..a8406460e2 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -1132,21 +1132,21 @@ func profilem(mp *m, thread uintptr) { c.contextflags = _CONTEXT_CONTROL stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) - gp := gFromTLS(mp) + gp := gFromSP(mp, c.sp()) sigprof(c.ip(), c.sp(), c.lr(), gp, mp) } -func gFromTLS(mp *m) *g { - switch GOARCH { - case "arm": - tls := &mp.tls[0] - return **((***g)(unsafe.Pointer(tls))) - case "386", "amd64": - tls := &mp.tls[0] - return *((**g)(unsafe.Pointer(tls))) +func gFromSP(mp *m, sp uintptr) *g { + if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { + return gp + } + if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { + return gp + } + if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { + return gp } - throw("unsupported architecture") return nil } @@ -1295,7 +1295,7 @@ func preemptM(mp *m) { unlock(&suspendLock) // Does it want a preemption and is it safe to preempt? - gp := gFromTLS(mp) + gp := gFromSP(mp, c.sp()) if wantAsyncPreempt(gp) { if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok { // Inject call to asyncPreempt diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index a55f474d39..a30d63513a 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -350,9 +350,6 @@ TEXT runtime·tstart_stdcall(SB),NOSPLIT|NOFRAME,$0 MOVW R0, g_m(g) BL runtime·save_g(SB) - // do per-thread TLS initialization - BL init_thread_tls<>(SB) - // Layout new m scheduler stack on os stack. MOVW R13, R0 MOVW R0, g_stack+stack_hi(g) @@ -581,39 +578,8 @@ TEXT runtime·_initcgo(SB),NOSPLIT|NOFRAME,$0 MOVW $runtime·tls_g(SB), R1 MOVW R0, (R1) - BL init_thread_tls<>(SB) - MOVW R4, R13 MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} -// void init_thread_tls() -// -// Does per-thread TLS initialization. Saves a pointer to the TLS slot -// holding G, in the current m. -// -// g->m->tls[0] = &_TEB->TlsSlots[tls_g] -// -// The purpose of this is to enable the profiling handler to get the -// current g associated with the thread. We cannot use m->curg because curg -// only holds the current user g. If the thread is executing system code or -// external code, m->curg will be NULL. The thread's TLS slot always holds -// the current g, so save a reference to this location so the profiling -// handler can get the real g from the thread's m. -// -// Clobbers R0-R3 -TEXT init_thread_tls<>(SB),NOSPLIT|NOFRAME,$0 - // compute &_TEB->TlsSlots[tls_g] - MRC 15, 0, R0, C13, C0, 2 - ADD $0xe10, R0 - MOVW $runtime·tls_g(SB), R1 - MOVW (R1), R1 - MOVW R1<<2, R1 - ADD R1, R0 - - // save in g->m->tls[0] - MOVW g_m(g), R1 - MOVW R0, m_tls(R1) - RET - // Holds the TLS Slot, which was allocated by TlsAlloc() GLOBL runtime·tls_g+0(SB), NOPTR, $4 -- cgit v1.3-5-g9baa From 76ab626bfc52fad9ce8c12fac56177ce68ff744b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:01:00 -0500 Subject: runtime: factor common code out of defs_windows_*.go Also give up on the fiction that these files can be regenerated. They contain many manual edits, and they're fairly small anyway. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: Ib4e4e20a43d8beb1d5390fd184160c33607641f6 Reviewed-on: https://go-review.googlesource.com/c/go/+/288807 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Alex Brainman Reviewed-by: Jason A. Donenfeld Reviewed-by: Cherry Zhang --- src/runtime/defs_windows.go | 127 ++++++++++++++++++++------------------ src/runtime/defs_windows_386.go | 84 ++----------------------- src/runtime/defs_windows_amd64.go | 87 ++------------------------ src/runtime/defs_windows_arm.go | 79 +----------------------- src/runtime/signal_windows.go | 4 +- 5 files changed, 78 insertions(+), 303 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/defs_windows.go b/src/runtime/defs_windows.go index 43f358d56a..656fd2b8b6 100644 --- a/src/runtime/defs_windows.go +++ b/src/runtime/defs_windows.go @@ -2,77 +2,82 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build ignore - -/* -Input to cgo. - -GOARCH=amd64 go tool cgo -cdefs defs_windows.go > defs_windows_amd64.h -GOARCH=386 go tool cgo -cdefs defs_windows.go > defs_windows_386.h -*/ +// Windows architecture-independent definitions. package runtime -/* -#include -#include -#include -#include -#include +const ( + _PROT_NONE = 0 + _PROT_READ = 1 + _PROT_WRITE = 2 + _PROT_EXEC = 4 -#ifndef _X86_ -typedef struct {} FLOATING_SAVE_AREA; -#endif -#ifndef _AMD64_ -typedef struct {} M128A; -#endif -*/ -import "C" + _MAP_ANON = 1 + _MAP_PRIVATE = 2 -const ( - PROT_NONE = 0 - PROT_READ = 1 - PROT_WRITE = 2 - PROT_EXEC = 4 + _DUPLICATE_SAME_ACCESS = 0x2 + _THREAD_PRIORITY_HIGHEST = 0x2 - MAP_ANON = 1 - MAP_PRIVATE = 2 + _SIGINT = 0x2 + _SIGTERM = 0xF + _CTRL_C_EVENT = 0x0 + _CTRL_BREAK_EVENT = 0x1 + _CTRL_CLOSE_EVENT = 0x2 + _CTRL_LOGOFF_EVENT = 0x5 + _CTRL_SHUTDOWN_EVENT = 0x6 - DUPLICATE_SAME_ACCESS = C.DUPLICATE_SAME_ACCESS - THREAD_PRIORITY_HIGHEST = C.THREAD_PRIORITY_HIGHEST + _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 + _EXCEPTION_BREAKPOINT = 0x80000003 + _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d + _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e + _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f + _EXCEPTION_FLT_OVERFLOW = 0xc0000091 + _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 + _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 + _EXCEPTION_INT_OVERFLOW = 0xc0000095 - SIGINT = C.SIGINT - SIGTERM = C.SIGTERM - CTRL_C_EVENT = C.CTRL_C_EVENT - CTRL_BREAK_EVENT = C.CTRL_BREAK_EVENT - CTRL_CLOSE_EVENT = C.CTRL_CLOSE_EVENT - CTRL_LOGOFF_EVENT = C.CTRL_LOGOFF_EVENT - CTRL_SHUTDOWN_EVENT = C.CTRL_SHUTDOWN_EVENT + _INFINITE = 0xffffffff + _WAIT_TIMEOUT = 0x102 - CONTEXT_CONTROL = C.CONTEXT_CONTROL - CONTEXT_FULL = C.CONTEXT_FULL + _EXCEPTION_CONTINUE_EXECUTION = -0x1 + _EXCEPTION_CONTINUE_SEARCH = 0x0 +) - EXCEPTION_ACCESS_VIOLATION = C.STATUS_ACCESS_VIOLATION - EXCEPTION_BREAKPOINT = C.STATUS_BREAKPOINT - EXCEPTION_FLT_DENORMAL_OPERAND = C.STATUS_FLOAT_DENORMAL_OPERAND - EXCEPTION_FLT_DIVIDE_BY_ZERO = C.STATUS_FLOAT_DIVIDE_BY_ZERO - EXCEPTION_FLT_INEXACT_RESULT = C.STATUS_FLOAT_INEXACT_RESULT - EXCEPTION_FLT_OVERFLOW = C.STATUS_FLOAT_OVERFLOW - EXCEPTION_FLT_UNDERFLOW = C.STATUS_FLOAT_UNDERFLOW - EXCEPTION_INT_DIVIDE_BY_ZERO = C.STATUS_INTEGER_DIVIDE_BY_ZERO - EXCEPTION_INT_OVERFLOW = C.STATUS_INTEGER_OVERFLOW +type systeminfo struct { + anon0 [4]byte + dwpagesize uint32 + lpminimumapplicationaddress *byte + lpmaximumapplicationaddress *byte + dwactiveprocessormask uintptr + dwnumberofprocessors uint32 + dwprocessortype uint32 + dwallocationgranularity uint32 + wprocessorlevel uint16 + wprocessorrevision uint16 +} - INFINITE = C.INFINITE - WAIT_TIMEOUT = C.WAIT_TIMEOUT +type exceptionrecord struct { + exceptioncode uint32 + exceptionflags uint32 + exceptionrecord *exceptionrecord + exceptionaddress *byte + numberparameters uint32 + exceptioninformation [15]uintptr +} - EXCEPTION_CONTINUE_EXECUTION = C.EXCEPTION_CONTINUE_EXECUTION - EXCEPTION_CONTINUE_SEARCH = C.EXCEPTION_CONTINUE_SEARCH -) +type overlapped struct { + internal uintptr + internalhigh uintptr + anon0 [8]byte + hevent *byte +} -type SystemInfo C.SYSTEM_INFO -type ExceptionRecord C.EXCEPTION_RECORD -type FloatingSaveArea C.FLOATING_SAVE_AREA -type M128a C.M128A -type Context C.CONTEXT -type Overlapped C.OVERLAPPED -type MemoryBasicInformation C.MEMORY_BASIC_INFORMATION +type memoryBasicInformation struct { + baseAddress uintptr + allocationBase uintptr + allocationProtect uint32 + regionSize uintptr + state uint32 + protect uint32 + type_ uint32 +} diff --git a/src/runtime/defs_windows_386.go b/src/runtime/defs_windows_386.go index 3c5057b86f..37fe74c542 100644 --- a/src/runtime/defs_windows_386.go +++ b/src/runtime/defs_windows_386.go @@ -1,69 +1,10 @@ -// created by cgo -cdefs and then converted to Go -// cgo -cdefs defs_windows.go +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package runtime -const ( - _PROT_NONE = 0 - _PROT_READ = 1 - _PROT_WRITE = 2 - _PROT_EXEC = 4 - - _MAP_ANON = 1 - _MAP_PRIVATE = 2 - - _DUPLICATE_SAME_ACCESS = 0x2 - _THREAD_PRIORITY_HIGHEST = 0x2 - - _SIGINT = 0x2 - _SIGTERM = 0xF - _CTRL_C_EVENT = 0x0 - _CTRL_BREAK_EVENT = 0x1 - _CTRL_CLOSE_EVENT = 0x2 - _CTRL_LOGOFF_EVENT = 0x5 - _CTRL_SHUTDOWN_EVENT = 0x6 - - _CONTEXT_CONTROL = 0x10001 - _CONTEXT_FULL = 0x10007 - - _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 - _EXCEPTION_BREAKPOINT = 0x80000003 - _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d - _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e - _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f - _EXCEPTION_FLT_OVERFLOW = 0xc0000091 - _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 - _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 - _EXCEPTION_INT_OVERFLOW = 0xc0000095 - - _INFINITE = 0xffffffff - _WAIT_TIMEOUT = 0x102 - - _EXCEPTION_CONTINUE_EXECUTION = -0x1 - _EXCEPTION_CONTINUE_SEARCH = 0x0 -) - -type systeminfo struct { - anon0 [4]byte - dwpagesize uint32 - lpminimumapplicationaddress *byte - lpmaximumapplicationaddress *byte - dwactiveprocessormask uint32 - dwnumberofprocessors uint32 - dwprocessortype uint32 - dwallocationgranularity uint32 - wprocessorlevel uint16 - wprocessorrevision uint16 -} - -type exceptionrecord struct { - exceptioncode uint32 - exceptionflags uint32 - exceptionrecord *exceptionrecord - exceptionaddress *byte - numberparameters uint32 - exceptioninformation [15]uint32 -} +const _CONTEXT_CONTROL = 0x10001 type floatingsavearea struct { controlword uint32 @@ -130,20 +71,3 @@ func dumpregs(r *context) { print("fs ", hex(r.segfs), "\n") print("gs ", hex(r.seggs), "\n") } - -type overlapped struct { - internal uint32 - internalhigh uint32 - anon0 [8]byte - hevent *byte -} - -type memoryBasicInformation struct { - baseAddress uintptr - allocationBase uintptr - allocationProtect uint32 - regionSize uintptr - state uint32 - protect uint32 - type_ uint32 -} diff --git a/src/runtime/defs_windows_amd64.go b/src/runtime/defs_windows_amd64.go index ebb1506e2f..ac636a68ec 100644 --- a/src/runtime/defs_windows_amd64.go +++ b/src/runtime/defs_windows_amd64.go @@ -1,70 +1,10 @@ -// created by cgo -cdefs and then converted to Go -// cgo -cdefs defs_windows.go +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. package runtime -const ( - _PROT_NONE = 0 - _PROT_READ = 1 - _PROT_WRITE = 2 - _PROT_EXEC = 4 - - _MAP_ANON = 1 - _MAP_PRIVATE = 2 - - _DUPLICATE_SAME_ACCESS = 0x2 - _THREAD_PRIORITY_HIGHEST = 0x2 - - _SIGINT = 0x2 - _SIGTERM = 0xF - _CTRL_C_EVENT = 0x0 - _CTRL_BREAK_EVENT = 0x1 - _CTRL_CLOSE_EVENT = 0x2 - _CTRL_LOGOFF_EVENT = 0x5 - _CTRL_SHUTDOWN_EVENT = 0x6 - - _CONTEXT_CONTROL = 0x100001 - _CONTEXT_FULL = 0x10000b - - _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 - _EXCEPTION_BREAKPOINT = 0x80000003 - _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d - _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e - _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f - _EXCEPTION_FLT_OVERFLOW = 0xc0000091 - _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 - _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 - _EXCEPTION_INT_OVERFLOW = 0xc0000095 - - _INFINITE = 0xffffffff - _WAIT_TIMEOUT = 0x102 - - _EXCEPTION_CONTINUE_EXECUTION = -0x1 - _EXCEPTION_CONTINUE_SEARCH = 0x0 -) - -type systeminfo struct { - anon0 [4]byte - dwpagesize uint32 - lpminimumapplicationaddress *byte - lpmaximumapplicationaddress *byte - dwactiveprocessormask uint64 - dwnumberofprocessors uint32 - dwprocessortype uint32 - dwallocationgranularity uint32 - wprocessorlevel uint16 - wprocessorrevision uint16 -} - -type exceptionrecord struct { - exceptioncode uint32 - exceptionflags uint32 - exceptionrecord *exceptionrecord - exceptionaddress *byte - numberparameters uint32 - pad_cgo_0 [4]byte - exceptioninformation [15]uint64 -} +const _CONTEXT_CONTROL = 0x100001 type m128a struct { low uint64 @@ -123,7 +63,7 @@ type context struct { func (c *context) ip() uintptr { return uintptr(c.rip) } func (c *context) sp() uintptr { return uintptr(c.rsp) } -// Amd64 does not have link register, so this returns 0. +// AMD64 does not have link register, so this returns 0. func (c *context) lr() uintptr { return 0 } func (c *context) set_lr(x uintptr) {} @@ -152,20 +92,3 @@ func dumpregs(r *context) { print("fs ", hex(r.segfs), "\n") print("gs ", hex(r.seggs), "\n") } - -type overlapped struct { - internal uint64 - internalhigh uint64 - anon0 [8]byte - hevent *byte -} - -type memoryBasicInformation struct { - baseAddress uintptr - allocationBase uintptr - allocationProtect uint32 - regionSize uintptr - state uint32 - protect uint32 - type_ uint32 -} diff --git a/src/runtime/defs_windows_arm.go b/src/runtime/defs_windows_arm.go index b275b0572a..4021f77ba8 100644 --- a/src/runtime/defs_windows_arm.go +++ b/src/runtime/defs_windows_arm.go @@ -4,67 +4,7 @@ package runtime -const ( - _PROT_NONE = 0 - _PROT_READ = 1 - _PROT_WRITE = 2 - _PROT_EXEC = 4 - - _MAP_ANON = 1 - _MAP_PRIVATE = 2 - - _DUPLICATE_SAME_ACCESS = 0x2 - _THREAD_PRIORITY_HIGHEST = 0x2 - - _SIGINT = 0x2 - _SIGTERM = 0xF - _CTRL_C_EVENT = 0x0 - _CTRL_BREAK_EVENT = 0x1 - _CTRL_CLOSE_EVENT = 0x2 - _CTRL_LOGOFF_EVENT = 0x5 - _CTRL_SHUTDOWN_EVENT = 0x6 - - _CONTEXT_CONTROL = 0x10001 - _CONTEXT_FULL = 0x10007 - - _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 - _EXCEPTION_BREAKPOINT = 0x80000003 - _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d - _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e - _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f - _EXCEPTION_FLT_OVERFLOW = 0xc0000091 - _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 - _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 - _EXCEPTION_INT_OVERFLOW = 0xc0000095 - - _INFINITE = 0xffffffff - _WAIT_TIMEOUT = 0x102 - - _EXCEPTION_CONTINUE_EXECUTION = -0x1 - _EXCEPTION_CONTINUE_SEARCH = 0x0 -) - -type systeminfo struct { - anon0 [4]byte - dwpagesize uint32 - lpminimumapplicationaddress *byte - lpmaximumapplicationaddress *byte - dwactiveprocessormask uint32 - dwnumberofprocessors uint32 - dwprocessortype uint32 - dwallocationgranularity uint32 - wprocessorlevel uint16 - wprocessorrevision uint16 -} - -type exceptionrecord struct { - exceptioncode uint32 - exceptionflags uint32 - exceptionrecord *exceptionrecord - exceptionaddress *byte - numberparameters uint32 - exceptioninformation [15]uint32 -} +const _CONTEXT_CONTROL = 0x10001 type neon128 struct { low uint64 @@ -132,23 +72,6 @@ func dumpregs(r *context) { print("cpsr ", hex(r.cpsr), "\n") } -type overlapped struct { - internal uint32 - internalhigh uint32 - anon0 [8]byte - hevent *byte -} - -type memoryBasicInformation struct { - baseAddress uintptr - allocationBase uintptr - allocationProtect uint32 - regionSize uintptr - state uint32 - protect uint32 - type_ uint32 -} - func stackcheck() { // TODO: not implemented on ARM } diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index 3af2e39b08..89d12617f4 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -112,8 +112,8 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { // augmenting the stack frame would break // the unwinding code. gp.sig = info.exceptioncode - gp.sigcode0 = uintptr(info.exceptioninformation[0]) - gp.sigcode1 = uintptr(info.exceptioninformation[1]) + gp.sigcode0 = info.exceptioninformation[0] + gp.sigcode1 = info.exceptioninformation[1] gp.sigpc = r.ip() // Only push runtime·sigpanic if r.ip() != 0. -- cgit v1.3-5-g9baa From 75e273fc2c183896a11bf23f0688c38059933336 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:02:33 -0500 Subject: runtime: fix windows/arm CONTEXT_CONTROL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The constant was wrong, and the “right” constant doesn't work either. But with the actually-right constant (and possibly earlier fixes in this stack as well), profiling now works. Change-Id: If8caff1da556826db40961fb9bcfe2b1f31ea9f9 Reviewed-on: https://go-review.googlesource.com/c/go/+/288808 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/runtime/defs_windows_arm.go | 8 +++++++- src/runtime/pprof/pprof_test.go | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/defs_windows_arm.go b/src/runtime/defs_windows_arm.go index 4021f77ba8..370470e35d 100644 --- a/src/runtime/defs_windows_arm.go +++ b/src/runtime/defs_windows_arm.go @@ -4,7 +4,13 @@ package runtime -const _CONTEXT_CONTROL = 0x10001 +// NOTE(rsc): _CONTEXT_CONTROL is actually 0x200001 and should include PC, SP, and LR. +// However, empirically, LR doesn't come along on Windows 10 +// unless you also set _CONTEXT_INTEGER (0x200002). +// Without LR, we skip over the next-to-bottom function in profiles +// when the bottom function is frameless. +// So we set both here, to make a working _CONTEXT_CONTROL. +const _CONTEXT_CONTROL = 0x200003 type neon128 struct { low uint64 diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index 37f12de0d9..f7c1349bc6 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -285,10 +285,6 @@ func testCPUProfile(t *testing.T, matches matchFunc, need []string, avoid []stri if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" { broken = true } - case "windows": - if runtime.GOARCH == "arm" { - broken = true // See https://golang.org/issues/42862 - } } maxDuration := 5 * time.Second -- cgit v1.3-5-g9baa From a1e9148e3dbb20a18e0139583e7d835cc7a820bf Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Tue, 26 Jan 2021 21:26:01 -0500 Subject: runtime: print hex numbers with hex prefixes in traceback debug If traceback fails, it prints a helpful hex dump of the stack. But the hex numbers have no 0x prefix, which might make it a little unclear that they are hex. We only print two per line, so there is plenty of room for the 0x. Print it, which lets us delete a custom hex formatter. Also, in the translated hints, print off in hex (with a 0x prefix). The offsets were previously decimal, which could have been confused for hex since none of the hex had 0x prefixes. And decimal is kind of useless anyway since the offsets shown in the main traceback are hex, so you can't easily match them up without mental base conversions. Just print hex everywhere, clearly marked by 0x. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I72d26a4e41ada38b620bf8fe3576d787a2e59b47 Reviewed-on: https://go-review.googlesource.com/c/go/+/288809 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/print.go | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/print.go b/src/runtime/print.go index 64055a34cc..f15296cf02 100644 --- a/src/runtime/print.go +++ b/src/runtime/print.go @@ -216,13 +216,15 @@ func printint(v int64) { printuint(uint64(v)) } +var minhexdigits = 0 // protected by printlock + func printhex(v uint64) { const dig = "0123456789abcdef" var buf [100]byte i := len(buf) for i--; i > 0; i-- { buf[i] = dig[v%16] - if v < 16 { + if v < 16 && len(buf)-i >= minhexdigits { break } v /= 16 @@ -265,29 +267,16 @@ func printiface(i iface) { // and should return a character mark to appear just before that // word's value. It can return 0 to indicate no mark. func hexdumpWords(p, end uintptr, mark func(uintptr) byte) { - p1 := func(x uintptr) { - var buf [2 * sys.PtrSize]byte - for i := len(buf) - 1; i >= 0; i-- { - if x&0xF < 10 { - buf[i] = byte(x&0xF) + '0' - } else { - buf[i] = byte(x&0xF) - 10 + 'a' - } - x >>= 4 - } - gwrite(buf[:]) - } - printlock() var markbuf [1]byte markbuf[0] = ' ' + minhexdigits = int(unsafe.Sizeof(uintptr(0)) * 2) for i := uintptr(0); p+i < end; i += sys.PtrSize { if i%16 == 0 { if i != 0 { println() } - p1(p + i) - print(": ") + print(hex(p+i), ": ") } if mark != nil { @@ -298,15 +287,16 @@ func hexdumpWords(p, end uintptr, mark func(uintptr) byte) { } gwrite(markbuf[:]) val := *(*uintptr)(unsafe.Pointer(p + i)) - p1(val) + print(hex(val)) print(" ") // Can we symbolize val? fn := findfunc(val) if fn.valid() { - print("<", funcname(fn), "+", val-fn.entry, "> ") + print("<", funcname(fn), "+", hex(val-fn.entry), "> ") } } + minhexdigits = 0 println() printunlock() } -- cgit v1.3-5-g9baa From 38672d3dcf2eae297c45dc2a899c39528148f14b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:15:33 -0500 Subject: runtime: crash earlier on windows for runtime.abort The isAbort check was wrong for non-x86 systems. That was causing the exception chain to be passed back to Windows. That was causing some other kind of fault - not sure what. That was leading back to lastcontinuehandler to print a larger stack trace, and then the throwing = 1 print added runtime.abort, which made TestAbort pass even though it wasn't really working. Recognize abort properly and handle it as Go, not as something for Windows to try to handle. Keep the throwing = 1 print, because more detail on throw is always better. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: If614f4ab2884bd90410d29e28311bf969ceeac09 Reviewed-on: https://go-review.googlesource.com/c/go/+/288810 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/signal_windows.go | 45 +++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 21 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index 89d12617f4..18834b0ec5 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -43,13 +43,17 @@ func initExceptionHandler() { // //go:nosplit func isAbort(r *context) bool { - // In the case of an abort, the exception IP is one byte after - // the INT3 (this differs from UNIX OSes). - return isAbortPC(r.ip() - 1) + pc := r.ip() + if GOARCH == "386" || GOARCH == "amd64" { + // In the case of an abort, the exception IP is one byte after + // the INT3 (this differs from UNIX OSes). + pc-- + } + return isAbortPC(pc) } // isgoexception reports whether this exception should be translated -// into a Go panic. +// into a Go panic or throw. // // It is nosplit to avoid growing the stack in case we're aborting // because of a stack overflow. @@ -63,11 +67,6 @@ func isgoexception(info *exceptionrecord, r *context) bool { return false } - if isAbort(r) { - // Never turn abort into a panic. - return false - } - // Go will only handle some exceptions. switch info.exceptioncode { default: @@ -99,14 +98,16 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { return _EXCEPTION_CONTINUE_SEARCH } - // After this point, it is safe to grow the stack. - - if gp.throwsplit { - // We can't safely sigpanic because it may grow the - // stack. Let it fall through. - return _EXCEPTION_CONTINUE_SEARCH + if gp.throwsplit || isAbort(r) { + // We can't safely sigpanic because it may grow the stack. + // Or this is a call to abort. + // Don't go through any more of the Windows handler chain. + // Crash now. + winthrow(info, r, gp) } + // After this point, it is safe to grow the stack. + // Make it look like a call to the signal func. // Have to pass arguments out of band since // augmenting the stack frame would break @@ -181,6 +182,12 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { return _EXCEPTION_CONTINUE_SEARCH } + winthrow(info, r, gp) + return 0 // not reached +} + +//go:nosplit +func winthrow(info *exceptionrecord, r *context, gp *g) { _g_ := getg() if panicking != 0 { // traceback already printed @@ -206,11 +213,8 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { } print("\n") - // TODO(jordanrh1): This may be needed for 386/AMD64 as well. - if GOARCH == "arm" { - _g_.m.throwing = 1 - _g_.m.caughtsig.set(gp) - } + _g_.m.throwing = 1 + _g_.m.caughtsig.set(gp) level, _, docrash := gotraceback() if level > 0 { @@ -224,7 +228,6 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { } exit(2) - return 0 // not reached } func sigpanic() { -- cgit v1.3-5-g9baa From 91cc484ea914fc75e7321d23017d59c9751f5066 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:22:02 -0500 Subject: runtime: fix time on windows/arm under WINE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This code has clearly never run successfully, since one of the “tail calls" calls the wrong function, and both of them appear in functions with stack frames that are never going to be properly unwound. Probably there is no windows/arm under WINE at all. But might as well fix the code. Change-Id: I5fa62274b3661bc6bce098657b5bcf11d59655eb Reviewed-on: https://go-review.googlesource.com/c/go/+/288811 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/runtime/sys_windows_arm.s | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index a30d63513a..42278dcbe7 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -419,7 +419,7 @@ TEXT runtime·read_tls_fallback(SB),NOSPLIT|NOFRAME,$0 #define time_hi1 4 #define time_hi2 8 -TEXT runtime·nanotime1(SB),NOSPLIT,$0-8 +TEXT runtime·nanotime1(SB),NOSPLIT|NOFRAME,$0-8 MOVW $0, R0 MOVB runtime·useQPCTime(SB), R0 CMP $0, R0 @@ -443,9 +443,8 @@ loop: RET useQPC: B runtime·nanotimeQPC(SB) // tail call - RET -TEXT time·now(SB),NOSPLIT,$0-20 +TEXT time·now(SB),NOSPLIT|NOFRAME,$0-20 MOVW $0, R0 MOVB runtime·useQPCTime(SB), R0 CMP $0, R0 @@ -519,8 +518,7 @@ wall: MOVW R1,nsec+8(FP) RET useQPC: - B runtime·nanotimeQPC(SB) // tail call - RET + B runtime·nowQPC(SB) // tail call // save_g saves the g register (R10) into thread local memory // so that we can call externally compiled -- cgit v1.3-5-g9baa From 5421c37a1db5098659f86b21d011fc263d93524e Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:27:25 -0500 Subject: runtime: fix windows/arm externalthreadhandler Externalthreadhandler was not handling its own stack correctly. It incorrectly referred to the saved LR slot (uninitialized, it turned out) as holding the return value from the called function. Externalthreadhandler is used to call two different functions: profileloop1 and ctrlhandler1. Profileloop1 does not return, so no harm done. Ctrlhandler1 returns a boolean indicating whether the handler took care of the control event (if true, no other handlers run). It's hard to say exactly what uninitialized values are likely to have been returned instead of ctrlhandler1's result, but it probably wasn't helping matters. Change-Id: Ia02f1c033df618cb82c2193b3a8241ed048a8b18 Reviewed-on: https://go-review.googlesource.com/c/go/+/288812 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Jason A. Donenfeld --- src/runtime/sys_windows_arm.s | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 42278dcbe7..3f01714c66 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -250,16 +250,17 @@ TEXT runtime·profileloop(SB),NOSPLIT|NOFRAME,$0 // +----------------+ // 12| argument (r0) | //---+----------------+ -// 8 | param1 | +// 8 | param1 | (also return value for called Go function) // +----------------+ // 4 | param0 | // +----------------+ -// 0 | retval | +// 0 | slot for LR | // +----------------+ // TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} SUB $(m__size + g__size + 20), R13 // space for locals + MOVW R14, 0(R13) // push LR again for anything unwinding the stack MOVW R0, 12(R13) MOVW R1, 16(R13) @@ -298,7 +299,7 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 MOVW $0, g BL runtime·save_g(SB) - MOVW 0(R13), R0 // load return value + MOVW 8(R13), R0 // load return value ADD $(m__size + g__size + 20), R13 // free locals MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} -- cgit v1.3-5-g9baa From b19e7b518e564cd309d3eb68dfd2da8839a7433b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 28 Jan 2021 09:23:35 -0500 Subject: runtime: clean up windows a bit Document the various hard-coded architecture checks or remove them in favor of more general checks. This should be a no-op now but will make the arm64 port have fewer diffs. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: Ifd6b19e44e8c9ca4a0d2590f314928ce235821b3 Reviewed-on: https://go-review.googlesource.com/c/go/+/288813 Trust: Russ Cox Reviewed-by: Cherry Zhang Reviewed-by: Alex Brainman --- src/runtime/os_windows.go | 12 ++++++++---- src/runtime/signal_windows.go | 13 ++++++------- src/runtime/syscall_windows.go | 3 ++- 3 files changed, 16 insertions(+), 12 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index a8406460e2..375c34ed99 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -236,6 +236,8 @@ func windowsLoadSystemLib(name []byte) uintptr { } } +const haveCputicksAsm = GOARCH == "386" || GOARCH == "amd64" + func loadOptionalSyscalls() { var kernel32dll = []byte("kernel32.dll\000") k32 := stdcall1(_LoadLibraryA, uintptr(unsafe.Pointer(&kernel32dll[0]))) @@ -262,7 +264,7 @@ func loadOptionalSyscalls() { } _NtWaitForSingleObject = windowsFindfunc(n32, []byte("NtWaitForSingleObject\000")) - if GOARCH == "arm" { + if !haveCputicksAsm { _QueryPerformanceCounter = windowsFindfunc(k32, []byte("QueryPerformanceCounter\000")) if _QueryPerformanceCounter == nil { throw("could not find QPC syscalls") @@ -452,8 +454,10 @@ func createHighResTimer() uintptr { _SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE) } +const highResTimerSupported = GOARCH == "386" || GOARCH == "amd64" + func initHighResTimer() { - if GOARCH == "arm" { + if !highResTimerSupported { // TODO: Not yet implemented. return } @@ -1217,14 +1221,14 @@ func setThreadCPUProfiler(hz int32) { atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) } -const preemptMSupported = GOARCH != "arm" +const preemptMSupported = GOARCH == "386" || GOARCH == "amd64" // suspendLock protects simultaneous SuspendThread operations from // suspending each other. var suspendLock mutex func preemptM(mp *m) { - if GOARCH == "arm" { + if !preemptMSupported { // TODO: Implement call injection return } diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index 18834b0ec5..cb1fbe9f81 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -5,6 +5,7 @@ package runtime import ( + "runtime/internal/sys" "unsafe" ) @@ -132,16 +133,14 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { // overwrite the PC. (See issue #35773) if r.ip() != 0 && r.ip() != funcPC(asyncPreempt) { sp := unsafe.Pointer(r.sp()) - sp = add(sp, ^(unsafe.Sizeof(uintptr(0)) - 1)) // sp-- + delta := uintptr(sys.StackAlign) + sp = add(sp, -delta) r.set_sp(uintptr(sp)) - switch GOARCH { - default: - panic("unsupported architecture") - case "386", "amd64": - *((*uintptr)(sp)) = r.ip() - case "arm": + if usesLR { *((*uintptr)(sp)) = r.lr() r.set_lr(r.ip()) + } else { + *((*uintptr)(sp)) = r.ip() } } r.set_ip(funcPC(sigpanic)) diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index add40bb0b3..6052cc333c 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -116,13 +116,14 @@ func compileCallback(fn eface, cdecl bool) (code uintptr) { // registers and the stack. panic("compileCallback: argument size is larger than uintptr") } - if k := t.kind & kindMask; (GOARCH == "amd64" || GOARCH == "arm") && (k == kindFloat32 || k == kindFloat64) { + if k := t.kind & kindMask; GOARCH != "386" && (k == kindFloat32 || k == kindFloat64) { // In fastcall, floating-point arguments in // the first four positions are passed in // floating-point registers, which we don't // currently spill. arm passes floating-point // arguments in VFP registers, which we also // don't support. + // So basically we only support 386. panic("compileCallback: float arguments not supported") } -- cgit v1.3-5-g9baa From 09e059afb1270498f416f5b5c75a6a5683b6d1da Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 27 Jan 2021 11:34:42 -0500 Subject: runtime: enable framepointer on all arm64 Frame pointers were already enabled on linux, darwin, ios, but not freebsd, android, openbsd, netbsd. But the space was reserved on all platforms, leading to two different arm64 framepointer conditions in different parts of the code, one of which had no name (framepointer_enabled || GOARCH == "arm64", which might have been "framepointer_space_reserved"). So on the disabled systems, the stack layouts were still set up for frame pointers and the only difference was not actually maintaining the FP register in the generated code. Reduce complexity by just enabling the frame pointer completely on all the arm64 systems. This commit passes on freebsd, android, netbsd. I have not been able to try it on openbsd. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. This CL is, however, not windows/arm64-specific. It is cleanup meant to make the port (and future ports) easier. Change-Id: I83bd23369d24b76db4c6a648fa74f6917819a093 Reviewed-on: https://go-review.googlesource.com/c/go/+/288814 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/dwarfgen/dwarf.go | 3 +- src/cmd/compile/internal/ssagen/pgen.go | 3 +- src/cmd/internal/obj/arm64/a.out.go | 2 +- src/cmd/internal/obj/arm64/obj7.go | 293 +++++++++++++++-------------- src/cmd/internal/objabi/util.go | 2 +- src/runtime/runtime2.go | 2 +- src/runtime/traceback.go | 17 +- 7 files changed, 175 insertions(+), 147 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/dwarfgen/dwarf.go b/src/cmd/compile/internal/dwarfgen/dwarf.go index dd22c033cc..70168cffeb 100644 --- a/src/cmd/compile/internal/dwarfgen/dwarf.go +++ b/src/cmd/compile/internal/dwarfgen/dwarf.go @@ -271,8 +271,7 @@ func createSimpleVar(fnsym *obj.LSym, n *ir.Name) *dwarf.Var { if base.Ctxt.FixedFrameSize() == 0 { offs -= int64(types.PtrSize) } - if objabi.Framepointer_enabled || objabi.GOARCH == "arm64" { - // There is a word space for FP on ARM64 even if the frame pointer is disabled + if objabi.Framepointer_enabled { offs -= int64(types.PtrSize) } diff --git a/src/cmd/compile/internal/ssagen/pgen.go b/src/cmd/compile/internal/ssagen/pgen.go index 182f8408cf..40f07a8d45 100644 --- a/src/cmd/compile/internal/ssagen/pgen.go +++ b/src/cmd/compile/internal/ssagen/pgen.go @@ -213,8 +213,7 @@ func StackOffset(slot ssa.LocalSlot) int32 { if base.Ctxt.FixedFrameSize() == 0 { off -= int64(types.PtrSize) } - if objabi.Framepointer_enabled || objabi.GOARCH == "arm64" { - // There is a word space for FP on ARM64 even if the frame pointer is disabled + if objabi.Framepointer_enabled { off -= int64(types.PtrSize) } case ir.PPARAM, ir.PPARAMOUT: diff --git a/src/cmd/internal/obj/arm64/a.out.go b/src/cmd/internal/obj/arm64/a.out.go index 1d1bea505c..7ab9c1475f 100644 --- a/src/cmd/internal/obj/arm64/a.out.go +++ b/src/cmd/internal/obj/arm64/a.out.go @@ -239,7 +239,7 @@ const ( REGCTXT = REG_R26 // environment for closures REGTMP = REG_R27 // reserved for liblink REGG = REG_R28 // G - REGFP = REG_R29 // frame pointer, unused in the Go toolchain + REGFP = REG_R29 // frame pointer REGLINK = REG_R30 // ARM64 uses R31 as both stack pointer and zero register, diff --git a/src/cmd/internal/obj/arm64/obj7.go b/src/cmd/internal/obj/arm64/obj7.go index 3b88543852..8f7648e5d5 100644 --- a/src/cmd/internal/obj/arm64/obj7.go +++ b/src/cmd/internal/obj/arm64/obj7.go @@ -622,25 +622,24 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { prologueEnd.Pos = prologueEnd.Pos.WithXlogue(src.PosPrologueEnd) - if objabi.Framepointer_enabled { - q1 = obj.Appendp(q1, c.newprog) - q1.Pos = p.Pos - q1.As = AMOVD - q1.From.Type = obj.TYPE_REG - q1.From.Reg = REGFP - q1.To.Type = obj.TYPE_MEM - q1.To.Reg = REGSP - q1.To.Offset = -8 - - q1 = obj.Appendp(q1, c.newprog) - q1.Pos = p.Pos - q1.As = ASUB - q1.From.Type = obj.TYPE_CONST - q1.From.Offset = 8 - q1.Reg = REGSP - q1.To.Type = obj.TYPE_REG - q1.To.Reg = REGFP - } + // Frame pointer. + q1 = obj.Appendp(q1, c.newprog) + q1.Pos = p.Pos + q1.As = AMOVD + q1.From.Type = obj.TYPE_REG + q1.From.Reg = REGFP + q1.To.Type = obj.TYPE_MEM + q1.To.Reg = REGSP + q1.To.Offset = -8 + + q1 = obj.Appendp(q1, c.newprog) + q1.Pos = p.Pos + q1.As = ASUB + q1.From.Type = obj.TYPE_CONST + q1.From.Offset = 8 + q1.Reg = REGSP + q1.To.Type = obj.TYPE_REG + q1.To.Reg = REGFP if c.cursym.Func().Text.From.Sym.Wrapper() { // if(g->panic != nil && g->panic->argp == FP) g->panic->argp = bottom-of-frame @@ -765,28 +764,26 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { p.To.Reg = REGSP p.Spadj = -c.autosize - if objabi.Framepointer_enabled { - p = obj.Appendp(p, c.newprog) - p.As = ASUB - p.From.Type = obj.TYPE_CONST - p.From.Offset = 8 - p.Reg = REGSP - p.To.Type = obj.TYPE_REG - p.To.Reg = REGFP - } + // Frame pointer. + p = obj.Appendp(p, c.newprog) + p.As = ASUB + p.From.Type = obj.TYPE_CONST + p.From.Offset = 8 + p.Reg = REGSP + p.To.Type = obj.TYPE_REG + p.To.Reg = REGFP } } else { /* want write-back pre-indexed SP+autosize -> SP, loading REGLINK*/ - if objabi.Framepointer_enabled { - p.As = AMOVD - p.From.Type = obj.TYPE_MEM - p.From.Reg = REGSP - p.From.Offset = -8 - p.To.Type = obj.TYPE_REG - p.To.Reg = REGFP - p = obj.Appendp(p, c.newprog) - } + // Frame pointer. + p.As = AMOVD + p.From.Type = obj.TYPE_MEM + p.From.Reg = REGSP + p.From.Offset = -8 + p.To.Type = obj.TYPE_REG + p.To.Reg = REGFP + p = obj.Appendp(p, c.newprog) aoffset := c.autosize @@ -821,6 +818,28 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { } } + // If enabled, this code emits 'MOV PC, R27' before every 'MOV LR, PC', + // so that if you are debugging a low-level crash where PC and LR are zero, + // you can look at R27 to see what jumped to the zero. + // This is useful when bringing up Go on a new system. + // (There is similar code in ../ppc64/obj9.go:/if.false.) + const debugRETZERO = false + if debugRETZERO { + if p.As != obj.ARET { + q = newprog() + q.Pos = p.Pos + q.Link = p.Link + p.Link = q + p = q + } + p.As = AADR + p.From.Type = obj.TYPE_BRANCH + p.From.Offset = 0 + p.To.Type = obj.TYPE_REG + p.To.Reg = REGTMP + + } + if p.As != obj.ARET { q = newprog() q.Pos = p.Pos @@ -866,110 +885,106 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { } case obj.ADUFFCOPY: - if objabi.Framepointer_enabled { - // ADR ret_addr, R27 - // STP (FP, R27), -24(SP) - // SUB 24, SP, FP - // DUFFCOPY - // ret_addr: - // SUB 8, SP, FP - - q1 := p - // copy DUFFCOPY from q1 to q4 - q4 := obj.Appendp(p, c.newprog) - q4.Pos = p.Pos - q4.As = obj.ADUFFCOPY - q4.To = p.To - - q1.As = AADR - q1.From.Type = obj.TYPE_BRANCH - q1.To.Type = obj.TYPE_REG - q1.To.Reg = REG_R27 - - q2 := obj.Appendp(q1, c.newprog) - q2.Pos = p.Pos - q2.As = ASTP - q2.From.Type = obj.TYPE_REGREG - q2.From.Reg = REGFP - q2.From.Offset = int64(REG_R27) - q2.To.Type = obj.TYPE_MEM - q2.To.Reg = REGSP - q2.To.Offset = -24 - - // maintaine FP for DUFFCOPY - q3 := obj.Appendp(q2, c.newprog) - q3.Pos = p.Pos - q3.As = ASUB - q3.From.Type = obj.TYPE_CONST - q3.From.Offset = 24 - q3.Reg = REGSP - q3.To.Type = obj.TYPE_REG - q3.To.Reg = REGFP - - q5 := obj.Appendp(q4, c.newprog) - q5.Pos = p.Pos - q5.As = ASUB - q5.From.Type = obj.TYPE_CONST - q5.From.Offset = 8 - q5.Reg = REGSP - q5.To.Type = obj.TYPE_REG - q5.To.Reg = REGFP - q1.From.SetTarget(q5) - p = q5 - } + // ADR ret_addr, R27 + // STP (FP, R27), -24(SP) + // SUB 24, SP, FP + // DUFFCOPY + // ret_addr: + // SUB 8, SP, FP + + q1 := p + // copy DUFFCOPY from q1 to q4 + q4 := obj.Appendp(p, c.newprog) + q4.Pos = p.Pos + q4.As = obj.ADUFFCOPY + q4.To = p.To + + q1.As = AADR + q1.From.Type = obj.TYPE_BRANCH + q1.To.Type = obj.TYPE_REG + q1.To.Reg = REG_R27 + + q2 := obj.Appendp(q1, c.newprog) + q2.Pos = p.Pos + q2.As = ASTP + q2.From.Type = obj.TYPE_REGREG + q2.From.Reg = REGFP + q2.From.Offset = int64(REG_R27) + q2.To.Type = obj.TYPE_MEM + q2.To.Reg = REGSP + q2.To.Offset = -24 + + // maintain FP for DUFFCOPY + q3 := obj.Appendp(q2, c.newprog) + q3.Pos = p.Pos + q3.As = ASUB + q3.From.Type = obj.TYPE_CONST + q3.From.Offset = 24 + q3.Reg = REGSP + q3.To.Type = obj.TYPE_REG + q3.To.Reg = REGFP + + q5 := obj.Appendp(q4, c.newprog) + q5.Pos = p.Pos + q5.As = ASUB + q5.From.Type = obj.TYPE_CONST + q5.From.Offset = 8 + q5.Reg = REGSP + q5.To.Type = obj.TYPE_REG + q5.To.Reg = REGFP + q1.From.SetTarget(q5) + p = q5 case obj.ADUFFZERO: - if objabi.Framepointer_enabled { - // ADR ret_addr, R27 - // STP (FP, R27), -24(SP) - // SUB 24, SP, FP - // DUFFZERO - // ret_addr: - // SUB 8, SP, FP - - q1 := p - // copy DUFFZERO from q1 to q4 - q4 := obj.Appendp(p, c.newprog) - q4.Pos = p.Pos - q4.As = obj.ADUFFZERO - q4.To = p.To - - q1.As = AADR - q1.From.Type = obj.TYPE_BRANCH - q1.To.Type = obj.TYPE_REG - q1.To.Reg = REG_R27 - - q2 := obj.Appendp(q1, c.newprog) - q2.Pos = p.Pos - q2.As = ASTP - q2.From.Type = obj.TYPE_REGREG - q2.From.Reg = REGFP - q2.From.Offset = int64(REG_R27) - q2.To.Type = obj.TYPE_MEM - q2.To.Reg = REGSP - q2.To.Offset = -24 - - // maintaine FP for DUFFZERO - q3 := obj.Appendp(q2, c.newprog) - q3.Pos = p.Pos - q3.As = ASUB - q3.From.Type = obj.TYPE_CONST - q3.From.Offset = 24 - q3.Reg = REGSP - q3.To.Type = obj.TYPE_REG - q3.To.Reg = REGFP - - q5 := obj.Appendp(q4, c.newprog) - q5.Pos = p.Pos - q5.As = ASUB - q5.From.Type = obj.TYPE_CONST - q5.From.Offset = 8 - q5.Reg = REGSP - q5.To.Type = obj.TYPE_REG - q5.To.Reg = REGFP - q1.From.SetTarget(q5) - p = q5 - } + // ADR ret_addr, R27 + // STP (FP, R27), -24(SP) + // SUB 24, SP, FP + // DUFFZERO + // ret_addr: + // SUB 8, SP, FP + + q1 := p + // copy DUFFZERO from q1 to q4 + q4 := obj.Appendp(p, c.newprog) + q4.Pos = p.Pos + q4.As = obj.ADUFFZERO + q4.To = p.To + + q1.As = AADR + q1.From.Type = obj.TYPE_BRANCH + q1.To.Type = obj.TYPE_REG + q1.To.Reg = REG_R27 + + q2 := obj.Appendp(q1, c.newprog) + q2.Pos = p.Pos + q2.As = ASTP + q2.From.Type = obj.TYPE_REGREG + q2.From.Reg = REGFP + q2.From.Offset = int64(REG_R27) + q2.To.Type = obj.TYPE_MEM + q2.To.Reg = REGSP + q2.To.Offset = -24 + + // maintain FP for DUFFZERO + q3 := obj.Appendp(q2, c.newprog) + q3.Pos = p.Pos + q3.As = ASUB + q3.From.Type = obj.TYPE_CONST + q3.From.Offset = 24 + q3.Reg = REGSP + q3.To.Type = obj.TYPE_REG + q3.To.Reg = REGFP + + q5 := obj.Appendp(q4, c.newprog) + q5.Pos = p.Pos + q5.As = ASUB + q5.From.Type = obj.TYPE_CONST + q5.From.Offset = 8 + q5.Reg = REGSP + q5.To.Type = obj.TYPE_REG + q5.To.Reg = REGFP + q1.From.SetTarget(q5) + p = q5 } if p.To.Type == obj.TYPE_REG && p.To.Reg == REGSP && p.Spadj == 0 { diff --git a/src/cmd/internal/objabi/util.go b/src/cmd/internal/objabi/util.go index a73ab479a1..1f99f8ed5d 100644 --- a/src/cmd/internal/objabi/util.go +++ b/src/cmd/internal/objabi/util.go @@ -137,7 +137,7 @@ func init() { } // Note: must agree with runtime.framepointer_enabled. -var Framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" && (GOOS == "linux" || GOOS == "darwin" || GOOS == "ios") +var Framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" func addexp(s string) { // Could do general integer parsing here, but the runtime copy doesn't yet. diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 05520d07b2..5bd283d12f 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -1106,4 +1106,4 @@ var ( ) // Must agree with cmd/internal/objabi.Framepointer_enabled. -const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" && (GOOS == "linux" || GOOS == "darwin" || GOOS == "ios") +const framepointer_enabled = GOARCH == "amd64" || GOARCH == "arm64" diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 7321790b78..eb185eecd3 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -275,7 +275,22 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // For architectures with frame pointers, if there's // a frame, then there's a saved frame pointer here. - if frame.varp > frame.sp && (GOARCH == "amd64" || GOARCH == "arm64") { + // + // NOTE: This code is not as general as it looks. + // On x86, the ABI is to save the frame pointer word at the + // top of the stack frame, so we have to back down over it. + // On arm64, the frame pointer should be at the bottom of + // the stack (with R29 (aka FP) = RSP), in which case we would + // not want to do the subtraction here. But we started out without + // any frame pointer, and when we wanted to add it, we didn't + // want to break all the assembly doing direct writes to 8(RSP) + // to set the first parameter to a called function. + // So we decided to write the FP link *below* the stack pointer + // (with R29 = RSP - 8 in Go functions). + // This is technically ABI-compatible but not standard. + // And it happens to end up mimicking the x86 layout. + // Other architectures may make different decisions. + if frame.varp > frame.sp && framepointer_enabled { frame.varp -= sys.PtrSize } -- cgit v1.3-5-g9baa From 427bd7599d1aa2bc89faecff09777a5a662e5bf8 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 22 Jan 2021 15:16:13 -0500 Subject: runtime: generate windows/arm64 callback asm This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. Change-Id: I5e2b589797808626bcca771cdf860d5cb85586cb Reviewed-on: https://go-review.googlesource.com/c/go/+/288826 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Alex Brainman Reviewed-by: Jason A. Donenfeld --- src/runtime/syscall_windows.go | 4 +- src/runtime/wincallback.go | 29 + src/runtime/zcallback_windows_arm64.s | 4012 +++++++++++++++++++++++++++++++++ 3 files changed, 4043 insertions(+), 2 deletions(-) create mode 100644 src/runtime/zcallback_windows_arm64.s (limited to 'src/runtime') diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 6052cc333c..666ec5f69e 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -71,8 +71,8 @@ func callbackasmAddr(i int) uintptr { panic("unsupported architecture") case "386", "amd64": entrySize = 5 - case "arm": - // On ARM, each entry is a MOV instruction + case "arm", "arm64": + // On ARM and ARM64, each entry is a MOV instruction // followed by a branch instruction entrySize = 8 } diff --git a/src/runtime/wincallback.go b/src/runtime/wincallback.go index fb452222da..cf3327c6fe 100644 --- a/src/runtime/wincallback.go +++ b/src/runtime/wincallback.go @@ -72,6 +72,34 @@ TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 } } +func genasmArm64() { + var buf bytes.Buffer + + buf.WriteString(`// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. + +// External code calls into callbackasm at an offset corresponding +// to the callback index. Callbackasm is a table of MOV and B instructions. +// The MOV instruction loads R12 with the callback index, and the +// B instruction branches to callbackasm1. +// callbackasm1 takes the callback index from R12 and +// indexes into an array that stores information about each callback. +// It then calls the Go implementation for that callback. +#include "textflag.h" + +TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 +`) + for i := 0; i < maxCallback; i++ { + buf.WriteString(fmt.Sprintf("\tMOVD\t$%d, R12\n", i)) + buf.WriteString("\tB\truntime·callbackasm1(SB)\n") + } + + err := os.WriteFile("zcallback_windows_arm64.s", buf.Bytes(), 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "wincallback: %s\n", err) + os.Exit(2) + } +} + func gengo() { var buf bytes.Buffer @@ -91,5 +119,6 @@ const cb_max = %d // maximum number of windows callbacks allowed func main() { genasm386Amd64() genasmArm() + genasmArm64() gengo() } diff --git a/src/runtime/zcallback_windows_arm64.s b/src/runtime/zcallback_windows_arm64.s new file mode 100644 index 0000000000..69fb05788c --- /dev/null +++ b/src/runtime/zcallback_windows_arm64.s @@ -0,0 +1,4012 @@ +// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. + +// External code calls into callbackasm at an offset corresponding +// to the callback index. Callbackasm is a table of MOV and B instructions. +// The MOV instruction loads R12 with the callback index, and the +// B instruction branches to callbackasm1. +// callbackasm1 takes the callback index from R12 and +// indexes into an array that stores information about each callback. +// It then calls the Go implementation for that callback. +#include "textflag.h" + +TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 + MOVD $0, R12 + B runtime·callbackasm1(SB) + MOVD $1, R12 + B runtime·callbackasm1(SB) + MOVD $2, R12 + B runtime·callbackasm1(SB) + MOVD $3, R12 + B runtime·callbackasm1(SB) + MOVD $4, R12 + B runtime·callbackasm1(SB) + MOVD $5, R12 + B runtime·callbackasm1(SB) + MOVD $6, R12 + B runtime·callbackasm1(SB) + MOVD $7, R12 + B runtime·callbackasm1(SB) + MOVD $8, R12 + B runtime·callbackasm1(SB) + MOVD $9, R12 + B runtime·callbackasm1(SB) + MOVD $10, R12 + B runtime·callbackasm1(SB) + MOVD $11, R12 + B runtime·callbackasm1(SB) + MOVD $12, R12 + B runtime·callbackasm1(SB) + MOVD $13, R12 + B runtime·callbackasm1(SB) + MOVD $14, R12 + B runtime·callbackasm1(SB) + MOVD $15, R12 + B runtime·callbackasm1(SB) + MOVD $16, R12 + B runtime·callbackasm1(SB) + MOVD $17, R12 + B runtime·callbackasm1(SB) + MOVD $18, R12 + B runtime·callbackasm1(SB) + MOVD $19, R12 + B runtime·callbackasm1(SB) + MOVD $20, R12 + B runtime·callbackasm1(SB) + MOVD $21, R12 + B runtime·callbackasm1(SB) + MOVD $22, R12 + B runtime·callbackasm1(SB) + MOVD $23, R12 + B runtime·callbackasm1(SB) + MOVD $24, R12 + B runtime·callbackasm1(SB) + MOVD $25, R12 + B runtime·callbackasm1(SB) + MOVD $26, R12 + B runtime·callbackasm1(SB) + MOVD $27, R12 + B runtime·callbackasm1(SB) + MOVD $28, R12 + B runtime·callbackasm1(SB) + MOVD $29, R12 + B runtime·callbackasm1(SB) + MOVD $30, R12 + B runtime·callbackasm1(SB) + MOVD $31, R12 + B runtime·callbackasm1(SB) + MOVD $32, R12 + B runtime·callbackasm1(SB) + MOVD $33, R12 + B runtime·callbackasm1(SB) + MOVD $34, R12 + B runtime·callbackasm1(SB) + MOVD $35, R12 + B runtime·callbackasm1(SB) + MOVD $36, R12 + B runtime·callbackasm1(SB) + MOVD $37, R12 + B runtime·callbackasm1(SB) + MOVD $38, R12 + B runtime·callbackasm1(SB) + MOVD $39, R12 + B runtime·callbackasm1(SB) + MOVD $40, R12 + B runtime·callbackasm1(SB) + MOVD $41, R12 + B runtime·callbackasm1(SB) + MOVD $42, R12 + B runtime·callbackasm1(SB) + MOVD $43, R12 + B runtime·callbackasm1(SB) + MOVD $44, R12 + B runtime·callbackasm1(SB) + MOVD $45, R12 + B runtime·callbackasm1(SB) + MOVD $46, R12 + B runtime·callbackasm1(SB) + MOVD $47, R12 + B runtime·callbackasm1(SB) + MOVD $48, R12 + B runtime·callbackasm1(SB) + MOVD $49, R12 + B runtime·callbackasm1(SB) + MOVD $50, R12 + B runtime·callbackasm1(SB) + MOVD $51, R12 + B runtime·callbackasm1(SB) + MOVD $52, R12 + B runtime·callbackasm1(SB) + MOVD $53, R12 + B runtime·callbackasm1(SB) + MOVD $54, R12 + B runtime·callbackasm1(SB) + MOVD $55, R12 + B runtime·callbackasm1(SB) + MOVD $56, R12 + B runtime·callbackasm1(SB) + MOVD $57, R12 + B runtime·callbackasm1(SB) + MOVD $58, R12 + B runtime·callbackasm1(SB) + MOVD $59, R12 + B runtime·callbackasm1(SB) + MOVD $60, R12 + B runtime·callbackasm1(SB) + MOVD $61, R12 + B runtime·callbackasm1(SB) + MOVD $62, R12 + B runtime·callbackasm1(SB) + MOVD $63, R12 + B runtime·callbackasm1(SB) + MOVD $64, R12 + B runtime·callbackasm1(SB) + MOVD $65, R12 + B runtime·callbackasm1(SB) + MOVD $66, R12 + B runtime·callbackasm1(SB) + MOVD $67, R12 + B runtime·callbackasm1(SB) + MOVD $68, R12 + B runtime·callbackasm1(SB) + MOVD $69, R12 + B runtime·callbackasm1(SB) + MOVD $70, R12 + B runtime·callbackasm1(SB) + MOVD $71, R12 + B runtime·callbackasm1(SB) + MOVD $72, R12 + B runtime·callbackasm1(SB) + MOVD $73, R12 + B runtime·callbackasm1(SB) + MOVD $74, R12 + B runtime·callbackasm1(SB) + MOVD $75, R12 + B runtime·callbackasm1(SB) + MOVD $76, R12 + B runtime·callbackasm1(SB) + MOVD $77, R12 + B runtime·callbackasm1(SB) + MOVD $78, R12 + B runtime·callbackasm1(SB) + MOVD $79, R12 + B runtime·callbackasm1(SB) + MOVD $80, R12 + B runtime·callbackasm1(SB) + MOVD $81, R12 + B runtime·callbackasm1(SB) + MOVD $82, R12 + B runtime·callbackasm1(SB) + MOVD $83, R12 + B runtime·callbackasm1(SB) + MOVD $84, R12 + B runtime·callbackasm1(SB) + MOVD $85, R12 + B runtime·callbackasm1(SB) + MOVD $86, R12 + B runtime·callbackasm1(SB) + MOVD $87, R12 + B runtime·callbackasm1(SB) + MOVD $88, R12 + B runtime·callbackasm1(SB) + MOVD $89, R12 + B runtime·callbackasm1(SB) + MOVD $90, R12 + B runtime·callbackasm1(SB) + MOVD $91, R12 + B runtime·callbackasm1(SB) + MOVD $92, R12 + B runtime·callbackasm1(SB) + MOVD $93, R12 + B runtime·callbackasm1(SB) + MOVD $94, R12 + B runtime·callbackasm1(SB) + MOVD $95, R12 + B runtime·callbackasm1(SB) + MOVD $96, R12 + B runtime·callbackasm1(SB) + MOVD $97, R12 + B runtime·callbackasm1(SB) + MOVD $98, R12 + B runtime·callbackasm1(SB) + MOVD $99, R12 + B runtime·callbackasm1(SB) + MOVD $100, R12 + B runtime·callbackasm1(SB) + MOVD $101, R12 + B runtime·callbackasm1(SB) + MOVD $102, R12 + B runtime·callbackasm1(SB) + MOVD $103, R12 + B runtime·callbackasm1(SB) + MOVD $104, R12 + B runtime·callbackasm1(SB) + MOVD $105, R12 + B runtime·callbackasm1(SB) + MOVD $106, R12 + B runtime·callbackasm1(SB) + MOVD $107, R12 + B runtime·callbackasm1(SB) + MOVD $108, R12 + B runtime·callbackasm1(SB) + MOVD $109, R12 + B runtime·callbackasm1(SB) + MOVD $110, R12 + B runtime·callbackasm1(SB) + MOVD $111, R12 + B runtime·callbackasm1(SB) + MOVD $112, R12 + B runtime·callbackasm1(SB) + MOVD $113, R12 + B runtime·callbackasm1(SB) + MOVD $114, R12 + B runtime·callbackasm1(SB) + MOVD $115, R12 + B runtime·callbackasm1(SB) + MOVD $116, R12 + B runtime·callbackasm1(SB) + MOVD $117, R12 + B runtime·callbackasm1(SB) + MOVD $118, R12 + B runtime·callbackasm1(SB) + MOVD $119, R12 + B runtime·callbackasm1(SB) + MOVD $120, R12 + B runtime·callbackasm1(SB) + MOVD $121, R12 + B runtime·callbackasm1(SB) + MOVD $122, R12 + B runtime·callbackasm1(SB) + MOVD $123, R12 + B runtime·callbackasm1(SB) + MOVD $124, R12 + B runtime·callbackasm1(SB) + MOVD $125, R12 + B runtime·callbackasm1(SB) + MOVD $126, R12 + B runtime·callbackasm1(SB) + MOVD $127, R12 + B runtime·callbackasm1(SB) + MOVD $128, R12 + B runtime·callbackasm1(SB) + MOVD $129, R12 + B runtime·callbackasm1(SB) + MOVD $130, R12 + B runtime·callbackasm1(SB) + MOVD $131, R12 + B runtime·callbackasm1(SB) + MOVD $132, R12 + B runtime·callbackasm1(SB) + MOVD $133, R12 + B runtime·callbackasm1(SB) + MOVD $134, R12 + B runtime·callbackasm1(SB) + MOVD $135, R12 + B runtime·callbackasm1(SB) + MOVD $136, R12 + B runtime·callbackasm1(SB) + MOVD $137, R12 + B runtime·callbackasm1(SB) + MOVD $138, R12 + B runtime·callbackasm1(SB) + MOVD $139, R12 + B runtime·callbackasm1(SB) + MOVD $140, R12 + B runtime·callbackasm1(SB) + MOVD $141, R12 + B runtime·callbackasm1(SB) + MOVD $142, R12 + B runtime·callbackasm1(SB) + MOVD $143, R12 + B runtime·callbackasm1(SB) + MOVD $144, R12 + B runtime·callbackasm1(SB) + MOVD $145, R12 + B runtime·callbackasm1(SB) + MOVD $146, R12 + B runtime·callbackasm1(SB) + MOVD $147, R12 + B runtime·callbackasm1(SB) + MOVD $148, R12 + B runtime·callbackasm1(SB) + MOVD $149, R12 + B runtime·callbackasm1(SB) + MOVD $150, R12 + B runtime·callbackasm1(SB) + MOVD $151, R12 + B runtime·callbackasm1(SB) + MOVD $152, R12 + B runtime·callbackasm1(SB) + MOVD $153, R12 + B runtime·callbackasm1(SB) + MOVD $154, R12 + B runtime·callbackasm1(SB) + MOVD $155, R12 + B runtime·callbackasm1(SB) + MOVD $156, R12 + B runtime·callbackasm1(SB) + MOVD $157, R12 + B runtime·callbackasm1(SB) + MOVD $158, R12 + B runtime·callbackasm1(SB) + MOVD $159, R12 + B runtime·callbackasm1(SB) + MOVD $160, R12 + B runtime·callbackasm1(SB) + MOVD $161, R12 + B runtime·callbackasm1(SB) + MOVD $162, R12 + B runtime·callbackasm1(SB) + MOVD $163, R12 + B runtime·callbackasm1(SB) + MOVD $164, R12 + B runtime·callbackasm1(SB) + MOVD $165, R12 + B runtime·callbackasm1(SB) + MOVD $166, R12 + B runtime·callbackasm1(SB) + MOVD $167, R12 + B runtime·callbackasm1(SB) + MOVD $168, R12 + B runtime·callbackasm1(SB) + MOVD $169, R12 + B runtime·callbackasm1(SB) + MOVD $170, R12 + B runtime·callbackasm1(SB) + MOVD $171, R12 + B runtime·callbackasm1(SB) + MOVD $172, R12 + B runtime·callbackasm1(SB) + MOVD $173, R12 + B runtime·callbackasm1(SB) + MOVD $174, R12 + B runtime·callbackasm1(SB) + MOVD $175, R12 + B runtime·callbackasm1(SB) + MOVD $176, R12 + B runtime·callbackasm1(SB) + MOVD $177, R12 + B runtime·callbackasm1(SB) + MOVD $178, R12 + B runtime·callbackasm1(SB) + MOVD $179, R12 + B runtime·callbackasm1(SB) + MOVD $180, R12 + B runtime·callbackasm1(SB) + MOVD $181, R12 + B runtime·callbackasm1(SB) + MOVD $182, R12 + B runtime·callbackasm1(SB) + MOVD $183, R12 + B runtime·callbackasm1(SB) + MOVD $184, R12 + B runtime·callbackasm1(SB) + MOVD $185, R12 + B runtime·callbackasm1(SB) + MOVD $186, R12 + B runtime·callbackasm1(SB) + MOVD $187, R12 + B runtime·callbackasm1(SB) + MOVD $188, R12 + B runtime·callbackasm1(SB) + MOVD $189, R12 + B runtime·callbackasm1(SB) + MOVD $190, R12 + B runtime·callbackasm1(SB) + MOVD $191, R12 + B runtime·callbackasm1(SB) + MOVD $192, R12 + B runtime·callbackasm1(SB) + MOVD $193, R12 + B runtime·callbackasm1(SB) + MOVD $194, R12 + B runtime·callbackasm1(SB) + MOVD $195, R12 + B runtime·callbackasm1(SB) + MOVD $196, R12 + B runtime·callbackasm1(SB) + MOVD $197, R12 + B runtime·callbackasm1(SB) + MOVD $198, R12 + B runtime·callbackasm1(SB) + MOVD $199, R12 + B runtime·callbackasm1(SB) + MOVD $200, R12 + B runtime·callbackasm1(SB) + MOVD $201, R12 + B runtime·callbackasm1(SB) + MOVD $202, R12 + B runtime·callbackasm1(SB) + MOVD $203, R12 + B runtime·callbackasm1(SB) + MOVD $204, R12 + B runtime·callbackasm1(SB) + MOVD $205, R12 + B runtime·callbackasm1(SB) + MOVD $206, R12 + B runtime·callbackasm1(SB) + MOVD $207, R12 + B runtime·callbackasm1(SB) + MOVD $208, R12 + B runtime·callbackasm1(SB) + MOVD $209, R12 + B runtime·callbackasm1(SB) + MOVD $210, R12 + B runtime·callbackasm1(SB) + MOVD $211, R12 + B runtime·callbackasm1(SB) + MOVD $212, R12 + B runtime·callbackasm1(SB) + MOVD $213, R12 + B runtime·callbackasm1(SB) + MOVD $214, R12 + B runtime·callbackasm1(SB) + MOVD $215, R12 + B runtime·callbackasm1(SB) + MOVD $216, R12 + B runtime·callbackasm1(SB) + MOVD $217, R12 + B runtime·callbackasm1(SB) + MOVD $218, R12 + B runtime·callbackasm1(SB) + MOVD $219, R12 + B runtime·callbackasm1(SB) + MOVD $220, R12 + B runtime·callbackasm1(SB) + MOVD $221, R12 + B runtime·callbackasm1(SB) + MOVD $222, R12 + B runtime·callbackasm1(SB) + MOVD $223, R12 + B runtime·callbackasm1(SB) + MOVD $224, R12 + B runtime·callbackasm1(SB) + MOVD $225, R12 + B runtime·callbackasm1(SB) + MOVD $226, R12 + B runtime·callbackasm1(SB) + MOVD $227, R12 + B runtime·callbackasm1(SB) + MOVD $228, R12 + B runtime·callbackasm1(SB) + MOVD $229, R12 + B runtime·callbackasm1(SB) + MOVD $230, R12 + B runtime·callbackasm1(SB) + MOVD $231, R12 + B runtime·callbackasm1(SB) + MOVD $232, R12 + B runtime·callbackasm1(SB) + MOVD $233, R12 + B runtime·callbackasm1(SB) + MOVD $234, R12 + B runtime·callbackasm1(SB) + MOVD $235, R12 + B runtime·callbackasm1(SB) + MOVD $236, R12 + B runtime·callbackasm1(SB) + MOVD $237, R12 + B runtime·callbackasm1(SB) + MOVD $238, R12 + B runtime·callbackasm1(SB) + MOVD $239, R12 + B runtime·callbackasm1(SB) + MOVD $240, R12 + B runtime·callbackasm1(SB) + MOVD $241, R12 + B runtime·callbackasm1(SB) + MOVD $242, R12 + B runtime·callbackasm1(SB) + MOVD $243, R12 + B runtime·callbackasm1(SB) + MOVD $244, R12 + B runtime·callbackasm1(SB) + MOVD $245, R12 + B runtime·callbackasm1(SB) + MOVD $246, R12 + B runtime·callbackasm1(SB) + MOVD $247, R12 + B runtime·callbackasm1(SB) + MOVD $248, R12 + B runtime·callbackasm1(SB) + MOVD $249, R12 + B runtime·callbackasm1(SB) + MOVD $250, R12 + B runtime·callbackasm1(SB) + MOVD $251, R12 + B runtime·callbackasm1(SB) + MOVD $252, R12 + B runtime·callbackasm1(SB) + MOVD $253, R12 + B runtime·callbackasm1(SB) + MOVD $254, R12 + B runtime·callbackasm1(SB) + MOVD $255, R12 + B runtime·callbackasm1(SB) + MOVD $256, R12 + B runtime·callbackasm1(SB) + MOVD $257, R12 + B runtime·callbackasm1(SB) + MOVD $258, R12 + B runtime·callbackasm1(SB) + MOVD $259, R12 + B runtime·callbackasm1(SB) + MOVD $260, R12 + B runtime·callbackasm1(SB) + MOVD $261, R12 + B runtime·callbackasm1(SB) + MOVD $262, R12 + B runtime·callbackasm1(SB) + MOVD $263, R12 + B runtime·callbackasm1(SB) + MOVD $264, R12 + B runtime·callbackasm1(SB) + MOVD $265, R12 + B runtime·callbackasm1(SB) + MOVD $266, R12 + B runtime·callbackasm1(SB) + MOVD $267, R12 + B runtime·callbackasm1(SB) + MOVD $268, R12 + B runtime·callbackasm1(SB) + MOVD $269, R12 + B runtime·callbackasm1(SB) + MOVD $270, R12 + B runtime·callbackasm1(SB) + MOVD $271, R12 + B runtime·callbackasm1(SB) + MOVD $272, R12 + B runtime·callbackasm1(SB) + MOVD $273, R12 + B runtime·callbackasm1(SB) + MOVD $274, R12 + B runtime·callbackasm1(SB) + MOVD $275, R12 + B runtime·callbackasm1(SB) + MOVD $276, R12 + B runtime·callbackasm1(SB) + MOVD $277, R12 + B runtime·callbackasm1(SB) + MOVD $278, R12 + B runtime·callbackasm1(SB) + MOVD $279, R12 + B runtime·callbackasm1(SB) + MOVD $280, R12 + B runtime·callbackasm1(SB) + MOVD $281, R12 + B runtime·callbackasm1(SB) + MOVD $282, R12 + B runtime·callbackasm1(SB) + MOVD $283, R12 + B runtime·callbackasm1(SB) + MOVD $284, R12 + B runtime·callbackasm1(SB) + MOVD $285, R12 + B runtime·callbackasm1(SB) + MOVD $286, R12 + B runtime·callbackasm1(SB) + MOVD $287, R12 + B runtime·callbackasm1(SB) + MOVD $288, R12 + B runtime·callbackasm1(SB) + MOVD $289, R12 + B runtime·callbackasm1(SB) + MOVD $290, R12 + B runtime·callbackasm1(SB) + MOVD $291, R12 + B runtime·callbackasm1(SB) + MOVD $292, R12 + B runtime·callbackasm1(SB) + MOVD $293, R12 + B runtime·callbackasm1(SB) + MOVD $294, R12 + B runtime·callbackasm1(SB) + MOVD $295, R12 + B runtime·callbackasm1(SB) + MOVD $296, R12 + B runtime·callbackasm1(SB) + MOVD $297, R12 + B runtime·callbackasm1(SB) + MOVD $298, R12 + B runtime·callbackasm1(SB) + MOVD $299, R12 + B runtime·callbackasm1(SB) + MOVD $300, R12 + B runtime·callbackasm1(SB) + MOVD $301, R12 + B runtime·callbackasm1(SB) + MOVD $302, R12 + B runtime·callbackasm1(SB) + MOVD $303, R12 + B runtime·callbackasm1(SB) + MOVD $304, R12 + B runtime·callbackasm1(SB) + MOVD $305, R12 + B runtime·callbackasm1(SB) + MOVD $306, R12 + B runtime·callbackasm1(SB) + MOVD $307, R12 + B runtime·callbackasm1(SB) + MOVD $308, R12 + B runtime·callbackasm1(SB) + MOVD $309, R12 + B runtime·callbackasm1(SB) + MOVD $310, R12 + B runtime·callbackasm1(SB) + MOVD $311, R12 + B runtime·callbackasm1(SB) + MOVD $312, R12 + B runtime·callbackasm1(SB) + MOVD $313, R12 + B runtime·callbackasm1(SB) + MOVD $314, R12 + B runtime·callbackasm1(SB) + MOVD $315, R12 + B runtime·callbackasm1(SB) + MOVD $316, R12 + B runtime·callbackasm1(SB) + MOVD $317, R12 + B runtime·callbackasm1(SB) + MOVD $318, R12 + B runtime·callbackasm1(SB) + MOVD $319, R12 + B runtime·callbackasm1(SB) + MOVD $320, R12 + B runtime·callbackasm1(SB) + MOVD $321, R12 + B runtime·callbackasm1(SB) + MOVD $322, R12 + B runtime·callbackasm1(SB) + MOVD $323, R12 + B runtime·callbackasm1(SB) + MOVD $324, R12 + B runtime·callbackasm1(SB) + MOVD $325, R12 + B runtime·callbackasm1(SB) + MOVD $326, R12 + B runtime·callbackasm1(SB) + MOVD $327, R12 + B runtime·callbackasm1(SB) + MOVD $328, R12 + B runtime·callbackasm1(SB) + MOVD $329, R12 + B runtime·callbackasm1(SB) + MOVD $330, R12 + B runtime·callbackasm1(SB) + MOVD $331, R12 + B runtime·callbackasm1(SB) + MOVD $332, R12 + B runtime·callbackasm1(SB) + MOVD $333, R12 + B runtime·callbackasm1(SB) + MOVD $334, R12 + B runtime·callbackasm1(SB) + MOVD $335, R12 + B runtime·callbackasm1(SB) + MOVD $336, R12 + B runtime·callbackasm1(SB) + MOVD $337, R12 + B runtime·callbackasm1(SB) + MOVD $338, R12 + B runtime·callbackasm1(SB) + MOVD $339, R12 + B runtime·callbackasm1(SB) + MOVD $340, R12 + B runtime·callbackasm1(SB) + MOVD $341, R12 + B runtime·callbackasm1(SB) + MOVD $342, R12 + B runtime·callbackasm1(SB) + MOVD $343, R12 + B runtime·callbackasm1(SB) + MOVD $344, R12 + B runtime·callbackasm1(SB) + MOVD $345, R12 + B runtime·callbackasm1(SB) + MOVD $346, R12 + B runtime·callbackasm1(SB) + MOVD $347, R12 + B runtime·callbackasm1(SB) + MOVD $348, R12 + B runtime·callbackasm1(SB) + MOVD $349, R12 + B runtime·callbackasm1(SB) + MOVD $350, R12 + B runtime·callbackasm1(SB) + MOVD $351, R12 + B runtime·callbackasm1(SB) + MOVD $352, R12 + B runtime·callbackasm1(SB) + MOVD $353, R12 + B runtime·callbackasm1(SB) + MOVD $354, R12 + B runtime·callbackasm1(SB) + MOVD $355, R12 + B runtime·callbackasm1(SB) + MOVD $356, R12 + B runtime·callbackasm1(SB) + MOVD $357, R12 + B runtime·callbackasm1(SB) + MOVD $358, R12 + B runtime·callbackasm1(SB) + MOVD $359, R12 + B runtime·callbackasm1(SB) + MOVD $360, R12 + B runtime·callbackasm1(SB) + MOVD $361, R12 + B runtime·callbackasm1(SB) + MOVD $362, R12 + B runtime·callbackasm1(SB) + MOVD $363, R12 + B runtime·callbackasm1(SB) + MOVD $364, R12 + B runtime·callbackasm1(SB) + MOVD $365, R12 + B runtime·callbackasm1(SB) + MOVD $366, R12 + B runtime·callbackasm1(SB) + MOVD $367, R12 + B runtime·callbackasm1(SB) + MOVD $368, R12 + B runtime·callbackasm1(SB) + MOVD $369, R12 + B runtime·callbackasm1(SB) + MOVD $370, R12 + B runtime·callbackasm1(SB) + MOVD $371, R12 + B runtime·callbackasm1(SB) + MOVD $372, R12 + B runtime·callbackasm1(SB) + MOVD $373, R12 + B runtime·callbackasm1(SB) + MOVD $374, R12 + B runtime·callbackasm1(SB) + MOVD $375, R12 + B runtime·callbackasm1(SB) + MOVD $376, R12 + B runtime·callbackasm1(SB) + MOVD $377, R12 + B runtime·callbackasm1(SB) + MOVD $378, R12 + B runtime·callbackasm1(SB) + MOVD $379, R12 + B runtime·callbackasm1(SB) + MOVD $380, R12 + B runtime·callbackasm1(SB) + MOVD $381, R12 + B runtime·callbackasm1(SB) + MOVD $382, R12 + B runtime·callbackasm1(SB) + MOVD $383, R12 + B runtime·callbackasm1(SB) + MOVD $384, R12 + B runtime·callbackasm1(SB) + MOVD $385, R12 + B runtime·callbackasm1(SB) + MOVD $386, R12 + B runtime·callbackasm1(SB) + MOVD $387, R12 + B runtime·callbackasm1(SB) + MOVD $388, R12 + B runtime·callbackasm1(SB) + MOVD $389, R12 + B runtime·callbackasm1(SB) + MOVD $390, R12 + B runtime·callbackasm1(SB) + MOVD $391, R12 + B runtime·callbackasm1(SB) + MOVD $392, R12 + B runtime·callbackasm1(SB) + MOVD $393, R12 + B runtime·callbackasm1(SB) + MOVD $394, R12 + B runtime·callbackasm1(SB) + MOVD $395, R12 + B runtime·callbackasm1(SB) + MOVD $396, R12 + B runtime·callbackasm1(SB) + MOVD $397, R12 + B runtime·callbackasm1(SB) + MOVD $398, R12 + B runtime·callbackasm1(SB) + MOVD $399, R12 + B runtime·callbackasm1(SB) + MOVD $400, R12 + B runtime·callbackasm1(SB) + MOVD $401, R12 + B runtime·callbackasm1(SB) + MOVD $402, R12 + B runtime·callbackasm1(SB) + MOVD $403, R12 + B runtime·callbackasm1(SB) + MOVD $404, R12 + B runtime·callbackasm1(SB) + MOVD $405, R12 + B runtime·callbackasm1(SB) + MOVD $406, R12 + B runtime·callbackasm1(SB) + MOVD $407, R12 + B runtime·callbackasm1(SB) + MOVD $408, R12 + B runtime·callbackasm1(SB) + MOVD $409, R12 + B runtime·callbackasm1(SB) + MOVD $410, R12 + B runtime·callbackasm1(SB) + MOVD $411, R12 + B runtime·callbackasm1(SB) + MOVD $412, R12 + B runtime·callbackasm1(SB) + MOVD $413, R12 + B runtime·callbackasm1(SB) + MOVD $414, R12 + B runtime·callbackasm1(SB) + MOVD $415, R12 + B runtime·callbackasm1(SB) + MOVD $416, R12 + B runtime·callbackasm1(SB) + MOVD $417, R12 + B runtime·callbackasm1(SB) + MOVD $418, R12 + B runtime·callbackasm1(SB) + MOVD $419, R12 + B runtime·callbackasm1(SB) + MOVD $420, R12 + B runtime·callbackasm1(SB) + MOVD $421, R12 + B runtime·callbackasm1(SB) + MOVD $422, R12 + B runtime·callbackasm1(SB) + MOVD $423, R12 + B runtime·callbackasm1(SB) + MOVD $424, R12 + B runtime·callbackasm1(SB) + MOVD $425, R12 + B runtime·callbackasm1(SB) + MOVD $426, R12 + B runtime·callbackasm1(SB) + MOVD $427, R12 + B runtime·callbackasm1(SB) + MOVD $428, R12 + B runtime·callbackasm1(SB) + MOVD $429, R12 + B runtime·callbackasm1(SB) + MOVD $430, R12 + B runtime·callbackasm1(SB) + MOVD $431, R12 + B runtime·callbackasm1(SB) + MOVD $432, R12 + B runtime·callbackasm1(SB) + MOVD $433, R12 + B runtime·callbackasm1(SB) + MOVD $434, R12 + B runtime·callbackasm1(SB) + MOVD $435, R12 + B runtime·callbackasm1(SB) + MOVD $436, R12 + B runtime·callbackasm1(SB) + MOVD $437, R12 + B runtime·callbackasm1(SB) + MOVD $438, R12 + B runtime·callbackasm1(SB) + MOVD $439, R12 + B runtime·callbackasm1(SB) + MOVD $440, R12 + B runtime·callbackasm1(SB) + MOVD $441, R12 + B runtime·callbackasm1(SB) + MOVD $442, R12 + B runtime·callbackasm1(SB) + MOVD $443, R12 + B runtime·callbackasm1(SB) + MOVD $444, R12 + B runtime·callbackasm1(SB) + MOVD $445, R12 + B runtime·callbackasm1(SB) + MOVD $446, R12 + B runtime·callbackasm1(SB) + MOVD $447, R12 + B runtime·callbackasm1(SB) + MOVD $448, R12 + B runtime·callbackasm1(SB) + MOVD $449, R12 + B runtime·callbackasm1(SB) + MOVD $450, R12 + B runtime·callbackasm1(SB) + MOVD $451, R12 + B runtime·callbackasm1(SB) + MOVD $452, R12 + B runtime·callbackasm1(SB) + MOVD $453, R12 + B runtime·callbackasm1(SB) + MOVD $454, R12 + B runtime·callbackasm1(SB) + MOVD $455, R12 + B runtime·callbackasm1(SB) + MOVD $456, R12 + B runtime·callbackasm1(SB) + MOVD $457, R12 + B runtime·callbackasm1(SB) + MOVD $458, R12 + B runtime·callbackasm1(SB) + MOVD $459, R12 + B runtime·callbackasm1(SB) + MOVD $460, R12 + B runtime·callbackasm1(SB) + MOVD $461, R12 + B runtime·callbackasm1(SB) + MOVD $462, R12 + B runtime·callbackasm1(SB) + MOVD $463, R12 + B runtime·callbackasm1(SB) + MOVD $464, R12 + B runtime·callbackasm1(SB) + MOVD $465, R12 + B runtime·callbackasm1(SB) + MOVD $466, R12 + B runtime·callbackasm1(SB) + MOVD $467, R12 + B runtime·callbackasm1(SB) + MOVD $468, R12 + B runtime·callbackasm1(SB) + MOVD $469, R12 + B runtime·callbackasm1(SB) + MOVD $470, R12 + B runtime·callbackasm1(SB) + MOVD $471, R12 + B runtime·callbackasm1(SB) + MOVD $472, R12 + B runtime·callbackasm1(SB) + MOVD $473, R12 + B runtime·callbackasm1(SB) + MOVD $474, R12 + B runtime·callbackasm1(SB) + MOVD $475, R12 + B runtime·callbackasm1(SB) + MOVD $476, R12 + B runtime·callbackasm1(SB) + MOVD $477, R12 + B runtime·callbackasm1(SB) + MOVD $478, R12 + B runtime·callbackasm1(SB) + MOVD $479, R12 + B runtime·callbackasm1(SB) + MOVD $480, R12 + B runtime·callbackasm1(SB) + MOVD $481, R12 + B runtime·callbackasm1(SB) + MOVD $482, R12 + B runtime·callbackasm1(SB) + MOVD $483, R12 + B runtime·callbackasm1(SB) + MOVD $484, R12 + B runtime·callbackasm1(SB) + MOVD $485, R12 + B runtime·callbackasm1(SB) + MOVD $486, R12 + B runtime·callbackasm1(SB) + MOVD $487, R12 + B runtime·callbackasm1(SB) + MOVD $488, R12 + B runtime·callbackasm1(SB) + MOVD $489, R12 + B runtime·callbackasm1(SB) + MOVD $490, R12 + B runtime·callbackasm1(SB) + MOVD $491, R12 + B runtime·callbackasm1(SB) + MOVD $492, R12 + B runtime·callbackasm1(SB) + MOVD $493, R12 + B runtime·callbackasm1(SB) + MOVD $494, R12 + B runtime·callbackasm1(SB) + MOVD $495, R12 + B runtime·callbackasm1(SB) + MOVD $496, R12 + B runtime·callbackasm1(SB) + MOVD $497, R12 + B runtime·callbackasm1(SB) + MOVD $498, R12 + B runtime·callbackasm1(SB) + MOVD $499, R12 + B runtime·callbackasm1(SB) + MOVD $500, R12 + B runtime·callbackasm1(SB) + MOVD $501, R12 + B runtime·callbackasm1(SB) + MOVD $502, R12 + B runtime·callbackasm1(SB) + MOVD $503, R12 + B runtime·callbackasm1(SB) + MOVD $504, R12 + B runtime·callbackasm1(SB) + MOVD $505, R12 + B runtime·callbackasm1(SB) + MOVD $506, R12 + B runtime·callbackasm1(SB) + MOVD $507, R12 + B runtime·callbackasm1(SB) + MOVD $508, R12 + B runtime·callbackasm1(SB) + MOVD $509, R12 + B runtime·callbackasm1(SB) + MOVD $510, R12 + B runtime·callbackasm1(SB) + MOVD $511, R12 + B runtime·callbackasm1(SB) + MOVD $512, R12 + B runtime·callbackasm1(SB) + MOVD $513, R12 + B runtime·callbackasm1(SB) + MOVD $514, R12 + B runtime·callbackasm1(SB) + MOVD $515, R12 + B runtime·callbackasm1(SB) + MOVD $516, R12 + B runtime·callbackasm1(SB) + MOVD $517, R12 + B runtime·callbackasm1(SB) + MOVD $518, R12 + B runtime·callbackasm1(SB) + MOVD $519, R12 + B runtime·callbackasm1(SB) + MOVD $520, R12 + B runtime·callbackasm1(SB) + MOVD $521, R12 + B runtime·callbackasm1(SB) + MOVD $522, R12 + B runtime·callbackasm1(SB) + MOVD $523, R12 + B runtime·callbackasm1(SB) + MOVD $524, R12 + B runtime·callbackasm1(SB) + MOVD $525, R12 + B runtime·callbackasm1(SB) + MOVD $526, R12 + B runtime·callbackasm1(SB) + MOVD $527, R12 + B runtime·callbackasm1(SB) + MOVD $528, R12 + B runtime·callbackasm1(SB) + MOVD $529, R12 + B runtime·callbackasm1(SB) + MOVD $530, R12 + B runtime·callbackasm1(SB) + MOVD $531, R12 + B runtime·callbackasm1(SB) + MOVD $532, R12 + B runtime·callbackasm1(SB) + MOVD $533, R12 + B runtime·callbackasm1(SB) + MOVD $534, R12 + B runtime·callbackasm1(SB) + MOVD $535, R12 + B runtime·callbackasm1(SB) + MOVD $536, R12 + B runtime·callbackasm1(SB) + MOVD $537, R12 + B runtime·callbackasm1(SB) + MOVD $538, R12 + B runtime·callbackasm1(SB) + MOVD $539, R12 + B runtime·callbackasm1(SB) + MOVD $540, R12 + B runtime·callbackasm1(SB) + MOVD $541, R12 + B runtime·callbackasm1(SB) + MOVD $542, R12 + B runtime·callbackasm1(SB) + MOVD $543, R12 + B runtime·callbackasm1(SB) + MOVD $544, R12 + B runtime·callbackasm1(SB) + MOVD $545, R12 + B runtime·callbackasm1(SB) + MOVD $546, R12 + B runtime·callbackasm1(SB) + MOVD $547, R12 + B runtime·callbackasm1(SB) + MOVD $548, R12 + B runtime·callbackasm1(SB) + MOVD $549, R12 + B runtime·callbackasm1(SB) + MOVD $550, R12 + B runtime·callbackasm1(SB) + MOVD $551, R12 + B runtime·callbackasm1(SB) + MOVD $552, R12 + B runtime·callbackasm1(SB) + MOVD $553, R12 + B runtime·callbackasm1(SB) + MOVD $554, R12 + B runtime·callbackasm1(SB) + MOVD $555, R12 + B runtime·callbackasm1(SB) + MOVD $556, R12 + B runtime·callbackasm1(SB) + MOVD $557, R12 + B runtime·callbackasm1(SB) + MOVD $558, R12 + B runtime·callbackasm1(SB) + MOVD $559, R12 + B runtime·callbackasm1(SB) + MOVD $560, R12 + B runtime·callbackasm1(SB) + MOVD $561, R12 + B runtime·callbackasm1(SB) + MOVD $562, R12 + B runtime·callbackasm1(SB) + MOVD $563, R12 + B runtime·callbackasm1(SB) + MOVD $564, R12 + B runtime·callbackasm1(SB) + MOVD $565, R12 + B runtime·callbackasm1(SB) + MOVD $566, R12 + B runtime·callbackasm1(SB) + MOVD $567, R12 + B runtime·callbackasm1(SB) + MOVD $568, R12 + B runtime·callbackasm1(SB) + MOVD $569, R12 + B runtime·callbackasm1(SB) + MOVD $570, R12 + B runtime·callbackasm1(SB) + MOVD $571, R12 + B runtime·callbackasm1(SB) + MOVD $572, R12 + B runtime·callbackasm1(SB) + MOVD $573, R12 + B runtime·callbackasm1(SB) + MOVD $574, R12 + B runtime·callbackasm1(SB) + MOVD $575, R12 + B runtime·callbackasm1(SB) + MOVD $576, R12 + B runtime·callbackasm1(SB) + MOVD $577, R12 + B runtime·callbackasm1(SB) + MOVD $578, R12 + B runtime·callbackasm1(SB) + MOVD $579, R12 + B runtime·callbackasm1(SB) + MOVD $580, R12 + B runtime·callbackasm1(SB) + MOVD $581, R12 + B runtime·callbackasm1(SB) + MOVD $582, R12 + B runtime·callbackasm1(SB) + MOVD $583, R12 + B runtime·callbackasm1(SB) + MOVD $584, R12 + B runtime·callbackasm1(SB) + MOVD $585, R12 + B runtime·callbackasm1(SB) + MOVD $586, R12 + B runtime·callbackasm1(SB) + MOVD $587, R12 + B runtime·callbackasm1(SB) + MOVD $588, R12 + B runtime·callbackasm1(SB) + MOVD $589, R12 + B runtime·callbackasm1(SB) + MOVD $590, R12 + B runtime·callbackasm1(SB) + MOVD $591, R12 + B runtime·callbackasm1(SB) + MOVD $592, R12 + B runtime·callbackasm1(SB) + MOVD $593, R12 + B runtime·callbackasm1(SB) + MOVD $594, R12 + B runtime·callbackasm1(SB) + MOVD $595, R12 + B runtime·callbackasm1(SB) + MOVD $596, R12 + B runtime·callbackasm1(SB) + MOVD $597, R12 + B runtime·callbackasm1(SB) + MOVD $598, R12 + B runtime·callbackasm1(SB) + MOVD $599, R12 + B runtime·callbackasm1(SB) + MOVD $600, R12 + B runtime·callbackasm1(SB) + MOVD $601, R12 + B runtime·callbackasm1(SB) + MOVD $602, R12 + B runtime·callbackasm1(SB) + MOVD $603, R12 + B runtime·callbackasm1(SB) + MOVD $604, R12 + B runtime·callbackasm1(SB) + MOVD $605, R12 + B runtime·callbackasm1(SB) + MOVD $606, R12 + B runtime·callbackasm1(SB) + MOVD $607, R12 + B runtime·callbackasm1(SB) + MOVD $608, R12 + B runtime·callbackasm1(SB) + MOVD $609, R12 + B runtime·callbackasm1(SB) + MOVD $610, R12 + B runtime·callbackasm1(SB) + MOVD $611, R12 + B runtime·callbackasm1(SB) + MOVD $612, R12 + B runtime·callbackasm1(SB) + MOVD $613, R12 + B runtime·callbackasm1(SB) + MOVD $614, R12 + B runtime·callbackasm1(SB) + MOVD $615, R12 + B runtime·callbackasm1(SB) + MOVD $616, R12 + B runtime·callbackasm1(SB) + MOVD $617, R12 + B runtime·callbackasm1(SB) + MOVD $618, R12 + B runtime·callbackasm1(SB) + MOVD $619, R12 + B runtime·callbackasm1(SB) + MOVD $620, R12 + B runtime·callbackasm1(SB) + MOVD $621, R12 + B runtime·callbackasm1(SB) + MOVD $622, R12 + B runtime·callbackasm1(SB) + MOVD $623, R12 + B runtime·callbackasm1(SB) + MOVD $624, R12 + B runtime·callbackasm1(SB) + MOVD $625, R12 + B runtime·callbackasm1(SB) + MOVD $626, R12 + B runtime·callbackasm1(SB) + MOVD $627, R12 + B runtime·callbackasm1(SB) + MOVD $628, R12 + B runtime·callbackasm1(SB) + MOVD $629, R12 + B runtime·callbackasm1(SB) + MOVD $630, R12 + B runtime·callbackasm1(SB) + MOVD $631, R12 + B runtime·callbackasm1(SB) + MOVD $632, R12 + B runtime·callbackasm1(SB) + MOVD $633, R12 + B runtime·callbackasm1(SB) + MOVD $634, R12 + B runtime·callbackasm1(SB) + MOVD $635, R12 + B runtime·callbackasm1(SB) + MOVD $636, R12 + B runtime·callbackasm1(SB) + MOVD $637, R12 + B runtime·callbackasm1(SB) + MOVD $638, R12 + B runtime·callbackasm1(SB) + MOVD $639, R12 + B runtime·callbackasm1(SB) + MOVD $640, R12 + B runtime·callbackasm1(SB) + MOVD $641, R12 + B runtime·callbackasm1(SB) + MOVD $642, R12 + B runtime·callbackasm1(SB) + MOVD $643, R12 + B runtime·callbackasm1(SB) + MOVD $644, R12 + B runtime·callbackasm1(SB) + MOVD $645, R12 + B runtime·callbackasm1(SB) + MOVD $646, R12 + B runtime·callbackasm1(SB) + MOVD $647, R12 + B runtime·callbackasm1(SB) + MOVD $648, R12 + B runtime·callbackasm1(SB) + MOVD $649, R12 + B runtime·callbackasm1(SB) + MOVD $650, R12 + B runtime·callbackasm1(SB) + MOVD $651, R12 + B runtime·callbackasm1(SB) + MOVD $652, R12 + B runtime·callbackasm1(SB) + MOVD $653, R12 + B runtime·callbackasm1(SB) + MOVD $654, R12 + B runtime·callbackasm1(SB) + MOVD $655, R12 + B runtime·callbackasm1(SB) + MOVD $656, R12 + B runtime·callbackasm1(SB) + MOVD $657, R12 + B runtime·callbackasm1(SB) + MOVD $658, R12 + B runtime·callbackasm1(SB) + MOVD $659, R12 + B runtime·callbackasm1(SB) + MOVD $660, R12 + B runtime·callbackasm1(SB) + MOVD $661, R12 + B runtime·callbackasm1(SB) + MOVD $662, R12 + B runtime·callbackasm1(SB) + MOVD $663, R12 + B runtime·callbackasm1(SB) + MOVD $664, R12 + B runtime·callbackasm1(SB) + MOVD $665, R12 + B runtime·callbackasm1(SB) + MOVD $666, R12 + B runtime·callbackasm1(SB) + MOVD $667, R12 + B runtime·callbackasm1(SB) + MOVD $668, R12 + B runtime·callbackasm1(SB) + MOVD $669, R12 + B runtime·callbackasm1(SB) + MOVD $670, R12 + B runtime·callbackasm1(SB) + MOVD $671, R12 + B runtime·callbackasm1(SB) + MOVD $672, R12 + B runtime·callbackasm1(SB) + MOVD $673, R12 + B runtime·callbackasm1(SB) + MOVD $674, R12 + B runtime·callbackasm1(SB) + MOVD $675, R12 + B runtime·callbackasm1(SB) + MOVD $676, R12 + B runtime·callbackasm1(SB) + MOVD $677, R12 + B runtime·callbackasm1(SB) + MOVD $678, R12 + B runtime·callbackasm1(SB) + MOVD $679, R12 + B runtime·callbackasm1(SB) + MOVD $680, R12 + B runtime·callbackasm1(SB) + MOVD $681, R12 + B runtime·callbackasm1(SB) + MOVD $682, R12 + B runtime·callbackasm1(SB) + MOVD $683, R12 + B runtime·callbackasm1(SB) + MOVD $684, R12 + B runtime·callbackasm1(SB) + MOVD $685, R12 + B runtime·callbackasm1(SB) + MOVD $686, R12 + B runtime·callbackasm1(SB) + MOVD $687, R12 + B runtime·callbackasm1(SB) + MOVD $688, R12 + B runtime·callbackasm1(SB) + MOVD $689, R12 + B runtime·callbackasm1(SB) + MOVD $690, R12 + B runtime·callbackasm1(SB) + MOVD $691, R12 + B runtime·callbackasm1(SB) + MOVD $692, R12 + B runtime·callbackasm1(SB) + MOVD $693, R12 + B runtime·callbackasm1(SB) + MOVD $694, R12 + B runtime·callbackasm1(SB) + MOVD $695, R12 + B runtime·callbackasm1(SB) + MOVD $696, R12 + B runtime·callbackasm1(SB) + MOVD $697, R12 + B runtime·callbackasm1(SB) + MOVD $698, R12 + B runtime·callbackasm1(SB) + MOVD $699, R12 + B runtime·callbackasm1(SB) + MOVD $700, R12 + B runtime·callbackasm1(SB) + MOVD $701, R12 + B runtime·callbackasm1(SB) + MOVD $702, R12 + B runtime·callbackasm1(SB) + MOVD $703, R12 + B runtime·callbackasm1(SB) + MOVD $704, R12 + B runtime·callbackasm1(SB) + MOVD $705, R12 + B runtime·callbackasm1(SB) + MOVD $706, R12 + B runtime·callbackasm1(SB) + MOVD $707, R12 + B runtime·callbackasm1(SB) + MOVD $708, R12 + B runtime·callbackasm1(SB) + MOVD $709, R12 + B runtime·callbackasm1(SB) + MOVD $710, R12 + B runtime·callbackasm1(SB) + MOVD $711, R12 + B runtime·callbackasm1(SB) + MOVD $712, R12 + B runtime·callbackasm1(SB) + MOVD $713, R12 + B runtime·callbackasm1(SB) + MOVD $714, R12 + B runtime·callbackasm1(SB) + MOVD $715, R12 + B runtime·callbackasm1(SB) + MOVD $716, R12 + B runtime·callbackasm1(SB) + MOVD $717, R12 + B runtime·callbackasm1(SB) + MOVD $718, R12 + B runtime·callbackasm1(SB) + MOVD $719, R12 + B runtime·callbackasm1(SB) + MOVD $720, R12 + B runtime·callbackasm1(SB) + MOVD $721, R12 + B runtime·callbackasm1(SB) + MOVD $722, R12 + B runtime·callbackasm1(SB) + MOVD $723, R12 + B runtime·callbackasm1(SB) + MOVD $724, R12 + B runtime·callbackasm1(SB) + MOVD $725, R12 + B runtime·callbackasm1(SB) + MOVD $726, R12 + B runtime·callbackasm1(SB) + MOVD $727, R12 + B runtime·callbackasm1(SB) + MOVD $728, R12 + B runtime·callbackasm1(SB) + MOVD $729, R12 + B runtime·callbackasm1(SB) + MOVD $730, R12 + B runtime·callbackasm1(SB) + MOVD $731, R12 + B runtime·callbackasm1(SB) + MOVD $732, R12 + B runtime·callbackasm1(SB) + MOVD $733, R12 + B runtime·callbackasm1(SB) + MOVD $734, R12 + B runtime·callbackasm1(SB) + MOVD $735, R12 + B runtime·callbackasm1(SB) + MOVD $736, R12 + B runtime·callbackasm1(SB) + MOVD $737, R12 + B runtime·callbackasm1(SB) + MOVD $738, R12 + B runtime·callbackasm1(SB) + MOVD $739, R12 + B runtime·callbackasm1(SB) + MOVD $740, R12 + B runtime·callbackasm1(SB) + MOVD $741, R12 + B runtime·callbackasm1(SB) + MOVD $742, R12 + B runtime·callbackasm1(SB) + MOVD $743, R12 + B runtime·callbackasm1(SB) + MOVD $744, R12 + B runtime·callbackasm1(SB) + MOVD $745, R12 + B runtime·callbackasm1(SB) + MOVD $746, R12 + B runtime·callbackasm1(SB) + MOVD $747, R12 + B runtime·callbackasm1(SB) + MOVD $748, R12 + B runtime·callbackasm1(SB) + MOVD $749, R12 + B runtime·callbackasm1(SB) + MOVD $750, R12 + B runtime·callbackasm1(SB) + MOVD $751, R12 + B runtime·callbackasm1(SB) + MOVD $752, R12 + B runtime·callbackasm1(SB) + MOVD $753, R12 + B runtime·callbackasm1(SB) + MOVD $754, R12 + B runtime·callbackasm1(SB) + MOVD $755, R12 + B runtime·callbackasm1(SB) + MOVD $756, R12 + B runtime·callbackasm1(SB) + MOVD $757, R12 + B runtime·callbackasm1(SB) + MOVD $758, R12 + B runtime·callbackasm1(SB) + MOVD $759, R12 + B runtime·callbackasm1(SB) + MOVD $760, R12 + B runtime·callbackasm1(SB) + MOVD $761, R12 + B runtime·callbackasm1(SB) + MOVD $762, R12 + B runtime·callbackasm1(SB) + MOVD $763, R12 + B runtime·callbackasm1(SB) + MOVD $764, R12 + B runtime·callbackasm1(SB) + MOVD $765, R12 + B runtime·callbackasm1(SB) + MOVD $766, R12 + B runtime·callbackasm1(SB) + MOVD $767, R12 + B runtime·callbackasm1(SB) + MOVD $768, R12 + B runtime·callbackasm1(SB) + MOVD $769, R12 + B runtime·callbackasm1(SB) + MOVD $770, R12 + B runtime·callbackasm1(SB) + MOVD $771, R12 + B runtime·callbackasm1(SB) + MOVD $772, R12 + B runtime·callbackasm1(SB) + MOVD $773, R12 + B runtime·callbackasm1(SB) + MOVD $774, R12 + B runtime·callbackasm1(SB) + MOVD $775, R12 + B runtime·callbackasm1(SB) + MOVD $776, R12 + B runtime·callbackasm1(SB) + MOVD $777, R12 + B runtime·callbackasm1(SB) + MOVD $778, R12 + B runtime·callbackasm1(SB) + MOVD $779, R12 + B runtime·callbackasm1(SB) + MOVD $780, R12 + B runtime·callbackasm1(SB) + MOVD $781, R12 + B runtime·callbackasm1(SB) + MOVD $782, R12 + B runtime·callbackasm1(SB) + MOVD $783, R12 + B runtime·callbackasm1(SB) + MOVD $784, R12 + B runtime·callbackasm1(SB) + MOVD $785, R12 + B runtime·callbackasm1(SB) + MOVD $786, R12 + B runtime·callbackasm1(SB) + MOVD $787, R12 + B runtime·callbackasm1(SB) + MOVD $788, R12 + B runtime·callbackasm1(SB) + MOVD $789, R12 + B runtime·callbackasm1(SB) + MOVD $790, R12 + B runtime·callbackasm1(SB) + MOVD $791, R12 + B runtime·callbackasm1(SB) + MOVD $792, R12 + B runtime·callbackasm1(SB) + MOVD $793, R12 + B runtime·callbackasm1(SB) + MOVD $794, R12 + B runtime·callbackasm1(SB) + MOVD $795, R12 + B runtime·callbackasm1(SB) + MOVD $796, R12 + B runtime·callbackasm1(SB) + MOVD $797, R12 + B runtime·callbackasm1(SB) + MOVD $798, R12 + B runtime·callbackasm1(SB) + MOVD $799, R12 + B runtime·callbackasm1(SB) + MOVD $800, R12 + B runtime·callbackasm1(SB) + MOVD $801, R12 + B runtime·callbackasm1(SB) + MOVD $802, R12 + B runtime·callbackasm1(SB) + MOVD $803, R12 + B runtime·callbackasm1(SB) + MOVD $804, R12 + B runtime·callbackasm1(SB) + MOVD $805, R12 + B runtime·callbackasm1(SB) + MOVD $806, R12 + B runtime·callbackasm1(SB) + MOVD $807, R12 + B runtime·callbackasm1(SB) + MOVD $808, R12 + B runtime·callbackasm1(SB) + MOVD $809, R12 + B runtime·callbackasm1(SB) + MOVD $810, R12 + B runtime·callbackasm1(SB) + MOVD $811, R12 + B runtime·callbackasm1(SB) + MOVD $812, R12 + B runtime·callbackasm1(SB) + MOVD $813, R12 + B runtime·callbackasm1(SB) + MOVD $814, R12 + B runtime·callbackasm1(SB) + MOVD $815, R12 + B runtime·callbackasm1(SB) + MOVD $816, R12 + B runtime·callbackasm1(SB) + MOVD $817, R12 + B runtime·callbackasm1(SB) + MOVD $818, R12 + B runtime·callbackasm1(SB) + MOVD $819, R12 + B runtime·callbackasm1(SB) + MOVD $820, R12 + B runtime·callbackasm1(SB) + MOVD $821, R12 + B runtime·callbackasm1(SB) + MOVD $822, R12 + B runtime·callbackasm1(SB) + MOVD $823, R12 + B runtime·callbackasm1(SB) + MOVD $824, R12 + B runtime·callbackasm1(SB) + MOVD $825, R12 + B runtime·callbackasm1(SB) + MOVD $826, R12 + B runtime·callbackasm1(SB) + MOVD $827, R12 + B runtime·callbackasm1(SB) + MOVD $828, R12 + B runtime·callbackasm1(SB) + MOVD $829, R12 + B runtime·callbackasm1(SB) + MOVD $830, R12 + B runtime·callbackasm1(SB) + MOVD $831, R12 + B runtime·callbackasm1(SB) + MOVD $832, R12 + B runtime·callbackasm1(SB) + MOVD $833, R12 + B runtime·callbackasm1(SB) + MOVD $834, R12 + B runtime·callbackasm1(SB) + MOVD $835, R12 + B runtime·callbackasm1(SB) + MOVD $836, R12 + B runtime·callbackasm1(SB) + MOVD $837, R12 + B runtime·callbackasm1(SB) + MOVD $838, R12 + B runtime·callbackasm1(SB) + MOVD $839, R12 + B runtime·callbackasm1(SB) + MOVD $840, R12 + B runtime·callbackasm1(SB) + MOVD $841, R12 + B runtime·callbackasm1(SB) + MOVD $842, R12 + B runtime·callbackasm1(SB) + MOVD $843, R12 + B runtime·callbackasm1(SB) + MOVD $844, R12 + B runtime·callbackasm1(SB) + MOVD $845, R12 + B runtime·callbackasm1(SB) + MOVD $846, R12 + B runtime·callbackasm1(SB) + MOVD $847, R12 + B runtime·callbackasm1(SB) + MOVD $848, R12 + B runtime·callbackasm1(SB) + MOVD $849, R12 + B runtime·callbackasm1(SB) + MOVD $850, R12 + B runtime·callbackasm1(SB) + MOVD $851, R12 + B runtime·callbackasm1(SB) + MOVD $852, R12 + B runtime·callbackasm1(SB) + MOVD $853, R12 + B runtime·callbackasm1(SB) + MOVD $854, R12 + B runtime·callbackasm1(SB) + MOVD $855, R12 + B runtime·callbackasm1(SB) + MOVD $856, R12 + B runtime·callbackasm1(SB) + MOVD $857, R12 + B runtime·callbackasm1(SB) + MOVD $858, R12 + B runtime·callbackasm1(SB) + MOVD $859, R12 + B runtime·callbackasm1(SB) + MOVD $860, R12 + B runtime·callbackasm1(SB) + MOVD $861, R12 + B runtime·callbackasm1(SB) + MOVD $862, R12 + B runtime·callbackasm1(SB) + MOVD $863, R12 + B runtime·callbackasm1(SB) + MOVD $864, R12 + B runtime·callbackasm1(SB) + MOVD $865, R12 + B runtime·callbackasm1(SB) + MOVD $866, R12 + B runtime·callbackasm1(SB) + MOVD $867, R12 + B runtime·callbackasm1(SB) + MOVD $868, R12 + B runtime·callbackasm1(SB) + MOVD $869, R12 + B runtime·callbackasm1(SB) + MOVD $870, R12 + B runtime·callbackasm1(SB) + MOVD $871, R12 + B runtime·callbackasm1(SB) + MOVD $872, R12 + B runtime·callbackasm1(SB) + MOVD $873, R12 + B runtime·callbackasm1(SB) + MOVD $874, R12 + B runtime·callbackasm1(SB) + MOVD $875, R12 + B runtime·callbackasm1(SB) + MOVD $876, R12 + B runtime·callbackasm1(SB) + MOVD $877, R12 + B runtime·callbackasm1(SB) + MOVD $878, R12 + B runtime·callbackasm1(SB) + MOVD $879, R12 + B runtime·callbackasm1(SB) + MOVD $880, R12 + B runtime·callbackasm1(SB) + MOVD $881, R12 + B runtime·callbackasm1(SB) + MOVD $882, R12 + B runtime·callbackasm1(SB) + MOVD $883, R12 + B runtime·callbackasm1(SB) + MOVD $884, R12 + B runtime·callbackasm1(SB) + MOVD $885, R12 + B runtime·callbackasm1(SB) + MOVD $886, R12 + B runtime·callbackasm1(SB) + MOVD $887, R12 + B runtime·callbackasm1(SB) + MOVD $888, R12 + B runtime·callbackasm1(SB) + MOVD $889, R12 + B runtime·callbackasm1(SB) + MOVD $890, R12 + B runtime·callbackasm1(SB) + MOVD $891, R12 + B runtime·callbackasm1(SB) + MOVD $892, R12 + B runtime·callbackasm1(SB) + MOVD $893, R12 + B runtime·callbackasm1(SB) + MOVD $894, R12 + B runtime·callbackasm1(SB) + MOVD $895, R12 + B runtime·callbackasm1(SB) + MOVD $896, R12 + B runtime·callbackasm1(SB) + MOVD $897, R12 + B runtime·callbackasm1(SB) + MOVD $898, R12 + B runtime·callbackasm1(SB) + MOVD $899, R12 + B runtime·callbackasm1(SB) + MOVD $900, R12 + B runtime·callbackasm1(SB) + MOVD $901, R12 + B runtime·callbackasm1(SB) + MOVD $902, R12 + B runtime·callbackasm1(SB) + MOVD $903, R12 + B runtime·callbackasm1(SB) + MOVD $904, R12 + B runtime·callbackasm1(SB) + MOVD $905, R12 + B runtime·callbackasm1(SB) + MOVD $906, R12 + B runtime·callbackasm1(SB) + MOVD $907, R12 + B runtime·callbackasm1(SB) + MOVD $908, R12 + B runtime·callbackasm1(SB) + MOVD $909, R12 + B runtime·callbackasm1(SB) + MOVD $910, R12 + B runtime·callbackasm1(SB) + MOVD $911, R12 + B runtime·callbackasm1(SB) + MOVD $912, R12 + B runtime·callbackasm1(SB) + MOVD $913, R12 + B runtime·callbackasm1(SB) + MOVD $914, R12 + B runtime·callbackasm1(SB) + MOVD $915, R12 + B runtime·callbackasm1(SB) + MOVD $916, R12 + B runtime·callbackasm1(SB) + MOVD $917, R12 + B runtime·callbackasm1(SB) + MOVD $918, R12 + B runtime·callbackasm1(SB) + MOVD $919, R12 + B runtime·callbackasm1(SB) + MOVD $920, R12 + B runtime·callbackasm1(SB) + MOVD $921, R12 + B runtime·callbackasm1(SB) + MOVD $922, R12 + B runtime·callbackasm1(SB) + MOVD $923, R12 + B runtime·callbackasm1(SB) + MOVD $924, R12 + B runtime·callbackasm1(SB) + MOVD $925, R12 + B runtime·callbackasm1(SB) + MOVD $926, R12 + B runtime·callbackasm1(SB) + MOVD $927, R12 + B runtime·callbackasm1(SB) + MOVD $928, R12 + B runtime·callbackasm1(SB) + MOVD $929, R12 + B runtime·callbackasm1(SB) + MOVD $930, R12 + B runtime·callbackasm1(SB) + MOVD $931, R12 + B runtime·callbackasm1(SB) + MOVD $932, R12 + B runtime·callbackasm1(SB) + MOVD $933, R12 + B runtime·callbackasm1(SB) + MOVD $934, R12 + B runtime·callbackasm1(SB) + MOVD $935, R12 + B runtime·callbackasm1(SB) + MOVD $936, R12 + B runtime·callbackasm1(SB) + MOVD $937, R12 + B runtime·callbackasm1(SB) + MOVD $938, R12 + B runtime·callbackasm1(SB) + MOVD $939, R12 + B runtime·callbackasm1(SB) + MOVD $940, R12 + B runtime·callbackasm1(SB) + MOVD $941, R12 + B runtime·callbackasm1(SB) + MOVD $942, R12 + B runtime·callbackasm1(SB) + MOVD $943, R12 + B runtime·callbackasm1(SB) + MOVD $944, R12 + B runtime·callbackasm1(SB) + MOVD $945, R12 + B runtime·callbackasm1(SB) + MOVD $946, R12 + B runtime·callbackasm1(SB) + MOVD $947, R12 + B runtime·callbackasm1(SB) + MOVD $948, R12 + B runtime·callbackasm1(SB) + MOVD $949, R12 + B runtime·callbackasm1(SB) + MOVD $950, R12 + B runtime·callbackasm1(SB) + MOVD $951, R12 + B runtime·callbackasm1(SB) + MOVD $952, R12 + B runtime·callbackasm1(SB) + MOVD $953, R12 + B runtime·callbackasm1(SB) + MOVD $954, R12 + B runtime·callbackasm1(SB) + MOVD $955, R12 + B runtime·callbackasm1(SB) + MOVD $956, R12 + B runtime·callbackasm1(SB) + MOVD $957, R12 + B runtime·callbackasm1(SB) + MOVD $958, R12 + B runtime·callbackasm1(SB) + MOVD $959, R12 + B runtime·callbackasm1(SB) + MOVD $960, R12 + B runtime·callbackasm1(SB) + MOVD $961, R12 + B runtime·callbackasm1(SB) + MOVD $962, R12 + B runtime·callbackasm1(SB) + MOVD $963, R12 + B runtime·callbackasm1(SB) + MOVD $964, R12 + B runtime·callbackasm1(SB) + MOVD $965, R12 + B runtime·callbackasm1(SB) + MOVD $966, R12 + B runtime·callbackasm1(SB) + MOVD $967, R12 + B runtime·callbackasm1(SB) + MOVD $968, R12 + B runtime·callbackasm1(SB) + MOVD $969, R12 + B runtime·callbackasm1(SB) + MOVD $970, R12 + B runtime·callbackasm1(SB) + MOVD $971, R12 + B runtime·callbackasm1(SB) + MOVD $972, R12 + B runtime·callbackasm1(SB) + MOVD $973, R12 + B runtime·callbackasm1(SB) + MOVD $974, R12 + B runtime·callbackasm1(SB) + MOVD $975, R12 + B runtime·callbackasm1(SB) + MOVD $976, R12 + B runtime·callbackasm1(SB) + MOVD $977, R12 + B runtime·callbackasm1(SB) + MOVD $978, R12 + B runtime·callbackasm1(SB) + MOVD $979, R12 + B runtime·callbackasm1(SB) + MOVD $980, R12 + B runtime·callbackasm1(SB) + MOVD $981, R12 + B runtime·callbackasm1(SB) + MOVD $982, R12 + B runtime·callbackasm1(SB) + MOVD $983, R12 + B runtime·callbackasm1(SB) + MOVD $984, R12 + B runtime·callbackasm1(SB) + MOVD $985, R12 + B runtime·callbackasm1(SB) + MOVD $986, R12 + B runtime·callbackasm1(SB) + MOVD $987, R12 + B runtime·callbackasm1(SB) + MOVD $988, R12 + B runtime·callbackasm1(SB) + MOVD $989, R12 + B runtime·callbackasm1(SB) + MOVD $990, R12 + B runtime·callbackasm1(SB) + MOVD $991, R12 + B runtime·callbackasm1(SB) + MOVD $992, R12 + B runtime·callbackasm1(SB) + MOVD $993, R12 + B runtime·callbackasm1(SB) + MOVD $994, R12 + B runtime·callbackasm1(SB) + MOVD $995, R12 + B runtime·callbackasm1(SB) + MOVD $996, R12 + B runtime·callbackasm1(SB) + MOVD $997, R12 + B runtime·callbackasm1(SB) + MOVD $998, R12 + B runtime·callbackasm1(SB) + MOVD $999, R12 + B runtime·callbackasm1(SB) + MOVD $1000, R12 + B runtime·callbackasm1(SB) + MOVD $1001, R12 + B runtime·callbackasm1(SB) + MOVD $1002, R12 + B runtime·callbackasm1(SB) + MOVD $1003, R12 + B runtime·callbackasm1(SB) + MOVD $1004, R12 + B runtime·callbackasm1(SB) + MOVD $1005, R12 + B runtime·callbackasm1(SB) + MOVD $1006, R12 + B runtime·callbackasm1(SB) + MOVD $1007, R12 + B runtime·callbackasm1(SB) + MOVD $1008, R12 + B runtime·callbackasm1(SB) + MOVD $1009, R12 + B runtime·callbackasm1(SB) + MOVD $1010, R12 + B runtime·callbackasm1(SB) + MOVD $1011, R12 + B runtime·callbackasm1(SB) + MOVD $1012, R12 + B runtime·callbackasm1(SB) + MOVD $1013, R12 + B runtime·callbackasm1(SB) + MOVD $1014, R12 + B runtime·callbackasm1(SB) + MOVD $1015, R12 + B runtime·callbackasm1(SB) + MOVD $1016, R12 + B runtime·callbackasm1(SB) + MOVD $1017, R12 + B runtime·callbackasm1(SB) + MOVD $1018, R12 + B runtime·callbackasm1(SB) + MOVD $1019, R12 + B runtime·callbackasm1(SB) + MOVD $1020, R12 + B runtime·callbackasm1(SB) + MOVD $1021, R12 + B runtime·callbackasm1(SB) + MOVD $1022, R12 + B runtime·callbackasm1(SB) + MOVD $1023, R12 + B runtime·callbackasm1(SB) + MOVD $1024, R12 + B runtime·callbackasm1(SB) + MOVD $1025, R12 + B runtime·callbackasm1(SB) + MOVD $1026, R12 + B runtime·callbackasm1(SB) + MOVD $1027, R12 + B runtime·callbackasm1(SB) + MOVD $1028, R12 + B runtime·callbackasm1(SB) + MOVD $1029, R12 + B runtime·callbackasm1(SB) + MOVD $1030, R12 + B runtime·callbackasm1(SB) + MOVD $1031, R12 + B runtime·callbackasm1(SB) + MOVD $1032, R12 + B runtime·callbackasm1(SB) + MOVD $1033, R12 + B runtime·callbackasm1(SB) + MOVD $1034, R12 + B runtime·callbackasm1(SB) + MOVD $1035, R12 + B runtime·callbackasm1(SB) + MOVD $1036, R12 + B runtime·callbackasm1(SB) + MOVD $1037, R12 + B runtime·callbackasm1(SB) + MOVD $1038, R12 + B runtime·callbackasm1(SB) + MOVD $1039, R12 + B runtime·callbackasm1(SB) + MOVD $1040, R12 + B runtime·callbackasm1(SB) + MOVD $1041, R12 + B runtime·callbackasm1(SB) + MOVD $1042, R12 + B runtime·callbackasm1(SB) + MOVD $1043, R12 + B runtime·callbackasm1(SB) + MOVD $1044, R12 + B runtime·callbackasm1(SB) + MOVD $1045, R12 + B runtime·callbackasm1(SB) + MOVD $1046, R12 + B runtime·callbackasm1(SB) + MOVD $1047, R12 + B runtime·callbackasm1(SB) + MOVD $1048, R12 + B runtime·callbackasm1(SB) + MOVD $1049, R12 + B runtime·callbackasm1(SB) + MOVD $1050, R12 + B runtime·callbackasm1(SB) + MOVD $1051, R12 + B runtime·callbackasm1(SB) + MOVD $1052, R12 + B runtime·callbackasm1(SB) + MOVD $1053, R12 + B runtime·callbackasm1(SB) + MOVD $1054, R12 + B runtime·callbackasm1(SB) + MOVD $1055, R12 + B runtime·callbackasm1(SB) + MOVD $1056, R12 + B runtime·callbackasm1(SB) + MOVD $1057, R12 + B runtime·callbackasm1(SB) + MOVD $1058, R12 + B runtime·callbackasm1(SB) + MOVD $1059, R12 + B runtime·callbackasm1(SB) + MOVD $1060, R12 + B runtime·callbackasm1(SB) + MOVD $1061, R12 + B runtime·callbackasm1(SB) + MOVD $1062, R12 + B runtime·callbackasm1(SB) + MOVD $1063, R12 + B runtime·callbackasm1(SB) + MOVD $1064, R12 + B runtime·callbackasm1(SB) + MOVD $1065, R12 + B runtime·callbackasm1(SB) + MOVD $1066, R12 + B runtime·callbackasm1(SB) + MOVD $1067, R12 + B runtime·callbackasm1(SB) + MOVD $1068, R12 + B runtime·callbackasm1(SB) + MOVD $1069, R12 + B runtime·callbackasm1(SB) + MOVD $1070, R12 + B runtime·callbackasm1(SB) + MOVD $1071, R12 + B runtime·callbackasm1(SB) + MOVD $1072, R12 + B runtime·callbackasm1(SB) + MOVD $1073, R12 + B runtime·callbackasm1(SB) + MOVD $1074, R12 + B runtime·callbackasm1(SB) + MOVD $1075, R12 + B runtime·callbackasm1(SB) + MOVD $1076, R12 + B runtime·callbackasm1(SB) + MOVD $1077, R12 + B runtime·callbackasm1(SB) + MOVD $1078, R12 + B runtime·callbackasm1(SB) + MOVD $1079, R12 + B runtime·callbackasm1(SB) + MOVD $1080, R12 + B runtime·callbackasm1(SB) + MOVD $1081, R12 + B runtime·callbackasm1(SB) + MOVD $1082, R12 + B runtime·callbackasm1(SB) + MOVD $1083, R12 + B runtime·callbackasm1(SB) + MOVD $1084, R12 + B runtime·callbackasm1(SB) + MOVD $1085, R12 + B runtime·callbackasm1(SB) + MOVD $1086, R12 + B runtime·callbackasm1(SB) + MOVD $1087, R12 + B runtime·callbackasm1(SB) + MOVD $1088, R12 + B runtime·callbackasm1(SB) + MOVD $1089, R12 + B runtime·callbackasm1(SB) + MOVD $1090, R12 + B runtime·callbackasm1(SB) + MOVD $1091, R12 + B runtime·callbackasm1(SB) + MOVD $1092, R12 + B runtime·callbackasm1(SB) + MOVD $1093, R12 + B runtime·callbackasm1(SB) + MOVD $1094, R12 + B runtime·callbackasm1(SB) + MOVD $1095, R12 + B runtime·callbackasm1(SB) + MOVD $1096, R12 + B runtime·callbackasm1(SB) + MOVD $1097, R12 + B runtime·callbackasm1(SB) + MOVD $1098, R12 + B runtime·callbackasm1(SB) + MOVD $1099, R12 + B runtime·callbackasm1(SB) + MOVD $1100, R12 + B runtime·callbackasm1(SB) + MOVD $1101, R12 + B runtime·callbackasm1(SB) + MOVD $1102, R12 + B runtime·callbackasm1(SB) + MOVD $1103, R12 + B runtime·callbackasm1(SB) + MOVD $1104, R12 + B runtime·callbackasm1(SB) + MOVD $1105, R12 + B runtime·callbackasm1(SB) + MOVD $1106, R12 + B runtime·callbackasm1(SB) + MOVD $1107, R12 + B runtime·callbackasm1(SB) + MOVD $1108, R12 + B runtime·callbackasm1(SB) + MOVD $1109, R12 + B runtime·callbackasm1(SB) + MOVD $1110, R12 + B runtime·callbackasm1(SB) + MOVD $1111, R12 + B runtime·callbackasm1(SB) + MOVD $1112, R12 + B runtime·callbackasm1(SB) + MOVD $1113, R12 + B runtime·callbackasm1(SB) + MOVD $1114, R12 + B runtime·callbackasm1(SB) + MOVD $1115, R12 + B runtime·callbackasm1(SB) + MOVD $1116, R12 + B runtime·callbackasm1(SB) + MOVD $1117, R12 + B runtime·callbackasm1(SB) + MOVD $1118, R12 + B runtime·callbackasm1(SB) + MOVD $1119, R12 + B runtime·callbackasm1(SB) + MOVD $1120, R12 + B runtime·callbackasm1(SB) + MOVD $1121, R12 + B runtime·callbackasm1(SB) + MOVD $1122, R12 + B runtime·callbackasm1(SB) + MOVD $1123, R12 + B runtime·callbackasm1(SB) + MOVD $1124, R12 + B runtime·callbackasm1(SB) + MOVD $1125, R12 + B runtime·callbackasm1(SB) + MOVD $1126, R12 + B runtime·callbackasm1(SB) + MOVD $1127, R12 + B runtime·callbackasm1(SB) + MOVD $1128, R12 + B runtime·callbackasm1(SB) + MOVD $1129, R12 + B runtime·callbackasm1(SB) + MOVD $1130, R12 + B runtime·callbackasm1(SB) + MOVD $1131, R12 + B runtime·callbackasm1(SB) + MOVD $1132, R12 + B runtime·callbackasm1(SB) + MOVD $1133, R12 + B runtime·callbackasm1(SB) + MOVD $1134, R12 + B runtime·callbackasm1(SB) + MOVD $1135, R12 + B runtime·callbackasm1(SB) + MOVD $1136, R12 + B runtime·callbackasm1(SB) + MOVD $1137, R12 + B runtime·callbackasm1(SB) + MOVD $1138, R12 + B runtime·callbackasm1(SB) + MOVD $1139, R12 + B runtime·callbackasm1(SB) + MOVD $1140, R12 + B runtime·callbackasm1(SB) + MOVD $1141, R12 + B runtime·callbackasm1(SB) + MOVD $1142, R12 + B runtime·callbackasm1(SB) + MOVD $1143, R12 + B runtime·callbackasm1(SB) + MOVD $1144, R12 + B runtime·callbackasm1(SB) + MOVD $1145, R12 + B runtime·callbackasm1(SB) + MOVD $1146, R12 + B runtime·callbackasm1(SB) + MOVD $1147, R12 + B runtime·callbackasm1(SB) + MOVD $1148, R12 + B runtime·callbackasm1(SB) + MOVD $1149, R12 + B runtime·callbackasm1(SB) + MOVD $1150, R12 + B runtime·callbackasm1(SB) + MOVD $1151, R12 + B runtime·callbackasm1(SB) + MOVD $1152, R12 + B runtime·callbackasm1(SB) + MOVD $1153, R12 + B runtime·callbackasm1(SB) + MOVD $1154, R12 + B runtime·callbackasm1(SB) + MOVD $1155, R12 + B runtime·callbackasm1(SB) + MOVD $1156, R12 + B runtime·callbackasm1(SB) + MOVD $1157, R12 + B runtime·callbackasm1(SB) + MOVD $1158, R12 + B runtime·callbackasm1(SB) + MOVD $1159, R12 + B runtime·callbackasm1(SB) + MOVD $1160, R12 + B runtime·callbackasm1(SB) + MOVD $1161, R12 + B runtime·callbackasm1(SB) + MOVD $1162, R12 + B runtime·callbackasm1(SB) + MOVD $1163, R12 + B runtime·callbackasm1(SB) + MOVD $1164, R12 + B runtime·callbackasm1(SB) + MOVD $1165, R12 + B runtime·callbackasm1(SB) + MOVD $1166, R12 + B runtime·callbackasm1(SB) + MOVD $1167, R12 + B runtime·callbackasm1(SB) + MOVD $1168, R12 + B runtime·callbackasm1(SB) + MOVD $1169, R12 + B runtime·callbackasm1(SB) + MOVD $1170, R12 + B runtime·callbackasm1(SB) + MOVD $1171, R12 + B runtime·callbackasm1(SB) + MOVD $1172, R12 + B runtime·callbackasm1(SB) + MOVD $1173, R12 + B runtime·callbackasm1(SB) + MOVD $1174, R12 + B runtime·callbackasm1(SB) + MOVD $1175, R12 + B runtime·callbackasm1(SB) + MOVD $1176, R12 + B runtime·callbackasm1(SB) + MOVD $1177, R12 + B runtime·callbackasm1(SB) + MOVD $1178, R12 + B runtime·callbackasm1(SB) + MOVD $1179, R12 + B runtime·callbackasm1(SB) + MOVD $1180, R12 + B runtime·callbackasm1(SB) + MOVD $1181, R12 + B runtime·callbackasm1(SB) + MOVD $1182, R12 + B runtime·callbackasm1(SB) + MOVD $1183, R12 + B runtime·callbackasm1(SB) + MOVD $1184, R12 + B runtime·callbackasm1(SB) + MOVD $1185, R12 + B runtime·callbackasm1(SB) + MOVD $1186, R12 + B runtime·callbackasm1(SB) + MOVD $1187, R12 + B runtime·callbackasm1(SB) + MOVD $1188, R12 + B runtime·callbackasm1(SB) + MOVD $1189, R12 + B runtime·callbackasm1(SB) + MOVD $1190, R12 + B runtime·callbackasm1(SB) + MOVD $1191, R12 + B runtime·callbackasm1(SB) + MOVD $1192, R12 + B runtime·callbackasm1(SB) + MOVD $1193, R12 + B runtime·callbackasm1(SB) + MOVD $1194, R12 + B runtime·callbackasm1(SB) + MOVD $1195, R12 + B runtime·callbackasm1(SB) + MOVD $1196, R12 + B runtime·callbackasm1(SB) + MOVD $1197, R12 + B runtime·callbackasm1(SB) + MOVD $1198, R12 + B runtime·callbackasm1(SB) + MOVD $1199, R12 + B runtime·callbackasm1(SB) + MOVD $1200, R12 + B runtime·callbackasm1(SB) + MOVD $1201, R12 + B runtime·callbackasm1(SB) + MOVD $1202, R12 + B runtime·callbackasm1(SB) + MOVD $1203, R12 + B runtime·callbackasm1(SB) + MOVD $1204, R12 + B runtime·callbackasm1(SB) + MOVD $1205, R12 + B runtime·callbackasm1(SB) + MOVD $1206, R12 + B runtime·callbackasm1(SB) + MOVD $1207, R12 + B runtime·callbackasm1(SB) + MOVD $1208, R12 + B runtime·callbackasm1(SB) + MOVD $1209, R12 + B runtime·callbackasm1(SB) + MOVD $1210, R12 + B runtime·callbackasm1(SB) + MOVD $1211, R12 + B runtime·callbackasm1(SB) + MOVD $1212, R12 + B runtime·callbackasm1(SB) + MOVD $1213, R12 + B runtime·callbackasm1(SB) + MOVD $1214, R12 + B runtime·callbackasm1(SB) + MOVD $1215, R12 + B runtime·callbackasm1(SB) + MOVD $1216, R12 + B runtime·callbackasm1(SB) + MOVD $1217, R12 + B runtime·callbackasm1(SB) + MOVD $1218, R12 + B runtime·callbackasm1(SB) + MOVD $1219, R12 + B runtime·callbackasm1(SB) + MOVD $1220, R12 + B runtime·callbackasm1(SB) + MOVD $1221, R12 + B runtime·callbackasm1(SB) + MOVD $1222, R12 + B runtime·callbackasm1(SB) + MOVD $1223, R12 + B runtime·callbackasm1(SB) + MOVD $1224, R12 + B runtime·callbackasm1(SB) + MOVD $1225, R12 + B runtime·callbackasm1(SB) + MOVD $1226, R12 + B runtime·callbackasm1(SB) + MOVD $1227, R12 + B runtime·callbackasm1(SB) + MOVD $1228, R12 + B runtime·callbackasm1(SB) + MOVD $1229, R12 + B runtime·callbackasm1(SB) + MOVD $1230, R12 + B runtime·callbackasm1(SB) + MOVD $1231, R12 + B runtime·callbackasm1(SB) + MOVD $1232, R12 + B runtime·callbackasm1(SB) + MOVD $1233, R12 + B runtime·callbackasm1(SB) + MOVD $1234, R12 + B runtime·callbackasm1(SB) + MOVD $1235, R12 + B runtime·callbackasm1(SB) + MOVD $1236, R12 + B runtime·callbackasm1(SB) + MOVD $1237, R12 + B runtime·callbackasm1(SB) + MOVD $1238, R12 + B runtime·callbackasm1(SB) + MOVD $1239, R12 + B runtime·callbackasm1(SB) + MOVD $1240, R12 + B runtime·callbackasm1(SB) + MOVD $1241, R12 + B runtime·callbackasm1(SB) + MOVD $1242, R12 + B runtime·callbackasm1(SB) + MOVD $1243, R12 + B runtime·callbackasm1(SB) + MOVD $1244, R12 + B runtime·callbackasm1(SB) + MOVD $1245, R12 + B runtime·callbackasm1(SB) + MOVD $1246, R12 + B runtime·callbackasm1(SB) + MOVD $1247, R12 + B runtime·callbackasm1(SB) + MOVD $1248, R12 + B runtime·callbackasm1(SB) + MOVD $1249, R12 + B runtime·callbackasm1(SB) + MOVD $1250, R12 + B runtime·callbackasm1(SB) + MOVD $1251, R12 + B runtime·callbackasm1(SB) + MOVD $1252, R12 + B runtime·callbackasm1(SB) + MOVD $1253, R12 + B runtime·callbackasm1(SB) + MOVD $1254, R12 + B runtime·callbackasm1(SB) + MOVD $1255, R12 + B runtime·callbackasm1(SB) + MOVD $1256, R12 + B runtime·callbackasm1(SB) + MOVD $1257, R12 + B runtime·callbackasm1(SB) + MOVD $1258, R12 + B runtime·callbackasm1(SB) + MOVD $1259, R12 + B runtime·callbackasm1(SB) + MOVD $1260, R12 + B runtime·callbackasm1(SB) + MOVD $1261, R12 + B runtime·callbackasm1(SB) + MOVD $1262, R12 + B runtime·callbackasm1(SB) + MOVD $1263, R12 + B runtime·callbackasm1(SB) + MOVD $1264, R12 + B runtime·callbackasm1(SB) + MOVD $1265, R12 + B runtime·callbackasm1(SB) + MOVD $1266, R12 + B runtime·callbackasm1(SB) + MOVD $1267, R12 + B runtime·callbackasm1(SB) + MOVD $1268, R12 + B runtime·callbackasm1(SB) + MOVD $1269, R12 + B runtime·callbackasm1(SB) + MOVD $1270, R12 + B runtime·callbackasm1(SB) + MOVD $1271, R12 + B runtime·callbackasm1(SB) + MOVD $1272, R12 + B runtime·callbackasm1(SB) + MOVD $1273, R12 + B runtime·callbackasm1(SB) + MOVD $1274, R12 + B runtime·callbackasm1(SB) + MOVD $1275, R12 + B runtime·callbackasm1(SB) + MOVD $1276, R12 + B runtime·callbackasm1(SB) + MOVD $1277, R12 + B runtime·callbackasm1(SB) + MOVD $1278, R12 + B runtime·callbackasm1(SB) + MOVD $1279, R12 + B runtime·callbackasm1(SB) + MOVD $1280, R12 + B runtime·callbackasm1(SB) + MOVD $1281, R12 + B runtime·callbackasm1(SB) + MOVD $1282, R12 + B runtime·callbackasm1(SB) + MOVD $1283, R12 + B runtime·callbackasm1(SB) + MOVD $1284, R12 + B runtime·callbackasm1(SB) + MOVD $1285, R12 + B runtime·callbackasm1(SB) + MOVD $1286, R12 + B runtime·callbackasm1(SB) + MOVD $1287, R12 + B runtime·callbackasm1(SB) + MOVD $1288, R12 + B runtime·callbackasm1(SB) + MOVD $1289, R12 + B runtime·callbackasm1(SB) + MOVD $1290, R12 + B runtime·callbackasm1(SB) + MOVD $1291, R12 + B runtime·callbackasm1(SB) + MOVD $1292, R12 + B runtime·callbackasm1(SB) + MOVD $1293, R12 + B runtime·callbackasm1(SB) + MOVD $1294, R12 + B runtime·callbackasm1(SB) + MOVD $1295, R12 + B runtime·callbackasm1(SB) + MOVD $1296, R12 + B runtime·callbackasm1(SB) + MOVD $1297, R12 + B runtime·callbackasm1(SB) + MOVD $1298, R12 + B runtime·callbackasm1(SB) + MOVD $1299, R12 + B runtime·callbackasm1(SB) + MOVD $1300, R12 + B runtime·callbackasm1(SB) + MOVD $1301, R12 + B runtime·callbackasm1(SB) + MOVD $1302, R12 + B runtime·callbackasm1(SB) + MOVD $1303, R12 + B runtime·callbackasm1(SB) + MOVD $1304, R12 + B runtime·callbackasm1(SB) + MOVD $1305, R12 + B runtime·callbackasm1(SB) + MOVD $1306, R12 + B runtime·callbackasm1(SB) + MOVD $1307, R12 + B runtime·callbackasm1(SB) + MOVD $1308, R12 + B runtime·callbackasm1(SB) + MOVD $1309, R12 + B runtime·callbackasm1(SB) + MOVD $1310, R12 + B runtime·callbackasm1(SB) + MOVD $1311, R12 + B runtime·callbackasm1(SB) + MOVD $1312, R12 + B runtime·callbackasm1(SB) + MOVD $1313, R12 + B runtime·callbackasm1(SB) + MOVD $1314, R12 + B runtime·callbackasm1(SB) + MOVD $1315, R12 + B runtime·callbackasm1(SB) + MOVD $1316, R12 + B runtime·callbackasm1(SB) + MOVD $1317, R12 + B runtime·callbackasm1(SB) + MOVD $1318, R12 + B runtime·callbackasm1(SB) + MOVD $1319, R12 + B runtime·callbackasm1(SB) + MOVD $1320, R12 + B runtime·callbackasm1(SB) + MOVD $1321, R12 + B runtime·callbackasm1(SB) + MOVD $1322, R12 + B runtime·callbackasm1(SB) + MOVD $1323, R12 + B runtime·callbackasm1(SB) + MOVD $1324, R12 + B runtime·callbackasm1(SB) + MOVD $1325, R12 + B runtime·callbackasm1(SB) + MOVD $1326, R12 + B runtime·callbackasm1(SB) + MOVD $1327, R12 + B runtime·callbackasm1(SB) + MOVD $1328, R12 + B runtime·callbackasm1(SB) + MOVD $1329, R12 + B runtime·callbackasm1(SB) + MOVD $1330, R12 + B runtime·callbackasm1(SB) + MOVD $1331, R12 + B runtime·callbackasm1(SB) + MOVD $1332, R12 + B runtime·callbackasm1(SB) + MOVD $1333, R12 + B runtime·callbackasm1(SB) + MOVD $1334, R12 + B runtime·callbackasm1(SB) + MOVD $1335, R12 + B runtime·callbackasm1(SB) + MOVD $1336, R12 + B runtime·callbackasm1(SB) + MOVD $1337, R12 + B runtime·callbackasm1(SB) + MOVD $1338, R12 + B runtime·callbackasm1(SB) + MOVD $1339, R12 + B runtime·callbackasm1(SB) + MOVD $1340, R12 + B runtime·callbackasm1(SB) + MOVD $1341, R12 + B runtime·callbackasm1(SB) + MOVD $1342, R12 + B runtime·callbackasm1(SB) + MOVD $1343, R12 + B runtime·callbackasm1(SB) + MOVD $1344, R12 + B runtime·callbackasm1(SB) + MOVD $1345, R12 + B runtime·callbackasm1(SB) + MOVD $1346, R12 + B runtime·callbackasm1(SB) + MOVD $1347, R12 + B runtime·callbackasm1(SB) + MOVD $1348, R12 + B runtime·callbackasm1(SB) + MOVD $1349, R12 + B runtime·callbackasm1(SB) + MOVD $1350, R12 + B runtime·callbackasm1(SB) + MOVD $1351, R12 + B runtime·callbackasm1(SB) + MOVD $1352, R12 + B runtime·callbackasm1(SB) + MOVD $1353, R12 + B runtime·callbackasm1(SB) + MOVD $1354, R12 + B runtime·callbackasm1(SB) + MOVD $1355, R12 + B runtime·callbackasm1(SB) + MOVD $1356, R12 + B runtime·callbackasm1(SB) + MOVD $1357, R12 + B runtime·callbackasm1(SB) + MOVD $1358, R12 + B runtime·callbackasm1(SB) + MOVD $1359, R12 + B runtime·callbackasm1(SB) + MOVD $1360, R12 + B runtime·callbackasm1(SB) + MOVD $1361, R12 + B runtime·callbackasm1(SB) + MOVD $1362, R12 + B runtime·callbackasm1(SB) + MOVD $1363, R12 + B runtime·callbackasm1(SB) + MOVD $1364, R12 + B runtime·callbackasm1(SB) + MOVD $1365, R12 + B runtime·callbackasm1(SB) + MOVD $1366, R12 + B runtime·callbackasm1(SB) + MOVD $1367, R12 + B runtime·callbackasm1(SB) + MOVD $1368, R12 + B runtime·callbackasm1(SB) + MOVD $1369, R12 + B runtime·callbackasm1(SB) + MOVD $1370, R12 + B runtime·callbackasm1(SB) + MOVD $1371, R12 + B runtime·callbackasm1(SB) + MOVD $1372, R12 + B runtime·callbackasm1(SB) + MOVD $1373, R12 + B runtime·callbackasm1(SB) + MOVD $1374, R12 + B runtime·callbackasm1(SB) + MOVD $1375, R12 + B runtime·callbackasm1(SB) + MOVD $1376, R12 + B runtime·callbackasm1(SB) + MOVD $1377, R12 + B runtime·callbackasm1(SB) + MOVD $1378, R12 + B runtime·callbackasm1(SB) + MOVD $1379, R12 + B runtime·callbackasm1(SB) + MOVD $1380, R12 + B runtime·callbackasm1(SB) + MOVD $1381, R12 + B runtime·callbackasm1(SB) + MOVD $1382, R12 + B runtime·callbackasm1(SB) + MOVD $1383, R12 + B runtime·callbackasm1(SB) + MOVD $1384, R12 + B runtime·callbackasm1(SB) + MOVD $1385, R12 + B runtime·callbackasm1(SB) + MOVD $1386, R12 + B runtime·callbackasm1(SB) + MOVD $1387, R12 + B runtime·callbackasm1(SB) + MOVD $1388, R12 + B runtime·callbackasm1(SB) + MOVD $1389, R12 + B runtime·callbackasm1(SB) + MOVD $1390, R12 + B runtime·callbackasm1(SB) + MOVD $1391, R12 + B runtime·callbackasm1(SB) + MOVD $1392, R12 + B runtime·callbackasm1(SB) + MOVD $1393, R12 + B runtime·callbackasm1(SB) + MOVD $1394, R12 + B runtime·callbackasm1(SB) + MOVD $1395, R12 + B runtime·callbackasm1(SB) + MOVD $1396, R12 + B runtime·callbackasm1(SB) + MOVD $1397, R12 + B runtime·callbackasm1(SB) + MOVD $1398, R12 + B runtime·callbackasm1(SB) + MOVD $1399, R12 + B runtime·callbackasm1(SB) + MOVD $1400, R12 + B runtime·callbackasm1(SB) + MOVD $1401, R12 + B runtime·callbackasm1(SB) + MOVD $1402, R12 + B runtime·callbackasm1(SB) + MOVD $1403, R12 + B runtime·callbackasm1(SB) + MOVD $1404, R12 + B runtime·callbackasm1(SB) + MOVD $1405, R12 + B runtime·callbackasm1(SB) + MOVD $1406, R12 + B runtime·callbackasm1(SB) + MOVD $1407, R12 + B runtime·callbackasm1(SB) + MOVD $1408, R12 + B runtime·callbackasm1(SB) + MOVD $1409, R12 + B runtime·callbackasm1(SB) + MOVD $1410, R12 + B runtime·callbackasm1(SB) + MOVD $1411, R12 + B runtime·callbackasm1(SB) + MOVD $1412, R12 + B runtime·callbackasm1(SB) + MOVD $1413, R12 + B runtime·callbackasm1(SB) + MOVD $1414, R12 + B runtime·callbackasm1(SB) + MOVD $1415, R12 + B runtime·callbackasm1(SB) + MOVD $1416, R12 + B runtime·callbackasm1(SB) + MOVD $1417, R12 + B runtime·callbackasm1(SB) + MOVD $1418, R12 + B runtime·callbackasm1(SB) + MOVD $1419, R12 + B runtime·callbackasm1(SB) + MOVD $1420, R12 + B runtime·callbackasm1(SB) + MOVD $1421, R12 + B runtime·callbackasm1(SB) + MOVD $1422, R12 + B runtime·callbackasm1(SB) + MOVD $1423, R12 + B runtime·callbackasm1(SB) + MOVD $1424, R12 + B runtime·callbackasm1(SB) + MOVD $1425, R12 + B runtime·callbackasm1(SB) + MOVD $1426, R12 + B runtime·callbackasm1(SB) + MOVD $1427, R12 + B runtime·callbackasm1(SB) + MOVD $1428, R12 + B runtime·callbackasm1(SB) + MOVD $1429, R12 + B runtime·callbackasm1(SB) + MOVD $1430, R12 + B runtime·callbackasm1(SB) + MOVD $1431, R12 + B runtime·callbackasm1(SB) + MOVD $1432, R12 + B runtime·callbackasm1(SB) + MOVD $1433, R12 + B runtime·callbackasm1(SB) + MOVD $1434, R12 + B runtime·callbackasm1(SB) + MOVD $1435, R12 + B runtime·callbackasm1(SB) + MOVD $1436, R12 + B runtime·callbackasm1(SB) + MOVD $1437, R12 + B runtime·callbackasm1(SB) + MOVD $1438, R12 + B runtime·callbackasm1(SB) + MOVD $1439, R12 + B runtime·callbackasm1(SB) + MOVD $1440, R12 + B runtime·callbackasm1(SB) + MOVD $1441, R12 + B runtime·callbackasm1(SB) + MOVD $1442, R12 + B runtime·callbackasm1(SB) + MOVD $1443, R12 + B runtime·callbackasm1(SB) + MOVD $1444, R12 + B runtime·callbackasm1(SB) + MOVD $1445, R12 + B runtime·callbackasm1(SB) + MOVD $1446, R12 + B runtime·callbackasm1(SB) + MOVD $1447, R12 + B runtime·callbackasm1(SB) + MOVD $1448, R12 + B runtime·callbackasm1(SB) + MOVD $1449, R12 + B runtime·callbackasm1(SB) + MOVD $1450, R12 + B runtime·callbackasm1(SB) + MOVD $1451, R12 + B runtime·callbackasm1(SB) + MOVD $1452, R12 + B runtime·callbackasm1(SB) + MOVD $1453, R12 + B runtime·callbackasm1(SB) + MOVD $1454, R12 + B runtime·callbackasm1(SB) + MOVD $1455, R12 + B runtime·callbackasm1(SB) + MOVD $1456, R12 + B runtime·callbackasm1(SB) + MOVD $1457, R12 + B runtime·callbackasm1(SB) + MOVD $1458, R12 + B runtime·callbackasm1(SB) + MOVD $1459, R12 + B runtime·callbackasm1(SB) + MOVD $1460, R12 + B runtime·callbackasm1(SB) + MOVD $1461, R12 + B runtime·callbackasm1(SB) + MOVD $1462, R12 + B runtime·callbackasm1(SB) + MOVD $1463, R12 + B runtime·callbackasm1(SB) + MOVD $1464, R12 + B runtime·callbackasm1(SB) + MOVD $1465, R12 + B runtime·callbackasm1(SB) + MOVD $1466, R12 + B runtime·callbackasm1(SB) + MOVD $1467, R12 + B runtime·callbackasm1(SB) + MOVD $1468, R12 + B runtime·callbackasm1(SB) + MOVD $1469, R12 + B runtime·callbackasm1(SB) + MOVD $1470, R12 + B runtime·callbackasm1(SB) + MOVD $1471, R12 + B runtime·callbackasm1(SB) + MOVD $1472, R12 + B runtime·callbackasm1(SB) + MOVD $1473, R12 + B runtime·callbackasm1(SB) + MOVD $1474, R12 + B runtime·callbackasm1(SB) + MOVD $1475, R12 + B runtime·callbackasm1(SB) + MOVD $1476, R12 + B runtime·callbackasm1(SB) + MOVD $1477, R12 + B runtime·callbackasm1(SB) + MOVD $1478, R12 + B runtime·callbackasm1(SB) + MOVD $1479, R12 + B runtime·callbackasm1(SB) + MOVD $1480, R12 + B runtime·callbackasm1(SB) + MOVD $1481, R12 + B runtime·callbackasm1(SB) + MOVD $1482, R12 + B runtime·callbackasm1(SB) + MOVD $1483, R12 + B runtime·callbackasm1(SB) + MOVD $1484, R12 + B runtime·callbackasm1(SB) + MOVD $1485, R12 + B runtime·callbackasm1(SB) + MOVD $1486, R12 + B runtime·callbackasm1(SB) + MOVD $1487, R12 + B runtime·callbackasm1(SB) + MOVD $1488, R12 + B runtime·callbackasm1(SB) + MOVD $1489, R12 + B runtime·callbackasm1(SB) + MOVD $1490, R12 + B runtime·callbackasm1(SB) + MOVD $1491, R12 + B runtime·callbackasm1(SB) + MOVD $1492, R12 + B runtime·callbackasm1(SB) + MOVD $1493, R12 + B runtime·callbackasm1(SB) + MOVD $1494, R12 + B runtime·callbackasm1(SB) + MOVD $1495, R12 + B runtime·callbackasm1(SB) + MOVD $1496, R12 + B runtime·callbackasm1(SB) + MOVD $1497, R12 + B runtime·callbackasm1(SB) + MOVD $1498, R12 + B runtime·callbackasm1(SB) + MOVD $1499, R12 + B runtime·callbackasm1(SB) + MOVD $1500, R12 + B runtime·callbackasm1(SB) + MOVD $1501, R12 + B runtime·callbackasm1(SB) + MOVD $1502, R12 + B runtime·callbackasm1(SB) + MOVD $1503, R12 + B runtime·callbackasm1(SB) + MOVD $1504, R12 + B runtime·callbackasm1(SB) + MOVD $1505, R12 + B runtime·callbackasm1(SB) + MOVD $1506, R12 + B runtime·callbackasm1(SB) + MOVD $1507, R12 + B runtime·callbackasm1(SB) + MOVD $1508, R12 + B runtime·callbackasm1(SB) + MOVD $1509, R12 + B runtime·callbackasm1(SB) + MOVD $1510, R12 + B runtime·callbackasm1(SB) + MOVD $1511, R12 + B runtime·callbackasm1(SB) + MOVD $1512, R12 + B runtime·callbackasm1(SB) + MOVD $1513, R12 + B runtime·callbackasm1(SB) + MOVD $1514, R12 + B runtime·callbackasm1(SB) + MOVD $1515, R12 + B runtime·callbackasm1(SB) + MOVD $1516, R12 + B runtime·callbackasm1(SB) + MOVD $1517, R12 + B runtime·callbackasm1(SB) + MOVD $1518, R12 + B runtime·callbackasm1(SB) + MOVD $1519, R12 + B runtime·callbackasm1(SB) + MOVD $1520, R12 + B runtime·callbackasm1(SB) + MOVD $1521, R12 + B runtime·callbackasm1(SB) + MOVD $1522, R12 + B runtime·callbackasm1(SB) + MOVD $1523, R12 + B runtime·callbackasm1(SB) + MOVD $1524, R12 + B runtime·callbackasm1(SB) + MOVD $1525, R12 + B runtime·callbackasm1(SB) + MOVD $1526, R12 + B runtime·callbackasm1(SB) + MOVD $1527, R12 + B runtime·callbackasm1(SB) + MOVD $1528, R12 + B runtime·callbackasm1(SB) + MOVD $1529, R12 + B runtime·callbackasm1(SB) + MOVD $1530, R12 + B runtime·callbackasm1(SB) + MOVD $1531, R12 + B runtime·callbackasm1(SB) + MOVD $1532, R12 + B runtime·callbackasm1(SB) + MOVD $1533, R12 + B runtime·callbackasm1(SB) + MOVD $1534, R12 + B runtime·callbackasm1(SB) + MOVD $1535, R12 + B runtime·callbackasm1(SB) + MOVD $1536, R12 + B runtime·callbackasm1(SB) + MOVD $1537, R12 + B runtime·callbackasm1(SB) + MOVD $1538, R12 + B runtime·callbackasm1(SB) + MOVD $1539, R12 + B runtime·callbackasm1(SB) + MOVD $1540, R12 + B runtime·callbackasm1(SB) + MOVD $1541, R12 + B runtime·callbackasm1(SB) + MOVD $1542, R12 + B runtime·callbackasm1(SB) + MOVD $1543, R12 + B runtime·callbackasm1(SB) + MOVD $1544, R12 + B runtime·callbackasm1(SB) + MOVD $1545, R12 + B runtime·callbackasm1(SB) + MOVD $1546, R12 + B runtime·callbackasm1(SB) + MOVD $1547, R12 + B runtime·callbackasm1(SB) + MOVD $1548, R12 + B runtime·callbackasm1(SB) + MOVD $1549, R12 + B runtime·callbackasm1(SB) + MOVD $1550, R12 + B runtime·callbackasm1(SB) + MOVD $1551, R12 + B runtime·callbackasm1(SB) + MOVD $1552, R12 + B runtime·callbackasm1(SB) + MOVD $1553, R12 + B runtime·callbackasm1(SB) + MOVD $1554, R12 + B runtime·callbackasm1(SB) + MOVD $1555, R12 + B runtime·callbackasm1(SB) + MOVD $1556, R12 + B runtime·callbackasm1(SB) + MOVD $1557, R12 + B runtime·callbackasm1(SB) + MOVD $1558, R12 + B runtime·callbackasm1(SB) + MOVD $1559, R12 + B runtime·callbackasm1(SB) + MOVD $1560, R12 + B runtime·callbackasm1(SB) + MOVD $1561, R12 + B runtime·callbackasm1(SB) + MOVD $1562, R12 + B runtime·callbackasm1(SB) + MOVD $1563, R12 + B runtime·callbackasm1(SB) + MOVD $1564, R12 + B runtime·callbackasm1(SB) + MOVD $1565, R12 + B runtime·callbackasm1(SB) + MOVD $1566, R12 + B runtime·callbackasm1(SB) + MOVD $1567, R12 + B runtime·callbackasm1(SB) + MOVD $1568, R12 + B runtime·callbackasm1(SB) + MOVD $1569, R12 + B runtime·callbackasm1(SB) + MOVD $1570, R12 + B runtime·callbackasm1(SB) + MOVD $1571, R12 + B runtime·callbackasm1(SB) + MOVD $1572, R12 + B runtime·callbackasm1(SB) + MOVD $1573, R12 + B runtime·callbackasm1(SB) + MOVD $1574, R12 + B runtime·callbackasm1(SB) + MOVD $1575, R12 + B runtime·callbackasm1(SB) + MOVD $1576, R12 + B runtime·callbackasm1(SB) + MOVD $1577, R12 + B runtime·callbackasm1(SB) + MOVD $1578, R12 + B runtime·callbackasm1(SB) + MOVD $1579, R12 + B runtime·callbackasm1(SB) + MOVD $1580, R12 + B runtime·callbackasm1(SB) + MOVD $1581, R12 + B runtime·callbackasm1(SB) + MOVD $1582, R12 + B runtime·callbackasm1(SB) + MOVD $1583, R12 + B runtime·callbackasm1(SB) + MOVD $1584, R12 + B runtime·callbackasm1(SB) + MOVD $1585, R12 + B runtime·callbackasm1(SB) + MOVD $1586, R12 + B runtime·callbackasm1(SB) + MOVD $1587, R12 + B runtime·callbackasm1(SB) + MOVD $1588, R12 + B runtime·callbackasm1(SB) + MOVD $1589, R12 + B runtime·callbackasm1(SB) + MOVD $1590, R12 + B runtime·callbackasm1(SB) + MOVD $1591, R12 + B runtime·callbackasm1(SB) + MOVD $1592, R12 + B runtime·callbackasm1(SB) + MOVD $1593, R12 + B runtime·callbackasm1(SB) + MOVD $1594, R12 + B runtime·callbackasm1(SB) + MOVD $1595, R12 + B runtime·callbackasm1(SB) + MOVD $1596, R12 + B runtime·callbackasm1(SB) + MOVD $1597, R12 + B runtime·callbackasm1(SB) + MOVD $1598, R12 + B runtime·callbackasm1(SB) + MOVD $1599, R12 + B runtime·callbackasm1(SB) + MOVD $1600, R12 + B runtime·callbackasm1(SB) + MOVD $1601, R12 + B runtime·callbackasm1(SB) + MOVD $1602, R12 + B runtime·callbackasm1(SB) + MOVD $1603, R12 + B runtime·callbackasm1(SB) + MOVD $1604, R12 + B runtime·callbackasm1(SB) + MOVD $1605, R12 + B runtime·callbackasm1(SB) + MOVD $1606, R12 + B runtime·callbackasm1(SB) + MOVD $1607, R12 + B runtime·callbackasm1(SB) + MOVD $1608, R12 + B runtime·callbackasm1(SB) + MOVD $1609, R12 + B runtime·callbackasm1(SB) + MOVD $1610, R12 + B runtime·callbackasm1(SB) + MOVD $1611, R12 + B runtime·callbackasm1(SB) + MOVD $1612, R12 + B runtime·callbackasm1(SB) + MOVD $1613, R12 + B runtime·callbackasm1(SB) + MOVD $1614, R12 + B runtime·callbackasm1(SB) + MOVD $1615, R12 + B runtime·callbackasm1(SB) + MOVD $1616, R12 + B runtime·callbackasm1(SB) + MOVD $1617, R12 + B runtime·callbackasm1(SB) + MOVD $1618, R12 + B runtime·callbackasm1(SB) + MOVD $1619, R12 + B runtime·callbackasm1(SB) + MOVD $1620, R12 + B runtime·callbackasm1(SB) + MOVD $1621, R12 + B runtime·callbackasm1(SB) + MOVD $1622, R12 + B runtime·callbackasm1(SB) + MOVD $1623, R12 + B runtime·callbackasm1(SB) + MOVD $1624, R12 + B runtime·callbackasm1(SB) + MOVD $1625, R12 + B runtime·callbackasm1(SB) + MOVD $1626, R12 + B runtime·callbackasm1(SB) + MOVD $1627, R12 + B runtime·callbackasm1(SB) + MOVD $1628, R12 + B runtime·callbackasm1(SB) + MOVD $1629, R12 + B runtime·callbackasm1(SB) + MOVD $1630, R12 + B runtime·callbackasm1(SB) + MOVD $1631, R12 + B runtime·callbackasm1(SB) + MOVD $1632, R12 + B runtime·callbackasm1(SB) + MOVD $1633, R12 + B runtime·callbackasm1(SB) + MOVD $1634, R12 + B runtime·callbackasm1(SB) + MOVD $1635, R12 + B runtime·callbackasm1(SB) + MOVD $1636, R12 + B runtime·callbackasm1(SB) + MOVD $1637, R12 + B runtime·callbackasm1(SB) + MOVD $1638, R12 + B runtime·callbackasm1(SB) + MOVD $1639, R12 + B runtime·callbackasm1(SB) + MOVD $1640, R12 + B runtime·callbackasm1(SB) + MOVD $1641, R12 + B runtime·callbackasm1(SB) + MOVD $1642, R12 + B runtime·callbackasm1(SB) + MOVD $1643, R12 + B runtime·callbackasm1(SB) + MOVD $1644, R12 + B runtime·callbackasm1(SB) + MOVD $1645, R12 + B runtime·callbackasm1(SB) + MOVD $1646, R12 + B runtime·callbackasm1(SB) + MOVD $1647, R12 + B runtime·callbackasm1(SB) + MOVD $1648, R12 + B runtime·callbackasm1(SB) + MOVD $1649, R12 + B runtime·callbackasm1(SB) + MOVD $1650, R12 + B runtime·callbackasm1(SB) + MOVD $1651, R12 + B runtime·callbackasm1(SB) + MOVD $1652, R12 + B runtime·callbackasm1(SB) + MOVD $1653, R12 + B runtime·callbackasm1(SB) + MOVD $1654, R12 + B runtime·callbackasm1(SB) + MOVD $1655, R12 + B runtime·callbackasm1(SB) + MOVD $1656, R12 + B runtime·callbackasm1(SB) + MOVD $1657, R12 + B runtime·callbackasm1(SB) + MOVD $1658, R12 + B runtime·callbackasm1(SB) + MOVD $1659, R12 + B runtime·callbackasm1(SB) + MOVD $1660, R12 + B runtime·callbackasm1(SB) + MOVD $1661, R12 + B runtime·callbackasm1(SB) + MOVD $1662, R12 + B runtime·callbackasm1(SB) + MOVD $1663, R12 + B runtime·callbackasm1(SB) + MOVD $1664, R12 + B runtime·callbackasm1(SB) + MOVD $1665, R12 + B runtime·callbackasm1(SB) + MOVD $1666, R12 + B runtime·callbackasm1(SB) + MOVD $1667, R12 + B runtime·callbackasm1(SB) + MOVD $1668, R12 + B runtime·callbackasm1(SB) + MOVD $1669, R12 + B runtime·callbackasm1(SB) + MOVD $1670, R12 + B runtime·callbackasm1(SB) + MOVD $1671, R12 + B runtime·callbackasm1(SB) + MOVD $1672, R12 + B runtime·callbackasm1(SB) + MOVD $1673, R12 + B runtime·callbackasm1(SB) + MOVD $1674, R12 + B runtime·callbackasm1(SB) + MOVD $1675, R12 + B runtime·callbackasm1(SB) + MOVD $1676, R12 + B runtime·callbackasm1(SB) + MOVD $1677, R12 + B runtime·callbackasm1(SB) + MOVD $1678, R12 + B runtime·callbackasm1(SB) + MOVD $1679, R12 + B runtime·callbackasm1(SB) + MOVD $1680, R12 + B runtime·callbackasm1(SB) + MOVD $1681, R12 + B runtime·callbackasm1(SB) + MOVD $1682, R12 + B runtime·callbackasm1(SB) + MOVD $1683, R12 + B runtime·callbackasm1(SB) + MOVD $1684, R12 + B runtime·callbackasm1(SB) + MOVD $1685, R12 + B runtime·callbackasm1(SB) + MOVD $1686, R12 + B runtime·callbackasm1(SB) + MOVD $1687, R12 + B runtime·callbackasm1(SB) + MOVD $1688, R12 + B runtime·callbackasm1(SB) + MOVD $1689, R12 + B runtime·callbackasm1(SB) + MOVD $1690, R12 + B runtime·callbackasm1(SB) + MOVD $1691, R12 + B runtime·callbackasm1(SB) + MOVD $1692, R12 + B runtime·callbackasm1(SB) + MOVD $1693, R12 + B runtime·callbackasm1(SB) + MOVD $1694, R12 + B runtime·callbackasm1(SB) + MOVD $1695, R12 + B runtime·callbackasm1(SB) + MOVD $1696, R12 + B runtime·callbackasm1(SB) + MOVD $1697, R12 + B runtime·callbackasm1(SB) + MOVD $1698, R12 + B runtime·callbackasm1(SB) + MOVD $1699, R12 + B runtime·callbackasm1(SB) + MOVD $1700, R12 + B runtime·callbackasm1(SB) + MOVD $1701, R12 + B runtime·callbackasm1(SB) + MOVD $1702, R12 + B runtime·callbackasm1(SB) + MOVD $1703, R12 + B runtime·callbackasm1(SB) + MOVD $1704, R12 + B runtime·callbackasm1(SB) + MOVD $1705, R12 + B runtime·callbackasm1(SB) + MOVD $1706, R12 + B runtime·callbackasm1(SB) + MOVD $1707, R12 + B runtime·callbackasm1(SB) + MOVD $1708, R12 + B runtime·callbackasm1(SB) + MOVD $1709, R12 + B runtime·callbackasm1(SB) + MOVD $1710, R12 + B runtime·callbackasm1(SB) + MOVD $1711, R12 + B runtime·callbackasm1(SB) + MOVD $1712, R12 + B runtime·callbackasm1(SB) + MOVD $1713, R12 + B runtime·callbackasm1(SB) + MOVD $1714, R12 + B runtime·callbackasm1(SB) + MOVD $1715, R12 + B runtime·callbackasm1(SB) + MOVD $1716, R12 + B runtime·callbackasm1(SB) + MOVD $1717, R12 + B runtime·callbackasm1(SB) + MOVD $1718, R12 + B runtime·callbackasm1(SB) + MOVD $1719, R12 + B runtime·callbackasm1(SB) + MOVD $1720, R12 + B runtime·callbackasm1(SB) + MOVD $1721, R12 + B runtime·callbackasm1(SB) + MOVD $1722, R12 + B runtime·callbackasm1(SB) + MOVD $1723, R12 + B runtime·callbackasm1(SB) + MOVD $1724, R12 + B runtime·callbackasm1(SB) + MOVD $1725, R12 + B runtime·callbackasm1(SB) + MOVD $1726, R12 + B runtime·callbackasm1(SB) + MOVD $1727, R12 + B runtime·callbackasm1(SB) + MOVD $1728, R12 + B runtime·callbackasm1(SB) + MOVD $1729, R12 + B runtime·callbackasm1(SB) + MOVD $1730, R12 + B runtime·callbackasm1(SB) + MOVD $1731, R12 + B runtime·callbackasm1(SB) + MOVD $1732, R12 + B runtime·callbackasm1(SB) + MOVD $1733, R12 + B runtime·callbackasm1(SB) + MOVD $1734, R12 + B runtime·callbackasm1(SB) + MOVD $1735, R12 + B runtime·callbackasm1(SB) + MOVD $1736, R12 + B runtime·callbackasm1(SB) + MOVD $1737, R12 + B runtime·callbackasm1(SB) + MOVD $1738, R12 + B runtime·callbackasm1(SB) + MOVD $1739, R12 + B runtime·callbackasm1(SB) + MOVD $1740, R12 + B runtime·callbackasm1(SB) + MOVD $1741, R12 + B runtime·callbackasm1(SB) + MOVD $1742, R12 + B runtime·callbackasm1(SB) + MOVD $1743, R12 + B runtime·callbackasm1(SB) + MOVD $1744, R12 + B runtime·callbackasm1(SB) + MOVD $1745, R12 + B runtime·callbackasm1(SB) + MOVD $1746, R12 + B runtime·callbackasm1(SB) + MOVD $1747, R12 + B runtime·callbackasm1(SB) + MOVD $1748, R12 + B runtime·callbackasm1(SB) + MOVD $1749, R12 + B runtime·callbackasm1(SB) + MOVD $1750, R12 + B runtime·callbackasm1(SB) + MOVD $1751, R12 + B runtime·callbackasm1(SB) + MOVD $1752, R12 + B runtime·callbackasm1(SB) + MOVD $1753, R12 + B runtime·callbackasm1(SB) + MOVD $1754, R12 + B runtime·callbackasm1(SB) + MOVD $1755, R12 + B runtime·callbackasm1(SB) + MOVD $1756, R12 + B runtime·callbackasm1(SB) + MOVD $1757, R12 + B runtime·callbackasm1(SB) + MOVD $1758, R12 + B runtime·callbackasm1(SB) + MOVD $1759, R12 + B runtime·callbackasm1(SB) + MOVD $1760, R12 + B runtime·callbackasm1(SB) + MOVD $1761, R12 + B runtime·callbackasm1(SB) + MOVD $1762, R12 + B runtime·callbackasm1(SB) + MOVD $1763, R12 + B runtime·callbackasm1(SB) + MOVD $1764, R12 + B runtime·callbackasm1(SB) + MOVD $1765, R12 + B runtime·callbackasm1(SB) + MOVD $1766, R12 + B runtime·callbackasm1(SB) + MOVD $1767, R12 + B runtime·callbackasm1(SB) + MOVD $1768, R12 + B runtime·callbackasm1(SB) + MOVD $1769, R12 + B runtime·callbackasm1(SB) + MOVD $1770, R12 + B runtime·callbackasm1(SB) + MOVD $1771, R12 + B runtime·callbackasm1(SB) + MOVD $1772, R12 + B runtime·callbackasm1(SB) + MOVD $1773, R12 + B runtime·callbackasm1(SB) + MOVD $1774, R12 + B runtime·callbackasm1(SB) + MOVD $1775, R12 + B runtime·callbackasm1(SB) + MOVD $1776, R12 + B runtime·callbackasm1(SB) + MOVD $1777, R12 + B runtime·callbackasm1(SB) + MOVD $1778, R12 + B runtime·callbackasm1(SB) + MOVD $1779, R12 + B runtime·callbackasm1(SB) + MOVD $1780, R12 + B runtime·callbackasm1(SB) + MOVD $1781, R12 + B runtime·callbackasm1(SB) + MOVD $1782, R12 + B runtime·callbackasm1(SB) + MOVD $1783, R12 + B runtime·callbackasm1(SB) + MOVD $1784, R12 + B runtime·callbackasm1(SB) + MOVD $1785, R12 + B runtime·callbackasm1(SB) + MOVD $1786, R12 + B runtime·callbackasm1(SB) + MOVD $1787, R12 + B runtime·callbackasm1(SB) + MOVD $1788, R12 + B runtime·callbackasm1(SB) + MOVD $1789, R12 + B runtime·callbackasm1(SB) + MOVD $1790, R12 + B runtime·callbackasm1(SB) + MOVD $1791, R12 + B runtime·callbackasm1(SB) + MOVD $1792, R12 + B runtime·callbackasm1(SB) + MOVD $1793, R12 + B runtime·callbackasm1(SB) + MOVD $1794, R12 + B runtime·callbackasm1(SB) + MOVD $1795, R12 + B runtime·callbackasm1(SB) + MOVD $1796, R12 + B runtime·callbackasm1(SB) + MOVD $1797, R12 + B runtime·callbackasm1(SB) + MOVD $1798, R12 + B runtime·callbackasm1(SB) + MOVD $1799, R12 + B runtime·callbackasm1(SB) + MOVD $1800, R12 + B runtime·callbackasm1(SB) + MOVD $1801, R12 + B runtime·callbackasm1(SB) + MOVD $1802, R12 + B runtime·callbackasm1(SB) + MOVD $1803, R12 + B runtime·callbackasm1(SB) + MOVD $1804, R12 + B runtime·callbackasm1(SB) + MOVD $1805, R12 + B runtime·callbackasm1(SB) + MOVD $1806, R12 + B runtime·callbackasm1(SB) + MOVD $1807, R12 + B runtime·callbackasm1(SB) + MOVD $1808, R12 + B runtime·callbackasm1(SB) + MOVD $1809, R12 + B runtime·callbackasm1(SB) + MOVD $1810, R12 + B runtime·callbackasm1(SB) + MOVD $1811, R12 + B runtime·callbackasm1(SB) + MOVD $1812, R12 + B runtime·callbackasm1(SB) + MOVD $1813, R12 + B runtime·callbackasm1(SB) + MOVD $1814, R12 + B runtime·callbackasm1(SB) + MOVD $1815, R12 + B runtime·callbackasm1(SB) + MOVD $1816, R12 + B runtime·callbackasm1(SB) + MOVD $1817, R12 + B runtime·callbackasm1(SB) + MOVD $1818, R12 + B runtime·callbackasm1(SB) + MOVD $1819, R12 + B runtime·callbackasm1(SB) + MOVD $1820, R12 + B runtime·callbackasm1(SB) + MOVD $1821, R12 + B runtime·callbackasm1(SB) + MOVD $1822, R12 + B runtime·callbackasm1(SB) + MOVD $1823, R12 + B runtime·callbackasm1(SB) + MOVD $1824, R12 + B runtime·callbackasm1(SB) + MOVD $1825, R12 + B runtime·callbackasm1(SB) + MOVD $1826, R12 + B runtime·callbackasm1(SB) + MOVD $1827, R12 + B runtime·callbackasm1(SB) + MOVD $1828, R12 + B runtime·callbackasm1(SB) + MOVD $1829, R12 + B runtime·callbackasm1(SB) + MOVD $1830, R12 + B runtime·callbackasm1(SB) + MOVD $1831, R12 + B runtime·callbackasm1(SB) + MOVD $1832, R12 + B runtime·callbackasm1(SB) + MOVD $1833, R12 + B runtime·callbackasm1(SB) + MOVD $1834, R12 + B runtime·callbackasm1(SB) + MOVD $1835, R12 + B runtime·callbackasm1(SB) + MOVD $1836, R12 + B runtime·callbackasm1(SB) + MOVD $1837, R12 + B runtime·callbackasm1(SB) + MOVD $1838, R12 + B runtime·callbackasm1(SB) + MOVD $1839, R12 + B runtime·callbackasm1(SB) + MOVD $1840, R12 + B runtime·callbackasm1(SB) + MOVD $1841, R12 + B runtime·callbackasm1(SB) + MOVD $1842, R12 + B runtime·callbackasm1(SB) + MOVD $1843, R12 + B runtime·callbackasm1(SB) + MOVD $1844, R12 + B runtime·callbackasm1(SB) + MOVD $1845, R12 + B runtime·callbackasm1(SB) + MOVD $1846, R12 + B runtime·callbackasm1(SB) + MOVD $1847, R12 + B runtime·callbackasm1(SB) + MOVD $1848, R12 + B runtime·callbackasm1(SB) + MOVD $1849, R12 + B runtime·callbackasm1(SB) + MOVD $1850, R12 + B runtime·callbackasm1(SB) + MOVD $1851, R12 + B runtime·callbackasm1(SB) + MOVD $1852, R12 + B runtime·callbackasm1(SB) + MOVD $1853, R12 + B runtime·callbackasm1(SB) + MOVD $1854, R12 + B runtime·callbackasm1(SB) + MOVD $1855, R12 + B runtime·callbackasm1(SB) + MOVD $1856, R12 + B runtime·callbackasm1(SB) + MOVD $1857, R12 + B runtime·callbackasm1(SB) + MOVD $1858, R12 + B runtime·callbackasm1(SB) + MOVD $1859, R12 + B runtime·callbackasm1(SB) + MOVD $1860, R12 + B runtime·callbackasm1(SB) + MOVD $1861, R12 + B runtime·callbackasm1(SB) + MOVD $1862, R12 + B runtime·callbackasm1(SB) + MOVD $1863, R12 + B runtime·callbackasm1(SB) + MOVD $1864, R12 + B runtime·callbackasm1(SB) + MOVD $1865, R12 + B runtime·callbackasm1(SB) + MOVD $1866, R12 + B runtime·callbackasm1(SB) + MOVD $1867, R12 + B runtime·callbackasm1(SB) + MOVD $1868, R12 + B runtime·callbackasm1(SB) + MOVD $1869, R12 + B runtime·callbackasm1(SB) + MOVD $1870, R12 + B runtime·callbackasm1(SB) + MOVD $1871, R12 + B runtime·callbackasm1(SB) + MOVD $1872, R12 + B runtime·callbackasm1(SB) + MOVD $1873, R12 + B runtime·callbackasm1(SB) + MOVD $1874, R12 + B runtime·callbackasm1(SB) + MOVD $1875, R12 + B runtime·callbackasm1(SB) + MOVD $1876, R12 + B runtime·callbackasm1(SB) + MOVD $1877, R12 + B runtime·callbackasm1(SB) + MOVD $1878, R12 + B runtime·callbackasm1(SB) + MOVD $1879, R12 + B runtime·callbackasm1(SB) + MOVD $1880, R12 + B runtime·callbackasm1(SB) + MOVD $1881, R12 + B runtime·callbackasm1(SB) + MOVD $1882, R12 + B runtime·callbackasm1(SB) + MOVD $1883, R12 + B runtime·callbackasm1(SB) + MOVD $1884, R12 + B runtime·callbackasm1(SB) + MOVD $1885, R12 + B runtime·callbackasm1(SB) + MOVD $1886, R12 + B runtime·callbackasm1(SB) + MOVD $1887, R12 + B runtime·callbackasm1(SB) + MOVD $1888, R12 + B runtime·callbackasm1(SB) + MOVD $1889, R12 + B runtime·callbackasm1(SB) + MOVD $1890, R12 + B runtime·callbackasm1(SB) + MOVD $1891, R12 + B runtime·callbackasm1(SB) + MOVD $1892, R12 + B runtime·callbackasm1(SB) + MOVD $1893, R12 + B runtime·callbackasm1(SB) + MOVD $1894, R12 + B runtime·callbackasm1(SB) + MOVD $1895, R12 + B runtime·callbackasm1(SB) + MOVD $1896, R12 + B runtime·callbackasm1(SB) + MOVD $1897, R12 + B runtime·callbackasm1(SB) + MOVD $1898, R12 + B runtime·callbackasm1(SB) + MOVD $1899, R12 + B runtime·callbackasm1(SB) + MOVD $1900, R12 + B runtime·callbackasm1(SB) + MOVD $1901, R12 + B runtime·callbackasm1(SB) + MOVD $1902, R12 + B runtime·callbackasm1(SB) + MOVD $1903, R12 + B runtime·callbackasm1(SB) + MOVD $1904, R12 + B runtime·callbackasm1(SB) + MOVD $1905, R12 + B runtime·callbackasm1(SB) + MOVD $1906, R12 + B runtime·callbackasm1(SB) + MOVD $1907, R12 + B runtime·callbackasm1(SB) + MOVD $1908, R12 + B runtime·callbackasm1(SB) + MOVD $1909, R12 + B runtime·callbackasm1(SB) + MOVD $1910, R12 + B runtime·callbackasm1(SB) + MOVD $1911, R12 + B runtime·callbackasm1(SB) + MOVD $1912, R12 + B runtime·callbackasm1(SB) + MOVD $1913, R12 + B runtime·callbackasm1(SB) + MOVD $1914, R12 + B runtime·callbackasm1(SB) + MOVD $1915, R12 + B runtime·callbackasm1(SB) + MOVD $1916, R12 + B runtime·callbackasm1(SB) + MOVD $1917, R12 + B runtime·callbackasm1(SB) + MOVD $1918, R12 + B runtime·callbackasm1(SB) + MOVD $1919, R12 + B runtime·callbackasm1(SB) + MOVD $1920, R12 + B runtime·callbackasm1(SB) + MOVD $1921, R12 + B runtime·callbackasm1(SB) + MOVD $1922, R12 + B runtime·callbackasm1(SB) + MOVD $1923, R12 + B runtime·callbackasm1(SB) + MOVD $1924, R12 + B runtime·callbackasm1(SB) + MOVD $1925, R12 + B runtime·callbackasm1(SB) + MOVD $1926, R12 + B runtime·callbackasm1(SB) + MOVD $1927, R12 + B runtime·callbackasm1(SB) + MOVD $1928, R12 + B runtime·callbackasm1(SB) + MOVD $1929, R12 + B runtime·callbackasm1(SB) + MOVD $1930, R12 + B runtime·callbackasm1(SB) + MOVD $1931, R12 + B runtime·callbackasm1(SB) + MOVD $1932, R12 + B runtime·callbackasm1(SB) + MOVD $1933, R12 + B runtime·callbackasm1(SB) + MOVD $1934, R12 + B runtime·callbackasm1(SB) + MOVD $1935, R12 + B runtime·callbackasm1(SB) + MOVD $1936, R12 + B runtime·callbackasm1(SB) + MOVD $1937, R12 + B runtime·callbackasm1(SB) + MOVD $1938, R12 + B runtime·callbackasm1(SB) + MOVD $1939, R12 + B runtime·callbackasm1(SB) + MOVD $1940, R12 + B runtime·callbackasm1(SB) + MOVD $1941, R12 + B runtime·callbackasm1(SB) + MOVD $1942, R12 + B runtime·callbackasm1(SB) + MOVD $1943, R12 + B runtime·callbackasm1(SB) + MOVD $1944, R12 + B runtime·callbackasm1(SB) + MOVD $1945, R12 + B runtime·callbackasm1(SB) + MOVD $1946, R12 + B runtime·callbackasm1(SB) + MOVD $1947, R12 + B runtime·callbackasm1(SB) + MOVD $1948, R12 + B runtime·callbackasm1(SB) + MOVD $1949, R12 + B runtime·callbackasm1(SB) + MOVD $1950, R12 + B runtime·callbackasm1(SB) + MOVD $1951, R12 + B runtime·callbackasm1(SB) + MOVD $1952, R12 + B runtime·callbackasm1(SB) + MOVD $1953, R12 + B runtime·callbackasm1(SB) + MOVD $1954, R12 + B runtime·callbackasm1(SB) + MOVD $1955, R12 + B runtime·callbackasm1(SB) + MOVD $1956, R12 + B runtime·callbackasm1(SB) + MOVD $1957, R12 + B runtime·callbackasm1(SB) + MOVD $1958, R12 + B runtime·callbackasm1(SB) + MOVD $1959, R12 + B runtime·callbackasm1(SB) + MOVD $1960, R12 + B runtime·callbackasm1(SB) + MOVD $1961, R12 + B runtime·callbackasm1(SB) + MOVD $1962, R12 + B runtime·callbackasm1(SB) + MOVD $1963, R12 + B runtime·callbackasm1(SB) + MOVD $1964, R12 + B runtime·callbackasm1(SB) + MOVD $1965, R12 + B runtime·callbackasm1(SB) + MOVD $1966, R12 + B runtime·callbackasm1(SB) + MOVD $1967, R12 + B runtime·callbackasm1(SB) + MOVD $1968, R12 + B runtime·callbackasm1(SB) + MOVD $1969, R12 + B runtime·callbackasm1(SB) + MOVD $1970, R12 + B runtime·callbackasm1(SB) + MOVD $1971, R12 + B runtime·callbackasm1(SB) + MOVD $1972, R12 + B runtime·callbackasm1(SB) + MOVD $1973, R12 + B runtime·callbackasm1(SB) + MOVD $1974, R12 + B runtime·callbackasm1(SB) + MOVD $1975, R12 + B runtime·callbackasm1(SB) + MOVD $1976, R12 + B runtime·callbackasm1(SB) + MOVD $1977, R12 + B runtime·callbackasm1(SB) + MOVD $1978, R12 + B runtime·callbackasm1(SB) + MOVD $1979, R12 + B runtime·callbackasm1(SB) + MOVD $1980, R12 + B runtime·callbackasm1(SB) + MOVD $1981, R12 + B runtime·callbackasm1(SB) + MOVD $1982, R12 + B runtime·callbackasm1(SB) + MOVD $1983, R12 + B runtime·callbackasm1(SB) + MOVD $1984, R12 + B runtime·callbackasm1(SB) + MOVD $1985, R12 + B runtime·callbackasm1(SB) + MOVD $1986, R12 + B runtime·callbackasm1(SB) + MOVD $1987, R12 + B runtime·callbackasm1(SB) + MOVD $1988, R12 + B runtime·callbackasm1(SB) + MOVD $1989, R12 + B runtime·callbackasm1(SB) + MOVD $1990, R12 + B runtime·callbackasm1(SB) + MOVD $1991, R12 + B runtime·callbackasm1(SB) + MOVD $1992, R12 + B runtime·callbackasm1(SB) + MOVD $1993, R12 + B runtime·callbackasm1(SB) + MOVD $1994, R12 + B runtime·callbackasm1(SB) + MOVD $1995, R12 + B runtime·callbackasm1(SB) + MOVD $1996, R12 + B runtime·callbackasm1(SB) + MOVD $1997, R12 + B runtime·callbackasm1(SB) + MOVD $1998, R12 + B runtime·callbackasm1(SB) + MOVD $1999, R12 + B runtime·callbackasm1(SB) -- cgit v1.3-5-g9baa From 3527caa7d63eab821c9936383e6c442d7a013de1 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 22 Jan 2021 10:30:10 -0500 Subject: runtime: initial windows/arm64 implementation files This CL adds a few small files - defs, os, and rt0 - to start on windows/arm64 support for the runtime. It also copies sys_windows_arm.s to sys_windows_arm64.s, with the addition of "#ifdef NOT_PORTED" around the entire file. This is meant to make future CLs easier to review, since the general pattern is to translate the 32-bit ARM assembly into 64-bit ARM assembly. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. Change-Id: I922037eb3890e77bac48281ecaa8e489595675be Reviewed-on: https://go-review.googlesource.com/c/go/+/288827 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang Reviewed-by: Alex Brainman Reviewed-by: Jason A. Donenfeld --- src/runtime/defs_windows_arm64.go | 83 ++++++ src/runtime/os_windows_arm64.go | 14 + src/runtime/rt0_windows_arm64.s | 12 + src/runtime/sys_windows_arm64.s | 588 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 697 insertions(+) create mode 100644 src/runtime/defs_windows_arm64.go create mode 100644 src/runtime/os_windows_arm64.go create mode 100644 src/runtime/rt0_windows_arm64.s create mode 100644 src/runtime/sys_windows_arm64.s (limited to 'src/runtime') diff --git a/src/runtime/defs_windows_arm64.go b/src/runtime/defs_windows_arm64.go new file mode 100644 index 0000000000..9ccce46f09 --- /dev/null +++ b/src/runtime/defs_windows_arm64.go @@ -0,0 +1,83 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// NOTE(rsc): _CONTEXT_CONTROL is actually 0x400001 and should include PC, SP, and LR. +// However, empirically, LR doesn't come along on Windows 10 +// unless you also set _CONTEXT_INTEGER (0x400002). +// Without LR, we skip over the next-to-bottom function in profiles +// when the bottom function is frameless. +// So we set both here, to make a working _CONTEXT_CONTROL. +const _CONTEXT_CONTROL = 0x400003 + +type neon128 struct { + low uint64 + high int64 +} + +// See https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-arm64_nt_context +type context struct { + contextflags uint32 + cpsr uint32 + x [31]uint64 // fp is x[29], lr is x[30] + xsp uint64 + pc uint64 + v [32]neon128 + fpcr uint32 + fpsr uint32 + bcr [8]uint32 + bvr [8]uint64 + wcr [2]uint32 + wvr [2]uint64 +} + +func (c *context) ip() uintptr { return uintptr(c.pc) } +func (c *context) sp() uintptr { return uintptr(c.xsp) } +func (c *context) lr() uintptr { return uintptr(c.x[30]) } + +func (c *context) set_ip(x uintptr) { c.pc = uint64(x) } +func (c *context) set_sp(x uintptr) { c.xsp = uint64(x) } +func (c *context) set_lr(x uintptr) { c.x[30] = uint64(x) } + +func dumpregs(r *context) { + print("r0 ", hex(r.x[0]), "\n") + print("r1 ", hex(r.x[1]), "\n") + print("r2 ", hex(r.x[2]), "\n") + print("r3 ", hex(r.x[3]), "\n") + print("r4 ", hex(r.x[4]), "\n") + print("r5 ", hex(r.x[5]), "\n") + print("r6 ", hex(r.x[6]), "\n") + print("r7 ", hex(r.x[7]), "\n") + print("r8 ", hex(r.x[8]), "\n") + print("r9 ", hex(r.x[9]), "\n") + print("r10 ", hex(r.x[10]), "\n") + print("r11 ", hex(r.x[11]), "\n") + print("r12 ", hex(r.x[12]), "\n") + print("r13 ", hex(r.x[13]), "\n") + print("r14 ", hex(r.x[14]), "\n") + print("r15 ", hex(r.x[15]), "\n") + print("r16 ", hex(r.x[16]), "\n") + print("r17 ", hex(r.x[17]), "\n") + print("r18 ", hex(r.x[18]), "\n") + print("r19 ", hex(r.x[19]), "\n") + print("r20 ", hex(r.x[20]), "\n") + print("r21 ", hex(r.x[21]), "\n") + print("r22 ", hex(r.x[22]), "\n") + print("r23 ", hex(r.x[23]), "\n") + print("r24 ", hex(r.x[24]), "\n") + print("r25 ", hex(r.x[25]), "\n") + print("r26 ", hex(r.x[26]), "\n") + print("r27 ", hex(r.x[27]), "\n") + print("r28 ", hex(r.x[28]), "\n") + print("r29 ", hex(r.x[29]), "\n") + print("lr ", hex(r.x[30]), "\n") + print("sp ", hex(r.xsp), "\n") + print("pc ", hex(r.pc), "\n") + print("cpsr ", hex(r.cpsr), "\n") +} + +func stackcheck() { + // TODO: not implemented on ARM +} diff --git a/src/runtime/os_windows_arm64.go b/src/runtime/os_windows_arm64.go new file mode 100644 index 0000000000..7e413445ba --- /dev/null +++ b/src/runtime/os_windows_arm64.go @@ -0,0 +1,14 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +//go:nosplit +func cputicks() int64 { + var counter int64 + stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) + return counter +} diff --git a/src/runtime/rt0_windows_arm64.s b/src/runtime/rt0_windows_arm64.s new file mode 100644 index 0000000000..1e71a068d3 --- /dev/null +++ b/src/runtime/rt0_windows_arm64.s @@ -0,0 +1,12 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" + +// This is the entry point for the program from the +// kernel for an ordinary -buildmode=exe program. +TEXT _rt0_arm64_windows(SB),NOSPLIT|NOFRAME,$0 + B ·rt0_go(SB) diff --git a/src/runtime/sys_windows_arm64.s b/src/runtime/sys_windows_arm64.s new file mode 100644 index 0000000000..b279f25de8 --- /dev/null +++ b/src/runtime/sys_windows_arm64.s @@ -0,0 +1,588 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" + +#ifdef NOT_PORTED + +// Note: For system ABI, R0-R3 are args, R4-R11 are callee-save. + +// void runtime·asmstdcall(void *c); +TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} + MOVW R0, R4 // put libcall * in r4 + MOVW R13, R5 // save stack pointer in r5 + + // SetLastError(0) + MOVW $0, R0 + MRC 15, 0, R1, C13, C0, 2 + MOVW R0, 0x34(R1) + + MOVW 8(R4), R12 // libcall->args + + // Do we have more than 4 arguments? + MOVW 4(R4), R0 // libcall->n + SUB.S $4, R0, R2 + BLE loadregs + + // Reserve stack space for remaining args + SUB R2<<2, R13 + BIC $0x7, R13 // alignment for ABI + + // R0: count of arguments + // R1: + // R2: loop counter, from 0 to (n-4) + // R3: scratch + // R4: pointer to libcall struct + // R12: libcall->args + MOVW $0, R2 +stackargs: + ADD $4, R2, R3 // r3 = args[4 + i] + MOVW R3<<2(R12), R3 + MOVW R3, R2<<2(R13) // stack[i] = r3 + + ADD $1, R2 // i++ + SUB $4, R0, R3 // while (i < (n - 4)) + CMP R3, R2 + BLT stackargs + +loadregs: + CMP $3, R0 + MOVW.GT 12(R12), R3 + + CMP $2, R0 + MOVW.GT 8(R12), R2 + + CMP $1, R0 + MOVW.GT 4(R12), R1 + + CMP $0, R0 + MOVW.GT 0(R12), R0 + + BIC $0x7, R13 // alignment for ABI + MOVW 0(R4), R12 // branch to libcall->fn + BL (R12) + + MOVW R5, R13 // free stack space + MOVW R0, 12(R4) // save return value to libcall->r1 + MOVW R1, 16(R4) + + // GetLastError + MRC 15, 0, R1, C13, C0, 2 + MOVW 0x34(R1), R0 + MOVW R0, 20(R4) // store in libcall->err + + MOVM.IA.W (R13), [R4, R5, R15] + +TEXT runtime·badsignal2(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R14], (R13) // push {r4, lr} + MOVW R13, R4 // save original stack pointer + SUB $8, R13 // space for 2 variables + BIC $0x7, R13 // alignment for ABI + + // stderr + MOVW runtime·_GetStdHandle(SB), R1 + MOVW $-12, R0 + BL (R1) + + MOVW $runtime·badsignalmsg(SB), R1 // lpBuffer + MOVW $runtime·badsignallen(SB), R2 // lpNumberOfBytesToWrite + MOVW (R2), R2 + ADD $0x4, R13, R3 // lpNumberOfBytesWritten + MOVW $0, R12 // lpOverlapped + MOVW R12, (R13) + + MOVW runtime·_WriteFile(SB), R12 + BL (R12) + + MOVW R4, R13 // restore SP + MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} + +TEXT runtime·getlasterror(SB),NOSPLIT,$0 + MRC 15, 0, R0, C13, C0, 2 + MOVW 0x34(R0), R0 + MOVW R0, ret+0(FP) + RET + +// Called by Windows as a Vectored Exception Handler (VEH). +// First argument is pointer to struct containing +// exception record and context pointers. +// Handler function is stored in R1 +// Return 0 for 'not handled', -1 for handled. +// int32_t sigtramp( +// PEXCEPTION_POINTERS ExceptionInfo, +// func *GoExceptionHandler); +TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R0, R4-R11, R14], (R13) // push {r0, r4-r11, lr} (SP-=40) + SUB $(8+20), R13 // reserve space for g, sp, and + // parameters/retval to go call + + MOVW R0, R6 // Save param0 + MOVW R1, R7 // Save param1 + + BL runtime·load_g(SB) + CMP $0, g // is there a current g? + BL.EQ runtime·badsignal2(SB) + + // save g and SP in case of stack switch + MOVW R13, 24(R13) + MOVW g, 20(R13) + + // do we need to switch to the g0 stack? + MOVW g, R5 // R5 = g + MOVW g_m(R5), R2 // R2 = m + MOVW m_g0(R2), R4 // R4 = g0 + CMP R5, R4 // if curg == g0 + BEQ g0 + + // switch to g0 stack + MOVW R4, g // g = g0 + MOVW (g_sched+gobuf_sp)(g), R3 // R3 = g->gobuf.sp + BL runtime·save_g(SB) + + // make room for sighandler arguments + // and re-save old SP for restoring later. + // (note that the 24(R3) here must match the 24(R13) above.) + SUB $40, R3 + MOVW R13, 24(R3) // save old stack pointer + MOVW R3, R13 // switch stack + +g0: + MOVW 0(R6), R2 // R2 = ExceptionPointers->ExceptionRecord + MOVW 4(R6), R3 // R3 = ExceptionPointers->ContextRecord + + MOVW $0, R4 + MOVW R4, 0(R13) // No saved link register. + MOVW R2, 4(R13) // Move arg0 (ExceptionRecord) into position + MOVW R3, 8(R13) // Move arg1 (ContextRecord) into position + MOVW R5, 12(R13) // Move arg2 (original g) into position + BL (R7) // Call the go routine + MOVW 16(R13), R4 // Fetch return value from stack + + // switch back to original stack and g + MOVW 24(R13), R13 + MOVW 20(R13), g + BL runtime·save_g(SB) + +done: + MOVW R4, R0 // move retval into position + ADD $(8 + 20), R13 // free locals + MOVM.IA.W (R13), [R3, R4-R11, R14] // pop {r3, r4-r11, lr} + + // if return value is CONTINUE_SEARCH, do not set up control + // flow guard workaround + CMP $0, R0 + BEQ return + + // Check if we need to set up the control flow guard workaround. + // On Windows/ARM, the stack pointer must lie within system + // stack limits when we resume from exception. + // Store the resume SP and PC on the g0 stack, + // and return to returntramp on the g0 stack. returntramp + // pops the saved PC and SP from the g0 stack, resuming execution + // at the desired location. + // If returntramp has already been set up by a previous exception + // handler, don't clobber the stored SP and PC on the stack. + MOVW 4(R3), R3 // PEXCEPTION_POINTERS->Context + MOVW context_pc(R3), R2 // load PC from context record + MOVW $returntramp<>(SB), R1 + CMP R1, R2 + B.EQ return // do not clobber saved SP/PC + + // Save resume SP and PC into R0, R1. + MOVW context_spr(R3), R2 + MOVW R2, context_r0(R3) + MOVW context_pc(R3), R2 + MOVW R2, context_r1(R3) + + // Set up context record to return to returntramp on g0 stack + MOVW R12, context_spr(R3) + MOVW $returntramp<>(SB), R2 + MOVW R2, context_pc(R3) + +return: + B (R14) // return + +// Trampoline to resume execution from exception handler. +// This is part of the control flow guard workaround. +// It switches stacks and jumps to the continuation address. +// R0 and R1 are set above at the end of sigtramp<> +// in the context that starts executing at returntramp<>. +TEXT returntramp<>(SB),NOSPLIT|NOFRAME,$0 + // Important: do not smash LR, + // which is set to a live value when handling + // a signal by pushing a call to sigpanic onto the stack. + MOVW R0, R13 + B (R1) + +TEXT runtime·exceptiontramp(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·exceptionhandler(SB), R1 + B sigtramp<>(SB) + +TEXT runtime·firstcontinuetramp(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·firstcontinuehandler(SB), R1 + B sigtramp<>(SB) + +TEXT runtime·lastcontinuetramp(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·lastcontinuehandler(SB), R1 + B sigtramp<>(SB) + +TEXT runtime·ctrlhandler(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·ctrlhandler1(SB), R1 + B runtime·externalthreadhandler(SB) + +TEXT runtime·profileloop(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·profileloop1(SB), R1 + B runtime·externalthreadhandler(SB) + +// int32 externalthreadhandler(uint32 arg, int (*func)(uint32)) +// stack layout: +// +----------------+ +// | callee-save | +// | registers | +// +----------------+ +// | m | +// +----------------+ +// 20| g | +// +----------------+ +// 16| func ptr (r1) | +// +----------------+ +// 12| argument (r0) | +//---+----------------+ +// 8 | param1 | (also return value for called Go function) +// +----------------+ +// 4 | param0 | +// +----------------+ +// 0 | slot for LR | +// +----------------+ +// +TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 + MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} + SUB $(m__size + g__size + 20), R13 // space for locals + MOVW R14, 0(R13) // push LR again for anything unwinding the stack + MOVW R0, 12(R13) + MOVW R1, 16(R13) + + // zero out m and g structures + ADD $20, R13, R0 // compute pointer to g + MOVW R0, 4(R13) + MOVW $(m__size + g__size), R0 + MOVW R0, 8(R13) + BL runtime·memclrNoHeapPointers(SB) + + // initialize m and g structures + ADD $20, R13, R2 // R2 = g + ADD $(20 + g__size), R13, R3 // R3 = m + MOVW R2, m_g0(R3) // m->g0 = g + MOVW R3, g_m(R2) // g->m = m + MOVW R2, m_curg(R3) // m->curg = g + + MOVW R2, g + BL runtime·save_g(SB) + + // set up stackguard stuff + MOVW R13, R0 + MOVW R0, g_stack+stack_hi(g) + SUB $(32*1024), R0 + MOVW R0, (g_stack+stack_lo)(g) + MOVW R0, g_stackguard0(g) + MOVW R0, g_stackguard1(g) + + // move argument into position and call function + MOVW 12(R13), R0 + MOVW R0, 4(R13) + MOVW 16(R13), R1 + BL (R1) + + // clear g + MOVW $0, g + BL runtime·save_g(SB) + + MOVW 8(R13), R0 // load return value + ADD $(m__size + g__size + 20), R13 // free locals + MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + +GLOBL runtime·cbctxts(SB), NOPTR, $4 + +TEXT runtime·callbackasm1(SB),NOSPLIT|NOFRAME,$0 + // On entry, the trampoline in zcallback_windows_arm.s left + // the callback index in R12 (which is volatile in the C ABI). + + // Push callback register arguments r0-r3. We do this first so + // they're contiguous with stack arguments. + MOVM.DB.W [R0-R3], (R13) + // Push C callee-save registers r4-r11 and lr. + MOVM.DB.W [R4-R11, R14], (R13) + SUB $(16 + callbackArgs__size), R13 // space for locals + + // Create a struct callbackArgs on our stack. + MOVW R12, (16+callbackArgs_index)(R13) // callback index + MOVW $(16+callbackArgs__size+4*9)(R13), R0 + MOVW R0, (16+callbackArgs_args)(R13) // address of args vector + MOVW $0, R0 + MOVW R0, (16+callbackArgs_result)(R13) // result + + // Prepare for entry to Go. + BL runtime·load_g(SB) + + // Call cgocallback, which will call callbackWrap(frame). + MOVW $0, R0 + MOVW R0, 12(R13) // context + MOVW $16(R13), R1 // R1 = &callbackArgs{...} + MOVW R1, 8(R13) // frame (address of callbackArgs) + MOVW $·callbackWrap(SB), R1 + MOVW R1, 4(R13) // PC of function to call + BL runtime·cgocallback(SB) + + // Get callback result. + MOVW (16+callbackArgs_result)(R13), R0 + + ADD $(16 + callbackArgs__size), R13 // free locals + MOVM.IA.W (R13), [R4-R11, R12] // pop {r4-r11, lr=>r12} + ADD $(4*4), R13 // skip r0-r3 + B (R12) // return + +// uint32 tstart_stdcall(M *newm); +TEXT runtime·tstart_stdcall(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} + + MOVW m_g0(R0), g + MOVW R0, g_m(g) + BL runtime·save_g(SB) + + // Layout new m scheduler stack on os stack. + MOVW R13, R0 + MOVW R0, g_stack+stack_hi(g) + SUB $(64*1024), R0 + MOVW R0, (g_stack+stack_lo)(g) + MOVW R0, g_stackguard0(g) + MOVW R0, g_stackguard1(g) + + BL runtime·emptyfunc(SB) // fault if stack check is wrong + BL runtime·mstart(SB) + + // Exit the thread. + MOVW $0, R0 + MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g may be nil. +TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$0-4 + MOVW dt+0(FP), R0 + MOVM.DB.W [R4, R14], (R13) // push {r4, lr} + MOVW R13, R4 // Save SP + SUB $8, R13 // R13 = R13 - 8 + BIC $0x7, R13 // Align SP for ABI + RSB $0, R0, R3 // R3 = -R0 + MOVW $0, R1 // R1 = FALSE (alertable) + MOVW $-1, R0 // R0 = handle + MOVW R13, R2 // R2 = pTime + MOVW R3, 0(R2) // time_lo + MOVW R0, 4(R2) // time_hi + MOVW runtime·_NtWaitForSingleObject(SB), R3 + BL (R3) + MOVW R4, R13 // Restore SP + MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} + +// Runs on OS stack. +// duration (in -100ns units) is in dt+0(FP). +// g is valid. +// TODO: neeeds to be implemented properly. +TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$0-4 + B runtime·abort(SB) + +// Runs on OS stack. +TEXT runtime·switchtothread(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R14], (R13) // push {R4, lr} + MOVW R13, R4 + BIC $0x7, R13 // alignment for ABI + MOVW runtime·_SwitchToThread(SB), R0 + BL (R0) + MOVW R4, R13 // restore stack pointer + MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} + +TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 + B runtime·armPublicationBarrier(SB) + +// never called (cgo not supported) +TEXT runtime·read_tls_fallback(SB),NOSPLIT|NOFRAME,$0 + MOVW $0xabcd, R0 + MOVW R0, (R0) + RET + +// See http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/ +// Must read hi1, then lo, then hi2. The snapshot is valid if hi1 == hi2. +#define _INTERRUPT_TIME 0x7ffe0008 +#define _SYSTEM_TIME 0x7ffe0014 +#define time_lo 0 +#define time_hi1 4 +#define time_hi2 8 + +TEXT runtime·nanotime1(SB),NOSPLIT|NOFRAME,$0-8 + MOVW $0, R0 + MOVB runtime·useQPCTime(SB), R0 + CMP $0, R0 + BNE useQPC + MOVW $_INTERRUPT_TIME, R3 +loop: + MOVW time_hi1(R3), R1 + MOVW time_lo(R3), R0 + MOVW time_hi2(R3), R2 + CMP R1, R2 + BNE loop + + // wintime = R1:R0, multiply by 100 + MOVW $100, R2 + MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 + MULA R1, R2, R4, R4 + + // wintime*100 = R4:R3 + MOVW R3, ret_lo+0(FP) + MOVW R4, ret_hi+4(FP) + RET +useQPC: + B runtime·nanotimeQPC(SB) // tail call + +TEXT time·now(SB),NOSPLIT|NOFRAME,$0-20 + MOVW $0, R0 + MOVB runtime·useQPCTime(SB), R0 + CMP $0, R0 + BNE useQPC + MOVW $_INTERRUPT_TIME, R3 +loop: + MOVW time_hi1(R3), R1 + MOVW time_lo(R3), R0 + MOVW time_hi2(R3), R2 + CMP R1, R2 + BNE loop + + // wintime = R1:R0, multiply by 100 + MOVW $100, R2 + MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 + MULA R1, R2, R4, R4 + + // wintime*100 = R4:R3 + MOVW R3, mono+12(FP) + MOVW R4, mono+16(FP) + + MOVW $_SYSTEM_TIME, R3 +wall: + MOVW time_hi1(R3), R1 + MOVW time_lo(R3), R0 + MOVW time_hi2(R3), R2 + CMP R1, R2 + BNE wall + + // w = R1:R0 in 100ns untis + // convert to Unix epoch (but still 100ns units) + #define delta 116444736000000000 + SUB.S $(delta & 0xFFFFFFFF), R0 + SBC $(delta >> 32), R1 + + // Convert to nSec + MOVW $100, R2 + MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 + MULA R1, R2, R4, R4 + // w = R2:R1 in nSec + MOVW R3, R1 // R4:R3 -> R2:R1 + MOVW R4, R2 + + // multiply nanoseconds by reciprocal of 10**9 (scaled by 2**61) + // to get seconds (96 bit scaled result) + MOVW $0x89705f41, R3 // 2**61 * 10**-9 + MULLU R1,R3,(R6,R5) // R7:R6:R5 = R2:R1 * R3 + MOVW $0,R7 + MULALU R2,R3,(R7,R6) + + // unscale by discarding low 32 bits, shifting the rest by 29 + MOVW R6>>29,R6 // R7:R6 = (R7:R6:R5 >> 61) + ORR R7<<3,R6 + MOVW R7>>29,R7 + + // subtract (10**9 * sec) from nsec to get nanosecond remainder + MOVW $1000000000, R5 // 10**9 + MULLU R6,R5,(R9,R8) // R9:R8 = R7:R6 * R5 + MULA R7,R5,R9,R9 + SUB.S R8,R1 // R2:R1 -= R9:R8 + SBC R9,R2 + + // because reciprocal was a truncated repeating fraction, quotient + // may be slightly too small -- adjust to make remainder < 10**9 + CMP R5,R1 // if remainder > 10**9 + SUB.HS R5,R1 // remainder -= 10**9 + ADD.HS $1,R6 // sec += 1 + + MOVW R6,sec_lo+0(FP) + MOVW R7,sec_hi+4(FP) + MOVW R1,nsec+8(FP) + RET +useQPC: + B runtime·nowQPC(SB) // tail call + +// save_g saves the g register (R10) into thread local memory +// so that we can call externally compiled +// ARM code that will overwrite those registers. +// NOTE: runtime.gogo assumes that R1 is preserved by this function. +// runtime.mcall assumes this function only clobbers R0 and R11. +// Returns with g in R0. +// Save the value in the _TEB->TlsSlots array. +// Effectively implements TlsSetValue(). +// tls_g stores the TLS slot allocated TlsAlloc(). +TEXT runtime·save_g(SB),NOSPLIT|NOFRAME,$0 + MRC 15, 0, R0, C13, C0, 2 + ADD $0xe10, R0 + MOVW $runtime·tls_g(SB), R11 + MOVW (R11), R11 + MOVW g, R11<<2(R0) + MOVW g, R0 // preserve R0 across call to setg<> + RET + +// load_g loads the g register from thread-local memory, +// for use after calling externally compiled +// ARM code that overwrote those registers. +// Get the value from the _TEB->TlsSlots array. +// Effectively implements TlsGetValue(). +TEXT runtime·load_g(SB),NOSPLIT|NOFRAME,$0 + MRC 15, 0, R0, C13, C0, 2 + ADD $0xe10, R0 + MOVW $runtime·tls_g(SB), g + MOVW (g), g + MOVW g<<2(R0), g + RET + +// This is called from rt0_go, which runs on the system stack +// using the initial stack allocated by the OS. +// It calls back into standard C using the BL below. +// To do that, the stack pointer must be 8-byte-aligned. +TEXT runtime·_initcgo(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R14], (R13) // push {r4, lr} + + // Ensure stack is 8-byte aligned before calling C code + MOVW R13, R4 + BIC $0x7, R13 + + // Allocate a TLS slot to hold g across calls to external code + MOVW $runtime·_TlsAlloc(SB), R0 + MOVW (R0), R0 + BL (R0) + + // Assert that slot is less than 64 so we can use _TEB->TlsSlots + CMP $64, R0 + MOVW $runtime·abort(SB), R1 + BL.GE (R1) + + // Save Slot into tls_g + MOVW $runtime·tls_g(SB), R1 + MOVW R0, (R1) + + MOVW R4, R13 + MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} + +// Holds the TLS Slot, which was allocated by TlsAlloc() +GLOBL runtime·tls_g+0(SB), NOPTR, $4 + +#endif -- cgit v1.3-5-g9baa From c7c6c113be96b7b68f54696d2986f98dc9df64d6 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 1 Feb 2021 15:12:08 -0500 Subject: runtime: convert windows/arm64 assembly The assembly is mostly a straightforward conversion of the equivalent arm assembly. This CL is part of a stack adding windows/arm64 support (#36439), intended to land in the Go 1.17 cycle. Change-Id: I61b15d712ade4d3a7285c7680de8e0987aacba10 Reviewed-on: https://go-review.googlesource.com/c/go/+/288828 Trust: Russ Cox Trust: Jason A. Donenfeld Reviewed-by: Cherry Zhang --- src/runtime/asm_arm64.s | 7 + src/runtime/defs_windows.go | 1 + src/runtime/memclr_arm.s | 1 + src/runtime/memclr_arm64.s | 1 + src/runtime/os_windows.go | 3 + src/runtime/signal_windows.go | 1 + src/runtime/stubs_arm64.go | 2 + src/runtime/sys_windows_arm.s | 18 +- src/runtime/sys_windows_arm64.s | 807 ++++++++++++++++++++-------------------- src/runtime/syscall_windows.go | 1 + src/runtime/tls_arm64.h | 12 +- src/runtime/tls_arm64.s | 8 +- 12 files changed, 441 insertions(+), 421 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index d81759537e..699fc99d58 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -73,6 +73,10 @@ nocgo: BL runtime·check(SB) +#ifdef GOOS_windows + BL runtime·wintls(SB) +#endif + MOVW 8(RSP), R0 // copy argc MOVW R0, -8(RSP) MOVD 16(RSP), R0 // copy argv @@ -1111,6 +1115,9 @@ TEXT setg_gcc<>(SB),NOSPLIT,$8 MOVD savedR27-8(SP), R27 RET +TEXT runtime·emptyfunc(SB),0,$0-0 + RET + TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 MOVD ZR, R0 MOVD (R0), R0 diff --git a/src/runtime/defs_windows.go b/src/runtime/defs_windows.go index 656fd2b8b6..8d4e38120e 100644 --- a/src/runtime/defs_windows.go +++ b/src/runtime/defs_windows.go @@ -28,6 +28,7 @@ const ( _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 _EXCEPTION_BREAKPOINT = 0x80000003 + _EXCEPTION_ILLEGAL_INSTRUCTION = 0xc000001d _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f diff --git a/src/runtime/memclr_arm.s b/src/runtime/memclr_arm.s index f113a1aa2d..f02d058ead 100644 --- a/src/runtime/memclr_arm.s +++ b/src/runtime/memclr_arm.s @@ -33,6 +33,7 @@ // See memclrNoHeapPointers Go doc for important implementation constraints. // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) +// Also called from assembly in sys_windows_arm.s without g (but using Go stack convention). TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-8 MOVW ptr+0(FP), TO MOVW n+4(FP), N diff --git a/src/runtime/memclr_arm64.s b/src/runtime/memclr_arm64.s index bef77651e4..c1a0dcef58 100644 --- a/src/runtime/memclr_arm64.s +++ b/src/runtime/memclr_arm64.s @@ -7,6 +7,7 @@ // See memclrNoHeapPointers Go doc for important implementation constraints. // func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) +// Also called from assembly in sys_windows_arm64.s without g (but using Go stack convention). TEXT runtime·memclrNoHeapPointers(SB),NOSPLIT,$0-16 MOVD ptr+0(FP), R0 MOVD n+8(FP), R1 diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 375c34ed99..f4e21a93ed 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -148,6 +148,9 @@ func tstart_stdcall(newm *m) // Called by OS using stdcall ABI. func ctrlhandler() +// Init-time helper +func wintls() + type mOS struct { threadLock mutex // protects "thread" and prevents closing thread uintptr // thread handle diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index cb1fbe9f81..6215d0ba2d 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -81,6 +81,7 @@ func isgoexception(info *exceptionrecord, r *context) bool { case _EXCEPTION_FLT_OVERFLOW: case _EXCEPTION_FLT_UNDERFLOW: case _EXCEPTION_BREAKPOINT: + case _EXCEPTION_ILLEGAL_INSTRUCTION: // breakpoint arrives this way on arm64 } return true } diff --git a/src/runtime/stubs_arm64.go b/src/runtime/stubs_arm64.go index 6e6e7df6b8..f5e3bb4854 100644 --- a/src/runtime/stubs_arm64.go +++ b/src/runtime/stubs_arm64.go @@ -12,3 +12,5 @@ func save_g() //go:noescape func asmcgocall_no_g(fn, arg unsafe.Pointer) + +func emptyfunc() diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 3f01714c66..cd230ccffd 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -176,17 +176,17 @@ done: BEQ return // Check if we need to set up the control flow guard workaround. - // On Windows/ARM, the stack pointer must lie within system - // stack limits when we resume from exception. + // On Windows, the stack pointer in the context must lie within + // system stack limits when we resume from exception. // Store the resume SP and PC on the g0 stack, - // and return to returntramp on the g0 stack. returntramp + // and return to sigresume on the g0 stack. sigresume // pops the saved PC and SP from the g0 stack, resuming execution // at the desired location. - // If returntramp has already been set up by a previous exception + // If sigresume has already been set up by a previous exception // handler, don't clobber the stored SP and PC on the stack. MOVW 4(R3), R3 // PEXCEPTION_POINTERS->Context MOVW context_pc(R3), R2 // load PC from context record - MOVW $returntramp<>(SB), R1 + MOVW $sigresume<>(SB), R1 CMP R1, R2 B.EQ return // do not clobber saved SP/PC @@ -196,9 +196,9 @@ done: MOVW context_pc(R3), R2 MOVW R2, context_r1(R3) - // Set up context record to return to returntramp on g0 stack + // Set up context record to return to sigresume on g0 stack MOVW R12, context_spr(R3) - MOVW $returntramp<>(SB), R2 + MOVW $sigresume<>(SB), R2 MOVW R2, context_pc(R3) return: @@ -208,8 +208,8 @@ return: // This is part of the control flow guard workaround. // It switches stacks and jumps to the continuation address. // R0 and R1 are set above at the end of sigtramp<> -// in the context that starts executing at returntramp<>. -TEXT returntramp<>(SB),NOSPLIT|NOFRAME,$0 +// in the context that starts executing at sigresume<>. +TEXT sigresume<>(SB),NOSPLIT|NOFRAME,$0 // Important: do not smash LR, // which is set to a live value when handling // a signal by pushing a call to sigpanic onto the stack. diff --git a/src/runtime/sys_windows_arm64.s b/src/runtime/sys_windows_arm64.s index b279f25de8..53960488f9 100644 --- a/src/runtime/sys_windows_arm64.s +++ b/src/runtime/sys_windows_arm64.s @@ -5,108 +5,150 @@ #include "go_asm.h" #include "go_tls.h" #include "textflag.h" +#include "funcdata.h" -#ifdef NOT_PORTED +// Offsets into Thread Environment Block (pointer in R18) +#define TEB_error 0x68 +#define TEB_TlsSlots 0x1480 -// Note: For system ABI, R0-R3 are args, R4-R11 are callee-save. +// Note: R0-R7 are args, R8 is indirect return value address, +// R9-R15 are caller-save, R19-R29 are callee-save. +// +// load_g and save_g (in tls_arm64.s) clobber R27 (REGTMP) and R0. // void runtime·asmstdcall(void *c); TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} - MOVW R0, R4 // put libcall * in r4 - MOVW R13, R5 // save stack pointer in r5 + STP.W (R29, R30), -32(RSP) // allocate C ABI stack frame + STP (R19, R20), 16(RSP) // save old R19, R20 + MOVD R0, R19 // save libcall pointer + MOVD RSP, R20 // save stack pointer // SetLastError(0) - MOVW $0, R0 - MRC 15, 0, R1, C13, C0, 2 - MOVW R0, 0x34(R1) - - MOVW 8(R4), R12 // libcall->args - - // Do we have more than 4 arguments? - MOVW 4(R4), R0 // libcall->n - SUB.S $4, R0, R2 - BLE loadregs + MOVD $0, TEB_error(R18_PLATFORM) + MOVD libcall_args(R19), R12 // libcall->args + + // Do we have more than 8 arguments? + MOVD libcall_n(R19), R0 + CMP $0, R0; BEQ _0args + CMP $1, R0; BEQ _1args + CMP $2, R0; BEQ _2args + CMP $3, R0; BEQ _3args + CMP $4, R0; BEQ _4args + CMP $5, R0; BEQ _5args + CMP $6, R0; BEQ _6args + CMP $7, R0; BEQ _7args + CMP $8, R0; BEQ _8args // Reserve stack space for remaining args - SUB R2<<2, R13 - BIC $0x7, R13 // alignment for ABI - - // R0: count of arguments - // R1: - // R2: loop counter, from 0 to (n-4) - // R3: scratch - // R4: pointer to libcall struct - // R12: libcall->args - MOVW $0, R2 + SUB $8, R0, R2 + ADD $1, R2, R3 // make even number of words for stack alignment + AND $~1, R3 + LSL $3, R3 + SUB R3, RSP + + // R4: size of stack arguments (n-8)*8 + // R5: &args[8] + // R6: loop counter, from 0 to (n-8)*8 + // R7: scratch + // R8: copy of RSP - (R2)(RSP) assembles as (R2)(ZR) + SUB $8, R0, R4 + LSL $3, R4 + ADD $(8*8), R12, R5 + MOVD $0, R6 + MOVD RSP, R8 stackargs: - ADD $4, R2, R3 // r3 = args[4 + i] - MOVW R3<<2(R12), R3 - MOVW R3, R2<<2(R13) // stack[i] = r3 - - ADD $1, R2 // i++ - SUB $4, R0, R3 // while (i < (n - 4)) - CMP R3, R2 - BLT stackargs - -loadregs: - CMP $3, R0 - MOVW.GT 12(R12), R3 - - CMP $2, R0 - MOVW.GT 8(R12), R2 - - CMP $1, R0 - MOVW.GT 4(R12), R1 - - CMP $0, R0 - MOVW.GT 0(R12), R0 - - BIC $0x7, R13 // alignment for ABI - MOVW 0(R4), R12 // branch to libcall->fn + MOVD (R6)(R5), R7 + MOVD R7, (R6)(R8) + ADD $8, R6 + CMP R6, R4 + BNE stackargs + +_8args: + MOVD (7*8)(R12), R7 +_7args: + MOVD (6*8)(R12), R6 +_6args: + MOVD (5*8)(R12), R5 +_5args: + MOVD (4*8)(R12), R4 +_4args: + MOVD (3*8)(R12), R3 +_3args: + MOVD (2*8)(R12), R2 +_2args: + MOVD (1*8)(R12), R1 +_1args: + MOVD (0*8)(R12), R0 +_0args: + + MOVD libcall_fn(R19), R12 // branch to libcall->fn BL (R12) - MOVW R5, R13 // free stack space - MOVW R0, 12(R4) // save return value to libcall->r1 - MOVW R1, 16(R4) + MOVD R20, RSP // free stack space + MOVD R0, libcall_r1(R19) // save return value to libcall->r1 + // TODO(rsc) floating point like amd64 in libcall->r2? // GetLastError - MRC 15, 0, R1, C13, C0, 2 - MOVW 0x34(R1), R0 - MOVW R0, 20(R4) // store in libcall->err + MOVD TEB_error(R18_PLATFORM), R0 + MOVD R0, libcall_err(R19) - MOVM.IA.W (R13), [R4, R5, R15] + // Restore callee-saved registers. + LDP 16(RSP), (R19, R20) + LDP.P 32(RSP), (R29, R30) + RET -TEXT runtime·badsignal2(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R14], (R13) // push {r4, lr} - MOVW R13, R4 // save original stack pointer - SUB $8, R13 // space for 2 variables - BIC $0x7, R13 // alignment for ABI +TEXT runtime·badsignal2(SB),NOSPLIT,$16-0 + NO_LOCAL_POINTERS // stderr - MOVW runtime·_GetStdHandle(SB), R1 - MOVW $-12, R0 + MOVD runtime·_GetStdHandle(SB), R1 + MOVD $-12, R0 + SUB $16, RSP // skip over saved frame pointer below RSP BL (R1) - - MOVW $runtime·badsignalmsg(SB), R1 // lpBuffer - MOVW $runtime·badsignallen(SB), R2 // lpNumberOfBytesToWrite - MOVW (R2), R2 - ADD $0x4, R13, R3 // lpNumberOfBytesWritten - MOVW $0, R12 // lpOverlapped - MOVW R12, (R13) - - MOVW runtime·_WriteFile(SB), R12 + ADD $16, RSP + + // handle in R0 already + MOVD $runtime·badsignalmsg(SB), R1 // lpBuffer + MOVD $runtime·badsignallen(SB), R2 // lpNumberOfBytesToWrite + MOVD (R2), R2 + MOVD R13, R3 // lpNumberOfBytesWritten + MOVD $0, R4 // lpOverlapped + MOVD runtime·_WriteFile(SB), R12 + SUB $16, RSP // skip over saved frame pointer below RSP BL (R12) + ADD $16, RSP - MOVW R4, R13 // restore SP - MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} + RET -TEXT runtime·getlasterror(SB),NOSPLIT,$0 - MRC 15, 0, R0, C13, C0, 2 - MOVW 0x34(R0), R0 - MOVW R0, ret+0(FP) +TEXT runtime·getlasterror(SB),NOSPLIT|NOFRAME,$0 + MOVD TEB_error(R18_PLATFORM), R0 + MOVD R0, ret+0(FP) RET +#define SAVE_R19_TO_R28(offset) \ + MOVD R19, savedR19+((offset)+0*8)(SP); \ + MOVD R20, savedR20+((offset)+1*8)(SP); \ + MOVD R21, savedR21+((offset)+2*8)(SP); \ + MOVD R22, savedR22+((offset)+3*8)(SP); \ + MOVD R23, savedR23+((offset)+4*8)(SP); \ + MOVD R24, savedR24+((offset)+5*8)(SP); \ + MOVD R25, savedR25+((offset)+6*8)(SP); \ + MOVD R26, savedR26+((offset)+7*8)(SP); \ + MOVD R27, savedR27+((offset)+8*8)(SP); \ + MOVD g, savedR28+((offset)+9*8)(SP); + +#define RESTORE_R19_TO_R28(offset) \ + MOVD savedR19+((offset)+0*8)(SP), R19; \ + MOVD savedR20+((offset)+1*8)(SP), R20; \ + MOVD savedR21+((offset)+2*8)(SP), R21; \ + MOVD savedR22+((offset)+3*8)(SP), R22; \ + MOVD savedR23+((offset)+4*8)(SP), R23; \ + MOVD savedR24+((offset)+5*8)(SP), R24; \ + MOVD savedR25+((offset)+6*8)(SP), R25; \ + MOVD savedR26+((offset)+7*8)(SP), R26; \ + MOVD savedR27+((offset)+8*8)(SP), R27; \ + MOVD savedR28+((offset)+9*8)(SP), g; /* R28 */ + // Called by Windows as a Vectored Exception Handler (VEH). // First argument is pointer to struct containing // exception record and context pointers. @@ -116,61 +158,83 @@ TEXT runtime·getlasterror(SB),NOSPLIT,$0 // PEXCEPTION_POINTERS ExceptionInfo, // func *GoExceptionHandler); TEXT sigtramp<>(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R0, R4-R11, R14], (R13) // push {r0, r4-r11, lr} (SP-=40) - SUB $(8+20), R13 // reserve space for g, sp, and - // parameters/retval to go call + // Save R0, R1 (args) as well as LR, R27, R28 (callee-save). + MOVD R0, R5 + MOVD R1, R6 + MOVD LR, R7 + MOVD R27, R16 // saved R27 (callee-save) + MOVD g, R17 // saved R28 (callee-save from Windows, not really g) + + BL runtime·load_g(SB) // smashes R0, R27, R28 (g) + CMP $0, g // is there a current g? + BNE 2(PC) + BL runtime·badsignal2(SB) + + // Do we need to switch to the g0 stack? + MOVD g, R3 // R3 = oldg (for sigtramp_g0) + MOVD g_m(g), R2 // R2 = m + MOVD m_g0(R2), R2 // R2 = g0 + CMP g, R2 // if curg == g0 + BNE switch + + // No: on g0 stack already, tail call to sigtramp_g0. + // Restore all the callee-saves so sigtramp_g0 can return to our caller. + // We also pass R2 = g0, R3 = oldg, both set above. + MOVD R5, R0 + MOVD R6, R1 + MOVD R7, LR + MOVD R16, R27 // restore R27 + MOVD R17, g // restore R28 + B sigtramp_g0<>(SB) + +switch: + // switch to g0 stack (but do not update g - that's sigtramp_g0's job) + MOVD RSP, R8 + MOVD (g_sched+gobuf_sp)(R2), R4 // R4 = g->gobuf.sp + SUB $(6*8), R4 // alloc space for saves - 2 words below SP for frame pointer, 3 for us to use, 1 for alignment + MOVD R4, RSP // switch to g0 stack + + MOVD $0, (0*8)(RSP) // fake saved LR + MOVD R7, (1*8)(RSP) // saved LR + MOVD R8, (2*8)(RSP) // saved SP + + MOVD R5, R0 // original args + MOVD R6, R1 // original args + MOVD R16, R27 + MOVD R17, g // R28 + BL sigtramp_g0<>(SB) + + // switch back to original stack; g already updated + MOVD (1*8)(RSP), R7 // saved LR + MOVD (2*8)(RSP), R8 // saved SP + MOVD R7, LR + MOVD R8, RSP + RET - MOVW R0, R6 // Save param0 - MOVW R1, R7 // Save param1 +// sigtramp_g0 is running on the g0 stack, with R2 = g0, R3 = oldg. +// But g itself is not set - that's R28, a callee-save register, +// and it still holds the value from the Windows DLL caller. +TEXT sigtramp_g0<>(SB),NOSPLIT,$128 + NO_LOCAL_POINTERS - BL runtime·load_g(SB) - CMP $0, g // is there a current g? - BL.EQ runtime·badsignal2(SB) - - // save g and SP in case of stack switch - MOVW R13, 24(R13) - MOVW g, 20(R13) - - // do we need to switch to the g0 stack? - MOVW g, R5 // R5 = g - MOVW g_m(R5), R2 // R2 = m - MOVW m_g0(R2), R4 // R4 = g0 - CMP R5, R4 // if curg == g0 - BEQ g0 - - // switch to g0 stack - MOVW R4, g // g = g0 - MOVW (g_sched+gobuf_sp)(g), R3 // R3 = g->gobuf.sp - BL runtime·save_g(SB) - - // make room for sighandler arguments - // and re-save old SP for restoring later. - // (note that the 24(R3) here must match the 24(R13) above.) - SUB $40, R3 - MOVW R13, 24(R3) // save old stack pointer - MOVW R3, R13 // switch stack - -g0: - MOVW 0(R6), R2 // R2 = ExceptionPointers->ExceptionRecord - MOVW 4(R6), R3 // R3 = ExceptionPointers->ContextRecord - - MOVW $0, R4 - MOVW R4, 0(R13) // No saved link register. - MOVW R2, 4(R13) // Move arg0 (ExceptionRecord) into position - MOVW R3, 8(R13) // Move arg1 (ContextRecord) into position - MOVW R5, 12(R13) // Move arg2 (original g) into position - BL (R7) // Call the go routine - MOVW 16(R13), R4 // Fetch return value from stack - - // switch back to original stack and g - MOVW 24(R13), R13 - MOVW 20(R13), g - BL runtime·save_g(SB) - -done: - MOVW R4, R0 // move retval into position - ADD $(8 + 20), R13 // free locals - MOVM.IA.W (R13), [R3, R4-R11, R14] // pop {r3, r4-r11, lr} + // Push C callee-save registers R19-R28. LR, FP already saved. + SAVE_R19_TO_R28(-10*8) + + MOVD 0(R0), R5 // R5 = ExceptionPointers->ExceptionRecord + MOVD 8(R0), R6 // R6 = ExceptionPointers->ContextRecord + MOVD R6, context-(11*8)(SP) + + MOVD R2, g // g0 + BL runtime·save_g(SB) // smashes R0 + + MOVD R5, (1*8)(RSP) // arg0 (ExceptionRecord) + MOVD R6, (2*8)(RSP) // arg1 (ContextRecord) + MOVD R3, (3*8)(RSP) // arg2 (original g) + MOVD R3, oldg-(12*8)(SP) + BL (R1) + MOVD oldg-(12*8)(SP), g + BL runtime·save_g(SB) // smashes R0 + MOVW (4*8)(RSP), R0 // return value (0 or -1) // if return value is CONTINUE_SEARCH, do not set up control // flow guard workaround @@ -178,240 +242,232 @@ done: BEQ return // Check if we need to set up the control flow guard workaround. - // On Windows/ARM, the stack pointer must lie within system - // stack limits when we resume from exception. - // Store the resume SP and PC on the g0 stack, - // and return to returntramp on the g0 stack. returntramp - // pops the saved PC and SP from the g0 stack, resuming execution - // at the desired location. - // If returntramp has already been set up by a previous exception - // handler, don't clobber the stored SP and PC on the stack. - MOVW 4(R3), R3 // PEXCEPTION_POINTERS->Context - MOVW context_pc(R3), R2 // load PC from context record - MOVW $returntramp<>(SB), R1 + // On Windows, the stack pointer in the context must lie within + // system stack limits when we resume from exception. + // Store the resume SP and PC in alternate registers + // and return to sigresume on the g0 stack. + // sigresume makes no use of the stack at all, + // loading SP from R0 and jumping to R1. + // Note that smashing R0 and R1 is only safe because we know sigpanic + // will not actually return to the original frame, so the registers + // are effectively dead. But this does mean we can't use the + // same mechanism for async preemption. + MOVD context-(11*8)(SP), R6 + MOVD context_pc(R6), R2 // load PC from context record + MOVD $sigresume<>(SB), R1 + CMP R1, R2 - B.EQ return // do not clobber saved SP/PC + BEQ return // do not clobber saved SP/PC // Save resume SP and PC into R0, R1. - MOVW context_spr(R3), R2 - MOVW R2, context_r0(R3) - MOVW context_pc(R3), R2 - MOVW R2, context_r1(R3) + MOVD context_xsp(R6), R2 + MOVD R2, (context_x+0*8)(R6) + MOVD context_pc(R6), R2 + MOVD R2, (context_x+1*8)(R6) - // Set up context record to return to returntramp on g0 stack - MOVW R12, context_spr(R3) - MOVW $returntramp<>(SB), R2 - MOVW R2, context_pc(R3) + // Set up context record to return to sigresume on g0 stack + MOVD RSP, R2 + MOVD R2, context_xsp(R6) + MOVD $sigresume<>(SB), R2 + MOVD R2, context_pc(R6) return: - B (R14) // return + RESTORE_R19_TO_R28(-10*8) // smashes g + RET // Trampoline to resume execution from exception handler. // This is part of the control flow guard workaround. // It switches stacks and jumps to the continuation address. // R0 and R1 are set above at the end of sigtramp<> -// in the context that starts executing at returntramp<>. -TEXT returntramp<>(SB),NOSPLIT|NOFRAME,$0 +// in the context that starts executing at sigresume<>. +TEXT sigresume<>(SB),NOSPLIT|NOFRAME,$0 // Important: do not smash LR, // which is set to a live value when handling // a signal by pushing a call to sigpanic onto the stack. - MOVW R0, R13 + MOVD R0, RSP B (R1) TEXT runtime·exceptiontramp(SB),NOSPLIT|NOFRAME,$0 - MOVW $runtime·exceptionhandler(SB), R1 + MOVD $runtime·exceptionhandler(SB), R1 B sigtramp<>(SB) TEXT runtime·firstcontinuetramp(SB),NOSPLIT|NOFRAME,$0 - MOVW $runtime·firstcontinuehandler(SB), R1 + MOVD $runtime·firstcontinuehandler(SB), R1 B sigtramp<>(SB) TEXT runtime·lastcontinuetramp(SB),NOSPLIT|NOFRAME,$0 - MOVW $runtime·lastcontinuehandler(SB), R1 + MOVD $runtime·lastcontinuehandler(SB), R1 B sigtramp<>(SB) TEXT runtime·ctrlhandler(SB),NOSPLIT|NOFRAME,$0 - MOVW $runtime·ctrlhandler1(SB), R1 + MOVD $runtime·ctrlhandler1(SB), R1 B runtime·externalthreadhandler(SB) TEXT runtime·profileloop(SB),NOSPLIT|NOFRAME,$0 - MOVW $runtime·profileloop1(SB), R1 + MOVD $runtime·profileloop1(SB), R1 B runtime·externalthreadhandler(SB) -// int32 externalthreadhandler(uint32 arg, int (*func)(uint32)) -// stack layout: -// +----------------+ -// | callee-save | -// | registers | -// +----------------+ -// | m | -// +----------------+ -// 20| g | -// +----------------+ -// 16| func ptr (r1) | -// +----------------+ -// 12| argument (r0) | -//---+----------------+ -// 8 | param1 | (also return value for called Go function) -// +----------------+ -// 4 | param0 | -// +----------------+ -// 0 | slot for LR | -// +----------------+ -// -TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME|TOPFRAME,$0 - MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} - SUB $(m__size + g__size + 20), R13 // space for locals - MOVW R14, 0(R13) // push LR again for anything unwinding the stack - MOVW R0, 12(R13) - MOVW R1, 16(R13) - - // zero out m and g structures - ADD $20, R13, R0 // compute pointer to g - MOVW R0, 4(R13) - MOVW $(m__size + g__size), R0 - MOVW R0, 8(R13) +// externalthreadhander called with R0 = uint32 arg, R1 = Go function f. +// Need to call f(arg), which returns a uint32, and return it in R0. +TEXT runtime·externalthreadhandler(SB),NOSPLIT|TOPFRAME,$96-0 + NO_LOCAL_POINTERS + + // Push C callee-save registers R19-R28. LR, FP already saved. + SAVE_R19_TO_R28(-10*8) + + // Allocate space for args, saved R0+R1, g, and m structures. + // Hide from nosplit check. + #define extra ((64+g__size+m__size+15)&~15) + SUB $extra, RSP, R2 // hide from nosplit overflow check + MOVD R2, RSP + + // Save R0 and R1 (our args). + MOVD R0, 32(RSP) + MOVD R1, 40(RSP) + + // Zero out m and g structures. + MOVD $64(RSP), R0 + MOVD R0, 8(RSP) + MOVD $(m__size + g__size), R0 + MOVD R0, 16(RSP) + MOVD $0, 0(RSP) // not-saved LR BL runtime·memclrNoHeapPointers(SB) - // initialize m and g structures - ADD $20, R13, R2 // R2 = g - ADD $(20 + g__size), R13, R3 // R3 = m - MOVW R2, m_g0(R3) // m->g0 = g - MOVW R3, g_m(R2) // g->m = m - MOVW R2, m_curg(R3) // m->curg = g - - MOVW R2, g + // Initialize m and g structures. + MOVD $64(RSP), g + MOVD $g__size(g), R3 // m + MOVD R3, g_m(g) // g->m = m + MOVD g, m_g0(R3) // m->g0 = g + MOVD g, m_curg(R3) // m->curg = g + MOVD RSP, R0 + MOVD R0, g_stack+stack_hi(g) + SUB $(32*1024), R0 + MOVD R0, (g_stack+stack_lo)(g) + MOVD R0, g_stackguard0(g) + MOVD R0, g_stackguard1(g) BL runtime·save_g(SB) - // set up stackguard stuff - MOVW R13, R0 - MOVW R0, g_stack+stack_hi(g) - SUB $(32*1024), R0 - MOVW R0, (g_stack+stack_lo)(g) - MOVW R0, g_stackguard0(g) - MOVW R0, g_stackguard1(g) - - // move argument into position and call function - MOVW 12(R13), R0 - MOVW R0, 4(R13) - MOVW 16(R13), R1 + // Call function. + MOVD 32(RSP), R0 + MOVD 40(RSP), R1 + MOVW R0, 8(RSP) BL (R1) - // clear g - MOVW $0, g + // Clear g. + MOVD $0, g BL runtime·save_g(SB) - MOVW 8(R13), R0 // load return value - ADD $(m__size + g__size + 20), R13 // free locals - MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + // Load return value (save_g would have smashed) + MOVW (2*8)(RSP), R0 + + ADD $extra, RSP, R2 + MOVD R2, RSP + #undef extra + + RESTORE_R19_TO_R28(-10*8) + RET GLOBL runtime·cbctxts(SB), NOPTR, $4 -TEXT runtime·callbackasm1(SB),NOSPLIT|NOFRAME,$0 - // On entry, the trampoline in zcallback_windows_arm.s left +TEXT runtime·callbackasm1(SB),NOSPLIT,$208-0 + NO_LOCAL_POINTERS + + // On entry, the trampoline in zcallback_windows_arm64.s left // the callback index in R12 (which is volatile in the C ABI). - // Push callback register arguments r0-r3. We do this first so - // they're contiguous with stack arguments. - MOVM.DB.W [R0-R3], (R13) - // Push C callee-save registers r4-r11 and lr. - MOVM.DB.W [R4-R11, R14], (R13) - SUB $(16 + callbackArgs__size), R13 // space for locals + // Save callback register arguments R0-R7. + // We do this at the top of the frame so they're contiguous with stack arguments. + MOVD R0, arg0-(8*8)(SP) + MOVD R1, arg1-(7*8)(SP) + MOVD R2, arg2-(6*8)(SP) + MOVD R3, arg3-(5*8)(SP) + MOVD R4, arg4-(4*8)(SP) + MOVD R5, arg5-(3*8)(SP) + MOVD R6, arg6-(2*8)(SP) + MOVD R7, arg7-(1*8)(SP) + + // Push C callee-save registers R19-R28. + // LR, FP already saved. + SAVE_R19_TO_R28(-18*8) // Create a struct callbackArgs on our stack. - MOVW R12, (16+callbackArgs_index)(R13) // callback index - MOVW $(16+callbackArgs__size+4*9)(R13), R0 - MOVW R0, (16+callbackArgs_args)(R13) // address of args vector - MOVW $0, R0 - MOVW R0, (16+callbackArgs_result)(R13) // result - - // Prepare for entry to Go. - BL runtime·load_g(SB) + MOVD $cbargs-(18*8+callbackArgs__size)(SP), R13 + MOVD R12, callbackArgs_index(R13) // callback index + MOVD $arg0-(8*8)(SP), R0 + MOVD R0, callbackArgs_args(R13) // address of args vector + MOVD $0, R0 + MOVD R0, callbackArgs_result(R13) // result // Call cgocallback, which will call callbackWrap(frame). - MOVW $0, R0 - MOVW R0, 12(R13) // context - MOVW $16(R13), R1 // R1 = &callbackArgs{...} - MOVW R1, 8(R13) // frame (address of callbackArgs) - MOVW $·callbackWrap(SB), R1 - MOVW R1, 4(R13) // PC of function to call + MOVD $·callbackWrap(SB), R0 // PC of function to call + MOVD R13, R1 // frame (&callbackArgs{...}) + MOVD $0, R2 // context + MOVD R0, (1*8)(RSP) + MOVD R1, (2*8)(RSP) + MOVD R2, (3*8)(RSP) BL runtime·cgocallback(SB) // Get callback result. - MOVW (16+callbackArgs_result)(R13), R0 + MOVD $cbargs-(18*8+callbackArgs__size)(SP), R13 + MOVD callbackArgs_result(R13), R0 - ADD $(16 + callbackArgs__size), R13 // free locals - MOVM.IA.W (R13), [R4-R11, R12] // pop {r4-r11, lr=>r12} - ADD $(4*4), R13 // skip r0-r3 - B (R12) // return + RESTORE_R19_TO_R28(-18*8) + + RET // uint32 tstart_stdcall(M *newm); -TEXT runtime·tstart_stdcall(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} +TEXT runtime·tstart_stdcall(SB),NOSPLIT,$96-0 + SAVE_R19_TO_R28(-10*8) - MOVW m_g0(R0), g - MOVW R0, g_m(g) + MOVD m_g0(R0), g + MOVD R0, g_m(g) BL runtime·save_g(SB) - // Layout new m scheduler stack on os stack. - MOVW R13, R0 - MOVW R0, g_stack+stack_hi(g) + // Set up stack guards for OS stack. + MOVD RSP, R0 + MOVD R0, g_stack+stack_hi(g) SUB $(64*1024), R0 - MOVW R0, (g_stack+stack_lo)(g) - MOVW R0, g_stackguard0(g) - MOVW R0, g_stackguard1(g) + MOVD R0, (g_stack+stack_lo)(g) + MOVD R0, g_stackguard0(g) + MOVD R0, g_stackguard1(g) BL runtime·emptyfunc(SB) // fault if stack check is wrong BL runtime·mstart(SB) + RESTORE_R19_TO_R28(-10*8) + // Exit the thread. - MOVW $0, R0 - MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + MOVD $0, R0 + RET // Runs on OS stack. // duration (in -100ns units) is in dt+0(FP). // g may be nil. -TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$0-4 +TEXT runtime·usleep2(SB),NOSPLIT,$32-4 MOVW dt+0(FP), R0 - MOVM.DB.W [R4, R14], (R13) // push {r4, lr} - MOVW R13, R4 // Save SP - SUB $8, R13 // R13 = R13 - 8 - BIC $0x7, R13 // Align SP for ABI - RSB $0, R0, R3 // R3 = -R0 - MOVW $0, R1 // R1 = FALSE (alertable) - MOVW $-1, R0 // R0 = handle - MOVW R13, R2 // R2 = pTime - MOVW R3, 0(R2) // time_lo - MOVW R0, 4(R2) // time_hi - MOVW runtime·_NtWaitForSingleObject(SB), R3 + MOVD $16(RSP), R2 // R2 = pTime + MOVD R0, 0(R2) // *pTime = -dt + MOVD $-1, R0 // R0 = handle + MOVD $0, R1 // R1 = FALSE (alertable) + MOVD runtime·_NtWaitForSingleObject(SB), R3 + SUB $16, RSP // skip over saved frame pointer below RSP BL (R3) - MOVW R4, R13 // Restore SP - MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} + ADD $16, RSP + RET // Runs on OS stack. // duration (in -100ns units) is in dt+0(FP). // g is valid. // TODO: neeeds to be implemented properly. -TEXT runtime·usleep2HighRes(SB),NOSPLIT|NOFRAME,$0-4 +TEXT runtime·usleep2HighRes(SB),NOSPLIT,$0-4 B runtime·abort(SB) // Runs on OS stack. -TEXT runtime·switchtothread(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R14], (R13) // push {R4, lr} - MOVW R13, R4 - BIC $0x7, R13 // alignment for ABI - MOVW runtime·_SwitchToThread(SB), R0 +TEXT runtime·switchtothread(SB),NOSPLIT,$16-0 + MOVD runtime·_SwitchToThread(SB), R0 + SUB $16, RSP // skip over saved frame pointer below RSP BL (R0) - MOVW R4, R13 // restore stack pointer - MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} - -TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 - B runtime·armPublicationBarrier(SB) - -// never called (cgo not supported) -TEXT runtime·read_tls_fallback(SB),NOSPLIT|NOFRAME,$0 - MOVW $0xabcd, R0 - MOVW R0, (R0) + ADD $16, RSP RET // See http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/ @@ -423,166 +479,101 @@ TEXT runtime·read_tls_fallback(SB),NOSPLIT|NOFRAME,$0 #define time_hi2 8 TEXT runtime·nanotime1(SB),NOSPLIT|NOFRAME,$0-8 - MOVW $0, R0 MOVB runtime·useQPCTime(SB), R0 CMP $0, R0 BNE useQPC - MOVW $_INTERRUPT_TIME, R3 + MOVD $_INTERRUPT_TIME, R3 loop: - MOVW time_hi1(R3), R1 - MOVW time_lo(R3), R0 - MOVW time_hi2(R3), R2 + MOVWU time_hi1(R3), R1 + MOVWU time_lo(R3), R0 + MOVWU time_hi2(R3), R2 CMP R1, R2 BNE loop // wintime = R1:R0, multiply by 100 - MOVW $100, R2 - MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 - MULA R1, R2, R4, R4 - - // wintime*100 = R4:R3 - MOVW R3, ret_lo+0(FP) - MOVW R4, ret_hi+4(FP) + ORR R1<<32, R0 + MOVD $100, R1 + MUL R1, R0 + MOVD R0, ret+0(FP) RET useQPC: B runtime·nanotimeQPC(SB) // tail call -TEXT time·now(SB),NOSPLIT|NOFRAME,$0-20 - MOVW $0, R0 +TEXT time·now(SB),NOSPLIT|NOFRAME,$0-24 MOVB runtime·useQPCTime(SB), R0 CMP $0, R0 BNE useQPC - MOVW $_INTERRUPT_TIME, R3 + MOVD $_INTERRUPT_TIME, R3 loop: - MOVW time_hi1(R3), R1 - MOVW time_lo(R3), R0 - MOVW time_hi2(R3), R2 + MOVWU time_hi1(R3), R1 + MOVWU time_lo(R3), R0 + MOVWU time_hi2(R3), R2 CMP R1, R2 BNE loop // wintime = R1:R0, multiply by 100 - MOVW $100, R2 - MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 - MULA R1, R2, R4, R4 - - // wintime*100 = R4:R3 - MOVW R3, mono+12(FP) - MOVW R4, mono+16(FP) + ORR R1<<32, R0 + MOVD $100, R1 + MUL R1, R0 + MOVD R0, mono+16(FP) - MOVW $_SYSTEM_TIME, R3 + MOVD $_SYSTEM_TIME, R3 wall: - MOVW time_hi1(R3), R1 - MOVW time_lo(R3), R0 - MOVW time_hi2(R3), R2 + MOVWU time_hi1(R3), R1 + MOVWU time_lo(R3), R0 + MOVWU time_hi2(R3), R2 CMP R1, R2 BNE wall - // w = R1:R0 in 100ns untis + // w = R1:R0 in 100ns units // convert to Unix epoch (but still 100ns units) #define delta 116444736000000000 - SUB.S $(delta & 0xFFFFFFFF), R0 - SBC $(delta >> 32), R1 + ORR R1<<32, R0 + SUB $delta, R0 // Convert to nSec - MOVW $100, R2 - MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 - MULA R1, R2, R4, R4 - // w = R2:R1 in nSec - MOVW R3, R1 // R4:R3 -> R2:R1 - MOVW R4, R2 - - // multiply nanoseconds by reciprocal of 10**9 (scaled by 2**61) - // to get seconds (96 bit scaled result) - MOVW $0x89705f41, R3 // 2**61 * 10**-9 - MULLU R1,R3,(R6,R5) // R7:R6:R5 = R2:R1 * R3 - MOVW $0,R7 - MULALU R2,R3,(R7,R6) - - // unscale by discarding low 32 bits, shifting the rest by 29 - MOVW R6>>29,R6 // R7:R6 = (R7:R6:R5 >> 61) - ORR R7<<3,R6 - MOVW R7>>29,R7 - - // subtract (10**9 * sec) from nsec to get nanosecond remainder - MOVW $1000000000, R5 // 10**9 - MULLU R6,R5,(R9,R8) // R9:R8 = R7:R6 * R5 - MULA R7,R5,R9,R9 - SUB.S R8,R1 // R2:R1 -= R9:R8 - SBC R9,R2 - - // because reciprocal was a truncated repeating fraction, quotient - // may be slightly too small -- adjust to make remainder < 10**9 - CMP R5,R1 // if remainder > 10**9 - SUB.HS R5,R1 // remainder -= 10**9 - ADD.HS $1,R6 // sec += 1 - - MOVW R6,sec_lo+0(FP) - MOVW R7,sec_hi+4(FP) - MOVW R1,nsec+8(FP) + MOVD $100, R1 + MUL R1, R0 + + // Code stolen from compiler output for: + // + // var x uint64 + // func f() (sec uint64, nsec uint32) { return x / 1000000000, uint32(x % 100000000) } + // + LSR $1, R0, R1 + MOVD $-8543223759426509416, R2 + UMULH R2, R1, R1 + LSR $28, R1, R1 + MOVD R1, sec+0(FP) + MOVD $-6067343680855748867, R1 + UMULH R0, R1, R1 + LSR $26, R1, R1 + MOVD $100000000, R2 + MSUB R1, R0, R2, R0 + MOVW R0, nsec+8(FP) RET useQPC: B runtime·nowQPC(SB) // tail call -// save_g saves the g register (R10) into thread local memory -// so that we can call externally compiled -// ARM code that will overwrite those registers. -// NOTE: runtime.gogo assumes that R1 is preserved by this function. -// runtime.mcall assumes this function only clobbers R0 and R11. -// Returns with g in R0. -// Save the value in the _TEB->TlsSlots array. -// Effectively implements TlsSetValue(). -// tls_g stores the TLS slot allocated TlsAlloc(). -TEXT runtime·save_g(SB),NOSPLIT|NOFRAME,$0 - MRC 15, 0, R0, C13, C0, 2 - ADD $0xe10, R0 - MOVW $runtime·tls_g(SB), R11 - MOVW (R11), R11 - MOVW g, R11<<2(R0) - MOVW g, R0 // preserve R0 across call to setg<> - RET - -// load_g loads the g register from thread-local memory, -// for use after calling externally compiled -// ARM code that overwrote those registers. -// Get the value from the _TEB->TlsSlots array. -// Effectively implements TlsGetValue(). -TEXT runtime·load_g(SB),NOSPLIT|NOFRAME,$0 - MRC 15, 0, R0, C13, C0, 2 - ADD $0xe10, R0 - MOVW $runtime·tls_g(SB), g - MOVW (g), g - MOVW g<<2(R0), g - RET - // This is called from rt0_go, which runs on the system stack // using the initial stack allocated by the OS. // It calls back into standard C using the BL below. -// To do that, the stack pointer must be 8-byte-aligned. -TEXT runtime·_initcgo(SB),NOSPLIT|NOFRAME,$0 - MOVM.DB.W [R4, R14], (R13) // push {r4, lr} - - // Ensure stack is 8-byte aligned before calling C code - MOVW R13, R4 - BIC $0x7, R13 - +TEXT runtime·wintls(SB),NOSPLIT,$0 // Allocate a TLS slot to hold g across calls to external code - MOVW $runtime·_TlsAlloc(SB), R0 - MOVW (R0), R0 + MOVD runtime·_TlsAlloc(SB), R0 + SUB $16, RSP // skip over saved frame pointer below RSP BL (R0) + ADD $16, RSP // Assert that slot is less than 64 so we can use _TEB->TlsSlots CMP $64, R0 - MOVW $runtime·abort(SB), R1 - BL.GE (R1) - - // Save Slot into tls_g - MOVW $runtime·tls_g(SB), R1 - MOVW R0, (R1) - - MOVW R4, R13 - MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} - -// Holds the TLS Slot, which was allocated by TlsAlloc() -GLOBL runtime·tls_g+0(SB), NOPTR, $4 + BLT ok + MOVD $runtime·abort(SB), R1 + BL (R1) +ok: -#endif + // Save offset from R18 into tls_g. + LSL $3, R1 + ADD $TEB_TlsSlots, R1 + MOVD R1, runtime·tls_g(SB) + RET diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 666ec5f69e..7cf9318bdb 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -148,6 +148,7 @@ func compileCallback(fn eface, cdecl bool) (code uintptr) { } // cdecl, stdcall, fastcall, and arm pad arguments to word size. + // TODO(rsc): On arm and arm64 do we need to skip the caller's saved LR? src += sys.PtrSize // The Go ABI packs arguments. dst += t.size diff --git a/src/runtime/tls_arm64.h b/src/runtime/tls_arm64.h index 0804fa3502..fe5e4cee12 100644 --- a/src/runtime/tls_arm64.h +++ b/src/runtime/tls_arm64.h @@ -41,8 +41,16 @@ #define MRS_TPIDR_R0 WORD $0xd53bd040 // MRS TPIDR_EL0, R0 #endif +#ifdef GOOS_windows +#define TLS_windows +#endif +#ifdef TLS_windows +#define TLSG_IS_VARIABLE +#define MRS_TPIDR_R0 MOVD R18_PLATFORM, R0 +#endif + // Define something that will break the build if // the GOOS is unknown. -#ifndef TPIDR -#define MRS_TPIDR_R0 TPIDR_UNKNOWN +#ifndef MRS_TPIDR_R0 +#define MRS_TPIDR_R0 unknown_TLS_implementation_in_tls_arm64_h #endif diff --git a/src/runtime/tls_arm64.s b/src/runtime/tls_arm64.s index 085012f791..52b3e8f222 100644 --- a/src/runtime/tls_arm64.s +++ b/src/runtime/tls_arm64.s @@ -9,11 +9,13 @@ #include "tls_arm64.h" TEXT runtime·load_g(SB),NOSPLIT,$0 -#ifndef TLS_darwin +#ifndef GOOS_darwin #ifndef GOOS_openbsd +#ifndef GOOS_windows MOVB runtime·iscgo(SB), R0 CBZ R0, nocgo #endif +#endif #endif MRS_TPIDR_R0 @@ -28,11 +30,13 @@ nocgo: RET TEXT runtime·save_g(SB),NOSPLIT,$0 -#ifndef TLS_darwin +#ifndef GOOS_darwin #ifndef GOOS_openbsd +#ifndef GOOS_windows MOVB runtime·iscgo(SB), R0 CBZ R0, nocgo #endif +#endif #endif MRS_TPIDR_R0 -- cgit v1.3-5-g9baa From b110a43628526787f73db44e11829520d92e5b2b Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 15 Feb 2021 11:26:58 -0500 Subject: runtime: delete gosave (dead code) Change-Id: Ie811526534df8622d89c5b1b81dbe19ece1c962b Reviewed-on: https://go-review.googlesource.com/c/go/+/292110 Trust: Russ Cox Reviewed-by: Cherry Zhang Reviewed-by: Ian Lance Taylor --- src/runtime/proc.go | 2 +- src/runtime/stack.go | 2 +- src/runtime/stubs.go | 1 + src/runtime/sys_wasm.go | 2 +- src/runtime/sys_x86.go | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/proc.go b/src/runtime/proc.go index ccfe085691..dbb430fd25 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -3482,7 +3482,7 @@ func save(pc, sp uintptr) { // This is called only from the go syscall library and cgocall, // not from the low-level system calls used by the runtime. // -// Entersyscall cannot split the stack: the gosave must +// Entersyscall cannot split the stack: the save must // make g->sched refer to the caller's stack segment, because // entersyscall is going to return immediately after. // diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 8c90e7b46f..d971e5e26f 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -1089,7 +1089,7 @@ func nilfunc() { } // adjust Gobuf as if it executed a call to fn -// and then did an immediate gosave. +// and then stopped before the first instruction in fn. func gostartcallfn(gobuf *gobuf, fv *funcval) { var fn unsafe.Pointer if fv != nil { diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index c0cc95ec65..b9b313a711 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -169,6 +169,7 @@ func noescape(p unsafe.Pointer) unsafe.Pointer { // This in turn calls cgocallbackg, which is where we'll find // pointer-declared arguments. func cgocallback(fn, frame, ctxt uintptr) + func gogo(buf *gobuf) //go:noescape diff --git a/src/runtime/sys_wasm.go b/src/runtime/sys_wasm.go index 3ed621f92e..057ed4ccd9 100644 --- a/src/runtime/sys_wasm.go +++ b/src/runtime/sys_wasm.go @@ -27,7 +27,7 @@ func wasmTruncU() func wasmExit(code int32) // adjust Gobuf as it if executed a call to fn with context ctxt -// and then did an immediate gosave. +// and then stopped before the first instruction in fn. func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { sp := buf.sp sp -= sys.PtrSize diff --git a/src/runtime/sys_x86.go b/src/runtime/sys_x86.go index 5b7a666679..8f21585d28 100644 --- a/src/runtime/sys_x86.go +++ b/src/runtime/sys_x86.go @@ -12,7 +12,7 @@ import ( ) // adjust Gobuf as if it executed a call to fn with context ctxt -// and then did an immediate gosave. +// and then stopped before the first instruction in fn. func gostartcall(buf *gobuf, fn, ctxt unsafe.Pointer) { sp := buf.sp sp -= sys.PtrSize -- cgit v1.3-5-g9baa From b445d6ea34661328a7310beda285c64d6823624d Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 15 Feb 2021 17:02:30 -0500 Subject: runtime/pprof: expect tests to pass on macOS macOS tests have been disabled since CL 12429045 (Aug 2013). At the time, macOS required a kernel patch to get a working profiler (https://research.swtch.com/macpprof), which we didn't want to require, of course. macOS has improved - it no longer requires the kernel patch - but we never updated the list of exceptions. As far as I can tell, the builders have no problem passing the pprof test now. (It is possible that the iOS builders have trouble, but that is now a different GOOS.) Remove the exception for macOS. The test should now pass. Fixes #6047. Change-Id: Iab49036cacc1025e56f515bd19d084390c2f5357 Reviewed-on: https://go-review.googlesource.com/c/go/+/292229 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/pprof/pprof_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index f7c1349bc6..d7571953a9 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -279,7 +279,7 @@ func testCPUProfile(t *testing.T, matches matchFunc, need []string, avoid []stri broken := false switch runtime.GOOS { - case "darwin", "ios", "dragonfly", "netbsd", "illumos", "solaris": + case "ios", "dragonfly", "netbsd", "illumos", "solaris": broken = true case "openbsd": if runtime.GOARCH == "arm" || runtime.GOARCH == "arm64" { -- cgit v1.3-5-g9baa From 7764ee5614df2228e03326487af7670c7c5d268a Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Fri, 19 Feb 2021 14:31:57 +0800 Subject: runtime: fix invalid nil g check for for mips64x In CL 292109 we removed unnecessary writes to gp.sched.g but put wrong register to save g (R4 saves pointer to g) on mips64x Change-Id: I9777846a7b0a46e1af83dcfc73b74649e0dba3c9 Reviewed-on: https://go-review.googlesource.com/c/go/+/293989 TryBot-Result: Go Bot Reviewed-by: Joel Sing Trust: Meng Zhuo Run-TryBot: Meng Zhuo --- src/runtime/asm_mips64x.s | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index af27b9b555..c123e96a71 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -98,11 +98,11 @@ TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 MOVV buf+0(FP), R3 MOVV gobuf_g(R3), R4 - MOVV 0(R4), R5 // make sure g != nil + MOVV 0(R4), R0 // make sure g != nil JMP gogo<>(SB) TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 - MOVV R5, g + MOVV R4, g JAL runtime·save_g(SB) MOVV 0(g), R2 -- cgit v1.3-5-g9baa From 49add6ad90c3c6e150266b35ae98067d7b52c021 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 19 Feb 2021 05:22:35 -0500 Subject: runtime: fix spurious stack overflow detection The regabi builders are unhappy about badctxt calling throw calling systemstack calling gosave_systemstack_switch calling badctxt, all nosplit, repeating. This wouldn't actually happen since after one systemstack we'd end up on the system stack and the next one wouldn't call gosave_systemstack_switch at all. The badctxt call itself is in a very unlikely assertion failure inside gosave_systemstack_switch. Keep the assertion check but call runtime.abort instead on failure, breaking the detected (but not real) cycle. Change-Id: Iaf5c0fc065783b8c1c6d0f62d848f023a0714b96 Reviewed-on: https://go-review.googlesource.com/c/go/+/294069 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/asm_386.s | 2 +- src/runtime/asm_amd64.s | 2 +- src/runtime/asm_arm.s | 2 +- src/runtime/asm_arm64.s | 2 +- src/runtime/asm_mips64x.s | 2 +- src/runtime/asm_mipsx.s | 2 +- src/runtime/asm_ppc64x.s | 2 +- src/runtime/asm_riscv64.s | 2 +- src/runtime/asm_s390x.s | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 5b0852f780..5cf6827c21 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -618,7 +618,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 MOVL (g_sched+gobuf_ctxt)(BX), AX TESTL AX, AX JZ 2(PC) - CALL runtime·badctxt(SB) + CALL runtime·abort(SB) POPL BX POPL AX RET diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index a68dc72ae5..517c5a9d3e 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -677,7 +677,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0 MOVQ (g_sched+gobuf_ctxt)(R14), R9 TESTQ R9, R9 JZ 2(PC) - CALL runtime·badctxt(SB) + CALL runtime·abort(SB) RET // func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index f9535bb1bc..9896ab4383 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -536,7 +536,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVW (g_sched+gobuf_ctxt)(g), R11 TST R11, R11 B.EQ 2(PC) - BL runtime·badctxt(SB) + BL runtime·abort(SB) RET // func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 699fc99d58..3709f1d95e 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -875,7 +875,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R0 CBZ R0, 2(PC) - CALL runtime·badctxt(SB) + CALL runtime·abort(SB) RET // func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index c123e96a71..cee4b528bb 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -415,7 +415,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 // Assert ctxt is zero. See func save. MOVV (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) - JAL runtime·badctxt(SB) + JAL runtime·abort(SB) RET // func asmcgocall_no_g(fn, arg unsafe.Pointer) diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index 0c7d28dcf7..17fbc902c2 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -413,7 +413,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 // Assert ctxt is zero. See func save. MOVW (g_sched+gobuf_ctxt)(g), R1 BEQ R1, 2(PC) - JAL runtime·badctxt(SB) + JAL runtime·abort(SB) RET // func asmcgocall(fn, arg unsafe.Pointer) int32 diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 56e73742ea..6544048497 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -547,7 +547,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 MOVD (g_sched+gobuf_ctxt)(g), R31 CMP R0, R31 BEQ 2(PC) - BL runtime·badctxt(SB) + BL runtime·abort(SB) RET #ifdef GOOS_aix diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 30f2bd2e4a..d8d5252ed5 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -310,7 +310,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 // Assert ctxt is zero. See func save. MOV (g_sched+gobuf_ctxt)(g), X31 BEQ ZERO, X31, 2(PC) - CALL runtime·badctxt(SB) + CALL runtime·abort(SB) RET // func asmcgocall(fn, arg unsafe.Pointer) int32 diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index f9fb1a4c55..4748e00aa8 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -511,7 +511,7 @@ TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 // Assert ctxt is zero. See func save. MOVD (g_sched+gobuf_ctxt)(g), R1 CMPBEQ R1, $0, 2(PC) - BL runtime·badctxt(SB) + BL runtime·abort(SB) RET // func asmcgocall(fn, arg unsafe.Pointer) int32 -- cgit v1.3-5-g9baa From fa18f224c378f5831210077944e5df718efb8df5 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 19 Feb 2021 05:42:20 -0500 Subject: runtime/pprof: disable TestMorestack on macOS under race detector This is failing but only under the race detector. It doesn't really seem fair to expect pprof to find specific profile events with the race detector slowing everything down anyway. Change-Id: I4b353d3d63944c87884d117e07d119b2c7bf4684 Reviewed-on: https://go-review.googlesource.com/c/go/+/294071 Trust: Russ Cox Reviewed-by: Cherry Zhang --- src/runtime/pprof/pprof_test.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index d7571953a9..168c1d4496 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -11,6 +11,7 @@ import ( "context" "fmt" "internal/profile" + "internal/race" "internal/testenv" "io" "math/big" @@ -261,18 +262,13 @@ func parseProfile(t *testing.T, valBytes []byte, f func(uintptr, []*profile.Loca // as interpreted by matches, and returns the parsed profile. func testCPUProfile(t *testing.T, matches matchFunc, need []string, avoid []string, f func(dur time.Duration)) *profile.Profile { switch runtime.GOOS { - case "darwin", "ios": - switch runtime.GOARCH { - case "arm64": - // nothing - default: - out, err := exec.Command("uname", "-a").CombinedOutput() - if err != nil { - t.Fatal(err) - } - vers := string(out) - t.Logf("uname -a: %v", vers) + case "darwin": + out, err := exec.Command("uname", "-a").CombinedOutput() + if err != nil { + t.Fatal(err) } + vers := string(out) + t.Logf("uname -a: %v", vers) case "plan9": t.Skip("skipping on plan9") } @@ -588,6 +584,13 @@ func stackContainsAll(spec string, count uintptr, stk []*profile.Location, label } func TestMorestack(t *testing.T) { + if runtime.GOOS == "darwin" && race.Enabled { + // For whatever reason, using the race detector on macOS keeps us + // from finding the newstack/growstack calls in the profile. + // Not worth worrying about. + // https://build.golang.org/log/280d387327806e17c8aabeb38b9503dbbd942ed1 + t.Skip("skipping on darwin race detector") + } testCPUProfile(t, stackContainsAll, []string{"runtime.newstack,runtime/pprof.growstack"}, avoidFunctions(), func(duration time.Duration) { t := time.After(duration) c := make(chan bool) -- cgit v1.3-5-g9baa From 02e5a8fdfcc8e237f5b55618ccbe9ad845014427 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 19 Feb 2021 06:01:25 -0500 Subject: runtime: ignore SPWRITE in syscall functions netbsd/amd64's Syscall9 changes SP using ADD and SUB, which are treated as SPWRITEs (they are not accounted for in the sp-adjust tracking, and there are too many functions that would report mismatched stack adjustments at RET if they were). A traceback starting in Syscall9 as saved by entersyscall complains about the SPWRITE-ness unnecessarily, since the PC/SP are saved at the start of the function. Ignore SPWRITE in that case. netbsd/arm's Syscall6 also changes SP (R13), using a direct write. So even if we could handle the ADD/SUB in the amd64 case or rewrote that assembly, we'd still be stuck with a more difficult problem in this case. Ignoring the SPWRITE fixes it. Example crashes: https://build.golang.org/log/160fc7b051a2cf90782b75a99984fff129329e66 https://build.golang.org/log/7879e2fecdb400eee616294285e1f952e5b17301 Change-Id: I0c8e9696066e90dafed6d4a93d11697da23f0080 Reviewed-on: https://go-review.googlesource.com/c/go/+/294072 Reviewed-by: Cherry Zhang Trust: Russ Cox Run-TryBot: Russ Cox --- src/runtime/traceback.go | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index eb185eecd3..53eb689848 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -174,6 +174,12 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // So we don't need to exclude it with the other SP-writing functions. flag &^= funcFlag_SPWRITE } + if frame.pc == pc0 && frame.sp == sp0 && pc0 == gp.syscallpc && sp0 == gp.syscallsp { + // Some Syscall functions write to SP, but they do so only after + // saving the entry PC/SP using entersyscall. + // Since we are using the entry PC/SP, the later SP write doesn't matter. + flag &^= funcFlag_SPWRITE + } // Found an actual function. // Derive frame pointer and link register. -- cgit v1.3-5-g9baa