diff options
| author | Chressie Himpel <chressie@google.com> | 2022-02-03 19:10:54 +0100 |
|---|---|---|
| committer | Chressie Himpel <chressie@google.com> | 2022-02-03 19:30:02 +0100 |
| commit | e14fee553a4646d15123caa04f7ca6ddb7b48362 (patch) | |
| tree | 26086b9981918546f946d12547d097eb0974cc2e /src/runtime | |
| parent | d382493a20cf005c6631032ebb410a7c2bc768eb (diff) | |
| parent | 8384fe86a5b7f579a50c7ad423d4dd4eb2d1f117 (diff) | |
| download | go-e14fee553a4646d15123caa04f7ca6ddb7b48362.tar.xz | |
[dev.boringcrypto] all: merge master into dev.boringcrypto
Change-Id: I18dbf4f9fa7e2334fccedd862a523126cf38164e
Diffstat (limited to 'src/runtime')
196 files changed, 1895 insertions, 1176 deletions
diff --git a/src/runtime/abi_test.go b/src/runtime/abi_test.go index 5c1f1f4067..f9e8d701ce 100644 --- a/src/runtime/abi_test.go +++ b/src/runtime/abi_test.go @@ -78,7 +78,7 @@ func TestFinalizerRegisterABI(t *testing.T) { tests := []struct { name string - fin interface{} + fin any confirmValue int }{ {"Pointer", regFinalizerPointer, -1}, diff --git a/src/runtime/alg.go b/src/runtime/alg.go index 978a3b85dc..5d7d1c77f4 100644 --- a/src/runtime/alg.go +++ b/src/runtime/alg.go @@ -290,7 +290,7 @@ func int64Hash(i uint64, seed uintptr) uintptr { return memhash64(noescape(unsafe.Pointer(&i)), seed) } -func efaceHash(i interface{}, seed uintptr) uintptr { +func efaceHash(i any, seed uintptr) uintptr { return nilinterhash(noescape(unsafe.Pointer(&i)), seed) } diff --git a/src/runtime/asan.go b/src/runtime/asan.go index a22b56bb07..5f1e6370d2 100644 --- a/src/runtime/asan.go +++ b/src/runtime/asan.go @@ -11,23 +11,43 @@ import ( ) // Public address sanitizer API. - func ASanRead(addr unsafe.Pointer, len int) { - asanread(addr, uintptr(len)) + sp := getcallersp() + pc := getcallerpc() + doasanread(addr, uintptr(len), sp, pc) } func ASanWrite(addr unsafe.Pointer, len int) { - asanwrite(addr, uintptr(len)) + sp := getcallersp() + pc := getcallerpc() + doasanwrite(addr, uintptr(len), sp, pc) } // Private interface for the runtime. const asanenabled = true +// asan{read,write} are nosplit because they may be called between +// fork and exec, when the stack must not grow. See issue #50391. + +//go:nosplit +func asanread(addr unsafe.Pointer, sz uintptr) { + sp := getcallersp() + pc := getcallerpc() + doasanread(addr, sz, sp, pc) +} + +//go:nosplit +func asanwrite(addr unsafe.Pointer, sz uintptr) { + sp := getcallersp() + pc := getcallerpc() + doasanwrite(addr, sz, sp, pc) +} + //go:noescape -func asanread(addr unsafe.Pointer, sz uintptr) +func doasanread(addr unsafe.Pointer, sz, sp, pc uintptr) //go:noescape -func asanwrite(addr unsafe.Pointer, sz uintptr) +func doasanwrite(addr unsafe.Pointer, sz, sp, pc uintptr) //go:noescape func asanunpoison(addr unsafe.Pointer, sz uintptr) diff --git a/src/runtime/asan/asan.go b/src/runtime/asan/asan.go index eb66b3aab5..bab2362c51 100644 --- a/src/runtime/asan/asan.go +++ b/src/runtime/asan/asan.go @@ -14,38 +14,15 @@ package asan #include <stdint.h> #include <sanitizer/asan_interface.h> -extern void __asan_report_load1(void*); -extern void __asan_report_load2(void*); -extern void __asan_report_load4(void*); -extern void __asan_report_load8(void*); -extern void __asan_report_load_n(void*, uintptr_t); -extern void __asan_report_store1(void*); -extern void __asan_report_store2(void*); -extern void __asan_report_store4(void*); -extern void __asan_report_store8(void*); -extern void __asan_report_store_n(void*, uintptr_t); - -void __asan_read_go(void *addr, uintptr_t sz) { +void __asan_read_go(void *addr, uintptr_t sz, void *sp, void *pc) { if (__asan_region_is_poisoned(addr, sz)) { - switch (sz) { - case 1: __asan_report_load1(addr); break; - case 2: __asan_report_load2(addr); break; - case 4: __asan_report_load4(addr); break; - case 8: __asan_report_load8(addr); break; - default: __asan_report_load_n(addr, sz); break; - } + __asan_report_error(pc, 0, sp, addr, false, sz); } } -void __asan_write_go(void *addr, uintptr_t sz) { +void __asan_write_go(void *addr, uintptr_t sz, void *sp, void *pc) { if (__asan_region_is_poisoned(addr, sz)) { - switch (sz) { - case 1: __asan_report_store1(addr); break; - case 2: __asan_report_store2(addr); break; - case 4: __asan_report_store4(addr); break; - case 8: __asan_report_store8(addr); break; - default: __asan_report_store_n(addr, sz); break; - } + __asan_report_error(pc, 0, sp, addr, true, sz); } } diff --git a/src/runtime/asan_amd64.s b/src/runtime/asan_amd64.s index 01bd612dc3..3857350020 100644 --- a/src/runtime/asan_amd64.s +++ b/src/runtime/asan_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build asan -// +build asan #include "go_asm.h" #include "go_tls.h" @@ -16,25 +15,33 @@ #ifdef GOOS_windows #define RARG0 CX #define RARG1 DX +#define RARG2 R8 +#define RARG3 R9 #else #define RARG0 DI #define RARG1 SI +#define RARG2 DX +#define RARG3 CX #endif // Called from intrumented code. -// func runtime·asanread(addr unsafe.Pointer, sz uintptr) -TEXT runtime·asanread(SB), NOSPLIT, $0-16 +// func runtime·doasanread(addr unsafe.Pointer, sz, sp, pc uintptr) +TEXT runtime·doasanread(SB), NOSPLIT, $0-32 MOVQ addr+0(FP), RARG0 MOVQ size+8(FP), RARG1 - // void __asan_read_go(void *addr, uintptr_t sz); + MOVQ sp+16(FP), RARG2 + MOVQ pc+24(FP), RARG3 + // void __asan_read_go(void *addr, uintptr_t sz, void *sp, void *pc); MOVQ $__asan_read_go(SB), AX JMP asancall<>(SB) -// func runtime·asanwrite(addr unsafe.Pointer, sz uintptr) -TEXT runtime·asanwrite(SB), NOSPLIT, $0-16 +// func runtime·doasanwrite(addr unsafe.Pointer, sz, sp, pc uintptr) +TEXT runtime·doasanwrite(SB), NOSPLIT, $0-32 MOVQ addr+0(FP), RARG0 MOVQ size+8(FP), RARG1 - // void __asan_write_go(void *addr, uintptr_t sz); + MOVQ sp+16(FP), RARG2 + MOVQ pc+24(FP), RARG3 + // void __asan_write_go(void *addr, uintptr_t sz, void *sp, void *pc); MOVQ $__asan_write_go(SB), AX JMP asancall<>(SB) diff --git a/src/runtime/asan_arm64.s b/src/runtime/asan_arm64.s index eb0f9bd71e..5ed03c932b 100644 --- a/src/runtime/asan_arm64.s +++ b/src/runtime/asan_arm64.s @@ -3,29 +3,34 @@ // license that can be found in the LICENSE file. //go:build asan -// +build asan #include "go_asm.h" #include "textflag.h" #define RARG0 R0 #define RARG1 R1 -#define FARG R3 +#define RARG2 R2 +#define RARG3 R3 +#define FARG R4 // Called from instrumented code. -// func runtime·asanread(addr unsafe.Pointer, sz uintptr) -TEXT runtime·asanread(SB), NOSPLIT, $0-16 +// func runtime·doasanread(addr unsafe.Pointer, sz, sp, pc uintptr) +TEXT runtime·doasanread(SB), NOSPLIT, $0-32 MOVD addr+0(FP), RARG0 MOVD size+8(FP), RARG1 - // void __asan_read_go(void *addr, uintptr_t sz); + MOVD sp+16(FP), RARG2 + MOVD pc+24(FP), RARG3 + // void __asan_read_go(void *addr, uintptr_t sz, void *sp, void *pc); MOVD $__asan_read_go(SB), FARG JMP asancall<>(SB) -// func runtime·asanwrite(addr unsafe.Pointer, sz uintptr) -TEXT runtime·asanwrite(SB), NOSPLIT, $0-16 +// func runtime·doasanwrite(addr unsafe.Pointer, sz, sp, pc uintptr) +TEXT runtime·doasanwrite(SB), NOSPLIT, $0-32 MOVD addr+0(FP), RARG0 MOVD size+8(FP), RARG1 - // void __asan_write_go(void *addr, uintptr_t sz); + MOVD sp+16(FP), RARG2 + MOVD pc+24(FP), RARG3 + // void __asan_write_go(void *addr, uintptr_t sz, void *sp, void *pc); MOVD $__asan_write_go(SB), FARG JMP asancall<>(SB) @@ -54,8 +59,8 @@ TEXT asancall<>(SB), NOSPLIT, $0-0 CMP R11, g BEQ g0stack - MOVD (g_sched+gobuf_sp)(R11), R4 - MOVD R4, RSP + MOVD (g_sched+gobuf_sp)(R11), R5 + MOVD R5, RSP g0stack: BL (FARG) diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 0f0e5be21a..c08ae610fb 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -145,28 +145,14 @@ GLOBL bad_cpu_msg<>(SB), RODATA, $84 #endif -#ifdef GOAMD64_v1 -#define SKIP_GOAMD64_CHECK -#endif - -#ifndef GOAMD64_v1 -#ifndef GOAMD64_v2 -#ifndef GOAMD64_v3 -#ifndef GOAMD64_v4 -#define SKIP_GOAMD64_CHECK -#endif -#endif -#endif -#endif - TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 // copy arguments forward on an even stack MOVQ DI, AX // argc MOVQ SI, BX // argv - SUBQ $(4*8+7), SP // 2args 2auto + SUBQ $(5*8), SP // 3args 2auto ANDQ $~15, SP - MOVQ AX, 16(SP) - MOVQ BX, 24(SP) + MOVQ AX, 24(SP) + MOVQ BX, 32(SP) // create istack out of the given (operating system) stack. // _cgo_init may update stackguard. @@ -181,23 +167,8 @@ TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 MOVL $0, AX CPUID CMPL AX, $0 -#ifdef SKIP_GOAMD64_CHECK JE nocpuinfo -#else - JNE has_cpuinfo - -bad_cpu: // show that the program requires a certain microarchitecture level. - MOVQ $2, 0(SP) - MOVQ $bad_cpu_msg<>(SB), AX - MOVQ AX, 8(SP) - MOVQ $84, 16(SP) - CALL runtime·write(SB) - MOVQ $1, 0(SP) - CALL runtime·exit(SB) - CALL runtime·abort(SB) -#endif -has_cpuinfo: CMPL BX, $0x756E6547 // "Genu" JNE notintel CMPL DX, $0x49656E69 // "ineI" @@ -212,44 +183,6 @@ notintel: CPUID MOVL AX, runtime·processorVersionInfo(SB) -#ifdef NEED_FEATURES_CX - ANDL $NEED_FEATURES_CX, CX - CMPL CX, $NEED_FEATURES_CX - JNE bad_cpu -#endif - -#ifdef NEED_MAX_CPUID - MOVL $0x80000000, AX - CPUID - CMPL AX, $NEED_MAX_CPUID - JL bad_cpu -#endif - -#ifdef NEED_EXT_FEATURES_BX - MOVL $7, AX - MOVL $0, CX - CPUID - ANDL $NEED_EXT_FEATURES_BX, BX - CMPL BX, $NEED_EXT_FEATURES_BX - JNE bad_cpu -#endif - -#ifdef NEED_EXT_FEATURES_CX - MOVL $0x80000001, AX - CPUID - ANDL $NEED_EXT_FEATURES_CX, CX - CMPL CX, $NEED_EXT_FEATURES_CX - JNE bad_cpu -#endif - -#ifdef NEED_OS_SUPPORT_AX - XORL CX, CX - XGETBV - ANDL $NEED_OS_SUPPORT_AX, AX - CMPL AX, $NEED_OS_SUPPORT_AX - JNE bad_cpu -#endif - nocpuinfo: // if there is an _cgo_init, call it. MOVQ _cgo_init(SB), AX @@ -330,11 +263,59 @@ ok: MOVQ AX, g_m(CX) CLD // convention is D is always left cleared + + // Check GOAMD64 reqirements + // We need to do this after setting up TLS, so that + // we can report an error if there is a failure. See issue 49586. +#ifdef NEED_FEATURES_CX + MOVL $0, AX + CPUID + CMPL AX, $0 + JE bad_cpu + MOVL $1, AX + CPUID + ANDL $NEED_FEATURES_CX, CX + CMPL CX, $NEED_FEATURES_CX + JNE bad_cpu +#endif + +#ifdef NEED_MAX_CPUID + MOVL $0x80000000, AX + CPUID + CMPL AX, $NEED_MAX_CPUID + JL bad_cpu +#endif + +#ifdef NEED_EXT_FEATURES_BX + MOVL $7, AX + MOVL $0, CX + CPUID + ANDL $NEED_EXT_FEATURES_BX, BX + CMPL BX, $NEED_EXT_FEATURES_BX + JNE bad_cpu +#endif + +#ifdef NEED_EXT_FEATURES_CX + MOVL $0x80000001, AX + CPUID + ANDL $NEED_EXT_FEATURES_CX, CX + CMPL CX, $NEED_EXT_FEATURES_CX + JNE bad_cpu +#endif + +#ifdef NEED_OS_SUPPORT_AX + XORL CX, CX + XGETBV + ANDL $NEED_OS_SUPPORT_AX, AX + CMPL AX, $NEED_OS_SUPPORT_AX + JNE bad_cpu +#endif + CALL runtime·check(SB) - MOVL 16(SP), AX // copy argc + MOVL 24(SP), AX // copy argc MOVL AX, 0(SP) - MOVQ 24(SP), AX // copy argv + MOVQ 32(SP), AX // copy argv MOVQ AX, 8(SP) CALL runtime·args(SB) CALL runtime·osinit(SB) @@ -352,6 +333,17 @@ ok: CALL runtime·abort(SB) // mstart should never return RET +bad_cpu: // show that the program requires a certain microarchitecture level. + MOVQ $2, 0(SP) + MOVQ $bad_cpu_msg<>(SB), AX + MOVQ AX, 8(SP) + MOVQ $84, 16(SP) + CALL runtime·write(SB) + MOVQ $1, 0(SP) + CALL runtime·exit(SB) + CALL runtime·abort(SB) + RET + // Prevent dead-code elimination of debugCallV2, which is // intended to be called by debuggers. MOVQ $runtime·debugCallV2<ABIInternal>(SB), AX diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index e0e5cbb704..3597ebec57 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips64 || mips64le -// +build mips64 mips64le #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index 1b550719d1..4a086b8eb3 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips || mipsle -// +build mips mipsle #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 0e7ef7b2b8..ae14213999 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index 0e813189d4..2a4837b399 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -81,7 +81,10 @@ TEXT setg_gcc<>(SB),NOSPLIT,$0-0 // func cputicks() int64 TEXT runtime·cputicks(SB),NOSPLIT,$0-8 - RDCYCLE A0 + // RDTIME to emulate cpu ticks + // RDCYCLE reads counter that is per HART(core) based + // according to the riscv manual, see issue 46737 + RDTIME A0 MOV A0, ret+0(FP) RET diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index 5894fe5783..9159a67372 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -777,13 +777,12 @@ TEXT ·checkASM(SB),NOSPLIT,$0-1 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: // - R2 is the destination of the write // - R3 is the value being written at R2. -// It clobbers R10 (the temp register). +// It clobbers R10 (the temp register) and R1 (used by PLT stub). // It does not clobber any other general-purpose registers, // but may clobber others (e.g., floating point registers). -TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$104 +TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$96 // Save the registers clobbered by the fast path. - MOVD R1, 96(R15) - MOVD R4, 104(R15) + MOVD R4, 96(R15) MOVD g_m(g), R1 MOVD m_p(R1), R1 // Increment wbBuf.next position. @@ -798,8 +797,7 @@ TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$104 // Is the buffer full? CMPBEQ R4, R1, flush ret: - MOVD 96(R15), R1 - MOVD 104(R15), R4 + MOVD 96(R15), R4 // Do the write. MOVD R3, (R2) RET diff --git a/src/runtime/atomic_mips64x.s b/src/runtime/atomic_mips64x.s index e2118e6a20..dd6380ce40 100644 --- a/src/runtime/atomic_mips64x.s +++ b/src/runtime/atomic_mips64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips64 || mips64le -// +build mips64 mips64le #include "textflag.h" diff --git a/src/runtime/atomic_mipsx.s b/src/runtime/atomic_mipsx.s index 1eacd273b4..ac255fe7e6 100644 --- a/src/runtime/atomic_mipsx.s +++ b/src/runtime/atomic_mipsx.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips || mipsle -// +build mips mipsle #include "textflag.h" diff --git a/src/runtime/atomic_ppc64x.s b/src/runtime/atomic_ppc64x.s index b63de2dbd3..4742b6cf56 100644 --- a/src/runtime/atomic_ppc64x.s +++ b/src/runtime/atomic_ppc64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "textflag.h" diff --git a/src/runtime/cgo.go b/src/runtime/cgo.go index 395d54a66e..d90468240d 100644 --- a/src/runtime/cgo.go +++ b/src/runtime/cgo.go @@ -42,7 +42,7 @@ var cgoHasExtraM bool // 2) they keep the argument alive until the call site; the call is emitted after // the end of the (presumed) use of the argument by C. // cgoUse should not actually be called (see cgoAlwaysFalse). -func cgoUse(interface{}) { throw("cgoUse should not be called") } +func cgoUse(any) { throw("cgoUse should not be called") } // cgoAlwaysFalse is a boolean value that is always false. // The cgo-generated code says if cgoAlwaysFalse { cgoUse(p) }. diff --git a/src/runtime/cgo/gcc_amd64.S b/src/runtime/cgo/gcc_amd64.S index d75f864666..46699d1d9c 100644 --- a/src/runtime/cgo/gcc_amd64.S +++ b/src/runtime/cgo/gcc_amd64.S @@ -12,7 +12,7 @@ #endif /* - * void crosscall_amd64(void (*fn)(void)) + * void crosscall_amd64(void (*fn)(void), void (*setg_gcc)(void*), void *g) * * Calling into the 6c tool chain, where all registers are caller save. * Called from standard x86-64 ABI, where %rbx, %rbp, %r12-%r15 diff --git a/src/runtime/cgo/handle.go b/src/runtime/cgo/handle.go index 720acca802..d711900d79 100644 --- a/src/runtime/cgo/handle.go +++ b/src/runtime/cgo/handle.go @@ -59,6 +59,41 @@ import ( // void myprint(uintptr_t handle) { // MyGoPrint(handle); // } +// +// Some C functions accept a void* argument that points to an arbitrary +// data value supplied by the caller. It is not safe to coerce a cgo.Handle +// (an integer) to a Go unsafe.Pointer, but instead we can pass the address +// of the cgo.Handle to the void* parameter, as in this variant of the +// previous example: +// +// package main +// +// /* +// extern void MyGoPrint(void *context); +// static inline void myprint(void *context) { +// MyGoPrint(context); +// } +// */ +// import "C" +// import ( +// "runtime/cgo" +// "unsafe" +// ) +// +// //export MyGoPrint +// func MyGoPrint(context unsafe.Pointer) { +// h := *(*cgo.Handle)(context) +// val := h.Value().(string) +// println(val) +// h.Delete() +// } +// +// func main() { +// val := "hello Go" +// h := cgo.NewHandle(val) +// C.myprint(unsafe.Pointer(&h)) +// // Output: hello Go +// } type Handle uintptr // NewHandle returns a handle for a given value. @@ -70,7 +105,7 @@ type Handle uintptr // // The intended use is to pass the returned handle to C code, which // passes it back to Go, which calls Value. -func NewHandle(v interface{}) Handle { +func NewHandle(v any) Handle { h := atomic.AddUintptr(&handleIdx, 1) if h == 0 { panic("runtime/cgo: ran out of handle space") @@ -83,7 +118,7 @@ func NewHandle(v interface{}) Handle { // Value returns the associated Go value for a valid handle. // // The method panics if the handle is invalid. -func (h Handle) Value() interface{} { +func (h Handle) Value() any { v, ok := handles.Load(uintptr(h)) if !ok { panic("runtime/cgo: misuse of an invalid Handle") diff --git a/src/runtime/cgo/handle_test.go b/src/runtime/cgo/handle_test.go index 738051a0ea..b341c8e1e4 100644 --- a/src/runtime/cgo/handle_test.go +++ b/src/runtime/cgo/handle_test.go @@ -13,8 +13,8 @@ func TestHandle(t *testing.T) { v := 42 tests := []struct { - v1 interface{} - v2 interface{} + v1 any + v2 any }{ {v1: v, v2: v}, {v1: &v, v2: &v}, @@ -44,7 +44,7 @@ func TestHandle(t *testing.T) { } siz := 0 - handles.Range(func(k, v interface{}) bool { + handles.Range(func(k, v any) bool { siz++ return true }) diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 694b3e66cd..a0c9560fd0 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -389,7 +389,7 @@ var racecgosync uint64 // represents possible synchronization in C code // cgoCheckPointer checks if the argument contains a Go pointer that // points to a Go pointer, and panics if it does. -func cgoCheckPointer(ptr interface{}, arg interface{}) { +func cgoCheckPointer(ptr any, arg any) { if debug.cgocheck == 0 { return } @@ -628,7 +628,7 @@ func cgoInRange(p unsafe.Pointer, start, end uintptr) bool { // cgoCheckResult is called to check the result parameter of an // exported Go function. It panics if the result is or contains a Go // pointer. -func cgoCheckResult(val interface{}) { +func cgoCheckResult(val any) { if debug.cgocheck == 0 { return } diff --git a/src/runtime/chan_test.go b/src/runtime/chan_test.go index 355267c5e3..9471d4596c 100644 --- a/src/runtime/chan_test.go +++ b/src/runtime/chan_test.go @@ -494,7 +494,7 @@ func TestSelectFairness(t *testing.T) { func TestChanSendInterface(t *testing.T) { type mt struct{} m := &mt{} - c := make(chan interface{}, 1) + c := make(chan any, 1) c <- m select { case c <- m: diff --git a/src/runtime/checkptr_test.go b/src/runtime/checkptr_test.go index b3aea079c6..15011ec494 100644 --- a/src/runtime/checkptr_test.go +++ b/src/runtime/checkptr_test.go @@ -12,6 +12,12 @@ import ( ) func TestCheckPtr(t *testing.T) { + // This test requires rebuilding packages with -d=checkptr=1, + // so it's somewhat slow. + if testing.Short() { + t.Skip("skipping test in -short mode") + } + t.Parallel() testenv.MustHaveGoRun(t) @@ -57,6 +63,12 @@ func TestCheckPtr(t *testing.T) { } func TestCheckPtr2(t *testing.T) { + // This test requires rebuilding packages with -d=checkptr=2, + // so it's somewhat slow. + if testing.Short() { + t.Skip("skipping test in -short mode") + } + t.Parallel() testenv.MustHaveGoRun(t) diff --git a/src/runtime/cpuprof.go b/src/runtime/cpuprof.go index c81ab710c2..48cef46fe9 100644 --- a/src/runtime/cpuprof.go +++ b/src/runtime/cpuprof.go @@ -89,7 +89,7 @@ func SetCPUProfileRate(hz int) { // held at the time of the signal, nor can it use substantial amounts // of stack. //go:nowritebarrierrec -func (p *cpuProfile) add(gp *g, stk []uintptr) { +func (p *cpuProfile) add(tagPtr *unsafe.Pointer, stk []uintptr) { // Simple cas-lock to coordinate with setcpuprofilerate. for !atomic.Cas(&prof.signalLock, 0, 1) { osyield() @@ -104,15 +104,6 @@ func (p *cpuProfile) add(gp *g, stk []uintptr) { // because otherwise its write barrier behavior may not // be correct. See the long comment there before // changing the argument here. - // - // Note: it can happen on Windows, where we are calling - // p.add with a gp that is not the current g, that gp is nil, - // meaning we interrupted a system thread with no g. - // Avoid faulting in that case. - var tagPtr *unsafe.Pointer - if gp != nil { - tagPtr = &gp.labels - } cpuprof.log.write(tagPtr, nanotime(), hdr[:], stk) } @@ -209,6 +200,8 @@ func runtime_pprof_runtime_cyclesPerSecond() int64 { // If profiling is turned off and all the profile data accumulated while it was // on has been returned, readProfile returns eof=true. // The caller must save the returned data and tags before calling readProfile again. +// The returned data contains a whole number of records, and tags contains +// exactly one entry per record. // //go:linkname runtime_pprof_readProfile runtime/pprof.readProfile func runtime_pprof_readProfile() ([]uint64, []unsafe.Pointer, bool) { diff --git a/src/runtime/crash_cgo_test.go b/src/runtime/crash_cgo_test.go index e6d1742a38..9444554d37 100644 --- a/src/runtime/crash_cgo_test.go +++ b/src/runtime/crash_cgo_test.go @@ -7,7 +7,6 @@ package runtime_test import ( - "bytes" "fmt" "internal/testenv" "os" @@ -64,6 +63,10 @@ func TestCgoCallbackGC(t *testing.T) { t.Skip("too slow for mips64x builders") } } + if testenv.Builder() == "darwin-amd64-10_14" { + // TODO(#23011): When the 10.14 builders are gone, remove this skip. + t.Skip("skipping due to platform bug on macOS 10.14; see https://golang.org/issue/43926") + } got := runTestProg(t, "testprogcgo", "CgoCallbackGC") want := "OK\n" if got != want { @@ -90,23 +93,9 @@ func TestCgoExternalThreadSIGPROF(t *testing.T) { case "plan9", "windows": t.Skipf("no pthreads on %s", runtime.GOOS) } - if runtime.GOARCH == "ppc64" && runtime.GOOS == "linux" { - // TODO(austin) External linking not implemented on - // linux/ppc64 (issue #8912) - t.Skipf("no external linking on ppc64") - } - exe, err := buildTestProg(t, "testprogcgo", "-tags=threadprof") - if err != nil { - t.Fatal(err) - } - - got, err := testenv.CleanCmdEnv(exec.Command(exe, "CgoExternalThreadSIGPROF")).CombinedOutput() - if err != nil { - t.Fatalf("exit status: %v\n%s", err, got) - } - - if want := "OK\n"; string(got) != want { + got := runTestProg(t, "testprogcgo", "CgoExternalThreadSIGPROF", "GO_START_SIGPROF_THREAD=1") + if want := "OK\n"; got != want { t.Fatalf("expected %q, but got:\n%s", want, got) } } @@ -119,18 +108,8 @@ func TestCgoExternalThreadSignal(t *testing.T) { t.Skipf("no pthreads on %s", runtime.GOOS) } - exe, err := buildTestProg(t, "testprogcgo", "-tags=threadprof") - if err != nil { - t.Fatal(err) - } - - got, err := testenv.CleanCmdEnv(exec.Command(exe, "CgoExternalThreadSIGPROF")).CombinedOutput() - if err != nil { - t.Fatalf("exit status: %v\n%s", err, got) - } - - want := []byte("OK\n") - if !bytes.Equal(got, want) { + got := runTestProg(t, "testprogcgo", "CgoExternalThreadSignal") + if want := "OK\n"; got != want { t.Fatalf("expected %q, but got:\n%s", want, got) } } @@ -302,12 +281,7 @@ func testCgoPprof(t *testing.T, buildArg, runArg, top, bottom string) { t.Fatal(err) } - // pprofCgoTraceback is called whenever CGO code is executing and a signal - // is received. Disable signal preemption to increase the likelihood at - // least one SIGPROF signal fired to capture a sample. See issue #37201. cmd := testenv.CleanCmdEnv(exec.Command(exe, runArg)) - cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1") - got, err := cmd.CombinedOutput() if err != nil { if testenv.Builder() == "linux-amd64-alpine" { @@ -320,7 +294,7 @@ func testCgoPprof(t *testing.T, buildArg, runArg, top, bottom string) { defer os.Remove(fn) for try := 0; try < 2; try++ { - cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-traces")) + cmd := testenv.CleanCmdEnv(exec.Command(testenv.GoToolPath(t), "tool", "pprof", "-tagignore=ignore", "-traces")) // Check that pprof works both with and without explicit executable on command line. if try == 0 { cmd.Args = append(cmd.Args, exe, fn) @@ -628,15 +602,25 @@ func TestSegv(t *testing.T) { // No runtime errors like "runtime: unknown pc". switch runtime.GOOS { case "darwin", "illumos", "solaris": - // TODO(golang.org/issue/49182): Skip, runtime - // throws while attempting to generate - // traceback. - default: - nowant := "runtime: " - if strings.Contains(got, nowant) { - t.Errorf("unexpectedly saw %q in output", nowant) + // Runtime sometimes throws when generating the traceback. + testenv.SkipFlaky(t, 49182) + case "linux": + if runtime.GOARCH == "386" { + // Runtime throws when generating a traceback from + // a VDSO call via asmcgocall. + testenv.SkipFlaky(t, 50504) + } + if testenv.Builder() == "linux-mips64le-mengzhuo" && strings.Contains(got, "runtime: unknown pc") { + // Runtime sometimes throw "unknown pc" when generating the traceback. + // Curiously, that doesn't seem to happen on the linux-mips64le-rtrk + // builder. + testenv.SkipFlaky(t, 50605) } } + nowant := "runtime: " + if strings.Contains(got, nowant) { + t.Errorf("unexpectedly saw %q in output", nowant) + } }) } } @@ -702,3 +686,11 @@ func TestNeedmDeadlock(t *testing.T) { t.Fatalf("want %s, got %s\n", want, output) } } + +func TestCgoTracebackGoroutineProfile(t *testing.T) { + output := runTestProg(t, "testprogcgo", "GoroutineProfile") + want := "OK\n" + if output != want { + t.Fatalf("want %s, got %s\n", want, output) + } +} diff --git a/src/runtime/crash_nonunix_test.go b/src/runtime/crash_nonunix_test.go deleted file mode 100644 index 73c1cd3101..0000000000 --- a/src/runtime/crash_nonunix_test.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build windows || plan9 || (js && wasm) - -package runtime_test - -import "os" - -// sigquit is the signal to send to kill a hanging testdata program. -// On Unix we send SIGQUIT, but on non-Unix we only have os.Kill. -var sigquit = os.Kill diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go index e0c0bac892..9b9ab4f3e1 100644 --- a/src/runtime/crash_test.go +++ b/src/runtime/crash_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "bytes" + "errors" "flag" "fmt" "internal/testenv" @@ -14,11 +15,9 @@ import ( "path/filepath" "regexp" "runtime" - "strconv" "strings" "sync" "testing" - "time" ) var toRemove []string @@ -34,12 +33,13 @@ func TestMain(m *testing.M) { var testprog struct { sync.Mutex dir string - target map[string]buildexe + target map[string]*buildexe } type buildexe struct { - exe string - err error + once sync.Once + exe string + err error } func runTestProg(t *testing.T, binary, name string, env ...string) string { @@ -69,52 +69,19 @@ func runBuiltTestProg(t *testing.T, exe, name string, env ...string) string { if testing.Short() { cmd.Env = append(cmd.Env, "RUNTIME_TEST_SHORT=1") } - var b bytes.Buffer - cmd.Stdout = &b - cmd.Stderr = &b - if err := cmd.Start(); err != nil { - t.Fatalf("starting %s %s: %v", exe, name, err) - } - - // If the process doesn't complete within 1 minute, - // assume it is hanging and kill it to get a stack trace. - p := cmd.Process - done := make(chan bool) - go func() { - scale := 1 - // This GOARCH/GOOS test is copied from cmd/dist/test.go. - // TODO(iant): Have cmd/dist update the environment variable. - if runtime.GOARCH == "arm" || runtime.GOOS == "windows" { - scale = 2 - } - if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" { - if sc, err := strconv.Atoi(s); err == nil { - scale = sc - } - } - - select { - case <-done: - case <-time.After(time.Duration(scale) * time.Minute): - p.Signal(sigquit) - } - }() - - if err := cmd.Wait(); err != nil { - t.Logf("%s %s exit status: %v", exe, name, err) - } - close(done) - - return b.String() + out, _ := testenv.RunWithTimeout(t, cmd) + return string(out) } +var serializeBuild = make(chan bool, 2) + func buildTestProg(t *testing.T, binary string, flags ...string) (string, error) { if *flagQuick { t.Skip("-quick") } + testenv.MustHaveGoBuild(t) testprog.Lock() - defer testprog.Unlock() if testprog.dir == "" { dir, err := os.MkdirTemp("", "go-build") if err != nil { @@ -125,29 +92,48 @@ func buildTestProg(t *testing.T, binary string, flags ...string) (string, error) } if testprog.target == nil { - testprog.target = make(map[string]buildexe) + testprog.target = make(map[string]*buildexe) } name := binary if len(flags) > 0 { name += "_" + strings.Join(flags, "_") } target, ok := testprog.target[name] - if ok { - return target.exe, target.err - } - - exe := filepath.Join(testprog.dir, name+".exe") - cmd := exec.Command(testenv.GoToolPath(t), append([]string{"build", "-o", exe}, flags...)...) - cmd.Dir = "testdata/" + binary - out, err := testenv.CleanCmdEnv(cmd).CombinedOutput() - if err != nil { - target.err = fmt.Errorf("building %s %v: %v\n%s", binary, flags, err, out) + if !ok { + target = &buildexe{} testprog.target[name] = target - return "", target.err } - target.exe = exe - testprog.target[name] = target - return exe, nil + + dir := testprog.dir + + // Unlock testprog while actually building, so that other + // tests can look up executables that were already built. + testprog.Unlock() + + target.once.Do(func() { + // Only do two "go build"'s at a time, + // to keep load from getting too high. + serializeBuild <- true + defer func() { <-serializeBuild }() + + // Don't get confused if testenv.GoToolPath calls t.Skip. + target.err = errors.New("building test called t.Skip") + + exe := filepath.Join(dir, name+".exe") + + t.Logf("running go build -o %s %s", exe, strings.Join(flags, " ")) + cmd := exec.Command(testenv.GoToolPath(t), append([]string{"build", "-o", exe}, flags...)...) + cmd.Dir = "testdata/" + binary + out, err := testenv.CleanCmdEnv(cmd).CombinedOutput() + if err != nil { + target.err = fmt.Errorf("building %s %v: %v\n%s", binary, flags, err, out) + } else { + target.exe = exe + target.err = nil + } + }) + + return target.exe, target.err } func TestVDSO(t *testing.T) { @@ -417,7 +403,7 @@ func TestRuntimePanicWithRuntimeError(t *testing.T) { } } -func panicValue(fn func()) (recovered interface{}) { +func panicValue(fn func()) (recovered any) { defer func() { recovered = recover() }() @@ -691,6 +677,13 @@ func TestBadTraceback(t *testing.T) { } func TestTimePprof(t *testing.T) { + // This test is unreliable on any system in which nanotime + // calls into libc. + switch runtime.GOOS { + case "aix", "darwin", "illumos", "openbsd", "solaris": + t.Skipf("skipping on %s because nanotime calls libc", runtime.GOOS) + } + // Pass GOTRACEBACK for issue #41120 to try to get more // information on timeout. fn := runTestProg(t, "testprog", "TimeProf", "GOTRACEBACK=crash") diff --git a/src/runtime/crash_unix_test.go b/src/runtime/crash_unix_test.go index 0d9d22aa49..1eb10f9b60 100644 --- a/src/runtime/crash_unix_test.go +++ b/src/runtime/crash_unix_test.go @@ -13,7 +13,7 @@ import ( "os" "os/exec" "runtime" - "strings" + "runtime/debug" "sync" "syscall" "testing" @@ -21,16 +21,12 @@ import ( "unsafe" ) -// sigquit is the signal to send to kill a hanging testdata program. -// Send SIGQUIT to get a stack trace. -var sigquit = syscall.SIGQUIT - func init() { if runtime.Sigisblocked(int(syscall.SIGQUIT)) { // We can't use SIGQUIT to kill subprocesses because // it's blocked. Use SIGKILL instead. See issue // #19196 for an example of when this happens. - sigquit = syscall.SIGKILL + testenv.Sigquit = syscall.SIGKILL } } @@ -211,6 +207,11 @@ func TestPanicSystemstack(t *testing.T) { func init() { if len(os.Args) >= 2 && os.Args[1] == "testPanicSystemstackInternal" { + // Complete any in-flight GCs and disable future ones. We're going to + // block goroutines on runtime locks, which aren't ever preemptible for the + // GC to scan them. + runtime.GC() + debug.SetGCPercent(-1) // Get two threads running on the system stack with // something recognizable in the stack trace. runtime.GOMAXPROCS(2) @@ -244,9 +245,7 @@ func TestSignalExitStatus(t *testing.T) { func TestSignalIgnoreSIGTRAP(t *testing.T) { if runtime.GOOS == "openbsd" { - if bn := testenv.Builder(); strings.HasSuffix(bn, "-62") || strings.HasSuffix(bn, "-64") { - testenv.SkipFlaky(t, 17496) - } + testenv.SkipFlaky(t, 49725) } output := runTestProg(t, "testprognet", "SignalIgnoreSIGTRAP") diff --git a/src/runtime/debug/garbage_test.go b/src/runtime/debug/garbage_test.go index 69e769ecf2..7213bbe641 100644 --- a/src/runtime/debug/garbage_test.go +++ b/src/runtime/debug/garbage_test.go @@ -6,6 +6,7 @@ package debug_test import ( "internal/testenv" + "os" "runtime" . "runtime/debug" "testing" @@ -87,27 +88,71 @@ func TestReadGCStats(t *testing.T) { } } -var big = make([]byte, 1<<20) +var big []byte func TestFreeOSMemory(t *testing.T) { - var ms1, ms2 runtime.MemStats + // Tests FreeOSMemory by making big susceptible to collection + // and checking that at least that much memory is returned to + // the OS after. - if big == nil { - t.Skip("test is not reliable when run multiple times") - } - big = nil + const bigBytes = 32 << 20 + big = make([]byte, bigBytes) + + // Make sure any in-progress GCs are complete. runtime.GC() - runtime.ReadMemStats(&ms1) + + var before runtime.MemStats + runtime.ReadMemStats(&before) + + // Clear the last reference to the big allocation, making it + // susceptible to collection. + big = nil + + // FreeOSMemory runs a GC cycle before releasing memory, + // so it's fine to skip a GC here. + // + // It's possible the background scavenger runs concurrently + // with this function and does most of the work for it. + // If that happens, it's OK. What we want is a test that fails + // often if FreeOSMemory does not work correctly, and a test + // that passes every time if it does. FreeOSMemory() - runtime.ReadMemStats(&ms2) - if ms1.HeapReleased >= ms2.HeapReleased { - t.Errorf("released before=%d; released after=%d; did not go up", ms1.HeapReleased, ms2.HeapReleased) + + var after runtime.MemStats + runtime.ReadMemStats(&after) + + // Check to make sure that the big allocation (now freed) + // had its memory shift into HeapReleased as a result of that + // FreeOSMemory. + if after.HeapReleased <= before.HeapReleased { + t.Fatalf("no memory released: %d -> %d", before.HeapReleased, after.HeapReleased) + } + + // Check to make sure bigBytes was released, plus some slack. Pages may get + // allocated in between the two measurements above for a variety for reasons, + // most commonly for GC work bufs. Since this can get fairly high, depending + // on scheduling and what GOMAXPROCS is, give a lot of slack up-front. + // + // Add a little more slack too if the page size is bigger than the runtime page size. + // "big" could end up unaligned on its ends, forcing the scavenger to skip at worst + // 2x pages. + slack := uint64(bigBytes / 2) + pageSize := uint64(os.Getpagesize()) + if pageSize > 8<<10 { + slack += pageSize * 2 + } + if slack > bigBytes { + // We basically already checked this. + return + } + if after.HeapReleased-before.HeapReleased < bigBytes-slack { + t.Fatalf("less than %d released: %d -> %d", bigBytes, before.HeapReleased, after.HeapReleased) } } var ( - setGCPercentBallast interface{} - setGCPercentSink interface{} + setGCPercentBallast any + setGCPercentSink any ) func TestSetGCPercent(t *testing.T) { diff --git a/src/runtime/debug/mod.go b/src/runtime/debug/mod.go index 14b99f5735..14a496a8eb 100644 --- a/src/runtime/debug/mod.go +++ b/src/runtime/debug/mod.go @@ -57,8 +57,9 @@ type Module struct { // BuildSetting describes a setting that may be used to understand how the // binary was built. For example, VCS commit and dirty status is stored here. type BuildSetting struct { - // Key and Value describe the build setting. They must not contain tabs - // or newlines. + // Key and Value describe the build setting. + // Key must not contain an equals sign, space, tab, or newline. + // Value must not contain newlines ('\n'). Key, Value string } @@ -97,10 +98,13 @@ func (bi *BuildInfo) MarshalText() ([]byte, error) { formatMod("dep", *dep) } for _, s := range bi.Settings { - if strings.ContainsAny(s.Key, "\n\t") || strings.ContainsAny(s.Value, "\n\t") { - return nil, fmt.Errorf("build setting %q contains tab or newline", s.Key) + if strings.ContainsAny(s.Key, "= \t\n") { + return nil, fmt.Errorf("invalid build setting key %q", s.Key) } - fmt.Fprintf(buf, "build\t%s\t%s\n", s.Key, s.Value) + if strings.Contains(s.Value, "\n") { + return nil, fmt.Errorf("invalid build setting value for key %q: contains newline", s.Value) + } + fmt.Fprintf(buf, "build\t%s=%s\n", s.Key, s.Value) } return buf.Bytes(), nil @@ -185,14 +189,14 @@ func (bi *BuildInfo) UnmarshalText(data []byte) (err error) { } last = nil case bytes.HasPrefix(line, buildLine): - elem := bytes.Split(line[len(buildLine):], tab) - if len(elem) != 2 { - return fmt.Errorf("expected 2 columns for build setting; got %d", len(elem)) + key, val, ok := strings.Cut(string(line[len(buildLine):]), "=") + if !ok { + return fmt.Errorf("invalid build line") } - if len(elem[0]) == 0 { + if key == "" { return fmt.Errorf("empty key") } - bi.Settings = append(bi.Settings, BuildSetting{Key: string(elem[0]), Value: string(elem[1])}) + bi.Settings = append(bi.Settings, BuildSetting{Key: key, Value: val}) } lineNum++ } diff --git a/src/runtime/debug_test.go b/src/runtime/debug_test.go index b5db7a55f1..5bb0c5cee3 100644 --- a/src/runtime/debug_test.go +++ b/src/runtime/debug_test.go @@ -34,12 +34,22 @@ func startDebugCallWorker(t *testing.T) (g *runtime.G, after func()) { skipUnderDebugger(t) // This can deadlock if there aren't enough threads or if a GC - // tries to interrupt an atomic loop (see issue #10958). We - // use 8 Ps so there's room for the debug call worker, + // tries to interrupt an atomic loop (see issue #10958). Execute + // an extra GC to ensure even the sweep phase is done (out of + // caution to prevent #49370 from happening). + // TODO(mknyszek): This extra GC cycle is likely unnecessary + // because preemption (which may happen during the sweep phase) + // isn't much of an issue anymore thanks to asynchronous preemption. + // The biggest risk is having a write barrier in the debug call + // injection test code fire, because it runs in a signal handler + // and may not have a P. + // + // We use 8 Ps so there's room for the debug call worker, // something that's trying to preempt the call worker, and the // goroutine that's trying to stop the call worker. ogomaxprocs := runtime.GOMAXPROCS(8) ogcpercent := debug.SetGCPercent(-1) + runtime.GC() // ready is a buffered channel so debugCallWorker won't block // on sending to it. This makes it less likely we'll catch @@ -143,6 +153,7 @@ func TestDebugCall(t *testing.T) { x1: 42.0, } } + if _, err := runtime.InjectDebugCall(g, fn, ®s, args, debugCallTKill, false); err != nil { t.Fatal(err) } @@ -233,6 +244,12 @@ func TestDebugCallUnsafePoint(t *testing.T) { // This can deadlock if there aren't enough threads or if a GC // tries to interrupt an atomic loop (see issue #10958). defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8)) + + // InjectDebugCall cannot be executed while a GC is actively in + // progress. Wait until the current GC is done, and turn it off. + // + // See #49370. + runtime.GC() defer debug.SetGCPercent(debug.SetGCPercent(-1)) // Test that the runtime refuses call injection at unsafe points. @@ -256,6 +273,19 @@ func TestDebugCallPanic(t *testing.T) { // This can deadlock if there aren't enough threads. defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8)) + // InjectDebugCall cannot be executed while a GC is actively in + // progress. Wait until the current GC is done, and turn it off. + // + // See #10958 and #49370. + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + // TODO(mknyszek): This extra GC cycle is likely unnecessary + // because preemption (which may happen during the sweep phase) + // isn't much of an issue anymore thanks to asynchronous preemption. + // The biggest risk is having a write barrier in the debug call + // injection test code fire, because it runs in a signal handler + // and may not have a P. + runtime.GC() + ready := make(chan *runtime.G) var stop uint32 defer atomic.StoreUint32(&stop, 1) diff --git a/src/runtime/debugcall.go b/src/runtime/debugcall.go index 005a259f28..205971c428 100644 --- a/src/runtime/debugcall.go +++ b/src/runtime/debugcall.go @@ -16,7 +16,7 @@ const ( ) func debugCallV2() -func debugCallPanicked(val interface{}) +func debugCallPanicked(val any) // debugCallCheck checks whether it is safe to inject a debugger // function call with return PC pc. If not, it returns a string diff --git a/src/runtime/debuglog.go b/src/runtime/debuglog.go index 588b54d1f5..75b91c4216 100644 --- a/src/runtime/debuglog.go +++ b/src/runtime/debuglog.go @@ -266,7 +266,7 @@ func (l *dlogger) hex(x uint64) *dlogger { } //go:nosplit -func (l *dlogger) p(x interface{}) *dlogger { +func (l *dlogger) p(x any) *dlogger { if !dlogEnabled { return l } diff --git a/src/runtime/defer_test.go b/src/runtime/defer_test.go index 1d5745d60b..3a54951c31 100644 --- a/src/runtime/defer_test.go +++ b/src/runtime/defer_test.go @@ -433,7 +433,7 @@ func TestIssue43921(t *testing.T) { }() } -func expect(t *testing.T, n int, err interface{}) { +func expect(t *testing.T, n int, err any) { if n != err { t.Fatalf("have %v, want %v", err, n) } @@ -467,7 +467,6 @@ func TestIssue43920(t *testing.T) { } func step(t *testing.T, steps *int, want int) { - println("step", want) *steps++ if *steps != want { t.Fatalf("have %v, want %v", *steps, want) diff --git a/src/runtime/defs_linux_riscv64.go b/src/runtime/defs_linux_riscv64.go index 332720a8c8..747e26bc4b 100644 --- a/src/runtime/defs_linux_riscv64.go +++ b/src/runtime/defs_linux_riscv64.go @@ -122,10 +122,12 @@ func (tv *timeval) set_usec(x int32) { } type sigactiont struct { - sa_handler uintptr - sa_flags uint64 + sa_handler uintptr + sa_flags uint64 + sa_mask uint64 + // Linux on riscv64 does not have the sa_restorer field, but the setsig + // function references it (for x86). Not much harm to include it at the end. sa_restorer uintptr - sa_mask uint64 } type siginfoFields struct { diff --git a/src/runtime/duff_mips64x.s b/src/runtime/duff_mips64x.s index a897d7fd9b..3a8524c78b 100644 --- a/src/runtime/duff_mips64x.s +++ b/src/runtime/duff_mips64x.s @@ -3,7 +3,6 @@ // See mkduff.go for comments. //go:build mips64 || mips64le -// +build mips64 mips64le #include "textflag.h" diff --git a/src/runtime/duff_ppc64x.s b/src/runtime/duff_ppc64x.s index eeecf13df1..a3caaa8817 100644 --- a/src/runtime/duff_ppc64x.s +++ b/src/runtime/duff_ppc64x.s @@ -3,7 +3,6 @@ // See mkduff.go for comments. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "textflag.h" diff --git a/src/runtime/error.go b/src/runtime/error.go index 91f83ae126..43114f092e 100644 --- a/src/runtime/error.go +++ b/src/runtime/error.go @@ -210,7 +210,7 @@ type stringer interface { // printany prints an argument passed to panic. // If panic is called with a value that has a String or Error method, // it has already been converted into a string by preprintpanics. -func printany(i interface{}) { +func printany(i any) { switch v := i.(type) { case nil: print("nil") @@ -253,7 +253,7 @@ func printany(i interface{}) { } } -func printanycustomtype(i interface{}) { +func printanycustomtype(i any) { eface := efaceOf(&i) typestring := eface._type.string() diff --git a/src/runtime/export_debug_test.go b/src/runtime/export_debug_test.go index 032a9b9725..19a9ec135f 100644 --- a/src/runtime/export_debug_test.go +++ b/src/runtime/export_debug_test.go @@ -22,7 +22,7 @@ import ( // // On success, InjectDebugCall returns the panic value of fn or nil. // If fn did not panic, its results will be available in args. -func InjectDebugCall(gp *g, fn interface{}, regArgs *abi.RegArgs, stackArgs interface{}, tkill func(tid int) error, returnOnUnsafePoint bool) (interface{}, error) { +func InjectDebugCall(gp *g, fn any, regArgs *abi.RegArgs, stackArgs any, tkill func(tid int) error, returnOnUnsafePoint bool) (any, error) { if gp.lockedm == 0 { return nil, plainError("goroutine not locked to thread") } @@ -96,7 +96,7 @@ type debugCallHandler struct { regArgs *abi.RegArgs argp unsafe.Pointer argSize uintptr - panic interface{} + panic any handleF func(info *siginfo, ctxt *sigctxt, gp2 *g) bool @@ -107,6 +107,10 @@ type debugCallHandler struct { } func (h *debugCallHandler) inject(info *siginfo, ctxt *sigctxt, gp2 *g) bool { + // TODO(49370): This code is riddled with write barriers, but called from + // a signal handler. Add the go:nowritebarrierrec annotation and restructure + // this to avoid write barriers. + switch h.gp.atomicstatus { case _Grunning: if getg().m != h.mp { @@ -141,7 +145,11 @@ func (h *debugCallHandler) inject(info *siginfo, ctxt *sigctxt, gp2 *g) bool { } func (h *debugCallHandler) handle(info *siginfo, ctxt *sigctxt, gp2 *g) bool { - // Sanity check. + // TODO(49370): This code is riddled with write barriers, but called from + // a signal handler. Add the go:nowritebarrierrec annotation and restructure + // this to avoid write barriers. + + // Double-check m. if getg().m != h.mp { println("trap on wrong M", getg().m, h.mp) return false diff --git a/src/runtime/export_debuglog_test.go b/src/runtime/export_debuglog_test.go index 8cd943b438..1a9074e646 100644 --- a/src/runtime/export_debuglog_test.go +++ b/src/runtime/export_debuglog_test.go @@ -14,15 +14,15 @@ const DebugLogStringLimit = debugLogStringLimit var Dlog = dlog -func (l *dlogger) End() { l.end() } -func (l *dlogger) B(x bool) *dlogger { return l.b(x) } -func (l *dlogger) I(x int) *dlogger { return l.i(x) } -func (l *dlogger) I16(x int16) *dlogger { return l.i16(x) } -func (l *dlogger) U64(x uint64) *dlogger { return l.u64(x) } -func (l *dlogger) Hex(x uint64) *dlogger { return l.hex(x) } -func (l *dlogger) P(x interface{}) *dlogger { return l.p(x) } -func (l *dlogger) S(x string) *dlogger { return l.s(x) } -func (l *dlogger) PC(x uintptr) *dlogger { return l.pc(x) } +func (l *dlogger) End() { l.end() } +func (l *dlogger) B(x bool) *dlogger { return l.b(x) } +func (l *dlogger) I(x int) *dlogger { return l.i(x) } +func (l *dlogger) I16(x int16) *dlogger { return l.i16(x) } +func (l *dlogger) U64(x uint64) *dlogger { return l.u64(x) } +func (l *dlogger) Hex(x uint64) *dlogger { return l.hex(x) } +func (l *dlogger) P(x any) *dlogger { return l.p(x) } +func (l *dlogger) S(x string) *dlogger { return l.s(x) } +func (l *dlogger) PC(x uintptr) *dlogger { return l.pc(x) } func DumpDebugLog() string { g := getg() diff --git a/src/runtime/export_futex_test.go b/src/runtime/export_futex_test.go deleted file mode 100644 index 03157d8eed..0000000000 --- a/src/runtime/export_futex_test.go +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build dragonfly || freebsd || linux - -package runtime - -var Futexwakeup = futexwakeup - -//go:nosplit -func Futexsleep(addr *uint32, val uint32, ns int64) { - // Temporarily disable preemption so that a preemption signal - // doesn't interrupt the system call. - poff := debug.asyncpreemptoff - debug.asyncpreemptoff = 1 - futexsleep(addr, val, ns) - debug.asyncpreemptoff = poff -} diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index b2e64f14ad..0f21838721 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -75,7 +75,7 @@ func Netpoll(delta int64) { }) } -func GCMask(x interface{}) (ret []byte) { +func GCMask(x any) (ret []byte) { systemstack(func() { ret = getgcmask(x) }) @@ -218,7 +218,7 @@ func SetEnvs(e []string) { envs = e } // For benchmarking. -func BenchSetType(n int, x interface{}) { +func BenchSetType(n int, x any) { e := *efaceOf(&x) t := e._type var size uintptr @@ -546,7 +546,7 @@ func MapTombstoneCheck(m map[int]int) { // We should have a series of filled and emptyOne cells, followed by // a series of emptyRest cells. h := *(**hmap)(unsafe.Pointer(&m)) - i := interface{}(m) + i := any(m) t := *(**maptype)(unsafe.Pointer(&i)) for x := 0; x < 1<<h.B; x++ { @@ -1048,7 +1048,19 @@ func FreePageAlloc(pp *PageAlloc) { // // This should not be higher than 0x100*pallocChunkBytes to support // mips and mipsle, which only have 31-bit address spaces. -var BaseChunkIdx = ChunkIdx(chunkIndex(((0xc000*pageAlloc64Bit + 0x100*pageAlloc32Bit) * pallocChunkBytes) + arenaBaseOffset*goos.IsAix)) +var BaseChunkIdx = func() ChunkIdx { + var prefix uintptr + if pageAlloc64Bit != 0 { + prefix = 0xc000 + } else { + prefix = 0x100 + } + baseAddr := prefix * pallocChunkBytes + if goos.IsAix != 0 { + baseAddr += arenaBaseOffset + } + return ChunkIdx(chunkIndex(baseAddr)) +}() // PageBase returns an address given a chunk index and a page index // relative to that chunk. @@ -1299,11 +1311,22 @@ func (c *GCController) EndCycle(bytesMarked uint64, assistTime, elapsed int64, g c.commit(triggerRatio) } -var escapeSink interface{} +var escapeSink any //go:noinline -func escape(x interface{}) interface{} { +func escape(x any) any { escapeSink = x escapeSink = nil return x } + +// Acquirem blocks preemption. +func Acquirem() { + acquirem() +} + +func Releasem() { + releasem(getg().m) +} + +var Timediv = timediv diff --git a/src/runtime/futex_test.go b/src/runtime/futex_test.go deleted file mode 100644 index 188d0c6525..0000000000 --- a/src/runtime/futex_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Futex is only available on DragonFly BSD, FreeBSD and Linux. -// The race detector emits calls to split stack functions so it breaks -// the test. - -//go:build (dragonfly || freebsd || linux) && !race - -package runtime_test - -import ( - "runtime" - "sync" - "sync/atomic" - "testing" - "time" -) - -type futexsleepTest struct { - mtx uint32 - ns int64 - msg string - ch chan *futexsleepTest -} - -var futexsleepTests = []futexsleepTest{ - beforeY2038: {mtx: 0, ns: 86400 * 1e9, msg: "before the year 2038"}, - afterY2038: {mtx: 0, ns: (1<<31 + 100) * 1e9, msg: "after the year 2038"}, -} - -const ( - beforeY2038 = iota - afterY2038 -) - -func TestFutexsleep(t *testing.T) { - if runtime.GOMAXPROCS(0) > 1 { - // futexsleep doesn't handle EINTR or other signals, - // so spurious wakeups may happen. - t.Skip("skipping; GOMAXPROCS>1") - } - - start := time.Now() - var wg sync.WaitGroup - for i := range futexsleepTests { - tt := &futexsleepTests[i] - tt.mtx = 0 - tt.ch = make(chan *futexsleepTest, 1) - wg.Add(1) - go func(tt *futexsleepTest) { - runtime.Entersyscall() - runtime.Futexsleep(&tt.mtx, 0, tt.ns) - runtime.Exitsyscall() - tt.ch <- tt - wg.Done() - }(tt) - } -loop: - for { - select { - case tt := <-futexsleepTests[beforeY2038].ch: - t.Errorf("futexsleep test %q finished early after %s", tt.msg, time.Since(start)) - break loop - case tt := <-futexsleepTests[afterY2038].ch: - // Looks like FreeBSD 10 kernel has changed - // the semantics of timedwait on userspace - // mutex to make broken stuff look broken. - switch { - case runtime.GOOS == "freebsd" && runtime.GOARCH == "386": - t.Log("freebsd/386 may not work correctly after the year 2038, see golang.org/issue/7194") - default: - t.Errorf("futexsleep test %q finished early after %s", tt.msg, time.Since(start)) - break loop - } - case <-time.After(time.Second): - break loop - } - } - for i := range futexsleepTests { - tt := &futexsleepTests[i] - atomic.StoreUint32(&tt.mtx, 1) - runtime.Futexwakeup(&tt.mtx, 1) - } - wg.Wait() -} diff --git a/src/runtime/gc_test.go b/src/runtime/gc_test.go index 7b979afd55..9743dbbe2b 100644 --- a/src/runtime/gc_test.go +++ b/src/runtime/gc_test.go @@ -136,7 +136,7 @@ func TestGcLastTime(t *testing.T) { } } -var hugeSink interface{} +var hugeSink any func TestHugeGCInfo(t *testing.T) { // The test ensures that compiler can chew these huge types even on weakest machines. @@ -196,7 +196,10 @@ func TestPeriodicGC(t *testing.T) { func TestGcZombieReporting(t *testing.T) { // This test is somewhat sensitive to how the allocator works. - got := runTestProg(t, "testprog", "GCZombie") + // Pointers in zombies slice may cross-span, thus we + // add invalidptr=0 for avoiding the badPointer check. + // See issue https://golang.org/issues/49613/ + got := runTestProg(t, "testprog", "GCZombie", "GODEBUG=invalidptr=0") want := "found pointer to free object" if !strings.Contains(got, want) { t.Fatalf("expected %q in output, but got %q", want, got) @@ -454,7 +457,7 @@ func BenchmarkSetTypeNode1024Slice(b *testing.B) { benchSetType(b, make([]Node1024, 32)) } -func benchSetType(b *testing.B, x interface{}) { +func benchSetType(b *testing.B, x any) { v := reflect.ValueOf(x) t := v.Type() switch t.Kind() { @@ -520,7 +523,7 @@ func TestPrintGC(t *testing.T) { close(done) } -func testTypeSwitch(x interface{}) error { +func testTypeSwitch(x any) error { switch y := x.(type) { case nil: // ok @@ -530,14 +533,14 @@ func testTypeSwitch(x interface{}) error { return nil } -func testAssert(x interface{}) error { +func testAssert(x any) error { if y, ok := x.(error); ok { return y } return nil } -func testAssertVar(x interface{}) error { +func testAssertVar(x any) error { var y, ok = x.(error) if ok { return y @@ -548,7 +551,7 @@ func testAssertVar(x interface{}) error { var a bool //go:noinline -func testIfaceEqual(x interface{}) { +func testIfaceEqual(x any) { if x == "abc" { a = true } diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index 0808b416f0..f2c88ef1ab 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -66,7 +66,7 @@ func TestGCInfo(t *testing.T) { runtime.KeepAlive(x) } { - var x interface{} + var x any verifyGCInfo(t, "stack eface", &x, infoEface) runtime.KeepAlive(x) } @@ -84,12 +84,12 @@ func TestGCInfo(t *testing.T) { verifyGCInfo(t, "heap PtrScalar", escape(new(PtrScalar)), trimDead(infoPtrScalar)) verifyGCInfo(t, "heap BigStruct", escape(new(BigStruct)), trimDead(infoBigStruct())) verifyGCInfo(t, "heap string", escape(new(string)), trimDead(infoString)) - verifyGCInfo(t, "heap eface", escape(new(interface{})), trimDead(infoEface)) + verifyGCInfo(t, "heap eface", escape(new(any)), trimDead(infoEface)) verifyGCInfo(t, "heap iface", escape(new(Iface)), trimDead(infoIface)) } } -func verifyGCInfo(t *testing.T, name string, p interface{}, mask0 []byte) { +func verifyGCInfo(t *testing.T, name string, p any, mask0 []byte) { mask := runtime.GCMask(p) if !bytes.Equal(mask, mask0) { t.Errorf("bad GC program for %v:\nwant %+v\ngot %+v", name, mask0, mask) @@ -104,9 +104,9 @@ func trimDead(mask []byte) []byte { return mask } -var gcinfoSink interface{} +var gcinfoSink any -func escape(p interface{}) interface{} { +func escape(p any) any { gcinfoSink = p return p } @@ -194,18 +194,18 @@ var ( bssBigStruct BigStruct bssString string bssSlice []string - bssEface interface{} + bssEface any bssIface Iface // DATA - dataPtr = Ptr{new(byte)} - dataScalarPtr = ScalarPtr{q: 1} - dataPtrScalar = PtrScalar{w: 1} - dataBigStruct = BigStruct{w: 1} - dataString = "foo" - dataSlice = []string{"foo"} - dataEface interface{} = 42 - dataIface Iface = IfaceImpl(42) + dataPtr = Ptr{new(byte)} + dataScalarPtr = ScalarPtr{q: 1} + dataPtrScalar = PtrScalar{w: 1} + dataBigStruct = BigStruct{w: 1} + dataString = "foo" + dataSlice = []string{"foo"} + dataEface any = 42 + dataIface Iface = IfaceImpl(42) infoString = []byte{typePointer, typeScalar} infoSlice = []byte{typePointer, typeScalar, typeScalar} diff --git a/src/runtime/hash_test.go b/src/runtime/hash_test.go index 7048874a71..cf56c57a5f 100644 --- a/src/runtime/hash_test.go +++ b/src/runtime/hash_test.go @@ -382,7 +382,7 @@ func (k *Int64Key) name() string { } type EfaceKey struct { - i interface{} + i any } func (k *EfaceKey) clear() { diff --git a/src/runtime/iface.go b/src/runtime/iface.go index e2bec10948..a4d56dd33b 100644 --- a/src/runtime/iface.go +++ b/src/runtime/iface.go @@ -296,11 +296,11 @@ type ( ) var ( - uint16Eface interface{} = uint16InterfacePtr(0) - uint32Eface interface{} = uint32InterfacePtr(0) - uint64Eface interface{} = uint64InterfacePtr(0) - stringEface interface{} = stringInterfacePtr("") - sliceEface interface{} = sliceInterfacePtr(nil) + uint16Eface any = uint16InterfacePtr(0) + uint32Eface any = uint32InterfacePtr(0) + uint64Eface any = uint64InterfacePtr(0) + stringEface any = stringInterfacePtr("") + sliceEface any = sliceInterfacePtr(nil) uint16Type *_type = efaceOf(&uint16Eface)._type uint32Type *_type = efaceOf(&uint32Eface)._type diff --git a/src/runtime/iface_test.go b/src/runtime/iface_test.go index 4fab6c968a..06f6eeb952 100644 --- a/src/runtime/iface_test.go +++ b/src/runtime/iface_test.go @@ -44,8 +44,8 @@ func (Tstr) Method1() {} func (Tslice) Method1() {} var ( - e interface{} - e_ interface{} + e any + e_ any i1 I1 i2 I2 ts TS @@ -196,7 +196,7 @@ func BenchmarkAssertI2I(b *testing.B) { func BenchmarkAssertI2E(b *testing.B) { i1 = tm for i := 0; i < b.N; i++ { - e = i1.(interface{}) + e = i1.(any) } } @@ -224,33 +224,33 @@ func BenchmarkAssertE2T2Blank(b *testing.B) { func BenchmarkAssertI2E2(b *testing.B) { i1 = tm for i := 0; i < b.N; i++ { - e, ok = i1.(interface{}) + e, ok = i1.(any) } } func BenchmarkAssertI2E2Blank(b *testing.B) { i1 = tm for i := 0; i < b.N; i++ { - _, ok = i1.(interface{}) + _, ok = i1.(any) } } func BenchmarkAssertE2E2(b *testing.B) { e = tm for i := 0; i < b.N; i++ { - e_, ok = e.(interface{}) + e_, ok = e.(any) } } func BenchmarkAssertE2E2Blank(b *testing.B) { e = tm for i := 0; i < b.N; i++ { - _, ok = e.(interface{}) + _, ok = e.(any) } } func TestNonEscapingConvT2E(t *testing.T) { - m := make(map[interface{}]bool) + m := make(map[any]bool) m[42] = true if !m[42] { t.Fatalf("42 is not present in the map") diff --git a/src/runtime/internal/atomic/atomic_mips64x.s b/src/runtime/internal/atomic/atomic_mips64x.s index fedfc4a175..b4411d87da 100644 --- a/src/runtime/internal/atomic/atomic_mips64x.s +++ b/src/runtime/internal/atomic/atomic_mips64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips64 || mips64le -// +build mips64 mips64le #include "textflag.h" diff --git a/src/runtime/internal/atomic/atomic_mipsx.s b/src/runtime/internal/atomic/atomic_mipsx.s index c0835d66ed..390e9ce7ac 100644 --- a/src/runtime/internal/atomic/atomic_mipsx.s +++ b/src/runtime/internal/atomic/atomic_mipsx.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips || mipsle -// +build mips mipsle #include "textflag.h" diff --git a/src/runtime/internal/atomic/atomic_ppc64x.s b/src/runtime/internal/atomic/atomic_ppc64x.s index 226b3b6216..04f0eadd06 100644 --- a/src/runtime/internal/atomic/atomic_ppc64x.s +++ b/src/runtime/internal/atomic/atomic_ppc64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "textflag.h" diff --git a/src/runtime/internal/atomic/bench_test.go b/src/runtime/internal/atomic/bench_test.go index 2476c06c52..efc0531a0d 100644 --- a/src/runtime/internal/atomic/bench_test.go +++ b/src/runtime/internal/atomic/bench_test.go @@ -9,7 +9,7 @@ import ( "testing" ) -var sink interface{} +var sink any func BenchmarkAtomicLoad64(b *testing.B) { var x uint64 diff --git a/src/runtime/internal/atomic/sys_nonlinux_arm.s b/src/runtime/internal/atomic/sys_nonlinux_arm.s index 04036ca970..b55bf908a2 100644 --- a/src/runtime/internal/atomic/sys_nonlinux_arm.s +++ b/src/runtime/internal/atomic/sys_nonlinux_arm.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !linux -// +build !linux #include "textflag.h" diff --git a/src/runtime/lfstack_test.go b/src/runtime/lfstack_test.go index fb4b45992d..d0a1b6ba06 100644 --- a/src/runtime/lfstack_test.go +++ b/src/runtime/lfstack_test.go @@ -24,7 +24,7 @@ func toMyNode(node *LFNode) *MyNode { return (*MyNode)(unsafe.Pointer(node)) } -var global interface{} +var global any func TestLFStack(t *testing.T) { stack := new(uint64) diff --git a/src/runtime/libfuzzer_amd64.s b/src/runtime/libfuzzer_amd64.s index 13645fc7af..253fe15198 100644 --- a/src/runtime/libfuzzer_amd64.s +++ b/src/runtime/libfuzzer_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build libfuzzer -// +build libfuzzer #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/libfuzzer_arm64.s b/src/runtime/libfuzzer_arm64.s index 4ad8242804..ae0efd8c9b 100644 --- a/src/runtime/libfuzzer_arm64.s +++ b/src/runtime/libfuzzer_arm64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build libfuzzer -// +build libfuzzer #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index e267e2df23..6ed6ceade2 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -201,15 +201,21 @@ const ( // we further limit it to 31 bits. // // On ios/arm64, although 64-bit pointers are presumably - // available, pointers are truncated to 33 bits. Furthermore, - // only the top 4 GiB of the address space are actually available - // to the application, but we allow the whole 33 bits anyway for - // simplicity. - // TODO(mknyszek): Consider limiting it to 32 bits and using - // arenaBaseOffset to offset into the top 4 GiB. + // available, pointers are truncated to 33 bits in iOS <14. + // Furthermore, only the top 4 GiB of the address space are + // actually available to the application. In iOS >=14, more + // of the address space is available, and the OS can now + // provide addresses outside of those 33 bits. Pick 40 bits + // as a reasonable balance between address space usage by the + // page allocator, and flexibility for what mmap'd regions + // we'll accept for the heap. We can't just move to the full + // 48 bits because this uses too much address space for older + // iOS versions. + // TODO(mknyszek): Once iOS <14 is deprecated, promote ios/arm64 + // to a 48-bit address space like every other arm64 platform. // // WebAssembly currently has a limit of 4GB linear memory. - heapAddrBits = (_64bit*(1-goarch.IsWasm)*(1-goos.IsIos*goarch.IsArm64))*48 + (1-_64bit+goarch.IsWasm)*(32-(goarch.IsMips+goarch.IsMipsle)) + 33*goos.IsIos*goarch.IsArm64 + heapAddrBits = (_64bit*(1-goarch.IsWasm)*(1-goos.IsIos*goarch.IsArm64))*48 + (1-_64bit+goarch.IsWasm)*(32-(goarch.IsMips+goarch.IsMipsle)) + 40*goos.IsIos*goarch.IsArm64 // maxAlloc is the maximum size of an allocation. On 64-bit, // it's theoretically possible to allocate 1<<heapAddrBits bytes. On diff --git a/src/runtime/malloc_test.go b/src/runtime/malloc_test.go index e028554b23..8ff88687bd 100644 --- a/src/runtime/malloc_test.go +++ b/src/runtime/malloc_test.go @@ -33,14 +33,14 @@ func TestMemStats(t *testing.T) { st := new(MemStats) ReadMemStats(st) - nz := func(x interface{}) error { + nz := func(x any) error { if x != reflect.Zero(reflect.TypeOf(x)).Interface() { return nil } return fmt.Errorf("zero value") } - le := func(thresh float64) func(interface{}) error { - return func(x interface{}) error { + le := func(thresh float64) func(any) error { + return func(x any) error { // These sanity tests aren't necessarily valid // with high -test.count values, so only run // them once. @@ -54,8 +54,8 @@ func TestMemStats(t *testing.T) { return fmt.Errorf("insanely high value (overflow?); want <= %v", thresh) } } - eq := func(x interface{}) func(interface{}) error { - return func(y interface{}) error { + eq := func(x any) func(any) error { + return func(y any) error { if x == y { return nil } @@ -64,7 +64,7 @@ func TestMemStats(t *testing.T) { } // Of the uint fields, HeapReleased, HeapIdle can be 0. // PauseTotalNs can be 0 if timer resolution is poor. - fields := map[string][]func(interface{}) error{ + fields := map[string][]func(any) error{ "Alloc": {nz, le(1e10)}, "TotalAlloc": {nz, le(1e11)}, "Sys": {nz, le(1e10)}, "Lookups": {eq(uint64(0))}, "Mallocs": {nz, le(1e10)}, "Frees": {nz, le(1e10)}, "HeapAlloc": {nz, le(1e10)}, "HeapSys": {nz, le(1e10)}, "HeapIdle": {le(1e10)}, @@ -198,6 +198,10 @@ func TestTinyAllocIssue37262(t *testing.T) { runtime.GC() runtime.GC() + // Disable preemption so we stay on one P's tiny allocator and + // nothing else allocates from it. + runtime.Acquirem() + // Make 1-byte allocations until we get a fresh tiny slot. aligned := false for i := 0; i < 16; i++ { @@ -208,6 +212,7 @@ func TestTinyAllocIssue37262(t *testing.T) { } } if !aligned { + runtime.Releasem() t.Fatal("unable to get a fresh tiny slot") } @@ -229,6 +234,8 @@ func TestTinyAllocIssue37262(t *testing.T) { tinyByteSink = nil tinyUint32Sink = nil tinyObj12Sink = nil + + runtime.Releasem() } func TestPageCacheLeak(t *testing.T) { diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index d0becc9ddb..b46d2a4727 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -488,20 +488,20 @@ func BenchmarkMapStringConversion(b *testing.B) { var BoolSink bool func BenchmarkMapInterfaceString(b *testing.B) { - m := map[interface{}]bool{} + m := map[any]bool{} for i := 0; i < 100; i++ { m[fmt.Sprintf("%d", i)] = true } - key := (interface{})("A") + key := (any)("A") b.ResetTimer() for i := 0; i < b.N; i++ { BoolSink = m[key] } } func BenchmarkMapInterfacePtr(b *testing.B) { - m := map[interface{}]bool{} + m := map[any]bool{} for i := 0; i < 100; i++ { i := i diff --git a/src/runtime/map_test.go b/src/runtime/map_test.go index f78cad5a77..0c83dd4ddf 100644 --- a/src/runtime/map_test.go +++ b/src/runtime/map_test.go @@ -1050,7 +1050,7 @@ func BenchmarkMapDelete(b *testing.B) { func TestDeferDeleteSlow(t *testing.T) { ks := []complex128{0, 1, 2, 3} - m := make(map[interface{}]int) + m := make(map[any]int) for i, k := range ks { m[k] = i } @@ -1193,14 +1193,14 @@ func TestMapInterfaceKey(t *testing.T) { c64 complex64 c128 complex128 s string - i0 interface{} + i0 any i1 interface { String() string } a [4]string } - m := map[interface{}]bool{} + m := map[any]bool{} // Put a bunch of data in m, so that a bad hash is likely to // lead to a bad bucket, which will lead to a missed lookup. for i := 0; i < 1000; i++ { diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 3330ddd62e..1c6f3f959f 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -1952,7 +1952,7 @@ func getgcmaskcb(frame *stkframe, ctxt unsafe.Pointer) bool { // gcbits returns the GC type info for x, for testing. // The result is the bitmap entries (0 or 1), one entry per byte. //go:linkname reflect_gcbits reflect.gcbits -func reflect_gcbits(x interface{}) []byte { +func reflect_gcbits(x any) []byte { ret := getgcmask(x) typ := (*ptrtype)(unsafe.Pointer(efaceOf(&x)._type)).elem nptr := typ.ptrdata / goarch.PtrSize @@ -1965,7 +1965,7 @@ func reflect_gcbits(x interface{}) []byte { // Returns GC type info for the pointer stored in ep for testing. // If ep points to the stack, only static live information will be returned // (i.e. not for objects which are only dynamically live stack objects). -func getgcmask(ep interface{}) (mask []byte) { +func getgcmask(ep any) (mask []byte) { e := *efaceOf(&ep) p := e.data t := e._type diff --git a/src/runtime/mem_aix.go b/src/runtime/mem_aix.go index 957aa4dcc2..489d7928e1 100644 --- a/src/runtime/mem_aix.go +++ b/src/runtime/mem_aix.go @@ -72,6 +72,7 @@ func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { throw("runtime: out of memory") } if err != 0 { + print("runtime: mprotect(", v, ", ", n, ") returned ", err, "\n") throw("runtime: cannot map pages in arena address space") } } diff --git a/src/runtime/mem_bsd.go b/src/runtime/mem_bsd.go index b152571792..49337eafbf 100644 --- a/src/runtime/mem_bsd.go +++ b/src/runtime/mem_bsd.go @@ -73,6 +73,7 @@ func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { throw("runtime: out of memory") } if p != v || err != 0 { + print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n") throw("runtime: cannot map pages in arena address space") } } diff --git a/src/runtime/mem_darwin.go b/src/runtime/mem_darwin.go index 7fccd2bb8e..9f836c0818 100644 --- a/src/runtime/mem_darwin.go +++ b/src/runtime/mem_darwin.go @@ -66,6 +66,7 @@ func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { throw("runtime: out of memory") } if p != v || err != 0 { + print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n") throw("runtime: cannot map pages in arena address space") } } diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go index f8f9c53170..f8333014c2 100644 --- a/src/runtime/mem_linux.go +++ b/src/runtime/mem_linux.go @@ -189,6 +189,7 @@ func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { throw("runtime: out of memory") } if p != v || err != 0 { + print("runtime: mmap(", v, ", ", n, ") returned ", p, ", ", err, "\n") throw("runtime: cannot map pages in arena address space") } } diff --git a/src/runtime/memclr_386.s b/src/runtime/memclr_386.s index 2627792ced..a72e5f228d 100644 --- a/src/runtime/memclr_386.s +++ b/src/runtime/memclr_386.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !plan9 -// +build !plan9 #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/memclr_amd64.s b/src/runtime/memclr_amd64.s index 918a4b9e0e..700bbd7b9b 100644 --- a/src/runtime/memclr_amd64.s +++ b/src/runtime/memclr_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !plan9 -// +build !plan9 #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/memclr_mips64x.s b/src/runtime/memclr_mips64x.s index bc037013fe..cf3a9c4ab4 100644 --- a/src/runtime/memclr_mips64x.s +++ b/src/runtime/memclr_mips64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips64 || mips64le -// +build mips64 mips64le #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/memclr_mipsx.s b/src/runtime/memclr_mipsx.s index 3d21c3c414..ee3009d46b 100644 --- a/src/runtime/memclr_mipsx.s +++ b/src/runtime/memclr_mipsx.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips || mipsle -// +build mips mipsle #include "textflag.h" diff --git a/src/runtime/memclr_ppc64x.s b/src/runtime/memclr_ppc64x.s index 91aa417ca2..64132cee96 100644 --- a/src/runtime/memclr_ppc64x.s +++ b/src/runtime/memclr_ppc64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "textflag.h" diff --git a/src/runtime/memmove_386.s b/src/runtime/memmove_386.s index 389ef88477..6d7e17fcbc 100644 --- a/src/runtime/memmove_386.s +++ b/src/runtime/memmove_386.s @@ -24,7 +24,6 @@ // THE SOFTWARE. //go:build !plan9 -// +build !plan9 #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/memmove_amd64.s b/src/runtime/memmove_amd64.s index fa0c0e414f..eeb5033fd9 100644 --- a/src/runtime/memmove_amd64.s +++ b/src/runtime/memmove_amd64.s @@ -24,7 +24,6 @@ // THE SOFTWARE. //go:build !plan9 -// +build !plan9 #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/memmove_mips64x.s b/src/runtime/memmove_mips64x.s index fef3c6be82..b69178ccd3 100644 --- a/src/runtime/memmove_mips64x.s +++ b/src/runtime/memmove_mips64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips64 || mips64le -// +build mips64 mips64le #include "textflag.h" diff --git a/src/runtime/memmove_mipsx.s b/src/runtime/memmove_mipsx.s index cd02fc25c4..494288cf33 100644 --- a/src/runtime/memmove_mipsx.s +++ b/src/runtime/memmove_mipsx.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips || mipsle -// +build mips mipsle #include "textflag.h" diff --git a/src/runtime/memmove_ppc64x.s b/src/runtime/memmove_ppc64x.s index b36b23f8ef..e69e71a4a1 100644 --- a/src/runtime/memmove_ppc64x.s +++ b/src/runtime/memmove_ppc64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "textflag.h" diff --git a/src/runtime/mfinal.go b/src/runtime/mfinal.go index 3cdb81e2fb..e2ac5d4993 100644 --- a/src/runtime/mfinal.go +++ b/src/runtime/mfinal.go @@ -200,7 +200,7 @@ func runfinq() { framesz = f.nret } else { // Need to pass arguments on the stack too. - framesz = unsafe.Sizeof((interface{})(nil)) + f.nret + framesz = unsafe.Sizeof((any)(nil)) + f.nret } if framecap < framesz { // The frame does not contain pointers interesting for GC, @@ -329,7 +329,7 @@ func runfinq() { // A single goroutine runs all finalizers for a program, sequentially. // If a finalizer must run for a long time, it should do so by starting // a new goroutine. -func SetFinalizer(obj interface{}, finalizer interface{}) { +func SetFinalizer(obj any, finalizer any) { if debug.sbrk != 0 { // debug.sbrk never frees memory, so no finalizers run // (and we don't have the data structures to record them). @@ -470,7 +470,7 @@ okarg: // Note: KeepAlive should only be used to prevent finalizers from // running prematurely. In particular, when used with unsafe.Pointer, // the rules for valid uses of unsafe.Pointer still apply. -func KeepAlive(x interface{}) { +func KeepAlive(x any) { // Introduce a use of x that the compiler can't eliminate. // This makes sure x is alive on entry. We need x to be alive // on entry for "defer runtime.KeepAlive(x)"; see issue 21402. diff --git a/src/runtime/mfinal_test.go b/src/runtime/mfinal_test.go index 3ca8d31c60..04ba7a6830 100644 --- a/src/runtime/mfinal_test.go +++ b/src/runtime/mfinal_test.go @@ -34,14 +34,14 @@ func TestFinalizerType(t *testing.T) { } var finalizerTests = []struct { - convert func(*int) interface{} - finalizer interface{} + convert func(*int) any + finalizer any }{ - {func(x *int) interface{} { return x }, func(v *int) { finalize(v) }}, - {func(x *int) interface{} { return Tintptr(x) }, func(v Tintptr) { finalize(v) }}, - {func(x *int) interface{} { return Tintptr(x) }, func(v *int) { finalize(v) }}, - {func(x *int) interface{} { return (*Tint)(x) }, func(v *Tint) { finalize((*int)(v)) }}, - {func(x *int) interface{} { return (*Tint)(x) }, func(v Tinter) { finalize((*int)(v.(*Tint))) }}, + {func(x *int) any { return x }, func(v *int) { finalize(v) }}, + {func(x *int) any { return Tintptr(x) }, func(v Tintptr) { finalize(v) }}, + {func(x *int) any { return Tintptr(x) }, func(v *int) { finalize(v) }}, + {func(x *int) any { return (*Tint)(x) }, func(v *Tint) { finalize((*int)(v)) }}, + {func(x *int) any { return (*Tint)(x) }, func(v Tinter) { finalize((*int)(v.(*Tint))) }}, } for i, tt := range finalizerTests { @@ -85,7 +85,7 @@ func TestFinalizerInterfaceBig(t *testing.T) { go func() { v := &bigValue{0xDEADBEEFDEADBEEF, true, "It matters not how strait the gate"} old := *v - runtime.SetFinalizer(v, func(v interface{}) { + runtime.SetFinalizer(v, func(v any) { i, ok := v.(*bigValue) if !ok { t.Errorf("finalizer called with type %T, want *bigValue", v) diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 96f4157b59..44b96154e7 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -320,11 +320,20 @@ var work struct { nwait uint32 // Number of roots of various root types. Set by gcMarkRootPrepare. + // + // nStackRoots == len(stackRoots), but we have nStackRoots for + // consistency. nDataRoots, nBSSRoots, nSpanRoots, nStackRoots int // Base indexes of each root type. Set by gcMarkRootPrepare. baseData, baseBSS, baseSpans, baseStacks, baseEnd uint32 + // stackRoots is a snapshot of all of the Gs that existed + // before the beginning of concurrent marking. The backing + // store of this must not be modified because it might be + // shared with allgs. + stackRoots []*g + // Each type of GC state transition is protected by a lock. // Since multiple threads can simultaneously detect the state // transition condition, any thread that detects a transition @@ -545,7 +554,7 @@ func (t gcTrigger) test() bool { // own write. return gcController.heapLive >= gcController.trigger case gcTriggerTime: - if atomic.Loadint32(&gcController.gcPercent) < 0 { + if gcController.gcPercent.Load() < 0 { return false } lastgc := int64(atomic.Load64(&memstats.last_gc_nanotime)) @@ -1314,7 +1323,7 @@ func gcBgMarkWorker() { // point, signal the main GC goroutine. if incnwait == work.nproc && !gcMarkWorkAvailable(nil) { // We don't need the P-local buffers here, allow - // preemption becuse we may schedule like a regular + // preemption because we may schedule like a regular // goroutine in gcMarkDone (block on locks, etc). releasem(node.m.ptr()) node.m.set(nil) @@ -1368,6 +1377,11 @@ func gcMark(startTime int64) { throw("work.full != 0") } + // Drop allg snapshot. allgs may have grown, in which case + // this is the only reference to the old backing store and + // there's no need to keep it around. + work.stackRoots = nil + // Clear out buffers and double-check that all gcWork caches // are empty. This should be ensured by gcMarkDone before we // enter mark termination. diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index a5129bd1ee..0bf044e314 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -102,7 +102,8 @@ func gcMarkRootPrepare() { // ignore them because they begin life without any roots, so // there's nothing to scan, and any roots they create during // the concurrent phase will be caught by the write barrier. - work.nStackRoots = int(atomic.Loaduintptr(&allglen)) + work.stackRoots = allGsSnapshot() + work.nStackRoots = len(work.stackRoots) work.markrootNext = 0 work.markrootJobs = uint32(fixedRootCount + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots) @@ -194,15 +195,12 @@ func markroot(gcw *gcWork, i uint32, flushBgCredit bool) int64 { default: // the rest is scanning goroutine stacks workCounter = &gcController.stackScanWork - var gp *g - if work.baseStacks <= i && i < work.baseEnd { - // N.B. Atomic read of allglen in gcMarkRootPrepare - // acts as a barrier to ensure that allgs must be large - // enough to contain all relevant Gs. - gp = allgs[i-work.baseStacks] - } else { + if i < work.baseStacks || work.baseEnd <= i { + printlock() + print("runtime: markroot index ", i, " not in stack roots range [", work.baseStacks, ", ", work.baseEnd, ")\n") throw("markroot: bad index") } + gp := work.stackRoots[i-work.baseStacks] // remember when we've first observed the G blocked // needed only to output in traceback @@ -403,7 +401,7 @@ func markrootSpans(gcw *gcWork, shard int) { } // gcAssistAlloc performs GC work to make gp's assist debt positive. -// gp must be the calling user gorountine. +// gp must be the calling user goroutine. // // This must be called with preemption enabled. func gcAssistAlloc(gp *g) { @@ -1563,7 +1561,7 @@ func gcmarknewobject(span *mspan, obj, size, scanSize uintptr) { if !goexperiment.PacerRedesign { // The old pacer counts newly allocated memory toward // heapScanWork because heapScan is continuously updated - // throughout the GC cyle with newly allocated memory. However, + // throughout the GC cycle with newly allocated memory. However, // these objects are never actually scanned, so we need // to account for them in heapScanWork here, "faking" their work. // Otherwise the pacer will think it's always behind, potentially diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index 5b699cb298..6df8af45a8 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -53,8 +53,8 @@ const ( gcOverAssistWork = 64 << 10 // defaultHeapMinimum is the value of heapMinimum for GOGC==100. - defaultHeapMinimum = goexperiment.PacerRedesignInt*(512<<10) + - (1-goexperiment.PacerRedesignInt)*(4<<20) + defaultHeapMinimum = (goexperiment.HeapMinimum512KiBInt)*(512<<10) + + (1-goexperiment.HeapMinimum512KiBInt)*(4<<20) // scannableStackSizeSlack is the bytes of stack space allocated or freed // that can accumulate on a P before updating gcController.stackSize. @@ -84,12 +84,9 @@ func init() { var gcController gcControllerState type gcControllerState struct { - // Initialized from $GOGC. GOGC=off means no GC. - // - // Updated atomically with mheap_.lock held or during a STW. - // Safe to read atomically at any time, or non-atomically with - // mheap_.lock or STW. - gcPercent int32 + + // Initialized from GOGC. GOGC=off means no GC. + gcPercent atomic.Int32 _ uint32 // padding so following 64-bit values are 8-byte aligned @@ -479,7 +476,7 @@ func (c *gcControllerState) startCycle(markStartTime int64, procs int) { // is when assists are enabled and the necessary statistics are // available). func (c *gcControllerState) revise() { - gcPercent := atomic.Loadint32(&c.gcPercent) + gcPercent := c.gcPercent.Load() if gcPercent < 0 { // If GC is disabled but we're running a forced GC, // act like GOGC is huge for the below calculations. @@ -969,8 +966,8 @@ func (c *gcControllerState) commit(triggerRatio float64) { // has grown by GOGC/100 over where it started the last cycle, // plus additional runway for non-heap sources of GC work. goal := ^uint64(0) - if c.gcPercent >= 0 { - goal = c.heapMarked + (c.heapMarked+atomic.Load64(&c.stackScan)+atomic.Load64(&c.globalsScan))*uint64(c.gcPercent)/100 + if gcPercent := c.gcPercent.Load(); gcPercent >= 0 { + goal = c.heapMarked + (c.heapMarked+atomic.Load64(&c.stackScan)+atomic.Load64(&c.globalsScan))*uint64(gcPercent)/100 } // Don't trigger below the minimum heap size. @@ -1084,17 +1081,19 @@ func (c *gcControllerState) commit(triggerRatio float64) { // // For !goexperiment.PacerRedesign. func (c *gcControllerState) oldCommit(triggerRatio float64) { + gcPercent := c.gcPercent.Load() + // Compute the next GC goal, which is when the allocated heap // has grown by GOGC/100 over the heap marked by the last // cycle. goal := ^uint64(0) - if c.gcPercent >= 0 { - goal = c.heapMarked + c.heapMarked*uint64(c.gcPercent)/100 + if gcPercent >= 0 { + goal = c.heapMarked + c.heapMarked*uint64(gcPercent)/100 } // Set the trigger ratio, capped to reasonable bounds. - if c.gcPercent >= 0 { - scalingFactor := float64(c.gcPercent) / 100 + if gcPercent >= 0 { + scalingFactor := float64(gcPercent) / 100 // Ensure there's always a little margin so that the // mutator assist ratio isn't infinity. maxTriggerRatio := 0.95 * scalingFactor @@ -1134,7 +1133,7 @@ func (c *gcControllerState) oldCommit(triggerRatio float64) { // We trigger the next GC cycle when the allocated heap has // grown by the trigger ratio over the marked heap size. trigger := ^uint64(0) - if c.gcPercent >= 0 { + if gcPercent >= 0 { trigger = uint64(float64(c.heapMarked) * (1 + triggerRatio)) // Don't trigger below the minimum heap size. minTrigger := c.heapMinimum @@ -1210,13 +1209,12 @@ func (c *gcControllerState) setGCPercent(in int32) int32 { assertWorldStoppedOrLockHeld(&mheap_.lock) } - out := c.gcPercent + out := c.gcPercent.Load() if in < 0 { in = -1 } - // Write it atomically so readers like revise() can read it safely. - atomic.Storeint32(&c.gcPercent, in) - c.heapMinimum = defaultHeapMinimum * uint64(c.gcPercent) / 100 + c.heapMinimum = defaultHeapMinimum * uint64(in) / 100 + c.gcPercent.Store(in) // Update pacing in response to gcPercent change. c.commit(c.triggerRatio) diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go index 4a7f2465fd..c27e189af9 100644 --- a/src/runtime/mgcscavenge.go +++ b/src/runtime/mgcscavenge.go @@ -263,7 +263,7 @@ func bgscavenge(c chan int) { scavenge.parked = true scavenge.timer = new(timer) - scavenge.timer.f = func(_ interface{}, _ uintptr) { + scavenge.timer.f = func(_ any, _ uintptr) { wakeScavenger() } @@ -326,8 +326,8 @@ func bgscavenge(c chan int) { // Accumulate the amount of time spent scavenging. start := nanotime() - released = mheap_.pages.scavenge(scavengeQuantum) - atomic.Xadduintptr(&mheap_.pages.scav.released, released) + r := mheap_.pages.scavenge(scavengeQuantum) + atomic.Xadduintptr(&mheap_.pages.scav.released, r) end := nanotime() // On some platforms we may see end >= start if the time it takes to scavenge @@ -339,10 +339,16 @@ func bgscavenge(c chan int) { // on timing. const approxCritNSPerPhysicalPage = 10e3 if end <= start { - crit += approxCritNSPerPhysicalPage * float64(released/physPageSize) + crit += approxCritNSPerPhysicalPage * float64(r/physPageSize) } else { crit += float64(end - start) } + released += r + + // When using fake time just do one loop. + if faketime != 0 { + break + } } if released == 0 { diff --git a/src/runtime/mgcscavenge_test.go b/src/runtime/mgcscavenge_test.go index b186cad2f4..0659293c60 100644 --- a/src/runtime/mgcscavenge_test.go +++ b/src/runtime/mgcscavenge_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "fmt" + "internal/goos" "math/rand" . "runtime" "testing" @@ -408,7 +409,9 @@ func TestPageAllocScavenge(t *testing.T) { }, }, } - if PageAlloc64Bit != 0 { + // Disable these tests on iOS since we have a small address space. + // See #46860. + if PageAlloc64Bit != 0 && goos.IsIos == 0 { tests["ScavAllVeryDiscontiguous"] = setup{ beforeAlloc: map[ChunkIdx][]BitRange{ BaseChunkIdx: {}, diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index fdbec30cf1..a46f4ec2c6 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -393,7 +393,7 @@ func sweepone() uintptr { // The scavenger is signaled by the last sweeper because once // sweeping is done, we will definitely have useful work for // the scavenger to do, since the scavenger only runs over the - // heap once per GC cyle. This update is not done during sweep + // heap once per GC cycle. This update is not done during sweep // termination because in some cases there may be a long delay // between sweep done and sweep termination (e.g. not enough // allocations to trigger a GC) which would be nice to fill in diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index e8c23d485a..17c9b75d69 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -129,7 +129,7 @@ func header(arch string) { fmt.Fprintf(out, "TEXT ·asyncPreempt(SB),NOSPLIT|NOFRAME,$0-0\n") } -func p(f string, args ...interface{}) { +func p(f string, args ...any) { fmted := fmt.Sprintf(f, args...) fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) } diff --git a/src/runtime/mpagealloc_32bit.go b/src/runtime/mpagealloc_32bit.go index 1d863f2fda..8c83b93412 100644 --- a/src/runtime/mpagealloc_32bit.go +++ b/src/runtime/mpagealloc_32bit.go @@ -2,19 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build 386 || arm || mips || mipsle || wasm || (ios && arm64) +//go:build 386 || arm || mips || mipsle || wasm // wasm is a treated as a 32-bit architecture for the purposes of the page // allocator, even though it has 64-bit pointers. This is because any wasm // pointer always has its top 32 bits as zero, so the effective heap address // space is only 2^32 bytes in size (see heapAddrBits). -// ios/arm64 is treated as a 32-bit architecture for the purposes of the -// page allocator, even though it has 64-bit pointers and a 33-bit address -// space (see heapAddrBits). The 33 bit address space cannot be rounded up -// to 64 bits because there are too many summary levels to fit in just 33 -// bits. - package runtime import "unsafe" diff --git a/src/runtime/mpagealloc_64bit.go b/src/runtime/mpagealloc_64bit.go index 782628c91d..1bacfbe0fa 100644 --- a/src/runtime/mpagealloc_64bit.go +++ b/src/runtime/mpagealloc_64bit.go @@ -2,9 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build amd64 || (!ios && arm64) || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x - -// See mpagealloc_32bit.go for why ios/arm64 is excluded here. +//go:build amd64 || arm64 || mips64 || mips64le || ppc64 || ppc64le || riscv64 || s390x package runtime diff --git a/src/runtime/mpagealloc_test.go b/src/runtime/mpagealloc_test.go index 5d979fa95b..f2b82e3f50 100644 --- a/src/runtime/mpagealloc_test.go +++ b/src/runtime/mpagealloc_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "fmt" + "internal/goos" . "runtime" "testing" ) @@ -165,7 +166,9 @@ func TestPageAllocGrow(t *testing.T) { }, }, } - if PageAlloc64Bit != 0 { + // Disable these tests on iOS since we have a small address space. + // See #46860. + if PageAlloc64Bit != 0 && goos.IsIos == 0 { tests["ExtremelyDiscontiguous"] = test{ chunks: []ChunkIdx{ BaseChunkIdx, @@ -571,7 +574,9 @@ func TestPageAllocAlloc(t *testing.T) { }, }, } - if PageAlloc64Bit != 0 { + // Disable these tests on iOS since we have a small address space. + // See #46860. + if PageAlloc64Bit != 0 && goos.IsIos == 0 { const chunkIdxBigJump = 0x100000 // chunk index offset which translates to O(TiB) // This test attempts to trigger a bug wherein we look at unmapped summary diff --git a/src/runtime/mpagecache_test.go b/src/runtime/mpagecache_test.go index 69084f9a84..6cb0620f7b 100644 --- a/src/runtime/mpagecache_test.go +++ b/src/runtime/mpagecache_test.go @@ -5,6 +5,7 @@ package runtime_test import ( + "internal/goos" "math/rand" . "runtime" "testing" @@ -372,7 +373,9 @@ func TestPageAllocAllocToCache(t *testing.T) { }, }, } - if PageAlloc64Bit != 0 { + // Disable these tests on iOS since we have a small address space. + // See #46860. + if PageAlloc64Bit != 0 && goos.IsIos == 0 { const chunkIdxBigJump = 0x100000 // chunk index offset which translates to O(TiB) // This test is similar to the one with the same name for diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index b4de8f53a9..569c17f0a7 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -805,7 +805,11 @@ func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int // truncated profile than to crash the entire process. return } - saveg(^uintptr(0), ^uintptr(0), gp1, &r[0]) + // saveg calls gentraceback, which may call cgo traceback functions. + // The world is stopped, so it cannot use cgocall (which will be + // blocked at exitsyscall). Do it on the system stack so it won't + // call into the schedular (see traceback.go:cgoContextPCs). + systemstack(func() { saveg(^uintptr(0), ^uintptr(0), gp1, &r[0]) }) if labels != nil { lbl[0] = gp1.labels lbl = lbl[1:] diff --git a/src/runtime/msan_amd64.s b/src/runtime/msan_amd64.s index 1bb57a3b7e..89ed3048d0 100644 --- a/src/runtime/msan_amd64.s +++ b/src/runtime/msan_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build msan -// +build msan #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/msan_arm64.s b/src/runtime/msan_arm64.s index 93ade8dd89..b9eff34ab6 100644 --- a/src/runtime/msan_arm64.s +++ b/src/runtime/msan_arm64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build msan -// +build msan #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index f60e62dec7..bb3dd35317 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -71,31 +71,99 @@ const pollBlockSize = 4 * 1024 //go:notinheap type pollDesc struct { link *pollDesc // in pollcache, protected by pollcache.lock + fd uintptr // constant for pollDesc usage lifetime + + // atomicInfo holds bits from closing, rd, and wd, + // which are only ever written while holding the lock, + // summarized for use by netpollcheckerr, + // which cannot acquire the lock. + // After writing these fields under lock in a way that + // might change the summary, code must call publishInfo + // before releasing the lock. + // Code that changes fields and then calls netpollunblock + // (while still holding the lock) must call publishInfo + // before calling netpollunblock, because publishInfo is what + // stops netpollblock from blocking anew + // (by changing the result of netpollcheckerr). + // atomicInfo also holds the eventErr bit, + // recording whether a poll event on the fd got an error; + // atomicInfo is the only source of truth for that bit. + atomicInfo atomic.Uint32 // atomic pollInfo + + // rg, wg are accessed atomically and hold g pointers. + // (Using atomic.Uintptr here is similar to using guintptr elsewhere.) + rg atomic.Uintptr // pdReady, pdWait, G waiting for read or nil + wg atomic.Uintptr // pdReady, pdWait, G waiting for write or nil - // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations. - // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime. - // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO readiness notification) - // proceed w/o taking the lock. So closing, everr, rg, rd, wg and wd are manipulated - // in a lock-free way by all operations. - // TODO(golang.org/issue/49008): audit these lock-free fields for continued correctness. - // NOTE(dvyukov): the following code uses uintptr to store *g (rg/wg), - // that will blow up when GC starts moving objects. lock mutex // protects the following fields - fd uintptr closing bool - everr bool // marks event scanning error happened user uint32 // user settable cookie rseq uintptr // protects from stale read timers - rg uintptr // pdReady, pdWait, G waiting for read or nil. Accessed atomically. rt timer // read deadline timer (set if rt.f != nil) - rd int64 // read deadline + rd int64 // read deadline (a nanotime in the future, -1 when expired) wseq uintptr // protects from stale write timers - wg uintptr // pdReady, pdWait, G waiting for write or nil. Accessed atomically. wt timer // write deadline timer - wd int64 // write deadline + wd int64 // write deadline (a nanotime in the future, -1 when expired) self *pollDesc // storage for indirect interface. See (*pollDesc).makeArg. } +// pollInfo is the bits needed by netpollcheckerr, stored atomically, +// mostly duplicating state that is manipulated under lock in pollDesc. +// The one exception is the pollEventErr bit, which is maintained only +// in the pollInfo. +type pollInfo uint32 + +const ( + pollClosing = 1 << iota + pollEventErr + pollExpiredReadDeadline + pollExpiredWriteDeadline +) + +func (i pollInfo) closing() bool { return i&pollClosing != 0 } +func (i pollInfo) eventErr() bool { return i&pollEventErr != 0 } +func (i pollInfo) expiredReadDeadline() bool { return i&pollExpiredReadDeadline != 0 } +func (i pollInfo) expiredWriteDeadline() bool { return i&pollExpiredWriteDeadline != 0 } + +// info returns the pollInfo corresponding to pd. +func (pd *pollDesc) info() pollInfo { + return pollInfo(pd.atomicInfo.Load()) +} + +// publishInfo updates pd.atomicInfo (returned by pd.info) +// using the other values in pd. +// It must be called while holding pd.lock, +// and it must be called after changing anything +// that might affect the info bits. +// In practice this means after changing closing +// or changing rd or wd from < 0 to >= 0. +func (pd *pollDesc) publishInfo() { + var info uint32 + if pd.closing { + info |= pollClosing + } + if pd.rd < 0 { + info |= pollExpiredReadDeadline + } + if pd.wd < 0 { + info |= pollExpiredWriteDeadline + } + + // Set all of x except the pollEventErr bit. + x := pd.atomicInfo.Load() + for !pd.atomicInfo.CompareAndSwap(x, (x&pollEventErr)|info) { + x = pd.atomicInfo.Load() + } +} + +// setEventErr sets the result of pd.info().eventErr() to b. +func (pd *pollDesc) setEventErr(b bool) { + x := pd.atomicInfo.Load() + for (x&pollEventErr != 0) != b && !pd.atomicInfo.CompareAndSwap(x, x^pollEventErr) { + x = pd.atomicInfo.Load() + } +} + type pollCache struct { lock mutex first *pollDesc @@ -147,24 +215,25 @@ func poll_runtime_isPollServerDescriptor(fd uintptr) bool { func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) { pd := pollcache.alloc() lock(&pd.lock) - wg := atomic.Loaduintptr(&pd.wg) + wg := pd.wg.Load() if wg != 0 && wg != pdReady { throw("runtime: blocked write on free polldesc") } - rg := atomic.Loaduintptr(&pd.rg) + rg := pd.rg.Load() if rg != 0 && rg != pdReady { throw("runtime: blocked read on free polldesc") } pd.fd = fd pd.closing = false - pd.everr = false + pd.setEventErr(false) pd.rseq++ - atomic.Storeuintptr(&pd.rg, 0) + pd.rg.Store(0) pd.rd = 0 pd.wseq++ - atomic.Storeuintptr(&pd.wg, 0) + pd.wg.Store(0) pd.wd = 0 pd.self = pd + pd.publishInfo() unlock(&pd.lock) errno := netpollopen(fd, pd) @@ -180,11 +249,11 @@ func poll_runtime_pollClose(pd *pollDesc) { if !pd.closing { throw("runtime: close polldesc w/o unblock") } - wg := atomic.Loaduintptr(&pd.wg) + wg := pd.wg.Load() if wg != 0 && wg != pdReady { throw("runtime: blocked write on closing polldesc") } - rg := atomic.Loaduintptr(&pd.rg) + rg := pd.rg.Load() if rg != 0 && rg != pdReady { throw("runtime: blocked read on closing polldesc") } @@ -209,9 +278,9 @@ func poll_runtime_pollReset(pd *pollDesc, mode int) int { return errcode } if mode == 'r' { - atomic.Storeuintptr(&pd.rg, 0) + pd.rg.Store(0) } else if mode == 'w' { - atomic.Storeuintptr(&pd.wg, 0) + pd.wg.Store(0) } return pollNoError } @@ -273,6 +342,7 @@ func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { if mode == 'w' || mode == 'r'+'w' { pd.wd = d } + pd.publishInfo() combo := pd.rd > 0 && pd.rd == pd.wd rtf := netpollReadDeadline if combo { @@ -314,15 +384,13 @@ func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { } } // If we set the new deadline in the past, unblock currently pending IO if any. + // Note that pd.publishInfo has already been called, above, immediately after modifying rd and wd. var rg, wg *g - if pd.rd < 0 || pd.wd < 0 { - atomic.StorepNoWB(noescape(unsafe.Pointer(&wg)), nil) // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock - if pd.rd < 0 { - rg = netpollunblock(pd, 'r', false) - } - if pd.wd < 0 { - wg = netpollunblock(pd, 'w', false) - } + if pd.rd < 0 { + rg = netpollunblock(pd, 'r', false) + } + if pd.wd < 0 { + wg = netpollunblock(pd, 'w', false) } unlock(&pd.lock) if rg != nil { @@ -343,7 +411,7 @@ func poll_runtime_pollUnblock(pd *pollDesc) { pd.rseq++ pd.wseq++ var rg, wg *g - atomic.StorepNoWB(noescape(unsafe.Pointer(&rg)), nil) // full memory barrier between store to closing and read of rg/wg in netpollunblock + pd.publishInfo() rg = netpollunblock(pd, 'r', false) wg = netpollunblock(pd, 'w', false) if pd.rt.f != nil { @@ -388,16 +456,17 @@ func netpollready(toRun *gList, pd *pollDesc, mode int32) { } func netpollcheckerr(pd *pollDesc, mode int32) int { - if pd.closing { + info := pd.info() + if info.closing() { return pollErrClosing } - if (mode == 'r' && pd.rd < 0) || (mode == 'w' && pd.wd < 0) { + if (mode == 'r' && info.expiredReadDeadline()) || (mode == 'w' && info.expiredWriteDeadline()) { return pollErrTimeout } // Report an event scanning error only on a read event. // An error on a write event will be captured in a subsequent // write call that is able to report a more specific error. - if mode == 'r' && pd.everr { + if mode == 'r' && info.eventErr() { return pollErrNotPollable } return pollNoError @@ -432,28 +501,28 @@ func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { // set the gpp semaphore to pdWait for { // Consume notification if already ready. - if atomic.Casuintptr(gpp, pdReady, 0) { + if gpp.CompareAndSwap(pdReady, 0) { return true } - if atomic.Casuintptr(gpp, 0, pdWait) { + if gpp.CompareAndSwap(0, pdWait) { break } // Double check that this isn't corrupt; otherwise we'd loop // forever. - if v := atomic.Loaduintptr(gpp); v != pdReady && v != 0 { + if v := gpp.Load(); v != pdReady && v != 0 { throw("runtime: double wait") } } // need to recheck error states after setting gpp to pdWait // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl - // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg + // do the opposite: store to closing/rd/wd, publishInfo, load of rg/wg if waitio || netpollcheckerr(pd, mode) == pollNoError { gopark(netpollblockcommit, unsafe.Pointer(gpp), waitReasonIOWait, traceEvGoBlockNet, 5) } // be careful to not lose concurrent pdReady notification - old := atomic.Xchguintptr(gpp, 0) + old := gpp.Swap(0) if old > pdWait { throw("runtime: corrupted polldesc") } @@ -467,7 +536,7 @@ func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g { } for { - old := atomic.Loaduintptr(gpp) + old := gpp.Load() if old == pdReady { return nil } @@ -480,7 +549,7 @@ func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g { if ioready { new = pdReady } - if atomic.Casuintptr(gpp, old, new) { + if gpp.CompareAndSwap(old, new) { if old == pdWait { old = 0 } @@ -508,7 +577,7 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { throw("runtime: inconsistent read deadline") } pd.rd = -1 - atomic.StorepNoWB(unsafe.Pointer(&pd.rt.f), nil) // full memory barrier between store to rd and load of rg in netpollunblock + pd.publishInfo() rg = netpollunblock(pd, 'r', false) } var wg *g @@ -517,7 +586,7 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { throw("runtime: inconsistent write deadline") } pd.wd = -1 - atomic.StorepNoWB(unsafe.Pointer(&pd.wt.f), nil) // full memory barrier between store to wd and load of wg in netpollunblock + pd.publishInfo() wg = netpollunblock(pd, 'w', false) } unlock(&pd.lock) @@ -529,15 +598,15 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { } } -func netpollDeadline(arg interface{}, seq uintptr) { +func netpollDeadline(arg any, seq uintptr) { netpolldeadlineimpl(arg.(*pollDesc), seq, true, true) } -func netpollReadDeadline(arg interface{}, seq uintptr) { +func netpollReadDeadline(arg any, seq uintptr) { netpolldeadlineimpl(arg.(*pollDesc), seq, true, false) } -func netpollWriteDeadline(arg interface{}, seq uintptr) { +func netpollWriteDeadline(arg any, seq uintptr) { netpolldeadlineimpl(arg.(*pollDesc), seq, false, true) } @@ -570,7 +639,7 @@ func (c *pollCache) alloc() *pollDesc { // a conversion requires an allocation because pointers to // go:notinheap types (which pollDesc is) must be stored // in interfaces indirectly. See issue 42076. -func (pd *pollDesc) makeArg() (i interface{}) { +func (pd *pollDesc) makeArg() (i any) { x := (*eface)(unsafe.Pointer(&i)) x._type = pdType x.data = unsafe.Pointer(&pd.self) @@ -578,6 +647,6 @@ func (pd *pollDesc) makeArg() (i interface{}) { } var ( - pdEface interface{} = (*pollDesc)(nil) - pdType *_type = efaceOf(&pdEface)._type + pdEface any = (*pollDesc)(nil) + pdType *_type = efaceOf(&pdEface)._type ) diff --git a/src/runtime/netpoll_aix.go b/src/runtime/netpoll_aix.go index 4590ed81a6..90950af444 100644 --- a/src/runtime/netpoll_aix.go +++ b/src/runtime/netpoll_aix.go @@ -212,10 +212,7 @@ retry: pfd.events &= ^_POLLOUT } if mode != 0 { - pds[i].everr = false - if pfd.revents == _POLLERR { - pds[i].everr = true - } + pds[i].setEventErr(pfd.revents == _POLLERR) netpollready(&toRun, pds[i], mode) n-- } diff --git a/src/runtime/netpoll_epoll.go b/src/runtime/netpoll_epoll.go index e0fb877d50..b7d6199965 100644 --- a/src/runtime/netpoll_epoll.go +++ b/src/runtime/netpoll_epoll.go @@ -168,10 +168,7 @@ retry: } if mode != 0 { pd := *(**pollDesc)(unsafe.Pointer(&ev.data)) - pd.everr = false - if ev.events == _EPOLLERR { - pd.everr = true - } + pd.setEventErr(ev.events == _EPOLLERR) netpollready(&toRun, pd, mode) } } diff --git a/src/runtime/netpoll_kqueue.go b/src/runtime/netpoll_kqueue.go index 2f7f2848d2..1694753b6f 100644 --- a/src/runtime/netpoll_kqueue.go +++ b/src/runtime/netpoll_kqueue.go @@ -179,10 +179,7 @@ retry: } if mode != 0 { pd := (*pollDesc)(unsafe.Pointer(ev.udata)) - pd.everr = false - if ev.flags == _EV_ERROR { - pd.everr = true - } + pd.setEventErr(ev.flags == _EV_ERROR) netpollready(&toRun, pd, mode) } } diff --git a/src/runtime/netpoll_solaris.go b/src/runtime/netpoll_solaris.go index d217d5b160..6e545b3d31 100644 --- a/src/runtime/netpoll_solaris.go +++ b/src/runtime/netpoll_solaris.go @@ -158,7 +158,7 @@ func netpollclose(fd uintptr) int32 { // this call, port_getn will return one and only one event for that // particular descriptor, so this function needs to be called again. func netpollupdate(pd *pollDesc, set, clear uint32) { - if pd.closing { + if pd.info().closing() { return } diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 0e17e75e3e..c76add7802 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -40,7 +40,6 @@ const ( //go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll" //go:cgo_import_dynamic runtime._LoadLibraryA LoadLibraryA%1 "kernel32.dll" //go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll" -//go:cgo_import_dynamic runtime._RaiseException RaiseException%4 "kernel32.dll" //go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll" //go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll" @@ -94,7 +93,6 @@ var ( _PostQueuedCompletionStatus, _QueryPerformanceCounter, _QueryPerformanceFrequency, - _RaiseException, _ResumeThread, _SetConsoleCtrlHandler, _SetErrorMode, @@ -122,7 +120,6 @@ var ( _AddVectoredContinueHandler, _LoadLibraryExA, _LoadLibraryExW, - _WerSetFlags, _ stdFunction // Use RtlGenRandom to generate cryptographically random data. @@ -257,7 +254,6 @@ func loadOptionalSyscalls() { _AddVectoredContinueHandler = windowsFindfunc(k32, []byte("AddVectoredContinueHandler\000")) _LoadLibraryExA = windowsFindfunc(k32, []byte("LoadLibraryExA\000")) _LoadLibraryExW = windowsFindfunc(k32, []byte("LoadLibraryExW\000")) - _WerSetFlags = windowsFindfunc(k32, []byte("WerSetFlags\000")) useLoadLibraryEx = (_LoadLibraryExW != nil && _LoadLibraryExA != nil && _AddDllDirectory != nil) var advapi32dll = []byte("advapi32.dll\000") @@ -327,7 +323,7 @@ func monitorSuspendResume() { if powerRegisterSuspendResumeNotification == nil { return // Running on Windows 7, where we don't need it anyway. } - var fn interface{} = func(context uintptr, changeType uint32, setting uintptr) uintptr { + var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr { for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink { if mp.resumesema != 0 { stdcall1(_SetEvent, mp.resumesema) @@ -686,7 +682,7 @@ func goenvs() { // We call these all the way here, late in init, so that malloc works // for the callback functions these generate. - var fn interface{} = ctrlHandler + var fn any = ctrlHandler ctrlHandlerPC := compileCallback(*efaceOf(&fn), true) stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1) @@ -1198,8 +1194,10 @@ func ctrlHandler(_type uint32) uintptr { if sigsend(s) { if s == _SIGTERM { // Windows terminates the process after this handler returns. - // Block indefinitely to give signal handlers a chance to clean up. - stdcall1(_Sleep, uintptr(_INFINITE)) + // Block indefinitely to give signal handlers a chance to clean up, + // but make sure to be properly parked first, so the rest of the + // program can continue executing. + block() } return 1 } @@ -1308,18 +1306,13 @@ func setThreadCPUProfiler(hz int32) { atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) } -const preemptMSupported = GOARCH == "386" || GOARCH == "amd64" +const preemptMSupported = true // suspendLock protects simultaneous SuspendThread operations from // suspending each other. var suspendLock mutex func preemptM(mp *m) { - if !preemptMSupported { - // TODO: Implement call injection - return - } - if mp == getg().m { throw("self-preempt") } @@ -1401,8 +1394,31 @@ func preemptM(mp *m) { *(*uintptr)(unsafe.Pointer(sp)) = newpc c.set_sp(sp) c.set_ip(targetPC) - } + case "arm": + // Push LR. The injected call is responsible + // for restoring LR. gentraceback is aware of + // this extra slot. See sigctxt.pushCall in + // signal_arm.go, which is similar except we + // subtract 1 from IP here. + sp := c.sp() + sp -= goarch.PtrSize + c.set_sp(sp) + *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr()) + c.set_lr(newpc - 1) + c.set_ip(targetPC) + + case "arm64": + // Push LR. The injected call is responsible + // for restoring LR. gentraceback is aware of + // this extra slot. See sigctxt.pushCall in + // signal_arm64.go. + sp := c.sp() - 16 // SP needs 16-byte alignment + c.set_sp(sp) + *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr()) + c.set_lr(newpc) + c.set_ip(targetPC) + } stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c))) } } diff --git a/src/runtime/panic.go b/src/runtime/panic.go index eec69dfdc6..6600410cb6 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -755,7 +755,7 @@ func deferCallSave(p *_panic, fn func()) { } // The implementation of the predeclared function panic. -func gopanic(e interface{}) { +func gopanic(e any) { gp := getg() if gp.m.curg != gp { print("panic: ") @@ -957,7 +957,7 @@ func getargp() uintptr { // TODO(rsc): Once we commit to CopyStackAlways, // this doesn't need to be nosplit. //go:nosplit -func gorecover(argp uintptr) interface{} { +func gorecover(argp uintptr) any { // Must be in a function running as part of a deferred call during the panic. // Must be called from the topmost function of the call // (the function used in the defer statement). @@ -1002,11 +1002,6 @@ var runningPanicDefers uint32 // panicking is incremented and decremented atomically. var panicking uint32 -// tracebackprinted is zero before gopanic() prints the traceback. After -// traceback is printed, it sets to 1 so that the subsequent exception handler -// won't print the traceback again. -var tracebackprinted uint32 - // paniclk is held while printing the panic information and stack trace, // so that two concurrent panics don't overlap their output. var paniclk mutex @@ -1050,9 +1045,6 @@ func fatalthrow() { startpanic_m() if dopanic_m(gp, pc, sp) { - // At this point, traceback has already been printed. - // Set tracebackprinted to 1 to avoid printing traceback again - tracebackprinted = 1 // crash uses a decent amount of nosplit stack and we're already // low on stack in throw, so crash on the system stack (unlike // fatalpanic). @@ -1094,9 +1086,6 @@ func fatalpanic(msgs *_panic) { }) if docrash { - // At this point, traceback has already been printed. - // Set tracebackprinted to 1 to avoid printing traceback again - tracebackprinted = 1 // By crashing outside the above systemstack call, debuggers // will not be confused when generating a backtrace. // Function crash is marked nosplit to avoid stack growth. diff --git a/src/runtime/plugin.go b/src/runtime/plugin.go index f37854f915..a61dcc3b5d 100644 --- a/src/runtime/plugin.go +++ b/src/runtime/plugin.go @@ -7,7 +7,7 @@ package runtime import "unsafe" //go:linkname plugin_lastmoduleinit plugin.lastmoduleinit -func plugin_lastmoduleinit() (path string, syms map[string]interface{}, errstr string) { +func plugin_lastmoduleinit() (path string, syms map[string]any, errstr string) { var md *moduledata for pmd := firstmoduledata.next; pmd != nil; pmd = pmd.next { if pmd.bad { @@ -76,11 +76,11 @@ func plugin_lastmoduleinit() (path string, syms map[string]interface{}, errstr s // Because functions are handled specially in the plugin package, // function symbol names are prefixed here with '.' to avoid // a dependency on the reflect package. - syms = make(map[string]interface{}, len(md.ptab)) + syms = make(map[string]any, len(md.ptab)) for _, ptab := range md.ptab { symName := resolveNameOff(unsafe.Pointer(md.types), ptab.name) t := (*_type)(unsafe.Pointer(md.types)).typeOff(ptab.typ) - var val interface{} + var val any valp := (*[2]unsafe.Pointer)(unsafe.Pointer(&val)) (*valp)[0] = unsafe.Pointer(t) diff --git a/src/runtime/pprof/mprof_test.go b/src/runtime/pprof/mprof_test.go index ab8341d32f..665487a7c4 100644 --- a/src/runtime/pprof/mprof_test.go +++ b/src/runtime/pprof/mprof_test.go @@ -17,7 +17,7 @@ import ( "unsafe" ) -var memSink interface{} +var memSink any func allocateTransient1M() { for i := 0; i < 1024; i++ { diff --git a/src/runtime/pprof/pprof.go b/src/runtime/pprof/pprof.go index 000abf935c..d75efce7a8 100644 --- a/src/runtime/pprof/pprof.go +++ b/src/runtime/pprof/pprof.go @@ -134,7 +134,7 @@ import ( type Profile struct { name string mu sync.Mutex - m map[interface{}][]uintptr + m map[any][]uintptr count func() int write func(io.Writer, int) error } @@ -217,7 +217,7 @@ func NewProfile(name string) *Profile { } p := &Profile{ name: name, - m: map[interface{}][]uintptr{}, + m: map[any][]uintptr{}, } profiles.m[name] = p return p @@ -277,7 +277,7 @@ func (p *Profile) Count() int { // Passing skip=0 begins the stack trace at the call to Add inside rpc.NewClient. // Passing skip=1 begins the stack trace at the call to NewClient inside mypkg.Run. // -func (p *Profile) Add(value interface{}, skip int) { +func (p *Profile) Add(value any, skip int) { if p.name == "" { panic("pprof: use of uninitialized Profile") } @@ -303,7 +303,7 @@ func (p *Profile) Add(value interface{}, skip int) { // Remove removes the execution stack associated with value from the profile. // It is a no-op if the value is not in the profile. -func (p *Profile) Remove(value interface{}) { +func (p *Profile) Remove(value any) { p.mu.Lock() defer p.mu.Unlock() delete(p.m, value) diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index 417d5034a6..1a44ab7ad7 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -79,10 +79,6 @@ func cpuHog2(x int) int { return foo } -func cpuHog3(x int) int { - return cpuHog0(x, 1e5) -} - // Return a list of functions that we don't want to ever appear in CPU // profiles. For gccgo, that list includes the sigprof handler itself. func avoidFunctions() []string { @@ -93,14 +89,16 @@ func avoidFunctions() []string { } func TestCPUProfile(t *testing.T) { - testCPUProfile(t, stackContains, []string{"runtime/pprof.cpuHog1"}, avoidFunctions(), func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContains, []string{"runtime/pprof.cpuHog1"}, avoidFunctions()) + testCPUProfile(t, matches, func(dur time.Duration) { cpuHogger(cpuHog1, &salt1, dur) }) } func TestCPUProfileMultithreaded(t *testing.T) { defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2)) - testCPUProfile(t, stackContains, []string{"runtime/pprof.cpuHog1", "runtime/pprof.cpuHog2"}, avoidFunctions(), func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContains, []string{"runtime/pprof.cpuHog1", "runtime/pprof.cpuHog2"}, avoidFunctions()) + testCPUProfile(t, matches, func(dur time.Duration) { c := make(chan int) go func() { cpuHogger(cpuHog1, &salt1, dur) @@ -116,6 +114,30 @@ func TestCPUProfileMultithreadMagnitude(t *testing.T) { t.Skip("issue 35057 is only confirmed on Linux") } + // Linux [5.9,5.16) has a kernel bug that can break CPU timers on newly + // created threads, breaking our CPU accounting. + major, minor, patch, err := linuxKernelVersion() + if err != nil { + t.Errorf("Error determining kernel version: %v", err) + } + t.Logf("Running on Linux %d.%d.%d", major, minor, patch) + defer func() { + if t.Failed() { + t.Logf("Failure of this test may indicate that your system suffers from a known Linux kernel bug fixed on newer kernels. See https://golang.org/issue/49065.") + } + }() + + // Disable on affected builders to avoid flakiness, but otherwise keep + // it enabled to potentially warn users that they are on a broken + // kernel. + if testenv.Builder() != "" && (runtime.GOARCH == "386" || runtime.GOARCH == "amd64") { + have59 := major > 5 || (major == 5 && minor >= 9) + have516 := major > 5 || (major == 5 && minor >= 16) + if have59 && !have516 { + testenv.SkipFlaky(t, 49065) + } + } + // Run a workload in a single goroutine, then run copies of the same // workload in several goroutines. For both the serial and parallel cases, // the CPU time the process measures with its own profiler should match the @@ -132,90 +154,93 @@ func TestCPUProfileMultithreadMagnitude(t *testing.T) { maxDiff = 0.40 } - parallelism := runtime.GOMAXPROCS(0) - - // This test compares the process's total CPU time against the CPU - // profiler's view of time spent in direct execution of user code. - // Background work, especially from the garbage collector, adds noise to - // that measurement. Disable automatic triggering of the GC, and then - // request a complete GC cycle (up through sweep termination). - defer debug.SetGCPercent(debug.SetGCPercent(-1)) - runtime.GC() - - var cpuTime1, cpuTimeN time.Duration - p := testCPUProfile(t, stackContains, []string{"runtime/pprof.cpuHog1", "runtime/pprof.cpuHog3"}, avoidFunctions(), func(dur time.Duration) { - cpuTime1 = diffCPUTime(t, func() { - // Consume CPU in one goroutine - cpuHogger(cpuHog1, &salt1, dur) - }) - - cpuTimeN = diffCPUTime(t, func() { - // Next, consume CPU in several goroutines - var wg sync.WaitGroup - var once sync.Once - for i := 0; i < parallelism; i++ { - wg.Add(1) - go func() { - defer wg.Done() - var salt = 0 - cpuHogger(cpuHog3, &salt, dur) - once.Do(func() { salt1 = salt }) - }() - } - wg.Wait() - }) - }) - - for i, unit := range []string{"count", "nanoseconds"} { - if have, want := p.SampleType[i].Unit, unit; have != want { - t.Errorf("pN SampleType[%d]; %q != %q", i, have, want) + compare := func(a, b time.Duration, maxDiff float64) error { + if a <= 0 || b <= 0 { + return fmt.Errorf("Expected both time reports to be positive") } - } - var value1, valueN time.Duration - for _, sample := range p.Sample { - if stackContains("runtime/pprof.cpuHog1", uintptr(sample.Value[0]), sample.Location, sample.Label) { - value1 += time.Duration(sample.Value[1]) * time.Nanosecond + if a < b { + a, b = b, a } - if stackContains("runtime/pprof.cpuHog3", uintptr(sample.Value[0]), sample.Location, sample.Label) { - valueN += time.Duration(sample.Value[1]) * time.Nanosecond + + diff := float64(a-b) / float64(a) + if diff > maxDiff { + return fmt.Errorf("CPU usage reports are too different (limit -%.1f%%, got -%.1f%%)", maxDiff*100, diff*100) } + + return nil } - compare := func(a, b time.Duration, maxDiff float64) func(*testing.T) { - return func(t *testing.T) { - t.Logf("compare %s vs %s", a, b) - if a <= 0 || b <= 0 { - t.Errorf("Expected both time reports to be positive") - return + for _, tc := range []struct { + name string + workers int + }{ + { + name: "serial", + workers: 1, + }, + { + name: "parallel", + workers: runtime.GOMAXPROCS(0), + }, + } { + // check that the OS's perspective matches what the Go runtime measures. + t.Run(tc.name, func(t *testing.T) { + t.Logf("Running with %d workers", tc.workers) + + var cpuTime time.Duration + matches := matchAndAvoidStacks(stackContains, []string{"runtime/pprof.cpuHog1"}, avoidFunctions()) + p := testCPUProfile(t, matches, func(dur time.Duration) { + cpuTime = diffCPUTime(t, func() { + var wg sync.WaitGroup + var once sync.Once + for i := 0; i < tc.workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + var salt = 0 + cpuHogger(cpuHog1, &salt, dur) + once.Do(func() { salt1 = salt }) + }() + } + wg.Wait() + }) + }) + + for i, unit := range []string{"count", "nanoseconds"} { + if have, want := p.SampleType[i].Unit, unit; have != want { + t.Errorf("pN SampleType[%d]; %q != %q", i, have, want) + } } - if a < b { - a, b = b, a + // cpuHog1 called above is the primary source of CPU + // load, but there may be some background work by the + // runtime. Since the OS rusage measurement will + // include all work done by the process, also compare + // against all samples in our profile. + var value time.Duration + for _, sample := range p.Sample { + value += time.Duration(sample.Value[1]) * time.Nanosecond } - diff := float64(a-b) / float64(a) - if diff > maxDiff { - t.Errorf("CPU usage reports are too different (limit -%.1f%%, got -%.1f%%)", maxDiff*100, diff*100) + t.Logf("compare %s vs %s", cpuTime, value) + if err := compare(cpuTime, value, maxDiff); err != nil { + t.Errorf("compare got %v want nil", err) } - } + }) } - - // check that the OS's perspective matches what the Go runtime measures - t.Run("serial execution OS vs pprof", compare(cpuTime1, value1, maxDiff)) - t.Run("parallel execution OS vs pprof", compare(cpuTimeN, valueN, maxDiff)) } // containsInlinedCall reports whether the function body for the function f is // known to contain an inlined function call within the first maxBytes bytes. -func containsInlinedCall(f interface{}, maxBytes int) bool { +func containsInlinedCall(f any, maxBytes int) bool { _, found := findInlinedCall(f, maxBytes) return found } // findInlinedCall returns the PC of an inlined function call within // the function body for the function f if any. -func findInlinedCall(f interface{}, maxBytes int) (pc uint64, found bool) { +func findInlinedCall(f any, maxBytes int) (pc uint64, found bool) { fFunc := runtime.FuncForPC(uintptr(abi.FuncPCABIInternal(f))) if fFunc == nil || fFunc.Entry() == 0 { panic("failed to locate function entry") @@ -248,7 +273,8 @@ func TestCPUProfileInlining(t *testing.T) { t.Skip("Can't determine whether inlinedCallee was inlined into inlinedCaller.") } - p := testCPUProfile(t, stackContains, []string{"runtime/pprof.inlinedCallee", "runtime/pprof.inlinedCaller"}, avoidFunctions(), func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContains, []string{"runtime/pprof.inlinedCallee", "runtime/pprof.inlinedCaller"}, avoidFunctions()) + p := testCPUProfile(t, matches, func(dur time.Duration) { cpuHogger(inlinedCaller, &salt1, dur) }) @@ -298,7 +324,8 @@ func inlinedCalleeDump(pcs []uintptr) { } func TestCPUProfileRecursion(t *testing.T) { - p := testCPUProfile(t, stackContains, []string{"runtime/pprof.inlinedCallee", "runtime/pprof.recursionCallee", "runtime/pprof.recursionCaller"}, avoidFunctions(), func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContains, []string{"runtime/pprof.inlinedCallee", "runtime/pprof.recursionCallee", "runtime/pprof.recursionCaller"}, avoidFunctions()) + p := testCPUProfile(t, matches, func(dur time.Duration) { cpuHogger(recursionCaller, &salt1, dur) }) @@ -383,7 +410,7 @@ func cpuProfilingBroken() bool { // testCPUProfile runs f under the CPU profiler, checking for some conditions specified by need, // as interpreted by matches, and returns the parsed profile. -func testCPUProfile(t *testing.T, matches matchFunc, need []string, avoid []string, f func(dur time.Duration)) *profile.Profile { +func testCPUProfile(t *testing.T, matches profileMatchFunc, f func(dur time.Duration)) *profile.Profile { switch runtime.GOOS { case "darwin": out, err := exec.Command("uname", "-a").CombinedOutput() @@ -424,7 +451,7 @@ func testCPUProfile(t *testing.T, matches matchFunc, need []string, avoid []stri f(duration) StopCPUProfile() - if p, ok := profileOk(t, matches, need, avoid, prof, duration); ok { + if p, ok := profileOk(t, matches, prof, duration); ok { return p } @@ -480,35 +507,18 @@ func stackContains(spec string, count uintptr, stk []*profile.Location, labels m return false } -type matchFunc func(spec string, count uintptr, stk []*profile.Location, labels map[string][]string) bool +type sampleMatchFunc func(spec string, count uintptr, stk []*profile.Location, labels map[string][]string) bool -func profileOk(t *testing.T, matches matchFunc, need []string, avoid []string, prof bytes.Buffer, duration time.Duration) (_ *profile.Profile, ok bool) { +func profileOk(t *testing.T, matches profileMatchFunc, prof bytes.Buffer, duration time.Duration) (_ *profile.Profile, ok bool) { ok = true - // Check that profile is well formed, contains 'need', and does not contain - // anything from 'avoid'. - have := make([]uintptr, len(need)) - avoidSamples := make([]uintptr, len(avoid)) var samples uintptr var buf bytes.Buffer p := parseProfile(t, prof.Bytes(), func(count uintptr, stk []*profile.Location, labels map[string][]string) { fmt.Fprintf(&buf, "%d:", count) fprintStack(&buf, stk) + fmt.Fprintf(&buf, " labels: %v\n", labels) samples += count - for i, spec := range need { - if matches(spec, count, stk, labels) { - have[i] += count - } - } - for i, name := range avoid { - for _, loc := range stk { - for _, line := range loc.Line { - if strings.Contains(line.Function.Name, name) { - avoidSamples[i] += count - } - } - } - } fmt.Fprintf(&buf, "\n") }) t.Logf("total %d CPU profile samples collected:\n%s", samples, buf.String()) @@ -531,39 +541,77 @@ func profileOk(t *testing.T, matches matchFunc, need []string, avoid []string, p ok = false } - for i, name := range avoid { - bad := avoidSamples[i] - if bad != 0 { - t.Logf("found %d samples in avoid-function %s\n", bad, name) - ok = false - } + if matches != nil && !matches(t, p) { + ok = false } - if len(need) == 0 { - return p, ok - } + return p, ok +} - var total uintptr - for i, name := range need { - total += have[i] - t.Logf("%s: %d\n", name, have[i]) - } - if total == 0 { - t.Logf("no samples in expected functions") - ok = false - } - // We'd like to check a reasonable minimum, like - // total / len(have) / smallconstant, but this test is - // pretty flaky (see bug 7095). So we'll just test to - // make sure we got at least one sample. - min := uintptr(1) - for i, name := range need { - if have[i] < min { - t.Logf("%s has %d samples out of %d, want at least %d, ideally %d", name, have[i], total, min, total/uintptr(len(have))) +type profileMatchFunc func(*testing.T, *profile.Profile) bool + +func matchAndAvoidStacks(matches sampleMatchFunc, need []string, avoid []string) profileMatchFunc { + return func(t *testing.T, p *profile.Profile) (ok bool) { + ok = true + + // Check that profile is well formed, contains 'need', and does not contain + // anything from 'avoid'. + have := make([]uintptr, len(need)) + avoidSamples := make([]uintptr, len(avoid)) + + for _, sample := range p.Sample { + count := uintptr(sample.Value[0]) + for i, spec := range need { + if matches(spec, count, sample.Location, sample.Label) { + have[i] += count + } + } + for i, name := range avoid { + for _, loc := range sample.Location { + for _, line := range loc.Line { + if strings.Contains(line.Function.Name, name) { + avoidSamples[i] += count + } + } + } + } + } + + for i, name := range avoid { + bad := avoidSamples[i] + if bad != 0 { + t.Logf("found %d samples in avoid-function %s\n", bad, name) + ok = false + } + } + + if len(need) == 0 { + return + } + + var total uintptr + for i, name := range need { + total += have[i] + t.Logf("%s: %d\n", name, have[i]) + } + if total == 0 { + t.Logf("no samples in expected functions") ok = false } + + // We'd like to check a reasonable minimum, like + // total / len(have) / smallconstant, but this test is + // pretty flaky (see bug 7095). So we'll just test to + // make sure we got at least one sample. + min := uintptr(1) + for i, name := range need { + if have[i] < min { + t.Logf("%s has %d samples out of %d, want at least %d, ideally %d", name, have[i], total, min, total/uintptr(len(have))) + ok = false + } + } + return } - return p, ok } // Fork can hang if preempted with signals frequently enough (see issue 5517). @@ -675,12 +723,11 @@ func fprintStack(w io.Writer, stk []*profile.Location) { } fmt.Fprintf(w, ")") } - fmt.Fprintf(w, "\n") } // Test that profiling of division operations is okay, especially on ARM. See issue 6681. func TestMathBigDivide(t *testing.T) { - testCPUProfile(t, nil, nil, nil, func(duration time.Duration) { + testCPUProfile(t, nil, func(duration time.Duration) { t := time.After(duration) pi := new(big.Int) for { @@ -709,7 +756,8 @@ func stackContainsAll(spec string, count uintptr, stk []*profile.Location, label } func TestMorestack(t *testing.T) { - testCPUProfile(t, stackContainsAll, []string{"runtime.newstack,runtime/pprof.growstack"}, avoidFunctions(), func(duration time.Duration) { + matches := matchAndAvoidStacks(stackContainsAll, []string{"runtime.newstack,runtime/pprof.growstack"}, avoidFunctions()) + testCPUProfile(t, matches, func(duration time.Duration) { t := time.After(duration) c := make(chan bool) for { @@ -1340,7 +1388,8 @@ func stackContainsLabeled(spec string, count uintptr, stk []*profile.Location, l } func TestCPUProfileLabel(t *testing.T) { - testCPUProfile(t, stackContainsLabeled, []string{"runtime/pprof.cpuHogger;key=value"}, avoidFunctions(), func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContainsLabeled, []string{"runtime/pprof.cpuHogger;key=value"}, avoidFunctions()) + testCPUProfile(t, matches, func(dur time.Duration) { Do(context.Background(), Labels("key", "value"), func(context.Context) { cpuHogger(cpuHog1, &salt1, dur) }) @@ -1351,7 +1400,8 @@ func TestLabelRace(t *testing.T) { // Test the race detector annotations for synchronization // between settings labels and consuming them from the // profile. - testCPUProfile(t, stackContainsLabeled, []string{"runtime/pprof.cpuHogger;key=value"}, nil, func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContainsLabeled, []string{"runtime/pprof.cpuHogger;key=value"}, nil) + testCPUProfile(t, matches, func(dur time.Duration) { start := time.Now() var wg sync.WaitGroup for time.Since(start) < dur { @@ -1370,6 +1420,126 @@ func TestLabelRace(t *testing.T) { }) } +// TestLabelSystemstack makes sure CPU profiler samples of goroutines running +// on systemstack include the correct pprof labels. See issue #48577 +func TestLabelSystemstack(t *testing.T) { + // Grab and re-set the initial value before continuing to ensure + // GOGC doesn't actually change following the test. + gogc := debug.SetGCPercent(100) + debug.SetGCPercent(gogc) + + matches := matchAndAvoidStacks(stackContainsLabeled, []string{"runtime.systemstack;key=value"}, avoidFunctions()) + p := testCPUProfile(t, matches, func(dur time.Duration) { + Do(context.Background(), Labels("key", "value"), func(ctx context.Context) { + parallelLabelHog(ctx, dur, gogc) + }) + }) + + // Two conditions to check: + // * labelHog should always be labeled. + // * The label should _only_ appear on labelHog and the Do call above. + for _, s := range p.Sample { + isLabeled := s.Label != nil && contains(s.Label["key"], "value") + var ( + mayBeLabeled bool + mustBeLabeled bool + mustNotBeLabeled bool + ) + for _, loc := range s.Location { + for _, l := range loc.Line { + switch l.Function.Name { + case "runtime/pprof.labelHog", "runtime/pprof.parallelLabelHog", "runtime/pprof.parallelLabelHog.func1": + mustBeLabeled = true + case "runtime/pprof.Do": + // Do sets the labels, so samples may + // or may not be labeled depending on + // which part of the function they are + // at. + mayBeLabeled = true + case "runtime.bgsweep", "runtime.bgscavenge", "runtime.forcegchelper", "runtime.gcBgMarkWorker", "runtime.runfinq", "runtime.sysmon": + // Runtime system goroutines or threads + // (such as those identified by + // runtime.isSystemGoroutine). These + // should never be labeled. + mustNotBeLabeled = true + case "gogo", "gosave_systemstack_switch", "racecall": + // These are context switch/race + // critical that we can't do a full + // traceback from. Typically this would + // be covered by the runtime check + // below, but these symbols don't have + // the package name. + mayBeLabeled = true + } + + if strings.HasPrefix(l.Function.Name, "runtime.") { + // There are many places in the runtime + // where we can't do a full traceback. + // Ideally we'd list them all, but + // barring that allow anything in the + // runtime, unless explicitly excluded + // above. + mayBeLabeled = true + } + } + } + if mustNotBeLabeled { + // If this must not be labeled, then mayBeLabeled hints + // are not relevant. + mayBeLabeled = false + } + if mustBeLabeled && !isLabeled { + var buf bytes.Buffer + fprintStack(&buf, s.Location) + t.Errorf("Sample labeled got false want true: %s", buf.String()) + } + if mustNotBeLabeled && isLabeled { + var buf bytes.Buffer + fprintStack(&buf, s.Location) + t.Errorf("Sample labeled got true want false: %s", buf.String()) + } + if isLabeled && !(mayBeLabeled || mustBeLabeled) { + var buf bytes.Buffer + fprintStack(&buf, s.Location) + t.Errorf("Sample labeled got true want false: %s", buf.String()) + } + } +} + +// labelHog is designed to burn CPU time in a way that a high number of CPU +// samples end up running on systemstack. +func labelHog(stop chan struct{}, gogc int) { + // Regression test for issue 50032. We must give GC an opportunity to + // be initially triggered by a labelled goroutine. + runtime.GC() + + for i := 0; ; i++ { + select { + case <-stop: + return + default: + debug.SetGCPercent(gogc) + } + } +} + +// parallelLabelHog runs GOMAXPROCS goroutines running labelHog. +func parallelLabelHog(ctx context.Context, dur time.Duration, gogc int) { + var wg sync.WaitGroup + stop := make(chan struct{}) + for i := 0; i < runtime.GOMAXPROCS(0); i++ { + wg.Add(1) + go func() { + defer wg.Done() + labelHog(stop, gogc) + }() + } + + time.Sleep(dur) + close(stop) + wg.Wait() +} + // Check that there is no deadlock when the program receives SIGPROF while in // 64bit atomics' critical section. Used to happen on mips{,le}. See #20146. func TestAtomicLoadStore64(t *testing.T) { @@ -1476,6 +1646,7 @@ func TestTryAdd(t *testing.T) { testCases := []struct { name string input []uint64 // following the input format assumed by profileBuilder.addCPUData. + count int // number of records in input. wantLocs [][]string // ordered location entries with function names. wantSamples []*profile.Sample // ordered samples, we care only about Value and the profile location IDs. }{{ @@ -1485,6 +1656,7 @@ func TestTryAdd(t *testing.T) { 3, 0, 500, // hz = 500. Must match the period. 5, 0, 50, inlinedCallerStack[0], inlinedCallerStack[1], }, + count: 2, wantLocs: [][]string{ {"runtime/pprof.inlinedCalleeDump", "runtime/pprof.inlinedCallerDump"}, }, @@ -1501,6 +1673,7 @@ func TestTryAdd(t *testing.T) { 7, 0, 10, inlinedCallerStack[0], inlinedCallerStack[1], inlinedCallerStack[0], inlinedCallerStack[1], 5, 0, 20, inlinedCallerStack[0], inlinedCallerStack[1], }, + count: 3, wantLocs: [][]string{{"runtime/pprof.inlinedCalleeDump", "runtime/pprof.inlinedCallerDump"}}, wantSamples: []*profile.Sample{ {Value: []int64{10, 10 * period}, Location: []*profile.Location{{ID: 1}, {ID: 1}}}, @@ -1514,6 +1687,7 @@ func TestTryAdd(t *testing.T) { // entry. The "stk" entry is actually the count. 4, 0, 0, 4242, }, + count: 2, wantLocs: [][]string{{"runtime/pprof.lostProfileEvent"}}, wantSamples: []*profile.Sample{ {Value: []int64{4242, 4242 * period}, Location: []*profile.Location{{ID: 1}}}, @@ -1532,6 +1706,7 @@ func TestTryAdd(t *testing.T) { 5, 0, 30, inlinedCallerStack[0], inlinedCallerStack[0], 4, 0, 40, inlinedCallerStack[0], }, + count: 3, // inlinedCallerDump shows up here because // runtime_expandFinalInlineFrame adds it to the stack frame. wantLocs: [][]string{{"runtime/pprof.inlinedCalleeDump"}, {"runtime/pprof.inlinedCallerDump"}}, @@ -1545,6 +1720,7 @@ func TestTryAdd(t *testing.T) { 3, 0, 500, // hz = 500. Must match the period. 9, 0, 10, recursionStack[0], recursionStack[1], recursionStack[2], recursionStack[3], recursionStack[4], recursionStack[5], }, + count: 2, wantLocs: [][]string{ {"runtime/pprof.recursionChainBottom"}, { @@ -1568,6 +1744,7 @@ func TestTryAdd(t *testing.T) { 5, 0, 50, inlinedCallerStack[0], inlinedCallerStack[1], 4, 0, 60, inlinedCallerStack[0], }, + count: 3, wantLocs: [][]string{{"runtime/pprof.inlinedCalleeDump", "runtime/pprof.inlinedCallerDump"}}, wantSamples: []*profile.Sample{ {Value: []int64{50, 50 * period}, Location: []*profile.Location{{ID: 1}}}, @@ -1580,6 +1757,7 @@ func TestTryAdd(t *testing.T) { 4, 0, 70, inlinedCallerStack[0], 5, 0, 80, inlinedCallerStack[0], inlinedCallerStack[1], }, + count: 3, wantLocs: [][]string{{"runtime/pprof.inlinedCalleeDump", "runtime/pprof.inlinedCallerDump"}}, wantSamples: []*profile.Sample{ {Value: []int64{70, 70 * period}, Location: []*profile.Location{{ID: 1}}}, @@ -1592,6 +1770,7 @@ func TestTryAdd(t *testing.T) { 3, 0, 500, // hz = 500. Must match the period. 4, 0, 70, inlinedCallerStack[0], }, + count: 2, wantLocs: [][]string{{"runtime/pprof.inlinedCalleeDump", "runtime/pprof.inlinedCallerDump"}}, wantSamples: []*profile.Sample{ {Value: []int64{70, 70 * period}, Location: []*profile.Location{{ID: 1}}}, @@ -1607,6 +1786,7 @@ func TestTryAdd(t *testing.T) { // from getting merged into above. 5, 0, 80, inlinedCallerStack[1], inlinedCallerStack[0], }, + count: 3, wantLocs: [][]string{ {"runtime/pprof.inlinedCalleeDump", "runtime/pprof.inlinedCallerDump"}, {"runtime/pprof.inlinedCallerDump"}, @@ -1619,7 +1799,7 @@ func TestTryAdd(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - p, err := translateCPUProfile(tc.input) + p, err := translateCPUProfile(tc.input, tc.count) if err != nil { t.Fatalf("translating profile: %v", err) } @@ -1662,7 +1842,8 @@ func TestTimeVDSO(t *testing.T) { testenv.SkipFlaky(t, 48655) } - p := testCPUProfile(t, stackContains, []string{"time.now"}, avoidFunctions(), func(dur time.Duration) { + matches := matchAndAvoidStacks(stackContains, []string{"time.now"}, avoidFunctions()) + p := testCPUProfile(t, matches, func(dur time.Duration) { t0 := time.Now() for { t := time.Now() diff --git a/src/runtime/pprof/proto.go b/src/runtime/pprof/proto.go index 54e7a80183..073a076802 100644 --- a/src/runtime/pprof/proto.go +++ b/src/runtime/pprof/proto.go @@ -266,8 +266,9 @@ func newProfileBuilder(w io.Writer) *profileBuilder { } // addCPUData adds the CPU profiling data to the profile. -// The data must be a whole number of records, -// as delivered by the runtime. +// +// The data must be a whole number of records, as delivered by the runtime. +// len(tags) must be equal to the number of records in data. func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error { if !b.havePeriod { // first record is period @@ -282,6 +283,9 @@ func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error b.period = 1e9 / int64(data[2]) b.havePeriod = true data = data[3:] + // Consume tag slot. Note that there isn't a meaningful tag + // value for this record. + tags = tags[1:] } // Parse CPU samples from the profile. @@ -306,14 +310,14 @@ func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error if data[0] < 3 || tags != nil && len(tags) < 1 { return fmt.Errorf("malformed profile") } + if len(tags) < 1 { + return fmt.Errorf("mismatched profile records and tags") + } count := data[2] stk := data[3:data[0]] data = data[data[0]:] - var tag unsafe.Pointer - if tags != nil { - tag = tags[0] - tags = tags[1:] - } + tag := tags[0] + tags = tags[1:] if count == 0 && len(stk) == 1 { // overflow record @@ -327,6 +331,10 @@ func (b *profileBuilder) addCPUData(data []uint64, tags []unsafe.Pointer) error } b.m.lookup(stk, tag).count += int64(count) } + + if len(tags) != 0 { + return fmt.Errorf("mismatched profile records and tags") + } return nil } diff --git a/src/runtime/pprof/proto_test.go b/src/runtime/pprof/proto_test.go index 4a9749a83f..84a051a536 100644 --- a/src/runtime/pprof/proto_test.go +++ b/src/runtime/pprof/proto_test.go @@ -17,16 +17,20 @@ import ( "runtime" "strings" "testing" + "unsafe" ) // translateCPUProfile parses binary CPU profiling stack trace data // generated by runtime.CPUProfile() into a profile struct. // This is only used for testing. Real conversions stream the // data into the profileBuilder as it becomes available. -func translateCPUProfile(data []uint64) (*profile.Profile, error) { +// +// count is the number of records in data. +func translateCPUProfile(data []uint64, count int) (*profile.Profile, error) { var buf bytes.Buffer b := newProfileBuilder(&buf) - if err := b.addCPUData(data, nil); err != nil { + tags := make([]unsafe.Pointer, count) + if err := b.addCPUData(data, tags); err != nil { return nil, err } b.build() @@ -36,7 +40,7 @@ func translateCPUProfile(data []uint64) (*profile.Profile, error) { // fmtJSON returns a pretty-printed JSON form for x. // It works reasonbly well for printing protocol-buffer // data structures like profile.Profile. -func fmtJSON(x interface{}) string { +func fmtJSON(x any) string { js, _ := json.MarshalIndent(x, "", "\t") return string(js) } @@ -46,7 +50,7 @@ func TestConvertCPUProfileEmpty(t *testing.T) { var buf bytes.Buffer b := []uint64{3, 0, 500} // empty profile at 500 Hz (2ms sample period) - p, err := translateCPUProfile(b) + p, err := translateCPUProfile(b, 1) if err != nil { t.Fatalf("translateCPUProfile: %v", err) } @@ -120,7 +124,7 @@ func TestConvertCPUProfile(t *testing.T) { 5, 0, 40, uint64(addr2 + 1), uint64(addr2 + 2), // 40 samples in addr2 5, 0, 10, uint64(addr1 + 1), uint64(addr1 + 2), // 10 samples in addr1 } - p, err := translateCPUProfile(b) + p, err := translateCPUProfile(b, 4) if err != nil { t.Fatalf("translating profile: %v", err) } @@ -429,7 +433,7 @@ func TestEmptyStack(t *testing.T) { 3, 0, 500, // hz = 500 3, 0, 10, // 10 samples with an empty stack trace } - _, err := translateCPUProfile(b) + _, err := translateCPUProfile(b, 2) if err != nil { t.Fatalf("translating profile: %v", err) } diff --git a/src/runtime/pprof/uname_linux_test.go b/src/runtime/pprof/uname_linux_test.go new file mode 100644 index 0000000000..8374c83f74 --- /dev/null +++ b/src/runtime/pprof/uname_linux_test.go @@ -0,0 +1,61 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux + +package pprof + +import ( + "fmt" + "regexp" + "strconv" + "syscall" +) + +var versionRe = regexp.MustCompile(`^(\d+)(?:\.(\d+)(?:\.(\d+))).*$`) + +func linuxKernelVersion() (major, minor, patch int, err error) { + var uname syscall.Utsname + if err := syscall.Uname(&uname); err != nil { + return 0, 0, 0, err + } + + buf := make([]byte, 0, len(uname.Release)) + for _, b := range uname.Release { + if b == 0 { + break + } + buf = append(buf, byte(b)) + } + rl := string(buf) + + m := versionRe.FindStringSubmatch(rl) + if m == nil { + return 0, 0, 0, fmt.Errorf("error matching version number in %q", rl) + } + + v, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("error parsing major version %q in %s: %w", m[1], rl, err) + } + major = int(v) + + if len(m) >= 3 { + v, err := strconv.ParseInt(m[2], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("error parsing minor version %q in %s: %w", m[2], rl, err) + } + minor = int(v) + } + + if len(m) >= 4 { + v, err := strconv.ParseInt(m[3], 10, 64) + if err != nil { + return 0, 0, 0, fmt.Errorf("error parsing patch version %q in %s: %w", m[3], rl, err) + } + patch = int(v) + } + + return +} diff --git a/src/runtime/pprof/uname_other_test.go b/src/runtime/pprof/uname_other_test.go new file mode 100644 index 0000000000..327640755b --- /dev/null +++ b/src/runtime/pprof/uname_other_test.go @@ -0,0 +1,15 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !linux + +package pprof + +import ( + "errors" +) + +func linuxKernelVersion() (major, minor, patch int, err error) { + return 0, 0, 0, errors.New("not running on linux") +} diff --git a/src/runtime/preempt_mips64x.s b/src/runtime/preempt_mips64x.s index c1249e382e..996b592ae0 100644 --- a/src/runtime/preempt_mips64x.s +++ b/src/runtime/preempt_mips64x.s @@ -1,7 +1,6 @@ // Code generated by mkpreempt.go; DO NOT EDIT. //go:build mips64 || mips64le -// +build mips64 mips64le #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/preempt_mipsx.s b/src/runtime/preempt_mipsx.s index 70b79e05b9..7b169acd99 100644 --- a/src/runtime/preempt_mipsx.s +++ b/src/runtime/preempt_mipsx.s @@ -1,7 +1,6 @@ // Code generated by mkpreempt.go; DO NOT EDIT. //go:build mips || mipsle -// +build mips mipsle #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/preempt_ppc64x.s b/src/runtime/preempt_ppc64x.s index 7ed4021dde..2c4d02edfe 100644 --- a/src/runtime/preempt_ppc64x.s +++ b/src/runtime/preempt_ppc64x.s @@ -1,7 +1,6 @@ // Code generated by mkpreempt.go; DO NOT EDIT. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/proc.go b/src/runtime/proc.go index bf5fa8e4fc..1be7a60830 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -547,6 +547,20 @@ func allgadd(gp *g) { unlock(&allglock) } +// allGsSnapshot returns a snapshot of the slice of all Gs. +// +// The world must be stopped or allglock must be held. +func allGsSnapshot() []*g { + assertWorldStoppedOrLockHeld(&allglock) + + // Because the world is stopped or allglock is held, allgadd + // cannot happen concurrently with this. allgs grows + // monotonically and existing entries never change, so we can + // simply return a copy of the slice header. For added safety, + // we trim everything past len because that can still change. + return allgs[:len(allgs):len(allgs)] +} + // atomicAllG returns &allgs[0] and len(allgs) for use with atomicAllGIndex. func atomicAllG() (**g, uintptr) { length := atomic.Loaduintptr(&allglen) @@ -980,17 +994,18 @@ func casgstatus(gp *g, oldval, newval uint32) { gp.trackingSeq++ } if gp.tracking { - now := nanotime() if oldval == _Grunnable { // We transitioned out of runnable, so measure how much // time we spent in this state and add it to // runnableTime. + now := nanotime() gp.runnableTime += now - gp.runnableStamp gp.runnableStamp = 0 } if newval == _Grunnable { // We just transitioned into runnable, so record what // time that happened. + now := nanotime() gp.runnableStamp = now } else if newval == _Grunning { // We're transitioning into running, so turn off @@ -4285,11 +4300,13 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr) *g { newg.gopc = callerpc newg.ancestors = saveAncestors(callergp) newg.startpc = fn.fn - if _g_.m.curg != nil { - newg.labels = _g_.m.curg.labels - } if isSystemGoroutine(newg, false) { atomic.Xadd(&sched.ngsys, +1) + } else { + // Only user goroutines inherit pprof labels. + if _g_.m.curg != nil { + newg.labels = _g_.m.curg.labels + } } // Track initial transition? newg.trackingSeq = uint8(fastrand()) @@ -4654,7 +4671,6 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { getg().m.mallocing++ var stk [maxCPUProfStack]uintptr - flags := uint(_TraceJumpStack) n := 0 if mp.ncgo > 0 && mp.curg != nil && mp.curg.syscallpc != 0 && mp.curg.syscallsp != 0 { cgoOff := 0 @@ -4672,12 +4688,12 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { } // Collect Go stack that leads to the cgo call. - n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[cgoOff], len(stk)-cgoOff, nil, nil, flags) + n = gentraceback(mp.curg.syscallpc, mp.curg.syscallsp, 0, mp.curg, 0, &stk[cgoOff], len(stk)-cgoOff, nil, nil, 0) if n > 0 { n += cgoOff } } else { - n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap|flags) + n = gentraceback(pc, sp, lr, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap|_TraceJumpStack) } if n <= 0 { @@ -4687,10 +4703,10 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { if usesLibcall() && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 { // Libcall, i.e. runtime syscall on windows. // Collect Go stack that leads to the call. - n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, flags) + n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0) } if n == 0 && mp != nil && mp.vdsoSP != 0 { - n = gentraceback(mp.vdsoPC, mp.vdsoSP, 0, gp, 0, &stk[0], len(stk), nil, nil, flags) + n = gentraceback(mp.vdsoPC, mp.vdsoSP, 0, gp, 0, &stk[0], len(stk), nil, nil, _TraceTrap|_TraceJumpStack) } if n == 0 { // If all of the above has failed, account it against abstract "System" or "GC". @@ -4711,7 +4727,14 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { } if prof.hz != 0 { - cpuprof.add(gp, stk[:n]) + // Note: it can happen on Windows that we interrupted a system thread + // with no g, so gp could nil. The other nil checks are done out of + // caution, but not expected to be nil in practice. + var tagPtr *unsafe.Pointer + if gp != nil && gp.m != nil && gp.m.curg != nil { + tagPtr = &gp.m.curg.labels + } + cpuprof.add(tagPtr, stk[:n]) } getg().m.mallocing-- } diff --git a/src/runtime/proc_test.go b/src/runtime/proc_test.go index 53cafe8907..719d0d1aee 100644 --- a/src/runtime/proc_test.go +++ b/src/runtime/proc_test.go @@ -119,6 +119,10 @@ func TestGoroutineParallelism(t *testing.T) { // since the goroutines can't be stopped/preempted. // Disable GC for this test (see issue #10958). defer debug.SetGCPercent(debug.SetGCPercent(-1)) + // SetGCPercent waits until the mark phase is over, but the runtime + // also preempts at the start of the sweep phase, so make sure that's + // done too. See #45867. + runtime.GC() for try := 0; try < N; try++ { done := make(chan bool) x := uint32(0) @@ -163,6 +167,10 @@ func testGoroutineParallelism2(t *testing.T, load, netpoll bool) { // since the goroutines can't be stopped/preempted. // Disable GC for this test (see issue #10958). defer debug.SetGCPercent(debug.SetGCPercent(-1)) + // SetGCPercent waits until the mark phase is over, but the runtime + // also preempts at the start of the sweep phase, so make sure that's + // done too. See #45867. + runtime.GC() for try := 0; try < N; try++ { if load { // Create P goroutines and wait until they all run. @@ -623,6 +631,10 @@ func TestSchedLocalQueueEmpty(t *testing.T) { // If runtime triggers a forced GC during this test then it will deadlock, // since the goroutines can't be stopped/preempted during spin wait. defer debug.SetGCPercent(debug.SetGCPercent(-1)) + // SetGCPercent waits until the mark phase is over, but the runtime + // also preempts at the start of the sweep phase, so make sure that's + // done too. See #45867. + runtime.GC() iters := int(1e5) if testing.Short() { @@ -1032,7 +1044,7 @@ func testPreemptionAfterSyscall(t *testing.T, syscallDuration time.Duration) { interations = 1 } const ( - maxDuration = 3 * time.Second + maxDuration = 5 * time.Second nroutines = 8 ) @@ -1068,6 +1080,10 @@ func testPreemptionAfterSyscall(t *testing.T, syscallDuration time.Duration) { } func TestPreemptionAfterSyscall(t *testing.T) { + if runtime.GOOS == "plan9" { + testenv.SkipFlaky(t, 41015) + } + for _, i := range []time.Duration{10, 100, 1000} { d := i * time.Microsecond t.Run(fmt.Sprint(d), func(t *testing.T) { diff --git a/src/runtime/race/README b/src/runtime/race/README index 3b188a0361..d3c55182ef 100644 --- a/src/runtime/race/README +++ b/src/runtime/race/README @@ -4,12 +4,12 @@ the LLVM project (https://github.com/llvm/llvm-project/tree/main/compiler-rt). To update the .syso files use golang.org/x/build/cmd/racebuild. -race_darwin_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. +race_darwin_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 with https://reviews.llvm.org/D114825 applied and Go 7ccbcc90560468937f02609a43cb39a6e13ff797. race_freebsd_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. race_linux_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. race_linux_ppc64le.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. race_netbsd_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. race_windows_amd64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. race_linux_arm64.syso built with LLVM 89f7ccea6f6488c443655880229c54db1f180153 and Go f62d3202bf9dbb3a00ad2a2c63ff4fa4188c5d3b. -race_darwin_arm64.syso built with LLVM 00da38ce2d36c07f12c287dc515d37bb7bc410e9 and Go fe70a3a0fd31441bcbb9932ecab11a6083cf2119. +race_darwin_arm64.syso built with LLVM 00da38ce2d36c07f12c287dc515d37bb7bc410e9 with https://reviews.llvm.org/D114825 applied and Go 7ccbcc90560468937f02609a43cb39a6e13ff797. race_openbsd_amd64.syso built with LLVM fcf6ae2f070eba73074b6ec8d8281e54d29dbeeb and Go 8f2db14cd35bbd674cb2988a508306de6655e425. diff --git a/src/runtime/race/output_test.go b/src/runtime/race/output_test.go index 46cdfcd0e9..0dcdabe641 100644 --- a/src/runtime/race/output_test.go +++ b/src/runtime/race/output_test.go @@ -207,7 +207,7 @@ func TestFail(t *testing.T) { } `, []string{` ================== ---- FAIL: TestFail \(0...s\) +--- FAIL: TestFail \([0-9.]+s\) .*main_test.go:14: true .*testing.go:.*: race detected during execution of test FAIL`}}, @@ -363,7 +363,7 @@ func TestPass(t *testing.T) { } `, []string{` ================== ---- FAIL: TestFail \(0...s\) +--- FAIL: TestFail \([0-9.]+s\) .*testing.go:.*: race detected during execution of test FAIL`}}, {"mutex", "run", "", "atexit_sleep_ms=0", ` diff --git a/src/runtime/race/race_darwin_amd64.syso b/src/runtime/race/race_darwin_amd64.syso Binary files differindex 3f95ecc8ee..6fbe140026 100644 --- a/src/runtime/race/race_darwin_amd64.syso +++ b/src/runtime/race/race_darwin_amd64.syso diff --git a/src/runtime/race/race_darwin_arm64.syso b/src/runtime/race/race_darwin_arm64.syso Binary files differindex f6eaa62ae3..207099eb1d 100644 --- a/src/runtime/race/race_darwin_arm64.syso +++ b/src/runtime/race/race_darwin_arm64.syso diff --git a/src/runtime/race/race_test.go b/src/runtime/race/race_test.go index 1677e13986..4fe61683eb 100644 --- a/src/runtime/race/race_test.go +++ b/src/runtime/race/race_test.go @@ -187,7 +187,7 @@ func runTests(t *testing.T) ([]byte, error) { func TestIssue8102(t *testing.T) { // If this compiles with -race, the test passes. type S struct { - x interface{} + x any i int } c := make(chan int) diff --git a/src/runtime/race/testdata/issue12664_test.go b/src/runtime/race/testdata/issue12664_test.go index c9f790edc8..714e83d318 100644 --- a/src/runtime/race/testdata/issue12664_test.go +++ b/src/runtime/race/testdata/issue12664_test.go @@ -56,7 +56,7 @@ func TestRaceIssue12664_3(t *testing.T) { close(c) }() var r MyT - var i interface{} = r + var i any = r issue12664_3 = i.(MyT) <-c } diff --git a/src/runtime/race/testdata/mop_test.go b/src/runtime/race/testdata/mop_test.go index b60cabfe86..94b6e58de0 100644 --- a/src/runtime/race/testdata/mop_test.go +++ b/src/runtime/race/testdata/mop_test.go @@ -255,7 +255,7 @@ func TestRaceCaseIssue6418(t *testing.T) { func TestRaceCaseType(t *testing.T) { var x, y int - var i interface{} = x + var i any = x c := make(chan int, 1) go func() { switch i.(type) { @@ -270,7 +270,7 @@ func TestRaceCaseType(t *testing.T) { func TestRaceCaseTypeBody(t *testing.T) { var x, y int - var i interface{} = &x + var i any = &x c := make(chan int, 1) go func() { switch i := i.(type) { @@ -288,8 +288,8 @@ func TestRaceCaseTypeIssue5890(t *testing.T) { // spurious extra instrumentation of the initial interface // value. var x, y int - m := make(map[int]map[int]interface{}) - m[0] = make(map[int]interface{}) + m := make(map[int]map[int]any) + m[0] = make(map[int]any) c := make(chan int, 1) go func() { switch i := m[0][1].(type) { @@ -758,7 +758,7 @@ func TestRaceStructFieldRW3(t *testing.T) { } func TestRaceEfaceWW(t *testing.T) { - var a, b interface{} + var a, b any ch := make(chan bool, 1) go func() { a = 1 @@ -810,7 +810,7 @@ func TestRaceEfaceConv(t *testing.T) { c := make(chan bool) v := 0 go func() { - go func(x interface{}) { + go func(x any) { }(v) c <- true }() @@ -1127,7 +1127,7 @@ func TestRaceRune(t *testing.T) { func TestRaceEmptyInterface1(t *testing.T) { c := make(chan bool) - var x interface{} + var x any go func() { x = nil c <- true @@ -1138,7 +1138,7 @@ func TestRaceEmptyInterface1(t *testing.T) { func TestRaceEmptyInterface2(t *testing.T) { c := make(chan bool) - var x interface{} + var x any go func() { x = &Point{} c <- true @@ -1579,7 +1579,7 @@ func TestRaceAddrExpr(t *testing.T) { func TestRaceTypeAssert(t *testing.T) { c := make(chan bool, 1) x := 0 - var i interface{} = x + var i any = x go func() { y := 0 i = y diff --git a/src/runtime/race/testdata/pool_test.go b/src/runtime/race/testdata/pool_test.go index 161f4b7c23..a96913e381 100644 --- a/src/runtime/race/testdata/pool_test.go +++ b/src/runtime/race/testdata/pool_test.go @@ -15,7 +15,7 @@ func TestRacePool(t *testing.T) { // Repeat so that at least one iteration gets reuse. for i := 0; i < 10; i++ { c := make(chan int) - p := &sync.Pool{New: func() interface{} { return make([]byte, 10) }} + p := &sync.Pool{New: func() any { return make([]byte, 10) }} x := p.Get().([]byte) x[0] = 1 p.Put(x) @@ -31,7 +31,7 @@ func TestRacePool(t *testing.T) { func TestNoRacePool(t *testing.T) { for i := 0; i < 10; i++ { - p := &sync.Pool{New: func() interface{} { return make([]byte, 10) }} + p := &sync.Pool{New: func() any { return make([]byte, 10) }} x := p.Get().([]byte) x[0] = 1 p.Put(x) diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index d42e415dca..f055acf77d 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build race -// +build race #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/race_arm64.s b/src/runtime/race_arm64.s index 2b2413b6b7..798e23294a 100644 --- a/src/runtime/race_arm64.s +++ b/src/runtime/race_arm64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build race -// +build race #include "go_asm.h" #include "funcdata.h" diff --git a/src/runtime/race_ppc64le.s b/src/runtime/race_ppc64le.s index 625c81a255..68cc5c8805 100644 --- a/src/runtime/race_ppc64le.s +++ b/src/runtime/race_ppc64le.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build race -// +build race #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/rt0_linux_mips64x.s b/src/runtime/rt0_linux_mips64x.s index fabd8570b5..e9328b7326 100644 --- a/src/runtime/rt0_linux_mips64x.s +++ b/src/runtime/rt0_linux_mips64x.s @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. //go:build linux && (mips64 || mips64le) -// +build linux -// +build mips64 mips64le #include "textflag.h" diff --git a/src/runtime/rt0_linux_mipsx.s b/src/runtime/rt0_linux_mipsx.s index 9f5842b51a..3cbb7fc377 100644 --- a/src/runtime/rt0_linux_mipsx.s +++ b/src/runtime/rt0_linux_mipsx.s @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. //go:build linux && (mips || mipsle) -// +build linux -// +build mips mipsle #include "textflag.h" diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 4a0f489c2f..7e8723e15f 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -153,8 +153,8 @@ func TestGdbPython(t *testing.T) { } func TestGdbPythonCgo(t *testing.T) { - if runtime.GOARCH == "mips" || runtime.GOARCH == "mipsle" || runtime.GOARCH == "mips64" { - testenv.SkipFlaky(t, 18784) + if strings.HasPrefix(runtime.GOARCH, "mips") { + testenv.SkipFlaky(t, 37794) } testGdbPython(t, true) } @@ -424,7 +424,7 @@ func TestGdbBacktrace(t *testing.T) { "-ex", "continue", filepath.Join(dir, "a.exe"), } - got, err := exec.Command("gdb", args...).CombinedOutput() + got, err := testenv.RunWithTimeout(t, exec.Command("gdb", args...)) t.Logf("gdb output:\n%s", got) if err != nil { t.Fatalf("gdb exited with error: %v", err) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index bfd857e8d5..3eada37840 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -209,7 +209,7 @@ type eface struct { data unsafe.Pointer } -func efaceOf(ep *interface{}) *eface { +func efaceOf(ep *any) *eface { return (*eface)(unsafe.Pointer(ep)) } @@ -255,6 +255,8 @@ func efaceOf(ep *interface{}) *eface { // so I can't see them ever moving. If we did want to start moving data // in the GC, we'd need to allocate the goroutine structs from an // alternate arena. Using guintptr doesn't make that problem any worse. +// Note that pollDesc.rg, pollDesc.wg also store g in uintptr form, +// so they would need to be updated too if g's start moving. type guintptr uintptr //go:nosplit @@ -939,7 +941,7 @@ func extendRandom(r []byte, n int) { } // A _defer holds an entry on the list of deferred calls. -// If you add a field here, add code to clear it in freedefer and deferProcStack +// If you add a field here, add code to clear it in deferProcStack. // This struct must match the code in cmd/compile/internal/ssagen/ssa.go:deferstruct // and cmd/compile/internal/ssagen/ssa.go:(*state).call. // Some defers will be allocated on the stack and some on the heap. @@ -982,7 +984,7 @@ type _defer struct { // adjustment takes care of them. type _panic struct { argp unsafe.Pointer // pointer to arguments of deferred call run during panic; cannot move - known to liblink - arg interface{} // argument to panic + arg any // argument to panic link *_panic // link to earlier panic pc uintptr // where to return to in runtime if this panic is bypassed sp unsafe.Pointer // where to return to in runtime if this panic is bypassed diff --git a/src/runtime/runtime_test.go b/src/runtime/runtime_test.go index 4572a25195..12f261bdd2 100644 --- a/src/runtime/runtime_test.go +++ b/src/runtime/runtime_test.go @@ -6,6 +6,7 @@ package runtime_test import ( "flag" + "fmt" "io" . "runtime" "runtime/debug" @@ -53,8 +54,8 @@ func BenchmarkIfaceCmpNil100(b *testing.B) { } } -var efaceCmp1 interface{} -var efaceCmp2 interface{} +var efaceCmp1 any +var efaceCmp2 any func BenchmarkEfaceCmpDiff(b *testing.B) { x := 5 @@ -362,3 +363,78 @@ func TestVersion(t *testing.T) { t.Fatalf("cr/nl in version: %q", vers) } } + +func TestTimediv(t *testing.T) { + for _, tc := range []struct { + num int64 + div int32 + ret int32 + rem int32 + }{ + { + num: 8, + div: 2, + ret: 4, + rem: 0, + }, + { + num: 9, + div: 2, + ret: 4, + rem: 1, + }, + { + // Used by runtime.check. + num: 12345*1000000000 + 54321, + div: 1000000000, + ret: 12345, + rem: 54321, + }, + { + num: 1<<32 - 1, + div: 2, + ret: 1<<31 - 1, // no overflow. + rem: 1, + }, + { + num: 1 << 32, + div: 2, + ret: 1<<31 - 1, // overflow. + rem: 0, + }, + { + num: 1 << 40, + div: 2, + ret: 1<<31 - 1, // overflow. + rem: 0, + }, + { + num: 1<<40 + 1, + div: 1 << 10, + ret: 1 << 30, + rem: 1, + }, + } { + name := fmt.Sprintf("%d div %d", tc.num, tc.div) + t.Run(name, func(t *testing.T) { + // Double check that the inputs make sense using + // standard 64-bit division. + ret64 := tc.num / int64(tc.div) + rem64 := tc.num % int64(tc.div) + if ret64 != int64(int32(ret64)) { + // Simulate timediv overflow value. + ret64 = 1<<31 - 1 + rem64 = 0 + } + if ret64 != int64(tc.ret) { + t.Errorf("%d / %d got ret %d rem %d want ret %d rem %d", tc.num, tc.div, ret64, rem64, tc.ret, tc.rem) + } + + var rem int32 + ret := Timediv(tc.num, tc.div, &rem) + if ret != tc.ret || rem != tc.rem { + t.Errorf("timediv %d / %d got ret %d rem %d want ret %d rem %d", tc.num, tc.div, ret, rem, tc.ret, tc.rem) + } + }) + } +} diff --git a/src/runtime/rwmutex_test.go b/src/runtime/rwmutex_test.go index 291a32ea5e..f15d367b32 100644 --- a/src/runtime/rwmutex_test.go +++ b/src/runtime/rwmutex_test.go @@ -55,6 +55,11 @@ func TestParallelRWMutexReaders(t *testing.T) { // since the goroutines can't be stopped/preempted. // Disable GC for this test (see issue #10958). defer debug.SetGCPercent(debug.SetGCPercent(-1)) + // SetGCPercent waits until the mark phase is over, but the runtime + // also preempts at the start of the sweep phase, so make sure that's + // done too. + GC() + doTestParallelReaders(1) doTestParallelReaders(3) doTestParallelReaders(4) diff --git a/src/runtime/semasleep_test.go b/src/runtime/semasleep_test.go index cf4ef18208..d56733c0cf 100644 --- a/src/runtime/semasleep_test.go +++ b/src/runtime/semasleep_test.go @@ -7,6 +7,7 @@ package runtime_test import ( + "io" "os/exec" "syscall" "testing" @@ -20,43 +21,83 @@ func TestSpuriousWakeupsNeverHangSemasleep(t *testing.T) { if *flagQuick { t.Skip("-quick") } + t.Parallel() // Waits for a program to sleep for 1s. exe, err := buildTestProg(t, "testprog") if err != nil { t.Fatal(err) } - start := time.Now() cmd := exec.Command(exe, "After1") + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatalf("StdoutPipe: %v", err) + } + beforeStart := time.Now() if err := cmd.Start(); err != nil { t.Fatalf("Failed to start command: %v", err) } doneCh := make(chan error, 1) go func() { doneCh <- cmd.Wait() + close(doneCh) }() + t.Cleanup(func() { + cmd.Process.Kill() + <-doneCh + }) + + // Wait for After1 to close its stdout so that we know the runtime's SIGIO + // handler is registered. + b, err := io.ReadAll(stdout) + if len(b) > 0 { + t.Logf("read from testprog stdout: %s", b) + } + if err != nil { + t.Fatalf("error reading from testprog: %v", err) + } + + // Wait for an arbitrary timeout longer than one second. The subprocess itself + // attempts to sleep for one second, but if the machine running the test is + // heavily loaded that subprocess may not schedule very quickly even if the + // bug remains fixed. (This is fine, because if the bug really is unfixed we + // can keep the process hung indefinitely, as long as we signal it often + // enough.) + timeout := 10 * time.Second + + // The subprocess begins sleeping for 1s after it writes to stdout, so measure + // the timeout from here (not from when we started creating the process). + // That should reduce noise from process startup overhead. + ready := time.Now() // With the repro running, we can continuously send to it - // a non-terminal signal such as SIGIO, to spuriously - // wakeup pthread_cond_timedwait_relative_np. - unfixedTimer := time.NewTimer(2 * time.Second) + // a signal that the runtime considers non-terminal, + // such as SIGIO, to spuriously wake up + // pthread_cond_timedwait_relative_np. + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() for { select { - case <-time.After(200 * time.Millisecond): + case now := <-ticker.C: + if now.Sub(ready) > timeout { + t.Error("Program failed to return on time and has to be killed, issue #27520 still exists") + // Send SIGQUIT to get a goroutine dump. + // Stop sending SIGIO so that the program can clean up and actually terminate. + cmd.Process.Signal(syscall.SIGQUIT) + return + } + // Send the pesky signal that toggles spinning // indefinitely if #27520 is not fixed. cmd.Process.Signal(syscall.SIGIO) - case <-unfixedTimer.C: - t.Error("Program failed to return on time and has to be killed, issue #27520 still exists") - cmd.Process.Signal(syscall.SIGKILL) - return - case err := <-doneCh: if err != nil { t.Fatalf("The program returned but unfortunately with an error: %v", err) } - if time.Since(start) < 100*time.Millisecond { + if time.Since(beforeStart) < 1*time.Second { + // The program was supposed to sleep for a full (monotonic) second; + // it should not return before that has elapsed. t.Fatalf("The program stopped too quickly.") } return diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index dbcbfc67bc..08f266cc67 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -271,7 +271,24 @@ func setProcessCPUProfilerTimer(hz int32) { if hz != 0 { // Enable the Go signal handler if not enabled. if atomic.Cas(&handlingSig[_SIGPROF], 0, 1) { - atomic.Storeuintptr(&fwdSig[_SIGPROF], getsig(_SIGPROF)) + h := getsig(_SIGPROF) + // If no signal handler was installed before, then we record + // _SIG_IGN here. When we turn off profiling (below) we'll start + // ignoring SIGPROF signals. We do this, rather than change + // to SIG_DFL, because there may be a pending SIGPROF + // signal that has not yet been delivered to some other thread. + // If we change to SIG_DFL when turning off profiling, the + // program will crash when that SIGPROF is delivered. We assume + // that programs that use profiling don't want to crash on a + // stray SIGPROF. See issue 19320. + // We do the change here instead of when turning off profiling, + // because there we may race with a signal handler running + // concurrently, in particular, sigfwdgo may observe _SIG_DFL and + // die. See issue 43828. + if h == _SIG_DFL { + h = _SIG_IGN + } + atomic.Storeuintptr(&fwdSig[_SIGPROF], h) setsig(_SIGPROF, abi.FuncPCABIInternal(sighandler)) } @@ -288,21 +305,9 @@ func setProcessCPUProfilerTimer(hz int32) { // when we enabled profiling. We don't try to handle the case // of a program that changes the SIGPROF handler while Go // profiling is enabled. - // - // If no signal handler was installed before, then start - // ignoring SIGPROF signals. We do this, rather than change - // to SIG_DFL, because there may be a pending SIGPROF - // signal that has not yet been delivered to some other thread. - // If we change to SIG_DFL here, the program will crash - // when that SIGPROF is delivered. We assume that programs - // that use profiling don't want to crash on a stray SIGPROF. - // See issue 19320. if !sigInstallGoHandler(_SIGPROF) { if atomic.Cas(&handlingSig[_SIGPROF], 1, 0) { h := atomic.Loaduintptr(&fwdSig[_SIGPROF]) - if h == _SIG_DFL { - h = _SIG_IGN - } setsig(_SIGPROF, h) } } diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index b036f3c965..16c36d07f1 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -22,38 +22,6 @@ func disableWER() { stdcall1(_SetErrorMode, uintptr(errormode)|SEM_FAILCRITICALERRORS|SEM_NOGPFAULTERRORBOX|SEM_NOOPENFILEERRORBOX) } -// isWin7 returns true on Windows 7. Otherwise it returns false. -// -//go:nosplit -func isWin7() bool { - var maj, min, build uint32 - stdcall3(_RtlGetNtVersionNumbers, uintptr(unsafe.Pointer(&maj)), uintptr(unsafe.Pointer(&min)), uintptr(unsafe.Pointer(&build))) - return maj < 6 || (maj == 6 && min <= 1) -} - -// enableWERNoUI re-enables Windows error reporting without fault reporting UI. -// -// This is marked nosplit since it is used during crash. -// -//go:nosplit -func enableWERNoUI() bool { - if _WerSetFlags == nil { - return false - } - - // Disable Fault reporting UI - const ( - WER_FAULT_REPORTING_NO_UI = 0x0020 - ) - if stdcall1(_WerSetFlags, WER_FAULT_REPORTING_NO_UI) != 0 { - return false - } - - // re-enable Windows Error Reporting - stdcall1(_SetErrorMode, 0) - return true -} - // in sys_windows_386.s and sys_windows_amd64.s func exceptiontramp() func firstcontinuetramp() @@ -140,7 +108,6 @@ func exceptionhandler(info *exceptionrecord, r *context, gp *g) int32 { // Don't go through any more of the Windows handler chain. // Crash now. winthrow(info, r, gp) - exit(2) } // After this point, it is safe to grow the stack. @@ -229,20 +196,6 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { } winthrow(info, r, gp) - - _, _, docrash := gotraceback() - if docrash { - // Windows 7 apears to ignore WER_FAULT_REPORTING_NO_UI - // WerSetFlags API flag. So do not call enableWERNoUI - // on Windows 7. - if !isWin7() { - // trigger crash dump creation - if enableWERNoUI() { - return _EXCEPTION_CONTINUE_SEARCH - } - } - } - exit(2) return 0 // not reached } @@ -250,6 +203,11 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { func winthrow(info *exceptionrecord, r *context, gp *g) { _g_ := getg() + if panicking != 0 { // traceback already printed + exit(2) + } + panicking = 1 + // In case we're handling a g0 stack overflow, blow away the // g0 stack bounds so we have room to print the traceback. If // this somehow overflows the stack, the OS will trap it. @@ -271,16 +229,18 @@ func winthrow(info *exceptionrecord, r *context, gp *g) { _g_.m.throwing = 1 _g_.m.caughtsig.set(gp) - level, _, _ := gotraceback() + level, _, docrash := gotraceback() if level > 0 { - // only print traceback when it hasn't been printed - if tracebackprinted == 0 { - tracebacktrap(r.ip(), r.sp(), r.lr(), gp) - tracebackothers(gp) - tracebackprinted = 1 - } + tracebacktrap(r.ip(), r.sp(), r.lr(), gp) + tracebackothers(gp) dumpregs(r) } + + if docrash { + crash() + } + + exit(2) } func sigpanic() { @@ -352,17 +312,14 @@ func signame(sig uint32) string { //go:nosplit func crash() { - // When GOTRACEBACK==crash, raise the same exception - // from kernel32.dll, so that Windows gets a chance - // to handle the exception by creating a crash dump. - - // Get the Exception code that caused the crash - gp := getg() - exceptionCode := gp.sig - - // RaiseException() here will not be handled in exceptionhandler() - // because it comes from kernel32.dll - stdcall4(_RaiseException, uintptr(unsafe.Pointer(&exceptionCode)), 0, 0, 0) + // TODO: This routine should do whatever is needed + // to make the Windows program abort/crash as it + // would if Go was not intercepting signals. + // On Unix the routine would remove the custom signal + // handler and then raise a signal (like SIGABRT). + // Something like that should happen here. + // It's okay to leave this empty for now: if crash returns + // the ordinary exit-after-panic happens. } // gsignalStack is unused on Windows. diff --git a/src/runtime/sizeof_test.go b/src/runtime/sizeof_test.go index bbbd1becf7..ebf544ad3b 100644 --- a/src/runtime/sizeof_test.go +++ b/src/runtime/sizeof_test.go @@ -17,9 +17,9 @@ func TestSizeof(t *testing.T) { const _64bit = unsafe.Sizeof(uintptr(0)) == 8 var tests = []struct { - val interface{} // type as a value - _32bit uintptr // size on 32bit platforms - _64bit uintptr // size on 64bit platforms + val any // type as a value + _32bit uintptr // size on 32bit platforms + _64bit uintptr // size on 64bit platforms }{ {runtime.G{}, 236, 392}, // g, but exported for testing {runtime.Sudog{}, 56, 88}, // sudog, but exported for testing diff --git a/src/runtime/softfloat64_test.go b/src/runtime/softfloat64_test.go index 7347aff8f1..3f53e8bc55 100644 --- a/src/runtime/softfloat64_test.go +++ b/src/runtime/softfloat64_test.go @@ -127,7 +127,7 @@ func fromint64sw(f float64) float64 { var nerr int -func err(t *testing.T, format string, args ...interface{}) { +func err(t *testing.T, format string, args ...any) { t.Errorf(format, args...) // cut errors off after a while. diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 25a6f5bbb4..edc37d4878 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -1358,7 +1358,7 @@ func getStackMap(frame *stkframe, cache *pcvalueCache, debug bool) (locals, args var methodValueCallFrameObjs [1]stackObjectRecord // initialized in stackobjectinit func stkobjinit() { - var abiRegArgsEface interface{} = abi.RegArgs{} + var abiRegArgsEface any = abi.RegArgs{} abiRegArgsType := efaceOf(&abiRegArgsEface)._type if abiRegArgsType.kind&kindGCProg != 0 { throw("abiRegArgsType needs GC Prog, update methodValueCallFrameObjs") diff --git a/src/runtime/stack_test.go b/src/runtime/stack_test.go index 3f02243a1e..1a59086901 100644 --- a/src/runtime/stack_test.go +++ b/src/runtime/stack_test.go @@ -7,11 +7,9 @@ package runtime_test import ( "bytes" "fmt" - "os" "reflect" "regexp" . "runtime" - "strconv" "strings" "sync" "sync/atomic" @@ -83,12 +81,7 @@ func TestStackGrowth(t *testing.T) { t.Skip("-quick") } - if GOARCH == "wasm" { - t.Skip("fails on wasm (too slow?)") - } - - // Don't make this test parallel as this makes the 20 second - // timeout unreliable on slow builders. (See issue #19381.) + t.Parallel() var wg sync.WaitGroup @@ -102,6 +95,7 @@ func TestStackGrowth(t *testing.T) { growDuration = time.Since(start) }() wg.Wait() + t.Log("first growStack took", growDuration) // in locked goroutine wg.Add(1) @@ -114,48 +108,38 @@ func TestStackGrowth(t *testing.T) { wg.Wait() // in finalizer + var finalizerStart time.Time + var started, progress uint32 wg.Add(1) - go func() { + s := new(string) // Must be of a type that avoids the tiny allocator, or else the finalizer might not run. + SetFinalizer(s, func(ss *string) { defer wg.Done() - done := make(chan bool) - var startTime time.Time - var started, progress uint32 - go func() { - s := new(string) - SetFinalizer(s, func(ss *string) { - startTime = time.Now() - atomic.StoreUint32(&started, 1) - growStack(&progress) - done <- true - }) - s = nil - done <- true - }() - <-done - GC() - - timeout := 20 * time.Second - if s := os.Getenv("GO_TEST_TIMEOUT_SCALE"); s != "" { - scale, err := strconv.Atoi(s) - if err == nil { - timeout *= time.Duration(scale) - } - } + finalizerStart = time.Now() + atomic.StoreUint32(&started, 1) + growStack(&progress) + }) + setFinalizerTime := time.Now() + s = nil - select { - case <-done: - case <-time.After(timeout): + if d, ok := t.Deadline(); ok { + // Pad the timeout by an arbitrary 5% to give the AfterFunc time to run. + timeout := time.Until(d) * 19 / 20 + timer := time.AfterFunc(timeout, func() { + // Panic — instead of calling t.Error and returning from the test — so + // that we get a useful goroutine dump if the test times out, especially + // if GOTRACEBACK=system or GOTRACEBACK=crash is set. if atomic.LoadUint32(&started) == 0 { - t.Log("finalizer did not start") + panic("finalizer did not start") } else { - t.Logf("finalizer started %s ago and finished %d iterations", time.Since(startTime), atomic.LoadUint32(&progress)) + panic(fmt.Sprintf("finalizer started %s ago (%s after registration) and ran %d iterations, but did not return", time.Since(finalizerStart), finalizerStart.Sub(setFinalizerTime), atomic.LoadUint32(&progress))) } - t.Log("first growStack took", growDuration) - t.Error("finalizer did not run") - return - } - }() + }) + defer timer.Stop() + } + + GC() wg.Wait() + t.Logf("finalizer started after %s and ran %d iterations in %v", finalizerStart.Sub(setFinalizerTime), atomic.LoadUint32(&progress), time.Since(finalizerStart)) } // ... and in init @@ -879,7 +863,7 @@ func deferHeapAndStack(n int) (r int) { } // Pass a value to escapeMe to force it to escape. -var escapeMe = func(x interface{}) {} +var escapeMe = func(x any) {} // Test that when F -> G is inlined and F is excluded from stack // traces, G still appears. diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 3237a6b708..21dd95a397 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -408,7 +408,7 @@ type pcHeader struct { // moduledata records information about the layout of the executable // image. It is written by the linker. Any changes here must be -// matched changes to the code in cmd/internal/ld/symtab.go:symtab. +// matched changes to the code in cmd/link/internal/ld/symtab.go:symtab. // moduledata is stored in statically allocated non-pointer memory; // none of the pointers here are visible to the garbage collector. type moduledata struct { diff --git a/src/runtime/sys_aix_ppc64.s b/src/runtime/sys_aix_ppc64.s index c171c191c0..217ebb8878 100644 --- a/src/runtime/sys_aix_ppc64.s +++ b/src/runtime/sys_aix_ppc64.s @@ -25,7 +25,12 @@ TEXT callCfunction<>(SB), NOSPLIT|NOFRAME,$0 // stored in libcall_fn and store the results in libcall struture // Up to 6 arguments can be passed to this C function // Called by runtime.asmcgocall -// It reserves a stack of 288 bytes for the C function. +// It reserves a stack of 288 bytes for the C function. It must +// follow AIX convention, thus the first local variable must +// be stored at the offset 112, after the linker area (48 bytes) +// and the argument area (64). +// The AIX convention is described here: +// https://www.ibm.com/docs/en/aix/7.2?topic=overview-runtime-process-stack // NOT USING GO CALLING CONVENTION // runtime.asmsyscall6 is a function descriptor to the real asmsyscall6. DATA runtime·asmsyscall6+0(SB)/8, $asmsyscall6<>(SB) @@ -34,7 +39,8 @@ DATA runtime·asmsyscall6+16(SB)/8, $0 GLOBL runtime·asmsyscall6(SB), NOPTR, $24 TEXT asmsyscall6<>(SB),NOSPLIT,$256 - MOVD R3, 48(R1) // Save libcall for later + // Save libcall for later + MOVD R3, 112(R1) MOVD libcall_fn(R3), R12 MOVD libcall_args(R3), R9 MOVD 0(R9), R3 @@ -50,7 +56,7 @@ TEXT asmsyscall6<>(SB),NOSPLIT,$256 MOVD 40(R1), R2 // Store result in libcall - MOVD 48(R1), R5 + MOVD 112(R1), R5 MOVD R3, (libcall_r1)(R5) MOVD $-1, R6 CMP R6, R3 diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 0f91685d6c..80dd1a0378 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -105,28 +105,38 @@ func syscallNoErr() //go:nosplit //go:cgo_unsafe_args func pthread_attr_init(attr *pthreadattr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + return ret } func pthread_attr_init_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_getstacksize(attr *pthreadattr, size *uintptr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_getstacksize_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_getstacksize_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + KeepAlive(size) + return ret } func pthread_attr_getstacksize_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_setdetachstate(attr *pthreadattr, state int) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + return ret } func pthread_attr_setdetachstate_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_create_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_create_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + KeepAlive(arg) // Just for consistency. Arg of course needs to be kept alive for the start function. + return ret } func pthread_create_trampoline() @@ -156,7 +166,7 @@ func pthread_kill_trampoline() // mmap is used to do low-level memory allocation via mmap. Don't allow stack // splits, since this function (used by sysAlloc) is called in a lot of low-level // parts of the runtime and callers often assume it won't acquire any locks. -// go:nosplit +//go:nosplit func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { args := struct { addr unsafe.Pointer @@ -175,6 +185,7 @@ func mmap_trampoline() //go:cgo_unsafe_args func munmap(addr unsafe.Pointer, n uintptr) { libcCall(unsafe.Pointer(abi.FuncPCABI0(munmap_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. } func munmap_trampoline() @@ -182,6 +193,7 @@ func munmap_trampoline() //go:cgo_unsafe_args func madvise(addr unsafe.Pointer, n uintptr, flags int32) { libcCall(unsafe.Pointer(abi.FuncPCABI0(madvise_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. } func madvise_trampoline() @@ -189,13 +201,16 @@ func madvise_trampoline() //go:cgo_unsafe_args func mlock(addr unsafe.Pointer, n uintptr) { libcCall(unsafe.Pointer(abi.FuncPCABI0(mlock_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. } func mlock_trampoline() //go:nosplit //go:cgo_unsafe_args func read(fd int32, p unsafe.Pointer, n int32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(read_trampoline)), unsafe.Pointer(&fd)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(read_trampoline)), unsafe.Pointer(&fd)) + KeepAlive(p) + return ret } func read_trampoline() @@ -239,14 +254,18 @@ func usleep_no_g(usec uint32) { //go:nosplit //go:cgo_unsafe_args func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(write_trampoline)), unsafe.Pointer(&fd)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(write_trampoline)), unsafe.Pointer(&fd)) + KeepAlive(p) + return ret } func write_trampoline() //go:nosplit //go:cgo_unsafe_args func open(name *byte, mode, perm int32) (ret int32) { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(open_trampoline)), unsafe.Pointer(&name)) + ret = libcCall(unsafe.Pointer(abi.FuncPCABI0(open_trampoline)), unsafe.Pointer(&name)) + KeepAlive(name) + return } func open_trampoline() @@ -285,6 +304,8 @@ func walltime_trampoline() //go:cgo_unsafe_args func sigaction(sig uint32, new *usigactiont, old *usigactiont) { libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaction_trampoline)), unsafe.Pointer(&sig)) + KeepAlive(new) + KeepAlive(old) } func sigaction_trampoline() @@ -292,6 +313,8 @@ func sigaction_trampoline() //go:cgo_unsafe_args func sigprocmask(how uint32, new *sigset, old *sigset) { libcCall(unsafe.Pointer(abi.FuncPCABI0(sigprocmask_trampoline)), unsafe.Pointer(&how)) + KeepAlive(new) + KeepAlive(old) } func sigprocmask_trampoline() @@ -306,6 +329,8 @@ func sigaltstack(new *stackt, old *stackt) { new.ss_size = 32768 } libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaltstack_trampoline)), unsafe.Pointer(&new)) + KeepAlive(new) + KeepAlive(old) } func sigaltstack_trampoline() @@ -320,20 +345,32 @@ func raiseproc_trampoline() //go:cgo_unsafe_args func setitimer(mode int32, new, old *itimerval) { libcCall(unsafe.Pointer(abi.FuncPCABI0(setitimer_trampoline)), unsafe.Pointer(&mode)) + KeepAlive(new) + KeepAlive(old) } func setitimer_trampoline() //go:nosplit //go:cgo_unsafe_args func sysctl(mib *uint32, miblen uint32, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctl_trampoline)), unsafe.Pointer(&mib)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctl_trampoline)), unsafe.Pointer(&mib)) + KeepAlive(mib) + KeepAlive(oldp) + KeepAlive(oldlenp) + KeepAlive(newp) + return ret } func sysctl_trampoline() //go:nosplit //go:cgo_unsafe_args func sysctlbyname(name *byte, oldp *byte, oldlenp *uintptr, newp *byte, newlen uintptr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctlbyname_trampoline)), unsafe.Pointer(&name)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctlbyname_trampoline)), unsafe.Pointer(&name)) + KeepAlive(name) + KeepAlive(oldp) + KeepAlive(oldlenp) + KeepAlive(newp) + return ret } func sysctlbyname_trampoline() @@ -355,56 +392,79 @@ func kqueue_trampoline() //go:nosplit //go:cgo_unsafe_args func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(kevent_trampoline)), unsafe.Pointer(&kq)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(kevent_trampoline)), unsafe.Pointer(&kq)) + KeepAlive(ch) + KeepAlive(ev) + KeepAlive(ts) + return ret } func kevent_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_mutex_init(m *pthreadmutex, attr *pthreadmutexattr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_mutex_init_trampoline)), unsafe.Pointer(&m)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_mutex_init_trampoline)), unsafe.Pointer(&m)) + KeepAlive(m) + KeepAlive(attr) + return ret } func pthread_mutex_init_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_mutex_lock(m *pthreadmutex) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_mutex_lock_trampoline)), unsafe.Pointer(&m)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_mutex_lock_trampoline)), unsafe.Pointer(&m)) + KeepAlive(m) + return ret } func pthread_mutex_lock_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_mutex_unlock(m *pthreadmutex) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_mutex_unlock_trampoline)), unsafe.Pointer(&m)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_mutex_unlock_trampoline)), unsafe.Pointer(&m)) + KeepAlive(m) + return ret } func pthread_mutex_unlock_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_cond_init(c *pthreadcond, attr *pthreadcondattr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_init_trampoline)), unsafe.Pointer(&c)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_init_trampoline)), unsafe.Pointer(&c)) + KeepAlive(c) + KeepAlive(attr) + return ret } func pthread_cond_init_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_cond_wait(c *pthreadcond, m *pthreadmutex) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_wait_trampoline)), unsafe.Pointer(&c)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_wait_trampoline)), unsafe.Pointer(&c)) + KeepAlive(c) + KeepAlive(m) + return ret } func pthread_cond_wait_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_cond_timedwait_relative_np(c *pthreadcond, m *pthreadmutex, t *timespec) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_timedwait_relative_np_trampoline)), unsafe.Pointer(&c)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_timedwait_relative_np_trampoline)), unsafe.Pointer(&c)) + KeepAlive(c) + KeepAlive(m) + KeepAlive(t) + return ret } func pthread_cond_timedwait_relative_np_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_cond_signal(c *pthreadcond) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_signal_trampoline)), unsafe.Pointer(&c)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_cond_signal_trampoline)), unsafe.Pointer(&c)) + KeepAlive(c) + return ret } func pthread_cond_signal_trampoline() diff --git a/src/runtime/sys_darwin_arm64.go b/src/runtime/sys_darwin_arm64.go index e6d4c1be48..6170f4fdda 100644 --- a/src/runtime/sys_darwin_arm64.go +++ b/src/runtime/sys_darwin_arm64.go @@ -15,7 +15,9 @@ import ( //go:nosplit //go:cgo_unsafe_args func g0_pthread_key_create(k *pthreadkey, destructor uintptr) int32 { - return asmcgocall(unsafe.Pointer(abi.FuncPCABI0(pthread_key_create_trampoline)), unsafe.Pointer(&k)) + ret := asmcgocall(unsafe.Pointer(abi.FuncPCABI0(pthread_key_create_trampoline)), unsafe.Pointer(&k)) + KeepAlive(k) + return ret } func pthread_key_create_trampoline() diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index 65935de99f..ca443b699f 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -456,7 +456,7 @@ finish: MOVW $1000000000, R3 MULLU R0, R3, (R1, R0) ADD.S R2, R0 - ADC R4, R1 + ADC $0, R1 // Add carry bit to upper half. MOVW R0, ret_lo+0(FP) MOVW R1, ret_hi+4(FP) diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 08e44d671b..0df2597993 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. //go:build linux && (mips64 || mips64le) -// +build linux -// +build mips64 mips64le // // System calls and other sys.stuff for mips64, Linux diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index c828431899..2207e9ab98 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. //go:build linux && (mips || mipsle) -// +build linux -// +build mips mipsle // // System calls and other sys.stuff for mips, Linux diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index 9347afaf19..dc3d89fae7 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -3,8 +3,6 @@ // license that can be found in the LICENSE file. //go:build linux && (ppc64 || ppc64le) -// +build linux -// +build ppc64 ppc64le // // System calls and other sys.stuff for ppc64, Linux diff --git a/src/runtime/sys_netbsd_arm64.s b/src/runtime/sys_netbsd_arm64.s index 2d0b894d47..8a0496e807 100644 --- a/src/runtime/sys_netbsd_arm64.s +++ b/src/runtime/sys_netbsd_arm64.s @@ -279,8 +279,8 @@ fail: TEXT sigreturn_tramp<>(SB),NOSPLIT,$-8 MOVD g, R0 SVC $SYS_setcontext - MOVD $0x4242, R0 // Something failed, return magic number - SVC $SYS_exit + MOVD $0, R0 + MOVD R0, (R0) // crash TEXT runtime·sigaction(SB),NOSPLIT,$-8 MOVW sig+0(FP), R0 // arg 1 - signum diff --git a/src/runtime/sys_openbsd.go b/src/runtime/sys_openbsd.go index 9f3a25fcf8..c4b8489612 100644 --- a/src/runtime/sys_openbsd.go +++ b/src/runtime/sys_openbsd.go @@ -17,35 +17,47 @@ import ( //go:nosplit //go:cgo_unsafe_args func pthread_attr_init(attr *pthreadattr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + return ret } func pthread_attr_init_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_destroy(attr *pthreadattr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_destroy_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_destroy_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + return ret } func pthread_attr_destroy_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_getstacksize(attr *pthreadattr, size *uintptr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_getstacksize_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_getstacksize_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + KeepAlive(size) + return ret } func pthread_attr_getstacksize_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_attr_setdetachstate(attr *pthreadattr, state int) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + return ret } func pthread_attr_setdetachstate_trampoline() //go:nosplit //go:cgo_unsafe_args func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_create_trampoline)), unsafe.Pointer(&attr)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(pthread_create_trampoline)), unsafe.Pointer(&attr)) + KeepAlive(attr) + KeepAlive(arg) // Just for consistency. Arg of course needs to be kept alive for the start function. + return ret } func pthread_create_trampoline() diff --git a/src/runtime/sys_openbsd1.go b/src/runtime/sys_openbsd1.go index 4b80f60226..d852e3c58a 100644 --- a/src/runtime/sys_openbsd1.go +++ b/src/runtime/sys_openbsd1.go @@ -14,7 +14,10 @@ import ( //go:nosplit //go:cgo_unsafe_args func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(thrsleep_trampoline)), unsafe.Pointer(&ident)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(thrsleep_trampoline)), unsafe.Pointer(&ident)) + KeepAlive(tsp) + KeepAlive(abort) + return ret } func thrsleep_trampoline() diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go index 7024cfa86d..4d50b4f6b1 100644 --- a/src/runtime/sys_openbsd2.go +++ b/src/runtime/sys_openbsd2.go @@ -45,7 +45,7 @@ func thrkill_trampoline() // mmap is used to do low-level memory allocation via mmap. Don't allow stack // splits, since this function (used by sysAlloc) is called in a lot of low-level // parts of the runtime and callers often assume it won't acquire any locks. -// go:nosplit +//go:nosplit func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { args := struct { addr unsafe.Pointer @@ -56,6 +56,7 @@ func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (un ret2 int }{addr, n, prot, flags, fd, off, nil, 0} libcCall(unsafe.Pointer(abi.FuncPCABI0(mmap_trampoline)), unsafe.Pointer(&args)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. return args.ret1, args.ret2 } func mmap_trampoline() @@ -64,6 +65,7 @@ func mmap_trampoline() //go:cgo_unsafe_args func munmap(addr unsafe.Pointer, n uintptr) { libcCall(unsafe.Pointer(abi.FuncPCABI0(munmap_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. } func munmap_trampoline() @@ -71,13 +73,16 @@ func munmap_trampoline() //go:cgo_unsafe_args func madvise(addr unsafe.Pointer, n uintptr, flags int32) { libcCall(unsafe.Pointer(abi.FuncPCABI0(madvise_trampoline)), unsafe.Pointer(&addr)) + KeepAlive(addr) // Just for consistency. Hopefully addr is not a Go address. } func madvise_trampoline() //go:nosplit //go:cgo_unsafe_args func open(name *byte, mode, perm int32) (ret int32) { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(open_trampoline)), unsafe.Pointer(&name)) + ret = libcCall(unsafe.Pointer(abi.FuncPCABI0(open_trampoline)), unsafe.Pointer(&name)) + KeepAlive(name) + return } func open_trampoline() @@ -91,14 +96,18 @@ func close_trampoline() //go:nosplit //go:cgo_unsafe_args func read(fd int32, p unsafe.Pointer, n int32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(read_trampoline)), unsafe.Pointer(&fd)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(read_trampoline)), unsafe.Pointer(&fd)) + KeepAlive(p) + return ret } func read_trampoline() //go:nosplit //go:cgo_unsafe_args func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(write_trampoline)), unsafe.Pointer(&fd)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(write_trampoline)), unsafe.Pointer(&fd)) + KeepAlive(p) + return ret } func write_trampoline() @@ -121,6 +130,8 @@ func pipe2_trampoline() //go:cgo_unsafe_args func setitimer(mode int32, new, old *itimerval) { libcCall(unsafe.Pointer(abi.FuncPCABI0(setitimer_trampoline)), unsafe.Pointer(&mode)) + KeepAlive(new) + KeepAlive(old) } func setitimer_trampoline() @@ -140,7 +151,12 @@ func usleep_no_g(usec uint32) { //go:nosplit //go:cgo_unsafe_args func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctl_trampoline)), unsafe.Pointer(&mib)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(sysctl_trampoline)), unsafe.Pointer(&mib)) + KeepAlive(mib) + KeepAlive(out) + KeepAlive(size) + KeepAlive(dst) + return ret } func sysctl_trampoline() @@ -158,7 +174,13 @@ func nanotime1() int64 { clock_id int32 tp unsafe.Pointer }{_CLOCK_MONOTONIC, unsafe.Pointer(&ts)} - libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)) + if errno := libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)); errno < 0 { + // Avoid growing the nosplit stack. + systemstack(func() { + println("runtime: errno", -errno) + throw("clock_gettime failed") + }) + } return ts.tv_sec*1e9 + int64(ts.tv_nsec) } func clock_gettime_trampoline() @@ -170,7 +192,13 @@ func walltime() (int64, int32) { clock_id int32 tp unsafe.Pointer }{_CLOCK_REALTIME, unsafe.Pointer(&ts)} - libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)) + if errno := libcCall(unsafe.Pointer(abi.FuncPCABI0(clock_gettime_trampoline)), unsafe.Pointer(&args)); errno < 0 { + // Avoid growing the nosplit stack. + systemstack(func() { + println("runtime: errno", -errno) + throw("clock_gettime failed") + }) + } return ts.tv_sec, int32(ts.tv_nsec) } @@ -184,7 +212,11 @@ func kqueue_trampoline() //go:nosplit //go:cgo_unsafe_args func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { - return libcCall(unsafe.Pointer(abi.FuncPCABI0(kevent_trampoline)), unsafe.Pointer(&kq)) + ret := libcCall(unsafe.Pointer(abi.FuncPCABI0(kevent_trampoline)), unsafe.Pointer(&kq)) + KeepAlive(ch) + KeepAlive(ev) + KeepAlive(ts) + return ret } func kevent_trampoline() @@ -192,6 +224,8 @@ func kevent_trampoline() //go:cgo_unsafe_args func sigaction(sig uint32, new *sigactiont, old *sigactiont) { libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaction_trampoline)), unsafe.Pointer(&sig)) + KeepAlive(new) + KeepAlive(old) } func sigaction_trampoline() @@ -201,6 +235,8 @@ func sigprocmask(how uint32, new *sigset, old *sigset) { // sigprocmask is called from sigsave, which is called from needm. // As such, we have to be able to run with no g here. asmcgocall_no_g(unsafe.Pointer(abi.FuncPCABI0(sigprocmask_trampoline)), unsafe.Pointer(&how)) + KeepAlive(new) + KeepAlive(old) } func sigprocmask_trampoline() @@ -208,6 +244,8 @@ func sigprocmask_trampoline() //go:cgo_unsafe_args func sigaltstack(new *stackt, old *stackt) { libcCall(unsafe.Pointer(abi.FuncPCABI0(sigaltstack_trampoline)), unsafe.Pointer(&new)) + KeepAlive(new) + KeepAlive(old) } func sigaltstack_trampoline() diff --git a/src/runtime/sys_openbsd_386.s b/src/runtime/sys_openbsd_386.s index 7830b61b7d..890b96b673 100644 --- a/src/runtime/sys_openbsd_386.s +++ b/src/runtime/sys_openbsd_386.s @@ -520,8 +520,11 @@ TEXT runtime·clock_gettime_trampoline(SB),NOSPLIT,$0 MOVL BX, 4(SP) // arg 2 - clock_id CALL libc_clock_gettime(SB) CMPL AX, $-1 - JNE 2(PC) - MOVL $0xf1, 0xf1 // crash on failure + JNE noerr + CALL libc_errno(SB) + MOVL (AX), AX + NEGL AX // caller expects negative errno +noerr: MOVL BP, SP POPL BP RET diff --git a/src/runtime/sys_openbsd_amd64.s b/src/runtime/sys_openbsd_amd64.s index fc89ee6cbb..fc6d5dc387 100644 --- a/src/runtime/sys_openbsd_amd64.s +++ b/src/runtime/sys_openbsd_amd64.s @@ -369,8 +369,11 @@ TEXT runtime·clock_gettime_trampoline(SB),NOSPLIT,$0 MOVL 0(DI), DI // arg 1 clock_id CALL libc_clock_gettime(SB) TESTL AX, AX - JEQ 2(PC) - MOVL $0xf1, 0xf1 // crash + JEQ noerr + CALL libc_errno(SB) + MOVL (AX), AX // errno + NEGL AX // caller expects negative errno value +noerr: POPQ BP RET diff --git a/src/runtime/sys_openbsd_arm.s b/src/runtime/sys_openbsd_arm.s index 143fcf0518..a9cb1fbafe 100644 --- a/src/runtime/sys_openbsd_arm.s +++ b/src/runtime/sys_openbsd_arm.s @@ -407,9 +407,11 @@ TEXT runtime·clock_gettime_trampoline(SB),NOSPLIT,$0 MOVW 0(R0), R0 // arg 1 clock_id BL libc_clock_gettime(SB) CMP $-1, R0 - BNE 3(PC) - MOVW $0, R8 // crash on failure - MOVW R8, (R8) + BNE noerr + BL libc_errno(SB) + MOVW (R0), R0 // errno + RSB.CS $0, R0 // caller expects negative errno +noerr: MOVW R9, R13 RET diff --git a/src/runtime/sys_openbsd_arm64.s b/src/runtime/sys_openbsd_arm64.s index 9b4acc90a5..3fa7e1ede2 100644 --- a/src/runtime/sys_openbsd_arm64.s +++ b/src/runtime/sys_openbsd_arm64.s @@ -359,9 +359,11 @@ TEXT runtime·clock_gettime_trampoline(SB),NOSPLIT,$0 MOVD 0(R0), R0 // arg 1 - clock_id CALL libc_clock_gettime(SB) CMP $-1, R0 - BNE 3(PC) - MOVD $0, R0 // crash on failure - MOVD R0, (R0) + BNE noerr + CALL libc_errno(SB) + MOVW (R0), R0 // errno + NEG R0, R0 // caller expects negative errno value +noerr: RET TEXT runtime·fcntl_trampoline(SB),NOSPLIT,$0 diff --git a/src/runtime/syscall_solaris.go b/src/runtime/syscall_solaris.go index 15be8e1c61..e270e271c0 100644 --- a/src/runtime/syscall_solaris.go +++ b/src/runtime/syscall_solaris.go @@ -312,6 +312,8 @@ func syscall_wait4(pid uintptr, wstatus *uint32, options uintptr, rusage unsafe. entersyscallblock() asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&call)) exitsyscall() + KeepAlive(wstatus) + KeepAlive(rusage) return int(call.r1), call.err } diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index da181f2a8d..e76b403ade 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -422,6 +422,8 @@ func syscall_loadsystemlibrary(filename *uint16, absoluteFilepath *uint16) (hand } cgocall(asmstdcallAddr, unsafe.Pointer(c)) + KeepAlive(filename) + KeepAlive(absoluteFilepath) handle = c.r1 if handle == 0 { err = c.err @@ -441,6 +443,7 @@ func syscall_loadlibrary(filename *uint16) (handle, err uintptr) { c.n = 1 c.args = uintptr(noescape(unsafe.Pointer(&filename))) cgocall(asmstdcallAddr, unsafe.Pointer(c)) + KeepAlive(filename) handle = c.r1 if handle == 0 { err = c.err @@ -459,6 +462,7 @@ func syscall_getprocaddress(handle uintptr, procname *byte) (outhandle, err uint c.n = 2 c.args = uintptr(noescape(unsafe.Pointer(&handle))) cgocall(asmstdcallAddr, unsafe.Pointer(c)) + KeepAlive(procname) outhandle = c.r1 if outhandle == 0 { err = c.err diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go index 65f74b32fb..034a1d84db 100644 --- a/src/runtime/syscall_windows_test.go +++ b/src/runtime/syscall_windows_test.go @@ -288,7 +288,7 @@ func TestCallbackInAnotherThread(t *testing.T) { } type cbFunc struct { - goFunc interface{} + goFunc any } func (f cbFunc) cName(cdecl bool) string { @@ -628,6 +628,9 @@ func TestOutputDebugString(t *testing.T) { } func TestRaiseException(t *testing.T) { + if testenv.Builder() == "windows-amd64-2012" { + testenv.SkipFlaky(t, 49681) + } o := runTestProg(t, "testprog", "RaiseException") if strings.Contains(o, "RaiseException should not return") { t.Fatalf("RaiseException did not crash program: %v", o) @@ -770,6 +773,7 @@ func TestSyscallN(t *testing.T) { for arglen := 0; arglen <= runtime.MaxArgs; arglen++ { arglen := arglen t.Run(fmt.Sprintf("arg-%d", arglen), func(t *testing.T) { + t.Parallel() args := make([]string, arglen) rets := make([]string, arglen+1) params := make([]uintptr, arglen) diff --git a/src/runtime/testdata/testprog/badtraceback.go b/src/runtime/testdata/testprog/badtraceback.go index d558adceec..09aa2b877e 100644 --- a/src/runtime/testdata/testprog/badtraceback.go +++ b/src/runtime/testdata/testprog/badtraceback.go @@ -17,6 +17,9 @@ func init() { func BadTraceback() { // Disable GC to prevent traceback at unexpected time. debug.SetGCPercent(-1) + // Out of an abundance of caution, also make sure that there are + // no GCs actively in progress. + runtime.GC() // Run badLR1 on its own stack to minimize the stack size and // exercise the stack bounds logic in the hex dump. diff --git a/src/runtime/testdata/testprog/gc.go b/src/runtime/testdata/testprog/gc.go index 74732cd9f4..215228ea05 100644 --- a/src/runtime/testdata/testprog/gc.go +++ b/src/runtime/testdata/testprog/gc.go @@ -90,7 +90,7 @@ func GCFairness2() { runtime.GOMAXPROCS(1) debug.SetGCPercent(1) var count [3]int64 - var sink [3]interface{} + var sink [3]any for i := range count { go func(i int) { for { @@ -132,81 +132,88 @@ func GCFairness2() { func GCPhys() { // This test ensures that heap-growth scavenging is working as intended. // - // It sets up a specific scenario: it allocates two pairs of objects whose - // sizes sum to size. One object in each pair is "small" (though must be - // large enough to be considered a large object by the runtime) and one is - // large. The small objects are kept while the large objects are freed, - // creating two large unscavenged holes in the heap. The heap goal should - // also be small as a result (so size must be at least as large as the - // minimum heap size). We then allocate one large object, bigger than both - // pairs of objects combined. This allocation, because it will tip - // HeapSys-HeapReleased well above the heap goal, should trigger heap-growth - // scavenging and scavenge most, if not all, of the large holes we created - // earlier. + // It attempts to construct a sizeable "swiss cheese" heap, with many + // allocChunk-sized holes. Then, it triggers a heap growth by trying to + // allocate as much memory as would fit in those holes. + // + // The heap growth should cause a large number of those holes to be + // returned to the OS. + const ( - // Size must be also large enough to be considered a large - // object (not in any size-segregated span). - size = 4 << 20 - split = 64 << 10 - objects = 2 + // The total amount of memory we're willing to allocate. + allocTotal = 32 << 20 // The page cache could hide 64 8-KiB pages from the scavenger today. maxPageCache = (8 << 10) * 64 - - // Reduce GOMAXPROCS down to 4 if it's greater. We need to bound the amount - // of memory held in the page cache because the scavenger can't reach it. - // The page cache will hold at most maxPageCache of memory per-P, so this - // bounds the amount of memory hidden from the scavenger to 4*maxPageCache - // at most. - maxProcs = 4 ) - // Set GOGC so that this test operates under consistent assumptions. - debug.SetGCPercent(100) - procs := runtime.GOMAXPROCS(-1) - if procs > maxProcs { - defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(maxProcs)) - procs = runtime.GOMAXPROCS(-1) + + // How big the allocations are needs to depend on the page size. + // If the page size is too big and the allocations are too small, + // they might not be aligned to the physical page size, so the scavenger + // will gloss over them. + pageSize := os.Getpagesize() + var allocChunk int + if pageSize <= 8<<10 { + allocChunk = 64 << 10 + } else { + allocChunk = 512 << 10 } - // Save objects which we want to survive, and condemn objects which we don't. - // Note that we condemn objects in this way and release them all at once in - // order to avoid having the GC start freeing up these objects while the loop - // is still running and filling in the holes we intend to make. - saved := make([][]byte, 0, objects+1) - condemned := make([][]byte, 0, objects) - for i := 0; i < 2*objects; i++ { + allocs := allocTotal / allocChunk + + // Set GC percent just so this test is a little more consistent in the + // face of varying environments. + debug.SetGCPercent(100) + + // Set GOMAXPROCS to 1 to minimize the amount of memory held in the page cache, + // and to reduce the chance that the background scavenger gets scheduled. + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(1)) + + // Allocate allocTotal bytes of memory in allocChunk byte chunks. + // Alternate between whether the chunk will be held live or will be + // condemned to GC to create holes in the heap. + saved := make([][]byte, allocs/2+1) + condemned := make([][]byte, allocs/2) + for i := 0; i < allocs; i++ { + b := make([]byte, allocChunk) if i%2 == 0 { - saved = append(saved, make([]byte, split)) + saved = append(saved, b) } else { - condemned = append(condemned, make([]byte, size-split)) + condemned = append(condemned, b) } } - condemned = nil - // Clean up the heap. This will free up every other object created above - // (i.e. everything in condemned) creating holes in the heap. - // Also, if the condemned objects are still being swept, its possible that - // the scavenging that happens as a result of the next allocation won't see - // the holes at all. We call runtime.GC() twice here so that when we allocate - // our large object there's no race with sweeping. - runtime.GC() + + // Run a GC cycle just so we're at a consistent state. runtime.GC() - // Perform one big allocation which should also scavenge any holes. - // - // The heap goal will rise after this object is allocated, so it's very - // important that we try to do all the scavenging in a single allocation - // that exceeds the heap goal. Otherwise the rising heap goal could foil our - // test. - saved = append(saved, make([]byte, objects*size)) - // Clean up the heap again just to put it in a known state. + + // Drop the only reference to all the condemned memory. + condemned = nil + + // Clear the condemned memory. runtime.GC() + + // At this point, the background scavenger is likely running + // and could pick up the work, so the next line of code doesn't + // end up doing anything. That's fine. What's important is that + // this test fails somewhat regularly if the runtime doesn't + // scavenge on heap growth, and doesn't fail at all otherwise. + + // Make a large allocation that in theory could fit, but won't + // because we turned the heap into swiss cheese. + saved = append(saved, make([]byte, allocTotal/2)) + // heapBacked is an estimate of the amount of physical memory used by // this test. HeapSys is an estimate of the size of the mapped virtual // address space (which may or may not be backed by physical pages) // whereas HeapReleased is an estimate of the amount of bytes returned // to the OS. Their difference then roughly corresponds to the amount // of virtual address space that is backed by physical pages. + // + // heapBacked also subtracts out maxPageCache bytes of memory because + // this is memory that may be hidden from the scavenger per-P. Since + // GOMAXPROCS=1 here, subtracting it out once is fine. var stats runtime.MemStats runtime.ReadMemStats(&stats) - heapBacked := stats.HeapSys - stats.HeapReleased + heapBacked := stats.HeapSys - stats.HeapReleased - maxPageCache // If heapBacked does not exceed the heap goal by more than retainExtraPercent // then the scavenger is working as expected; the newly-created holes have been // scavenged immediately as part of the allocations which cannot fit in the holes. @@ -216,19 +223,14 @@ func GCPhys() { // to other allocations that happen during this test we may still see some physical // memory over-use. overuse := (float64(heapBacked) - float64(stats.HeapAlloc)) / float64(stats.HeapAlloc) - // Compute the threshold. + // Check against our overuse threshold, which is what the scavenger always reserves + // to encourage allocation of memory that doesn't need to be faulted in. // - // In theory, this threshold should just be zero, but that's not possible in practice. - // Firstly, the runtime's page cache can hide up to maxPageCache of free memory from the - // scavenger per P. To account for this, we increase the threshold by the ratio between the - // total amount the runtime could hide from the scavenger to the amount of memory we expect - // to be able to scavenge here, which is (size-split)*objects. This computation is the crux - // GOMAXPROCS above; if GOMAXPROCS is too high the threshold just becomes 100%+ since the - // amount of memory being allocated is fixed. Then we add 5% to account for noise, such as - // other allocations this test may have performed that we don't explicitly account for The - // baseline threshold here is around 11% for GOMAXPROCS=1, capping out at around 30% for - // GOMAXPROCS=4. - threshold := 0.05 + float64(procs)*maxPageCache/float64((size-split)*objects) + // Add additional slack in case the page size is large and the scavenger + // can't reach that memory because it doesn't constitute a complete aligned + // physical page. Assume the worst case: a full physical page out of each + // allocation. + threshold := 0.1 + float64(pageSize)/float64(allocChunk) if overuse <= threshold { fmt.Println("OK") return @@ -243,6 +245,7 @@ func GCPhys() { "(alloc: %d, goal: %d, sys: %d, rel: %d, objs: %d)\n", threshold*100, overuse*100, stats.HeapAlloc, stats.NextGC, stats.HeapSys, stats.HeapReleased, len(saved)) runtime.KeepAlive(saved) + runtime.KeepAlive(condemned) } // Test that defer closure is correctly scanned when the stack is scanned. @@ -263,9 +266,9 @@ func DeferLiveness() { } //go:noinline -func escape(x interface{}) { sink2 = x; sink2 = nil } +func escape(x any) { sink2 = x; sink2 = nil } -var sink2 interface{} +var sink2 any // Test zombie object detection and reporting. func GCZombie() { diff --git a/src/runtime/testdata/testprog/preempt.go b/src/runtime/testdata/testprog/preempt.go index 1c74d0e435..fb6755a372 100644 --- a/src/runtime/testdata/testprog/preempt.go +++ b/src/runtime/testdata/testprog/preempt.go @@ -20,6 +20,10 @@ func AsyncPreempt() { runtime.GOMAXPROCS(1) // Disable GC so we have complete control of what we're testing. debug.SetGCPercent(-1) + // Out of an abundance of caution, also make sure that there are + // no GCs actively in progress. The sweep phase of a GC cycle + // for instance tries to preempt Ps at the very beginning. + runtime.GC() // Start a goroutine with no sync safe-points. var ready, ready2 uint32 diff --git a/src/runtime/testdata/testprog/signal.go b/src/runtime/testdata/testprog/signal.go index 417e105c68..cc5ac8af58 100644 --- a/src/runtime/testdata/testprog/signal.go +++ b/src/runtime/testdata/testprog/signal.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !windows && !plan9 // +build !windows,!plan9 package main diff --git a/src/runtime/testdata/testprog/sleep.go b/src/runtime/testdata/testprog/sleep.go index 86e2f6cfe6..b230e60181 100644 --- a/src/runtime/testdata/testprog/sleep.go +++ b/src/runtime/testdata/testprog/sleep.go @@ -4,7 +4,10 @@ package main -import "time" +import ( + "os" + "time" +) // for golang.org/issue/27250 @@ -13,5 +16,7 @@ func init() { } func After1() { + os.Stdout.WriteString("ready\n") + os.Stdout.Close() <-time.After(1 * time.Second) } diff --git a/src/runtime/testdata/testprog/syscalls_none.go b/src/runtime/testdata/testprog/syscalls_none.go index 7f8ded3994..068bb59af3 100644 --- a/src/runtime/testdata/testprog/syscalls_none.go +++ b/src/runtime/testdata/testprog/syscalls_none.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !linux // +build !linux package main diff --git a/src/runtime/testdata/testprogcgo/callback.go b/src/runtime/testdata/testprogcgo/callback.go index be0409f39d..25f07159b8 100644 --- a/src/runtime/testdata/testprogcgo/callback.go +++ b/src/runtime/testdata/testprogcgo/callback.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main @@ -65,7 +66,7 @@ func grow1(x, sum *int) int { func CgoCallbackGC() { P := 100 - if os.Getenv("RUNTIME_TESTING_SHORT") != "" { + if os.Getenv("RUNTIME_TEST_SHORT") != "" { P = 10 } done := make(chan bool) diff --git a/src/runtime/testdata/testprogcgo/catchpanic.go b/src/runtime/testdata/testprogcgo/catchpanic.go index 55a606d1bc..c722d40a19 100644 --- a/src/runtime/testdata/testprogcgo/catchpanic.go +++ b/src/runtime/testdata/testprogcgo/catchpanic.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/dropm.go b/src/runtime/testdata/testprogcgo/dropm.go index 9e782f504f..700b7fa0c2 100644 --- a/src/runtime/testdata/testprogcgo/dropm.go +++ b/src/runtime/testdata/testprogcgo/dropm.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows // Test that a sequence of callbacks from C to Go get the same m. diff --git a/src/runtime/testdata/testprogcgo/eintr.go b/src/runtime/testdata/testprogcgo/eintr.go index 1722a75eb9..b35b280a76 100644 --- a/src/runtime/testdata/testprogcgo/eintr.go +++ b/src/runtime/testdata/testprogcgo/eintr.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/exec.go b/src/runtime/testdata/testprogcgo/exec.go index 15723c7369..c268bcd9b2 100644 --- a/src/runtime/testdata/testprogcgo/exec.go +++ b/src/runtime/testdata/testprogcgo/exec.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/gprof.go b/src/runtime/testdata/testprogcgo/gprof.go new file mode 100644 index 0000000000..d453b4d0ce --- /dev/null +++ b/src/runtime/testdata/testprogcgo/gprof.go @@ -0,0 +1,46 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +// Test taking a goroutine profile with C traceback. + +/* +// Defined in gprof_c.c. +void CallGoSleep(void); +void gprofCgoTraceback(void* parg); +void gprofCgoContext(void* parg); +*/ +import "C" + +import ( + "fmt" + "io" + "runtime" + "runtime/pprof" + "time" + "unsafe" +) + +func init() { + register("GoroutineProfile", GoroutineProfile) +} + +func GoroutineProfile() { + runtime.SetCgoTraceback(0, unsafe.Pointer(C.gprofCgoTraceback), unsafe.Pointer(C.gprofCgoContext), nil) + + go C.CallGoSleep() + go C.CallGoSleep() + go C.CallGoSleep() + time.Sleep(1 * time.Second) + + prof := pprof.Lookup("goroutine") + prof.WriteTo(io.Discard, 1) + fmt.Println("OK") +} + +//export GoSleep +func GoSleep() { + time.Sleep(time.Hour) +} diff --git a/src/runtime/testdata/testprogcgo/gprof_c.c b/src/runtime/testdata/testprogcgo/gprof_c.c new file mode 100644 index 0000000000..5c7cd77022 --- /dev/null +++ b/src/runtime/testdata/testprogcgo/gprof_c.c @@ -0,0 +1,30 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// The C definitions for gprof.go. That file uses //export so +// it can't put function definitions in the "C" import comment. + +#include <stdint.h> +#include <stdlib.h> + +// Functions exported from Go. +extern void GoSleep(); + +struct cgoContextArg { + uintptr_t context; +}; + +void gprofCgoContext(void *arg) { + ((struct cgoContextArg*)arg)->context = 1; +} + +void gprofCgoTraceback(void *arg) { + // spend some time here so the P is more likely to be retaken. + volatile int i; + for (i = 0; i < 123456789; i++); +} + +void CallGoSleep() { + GoSleep(); +} diff --git a/src/runtime/testdata/testprogcgo/lockosthread.go b/src/runtime/testdata/testprogcgo/lockosthread.go index 36423d9eb0..8fcea35f52 100644 --- a/src/runtime/testdata/testprogcgo/lockosthread.go +++ b/src/runtime/testdata/testprogcgo/lockosthread.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/needmdeadlock.go b/src/runtime/testdata/testprogcgo/needmdeadlock.go index 5a9c359006..b95ec77468 100644 --- a/src/runtime/testdata/testprogcgo/needmdeadlock.go +++ b/src/runtime/testdata/testprogcgo/needmdeadlock.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/numgoroutine.go b/src/runtime/testdata/testprogcgo/numgoroutine.go index 5bdfe52ed4..1b9f202f46 100644 --- a/src/runtime/testdata/testprogcgo/numgoroutine.go +++ b/src/runtime/testdata/testprogcgo/numgoroutine.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/pprof.go b/src/runtime/testdata/testprogcgo/pprof.go index 3b73fa0bdd..8870d0c415 100644 --- a/src/runtime/testdata/testprogcgo/pprof.go +++ b/src/runtime/testdata/testprogcgo/pprof.go @@ -29,8 +29,6 @@ void cpuHog() { void cpuHog2() { } -static int cpuHogCount; - struct cgoTracebackArg { uintptr_t context; uintptr_t sigContext; @@ -47,13 +45,6 @@ void pprofCgoTraceback(void* parg) { arg->buf[0] = (uintptr_t)(cpuHog) + 0x10; arg->buf[1] = (uintptr_t)(cpuHog2) + 0x4; arg->buf[2] = 0; - ++cpuHogCount; -} - -// getCpuHogCount fetches the number of times we've seen cpuHog in the -// traceback. -int getCpuHogCount() { - return cpuHogCount; } */ import "C" @@ -86,7 +77,7 @@ func CgoPprof() { } t0 := time.Now() - for C.getCpuHogCount() < 2 && time.Since(t0) < time.Second { + for time.Since(t0) < time.Second { C.cpuHog() } diff --git a/src/runtime/testdata/testprogcgo/raceprof.go b/src/runtime/testdata/testprogcgo/raceprof.go index f7ca629789..c098e16196 100644 --- a/src/runtime/testdata/testprogcgo/raceprof.go +++ b/src/runtime/testdata/testprogcgo/raceprof.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build (linux && amd64) || (freebsd && amd64) // +build linux,amd64 freebsd,amd64 package main diff --git a/src/runtime/testdata/testprogcgo/racesig.go b/src/runtime/testdata/testprogcgo/racesig.go index a079b3fd1a..9352679714 100644 --- a/src/runtime/testdata/testprogcgo/racesig.go +++ b/src/runtime/testdata/testprogcgo/racesig.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build (linux && amd64) || (freebsd && amd64) // +build linux,amd64 freebsd,amd64 package main diff --git a/src/runtime/testdata/testprogcgo/segv.go b/src/runtime/testdata/testprogcgo/segv.go index 3237a8c69c..0632475228 100644 --- a/src/runtime/testdata/testprogcgo/segv.go +++ b/src/runtime/testdata/testprogcgo/segv.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main diff --git a/src/runtime/testdata/testprogcgo/sigstack.go b/src/runtime/testdata/testprogcgo/sigstack.go index 21b668d6c0..12ca661033 100644 --- a/src/runtime/testdata/testprogcgo/sigstack.go +++ b/src/runtime/testdata/testprogcgo/sigstack.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows // Test handling of Go-allocated signal stacks when calling from diff --git a/src/runtime/testdata/testprogcgo/threadpanic.go b/src/runtime/testdata/testprogcgo/threadpanic.go index f9b48a9026..2d24fe68ea 100644 --- a/src/runtime/testdata/testprogcgo/threadpanic.go +++ b/src/runtime/testdata/testprogcgo/threadpanic.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 // +build !plan9 package main diff --git a/src/runtime/testdata/testprogcgo/threadpprof.go b/src/runtime/testdata/testprogcgo/threadpprof.go index feb774ba59..ec5e750da9 100644 --- a/src/runtime/testdata/testprogcgo/threadpprof.go +++ b/src/runtime/testdata/testprogcgo/threadpprof.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 && !windows // +build !plan9,!windows package main @@ -33,8 +34,6 @@ void cpuHogThread() { void cpuHogThread2() { } -static int cpuHogThreadCount; - struct cgoTracebackArg { uintptr_t context; uintptr_t sigContext; @@ -49,13 +48,6 @@ void pprofCgoThreadTraceback(void* parg) { arg->buf[0] = (uintptr_t)(cpuHogThread) + 0x10; arg->buf[1] = (uintptr_t)(cpuHogThread2) + 0x4; arg->buf[2] = 0; - __sync_add_and_fetch(&cpuHogThreadCount, 1); -} - -// getCPUHogThreadCount fetches the number of times we've seen cpuHogThread -// in the traceback. -int getCPUHogThreadCount() { - return __sync_add_and_fetch(&cpuHogThreadCount, 0); } static void* cpuHogDriver(void* arg __attribute__ ((unused))) { @@ -73,6 +65,7 @@ void runCPUHogThread(void) { import "C" import ( + "context" "fmt" "os" "runtime" @@ -107,12 +100,16 @@ func pprofThread() { os.Exit(2) } - C.runCPUHogThread() + // This goroutine may receive a profiling signal while creating the C-owned + // thread. If it does, the SetCgoTraceback handler will make the leaf end of + // the stack look almost (but not exactly) like the stacks the test case is + // trying to find. Attach a profiler label so the test can filter out those + // confusing samples. + pprof.Do(context.Background(), pprof.Labels("ignore", "ignore"), func(ctx context.Context) { + C.runCPUHogThread() + }) - t0 := time.Now() - for C.getCPUHogThreadCount() < 2 && time.Since(t0) < time.Second { - time.Sleep(100 * time.Millisecond) - } + time.Sleep(1 * time.Second) pprof.StopCPUProfile() diff --git a/src/runtime/testdata/testprogcgo/threadprof.go b/src/runtime/testdata/testprogcgo/threadprof.go index 2d4c1039fb..d62d4b4be8 100644 --- a/src/runtime/testdata/testprogcgo/threadprof.go +++ b/src/runtime/testdata/testprogcgo/threadprof.go @@ -2,21 +2,22 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// We only build this file with the tag "threadprof", since it starts -// a thread running a busy loop at constructor time. - +//go:build !plan9 && !windows // +build !plan9,!windows -// +build threadprof package main /* #include <stdint.h> +#include <stdlib.h> #include <signal.h> #include <pthread.h> volatile int32_t spinlock; +// Note that this thread is only started if GO_START_SIGPROF_THREAD +// is set in the environment, which is only done when running the +// CgoExternalThreadSIGPROF test. static void *thread1(void *p) { (void)p; while (spinlock == 0) @@ -26,9 +27,13 @@ static void *thread1(void *p) { return NULL; } +// This constructor function is run when the program starts. +// It is used for the CgoExternalThreadSIGPROF test. __attribute__((constructor)) void issue9456() { - pthread_t tid; - pthread_create(&tid, 0, thread1, NULL); + if (getenv("GO_START_SIGPROF_THREAD") != NULL) { + pthread_t tid; + pthread_create(&tid, 0, thread1, NULL); + } } void **nullptr; diff --git a/src/runtime/testdata/testprognet/signal.go b/src/runtime/testdata/testprognet/signal.go index 4d2de79d97..dfa2e10e7a 100644 --- a/src/runtime/testdata/testprognet/signal.go +++ b/src/runtime/testdata/testprognet/signal.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !windows && !plan9 // +build !windows,!plan9 // This is in testprognet instead of testprog because testprog diff --git a/src/runtime/testdata/testprognet/signalexec.go b/src/runtime/testdata/testprognet/signalexec.go index 4a988ef6c1..62ebce7176 100644 --- a/src/runtime/testdata/testprognet/signalexec.go +++ b/src/runtime/testdata/testprognet/signalexec.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd // +build darwin dragonfly freebsd linux netbsd openbsd // This is in testprognet instead of testprog because testprog diff --git a/src/runtime/testdata/testwinlib/main.c b/src/runtime/testdata/testwinlib/main.c index e84a32f753..c3fe3cb071 100644 --- a/src/runtime/testdata/testwinlib/main.c +++ b/src/runtime/testdata/testwinlib/main.c @@ -41,17 +41,20 @@ int main() if (NULL == exceptionHandlerHandle) { printf("cannot add vectored exception handler\n"); + fflush(stdout); return 2; } void *continueHandlerHandle = AddVectoredContinueHandler(0, customContinueHandlder); if (NULL == continueHandlerHandle) { printf("cannot add vectored continue handler\n"); + fflush(stdout); return 2; } CallMeBack(throwFromC); RemoveVectoredContinueHandler(continueHandlerHandle); RemoveVectoredExceptionHandler(exceptionHandlerHandle); printf("exceptionCount: %d\ncontinueCount: %d\n", exceptionCount, continueCount); + fflush(stdout); return 0; -}
\ No newline at end of file +} diff --git a/src/runtime/testdata/testwinlib/main.go b/src/runtime/testdata/testwinlib/main.go index 400eaa1c82..025ef913e5 100644 --- a/src/runtime/testdata/testwinlib/main.go +++ b/src/runtime/testdata/testwinlib/main.go @@ -1,3 +1,4 @@ +//go:build windows && cgo // +build windows,cgo package main diff --git a/src/runtime/time.go b/src/runtime/time.go index 46e9a8c2ab..a9ad620776 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -28,8 +28,8 @@ type timer struct { // when must be positive on an active timer. when int64 period int64 - f func(interface{}, uintptr) - arg interface{} + f func(any, uintptr) + arg any seq uintptr // What to set the when field to in timerModifiedXX status. @@ -232,14 +232,14 @@ func resetTimer(t *timer, when int64) bool { // modTimer modifies an existing timer. //go:linkname modTimer time.modTimer -func modTimer(t *timer, when, period int64, f func(interface{}, uintptr), arg interface{}, seq uintptr) { +func modTimer(t *timer, when, period int64, f func(any, uintptr), arg any, seq uintptr) { modtimer(t, when, period, f, arg, seq) } // Go runtime. // Ready the goroutine arg. -func goroutineReady(arg interface{}, seq uintptr) { +func goroutineReady(arg any, seq uintptr) { goready(arg.(*g), 0) } @@ -421,7 +421,7 @@ func dodeltimer0(pp *p) { // modtimer modifies an existing timer. // This is called by the netpoll code or time.Ticker.Reset or time.Timer.Reset. // Reports whether the timer was modified before it was run. -func modtimer(t *timer, when, period int64, f func(interface{}, uintptr), arg interface{}, seq uintptr) bool { +func modtimer(t *timer, when, period int64, f func(any, uintptr), arg any, seq uintptr) bool { if when <= 0 { throw("timer when must be positive") } diff --git a/src/runtime/time_fake.go b/src/runtime/time_fake.go index 107f6be335..b5e0463588 100644 --- a/src/runtime/time_fake.go +++ b/src/runtime/time_fake.go @@ -41,6 +41,10 @@ func time_now() (sec int64, nsec int32, mono int64) { return faketime / 1e9, int32(faketime % 1e9), faketime } +// write is like the Unix write system call. +// We have to avoid write barriers to avoid potential deadlock +// on write calls. +//go:nowritebarrierrec func write(fd uintptr, p unsafe.Pointer, n int32) int32 { if !(fd == 1 || fd == 2) { // Do an ordinary write. diff --git a/src/runtime/time_linux_amd64.s b/src/runtime/time_linux_amd64.s index 67cfdd8fdf..1416d23230 100644 --- a/src/runtime/time_linux_amd64.s +++ b/src/runtime/time_linux_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !faketime -// +build !faketime #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/time_windows_386.s b/src/runtime/time_windows_386.s index 19ce6910d7..b8b636ef30 100644 --- a/src/runtime/time_windows_386.s +++ b/src/runtime/time_windows_386.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !faketime -// +build !faketime #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/time_windows_amd64.s b/src/runtime/time_windows_amd64.s index 70f6a008cd..226f2b5136 100644 --- a/src/runtime/time_windows_amd64.s +++ b/src/runtime/time_windows_amd64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !faketime -// +build !faketime #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/time_windows_arm.s b/src/runtime/time_windows_arm.s index 6552d75ff1..711af88307 100644 --- a/src/runtime/time_windows_arm.s +++ b/src/runtime/time_windows_arm.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !faketime -// +build !faketime #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/time_windows_arm64.s b/src/runtime/time_windows_arm64.s index ef5b848473..e0c7d28e15 100644 --- a/src/runtime/time_windows_arm64.s +++ b/src/runtime/time_windows_arm64.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !faketime -// +build !faketime #include "go_asm.h" #include "textflag.h" diff --git a/src/runtime/tls_arm.s b/src/runtime/tls_arm.s index 879caac9e1..83fd37e6ec 100644 --- a/src/runtime/tls_arm.s +++ b/src/runtime/tls_arm.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !windows -// +build !windows #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/tls_mips64x.s b/src/runtime/tls_mips64x.s index 779d64ba31..ec2748e5b2 100644 --- a/src/runtime/tls_mips64x.s +++ b/src/runtime/tls_mips64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips64 || mips64le -// +build mips64 mips64le #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/tls_mipsx.s b/src/runtime/tls_mipsx.s index ada8d06a9e..acc3eb5a17 100644 --- a/src/runtime/tls_mipsx.s +++ b/src/runtime/tls_mipsx.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build mips || mipsle -// +build mips mipsle #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/tls_ppc64x.s b/src/runtime/tls_ppc64x.s index 7e935d0eb2..17aec9fc1e 100644 --- a/src/runtime/tls_ppc64x.s +++ b/src/runtime/tls_ppc64x.s @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build ppc64 || ppc64le -// +build ppc64 ppc64le #include "go_asm.h" #include "go_tls.h" diff --git a/src/runtime/trace.go b/src/runtime/trace.go index 5b14a5f553..71a29d4316 100644 --- a/src/runtime/trace.go +++ b/src/runtime/trace.go @@ -426,6 +426,9 @@ func ReadTrace() []byte { trace.footerWritten = true // Use float64 because (trace.ticksEnd - trace.ticksStart) * 1e9 can overflow int64. freq := float64(trace.ticksEnd-trace.ticksStart) * 1e9 / float64(trace.timeEnd-trace.timeStart) / traceTickDiv + if freq <= 0 { + throw("trace: ReadTrace got invalid frequency") + } trace.lockOwner = nil unlock(&trace.lock) var data []byte diff --git a/src/runtime/trace/annotation.go b/src/runtime/trace/annotation.go index 6e18bfb755..d05b5e2261 100644 --- a/src/runtime/trace/annotation.go +++ b/src/runtime/trace/annotation.go @@ -98,7 +98,7 @@ func Log(ctx context.Context, category, message string) { } // Logf is like Log, but the value is formatted using the specified format spec. -func Logf(ctx context.Context, category, format string, args ...interface{}) { +func Logf(ctx context.Context, category, format string, args ...any) { if IsEnabled() { // Ideally this should be just Log, but that will // add one more frame in the stack trace. diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 36627a6735..73bd0e11a9 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -96,6 +96,20 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } } + // runtime/internal/atomic functions call into kernel helpers on + // arm < 7. See runtime/internal/atomic/sys_linux_arm.s. + // + // Start in the caller's frame. + if GOARCH == "arm" && goarm < 7 && GOOS == "linux" && frame.pc&0xffff0000 == 0xffff0000 { + // Note that the calls are simple BL without pushing the return + // address, so we use LR directly. + // + // The kernel helpers are frameless leaf functions, so SP and + // LR are not touched. + frame.pc = frame.lr + frame.lr = 0 + } + f := findfunc(frame.pc) if !f.valid() { if callback != nil || printing { diff --git a/src/runtime/zcallback_windows.s b/src/runtime/zcallback_windows.s index 561527c90d..bd23d71333 100644 --- a/src/runtime/zcallback_windows.s +++ b/src/runtime/zcallback_windows.s @@ -1,7 +1,6 @@ // Code generated by wincallback.go using 'go generate'. DO NOT EDIT. //go:build 386 || amd64 -// +build 386 amd64 // runtime·callbackasm is called by external code to // execute Go implemented callback function. It is not |