From a69395eab2a1f704a549d15260c896d01fc9ab34 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 17 Mar 2025 11:45:52 -0400 Subject: runtime/_mkmalloc: add a copy of cloneNode cloneNode is defined in golang.org/x/tools/internal/astutil. Make a copy of it so we can easily clone AST nodes. Change-Id: I6a6a6964132e663e64faa00fa6037cf6e8d4cbc6 Reviewed-on: https://go-review.googlesource.com/c/go/+/703396 LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek Reviewed-by: Michael Matloob --- src/runtime/_mkmalloc/astutil/clone.go | 73 ++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 src/runtime/_mkmalloc/astutil/clone.go (limited to 'src/runtime') diff --git a/src/runtime/_mkmalloc/astutil/clone.go b/src/runtime/_mkmalloc/astutil/clone.go new file mode 100644 index 0000000000..16ea7163ca --- /dev/null +++ b/src/runtime/_mkmalloc/astutil/clone.go @@ -0,0 +1,73 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is a copy of golang.org/x/tools/internal/astutil/clone.go + +package astutil + +import ( + "go/ast" + "reflect" +) + +// CloneNode returns a deep copy of a Node. +// It omits pointers to ast.{Scope,Object} variables. +func CloneNode[T ast.Node](n T) T { + return cloneNode(n).(T) +} + +func cloneNode(n ast.Node) ast.Node { + var clone func(x reflect.Value) reflect.Value + set := func(dst, src reflect.Value) { + src = clone(src) + if src.IsValid() { + dst.Set(src) + } + } + clone = func(x reflect.Value) reflect.Value { + switch x.Kind() { + case reflect.Pointer: + if x.IsNil() { + return x + } + // Skip fields of types potentially involved in cycles. + switch x.Interface().(type) { + case *ast.Object, *ast.Scope: + return reflect.Zero(x.Type()) + } + y := reflect.New(x.Type().Elem()) + set(y.Elem(), x.Elem()) + return y + + case reflect.Struct: + y := reflect.New(x.Type()).Elem() + for i := 0; i < x.Type().NumField(); i++ { + set(y.Field(i), x.Field(i)) + } + return y + + case reflect.Slice: + if x.IsNil() { + return x + } + y := reflect.MakeSlice(x.Type(), x.Len(), x.Cap()) + for i := 0; i < x.Len(); i++ { + set(y.Index(i), x.Index(i)) + } + return y + + case reflect.Interface: + y := reflect.New(x.Type()).Elem() + set(y, x.Elem()) + return y + + case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer: + panic(x) // unreachable in AST + + default: + return x // bool, string, number + } + } + return clone(reflect.ValueOf(n)).Interface().(ast.Node) +} -- cgit v1.3-5-g9baa From 465b85eb760bfdb114f6b6ebccf374aba3977929 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Wed, 17 Sep 2025 21:04:56 +0000 Subject: runtime: fix CheckScavengedBitsCleared with randomized heap base We cannot easily determine the base address of the arena we added the random padding pages to, which can cause confusing TestScavengedBitsCleared failures when the arena we padded does not have the lowest address of all of the arenas. Instead of attempting to determine the correct arena to ignore when checking the scav and alloc bits, switch to just tolerating _one_ arena having mismatches, which is expected when randomizedHeapBase64 is enabled. Any other number of arenas having mismatches is likely a real error. Fixes #75502 Change-Id: Iacc445b2905824f9f71970c7abd33f187793cf39 Reviewed-on: https://go-review.googlesource.com/c/go/+/704855 Reviewed-by: Michael Knyszek Auto-Submit: Roland Shoemaker LUCI-TryBot-Result: Go LUCI --- src/runtime/export_test.go | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 1f55717f0a..99c665c6d0 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -1122,8 +1122,6 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { // Lock so that we can safely access the bitmap. lock(&mheap_.lock) - heapBase := mheap_.pages.inUse.ranges[0].base.addr() - secondArenaBase := arenaBase(arenaIndex(heapBase) + 1) chunkLoop: for i := mheap_.pages.start; i < mheap_.pages.end; i++ { chunk := mheap_.pages.tryChunkOf(i) @@ -1140,14 +1138,6 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { want := chunk.scavenged[j] &^ chunk.pallocBits[j] got := chunk.scavenged[j] if want != got { - // When goexperiment.RandomizedHeapBase64 is set we use a - // series of padding pages to generate randomized heap base - // address which have both the alloc and scav bits set. If - // we see this for a chunk between the address of the heap - // base, and the address of the second arena continue. - if goexperiment.RandomizedHeapBase64 && (cb >= heapBase && cb < secondArenaBase) { - continue - } ok = false if n >= len(mismatches) { break chunkLoop @@ -1165,6 +1155,37 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { getg().m.mallocing-- }) + + if goexperiment.RandomizedHeapBase64 && len(mismatches) > 0 { + // When goexperiment.RandomizedHeapBase64 is set we use a series of + // padding pages to generate randomized heap base address which have + // both the alloc and scav bits set. Because of this we expect exactly + // one arena will have mismatches, so check for that explicitly and + // remove the mismatches if that property holds. If we see more than one + // arena with this property, that is an indication something has + // actually gone wrong, so return the mismatches. + // + // We do this, instead of ignoring the mismatches in the chunkLoop, because + // it's not easy to determine which arena we added the padding pages to + // programmatically, without explicitly recording the base address somewhere + // in a global variable (which we'd rather not do as the address of that variable + // is likely to be somewhat predictable, potentially defeating the purpose + // of our randomization). + affectedArenas := map[arenaIdx]bool{} + for _, mismatch := range mismatches { + if mismatch.Base > 0 { + affectedArenas[arenaIndex(mismatch.Base)] = true + } + } + if len(affectedArenas) == 1 { + ok = true + // zero the mismatches + for i := range n { + mismatches[i] = BitsMismatch{} + } + } + } + return } -- cgit v1.3-5-g9baa From 77aac7bb75edc222dd7b350e8b76c20c79da5f43 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Wed, 17 Sep 2025 21:10:11 +0000 Subject: runtime: don't enable heap randomization if MSAN or ASAN is enabled MSAN and ASAN do confusing things to the memory layout, which are likely to conflict with heap base randomization, so if they are enabled, ignore randomizedHeapBase64. We already didn't turn it on when TSAN was enabled. Change-Id: I41e59dfc33d8bb059c208a9595442571fb31eea3 Reviewed-on: https://go-review.googlesource.com/c/go/+/704856 LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek Auto-Submit: Roland Shoemaker --- src/runtime/export_test.go | 3 +-- src/runtime/malloc.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 99c665c6d0..607281d382 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -9,7 +9,6 @@ package runtime import ( "internal/abi" "internal/goarch" - "internal/goexperiment" "internal/goos" "internal/runtime/atomic" "internal/runtime/gc" @@ -1156,7 +1155,7 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) { getg().m.mallocing-- }) - if goexperiment.RandomizedHeapBase64 && len(mismatches) > 0 { + if randomizeHeapBase && len(mismatches) > 0 { // When goexperiment.RandomizedHeapBase64 is set we use a series of // padding pages to generate randomized heap base address which have // both the alloc and scav bits set. Because of this we expect exactly diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index ec5f0765ba..5b5a633d9a 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -349,7 +349,7 @@ const ( // randomizeHeapBase indicates if the heap base address should be randomized. // See comment in mallocinit for how the randomization is performed. - randomizeHeapBase = goexperiment.RandomizedHeapBase64 && goarch.PtrSize == 8 && !isSbrkPlatform + randomizeHeapBase = goexperiment.RandomizedHeapBase64 && goarch.PtrSize == 8 && !isSbrkPlatform && !raceenabled && !msanenabled && !asanenabled // randHeapBasePrefixMask is used to extract the top byte of the randomized // heap base address. -- cgit v1.3-5-g9baa From 3032894e045fd3628198061a44c56d4a1fb73d93 Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Wed, 17 Sep 2025 13:25:03 -0400 Subject: runtime: make explicit nil check in heapSetTypeSmallHeader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is another case very similar to CL 684015 and #74375. In spans with type headers, mallocgc always writes to the page before returning the allocated memory. This initial write is done by runtime.heapSetTypeSmallHeader. Prior to the write, the compiler inserts a nil check, implemented as a dummy instruction reading from memory. On a freshly mapped page, this read triggers a page fault, mapping the zero page read-only. Immediately afterwards, the write triggers another page fault, copying to a writeable page and performing a TLB flush. This problem is exacerbated as the process scales up. At GOMAXPROCS=6, the tile38 sweet benchmark spends around 0.1% of cycles directly handling these page faults. On the same machine at GOMAXPROCS=192, it spends about 2.7% of cycles directly handling these page faults. Replacing the read with an explicit nil check reduces the direct cost of these page faults down to around 0.1% at GOMAXPROCS=192. There are additional positive side-effects due to reduced contention, so the overall time spent in page faults drops from around 12.8% to 6.8%. Most of the remaining time in page faults is spent on automatic NUMA page migration (completely unrelated to this issue). Impact on the tile38 benchmark results: │ baseline │ cl704755 │ │ sec/op │ sec/op vs base │ Tile38QueryLoad-192 1.638m ± 3% 1.494m ± 5% -8.79% (p=0.002 n=6) │ baseline │ cl704755 │ │ average-RSS-bytes │ average-RSS-bytes vs base │ Tile38QueryLoad-192 5.384Gi ± 3% 5.399Gi ± 3% ~ (p=0.818 n=6) │ baseline │ cl704755 │ │ peak-RSS-bytes │ peak-RSS-bytes vs base │ Tile38QueryLoad-192 5.818Gi ± 1% 5.864Gi ± 2% ~ (p=0.394 n=6) │ baseline │ cl704755 │ │ peak-VM-bytes │ peak-VM-bytes vs base │ Tile38QueryLoad-192 7.121Gi ± 1% 7.180Gi ± 2% ~ (p=0.818 n=6) │ baseline │ cl704755 │ │ p50-latency-sec │ p50-latency-sec vs base │ Tile38QueryLoad-192 343.2µ ± 1% 313.2µ ± 3% -8.73% (p=0.002 n=6) │ baseline │ cl704755 │ │ p90-latency-sec │ p90-latency-sec vs base │ Tile38QueryLoad-192 1.662m ± 2% 1.603m ± 5% ~ (p=0.093 n=6) │ baseline │ cl704755 │ │ p99-latency-sec │ p99-latency-sec vs base │ Tile38QueryLoad-192 41.56m ± 8% 35.26m ± 18% -15.17% (p=0.026 n=6) │ baseline │ cl704755 │ │ ops/s │ ops/s vs base │ Tile38QueryLoad-192 87.89k ± 3% 96.36k ± 4% +9.64% (p=0.002 n=6) Updates #74375. Change-Id: I6a6a636c1a16261b6d5076f2e1b08524a6544d33 Reviewed-on: https://go-review.googlesource.com/c/go/+/704755 LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek Auto-Submit: Michael Pratt --- src/runtime/mbitmap.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 9872e5297f..508de9a115 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -714,6 +714,26 @@ func heapSetTypeNoHeader(x, dataSize uintptr, typ *_type, span *mspan) uintptr { } func heapSetTypeSmallHeader(x, dataSize uintptr, typ *_type, header **_type, span *mspan) uintptr { + if header == nil { + // This nil check and throw is almost pointless. Normally we would + // expect header to never be nil. However, this is called on potentially + // freshly-allocated virtual memory. As of 2025, the compiler-inserted + // nil check is not a branch but a memory read that we expect to fault + // if the pointer really is nil. + // + // However, this causes a read of the page, and operating systems may + // take it as a hint to back the accessed memory with a read-only zero + // page. However, we immediately write to this memory, which can then + // force operating systems to have to update the page table and flush + // the TLB. + // + // This nil check is thus an explicit branch instead of what the compiler + // would insert circa 2025, which is a memory read instruction. + // + // See go.dev/issue/74375 for details of a similar issue in + // spanInlineMarkBits. + throw("runtime: pointer to heap type header nil?") + } *header = typ if doubleCheckHeapSetType { doubleCheckHeapType(x, dataSize, typ, header, span) -- cgit v1.3-5-g9baa From 3cf1aaf8b9c846c44ec8db679495dd5816d1ec30 Mon Sep 17 00:00:00 2001 From: Daniel Maslowski Date: Thu, 18 Sep 2025 15:43:42 +0000 Subject: runtime: use futexes with 64-bit time on Linux Linux introduced new syscalls to fix the year 2038 issue. To still be able to use the old ones, the Kconfig option COMPAT_32BIT_TIME would be necessary. Use the new syscall with 64-bit values for futex by default. Define _ENOSYS for detecting if it's not available. Add a fallback to use the older syscall in case the new one is not available, since Go runs on Linux from 2.6.32 on, per https://go.dev/wiki/MinimumRequirements. Updates #75133 Change-Id: I65daff0a3d06b55440ff05d8f5a9aa1c07eb201d GitHub-Last-Rev: 96dd1bd84bd12d898e971157fc83da562cc4f6b4 GitHub-Pull-Request: golang/go#75306 Reviewed-on: https://go-review.googlesource.com/c/go/+/701615 Reviewed-by: Michael Knyszek Reviewed-by: Mark Freeman Reviewed-by: Jorropo LUCI-TryBot-Result: Go LUCI --- src/runtime/defs2_linux.go | 1 + src/runtime/defs_linux.go | 1 + src/runtime/defs_linux_386.go | 23 ++++++++++++++++++---- src/runtime/defs_linux_amd64.go | 1 + src/runtime/defs_linux_arm.go | 23 ++++++++++++++++++---- src/runtime/defs_linux_arm64.go | 1 + src/runtime/defs_linux_loong64.go | 1 + src/runtime/defs_linux_mips64x.go | 1 + src/runtime/defs_linux_mipsx.go | 23 ++++++++++++++++++---- src/runtime/defs_linux_ppc64.go | 1 + src/runtime/defs_linux_ppc64le.go | 1 + src/runtime/defs_linux_riscv64.go | 1 + src/runtime/defs_linux_s390x.go | 1 + src/runtime/os_linux.go | 5 +---- src/runtime/os_linux_futex32.go | 40 +++++++++++++++++++++++++++++++++++++++ src/runtime/os_linux_futex64.go | 14 ++++++++++++++ src/runtime/sys_linux_386.s | 21 ++++++++++++++++++-- src/runtime/sys_linux_arm.s | 21 ++++++++++++++++++-- src/runtime/sys_linux_mipsx.s | 28 +++++++++++++++++++++++++-- 19 files changed, 186 insertions(+), 22 deletions(-) create mode 100644 src/runtime/os_linux_futex32.go create mode 100644 src/runtime/os_linux_futex64.go (limited to 'src/runtime') diff --git a/src/runtime/defs2_linux.go b/src/runtime/defs2_linux.go index 5d6730a7ad..597073c39d 100644 --- a/src/runtime/defs2_linux.go +++ b/src/runtime/defs2_linux.go @@ -48,6 +48,7 @@ const ( EINTR = C.EINTR EAGAIN = C.EAGAIN ENOMEM = C.ENOMEM + ENOSYS = C.ENOSYS PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ diff --git a/src/runtime/defs_linux.go b/src/runtime/defs_linux.go index 296fcb4bfd..d2b619ecab 100644 --- a/src/runtime/defs_linux.go +++ b/src/runtime/defs_linux.go @@ -37,6 +37,7 @@ const ( EINTR = C.EINTR EAGAIN = C.EAGAIN ENOMEM = C.ENOMEM + ENOSYS = C.ENOSYS PROT_NONE = C.PROT_NONE PROT_READ = C.PROT_READ diff --git a/src/runtime/defs_linux_386.go b/src/runtime/defs_linux_386.go index 5fef55610f..e902d8175c 100644 --- a/src/runtime/defs_linux_386.go +++ b/src/runtime/defs_linux_386.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 @@ -136,16 +137,30 @@ type fpstate struct { anon0 [48]byte } -type timespec struct { +// The timespec structs and types are defined in Linux in +// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h. +type timespec32 struct { tv_sec int32 tv_nsec int32 } //go:nosplit -func (ts *timespec) setNsec(ns int64) { +func (ts *timespec32) setNsec(ns int64) { ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec) } +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +//go:nosplit +func (ts *timespec) setNsec(ns int64) { + var newNS int32 + ts.tv_sec = int64(timediv(ns, 1e9, &newNS)) + ts.tv_nsec = int64(newNS) +} + type timeval struct { tv_sec int32 tv_usec int32 @@ -223,8 +238,8 @@ type ucontext struct { } type itimerspec struct { - it_interval timespec - it_value timespec + it_interval timespec32 + it_value timespec32 } type itimerval struct { diff --git a/src/runtime/defs_linux_amd64.go b/src/runtime/defs_linux_amd64.go index dce7799b6a..9a908c9400 100644 --- a/src/runtime/defs_linux_amd64.go +++ b/src/runtime/defs_linux_amd64.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_arm.go b/src/runtime/defs_linux_arm.go index 71cf8c6d50..35c4faf964 100644 --- a/src/runtime/defs_linux_arm.go +++ b/src/runtime/defs_linux_arm.go @@ -11,6 +11,7 @@ const ( _EINTR = 0x4 _ENOMEM = 0xc _EAGAIN = 0xb + _ENOSYS = 0x26 _PROT_NONE = 0 _PROT_READ = 0x1 @@ -95,16 +96,30 @@ const ( _SOCK_DGRAM = 0x2 ) -type timespec struct { +// The timespec structs and types are defined in Linux in +// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h. +type timespec32 struct { tv_sec int32 tv_nsec int32 } //go:nosplit -func (ts *timespec) setNsec(ns int64) { +func (ts *timespec32) setNsec(ns int64) { ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec) } +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +//go:nosplit +func (ts *timespec) setNsec(ns int64) { + var newNS int32 + ts.tv_sec = int64(timediv(ns, 1e9, &newNS)) + ts.tv_nsec = int64(newNS) +} + type stackt struct { ss_sp *byte ss_flags int32 @@ -155,8 +170,8 @@ func (tv *timeval) set_usec(x int32) { } type itimerspec struct { - it_interval timespec - it_value timespec + it_interval timespec32 + it_value timespec32 } type itimerval struct { diff --git a/src/runtime/defs_linux_arm64.go b/src/runtime/defs_linux_arm64.go index 606cd70494..4992e91ea6 100644 --- a/src/runtime/defs_linux_arm64.go +++ b/src/runtime/defs_linux_arm64.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_loong64.go b/src/runtime/defs_linux_loong64.go index b983725160..670d4c318d 100644 --- a/src/runtime/defs_linux_loong64.go +++ b/src/runtime/defs_linux_loong64.go @@ -10,6 +10,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_mips64x.go b/src/runtime/defs_linux_mips64x.go index 8a0af41234..7449d2cfac 100644 --- a/src/runtime/defs_linux_mips64x.go +++ b/src/runtime/defs_linux_mips64x.go @@ -12,6 +12,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_mipsx.go b/src/runtime/defs_linux_mipsx.go index 8322beab2b..cec504c885 100644 --- a/src/runtime/defs_linux_mipsx.go +++ b/src/runtime/defs_linux_mipsx.go @@ -12,6 +12,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 @@ -93,16 +94,30 @@ const ( _SIGEV_THREAD_ID = 0x4 ) -type timespec struct { +// The timespec structs and types are defined in Linux in +// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h. +type timespec32 struct { tv_sec int32 tv_nsec int32 } //go:nosplit -func (ts *timespec) setNsec(ns int64) { +func (ts *timespec32) setNsec(ns int64) { ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec) } +type timespec struct { + tv_sec int64 + tv_nsec int64 +} + +//go:nosplit +func (ts *timespec) setNsec(ns int64) { + var newNS int32 + ts.tv_sec = int64(timediv(ns, 1e9, &newNS)) + ts.tv_nsec = int64(newNS) +} + type timeval struct { tv_sec int32 tv_usec int32 @@ -138,8 +153,8 @@ type siginfo struct { } type itimerspec struct { - it_interval timespec - it_value timespec + it_interval timespec32 + it_value timespec32 } type itimerval struct { diff --git a/src/runtime/defs_linux_ppc64.go b/src/runtime/defs_linux_ppc64.go index f87924affe..dc45f37b7c 100644 --- a/src/runtime/defs_linux_ppc64.go +++ b/src/runtime/defs_linux_ppc64.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_ppc64le.go b/src/runtime/defs_linux_ppc64le.go index f87924affe..dc45f37b7c 100644 --- a/src/runtime/defs_linux_ppc64le.go +++ b/src/runtime/defs_linux_ppc64le.go @@ -9,6 +9,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_riscv64.go b/src/runtime/defs_linux_riscv64.go index 29b1ef2a50..b73e208ac3 100644 --- a/src/runtime/defs_linux_riscv64.go +++ b/src/runtime/defs_linux_riscv64.go @@ -10,6 +10,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/defs_linux_s390x.go b/src/runtime/defs_linux_s390x.go index b0280213b3..c03d0f2117 100644 --- a/src/runtime/defs_linux_s390x.go +++ b/src/runtime/defs_linux_s390x.go @@ -10,6 +10,7 @@ const ( _EINTR = 0x4 _EAGAIN = 0xb _ENOMEM = 0xc + _ENOSYS = 0x26 _PROT_NONE = 0x0 _PROT_READ = 0x1 diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index f9fe1b5f33..080dd96532 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -40,9 +40,6 @@ type mOS struct { waitsema uint32 // semaphore for parking on locks } -//go:noescape -func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32 - // Linux futex. // // futexsleep(uint32 *addr, uint32 val) @@ -79,7 +76,7 @@ func futexsleep(addr *uint32, val uint32, ns int64) { var ts timespec ts.setNsec(ns) - futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0) + futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, &ts, nil, 0) } // If any procs are sleeping on addr, wake up at most cnt. diff --git a/src/runtime/os_linux_futex32.go b/src/runtime/os_linux_futex32.go new file mode 100644 index 0000000000..fdf99e5669 --- /dev/null +++ b/src/runtime/os_linux_futex32.go @@ -0,0 +1,40 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (386 || arm || mips || mipsle || ppc) + +package runtime + +import ( + "internal/runtime/atomic" + "unsafe" +) + +//go:noescape +func futex_time32(addr unsafe.Pointer, op int32, val uint32, ts *timespec32, addr2 unsafe.Pointer, val3 uint32) int32 + +//go:noescape +func futex_time64(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 + +var is32bitOnly atomic.Bool + +//go:nosplit +func futex(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 { + if !is32bitOnly.Load() { + ret := futex_time64(addr, op, val, ts, addr2, val3) + // futex_time64 is only supported on Linux 5.0+ + if ret != -_ENOSYS { + return ret + } + is32bitOnly.Store(true) + } + // Downgrade ts. + var ts32 timespec32 + var pts32 *timespec32 + if ts != nil { + ts32.setNsec(ts.tv_sec*1e9 + ts.tv_nsec) + pts32 = &ts32 + } + return futex_time32(addr, op, val, pts32, addr2, val3) +} diff --git a/src/runtime/os_linux_futex64.go b/src/runtime/os_linux_futex64.go new file mode 100644 index 0000000000..487d0e0397 --- /dev/null +++ b/src/runtime/os_linux_futex64.go @@ -0,0 +1,14 @@ +// Copyright 2025 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && !(386 || arm || mips || mipsle || ppc || s390) + +package runtime + +import ( + "unsafe" +) + +//go:noescape +func futex(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 8e832687e0..1c3f1ff3e6 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -48,6 +48,7 @@ #define SYS_madvise 219 #define SYS_gettid 224 #define SYS_futex 240 +#define SYS_futex_time64 422 #define SYS_sched_getaffinity 242 #define SYS_set_thread_area 243 #define SYS_exit_group 252 @@ -532,9 +533,10 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL AX, ret+12(FP) RET +// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME // int32 futex(int32 *uaddr, int32 op, int32 val, -// struct timespec *timeout, int32 *uaddr2, int32 val2); -TEXT runtime·futex(SB),NOSPLIT,$0 +// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time32(SB),NOSPLIT,$0 MOVL $SYS_futex, AX MOVL addr+0(FP), BX MOVL op+4(FP), CX @@ -546,6 +548,21 @@ TEXT runtime·futex(SB),NOSPLIT,$0 MOVL AX, ret+24(FP) RET +// Linux: kernel/futex/syscalls.c +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time64(SB),NOSPLIT,$0 + MOVL $SYS_futex_time64, AX + MOVL addr+0(FP), BX + MOVL op+4(FP), CX + MOVL val+8(FP), DX + MOVL ts+12(FP), SI + MOVL addr2+16(FP), DI + MOVL val3+20(FP), BP + INVOKE_SYSCALL + MOVL AX, ret+24(FP) + RET + // int32 clone(int32 flags, void *stack, M *mp, G *gp, void (*fn)(void)); TEXT runtime·clone(SB),NOSPLIT,$0 MOVL $SYS_clone, AX diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index 992d32ab6c..44b56ccb9f 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -30,6 +30,7 @@ #define SYS_sigaltstack (SYS_BASE + 186) #define SYS_mmap2 (SYS_BASE + 192) #define SYS_futex (SYS_BASE + 240) +#define SYS_futex_time64 (SYS_BASE + 422) #define SYS_exit_group (SYS_BASE + 248) #define SYS_munmap (SYS_BASE + 91) #define SYS_madvise (SYS_BASE + 220) @@ -403,9 +404,10 @@ finish: RET +// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME // int32 futex(int32 *uaddr, int32 op, int32 val, -// struct timespec *timeout, int32 *uaddr2, int32 val2); -TEXT runtime·futex(SB),NOSPLIT,$0 +// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time32(SB),NOSPLIT,$0 MOVW addr+0(FP), R0 MOVW op+4(FP), R1 MOVW val+8(FP), R2 @@ -417,6 +419,21 @@ TEXT runtime·futex(SB),NOSPLIT,$0 MOVW R0, ret+24(FP) RET +// Linux: kernel/futex/syscalls.c +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time64(SB),NOSPLIT,$0 + MOVW addr+0(FP), R0 + MOVW op+4(FP), R1 + MOVW val+8(FP), R2 + MOVW ts+12(FP), R3 + MOVW addr2+16(FP), R4 + MOVW val3+20(FP), R5 + MOVW $SYS_futex_time64, R7 + SWI $0 + MOVW R0, ret+24(FP) + RET + // int32 clone(int32 flags, void *stack, M *mp, G *gp, void (*fn)(void)); TEXT runtime·clone(SB),NOSPLIT,$0 MOVW flags+0(FP), R0 diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index 5e6b6c1504..6f11841efc 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -34,6 +34,7 @@ #define SYS_mincore 4217 #define SYS_gettid 4222 #define SYS_futex 4238 +#define SYS_futex_time64 4422 #define SYS_sched_getaffinity 4240 #define SYS_exit_group 4246 #define SYS_timer_create 4257 @@ -362,8 +363,10 @@ TEXT runtime·madvise(SB),NOSPLIT,$0-16 MOVW R2, ret+12(FP) RET -// int32 futex(int32 *uaddr, int32 op, int32 val, struct timespec *timeout, int32 *uaddr2, int32 val2); -TEXT runtime·futex(SB),NOSPLIT,$20-28 +// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time32(SB),NOSPLIT,$20-28 MOVW addr+0(FP), R4 MOVW op+4(FP), R5 MOVW val+8(FP), R6 @@ -382,6 +385,27 @@ TEXT runtime·futex(SB),NOSPLIT,$20-28 MOVW R2, ret+24(FP) RET +// Linux: kernel/futex/syscalls.c +// int32 futex(int32 *uaddr, int32 op, int32 val, +// struct timespec *timeout, int32 *uaddr2, int32 val2); +TEXT runtime·futex_time64(SB),NOSPLIT,$20-28 + MOVW addr+0(FP), R4 + MOVW op+4(FP), R5 + MOVW val+8(FP), R6 + MOVW ts+12(FP), R7 + + MOVW addr2+16(FP), R8 + MOVW val3+20(FP), R9 + + MOVW R8, 16(R29) + MOVW R9, 20(R29) + + MOVW $SYS_futex_time64, R2 + SYSCALL + BEQ R7, 2(PC) + SUBU R2, R0, R2 // caller expects negative errno + MOVW R2, ret+24(FP) + RET // int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void)); TEXT runtime·clone(SB),NOSPLIT|NOFRAME,$0-24 -- cgit v1.3-5-g9baa From 9693b94be057b58f35bcc4a81c937495b93d588e Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Tue, 16 Sep 2025 15:07:08 -0400 Subject: runtime: include stderr when objdump fails If objdump panics, we want the panic included in the test log. Change-Id: I6a6a636c13c5ba08acaa1c6c8714541a8e91542b Reviewed-on: https://go-review.googlesource.com/c/go/+/704338 Reviewed-by: Florian Lehner Auto-Submit: Michael Pratt Reviewed-by: Michael Knyszek LUCI-TryBot-Result: Go LUCI --- src/runtime/unsafepoint_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/unsafepoint_test.go b/src/runtime/unsafepoint_test.go index 2c97adead8..79f0171854 100644 --- a/src/runtime/unsafepoint_test.go +++ b/src/runtime/unsafepoint_test.go @@ -43,7 +43,7 @@ func TestUnsafePoint(t *testing.T) { cmd := exec.Command(testenv.GoToolPath(t), "tool", "objdump", "-s", "setGlobalPointer", os.Args[0]) out, err := cmd.CombinedOutput() if err != nil { - t.Fatalf("can't objdump %v", err) + t.Fatalf("can't objdump %v:\n%s", err, out) } lines := strings.Split(string(out), "\n")[1:] -- cgit v1.3-5-g9baa From e23edf5e55703cc6ddbb86a198a35487b34d863b Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Mon, 22 Sep 2025 17:36:03 +0000 Subject: runtime: don't re-read metrics before check in TestReadMetricsSched The two remaining popular flakes in TestReadMetricsSched are with waiting and not-in-go goroutines. I believe the reason for both of these is pollution due to other goroutines in the test binary, and we can be a little bit more robust to them. In particular, both of these tests spin until there's a particular condition met in the metrics. Then they both immediately re-read the metrics. The metrics being checked are not guaranteed to stay that way in the re-read. For example, for the waiting case, it's possible for >1000 to be waiting, but then some leftover goroutines from a previous test wake up off of some condition, causing the number to dip again on the re-read. This is supported by the fact that in some of these failures, the test takes very little time to execute (the condition that there are at least 1000 waiting goroutines is almost immediately satisfied) but it still fails (the re-read causes us to notice that there are <1000 waiting goroutines). This change makes the test more robust by not performing this re-read. This means more possible false negatives, but the error cases we're looking for (bad metrics) should still show up with some reasonable probability when something *is* truly wrong. For #75049. Change-Id: Idc3a8030bed8da51f24322efe15f3ff13da05623 Reviewed-on: https://go-review.googlesource.com/c/go/+/705875 Reviewed-by: Michael Pratt Auto-Submit: Michael Knyszek LUCI-TryBot-Result: Go LUCI --- src/runtime/metrics_test.go | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 3989b79293..af042f4445 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -1760,8 +1760,6 @@ func TestReadMetricsSched(t *testing.T) { metrics.Read(s[:]) return s[notInGo].Value.Uint64() >= count }) - - metrics.Read(s[:]) logMetrics(t, s[:]) check(t, &s[notInGo], count, count+generalSlack) @@ -1782,8 +1780,6 @@ func TestReadMetricsSched(t *testing.T) { metrics.Read(s[:]) return s[waiting].Value.Uint64() >= waitingCount }) - - metrics.Read(s[:]) logMetrics(t, s[:]) check(t, &s[waiting], waitingCount, waitingCount+waitingSlack) -- cgit v1.3-5-g9baa