aboutsummaryrefslogtreecommitdiff
path: root/src/runtime
diff options
context:
space:
mode:
authorCherry Mui <cherryyz@google.com>2025-09-23 10:32:03 -0400
committerCherry Mui <cherryyz@google.com>2025-09-23 10:32:03 -0400
commit2d8cb80d7c4af3dbcb507783938ceb0e071f64e3 (patch)
tree719d86801da431f6ba11a84a3b66c60b4e5c1f38 /src/runtime
parent63a09d6d3d68acedfc9e5fd2daf6febc35aca1d6 (diff)
parent9b2d39b75bcc8ced3eaab1c841d7d62e27867931 (diff)
downloadgo-2d8cb80d7c4af3dbcb507783938ceb0e071f64e3.tar.xz
[dev.simd] all: merge master (9b2d39b) into dev.simd
Conflicts: - src/internal/buildcfg/exp.go Merge List: + 2025-09-22 9b2d39b75b cmd/compile/internal/ssa: match style and formatting + 2025-09-22 e23edf5e55 runtime: don't re-read metrics before check in TestReadMetricsSched + 2025-09-22 177cd8d763 log/slog: use a pooled json encoder + 2025-09-22 2353c15785 cmd/cgo/internal/test: skip TestMultipleAssign when using UCRT on Windows + 2025-09-22 32dfd69282 cmd/dist: disable FIPS 140-3 mode when testing maphash with purego + 2025-09-19 7f6ff5ec3e cmd/compile: fix doc word + 2025-09-19 9693b94be0 runtime: include stderr when objdump fails + 2025-09-19 8616981ce6 log/slog: optimize slog Level.String() to avoid fmt.Sprintf + 2025-09-19 b8af744360 testing: fix example for unexported identifier + 2025-09-19 51dc5bfe6c Revert "cmd/go: disable cgo by default if CC unset and DefaultCC doesn't exist" + 2025-09-19 ee7bf06cb3 time: improve ParseDuration performance for invalid input + 2025-09-19 f9e61a9a32 cmd/compile: duplicate nil check to two branches of write barrier + 2025-09-18 3cf1aaf8b9 runtime: use futexes with 64-bit time on Linux + 2025-09-18 0ab038af62 cmd/compile/internal/abi: use clear built-in + 2025-09-18 00bf24fdca bytes: use clear in test + 2025-09-18 f9701d21d2 crypto: use clear built-in + 2025-09-18 a58afe44fa net: fix testHookCanceledDial race + 2025-09-18 3203a5da29 net/http: avoid connCount underflow race + 2025-09-18 8ca209ec39 context: don't return a non-nil from Err before Done is closed + 2025-09-18 3032894e04 runtime: make explicit nil check in heapSetTypeSmallHeader + 2025-09-17 ef05b66d61 cmd/internal/obj/riscv: add support for Zicond instructions + 2025-09-17 78ef487a6f cmd/compile: fix the issue of shift amount exceeding the valid range + 2025-09-17 77aac7bb75 runtime: don't enable heap randomization if MSAN or ASAN is enabled + 2025-09-17 465b85eb76 runtime: fix CheckScavengedBitsCleared with randomized heap base + 2025-09-17 909704b85e encoding/json/v2: fix typo in comment + 2025-09-17 3db5979e8c testing: use reflect.TypeAssert and reflect.TypeFor + 2025-09-17 6a8dbbecbf path/filepath: fix EvalSymlinks to return ENOTDIR on plan9 + 2025-09-17 bffe7ad9f1 go/parser: Add TestBothLineAndLeadComment + 2025-09-17 02a888e820 go/ast: document that (*ast.File).Comments is sorted by position + 2025-09-16 594deca981 cmd/link: simplify PE relocations mapping + 2025-09-16 9df1a289ac go/parser: simplify expectSemi + 2025-09-16 72ba117bda internal/buildcfg: enable randomizedHeapBase64 by default + 2025-09-16 796ea3bc2e os/user: align test file name and build tags + 2025-09-16 a69395eab2 runtime/_mkmalloc: add a copy of cloneNode + 2025-09-16 cbdad4fc3c cmd/go: check pattern for utf8 validity before call regexp.MustCompile + 2025-09-16 c2d85eb999 cmd/go: disable cgo by default if CC unset and DefaultCC doesn't exist + 2025-09-16 ac82fe68aa bytes,strings: remove reference to non-existent SplitFunc + 2025-09-16 0b26678db2 cmd/compile: fix mips zerorange implementation + 2025-09-16 e2cfc1eb3a cmd/internal/obj/riscv: improve handling of float point moves + 2025-09-16 281c632e6e crypto/x509/internal/macos: standardize package name + 2025-09-16 61dc7fe30d iter: document that calling yield after terminated range loop causes runtime panic Change-Id: Ic06019efc855913632003f41eb10c746b3410b0a
Diffstat (limited to 'src/runtime')
-rw-r--r--src/runtime/_mkmalloc/astutil/clone.go73
-rw-r--r--src/runtime/defs2_linux.go1
-rw-r--r--src/runtime/defs_linux.go1
-rw-r--r--src/runtime/defs_linux_386.go23
-rw-r--r--src/runtime/defs_linux_amd64.go1
-rw-r--r--src/runtime/defs_linux_arm.go23
-rw-r--r--src/runtime/defs_linux_arm64.go1
-rw-r--r--src/runtime/defs_linux_loong64.go1
-rw-r--r--src/runtime/defs_linux_mips64x.go1
-rw-r--r--src/runtime/defs_linux_mipsx.go23
-rw-r--r--src/runtime/defs_linux_ppc64.go1
-rw-r--r--src/runtime/defs_linux_ppc64le.go1
-rw-r--r--src/runtime/defs_linux_riscv64.go1
-rw-r--r--src/runtime/defs_linux_s390x.go1
-rw-r--r--src/runtime/export_test.go42
-rw-r--r--src/runtime/malloc.go2
-rw-r--r--src/runtime/mbitmap.go20
-rw-r--r--src/runtime/metrics_test.go4
-rw-r--r--src/runtime/os_linux.go5
-rw-r--r--src/runtime/os_linux_futex32.go40
-rw-r--r--src/runtime/os_linux_futex64.go14
-rw-r--r--src/runtime/sys_linux_386.s21
-rw-r--r--src/runtime/sys_linux_arm.s21
-rw-r--r--src/runtime/sys_linux_mipsx.s28
-rw-r--r--src/runtime/unsafepoint_test.go2
25 files changed, 312 insertions, 39 deletions
diff --git a/src/runtime/_mkmalloc/astutil/clone.go b/src/runtime/_mkmalloc/astutil/clone.go
new file mode 100644
index 0000000000..16ea7163ca
--- /dev/null
+++ b/src/runtime/_mkmalloc/astutil/clone.go
@@ -0,0 +1,73 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is a copy of golang.org/x/tools/internal/astutil/clone.go
+
+package astutil
+
+import (
+ "go/ast"
+ "reflect"
+)
+
+// CloneNode returns a deep copy of a Node.
+// It omits pointers to ast.{Scope,Object} variables.
+func CloneNode[T ast.Node](n T) T {
+ return cloneNode(n).(T)
+}
+
+func cloneNode(n ast.Node) ast.Node {
+ var clone func(x reflect.Value) reflect.Value
+ set := func(dst, src reflect.Value) {
+ src = clone(src)
+ if src.IsValid() {
+ dst.Set(src)
+ }
+ }
+ clone = func(x reflect.Value) reflect.Value {
+ switch x.Kind() {
+ case reflect.Pointer:
+ if x.IsNil() {
+ return x
+ }
+ // Skip fields of types potentially involved in cycles.
+ switch x.Interface().(type) {
+ case *ast.Object, *ast.Scope:
+ return reflect.Zero(x.Type())
+ }
+ y := reflect.New(x.Type().Elem())
+ set(y.Elem(), x.Elem())
+ return y
+
+ case reflect.Struct:
+ y := reflect.New(x.Type()).Elem()
+ for i := 0; i < x.Type().NumField(); i++ {
+ set(y.Field(i), x.Field(i))
+ }
+ return y
+
+ case reflect.Slice:
+ if x.IsNil() {
+ return x
+ }
+ y := reflect.MakeSlice(x.Type(), x.Len(), x.Cap())
+ for i := 0; i < x.Len(); i++ {
+ set(y.Index(i), x.Index(i))
+ }
+ return y
+
+ case reflect.Interface:
+ y := reflect.New(x.Type()).Elem()
+ set(y, x.Elem())
+ return y
+
+ case reflect.Array, reflect.Chan, reflect.Func, reflect.Map, reflect.UnsafePointer:
+ panic(x) // unreachable in AST
+
+ default:
+ return x // bool, string, number
+ }
+ }
+ return clone(reflect.ValueOf(n)).Interface().(ast.Node)
+}
diff --git a/src/runtime/defs2_linux.go b/src/runtime/defs2_linux.go
index 5d6730a7ad..597073c39d 100644
--- a/src/runtime/defs2_linux.go
+++ b/src/runtime/defs2_linux.go
@@ -48,6 +48,7 @@ const (
EINTR = C.EINTR
EAGAIN = C.EAGAIN
ENOMEM = C.ENOMEM
+ ENOSYS = C.ENOSYS
PROT_NONE = C.PROT_NONE
PROT_READ = C.PROT_READ
diff --git a/src/runtime/defs_linux.go b/src/runtime/defs_linux.go
index 296fcb4bfd..d2b619ecab 100644
--- a/src/runtime/defs_linux.go
+++ b/src/runtime/defs_linux.go
@@ -37,6 +37,7 @@ const (
EINTR = C.EINTR
EAGAIN = C.EAGAIN
ENOMEM = C.ENOMEM
+ ENOSYS = C.ENOSYS
PROT_NONE = C.PROT_NONE
PROT_READ = C.PROT_READ
diff --git a/src/runtime/defs_linux_386.go b/src/runtime/defs_linux_386.go
index 5fef55610f..e902d8175c 100644
--- a/src/runtime/defs_linux_386.go
+++ b/src/runtime/defs_linux_386.go
@@ -9,6 +9,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
@@ -136,16 +137,30 @@ type fpstate struct {
anon0 [48]byte
}
-type timespec struct {
+// The timespec structs and types are defined in Linux in
+// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h.
+type timespec32 struct {
tv_sec int32
tv_nsec int32
}
//go:nosplit
-func (ts *timespec) setNsec(ns int64) {
+func (ts *timespec32) setNsec(ns int64) {
ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec)
}
+type timespec struct {
+ tv_sec int64
+ tv_nsec int64
+}
+
+//go:nosplit
+func (ts *timespec) setNsec(ns int64) {
+ var newNS int32
+ ts.tv_sec = int64(timediv(ns, 1e9, &newNS))
+ ts.tv_nsec = int64(newNS)
+}
+
type timeval struct {
tv_sec int32
tv_usec int32
@@ -223,8 +238,8 @@ type ucontext struct {
}
type itimerspec struct {
- it_interval timespec
- it_value timespec
+ it_interval timespec32
+ it_value timespec32
}
type itimerval struct {
diff --git a/src/runtime/defs_linux_amd64.go b/src/runtime/defs_linux_amd64.go
index dce7799b6a..9a908c9400 100644
--- a/src/runtime/defs_linux_amd64.go
+++ b/src/runtime/defs_linux_amd64.go
@@ -9,6 +9,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_arm.go b/src/runtime/defs_linux_arm.go
index 71cf8c6d50..35c4faf964 100644
--- a/src/runtime/defs_linux_arm.go
+++ b/src/runtime/defs_linux_arm.go
@@ -11,6 +11,7 @@ const (
_EINTR = 0x4
_ENOMEM = 0xc
_EAGAIN = 0xb
+ _ENOSYS = 0x26
_PROT_NONE = 0
_PROT_READ = 0x1
@@ -95,16 +96,30 @@ const (
_SOCK_DGRAM = 0x2
)
-type timespec struct {
+// The timespec structs and types are defined in Linux in
+// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h.
+type timespec32 struct {
tv_sec int32
tv_nsec int32
}
//go:nosplit
-func (ts *timespec) setNsec(ns int64) {
+func (ts *timespec32) setNsec(ns int64) {
ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec)
}
+type timespec struct {
+ tv_sec int64
+ tv_nsec int64
+}
+
+//go:nosplit
+func (ts *timespec) setNsec(ns int64) {
+ var newNS int32
+ ts.tv_sec = int64(timediv(ns, 1e9, &newNS))
+ ts.tv_nsec = int64(newNS)
+}
+
type stackt struct {
ss_sp *byte
ss_flags int32
@@ -155,8 +170,8 @@ func (tv *timeval) set_usec(x int32) {
}
type itimerspec struct {
- it_interval timespec
- it_value timespec
+ it_interval timespec32
+ it_value timespec32
}
type itimerval struct {
diff --git a/src/runtime/defs_linux_arm64.go b/src/runtime/defs_linux_arm64.go
index 606cd70494..4992e91ea6 100644
--- a/src/runtime/defs_linux_arm64.go
+++ b/src/runtime/defs_linux_arm64.go
@@ -9,6 +9,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_loong64.go b/src/runtime/defs_linux_loong64.go
index b983725160..670d4c318d 100644
--- a/src/runtime/defs_linux_loong64.go
+++ b/src/runtime/defs_linux_loong64.go
@@ -10,6 +10,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_mips64x.go b/src/runtime/defs_linux_mips64x.go
index 8a0af41234..7449d2cfac 100644
--- a/src/runtime/defs_linux_mips64x.go
+++ b/src/runtime/defs_linux_mips64x.go
@@ -12,6 +12,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_mipsx.go b/src/runtime/defs_linux_mipsx.go
index 8322beab2b..cec504c885 100644
--- a/src/runtime/defs_linux_mipsx.go
+++ b/src/runtime/defs_linux_mipsx.go
@@ -12,6 +12,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
@@ -93,16 +94,30 @@ const (
_SIGEV_THREAD_ID = 0x4
)
-type timespec struct {
+// The timespec structs and types are defined in Linux in
+// include/uapi/linux/time_types.h and include/uapi/asm-generic/posix_types.h.
+type timespec32 struct {
tv_sec int32
tv_nsec int32
}
//go:nosplit
-func (ts *timespec) setNsec(ns int64) {
+func (ts *timespec32) setNsec(ns int64) {
ts.tv_sec = timediv(ns, 1e9, &ts.tv_nsec)
}
+type timespec struct {
+ tv_sec int64
+ tv_nsec int64
+}
+
+//go:nosplit
+func (ts *timespec) setNsec(ns int64) {
+ var newNS int32
+ ts.tv_sec = int64(timediv(ns, 1e9, &newNS))
+ ts.tv_nsec = int64(newNS)
+}
+
type timeval struct {
tv_sec int32
tv_usec int32
@@ -138,8 +153,8 @@ type siginfo struct {
}
type itimerspec struct {
- it_interval timespec
- it_value timespec
+ it_interval timespec32
+ it_value timespec32
}
type itimerval struct {
diff --git a/src/runtime/defs_linux_ppc64.go b/src/runtime/defs_linux_ppc64.go
index f87924affe..dc45f37b7c 100644
--- a/src/runtime/defs_linux_ppc64.go
+++ b/src/runtime/defs_linux_ppc64.go
@@ -9,6 +9,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_ppc64le.go b/src/runtime/defs_linux_ppc64le.go
index f87924affe..dc45f37b7c 100644
--- a/src/runtime/defs_linux_ppc64le.go
+++ b/src/runtime/defs_linux_ppc64le.go
@@ -9,6 +9,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_riscv64.go b/src/runtime/defs_linux_riscv64.go
index 29b1ef2a50..b73e208ac3 100644
--- a/src/runtime/defs_linux_riscv64.go
+++ b/src/runtime/defs_linux_riscv64.go
@@ -10,6 +10,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/defs_linux_s390x.go b/src/runtime/defs_linux_s390x.go
index b0280213b3..c03d0f2117 100644
--- a/src/runtime/defs_linux_s390x.go
+++ b/src/runtime/defs_linux_s390x.go
@@ -10,6 +10,7 @@ const (
_EINTR = 0x4
_EAGAIN = 0xb
_ENOMEM = 0xc
+ _ENOSYS = 0x26
_PROT_NONE = 0x0
_PROT_READ = 0x1
diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go
index fc77b535da..a7b51dd6c7 100644
--- a/src/runtime/export_test.go
+++ b/src/runtime/export_test.go
@@ -9,7 +9,6 @@ package runtime
import (
"internal/abi"
"internal/goarch"
- "internal/goexperiment"
"internal/goos"
"internal/runtime/atomic"
"internal/runtime/gc"
@@ -1122,8 +1121,6 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) {
// Lock so that we can safely access the bitmap.
lock(&mheap_.lock)
- heapBase := mheap_.pages.inUse.ranges[0].base.addr()
- secondArenaBase := arenaBase(arenaIndex(heapBase) + 1)
chunkLoop:
for i := mheap_.pages.start; i < mheap_.pages.end; i++ {
chunk := mheap_.pages.tryChunkOf(i)
@@ -1140,14 +1137,6 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) {
want := chunk.scavenged[j] &^ chunk.pallocBits[j]
got := chunk.scavenged[j]
if want != got {
- // When goexperiment.RandomizedHeapBase64 is set we use a
- // series of padding pages to generate randomized heap base
- // address which have both the alloc and scav bits set. If
- // we see this for a chunk between the address of the heap
- // base, and the address of the second arena continue.
- if goexperiment.RandomizedHeapBase64 && (cb >= heapBase && cb < secondArenaBase) {
- continue
- }
ok = false
if n >= len(mismatches) {
break chunkLoop
@@ -1165,6 +1154,37 @@ func CheckScavengedBitsCleared(mismatches []BitsMismatch) (n int, ok bool) {
getg().m.mallocing--
})
+
+ if randomizeHeapBase && len(mismatches) > 0 {
+ // When goexperiment.RandomizedHeapBase64 is set we use a series of
+ // padding pages to generate randomized heap base address which have
+ // both the alloc and scav bits set. Because of this we expect exactly
+ // one arena will have mismatches, so check for that explicitly and
+ // remove the mismatches if that property holds. If we see more than one
+ // arena with this property, that is an indication something has
+ // actually gone wrong, so return the mismatches.
+ //
+ // We do this, instead of ignoring the mismatches in the chunkLoop, because
+ // it's not easy to determine which arena we added the padding pages to
+ // programmatically, without explicitly recording the base address somewhere
+ // in a global variable (which we'd rather not do as the address of that variable
+ // is likely to be somewhat predictable, potentially defeating the purpose
+ // of our randomization).
+ affectedArenas := map[arenaIdx]bool{}
+ for _, mismatch := range mismatches {
+ if mismatch.Base > 0 {
+ affectedArenas[arenaIndex(mismatch.Base)] = true
+ }
+ }
+ if len(affectedArenas) == 1 {
+ ok = true
+ // zero the mismatches
+ for i := range n {
+ mismatches[i] = BitsMismatch{}
+ }
+ }
+ }
+
return
}
diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index ec5f0765ba..5b5a633d9a 100644
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -349,7 +349,7 @@ const (
// randomizeHeapBase indicates if the heap base address should be randomized.
// See comment in mallocinit for how the randomization is performed.
- randomizeHeapBase = goexperiment.RandomizedHeapBase64 && goarch.PtrSize == 8 && !isSbrkPlatform
+ randomizeHeapBase = goexperiment.RandomizedHeapBase64 && goarch.PtrSize == 8 && !isSbrkPlatform && !raceenabled && !msanenabled && !asanenabled
// randHeapBasePrefixMask is used to extract the top byte of the randomized
// heap base address.
diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index 9872e5297f..508de9a115 100644
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -714,6 +714,26 @@ func heapSetTypeNoHeader(x, dataSize uintptr, typ *_type, span *mspan) uintptr {
}
func heapSetTypeSmallHeader(x, dataSize uintptr, typ *_type, header **_type, span *mspan) uintptr {
+ if header == nil {
+ // This nil check and throw is almost pointless. Normally we would
+ // expect header to never be nil. However, this is called on potentially
+ // freshly-allocated virtual memory. As of 2025, the compiler-inserted
+ // nil check is not a branch but a memory read that we expect to fault
+ // if the pointer really is nil.
+ //
+ // However, this causes a read of the page, and operating systems may
+ // take it as a hint to back the accessed memory with a read-only zero
+ // page. However, we immediately write to this memory, which can then
+ // force operating systems to have to update the page table and flush
+ // the TLB.
+ //
+ // This nil check is thus an explicit branch instead of what the compiler
+ // would insert circa 2025, which is a memory read instruction.
+ //
+ // See go.dev/issue/74375 for details of a similar issue in
+ // spanInlineMarkBits.
+ throw("runtime: pointer to heap type header nil?")
+ }
*header = typ
if doubleCheckHeapSetType {
doubleCheckHeapType(x, dataSize, typ, header, span)
diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go
index 3989b79293..af042f4445 100644
--- a/src/runtime/metrics_test.go
+++ b/src/runtime/metrics_test.go
@@ -1760,8 +1760,6 @@ func TestReadMetricsSched(t *testing.T) {
metrics.Read(s[:])
return s[notInGo].Value.Uint64() >= count
})
-
- metrics.Read(s[:])
logMetrics(t, s[:])
check(t, &s[notInGo], count, count+generalSlack)
@@ -1782,8 +1780,6 @@ func TestReadMetricsSched(t *testing.T) {
metrics.Read(s[:])
return s[waiting].Value.Uint64() >= waitingCount
})
-
- metrics.Read(s[:])
logMetrics(t, s[:])
check(t, &s[waiting], waitingCount, waitingCount+waitingSlack)
diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
index f9fe1b5f33..080dd96532 100644
--- a/src/runtime/os_linux.go
+++ b/src/runtime/os_linux.go
@@ -40,9 +40,6 @@ type mOS struct {
waitsema uint32 // semaphore for parking on locks
}
-//go:noescape
-func futex(addr unsafe.Pointer, op int32, val uint32, ts, addr2 unsafe.Pointer, val3 uint32) int32
-
// Linux futex.
//
// futexsleep(uint32 *addr, uint32 val)
@@ -79,7 +76,7 @@ func futexsleep(addr *uint32, val uint32, ns int64) {
var ts timespec
ts.setNsec(ns)
- futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, unsafe.Pointer(&ts), nil, 0)
+ futex(unsafe.Pointer(addr), _FUTEX_WAIT_PRIVATE, val, &ts, nil, 0)
}
// If any procs are sleeping on addr, wake up at most cnt.
diff --git a/src/runtime/os_linux_futex32.go b/src/runtime/os_linux_futex32.go
new file mode 100644
index 0000000000..fdf99e5669
--- /dev/null
+++ b/src/runtime/os_linux_futex32.go
@@ -0,0 +1,40 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && (386 || arm || mips || mipsle || ppc)
+
+package runtime
+
+import (
+ "internal/runtime/atomic"
+ "unsafe"
+)
+
+//go:noescape
+func futex_time32(addr unsafe.Pointer, op int32, val uint32, ts *timespec32, addr2 unsafe.Pointer, val3 uint32) int32
+
+//go:noescape
+func futex_time64(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32
+
+var is32bitOnly atomic.Bool
+
+//go:nosplit
+func futex(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32 {
+ if !is32bitOnly.Load() {
+ ret := futex_time64(addr, op, val, ts, addr2, val3)
+ // futex_time64 is only supported on Linux 5.0+
+ if ret != -_ENOSYS {
+ return ret
+ }
+ is32bitOnly.Store(true)
+ }
+ // Downgrade ts.
+ var ts32 timespec32
+ var pts32 *timespec32
+ if ts != nil {
+ ts32.setNsec(ts.tv_sec*1e9 + ts.tv_nsec)
+ pts32 = &ts32
+ }
+ return futex_time32(addr, op, val, pts32, addr2, val3)
+}
diff --git a/src/runtime/os_linux_futex64.go b/src/runtime/os_linux_futex64.go
new file mode 100644
index 0000000000..487d0e0397
--- /dev/null
+++ b/src/runtime/os_linux_futex64.go
@@ -0,0 +1,14 @@
+// Copyright 2025 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && !(386 || arm || mips || mipsle || ppc || s390)
+
+package runtime
+
+import (
+ "unsafe"
+)
+
+//go:noescape
+func futex(addr unsafe.Pointer, op int32, val uint32, ts *timespec, addr2 unsafe.Pointer, val3 uint32) int32
diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s
index 8e832687e0..1c3f1ff3e6 100644
--- a/src/runtime/sys_linux_386.s
+++ b/src/runtime/sys_linux_386.s
@@ -48,6 +48,7 @@
#define SYS_madvise 219
#define SYS_gettid 224
#define SYS_futex 240
+#define SYS_futex_time64 422
#define SYS_sched_getaffinity 242
#define SYS_set_thread_area 243
#define SYS_exit_group 252
@@ -532,9 +533,10 @@ TEXT runtime·madvise(SB),NOSPLIT,$0
MOVL AX, ret+12(FP)
RET
+// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME
// int32 futex(int32 *uaddr, int32 op, int32 val,
-// struct timespec *timeout, int32 *uaddr2, int32 val2);
-TEXT runtime·futex(SB),NOSPLIT,$0
+// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2);
+TEXT runtime·futex_time32(SB),NOSPLIT,$0
MOVL $SYS_futex, AX
MOVL addr+0(FP), BX
MOVL op+4(FP), CX
@@ -546,6 +548,21 @@ TEXT runtime·futex(SB),NOSPLIT,$0
MOVL AX, ret+24(FP)
RET
+// Linux: kernel/futex/syscalls.c
+// int32 futex(int32 *uaddr, int32 op, int32 val,
+// struct timespec *timeout, int32 *uaddr2, int32 val2);
+TEXT runtime·futex_time64(SB),NOSPLIT,$0
+ MOVL $SYS_futex_time64, AX
+ MOVL addr+0(FP), BX
+ MOVL op+4(FP), CX
+ MOVL val+8(FP), DX
+ MOVL ts+12(FP), SI
+ MOVL addr2+16(FP), DI
+ MOVL val3+20(FP), BP
+ INVOKE_SYSCALL
+ MOVL AX, ret+24(FP)
+ RET
+
// int32 clone(int32 flags, void *stack, M *mp, G *gp, void (*fn)(void));
TEXT runtime·clone(SB),NOSPLIT,$0
MOVL $SYS_clone, AX
diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s
index 992d32ab6c..44b56ccb9f 100644
--- a/src/runtime/sys_linux_arm.s
+++ b/src/runtime/sys_linux_arm.s
@@ -30,6 +30,7 @@
#define SYS_sigaltstack (SYS_BASE + 186)
#define SYS_mmap2 (SYS_BASE + 192)
#define SYS_futex (SYS_BASE + 240)
+#define SYS_futex_time64 (SYS_BASE + 422)
#define SYS_exit_group (SYS_BASE + 248)
#define SYS_munmap (SYS_BASE + 91)
#define SYS_madvise (SYS_BASE + 220)
@@ -403,9 +404,10 @@ finish:
RET
+// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME
// int32 futex(int32 *uaddr, int32 op, int32 val,
-// struct timespec *timeout, int32 *uaddr2, int32 val2);
-TEXT runtime·futex(SB),NOSPLIT,$0
+// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2);
+TEXT runtime·futex_time32(SB),NOSPLIT,$0
MOVW addr+0(FP), R0
MOVW op+4(FP), R1
MOVW val+8(FP), R2
@@ -417,6 +419,21 @@ TEXT runtime·futex(SB),NOSPLIT,$0
MOVW R0, ret+24(FP)
RET
+// Linux: kernel/futex/syscalls.c
+// int32 futex(int32 *uaddr, int32 op, int32 val,
+// struct timespec *timeout, int32 *uaddr2, int32 val2);
+TEXT runtime·futex_time64(SB),NOSPLIT,$0
+ MOVW addr+0(FP), R0
+ MOVW op+4(FP), R1
+ MOVW val+8(FP), R2
+ MOVW ts+12(FP), R3
+ MOVW addr2+16(FP), R4
+ MOVW val3+20(FP), R5
+ MOVW $SYS_futex_time64, R7
+ SWI $0
+ MOVW R0, ret+24(FP)
+ RET
+
// int32 clone(int32 flags, void *stack, M *mp, G *gp, void (*fn)(void));
TEXT runtime·clone(SB),NOSPLIT,$0
MOVW flags+0(FP), R0
diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s
index 5e6b6c1504..6f11841efc 100644
--- a/src/runtime/sys_linux_mipsx.s
+++ b/src/runtime/sys_linux_mipsx.s
@@ -34,6 +34,7 @@
#define SYS_mincore 4217
#define SYS_gettid 4222
#define SYS_futex 4238
+#define SYS_futex_time64 4422
#define SYS_sched_getaffinity 4240
#define SYS_exit_group 4246
#define SYS_timer_create 4257
@@ -362,8 +363,10 @@ TEXT runtime·madvise(SB),NOSPLIT,$0-16
MOVW R2, ret+12(FP)
RET
-// int32 futex(int32 *uaddr, int32 op, int32 val, struct timespec *timeout, int32 *uaddr2, int32 val2);
-TEXT runtime·futex(SB),NOSPLIT,$20-28
+// Linux: kernel/futex/syscalls.c, requiring COMPAT_32BIT_TIME
+// int32 futex(int32 *uaddr, int32 op, int32 val,
+// struct old_timespec32 *timeout, int32 *uaddr2, int32 val2);
+TEXT runtime·futex_time32(SB),NOSPLIT,$20-28
MOVW addr+0(FP), R4
MOVW op+4(FP), R5
MOVW val+8(FP), R6
@@ -382,6 +385,27 @@ TEXT runtime·futex(SB),NOSPLIT,$20-28
MOVW R2, ret+24(FP)
RET
+// Linux: kernel/futex/syscalls.c
+// int32 futex(int32 *uaddr, int32 op, int32 val,
+// struct timespec *timeout, int32 *uaddr2, int32 val2);
+TEXT runtime·futex_time64(SB),NOSPLIT,$20-28
+ MOVW addr+0(FP), R4
+ MOVW op+4(FP), R5
+ MOVW val+8(FP), R6
+ MOVW ts+12(FP), R7
+
+ MOVW addr2+16(FP), R8
+ MOVW val3+20(FP), R9
+
+ MOVW R8, 16(R29)
+ MOVW R9, 20(R29)
+
+ MOVW $SYS_futex_time64, R2
+ SYSCALL
+ BEQ R7, 2(PC)
+ SUBU R2, R0, R2 // caller expects negative errno
+ MOVW R2, ret+24(FP)
+ RET
// int32 clone(int32 flags, void *stk, M *mp, G *gp, void (*fn)(void));
TEXT runtime·clone(SB),NOSPLIT|NOFRAME,$0-24
diff --git a/src/runtime/unsafepoint_test.go b/src/runtime/unsafepoint_test.go
index 2c97adead8..79f0171854 100644
--- a/src/runtime/unsafepoint_test.go
+++ b/src/runtime/unsafepoint_test.go
@@ -43,7 +43,7 @@ func TestUnsafePoint(t *testing.T) {
cmd := exec.Command(testenv.GoToolPath(t), "tool", "objdump", "-s", "setGlobalPointer", os.Args[0])
out, err := cmd.CombinedOutput()
if err != nil {
- t.Fatalf("can't objdump %v", err)
+ t.Fatalf("can't objdump %v:\n%s", err, out)
}
lines := strings.Split(string(out), "\n")[1:]