aboutsummaryrefslogtreecommitdiff
path: root/src/runtime
diff options
context:
space:
mode:
authorWang Deyu <wangdeyu.2021@bytedance.com>2026-02-12 17:00:24 +0800
committerGopher Robot <gobot@golang.org>2026-02-26 18:11:47 -0800
commit40a10063e89d4f0ae55631d1e50c31e3314fb812 (patch)
tree0d6fa3cda660480ac567d3e94e7f6a1bed5ae563 /src/runtime
parent657ed934e85dc575aad51356c4b437961e7c1313 (diff)
downloadgo-40a10063e89d4f0ae55631d1e50c31e3314fb812.tar.xz
runtime: fix scan size calculation for small arrays of only pointers
When allocating arrays, scan size should be the position of the last pointer in the last object. Small array allocations containing only pointers (for example, the backing store for a []*int) have a fast path in the allocator for unrolling their bits. However, this fast path doesn't correctly calculate the scan size for the object, and the result is that only the first pointer is actually counted. This might lead the GC to think it has less work to do than it actually does. Fixes #77573 Change-Id: I4b9db6db069e3fb64e4fcbbbc89b68585a71d483 Reviewed-on: https://go-review.googlesource.com/c/go/+/744840 Reviewed-by: Mark Freeman <markfreeman@google.com> Reviewed-by: Michael Knyszek <mknyszek@google.com> Auto-Submit: Wayne Zuo <wdvxdr1123@gmail.com> LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Diffstat (limited to 'src/runtime')
-rw-r--r--src/runtime/export_test.go5
-rw-r--r--src/runtime/malloc_generated.go130
-rw-r--r--src/runtime/malloc_stubs.go5
-rw-r--r--src/runtime/malloc_test.go32
-rw-r--r--src/runtime/mbitmap.go4
5 files changed, 148 insertions, 28 deletions
diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go
index 72f5c96564..f71494174c 100644
--- a/src/runtime/export_test.go
+++ b/src/runtime/export_test.go
@@ -2083,3 +2083,8 @@ func DumpPrintQuoted(s string) string {
return string(buf)
}
+
+func GetScanAlloc() uintptr {
+ c := getMCache(getg().m)
+ return c.scanAlloc
+}
diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go
index 8d7112646f..c72c10544a 100644
--- a/src/runtime/malloc_generated.go
+++ b/src/runtime/malloc_generated.go
@@ -84,15 +84,18 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 8
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -249,15 +252,18 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 16
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -414,15 +420,18 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 24
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -579,15 +588,18 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 32
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -744,15 +756,18 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 48
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -909,15 +924,18 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 64
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -1074,15 +1092,18 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 80
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -1239,15 +1260,18 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 96
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -1404,15 +1428,18 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf
const elemsize = 112
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -1569,15 +1596,18 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 128
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -1734,15 +1764,18 @@ func mallocgcSmallScanNoHeaderSC11(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 144
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -1899,15 +1932,18 @@ func mallocgcSmallScanNoHeaderSC12(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 160
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -2064,15 +2100,18 @@ func mallocgcSmallScanNoHeaderSC13(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 176
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -2229,15 +2268,18 @@ func mallocgcSmallScanNoHeaderSC14(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 192
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -2394,15 +2436,18 @@ func mallocgcSmallScanNoHeaderSC15(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 208
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -2559,15 +2604,18 @@ func mallocgcSmallScanNoHeaderSC16(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 224
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -2724,15 +2772,18 @@ func mallocgcSmallScanNoHeaderSC17(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 240
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -2889,15 +2940,18 @@ func mallocgcSmallScanNoHeaderSC18(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 256
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -3054,15 +3108,18 @@ func mallocgcSmallScanNoHeaderSC19(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 288
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -3219,15 +3276,18 @@ func mallocgcSmallScanNoHeaderSC20(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 320
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -3384,15 +3444,18 @@ func mallocgcSmallScanNoHeaderSC21(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 352
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -3549,15 +3612,18 @@ func mallocgcSmallScanNoHeaderSC22(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 384
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -3714,15 +3780,18 @@ func mallocgcSmallScanNoHeaderSC23(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 416
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -3879,15 +3948,18 @@ func mallocgcSmallScanNoHeaderSC24(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 448
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -4044,15 +4116,18 @@ func mallocgcSmallScanNoHeaderSC25(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 480
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
@@ -4209,15 +4284,18 @@ func mallocgcSmallScanNoHeaderSC26(size uintptr, typ *_type, needzero bool) unsa
const elemsize = 512
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+
+ scanSize = dataSize
} else {
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
diff --git a/src/runtime/malloc_stubs.go b/src/runtime/malloc_stubs.go
index b395172e4b..7729637f8f 100644
--- a/src/runtime/malloc_stubs.go
+++ b/src/runtime/malloc_stubs.go
@@ -572,10 +572,12 @@ func writeHeapBitsSmallStub(span *mspan, x, dataSize uintptr, typ *_type) uintpt
const elemsize = elemsize_
// Create repetitions of the bitmap if we have a small slice backing store.
- scanSize := typ.PtrBytes
+ var scanSize uintptr
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+ // This object is all pointers, so scanSize is just dataSize.
+ scanSize = dataSize
} else {
// N.B. We rely on dataSize being an exact multiple of the type size.
// The alternative is to be defensive and mask out src to the length
@@ -583,6 +585,7 @@ func writeHeapBitsSmallStub(span *mspan, x, dataSize uintptr, typ *_type) uintpt
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_
diff --git a/src/runtime/malloc_test.go b/src/runtime/malloc_test.go
index b76b0a02ac..22ed23c7de 100644
--- a/src/runtime/malloc_test.go
+++ b/src/runtime/malloc_test.go
@@ -8,6 +8,7 @@ import (
"flag"
"fmt"
"internal/asan"
+ "internal/goarch"
"internal/race"
"internal/testenv"
"os"
@@ -851,3 +852,34 @@ func TestMkmalloc(t *testing.T) {
t.Errorf("_mkmalloc tests failed: %v", err)
}
}
+
+func TestScanAllocIssue77573(t *testing.T) {
+ verifyScanAlloc := func(t *testing.T, f func(), expectSize uintptr) {
+ runtime.Acquirem()
+ defer runtime.Releasem()
+ for i := 0; i < 100; i++ {
+ before := runtime.GetScanAlloc()
+ f()
+ after := runtime.GetScanAlloc()
+ // When mcache refills a new mspan, scanAlloc will be set to 0.
+ // As a result, the calculated value is not the scanAlloc of the allocated object, just retry again.
+ if after > before {
+ actualSize := after - before
+ if actualSize != expectSize {
+ t.Errorf("wrong GC Scan Alloc Size:\nwant %+v\ngot %+v", expectSize, actualSize)
+ }
+ return
+ }
+ }
+ t.Error("always refill, it still fails after running multiple times")
+ }
+ t.Run("heap slice ([]*int, 1)", func(t *testing.T) {
+ verifyScanAlloc(t, func() { runtime.Escape(make([]*int, 1)) }, goarch.PtrSize)
+ })
+ t.Run("heap slice ([]*int, 2)", func(t *testing.T) {
+ verifyScanAlloc(t, func() { runtime.Escape(make([]*int, 2)) }, 2*goarch.PtrSize)
+ })
+ t.Run("heap slice ([]*int, 3)", func(t *testing.T) {
+ verifyScanAlloc(t, func() { runtime.Escape(make([]*int, 3)) }, 3*goarch.PtrSize)
+ })
+}
diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index 37a92c64bc..7c05cd6ea9 100644
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -626,10 +626,11 @@ func (span *mspan) writeHeapBitsSmall(x, dataSize uintptr, typ *_type) (scanSize
src0 := readUintptr(getGCMask(typ))
// Create repetitions of the bitmap if we have a small slice backing store.
- scanSize = typ.PtrBytes
src := src0
if typ.Size_ == goarch.PtrSize {
src = (1 << (dataSize / goarch.PtrSize)) - 1
+ // This object is all pointers, so scanSize is just dataSize.
+ scanSize = dataSize
} else {
// N.B. We rely on dataSize being an exact multiple of the type size.
// The alternative is to be defensive and mask out src to the length
@@ -637,6 +638,7 @@ func (span *mspan) writeHeapBitsSmall(x, dataSize uintptr, typ *_type) (scanSize
if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
}
+ scanSize = typ.PtrBytes
for i := typ.Size_; i < dataSize; i += typ.Size_ {
src |= src0 << (i / goarch.PtrSize)
scanSize += typ.Size_