From 40a10063e89d4f0ae55631d1e50c31e3314fb812 Mon Sep 17 00:00:00 2001 From: Wang Deyu Date: Thu, 12 Feb 2026 17:00:24 +0800 Subject: runtime: fix scan size calculation for small arrays of only pointers When allocating arrays, scan size should be the position of the last pointer in the last object. Small array allocations containing only pointers (for example, the backing store for a []*int) have a fast path in the allocator for unrolling their bits. However, this fast path doesn't correctly calculate the scan size for the object, and the result is that only the first pointer is actually counted. This might lead the GC to think it has less work to do than it actually does. Fixes #77573 Change-Id: I4b9db6db069e3fb64e4fcbbbc89b68585a71d483 Reviewed-on: https://go-review.googlesource.com/c/go/+/744840 Reviewed-by: Mark Freeman Reviewed-by: Michael Knyszek Auto-Submit: Wayne Zuo LUCI-TryBot-Result: Go LUCI --- src/runtime/malloc_generated.go | 130 ++++++++++++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 26 deletions(-) (limited to 'src/runtime/malloc_generated.go') diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go index 8d7112646f..c72c10544a 100644 --- a/src/runtime/malloc_generated.go +++ b/src/runtime/malloc_generated.go @@ -84,15 +84,18 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 8 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -249,15 +252,18 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 16 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -414,15 +420,18 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 24 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -579,15 +588,18 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 32 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -744,15 +756,18 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 48 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -909,15 +924,18 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 64 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -1074,15 +1092,18 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 80 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -1239,15 +1260,18 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 96 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -1404,15 +1428,18 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf const elemsize = 112 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -1569,15 +1596,18 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa const elemsize = 128 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -1734,15 +1764,18 @@ func mallocgcSmallScanNoHeaderSC11(size uintptr, typ *_type, needzero bool) unsa const elemsize = 144 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -1899,15 +1932,18 @@ func mallocgcSmallScanNoHeaderSC12(size uintptr, typ *_type, needzero bool) unsa const elemsize = 160 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -2064,15 +2100,18 @@ func mallocgcSmallScanNoHeaderSC13(size uintptr, typ *_type, needzero bool) unsa const elemsize = 176 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -2229,15 +2268,18 @@ func mallocgcSmallScanNoHeaderSC14(size uintptr, typ *_type, needzero bool) unsa const elemsize = 192 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -2394,15 +2436,18 @@ func mallocgcSmallScanNoHeaderSC15(size uintptr, typ *_type, needzero bool) unsa const elemsize = 208 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -2559,15 +2604,18 @@ func mallocgcSmallScanNoHeaderSC16(size uintptr, typ *_type, needzero bool) unsa const elemsize = 224 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -2724,15 +2772,18 @@ func mallocgcSmallScanNoHeaderSC17(size uintptr, typ *_type, needzero bool) unsa const elemsize = 240 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -2889,15 +2940,18 @@ func mallocgcSmallScanNoHeaderSC18(size uintptr, typ *_type, needzero bool) unsa const elemsize = 256 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -3054,15 +3108,18 @@ func mallocgcSmallScanNoHeaderSC19(size uintptr, typ *_type, needzero bool) unsa const elemsize = 288 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -3219,15 +3276,18 @@ func mallocgcSmallScanNoHeaderSC20(size uintptr, typ *_type, needzero bool) unsa const elemsize = 320 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -3384,15 +3444,18 @@ func mallocgcSmallScanNoHeaderSC21(size uintptr, typ *_type, needzero bool) unsa const elemsize = 352 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -3549,15 +3612,18 @@ func mallocgcSmallScanNoHeaderSC22(size uintptr, typ *_type, needzero bool) unsa const elemsize = 384 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -3714,15 +3780,18 @@ func mallocgcSmallScanNoHeaderSC23(size uintptr, typ *_type, needzero bool) unsa const elemsize = 416 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -3879,15 +3948,18 @@ func mallocgcSmallScanNoHeaderSC24(size uintptr, typ *_type, needzero bool) unsa const elemsize = 448 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -4044,15 +4116,18 @@ func mallocgcSmallScanNoHeaderSC25(size uintptr, typ *_type, needzero bool) unsa const elemsize = 480 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ @@ -4209,15 +4284,18 @@ func mallocgcSmallScanNoHeaderSC26(size uintptr, typ *_type, needzero bool) unsa const elemsize = 512 - scanSize := typ.PtrBytes + var scanSize uintptr src := src0 if typ.Size_ == goarch.PtrSize { src = (1 << (dataSize / goarch.PtrSize)) - 1 + + scanSize = dataSize } else { if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 { throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_") } + scanSize = typ.PtrBytes for i := typ.Size_; i < dataSize; i += typ.Size_ { src |= src0 << (i / goarch.PtrSize) scanSize += typ.Size_ -- cgit v1.3-5-g9baa