cmd/compile/internal/bitvec: remove 2GB allocation limit in NewBulk

Remove the "NewBulk too big" check that prevented bulk bit vector
allocations exceeding ~2GB. This limit was overly restrictive and
caused compilation failures for large generated code.

The actual constraint is in the runtime's stackmap reader, which uses
int32 arithmetic. Add overflow checks in cmd/compile/internal/liveness
before stackmap serialization to detect this condition, allowing
NewBulk itself to allocate bulk large bit vector.

Fixes #77170

Change-Id: Ib7f2cabb28683258fc4e85418ba7fa70b48620b0
Reviewed-on: https://go-review.googlesource.com/c/go/+/736240
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Keith Randall <khr@golang.org>
Auto-Submit: Keith Randall <khr@golang.org>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Junyang Shao <shaojunyang@google.com>

3 files changed, 27 insertions, 7 deletions

diff --git a/src/cmd/compile/internal/bitvec/bv.go b/src/cmd/compile/internal/bitvec/bv.go
index 9214aa6cd0..57a8237ab0 100644
--- a/src/cmd/compile/internal/bitvec/bv.go
+++ b/src/cmd/compile/internal/bitvec/bv.go
@@ -8,7 +8,6 @@ import (
 	"math/bits"
 
 	"cmd/compile/internal/base"
-	"cmd/internal/src"
 )
 
 const (
@@ -34,12 +33,9 @@ type Bulk struct {
 	nword int32
 }
 
-func NewBulk(nbit int32, count int32, pos src.XPos) Bulk {
+func NewBulk(nbit int32, count int32) Bulk {
 	nword := (nbit + wordBits - 1) / wordBits
 	size := int64(nword) * int64(count)
-	if int64(int32(size*4)) != size*4 {
-		base.FatalfAt(pos, "NewBulk too big: nbit=%d count=%d nword=%d size=%d", nbit, count, nword, size)
-	}
 	return Bulk{
 		words: make([]uint32, size),
 		nbit:  nbit,
diff --git a/src/cmd/compile/internal/liveness/arg.go b/src/cmd/compile/internal/liveness/arg.go
index 33e7f1856f..69ead6ecc5 100644
--- a/src/cmd/compile/internal/liveness/arg.go
+++ b/src/cmd/compile/internal/liveness/arg.go
@@ -132,7 +132,7 @@ func ArgLiveness(fn *ir.Func, f *ssa.Func, pp *objw.Progs) (blockIdx, valueIdx m
 	}
 
 	nargs := int32(len(lv.args))
-	bulk := bitvec.NewBulk(nargs, int32(len(f.Blocks)*2), fn.Pos())
+	bulk := bitvec.NewBulk(nargs, int32(len(f.Blocks)*2))
 	for _, b := range f.Blocks {
 		be := &lv.be[b.ID]
 		be.livein = bulk.Next()
diff --git a/src/cmd/compile/internal/liveness/plive.go b/src/cmd/compile/internal/liveness/plive.go
index 88e46af547..198f6418d6 100644
--- a/src/cmd/compile/internal/liveness/plive.go
+++ b/src/cmd/compile/internal/liveness/plive.go
@@ -17,6 +17,7 @@ package liveness
 import (
 	"cmp"
 	"fmt"
+	"math"
 	"os"
 	"slices"
 	"sort"
@@ -427,7 +428,7 @@ func newliveness(fn *ir.Func, f *ssa.Func, vars []*ir.Name, idx map[*ir.Name]int
 
 	nblocks := int32(len(f.Blocks))
 	nvars := int32(len(vars))
-	bulk := bitvec.NewBulk(nvars, nblocks*4, fn.Pos())
+	bulk := bitvec.NewBulk(nvars, nblocks*4)
 	for _, b := range f.Blocks {
 		be := lv.blockEffects(b)
 
@@ -1367,6 +1368,10 @@ func (lv *Liveness) emit() (argsSym, liveSym *obj.LSym) {
 	loff := objw.Uint32(&liveSymTmp, 0, uint32(len(lv.stackMaps))) // number of bitmaps
 	loff = objw.Uint32(&liveSymTmp, loff, uint32(locals.N))        // number of bits in each bitmap
 
+	// Check for overflow before serializing stackmaps
+	checkStackmapOverflow(args, len(lv.stackMaps), lv.fn.Pos())
+	checkStackmapOverflow(locals, len(lv.stackMaps), lv.fn.Pos())
+
 	for _, live := range lv.stackMaps {
 		args.Clear()
 		locals.Clear()
@@ -1567,6 +1572,10 @@ func WriteFuncMap(fn *ir.Func, abiInfo *abi.ABIParamResultInfo) {
 	if fn.Type().NumResults() > 0 {
 		nbitmap = 2
 	}
+
+	// defensive check: function arguments can't realistically be large enough for overflow here
+	checkStackmapOverflow(bv, nbitmap, fn.Pos())
+
 	lsym := base.Ctxt.Lookup(fn.LSym.Name + ".args_stackmap")
 	lsym.Set(obj.AttrLinkname, true) // allow args_stackmap referenced from assembly
 	off := objw.Uint32(lsym, 0, uint32(nbitmap))
@@ -1584,3 +1593,18 @@ func WriteFuncMap(fn *ir.Func, abiInfo *abi.ABIParamResultInfo) {
 
 	objw.Global(lsym, int32(off), obj.RODATA|obj.LOCAL)
 }
+
+// checkStackmapOverflow checks for potential overflow in runtime stackmap reading.
+// Runtime computes: n * ((nbit+7)/8) using int32 arithmetic.
+// See runtime.stackmapdata implementation.
+func checkStackmapOverflow(bv bitvec.BitVec, count int, pos src.XPos) {
+	if bv.N <= 0 || count <= 0 {
+		return
+	}
+	bytesPerBitVec := (int64(bv.N) + 7) >> 3
+	totalBytes := bytesPerBitVec * int64(count)
+	if totalBytes > math.MaxInt32 {
+		// runtime.stackmap has to support 64-bit values to avoid this restriction, see issue 77170
+		base.FatalfAt(pos, "liveness stackmaps are too large: nbit=%d count=%d totalBytes=%d exceeds MaxInt32", bv.N, count, totalBytes)
+	}
+}