runtime: use vDSO for getrandom() on linux

Linux 6.11 supports calling getrandom() from the vDSO. It operates on a
thread-local opaque state allocated with mmap using flags specified by
the vDSO.

Opaque states are allocated in chunks, ideally ncpu at a time as a hint,
rounding up to as many fit in a complete page. On first use, a state is
assigned to an m, which owns that state, until the m exits, at which
point it is given back to the pool.

Performance appears to be quite good:

           │    sec/op    │   sec/op       vs base                 │
Read/4-16    222.45n ± 3%   27.13n   ± 6%  -87.80% (p=0.000 n=10)
           │     B/s      │      B/s       vs base                 │
Read/4-16    17.15Mi ± 3%   140.61Mi ± 6%  +719.82% (p=0.000 n=10)

Fixes #69577.

Change-Id: Ib6f44e8f2f3940c94d970eaada0eb566ec297dc7
Reviewed-on: https://go-review.googlesource.com/c/go/+/614835
Reviewed-by: Filippo Valsorda <filippo@golang.org>
Reviewed-by: Cuong Manh Le <cuong.manhle.vn@gmail.com>
Auto-Submit: Jason Donenfeld <Jason@zx2c4.com>
Reviewed-by: Paul Murphy <murp@ibm.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: David Chase <drchase@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>

1 files changed, 100 insertions, 0 deletions

diff --git a/src/runtime/vgetrandom_linux.go b/src/runtime/vgetrandom_linux.go
new file mode 100644
index 0000000000..1e8c8ceaf0
--- /dev/null
+++ b/src/runtime/vgetrandom_linux.go
@@ -0,0 +1,100 @@
+// Copyright 2024 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && (amd64 || arm64 || arm64be || ppc64 || ppc64le || loong64 || s390x)
+
+package runtime
+
+import "unsafe"
+
+func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int
+
+var vgetrandomAlloc struct {
+	states     []uintptr
+	statesLock mutex
+	stateSize  uintptr
+	mmapProt   int32
+	mmapFlags  int32
+}
+
+func vgetrandomInit() {
+	if vdsoGetrandomSym == 0 {
+		return
+	}
+
+	var params struct {
+		SizeOfOpaqueState uint32
+		MmapProt          uint32
+		MmapFlags         uint32
+		reserved          [13]uint32
+	}
+	if vgetrandom1(nil, 0, 0, uintptr(unsafe.Pointer(&params)), ^uintptr(0)) != 0 {
+		return
+	}
+	vgetrandomAlloc.stateSize = uintptr(params.SizeOfOpaqueState)
+	vgetrandomAlloc.mmapProt = int32(params.MmapProt)
+	vgetrandomAlloc.mmapFlags = int32(params.MmapFlags)
+
+	lockInit(&vgetrandomAlloc.statesLock, lockRankLeafRank)
+}
+
+func vgetrandomGetState() uintptr {
+	lock(&vgetrandomAlloc.statesLock)
+	if len(vgetrandomAlloc.states) == 0 {
+		num := uintptr(ncpu) // Just a reasonable size hint to start.
+		allocSize := (num*vgetrandomAlloc.stateSize + physPageSize - 1) &^ (physPageSize - 1)
+		num = (physPageSize / vgetrandomAlloc.stateSize) * (allocSize / physPageSize)
+		p, err := mmap(nil, allocSize, vgetrandomAlloc.mmapProt, vgetrandomAlloc.mmapFlags, -1, 0)
+		if err != 0 {
+			unlock(&vgetrandomAlloc.statesLock)
+			return 0
+		}
+		newBlock := uintptr(p)
+		if vgetrandomAlloc.states == nil {
+			vgetrandomAlloc.states = make([]uintptr, 0, num)
+		}
+		for i := uintptr(0); i < num; i++ {
+			if (newBlock&(physPageSize-1))+vgetrandomAlloc.stateSize > physPageSize {
+				newBlock = (newBlock + physPageSize - 1) &^ (physPageSize - 1)
+			}
+			vgetrandomAlloc.states = append(vgetrandomAlloc.states, newBlock)
+			newBlock += vgetrandomAlloc.stateSize
+		}
+	}
+	state := vgetrandomAlloc.states[len(vgetrandomAlloc.states)-1]
+	vgetrandomAlloc.states = vgetrandomAlloc.states[:len(vgetrandomAlloc.states)-1]
+	unlock(&vgetrandomAlloc.statesLock)
+	return state
+}
+
+func vgetrandomPutState(state uintptr) {
+	lock(&vgetrandomAlloc.statesLock)
+	vgetrandomAlloc.states = append(vgetrandomAlloc.states, state)
+	unlock(&vgetrandomAlloc.statesLock)
+}
+
+// This is exported for use in internal/syscall/unix as well as x/sys/unix.
+//
+//go:linkname vgetrandom
+func vgetrandom(p []byte, flags uint32) (ret int, supported bool) {
+	if vgetrandomAlloc.stateSize == 0 {
+		return -1, false
+	}
+
+	mp := acquirem()
+	if mp.vgetrandomState == 0 {
+		state := vgetrandomGetState()
+		if state == 0 {
+			releasem(mp)
+			return -1, false
+		}
+		mp.vgetrandomState = state
+	}
+
+	ret = vgetrandom1(unsafe.SliceData(p), uintptr(len(p)), flags, mp.vgetrandomState, vgetrandomAlloc.stateSize)
+	supported = true
+
+	releasem(mp)
+	return
+}