From eb6f2c24cd17c0ca1df7e343f8d9187eef7d6e13 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Sun, 22 Sep 2024 03:45:29 +0200 Subject: runtime: use vDSO for getrandom() on linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux 6.11 supports calling getrandom() from the vDSO. It operates on a thread-local opaque state allocated with mmap using flags specified by the vDSO. Opaque states are allocated in chunks, ideally ncpu at a time as a hint, rounding up to as many fit in a complete page. On first use, a state is assigned to an m, which owns that state, until the m exits, at which point it is given back to the pool. Performance appears to be quite good: │ sec/op │ sec/op vs base │ Read/4-16 222.45n ± 3% 27.13n ± 6% -87.80% (p=0.000 n=10) │ B/s │ B/s vs base │ Read/4-16 17.15Mi ± 3% 140.61Mi ± 6% +719.82% (p=0.000 n=10) Fixes #69577. Change-Id: Ib6f44e8f2f3940c94d970eaada0eb566ec297dc7 Reviewed-on: https://go-review.googlesource.com/c/go/+/614835 Reviewed-by: Filippo Valsorda Reviewed-by: Cuong Manh Le Auto-Submit: Jason Donenfeld Reviewed-by: Paul Murphy LUCI-TryBot-Result: Go LUCI Reviewed-by: David Chase Reviewed-by: Michael Pratt --- src/runtime/vgetrandom_linux.go | 100 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 src/runtime/vgetrandom_linux.go (limited to 'src/runtime/vgetrandom_linux.go') diff --git a/src/runtime/vgetrandom_linux.go b/src/runtime/vgetrandom_linux.go new file mode 100644 index 0000000000..1e8c8ceaf0 --- /dev/null +++ b/src/runtime/vgetrandom_linux.go @@ -0,0 +1,100 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build linux && (amd64 || arm64 || arm64be || ppc64 || ppc64le || loong64 || s390x) + +package runtime + +import "unsafe" + +func vgetrandom1(buf *byte, length uintptr, flags uint32, state uintptr, stateSize uintptr) int + +var vgetrandomAlloc struct { + states []uintptr + statesLock mutex + stateSize uintptr + mmapProt int32 + mmapFlags int32 +} + +func vgetrandomInit() { + if vdsoGetrandomSym == 0 { + return + } + + var params struct { + SizeOfOpaqueState uint32 + MmapProt uint32 + MmapFlags uint32 + reserved [13]uint32 + } + if vgetrandom1(nil, 0, 0, uintptr(unsafe.Pointer(¶ms)), ^uintptr(0)) != 0 { + return + } + vgetrandomAlloc.stateSize = uintptr(params.SizeOfOpaqueState) + vgetrandomAlloc.mmapProt = int32(params.MmapProt) + vgetrandomAlloc.mmapFlags = int32(params.MmapFlags) + + lockInit(&vgetrandomAlloc.statesLock, lockRankLeafRank) +} + +func vgetrandomGetState() uintptr { + lock(&vgetrandomAlloc.statesLock) + if len(vgetrandomAlloc.states) == 0 { + num := uintptr(ncpu) // Just a reasonable size hint to start. + allocSize := (num*vgetrandomAlloc.stateSize + physPageSize - 1) &^ (physPageSize - 1) + num = (physPageSize / vgetrandomAlloc.stateSize) * (allocSize / physPageSize) + p, err := mmap(nil, allocSize, vgetrandomAlloc.mmapProt, vgetrandomAlloc.mmapFlags, -1, 0) + if err != 0 { + unlock(&vgetrandomAlloc.statesLock) + return 0 + } + newBlock := uintptr(p) + if vgetrandomAlloc.states == nil { + vgetrandomAlloc.states = make([]uintptr, 0, num) + } + for i := uintptr(0); i < num; i++ { + if (newBlock&(physPageSize-1))+vgetrandomAlloc.stateSize > physPageSize { + newBlock = (newBlock + physPageSize - 1) &^ (physPageSize - 1) + } + vgetrandomAlloc.states = append(vgetrandomAlloc.states, newBlock) + newBlock += vgetrandomAlloc.stateSize + } + } + state := vgetrandomAlloc.states[len(vgetrandomAlloc.states)-1] + vgetrandomAlloc.states = vgetrandomAlloc.states[:len(vgetrandomAlloc.states)-1] + unlock(&vgetrandomAlloc.statesLock) + return state +} + +func vgetrandomPutState(state uintptr) { + lock(&vgetrandomAlloc.statesLock) + vgetrandomAlloc.states = append(vgetrandomAlloc.states, state) + unlock(&vgetrandomAlloc.statesLock) +} + +// This is exported for use in internal/syscall/unix as well as x/sys/unix. +// +//go:linkname vgetrandom +func vgetrandom(p []byte, flags uint32) (ret int, supported bool) { + if vgetrandomAlloc.stateSize == 0 { + return -1, false + } + + mp := acquirem() + if mp.vgetrandomState == 0 { + state := vgetrandomGetState() + if state == 0 { + releasem(mp) + return -1, false + } + mp.vgetrandomState = state + } + + ret = vgetrandom1(unsafe.SliceData(p), uintptr(len(p)), flags, mp.vgetrandomState, vgetrandomAlloc.stateSize) + supported = true + + releasem(mp) + return +} -- cgit v1.3