syscall: honor prlimit set by a different process

On Linux one process can call prlimit to change the resource limit
of another process. With this change we treat that as though the
current process called prlimit (or setrlimit) to set its own limit.
The cost is one additional getrlimit system call per fork/exec,
for cases in which the rlimit Cur and Max values differ at startup.

This revealed a bug: the setrlimit (not Setrlimit) function should not
change the cached rlimit. That means that it must call prlimit1, not prlimit.

Fixes #66797

Change-Id: I46bfd06e09ab7273fe8dd9b5b744dffdf31d828b
Reviewed-on: https://go-review.googlesource.com/c/go/+/607516
Auto-Submit: Ian Lance Taylor <iant@golang.org>
Reviewed-by: Damien Neil <dneil@google.com>
Reviewed-by: Ian Lance Taylor <iant@google.com>
Reviewed-by: Aleksa Sarai <cyphar@cyphar.com>
Reviewed-by: Kirill Kolyshkin <kolyshkin@gmail.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Ian Lance Taylor <iant@google.com>

1 files changed, 17 insertions, 2 deletions

diff --git a/src/syscall/exec_linux.go b/src/syscall/exec_linux.go
index 5ef62450a8..415706c032 100644
--- a/src/syscall/exec_linux.go
+++ b/src/syscall/exec_linux.go
@@ -252,10 +252,12 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att
 		cred                      *Credential
 		ngroups, groups           uintptr
 		c                         uintptr
+		rlim                      *Rlimit
+		lim                       Rlimit
 	)
 	pidfd = -1
 
-	rlim := origRlimitNofile.Load()
+	rlim = origRlimitNofile.Load()
 
 	if sys.UidMappings != nil {
 		puid = []byte("/proc/self/uid_map\000")
@@ -632,7 +634,20 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att
 
 	// Restore original rlimit.
 	if rlim != nil {
-		RawSyscall6(SYS_PRLIMIT64, 0, RLIMIT_NOFILE, uintptr(unsafe.Pointer(rlim)), 0, 0, 0)
+		// Some other process may have changed our rlimit by
+		// calling prlimit. We can check for that case because
+		// our current rlimit will not be the value we set when
+		// caching the rlimit in the init function in rlimit.go.
+		//
+		// Note that this test is imperfect, since it won't catch
+		// the case in which some other process used prlimit to
+		// set our rlimits to max-1/max. In that case we will fall
+		// back to the original cur/max when starting the child.
+		// We hope that setting to max-1/max is unlikely.
+		_, _, err1 = RawSyscall6(SYS_PRLIMIT64, 0, RLIMIT_NOFILE, 0, uintptr(unsafe.Pointer(&lim)), 0, 0)
+		if err1 != 0 || (lim.Cur == rlim.Max-1 && lim.Max == rlim.Max) {
+			RawSyscall6(SYS_PRLIMIT64, 0, RLIMIT_NOFILE, uintptr(unsafe.Pointer(rlim)), 0, 0, 0)
+		}
 	}
 
 	// Enable tracing if requested.