runtime: move epoll syscalls to runtime/internal/syscall

This change moves Linux epoll's syscalls implementation to the
"runtime/internal/syscall" package. The intention in this CL was to
minimise behavioural changes but make the code more generalised. This
also will allow adding new syscalls (like epoll_pwait2) without the
need to implement assembly stubs for each arch.

It also drops epoll_create as not all architectures provide this call.
epoll_create1 was added to the kernel in version 2.6.27 and Go requires
Linux kernel version 2.6.32 or later since Go 1.18. So it is safe to
always use epoll_create1.

For #53824
For #51087

Change-Id: I9a6a26b7f2075a38e041de1bab4691da0ecb94fc
Reviewed-on: https://go-review.googlesource.com/c/go/+/421994
Reviewed-by: Michael Pratt <mpratt@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Run-TryBot: Michael Pratt <mpratt@google.com>
Auto-Submit: Michael Pratt <mpratt@google.com>

13 files changed, 361 insertions, 1 deletions

diff --git a/src/runtime/internal/syscall/defs_linux.go b/src/runtime/internal/syscall/defs_linux.go
new file mode 100644
index 0000000000..71f1fa1453
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux.go
@@ -0,0 +1,10 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	F_SETFD    = 2
+	FD_CLOEXEC = 1
+)
diff --git a/src/runtime/internal/syscall/defs_linux_386.go b/src/runtime/internal/syscall/defs_linux_386.go
new file mode 100644
index 0000000000..dc723a60b2
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_386.go
@@ -0,0 +1,29 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_FCNTL         = 55
+	SYS_EPOLL_CTL     = 255
+	SYS_EPOLL_PWAIT   = 319
+	SYS_EPOLL_CREATE1 = 329
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events uint32
+	Data   [8]byte // to match amd64
+}
diff --git a/src/runtime/internal/syscall/defs_linux_amd64.go b/src/runtime/internal/syscall/defs_linux_amd64.go
new file mode 100644
index 0000000000..886eb5bda2
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_amd64.go
@@ -0,0 +1,29 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_FCNTL         = 72
+	SYS_EPOLL_CTL     = 233
+	SYS_EPOLL_PWAIT   = 281
+	SYS_EPOLL_CREATE1 = 291
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events uint32
+	Data   [8]byte // unaligned uintptr
+}
diff --git a/src/runtime/internal/syscall/defs_linux_arm.go b/src/runtime/internal/syscall/defs_linux_arm.go
new file mode 100644
index 0000000000..8f812a2f68
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_arm.go
@@ -0,0 +1,30 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_FCNTL         = 55
+	SYS_EPOLL_CTL     = 251
+	SYS_EPOLL_PWAIT   = 346
+	SYS_EPOLL_CREATE1 = 357
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events uint32
+	_pad   uint32
+	Data   [8]byte // to match amd64
+}
diff --git a/src/runtime/internal/syscall/defs_linux_arm64.go b/src/runtime/internal/syscall/defs_linux_arm64.go
new file mode 100644
index 0000000000..48e11b0c51
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_arm64.go
@@ -0,0 +1,30 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_EPOLL_CREATE1 = 20
+	SYS_EPOLL_CTL     = 21
+	SYS_EPOLL_PWAIT   = 22
+	SYS_FCNTL         = 25
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events uint32
+	_pad   uint32
+	Data   [8]byte // to match amd64
+}
diff --git a/src/runtime/internal/syscall/defs_linux_loong64.go b/src/runtime/internal/syscall/defs_linux_loong64.go
new file mode 100644
index 0000000000..b78ef81861
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_loong64.go
@@ -0,0 +1,30 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_EPOLL_CREATE1 = 20
+	SYS_EPOLL_CTL     = 21
+	SYS_EPOLL_PWAIT   = 22
+	SYS_FCNTL         = 25
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events    uint32
+	pad_cgo_0 [4]byte
+	Data      [8]byte // unaligned uintptr
+}
diff --git a/src/runtime/internal/syscall/defs_linux_mips64x.go b/src/runtime/internal/syscall/defs_linux_mips64x.go
new file mode 100644
index 0000000000..92b49ca969
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_mips64x.go
@@ -0,0 +1,32 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && (mips64 || mips64le)
+
+package syscall
+
+const (
+	SYS_FCNTL         = 5070
+	SYS_EPOLL_CTL     = 5208
+	SYS_EPOLL_PWAIT   = 5272
+	SYS_EPOLL_CREATE1 = 5285
+	SYS_EPOLL_PWAIT2  = 5441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events    uint32
+	pad_cgo_0 [4]byte
+	Data      [8]byte // unaligned uintptr
+}
diff --git a/src/runtime/internal/syscall/defs_linux_mipsx.go b/src/runtime/internal/syscall/defs_linux_mipsx.go
new file mode 100644
index 0000000000..e28d09c7f1
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_mipsx.go
@@ -0,0 +1,32 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && (mips || mipsle)
+
+package syscall
+
+const (
+	SYS_FCNTL         = 4055
+	SYS_EPOLL_CTL     = 4249
+	SYS_EPOLL_PWAIT   = 4313
+	SYS_EPOLL_CREATE1 = 4326
+	SYS_EPOLL_PWAIT2  = 4441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events    uint32
+	pad_cgo_0 [4]byte
+	Data      uint64
+}
diff --git a/src/runtime/internal/syscall/defs_linux_ppc64x.go b/src/runtime/internal/syscall/defs_linux_ppc64x.go
new file mode 100644
index 0000000000..a74483eb6d
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_ppc64x.go
@@ -0,0 +1,32 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build linux && (ppc64 || ppc64le)
+
+package syscall
+
+const (
+	SYS_FCNTL         = 55
+	SYS_EPOLL_CTL     = 237
+	SYS_EPOLL_PWAIT   = 303
+	SYS_EPOLL_CREATE1 = 315
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events    uint32
+	pad_cgo_0 [4]byte
+	Data      [8]byte // unaligned uintptr
+}
diff --git a/src/runtime/internal/syscall/defs_linux_riscv64.go b/src/runtime/internal/syscall/defs_linux_riscv64.go
new file mode 100644
index 0000000000..b78ef81861
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_riscv64.go
@@ -0,0 +1,30 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_EPOLL_CREATE1 = 20
+	SYS_EPOLL_CTL     = 21
+	SYS_EPOLL_PWAIT   = 22
+	SYS_FCNTL         = 25
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events    uint32
+	pad_cgo_0 [4]byte
+	Data      [8]byte // unaligned uintptr
+}
diff --git a/src/runtime/internal/syscall/defs_linux_s390x.go b/src/runtime/internal/syscall/defs_linux_s390x.go
new file mode 100644
index 0000000000..a7bb1ba66d
--- /dev/null
+++ b/src/runtime/internal/syscall/defs_linux_s390x.go
@@ -0,0 +1,30 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall
+
+const (
+	SYS_FCNTL         = 55
+	SYS_EPOLL_CTL     = 250
+	SYS_EPOLL_PWAIT   = 312
+	SYS_EPOLL_CREATE1 = 327
+	SYS_EPOLL_PWAIT2  = 441
+
+	EPOLLIN       = 0x1
+	EPOLLOUT      = 0x4
+	EPOLLERR      = 0x8
+	EPOLLHUP      = 0x10
+	EPOLLRDHUP    = 0x2000
+	EPOLLET       = 0x80000000
+	EPOLL_CLOEXEC = 0x80000
+	EPOLL_CTL_ADD = 0x1
+	EPOLL_CTL_DEL = 0x2
+	EPOLL_CTL_MOD = 0x3
+)
+
+type EpollEvent struct {
+	Events    uint32
+	pad_cgo_0 [4]byte
+	Data      [8]byte // unaligned uintptr
+}
diff --git a/src/runtime/internal/syscall/syscall_linux.go b/src/runtime/internal/syscall/syscall_linux.go
index 7f268e8fba..a103d318c4 100644
--- a/src/runtime/internal/syscall/syscall_linux.go
+++ b/src/runtime/internal/syscall/syscall_linux.go
@@ -6,7 +6,7 @@
 package syscall
 
 import (
-	_ "unsafe" // for go:linkname
+	"unsafe"
 )
 
 // TODO(https://go.dev/issue/51087): This package is incomplete and currently
@@ -37,3 +37,30 @@ func Syscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr)
 func syscall_RawSyscall6(num, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, errno uintptr) {
 	return Syscall6(num, a1, a2, a3, a4, a5, a6)
 }
+
+func EpollCreate1(flags int32) (fd int32, errno uintptr) {
+	r1, _, e := Syscall6(SYS_EPOLL_CREATE1, uintptr(flags), 0, 0, 0, 0, 0)
+	return int32(r1), e
+}
+
+var _zero uintptr
+
+func EpollWait(epfd int32, events []EpollEvent, maxev, waitms int32) (n int32, errno uintptr) {
+	var ev unsafe.Pointer
+	if len(events) > 0 {
+		ev = unsafe.Pointer(&events[0])
+	} else {
+		ev = unsafe.Pointer(&_zero)
+	}
+	r1, _, e := Syscall6(SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(ev), uintptr(maxev), uintptr(waitms), 0, 0)
+	return int32(r1), e
+}
+
+func EpollCtl(epfd, op, fd int32, event *EpollEvent) (errno uintptr) {
+	_, _, e := Syscall6(SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)
+	return e
+}
+
+func CloseOnExec(fd int32) {
+	Syscall6(SYS_FCNTL, uintptr(fd), F_SETFD, FD_CLOEXEC, 0, 0, 0)
+}
diff --git a/src/runtime/internal/syscall/syscall_linux_test.go b/src/runtime/internal/syscall/syscall_linux_test.go
new file mode 100644
index 0000000000..1976da5c28
--- /dev/null
+++ b/src/runtime/internal/syscall/syscall_linux_test.go
@@ -0,0 +1,19 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syscall_test
+
+import (
+	"runtime/internal/syscall"
+	"testing"
+)
+
+func TestEpollctlErrorSign(t *testing.T) {
+	v := syscall.EpollCtl(-1, 1, -1, &syscall.EpollEvent{})
+
+	const EBADF = 0x09
+	if v != EBADF {
+		t.Errorf("epollctl = %v, want %v", v, EBADF)
+	}
+}