runtime: use vDSO for clock_gettime on linux/arm

Use the __vdso_clock_gettime fast path via the vDSO on linux/arm to
speed up nanotime and walltime. This results in the following
performance improvement for time.Now on a RaspberryPi 3 (running
32bit Raspbian, i.e. GOOS=linux/GOARCH=arm):

name     old time/op  new time/op  delta
TimeNow  0.99µs ± 0%  0.39µs ± 1%  -60.74%  (p=0.000 n=12+20)

Change-Id: I3598278a6c88d7f6a6ce66c56b9d25f9dd2f4c9a
Reviewed-on: https://go-review.googlesource.com/98095
Reviewed-by: Ian Lance Taylor <iant@golang.org>
Run-TryBot: Ian Lance Taylor <iant@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>

8 files changed, 110 insertions, 18 deletions

diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go
index 7ebca7f856..d8c1592a1d 100644
--- a/src/runtime/os_linux.go
+++ b/src/runtime/os_linux.go
@@ -268,6 +268,7 @@ func sysauxv(auxv []uintptr) int {
 		}
 
 		archauxv(tag, val)
+		vdsoauxv(tag, val)
 	}
 	return i / 2
 }
diff --git a/src/runtime/os_linux_noauxv.go b/src/runtime/os_linux_noauxv.go
index db6e5a0530..d6174c33fe 100644
--- a/src/runtime/os_linux_noauxv.go
+++ b/src/runtime/os_linux_noauxv.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build !386,!amd64,!arm,!arm64,!mips,!mipsle,!mips64,!mips64le,!s390x,!ppc64,!ppc64le
+// +build !arm,!arm64,!mips,!mipsle,!mips64,!mips64le,!s390x,!ppc64,!ppc64le
 
 package runtime
 
diff --git a/src/runtime/os_linux_novdso.go b/src/runtime/os_linux_novdso.go
new file mode 100644
index 0000000000..41acd957cf
--- /dev/null
+++ b/src/runtime/os_linux_novdso.go
@@ -0,0 +1,10 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !386,!amd64,!arm
+
+package runtime
+
+func vdsoauxv(tag, val uintptr) {
+}
diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s
index ef72ae0308..39046bb5c5 100644
--- a/src/runtime/sys_linux_arm.s
+++ b/src/runtime/sys_linux_arm.s
@@ -10,6 +10,9 @@
 #include "go_tls.h"
 #include "textflag.h"
 
+#define CLOCK_REALTIME	0
+#define CLOCK_MONOTONIC	1
+
 // for EABI, as we don't support OABI
 #define SYS_BASE 0x0
 
@@ -209,31 +212,90 @@ TEXT runtime·mincore(SB),NOSPLIT,$0
 	MOVW	R0, ret+12(FP)
 	RET
 
-TEXT runtime·walltime(SB), NOSPLIT, $32
-	MOVW	$0, R0  // CLOCK_REALTIME
-	MOVW	$8(R13), R1  // timespec
+TEXT runtime·walltime(SB),NOSPLIT,$0-12
+	// We don't know how much stack space the VDSO code will need,
+	// so switch to g0.
+
+	// Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets.
+	MOVW	R13, R4	// R4 is unchanged by C code.
+
+	MOVW	g_m(g), R1
+	MOVW	m_curg(R1), R0
+
+	CMP	R1, R0		// Only switch if on curg.
+	B.NE	noswitch
+
+	MOVW	m_g0(R1), R0
+	MOVW	(g_sched+gobuf_sp)(R0), R13	 // Set SP to g0 stack
+
+noswitch:
+	SUB	$24, R13	// Space for results
+	BIC	$0x7, R13	// Align for C code
+
+	MOVW	$CLOCK_REALTIME, R0
+	MOVW	$8(R13), R1	// timespec
+	MOVW	runtime·__vdso_clock_gettime_sym(SB), R11
+	CMP	$0, R11
+	B.EQ	fallback
+
+	BL	(R11)
+	JMP	finish
+
+fallback:
 	MOVW	$SYS_clock_gettime, R7
 	SWI	$0
-	
+
+finish:
 	MOVW	8(R13), R0  // sec
 	MOVW	12(R13), R2  // nsec
-	
+
+	MOVW	R4, R13		// Restore real SP
+
 	MOVW	R0, sec_lo+0(FP)
 	MOVW	$0, R1
 	MOVW	R1, sec_hi+4(FP)
 	MOVW	R2, nsec+8(FP)
-	RET	
+	RET
 
 // int64 nanotime(void)
-TEXT runtime·nanotime(SB),NOSPLIT,$32
-	MOVW	$1, R0  // CLOCK_MONOTONIC
-	MOVW	$8(R13), R1  // timespec
+TEXT runtime·nanotime(SB),NOSPLIT,$0-8
+	// Switch to g0 stack. See comment above in runtime·walltime.
+
+	// Save old SP. Use R13 instead of SP to avoid linker rewriting the offsets.
+	MOVW	R13, R4	// R4 is unchanged by C code.
+
+	MOVW	g_m(g), R1
+	MOVW	m_curg(R1), R0
+
+	CMP	R1, R0		// Only switch if on curg.
+	B.NE	noswitch
+
+	MOVW	m_g0(R1), R0
+	MOVW	(g_sched+gobuf_sp)(R0), R13	// Set SP to g0 stack
+
+noswitch:
+	SUB	$24, R13	// Space for results
+	BIC	$0x7, R13	// Align for C code
+
+	MOVW	$CLOCK_MONOTONIC, R0
+	MOVW	$8(R13), R1	// timespec
+	MOVW	runtime·__vdso_clock_gettime_sym(SB), R11
+	CMP	$0, R11
+	B.EQ	fallback
+
+	BL	(R11)
+	JMP	finish
+
+fallback:
 	MOVW	$SYS_clock_gettime, R7
 	SWI	$0
-	
-	MOVW	8(R13), R0  // sec
-	MOVW	12(R13), R2  // nsec
-	
+
+finish:
+	MOVW	8(R13), R0	// sec
+	MOVW	12(R13), R2	// nsec
+
+	MOVW	R4, R13		// Restore real SP
+
 	MOVW	$1000000000, R3
 	MULLU	R0, R3, (R1, R0)
 	MOVW	$0, R4
diff --git a/src/runtime/vdso_elf32.go b/src/runtime/vdso_elf32.go
index e1a693db90..2720f33eed 100644
--- a/src/runtime/vdso_elf32.go
+++ b/src/runtime/vdso_elf32.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386
+// +build 386 arm
 
 package runtime
 
diff --git a/src/runtime/vdso_linux.go b/src/runtime/vdso_linux.go
index 61872e3b27..46b92869ab 100644
--- a/src/runtime/vdso_linux.go
+++ b/src/runtime/vdso_linux.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64
+// +build 386 amd64 arm
 
 package runtime
 
@@ -264,7 +264,7 @@ func vdso_parse_symbols(info *vdso_info, version int32) {
 	}
 }
 
-func archauxv(tag, val uintptr) {
+func vdsoauxv(tag, val uintptr) {
 	switch tag {
 	case _AT_SYSINFO_EHDR:
 		if val == 0 {
diff --git a/src/runtime/vdso_linux_arm.go b/src/runtime/vdso_linux_arm.go
new file mode 100644
index 0000000000..0f5fd5a38c
--- /dev/null
+++ b/src/runtime/vdso_linux_arm.go
@@ -0,0 +1,19 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package runtime
+
+const (
+	// vdsoArrayMax is the byte-size of a maximally sized array on this architecture.
+	// See cmd/compile/internal/arm/galign.go arch.MAXWIDTH initialization, but must also
+	// be constrained to max +ve int.
+	vdsoArrayMax = 1<<31 - 1
+)
+
+var sym_keys = []symbol_key{
+	{"__vdso_clock_gettime", 0xd35ec75, 0x6e43a318, &__vdso_clock_gettime_sym},
+}
+
+// initialize to fall back to syscall
+var __vdso_clock_gettime_sym uintptr = 0
diff --git a/src/runtime/vdso_linux_test.go b/src/runtime/vdso_linux_test.go
index f507ee98ee..890f907d22 100644
--- a/src/runtime/vdso_linux_test.go
+++ b/src/runtime/vdso_linux_test.go
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build linux
-// +build 386 amd64
+// +build 386 amd64 arm
 
 package runtime_test