runtime, syscall: use the vdso page on linux x86 for faster syscalls instead of int $0x80.

8l: fix handling CALL $(constant) code generated by 8a.
8a,8l: add indirect call instruction: CALL *data(SB).

R=rsc, iant
CC=golang-dev
https://golang.org/cl/4817054

3 files changed, 59 insertions, 21 deletions

diff --git a/src/pkg/runtime/linux/386/rt0.s b/src/pkg/runtime/linux/386/rt0.s
index 223e6d2ea4..83149540ec 100644
--- a/src/pkg/runtime/linux/386/rt0.s
+++ b/src/pkg/runtime/linux/386/rt0.s
@@ -5,5 +5,13 @@
 // Darwin and Linux use the same linkage to main
 
 TEXT _rt0_386_linux(SB),7,$0
-	JMP	_rt0_386(SB)
+	CALL	runtime·linux_setup_vdso(SB)
+	JMP		_rt0_386(SB)
+
+TEXT _fallback_vdso(SB),7,$0
+	INT	$0x80
+	RET
+
+DATA	runtime·_vdso(SB)/4, $_fallback_vdso(SB)
+GLOBL	runtime·_vdso(SB), $4
 
diff --git a/src/pkg/runtime/linux/386/signal.c b/src/pkg/runtime/linux/386/signal.c
index 8916e10bd1..4045b2efc3 100644
--- a/src/pkg/runtime/linux/386/signal.c
+++ b/src/pkg/runtime/linux/386/signal.c
@@ -182,3 +182,27 @@ os·sigpipe(void)
 	sigaction(SIGPIPE, SIG_DFL, false);
 	runtime·raisesigpipe();
 }
+
+#define AT_NULL		0
+#define AT_SYSINFO	32
+extern uint32 runtime·_vdso;
+
+#pragma textflag 7
+void runtime·linux_setup_vdso(int32 argc, void *argv_list)
+{
+	byte **argv = &argv_list;
+	byte **envp;
+	uint32 *auxv;
+
+	// skip envp to get to ELF auxiliary vector.
+	for(envp = &argv[argc+1]; *envp != nil; envp++)
+		;
+	envp++;
+	
+	for(auxv=(uint32*)envp; auxv[0] != AT_NULL; auxv += 2) {
+		if(auxv[0] == AT_SYSINFO) {
+			runtime·_vdso = auxv[1];
+			break;
+		}
+	}		
+}
diff --git a/src/pkg/runtime/linux/386/sys.s b/src/pkg/runtime/linux/386/sys.s
index 0b4a34986c..f87420f788 100644
--- a/src/pkg/runtime/linux/386/sys.s
+++ b/src/pkg/runtime/linux/386/sys.s
@@ -11,14 +11,14 @@
 TEXT runtime·exit(SB),7,$0
 	MOVL	$252, AX	// syscall number
 	MOVL	4(SP), BX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	INT $3	// not reached
 	RET
 
 TEXT runtime·exit1(SB),7,$0
 	MOVL	$1, AX	// exit - exit the current os thread
 	MOVL	4(SP), BX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	INT $3	// not reached
 	RET
 
@@ -27,13 +27,13 @@ TEXT runtime·open(SB),7,$0
 	MOVL	4(SP), BX
 	MOVL	8(SP), CX
 	MOVL	12(SP), DX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·close(SB),7,$0
 	MOVL	$6, AX		// syscall - close
 	MOVL	4(SP), BX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·write(SB),7,$0
@@ -41,7 +41,7 @@ TEXT runtime·write(SB),7,$0
 	MOVL	4(SP), BX
 	MOVL	8(SP), CX
 	MOVL	12(SP), DX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·read(SB),7,$0
@@ -49,16 +49,16 @@ TEXT runtime·read(SB),7,$0
 	MOVL	4(SP), BX
 	MOVL	8(SP), CX
 	MOVL	12(SP), DX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·raisesigpipe(SB),7,$12
 	MOVL	$224, AX	// syscall - gettid
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	MOVL	AX, 0(SP)	// arg 1 tid
 	MOVL	$13, 4(SP)	// arg 2 SIGPIPE
 	MOVL	$238, AX	// syscall - tkill
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·setitimer(SB),7,$0-24
@@ -66,7 +66,7 @@ TEXT runtime·setitimer(SB),7,$0-24
 	MOVL	4(SP), BX
 	MOVL	8(SP), CX
 	MOVL	12(SP), DX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·mincore(SB),7,$0-24
@@ -74,7 +74,7 @@ TEXT runtime·mincore(SB),7,$0-24
 	MOVL	4(SP), BX
 	MOVL	8(SP), CX
 	MOVL	12(SP), DX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·gettime(SB), 7, $32
@@ -82,7 +82,7 @@ TEXT runtime·gettime(SB), 7, $32
 	LEAL	8(SP), BX
 	MOVL	$0, CX
 	MOVL	$0, DX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 
 	MOVL	8(SP), BX	// sec
 	MOVL	sec+0(FP), DI
@@ -100,7 +100,7 @@ TEXT runtime·rt_sigaction(SB),7,$0
 	MOVL	8(SP), CX
 	MOVL	12(SP), DX
 	MOVL	16(SP), SI
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 TEXT runtime·sigtramp(SB),7,$44
@@ -138,7 +138,9 @@ TEXT runtime·sigignore(SB),7,$0
 
 TEXT runtime·sigreturn(SB),7,$0
 	MOVL	$173, AX	// rt_sigreturn
-	INT $0x80
+	// Sigreturn expects same SP as signal handler,
+	// so cannot CALL *runtime._vsdo(SB) here.
+	INT	$0x80
 	INT $3	// not reached
 	RET
 
@@ -151,7 +153,7 @@ TEXT runtime·mmap(SB),7,$0
 	MOVL	20(SP), DI
 	MOVL	24(SP), BP
 	SHRL	$12, BP
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	CMPL	AX, $0xfffff001
 	JLS	3(PC)
 	NOTL	AX
@@ -162,7 +164,7 @@ TEXT runtime·munmap(SB),7,$0
 	MOVL	$91, AX	// munmap
 	MOVL	4(SP), BX
 	MOVL	8(SP), CX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	CMPL	AX, $0xfffff001
 	JLS	2(PC)
 	INT $3
@@ -178,7 +180,7 @@ TEXT runtime·futex(SB),7,$0
 	MOVL	16(SP), SI
 	MOVL	20(SP), DI
 	MOVL	24(SP), BP
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET
 
 // int32 clone(int32 flags, void *stack, M *m, G *g, void (*fn)(void));
@@ -199,6 +201,10 @@ TEXT runtime·clone(SB),7,$0
 	MOVL	SI, 8(CX)
 	MOVL	$1234, 12(CX)
 
+	// cannot use CALL *runtime·_vdso(SB) here, because
+	// the stack changes during the system call (after 
+	// CALL *runtime·_vdso(SB), the child is still using
+	// the parent's stack when executing its RET instruction).
 	INT	$0x80
 
 	// In parent, return.
@@ -214,7 +220,7 @@ TEXT runtime·clone(SB),7,$0
 
 	// Initialize AX to Linux tid
 	MOVL	$224, AX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 
 	// In child on new stack.  Reload registers (paranoia).
 	MOVL	0(SP), BX	// m
@@ -262,7 +268,7 @@ TEXT runtime·sigaltstack(SB),7,$-8
 	MOVL	$186, AX	// sigaltstack
 	MOVL	new+4(SP), BX
 	MOVL	old+8(SP), CX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	CMPL	AX, $0xfffff001
 	JLS	2(PC)
 	INT	$3
@@ -323,7 +329,7 @@ TEXT runtime·setldt(SB),7,$32
 	MOVL	AX, CX	// user_desc
 	MOVL	$16, DX	// sizeof(user_desc)
 	MOVL	$123, AX	// syscall - modify_ldt
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 
 	// breakpoint on error
 	CMPL AX, $0xfffff001
@@ -340,5 +346,5 @@ TEXT runtime·setldt(SB),7,$32
 
 TEXT runtime·osyield(SB),7,$0
 	MOVL	$158, AX
-	INT	$0x80
+	CALL	*runtime·_vdso(SB)
 	RET