runtime: faster entersyscall/exitsyscall

Replace cas with xadd in scheduler.
Suggested by Dmitriy in last code review.
Verified with Promela model.

When there's actual contention for the atomic word,
this avoids the looping that compare-and-swap requires.

benchmark                            old ns/op    new ns/op    delta
runtime_test.BenchmarkSyscall               32           26  -17.08%
runtime_test.BenchmarkSyscall-2            155           59  -61.81%
runtime_test.BenchmarkSyscall-3            112           52  -52.95%
runtime_test.BenchmarkSyscall-4             94           48  -48.57%
runtime_test.BenchmarkSyscallWork          871          872   +0.11%
runtime_test.BenchmarkSyscallWork-2        481          477   -0.83%
runtime_test.BenchmarkSyscallWork-3        338          335   -0.89%
runtime_test.BenchmarkSyscallWork-4        263          256   -2.66%

R=golang-dev, iant
CC=golang-dev
https://golang.org/cl/4800047

1 files changed, 16 insertions, 41 deletions

diff --git a/src/pkg/runtime/proc.c b/src/pkg/runtime/proc.c
index 6416651ce5..13bc52bb68 100644
--- a/src/pkg/runtime/proc.c
+++ b/src/pkg/runtime/proc.c
@@ -773,7 +773,7 @@ runtime·gosched(void)
 void
 runtime·entersyscall(void)
 {
-	uint32 v, w;
+	uint32 v;
 
 	if(runtime·sched.predawn)
 		return;
@@ -796,24 +796,14 @@ runtime·entersyscall(void)
 	//	mcpu--
 	//	gwait not true
 	//	waitstop && mcpu <= mcpumax not true
-	// If we can do the same with a single atomic read/write,
+	// If we can do the same with a single atomic add,
 	// then we can skip the locks.
-	for(;;) {
-		v = runtime·sched.atomic;
-		if(atomic_gwaiting(v))
-			break;
-		if(atomic_waitstop(v) && atomic_mcpu(v)-1 <= atomic_mcpumax(v))
-			break;
-		w = v;
-		w += (-1<<mcpuShift);
-		if(runtime·cas(&runtime·sched.atomic, v, w))
-			return;
-	}
+	v = runtime·xadd(&runtime·sched.atomic, -1<<mcpuShift);
+	if(!atomic_gwaiting(v) && (!atomic_waitstop(v) || atomic_mcpu(v) > atomic_mcpumax(v)))
+		return;
 
 	schedlock();
-
-	// atomic { mcpu--; }
-	v = runtime·xadd(&runtime·sched.atomic, (-1<<mcpuShift));
+	v = runtime·atomicload(&runtime·sched.atomic);
 	if(atomic_gwaiting(v)) {
 		matchmg();
 		v = runtime·atomicload(&runtime·sched.atomic);
@@ -837,43 +827,28 @@ runtime·entersyscall(void)
 void
 runtime·exitsyscall(void)
 {
-	uint32 v, w;
+	uint32 v;
 
 	if(runtime·sched.predawn)
 		return;
 
 	// Fast path.
-	// If we can do the mcpu-- bookkeeping and
+	// If we can do the mcpu++ bookkeeping and
 	// find that we still have mcpu <= mcpumax, then we can
 	// start executing Go code immediately, without having to
 	// schedlock/schedunlock.
-	for(;;) {
-		// If the profiler frequency needs updating,
-		// take the slow path.
-		if(m->profilehz != runtime·sched.profilehz)
-			break;
-
-		v = runtime·sched.atomic;
-		if(atomic_mcpu(v) >= atomic_mcpumax(v))
-			break;
-
-		w = v;
-		w += (1<<mcpuShift);
-		if(runtime·cas(&runtime·sched.atomic, v, w)) {
-			// There's a cpu for us, so we can run.
-			g->status = Grunning;
-			// Garbage collector isn't running (since we are),
-			// so okay to clear gcstack.
-			g->gcstack = nil;
-			return;
-		}
+	v = runtime·xadd(&runtime·sched.atomic, (1<<mcpuShift));
+	if(m->profilehz == runtime·sched.profilehz && atomic_mcpu(v) <= atomic_mcpumax(v)) {
+		// There's a cpu for us, so we can run.
+		g->status = Grunning;
+		// Garbage collector isn't running (since we are),
+		// so okay to clear gcstack.
+		g->gcstack = nil;
+		return;
 	}
 
 	schedlock();
 
-	// atomic { mcpu++; }
-	runtime·xadd(&runtime·sched.atomic, (1<<mcpuShift));
-
 	// Tell scheduler to put g back on the run queue:
 	// mostly equivalent to g->status = Grunning,
 	// but keeps the garbage collector from thinking