runtime: inline MCache_Alloc() into mallocgc()

benchmark                    old ns/op    new ns/op    delta
BenchmarkMalloc8                    68           62   -8.63%
BenchmarkMalloc16                   75           69   -7.94%
BenchmarkMallocTypeInfo8           102           98   -3.73%
BenchmarkMallocTypeInfo16          108          103   -4.63%

R=golang-dev, dave, khr
CC=golang-dev
https://golang.org/cl/9790043

3 files changed, 25 insertions, 30 deletions

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index cf18e8c9e5..47eb005894 100644
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -31,9 +31,10 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 	int32 sizeclass;
 	intgo rate;
 	MCache *c;
+	MCacheList *l;
 	uintptr npages;
 	MSpan *s;
-	void *v;
+	MLink *v;
 
 	if(runtime·gcwaiting && g != m->g0 && m->locks == 0 && dogc)
 		runtime·gosched();
@@ -56,9 +57,20 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		else
 			sizeclass = runtime·size_to_class128[(size-1024+127) >> 7];
 		size = runtime·class_to_size[sizeclass];
-		v = runtime·MCache_Alloc(c, sizeclass, size, zeroed);
-		if(v == nil)
-			runtime·throw("out of memory");
+		l = &c->list[sizeclass];
+		if(l->list == nil)
+			runtime·MCache_Refill(c, sizeclass);
+		v = l->list;
+		l->list = v->next;
+		l->nlist--;
+		if(zeroed) {
+			v->next = nil;
+			// block is zeroed iff second word is zero ...
+			if(size > sizeof(uintptr) && ((uintptr*)v)[1] != 0)
+				runtime·memclr((byte*)v, size);
+		}
+		c->local_cachealloc += size;
+		c->local_objects++;
 		c->local_alloc += size;
 		c->local_total_alloc += size;
 		c->local_by_size[sizeclass].nmalloc++;
diff --git a/src/pkg/runtime/malloc.h b/src/pkg/runtime/malloc.h
index 99a1966071..1085344ee1 100644
--- a/src/pkg/runtime/malloc.h
+++ b/src/pkg/runtime/malloc.h
@@ -305,7 +305,7 @@ struct MCache
 
 };
 
-void*	runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed);
+void	runtime·MCache_Refill(MCache *c, int32 sizeclass);
 void	runtime·MCache_Free(MCache *c, void *p, int32 sizeclass, uintptr size);
 void	runtime·MCache_ReleaseAll(MCache *c);
 
diff --git a/src/pkg/runtime/mcache.c b/src/pkg/runtime/mcache.c
index 3df0450fea..219eb8d4d6 100644
--- a/src/pkg/runtime/mcache.c
+++ b/src/pkg/runtime/mcache.c
@@ -10,35 +10,18 @@
 #include "arch_GOARCH.h"
 #include "malloc.h"
 
-void*
-runtime·MCache_Alloc(MCache *c, int32 sizeclass, uintptr size, int32 zeroed)
+void
+runtime·MCache_Refill(MCache *c, int32 sizeclass)
 {
 	MCacheList *l;
-	MLink *v;
 
-	// Allocate from list.
+	// Replenish using central lists.
 	l = &c->list[sizeclass];
-	if(l->list == nil) {
-		// Replenish using central lists.
-		l->nlist = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], &l->list);
-		if(l->list == nil)
-			runtime·throw("out of memory");
-	}
-	v = l->list;
-	l->list = v->next;
-	l->nlist--;
-
-	// v is zeroed except for the link pointer
-	// that we used above; zero that.
-	v->next = nil;
-	if(zeroed) {
-		// block is zeroed iff second word is zero ...
-		if(size > sizeof(uintptr) && ((uintptr*)v)[1] != 0)
-			runtime·memclr((byte*)v, size);
-	}
-	c->local_cachealloc += size;
-	c->local_objects++;
-	return v;
+	if(l->list)
+		runtime·throw("MCache_Refill: the list is not empty");
+	l->nlist = runtime·MCentral_AllocList(&runtime·mheap->central[sizeclass], &l->list);
+	if(l->list == nil)
+		runtime·throw("out of memory");
 }
 
 // Take n elements off l and return them to the central free list.