runtime: per-P defer pool

Instead of a per-goroutine stack of defers for all sizes,
introduce per-P defer pool for argument sizes 8, 24, 40, 56, 72 bytes.

For a program that starts 1e6 goroutines and then joins then:
old: rss=6.6g virtmem=10.2g time=4.85s
new: rss=4.5g virtmem= 8.2g time=3.48s

R=golang-codereviews, rsc
CC=golang-codereviews
https://golang.org/cl/42750044

1 files changed, 72 insertions, 86 deletions

diff --git a/src/pkg/runtime/panic.c b/src/pkg/runtime/panic.c
index d85a8fefba..7bd408aea8 100644
--- a/src/pkg/runtime/panic.c
+++ b/src/pkg/runtime/panic.c
@@ -13,108 +13,63 @@
 uint32 runtime·panicking;
 static Lock paniclk;
 
-enum
-{
-	DeferChunkSize = 2048
-};
+// Each P holds pool for defers with arg sizes 8, 24, 40, 56 and 72 bytes.
+// Memory block is 40 (24 for 32 bits) bytes larger due to Defer header.
+// This maps exactly to malloc size classes.
+
+// defer size class for arg size sz
+#define DEFERCLASS(sz) (((sz)+7)>>4)
+// total size of memory block for defer with arg size sz
+#define TOTALSIZE(sz) (sizeof(Defer) - sizeof(((Defer*)nil)->args) + ROUND(sz, sizeof(uintptr)))
 
-// Allocate a Defer, usually as part of the larger frame of deferred functions.
-// Each defer must be released with both popdefer and freedefer.
+// Allocate a Defer, usually using per-P pool.
+// Each defer must be released with freedefer.
 static Defer*
 newdefer(int32 siz)
 {
-	int32 total;
-	DeferChunk *c;
+	int32 total, sc;
 	Defer *d;
-	
-	c = g->dchunk;
-	total = sizeof(*d) + ROUND(siz, sizeof(uintptr)) - sizeof(d->args);
-	if(c == nil || total > DeferChunkSize - c->off) {
-		if(total > DeferChunkSize / 2) {
-			// Not worth putting in any chunk.
-			// Allocate a separate block.
-			d = runtime·malloc(total);
-			d->siz = siz;
-			d->special = 1;
-			d->free = 1;
-			d->link = g->defer;
-			g->defer = d;
-			return d;
-		}
+	P *p;
 
-		// Cannot fit in current chunk.
-		// Switch to next chunk, allocating if necessary.
-		c = g->dchunknext;
-		if(c == nil)
-			c = runtime·malloc(DeferChunkSize);
-		c->prev = g->dchunk;
-		c->off = sizeof(*c);
-		g->dchunk = c;
-		g->dchunknext = nil;
+	d = nil;
+	sc = DEFERCLASS(siz);
+	if(sc < nelem(p->deferpool)) {
+		p = m->p;
+		d = p->deferpool[sc];
+		if(d)
+			p->deferpool[sc] = d->link;
+	}
+	if(d == nil) {
+		// deferpool is empty or just a big defer
+		total = TOTALSIZE(siz);
+		d = runtime·malloc(total);
 	}
-
-	d = (Defer*)((byte*)c + c->off);
-	c->off += total;
 	d->siz = siz;
 	d->special = 0;
-	d->free = 0;
 	d->link = g->defer;
 	g->defer = d;
-	return d;	
-}
-
-// Pop the current defer from the defer stack.
-// Its contents are still valid until the goroutine begins executing again.
-// In particular it is safe to call reflect.call(d->fn, d->argp, d->siz) after
-// popdefer returns.
-static void
-popdefer(void)
-{
-	Defer *d;
-	DeferChunk *c;
-	int32 total;
-	
-	d = g->defer;
-	if(d == nil)
-		runtime·throw("runtime: popdefer nil");
-	g->defer = d->link;
-	if(d->special) {
-		// Nothing else to do.
-		return;
-	}
-	total = sizeof(*d) + ROUND(d->siz, sizeof(uintptr)) - sizeof(d->args);
-	c = g->dchunk;
-	if(c == nil || (byte*)d+total != (byte*)c+c->off)
-		runtime·throw("runtime: popdefer phase error");
-	c->off -= total;
-	if(c->off == sizeof(*c)) {
-		// Chunk now empty, so pop from stack.
-		// Save in dchunknext both to help with pingponging between frames
-		// and to make sure d is still valid on return.
-		if(g->dchunknext != nil)
-			runtime·free(g->dchunknext);
-		g->dchunknext = c;
-		g->dchunk = c->prev;
-	}
+	return d;
 }
 
 // Free the given defer.
-// For defers in the per-goroutine chunk this just clears the saved arguments.
-// For large defers allocated on the heap, this frees them.
 // The defer cannot be used after this call.
 static void
 freedefer(Defer *d)
 {
-	int32 total;
+	int32 sc;
+	P *p;
 
-	if(d->special) {
-		if(d->free)
-			runtime·free(d);
-	} else {
-		// Wipe out any possible pointers in argp/pc/fn/args.
-		total = sizeof(*d) + ROUND(d->siz, sizeof(uintptr)) - sizeof(d->args);
-		runtime·memclr((byte*)d, total);
-	}
+	if(d->special)
+		return;
+	sc = DEFERCLASS(d->siz);
+	if(sc < nelem(p->deferpool)) {
+		p = m->p;
+		d->link = p->deferpool[sc];
+		p->deferpool[sc] = d;
+		// No need to wipe out pointers in argp/pc/fn/args,
+		// because we empty the pool before GC.
+	} else
+		runtime·free(d);
 }
 
 // Create a new deferred function fn with siz bytes of arguments.
@@ -182,7 +137,7 @@ runtime·deferreturn(uintptr arg0)
 	m->locks++;
 	runtime·memmove(argp, d->args, d->siz);
 	fn = d->fn;
-	popdefer();
+	g->defer = d->link;
 	freedefer(d);
 	m->locks--;
 	if(m->locks == 0 && g->preempt)
@@ -190,6 +145,37 @@ runtime·deferreturn(uintptr arg0)
 	runtime·jmpdefer(fn, argp);
 }
 
+// Ensure that defer arg sizes that map to the same defer size class
+// also map to the same malloc size class.
+void
+runtime·testdefersizes(void)
+{
+	P *p;
+	int32 i, siz, defersc, mallocsc;
+	int32 map[nelem(p->deferpool)];
+
+	for(i=0; i<nelem(p->deferpool); i++)
+		map[i] = -1;
+	for(i=0;; i++) {
+		defersc = DEFERCLASS(i);
+		if(defersc >= nelem(p->deferpool))
+			break;
+		siz = TOTALSIZE(i);
+		mallocsc = runtime·SizeToClass(siz);
+		siz = runtime·class_to_size[mallocsc];
+		// runtime·printf("defer class %d: arg size %d, block size %d(%d)\n", defersc, i, siz, mallocsc);
+		if(map[defersc] < 0) {
+			map[defersc] = mallocsc;
+			continue;
+		}
+		if(map[defersc] != mallocsc) {
+			runtime·printf("bad defer size class: i=%d siz=%d mallocsc=%d/%d\n",
+				i, siz, map[defersc], mallocsc);
+			runtime·throw("bad defer size class");
+		}
+	}
+}
+
 // Run all deferred functions for the current goroutine.
 static void
 rundefer(void)
@@ -197,7 +183,7 @@ rundefer(void)
 	Defer *d;
 
 	while((d = g->defer) != nil) {
-		popdefer();
+		g->defer = d->link;
 		reflect·call(d->fn, (byte*)d->args, d->siz);
 		freedefer(d);
 	}
@@ -239,7 +225,7 @@ runtime·panic(Eface e)
 		if(d == nil)
 			break;
 		// take defer off list in case of recursive panic
-		popdefer();
+		g->defer = d->link;
 		g->ispanic = true;	// rock for newstack, where reflect.newstackcall ends up
 		argp = d->argp;
 		pc = d->pc;