runtime: refactor mallocgc

Make it accept type, combine flags.
Several reasons for the change:
1. mallocgc and settype must be atomic wrt GC
2. settype is called from only one place now
3. it will help performance (eventually settype
functionality must be combined with markallocated)
4. flags are easier to read now (no mallocgc(sz, 0, 1, 0) anymore)

R=golang-dev, iant, nightlyone, rsc, dave, khr, bradfitz, r
CC=golang-dev
https://golang.org/cl/10136043

1 files changed, 36 insertions, 85 deletions

diff --git a/src/pkg/runtime/malloc.goc b/src/pkg/runtime/malloc.goc
index 352f93f69d..f31f119082 100644
--- a/src/pkg/runtime/malloc.goc
+++ b/src/pkg/runtime/malloc.goc
@@ -28,8 +28,9 @@ extern volatile intgo runtime·MemProfileRate;
 // Allocate an object of at least size bytes.
 // Small objects are allocated from the per-thread cache's free lists.
 // Large objects (> 32 kB) are allocated straight from the heap.
+// If the block will be freed with runtime·free(), typ must be 0.
 void*
-runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
+runtime·mallocgc(uintptr size, uintptr typ, uint32 flag)
 {
 	int32 sizeclass;
 	intgo rate;
@@ -39,13 +40,20 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 	MSpan *s;
 	MLink *v;
 
-	if(runtime·gcwaiting && g != m->g0 && m->locks == 0 && dogc)
+	if(runtime·gcwaiting && g != m->g0 && m->locks == 0 && !(flag & FlagNoInvokeGC))
 		runtime·gosched();
+	if(size == 0) {
+		// All 0-length allocations use this pointer.
+		// The language does not require the allocations to
+		// have distinct values.
+		return &runtime·zerobase;
+	}
 	if(m->mallocing)
 		runtime·throw("malloc/free - deadlock");
+	// Disable preemption during settype_flush.
+	// We can not use m->mallocing for this, because settype_flush calls mallocgc.
+	m->locks++;
 	m->mallocing = 1;
-	if(size == 0)
-		size = 1;
 
 	if(DebugTypeAtBlockEnd)
 		size += sizeof(uintptr);
@@ -65,7 +73,7 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		v = l->list;
 		l->list = v->next;
 		l->nlist--;
-		if(zeroed) {
+		if(!(flag & FlagNoZero)) {
 			v->next = nil;
 			// block is zeroed iff second word is zero ...
 			if(size > sizeof(uintptr) && ((uintptr*)v)[1] != 0)
@@ -79,7 +87,7 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		npages = size >> PageShift;
 		if((size & PageMask) != 0)
 			npages++;
-		s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, zeroed);
+		s = runtime·MHeap_Alloc(&runtime·mheap, npages, 0, 1, !(flag & FlagNoZero));
 		if(s == nil)
 			runtime·throw("out of memory");
 		s->limit = (byte*)(s->start<<PageShift) + size;
@@ -94,10 +102,23 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		runtime·markallocated(v, size, (flag&FlagNoPointers) != 0);
 
 	if(DebugTypeAtBlockEnd)
-		*(uintptr*)((uintptr)v+size-sizeof(uintptr)) = 0;
+		*(uintptr*)((uintptr)v+size-sizeof(uintptr)) = typ;
+
+	if(UseSpanType && !(flag & FlagNoPointers) && typ != 0) {
+		uintptr *buf, i;
+
+		buf = m->settype_buf;
+		i = m->settype_bufsize;
+		buf[i++] = (uintptr)v;
+		buf[i++] = typ;
+		m->settype_bufsize = i;
+	}
 
 	m->mallocing = 0;
-	if(g->preempt)  // restore the preemption request in case we've cleared it in newstack
+	if(UseSpanType && !(flag & FlagNoPointers) && typ != 0 && m->settype_bufsize == nelem(m->settype_buf))
+		runtime·settype_flush(m, false);
+	m->locks--;
+	if(m->locks == 0 && g->preempt)  // restore the preemption request in case we've cleared it in newstack
 		g->stackguard0 = StackPreempt;
 
 	if(!(flag & FlagNoProfiling) && (rate = runtime·MemProfileRate) > 0) {
@@ -117,7 +138,7 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 		}
 	}
 
-	if(dogc && mstats.heap_alloc >= mstats.next_gc)
+	if(!(flag & FlagNoInvokeGC) && mstats.heap_alloc >= mstats.next_gc)
 		runtime·gc(0);
 
 	if(raceenabled) {
@@ -130,7 +151,7 @@ runtime·mallocgc(uintptr size, uint32 flag, int32 dogc, int32 zeroed)
 void*
 runtime·malloc(uintptr size)
 {
-	return runtime·mallocgc(size, 0, 0, 1);
+	return runtime·mallocgc(size, 0, FlagNoInvokeGC);
 }
 
 // Free the object whose base pointer is v.
@@ -586,7 +607,7 @@ runtime·settype_flush(M *mp, bool sysalloc)
 			nbytes3 = 8*sizeof(uintptr) + 1*ntypes;
 
 			if(!sysalloc) {
-				data3 = runtime·mallocgc(nbytes3, FlagNoProfiling|FlagNoPointers, 0, 1);
+				data3 = runtime·mallocgc(nbytes3, 0, FlagNoProfiling|FlagNoPointers|FlagNoInvokeGC);
 			} else {
 				data3 = runtime·SysAlloc(nbytes3);
 				if(data3 == nil)
@@ -624,7 +645,7 @@ runtime·settype_flush(M *mp, bool sysalloc)
 				nbytes2 = ntypes * sizeof(uintptr);
 
 				if(!sysalloc) {
-					data2 = runtime·mallocgc(nbytes2, FlagNoProfiling|FlagNoPointers, 0, 1);
+					data2 = runtime·mallocgc(nbytes2, 0, FlagNoProfiling|FlagNoPointers|FlagNoInvokeGC);
 				} else {
 					data2 = runtime·SysAlloc(nbytes2);
 					if(data2 == nil)
@@ -660,42 +681,6 @@ runtime·settype_flush(M *mp, bool sysalloc)
 	mp->settype_bufsize = 0;
 }
 
-// It is forbidden to use this function if it is possible that
-// explicit deallocation via calling runtime·free(v) may happen.
-void
-runtime·settype(void *v, uintptr t)
-{
-	M *mp;
-	uintptr *buf;
-	uintptr i;
-	MSpan *s;
-
-	if(t == 0)
-		runtime·throw("settype: zero type");
-
-	mp = m;
-	m->locks++;
-	buf = mp->settype_buf;
-	i = mp->settype_bufsize;
-	buf[i+0] = (uintptr)v;
-	buf[i+1] = t;
-	i += 2;
-	mp->settype_bufsize = i;
-
-	if(i == nelem(mp->settype_buf)) {
-		runtime·settype_flush(mp, false);
-	}
-
-	if(DebugTypeAtBlockEnd) {
-		s = runtime·MHeap_Lookup(&runtime·mheap, v);
-		*(uintptr*)((uintptr)v+s->elemsize-sizeof(uintptr)) = t;
-	}
-
-	m->locks--;
-	if(m->locks == 0 && g->preempt)  // restore the preemption request in case we've cleared it in newstack
-		g->stackguard0 = StackPreempt;
-}
-
 void
 runtime·settype_sysfree(MSpan *s)
 {
@@ -767,61 +752,27 @@ runtime·gettype(void *v)
 void*
 runtime·mal(uintptr n)
 {
-	return runtime·mallocgc(n, 0, 1, 1);
+	return runtime·mallocgc(n, 0, 0);
 }
 
 #pragma textflag 7
 void
 runtime·new(Type *typ, uint8 *ret)
 {
-	uint32 flag;
-
 	if(raceenabled)
 		m->racepc = runtime·getcallerpc(&typ);
-
-	if(typ->size == 0) {
-		// All 0-length allocations use this pointer.
-		// The language does not require the allocations to
-		// have distinct values.
-		ret = (uint8*)&runtime·zerobase;
-	} else {
-		flag = typ->kind&KindNoPointers ? FlagNoPointers : 0;
-		ret = runtime·mallocgc(typ->size, flag, 1, 1);
-
-		if(UseSpanType && !flag) {
-			if(false)
-				runtime·printf("new %S: %p\n", *typ->string, ret);
-			runtime·settype(ret, (uintptr)typ | TypeInfo_SingleObject);
-		}
-	}
-
+	ret = runtime·mallocgc(typ->size, (uintptr)typ | TypeInfo_SingleObject, typ->kind&KindNoPointers ? FlagNoPointers : 0);
 	FLUSH(&ret);
 }
 
 static void*
 cnew(Type *typ, intgo n, int32 objtyp)
 {
-	uint32 flag;
-	void *ret;
-
 	if((objtyp&(PtrSize-1)) != objtyp)
 		runtime·throw("runtime: invalid objtyp");
 	if(n < 0 || (typ->size > 0 && n > MaxMem/typ->size))
 		runtime·panicstring("runtime: allocation size out of range");
-	if(typ->size == 0 || n == 0) {
-		// All 0-length allocations use this pointer.
-		// The language does not require the allocations to
-		// have distinct values.
-		return &runtime·zerobase;
-	}
-	flag = typ->kind&KindNoPointers ? FlagNoPointers : 0;
-	ret = runtime·mallocgc(typ->size*n, flag, 1, 1);
-	if(UseSpanType && !flag) {
-		if(false)
-			runtime·printf("cnew [%D]%S: %p\n", (int64)n, *typ->string, ret);
-		runtime·settype(ret, (uintptr)typ | objtyp);
-	}
-	return ret;
+	return runtime·mallocgc(typ->size*n, (uintptr)typ | objtyp, typ->kind&KindNoPointers ? FlagNoPointers : 0);
 }
 
 // same as runtime·new, but callable from C