From e4f1d9cf2e948eb0f0bb91d7c253ab61dfff3a59 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Sun, 27 Mar 2016 17:29:53 -0700 Subject: runtime: make execution error panic values implement the Error interface Make execution panics implement Error as mandated by https://golang.org/ref/spec#Run_time_panics, instead of panics with strings. Fixes #14965 Change-Id: I7827f898b9b9c08af541db922cc24fa0800ff18a Reviewed-on: https://go-review.googlesource.com/21214 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/runtime/malloc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 5f1e2f64c0..ee4728c9a5 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -793,7 +793,7 @@ func newarray(typ *_type, n uintptr) unsafe.Pointer { flags |= flagNoScan } if int(n) < 0 || (typ.size > 0 && n > _MaxMem/typ.size) { - panic("runtime: allocation size out of range") + panic(plainError("runtime: allocation size out of range")) } return mallocgc(typ.size*n, typ, flags) } -- cgit v1.3 From a4dd6ea1524901fab5deac60772345babd058ae7 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Tue, 19 Apr 2016 09:18:59 -0700 Subject: runtime: add maxSliceCap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids expensive division calculations for many common slice element sizes. name old time/op new time/op delta MakeSlice-8 51.9ns ± 3% 35.1ns ± 2% -32.41% (p=0.000 n=10+10) GrowSliceBytes-8 44.1ns ± 2% 44.1ns ± 1% ~ (p=0.984 n=10+10) GrowSliceInts-8 60.9ns ± 3% 60.9ns ± 3% ~ (p=0.698 n=10+10) GrowSlicePtr-8 131ns ± 1% 120ns ± 2% -8.41% (p=0.000 n=8+10) GrowSliceStruct24Bytes-8 111ns ± 2% 103ns ± 3% -7.23% (p=0.000 n=8+8) Change-Id: I2630eb3d73c814db030cad16e620ea7fecbbd312 Reviewed-on: https://go-review.googlesource.com/22223 Reviewed-by: Keith Randall --- src/runtime/malloc.go | 2 +- src/runtime/slice.go | 35 +++++++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 11 deletions(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index ee4728c9a5..a3e55ec2fb 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -792,7 +792,7 @@ func newarray(typ *_type, n uintptr) unsafe.Pointer { if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } - if int(n) < 0 || (typ.size > 0 && n > _MaxMem/typ.size) { + if int(n) < 0 || n > maxSliceCap(typ.size) { panic(plainError("runtime: allocation size out of range")) } return mallocgc(typ.size*n, typ, flags) diff --git a/src/runtime/slice.go b/src/runtime/slice.go index f36ec0b466..f9414d7658 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -14,6 +14,28 @@ type slice struct { cap int } +// maxElems is a lookup table containing the maximum capacity for a slice. +// The index is the size of the slice element. +var maxElems = [...]uintptr{ + ^uintptr(0), + _MaxMem / 1, _MaxMem / 2, _MaxMem / 3, _MaxMem / 4, + _MaxMem / 5, _MaxMem / 6, _MaxMem / 7, _MaxMem / 8, + _MaxMem / 9, _MaxMem / 10, _MaxMem / 11, _MaxMem / 12, + _MaxMem / 13, _MaxMem / 14, _MaxMem / 15, _MaxMem / 16, + _MaxMem / 17, _MaxMem / 18, _MaxMem / 19, _MaxMem / 20, + _MaxMem / 21, _MaxMem / 22, _MaxMem / 23, _MaxMem / 24, + _MaxMem / 25, _MaxMem / 26, _MaxMem / 27, _MaxMem / 28, + _MaxMem / 29, _MaxMem / 30, _MaxMem / 31, _MaxMem / 32, +} + +// maxSliceCap returns the maximum capacity for a slice. +func maxSliceCap(elemsize uintptr) uintptr { + if elemsize < uintptr(len(maxElems)) { + return maxElems[elemsize] + } + return _MaxMem / elemsize +} + // TODO: take uintptrs instead of int64s? func makeslice(t *slicetype, len64, cap64 int64) slice { // NOTE: The len > maxElements check here is not strictly necessary, @@ -22,11 +44,7 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. - maxElements := ^uintptr(0) - if t.elem.size > 0 { - maxElements = _MaxMem / t.elem.size - } - + maxElements := maxSliceCap(t.elem.size) len := int(len64) if len64 < 0 || int64(len) != len64 || uintptr(len) > maxElements { panic(errorString("makeslice: len out of range")) @@ -84,27 +102,24 @@ func growslice(t *slicetype, old slice, cap int) slice { } } - var lenmem, capmem, maxcap uintptr + var lenmem, capmem uintptr const ptrSize = unsafe.Sizeof((*byte)(nil)) switch et.size { case 1: lenmem = uintptr(old.len) capmem = roundupsize(uintptr(newcap)) newcap = int(capmem) - maxcap = _MaxMem case ptrSize: lenmem = uintptr(old.len) * ptrSize capmem = roundupsize(uintptr(newcap) * ptrSize) newcap = int(capmem / ptrSize) - maxcap = _MaxMem / ptrSize default: lenmem = uintptr(old.len) * et.size capmem = roundupsize(uintptr(newcap) * et.size) newcap = int(capmem / et.size) - maxcap = _MaxMem / et.size } - if cap < old.cap || uintptr(newcap) > maxcap { + if cap < old.cap || uintptr(newcap) > maxSliceCap(et.size) { panic(errorString("growslice: cap out of range")) } -- cgit v1.3 From 001e8e8070e8ed3a578dbad93cc3f70257e965bd Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 19 Apr 2016 19:35:10 -0700 Subject: runtime: simplify mallocgc flag argument mallocgc can calculate noscan itself. The only remaining flag argument is needzero, so we just make that a boolean arg. Fixes #15379 Change-Id: I839a70790b2a0c9dbcee2600052bfbd6c8148e20 Reviewed-on: https://go-review.googlesource.com/22290 Reviewed-by: Josh Bleecher Snyder Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot --- src/runtime/chan.go | 2 +- src/runtime/malloc.go | 48 +++++++++++------------------------------------- src/runtime/mfinal.go | 2 +- src/runtime/os1_plan9.go | 4 ++-- src/runtime/panic.go | 2 +- src/runtime/select.go | 2 +- src/runtime/slice.go | 11 +++-------- src/runtime/string.go | 6 +++--- 8 files changed, 23 insertions(+), 54 deletions(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/chan.go b/src/runtime/chan.go index 8543cb4c9c..3fb0236785 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -74,7 +74,7 @@ func makechan(t *chantype, size int64) *hchan { // buf points into the same allocation, elemtype is persistent. // SudoG's are referenced from their owning thread so they can't be collected. // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. - c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, flagNoScan)) + c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, true)) if size > 0 && elem.size != 0 { c.buf = add(unsafe.Pointer(c), hchanSize) } else { diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index a3e55ec2fb..95d24a467a 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -87,9 +87,6 @@ import ( const ( debugMalloc = false - flagNoScan = _FlagNoScan - flagNoZero = _FlagNoZero - maxTinySize = _TinySize tinySizeClass = _TinySizeClass maxSmallSize = _MaxSmallSize @@ -487,16 +484,10 @@ func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer { // base address for all 0-byte allocations var zerobase uintptr -const ( - // flags to malloc - _FlagNoScan = 1 << 0 // GC doesn't have to scan object - _FlagNoZero = 1 << 1 // don't zero memory -) - // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. -func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { +func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") } @@ -505,10 +496,6 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { return unsafe.Pointer(&zerobase) } - if flags&flagNoScan == 0 && typ == nil { - throw("malloc missing type") - } - if debug.sbrk != 0 { align := uintptr(16) if typ != nil { @@ -553,14 +540,15 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { c := gomcache() var s *mspan var x unsafe.Pointer + noscan := typ == nil || typ.kind&kindNoPointers != 0 if size <= maxSmallSize { - if flags&flagNoScan != 0 && size < maxTinySize { + if noscan && size < maxTinySize { // Tiny allocator. // // Tiny allocator combines several tiny allocation requests // into a single memory block. The resulting memory block // is freed when all subobjects are unreachable. The subobjects - // must be FlagNoScan (don't have pointers), this ensures that + // must be noscan (don't have pointers), this ensures that // the amount of potentially wasted memory is bounded. // // Size of the memory block used for combining (maxTinySize) is tunable. @@ -650,7 +638,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { // prefetchnta offers best performance, see change list message. prefetchnta(uintptr(v.ptr().next)) x = unsafe.Pointer(v) - if flags&flagNoZero == 0 { + if needzero { v.ptr().next = 0 if size > 2*sys.PtrSize && ((*[2]uintptr)(x))[1] != 0 { memclr(unsafe.Pointer(v), size) @@ -661,13 +649,13 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { var s *mspan shouldhelpgc = true systemstack(func() { - s = largeAlloc(size, flags) + s = largeAlloc(size, needzero) }) x = unsafe.Pointer(uintptr(s.start << pageShift)) size = s.elemsize } - if flags&flagNoScan != 0 { + if noscan { // All objects are pre-marked as noscan. Nothing to do. } else { // If allocating a defer+arg block, now that we've picked a malloc size @@ -747,7 +735,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { return x } -func largeAlloc(size uintptr, flag uint32) *mspan { +func largeAlloc(size uintptr, needzero bool) *mspan { // print("largeAlloc size=", size, "\n") if size+_PageSize < size { @@ -763,7 +751,7 @@ func largeAlloc(size uintptr, flag uint32) *mspan { // pays the debt down to npage pages. deductSweepCredit(npages*_PageSize, npages) - s := mheap_.alloc(npages, 0, true, flag&_FlagNoZero == 0) + s := mheap_.alloc(npages, 0, true, needzero) if s == nil { throw("out of memory") } @@ -774,11 +762,7 @@ func largeAlloc(size uintptr, flag uint32) *mspan { // implementation of new builtin func newobject(typ *_type) unsafe.Pointer { - flags := uint32(0) - if typ.kind&kindNoPointers != 0 { - flags |= flagNoScan - } - return mallocgc(typ.size, typ, flags) + return mallocgc(typ.size, typ, true) } //go:linkname reflect_unsafe_New reflect.unsafe_New @@ -788,14 +772,10 @@ func reflect_unsafe_New(typ *_type) unsafe.Pointer { // implementation of make builtin for slices func newarray(typ *_type, n uintptr) unsafe.Pointer { - flags := uint32(0) - if typ.kind&kindNoPointers != 0 { - flags |= flagNoScan - } if int(n) < 0 || n > maxSliceCap(typ.size) { panic(plainError("runtime: allocation size out of range")) } - return mallocgc(typ.size*n, typ, flags) + return mallocgc(typ.size*n, typ, true) } //go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray @@ -803,12 +783,6 @@ func reflect_unsafe_NewArray(typ *_type, n uintptr) unsafe.Pointer { return newarray(typ, n) } -// rawmem returns a chunk of pointerless memory. It is -// not zeroed. -func rawmem(size uintptr) unsafe.Pointer { - return mallocgc(size, nil, flagNoScan|flagNoZero) -} - func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { mp.mcache.next_sample = nextSample() mProf_Malloc(x, size) diff --git a/src/runtime/mfinal.go b/src/runtime/mfinal.go index b862f019b6..f698e72709 100644 --- a/src/runtime/mfinal.go +++ b/src/runtime/mfinal.go @@ -172,7 +172,7 @@ func runfinq() { // all not yet finalized objects are stored in finq. // If we do not mark it as FlagNoScan, // the last finalized object is not collected. - frame = mallocgc(framesz, nil, flagNoScan) + frame = mallocgc(framesz, nil, true) framecap = framesz } diff --git a/src/runtime/os1_plan9.go b/src/runtime/os1_plan9.go index 2c257442ba..eb7a0c6481 100644 --- a/src/runtime/os1_plan9.go +++ b/src/runtime/os1_plan9.go @@ -17,10 +17,10 @@ func mpreinit(mp *m) { // Initialize stack and goroutine for note handling. mp.gsignal = malg(32 * 1024) mp.gsignal.m = mp - mp.notesig = (*int8)(mallocgc(_ERRMAX, nil, _FlagNoScan)) + mp.notesig = (*int8)(mallocgc(_ERRMAX, nil, true)) // Initialize stack for handling strings from the // errstr system call, as used in package syscall. - mp.errstr = (*byte)(mallocgc(_ERRMAX, nil, _FlagNoScan)) + mp.errstr = (*byte)(mallocgc(_ERRMAX, nil, true)) } func msigsave(mp *m) { diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 59fbc83369..382a20e4e7 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -205,7 +205,7 @@ func newdefer(siz int32) *_defer { if d == nil { // Allocate new defer+args. total := roundupsize(totaldefersize(uintptr(siz))) - d = (*_defer)(mallocgc(total, deferType, 0)) + d = (*_defer)(mallocgc(total, deferType, true)) } d.siz = siz gp := mp.curg diff --git a/src/runtime/select.go b/src/runtime/select.go index 9810db5453..433048fb79 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -626,7 +626,7 @@ const ( func reflect_rselect(cases []runtimeSelect) (chosen int, recvOK bool) { // flagNoScan is safe here, because all objects are also referenced from cases. size := selectsize(uintptr(len(cases))) - sel := (*hselect)(mallocgc(size, nil, flagNoScan)) + sel := (*hselect)(mallocgc(size, nil, true)) newselect(sel, int64(size), int32(len(cases))) r := new(bool) for i := range cases { diff --git a/src/runtime/slice.go b/src/runtime/slice.go index e86c1ce2c8..e15e6c4dc6 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -43,7 +43,6 @@ func makeslice(et *_type, len64, cap64 int64) slice { // when someone does make([]T, bignumber). 'cap out of range' is true too, // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. - maxElements := maxSliceCap(et.size) len := int(len64) if len64 < 0 || int64(len) != len64 || uintptr(len) > maxElements { @@ -55,11 +54,7 @@ func makeslice(et *_type, len64, cap64 int64) slice { panic(errorString("makeslice: cap out of range")) } - var flags uint32 - if et.kind&kindNoPointers != 0 { - flags = flagNoScan - } - p := mallocgc(et.size*uintptr(cap), et, flags) + p := mallocgc(et.size*uintptr(cap), et, true) return slice{p, len, cap} } @@ -128,12 +123,12 @@ func growslice(et *_type, old slice, cap int) slice { var p unsafe.Pointer if et.kind&kindNoPointers != 0 { - p = rawmem(capmem) + p = mallocgc(capmem, nil, false) memmove(p, old.array, lenmem) memclr(add(p, lenmem), capmem-lenmem) } else { // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory. - p = mallocgc(capmem, et, 0) + p = mallocgc(capmem, et, true) if !writeBarrier.enabled { memmove(p, old.array, lenmem) } else { diff --git a/src/runtime/string.go b/src/runtime/string.go index 2d20e0a9c3..112ce5d588 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -284,7 +284,7 @@ func stringiter2(s string, k int) (int, rune) { // The storage is not zeroed. Callers should use // b to set the string contents and then drop b. func rawstring(size int) (s string, b []byte) { - p := mallocgc(uintptr(size), nil, flagNoScan|flagNoZero) + p := mallocgc(uintptr(size), nil, false) stringStructOf(&s).str = p stringStructOf(&s).len = size @@ -302,7 +302,7 @@ func rawstring(size int) (s string, b []byte) { // rawbyteslice allocates a new byte slice. The byte slice is not zeroed. func rawbyteslice(size int) (b []byte) { cap := roundupsize(uintptr(size)) - p := mallocgc(cap, nil, flagNoScan|flagNoZero) + p := mallocgc(cap, nil, false) if cap != uintptr(size) { memclr(add(p, uintptr(size)), cap-uintptr(size)) } @@ -317,7 +317,7 @@ func rawruneslice(size int) (b []rune) { throw("out of memory") } mem := roundupsize(uintptr(size) * 4) - p := mallocgc(mem, nil, flagNoScan|flagNoZero) + p := mallocgc(mem, nil, false) if mem != uintptr(size)*4 { memclr(add(p, uintptr(size)*4), mem-uintptr(size)*4) } -- cgit v1.3 From 7e460e70d90295cf08ea627c0a0fff170aba5518 Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Wed, 20 Apr 2016 18:00:52 +0200 Subject: runtime: use type int to specify size for newarray MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consistently use type int for the size argument of runtime.newarray, runtime.reflect_unsafe_NewArray and reflect.unsafe_NewArray. Change-Id: Ic77bf2dde216c92ca8c49462f8eedc0385b6314e Reviewed-on: https://go-review.googlesource.com/22311 Reviewed-by: Keith Randall Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot --- src/runtime/chan.go | 2 +- src/runtime/hashmap.go | 4 ++-- src/runtime/malloc.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/chan.go b/src/runtime/chan.go index 3fb0236785..712ad8cef9 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -84,7 +84,7 @@ func makechan(t *chantype, size int64) *hchan { } } else { c = new(hchan) - c.buf = newarray(elem, uintptr(size)) + c.buf = newarray(elem, int(size)) } c.elemsize = uint16(elem.size) c.elemtype = elem diff --git a/src/runtime/hashmap.go b/src/runtime/hashmap.go index ff59faab5d..509cab2f0f 100644 --- a/src/runtime/hashmap.go +++ b/src/runtime/hashmap.go @@ -246,7 +246,7 @@ func makemap(t *maptype, hint int64, h *hmap, bucket unsafe.Pointer) *hmap { // If hint is large zeroing this memory could take a while. buckets := bucket if B != 0 { - buckets = newarray(t.bucket, uintptr(1)< maxSliceCap(typ.size) { +// newarray allocates an array of n elements of type typ. +func newarray(typ *_type, n int) unsafe.Pointer { + if n < 0 || uintptr(n) > maxSliceCap(typ.size) { panic(plainError("runtime: allocation size out of range")) } - return mallocgc(typ.size*n, typ, true) + return mallocgc(typ.size*uintptr(n), typ, true) } //go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray -func reflect_unsafe_NewArray(typ *_type, n uintptr) unsafe.Pointer { +func reflect_unsafe_NewArray(typ *_type, n int) unsafe.Pointer { return newarray(typ, n) } -- cgit v1.3 From 479501c14c9d36e27727bc4b4294d57c5ddc29d0 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sat, 16 Apr 2016 18:27:38 -0400 Subject: runtime: count black allocations toward scan work Currently we count black allocations toward the scannable heap size, but not toward the scan work we've done so far. This is clearly inconsistent (we have, in effect, scanned these allocations and since they're already black, we're not going to scan them again). Worse, it means we don't count black allocations toward the scannable heap size as of the *next* GC because this is based on the amount of scan work we did in this cycle. Fix this by counting black allocations as scan work. Currently the GC spends very little time in allocate-black mode, so this probably hasn't been a problem, but this will become important when we switch to always allocating black. Change-Id: If6ff693b070c385b65b6ecbbbbf76283a0f9d990 Reviewed-on: https://go-review.googlesource.com/22119 Reviewed-by: Rick Hudson Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot --- src/runtime/malloc.go | 8 +++++--- src/runtime/mgc.go | 9 +++++++-- src/runtime/mgcmark.go | 4 +++- 3 files changed, 15 insertions(+), 6 deletions(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 30f2a4fca5..3f437bc02f 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -655,6 +655,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { size = s.elemsize } + var scanSize uintptr if noscan { // All objects are pre-marked as noscan. Nothing to do. } else { @@ -673,11 +674,12 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // pointers, GC has to scan to the last // element. if typ.ptrdata != 0 { - c.local_scan += dataSize - typ.size + typ.ptrdata + scanSize = dataSize - typ.size + typ.ptrdata } } else { - c.local_scan += typ.ptrdata + scanSize = typ.ptrdata } + c.local_scan += scanSize // Ensure that the stores above that initialize x to // type-safe memory and set the heap bits occur before @@ -694,7 +696,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // a race marking the bit. if gcphase == _GCmarktermination || gcBlackenPromptly { systemstack(func() { - gcmarknewobject_m(uintptr(x), size) + gcmarknewobject_m(uintptr(x), size, scanSize) }) } diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 1c184db10b..d120dae05a 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -304,7 +304,8 @@ type gcControllerState struct { // scanWork is the total scan work performed this cycle. This // is updated atomically during the cycle. Updates occur in // bounded batches, since it is both written and read - // throughout the cycle. + // throughout the cycle. At the end of the cycle, this is how + // much of the retained heap is scannable. // // Currently this is the bytes of heap scanned. For most uses, // this is an opaque unit of work, but for estimation the @@ -1578,9 +1579,13 @@ func gcMark(start_time int64) { work.markrootDone = true for i := 0; i < int(gomaxprocs); i++ { - if !allp[i].gcw.empty() { + gcw := &allp[i].gcw + if !gcw.empty() { throw("P has cached GC work at end of mark termination") } + if gcw.scanWork != 0 || gcw.bytesMarked != 0 { + throw("P has unflushed stats at end of mark termination") + } } if trace.enabled { diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 1ab8315a29..0d05838987 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1134,12 +1134,14 @@ func gcDumpObject(label string, obj, off uintptr) { // If gcBlackenPromptly is true we are in the second mark phase phase so we allocate black. //go:nowritebarrier -func gcmarknewobject_m(obj, size uintptr) { +func gcmarknewobject_m(obj, size, scanSize uintptr) { if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } heapBitsForAddr(obj).setMarked() atomic.Xadd64(&work.bytesMarked, int64(size)) + gcw := &getg().m.p.ptr().gcw + gcw.scanWork += int64(scanSize) } // Checkmarking -- cgit v1.3 From 64a26b79ac781118d4fa364f884ce8080ba97870 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sun, 17 Apr 2016 11:42:37 -0400 Subject: runtime: simplify/optimize allocate-black a bit Currently allocating black switches to the system stack (which is probably a historical accident) and atomically updates the global bytes marked stat. Since we're about to depend on this much more, optimize it a bit by putting it back on the regular stack and updating the per-P bytes marked stat, which gets lazily folded into the global bytes marked stat. Change-Id: Ibbe16e5382d3fd2256e4381f88af342bf7020b04 Reviewed-on: https://go-review.googlesource.com/22170 Reviewed-by: Rick Hudson Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot --- src/runtime/malloc.go | 4 +--- src/runtime/mgcmark.go | 11 ++++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 3f437bc02f..9e1f47e1e6 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -695,9 +695,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // This may be racing with GC so do it atomically if there can be // a race marking the bit. if gcphase == _GCmarktermination || gcBlackenPromptly { - systemstack(func() { - gcmarknewobject_m(uintptr(x), size, scanSize) - }) + gcmarknewobject(uintptr(x), size, scanSize) } if raceenabled { diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 0d05838987..ad64b735a5 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1132,15 +1132,20 @@ func gcDumpObject(label string, obj, off uintptr) { } } -// If gcBlackenPromptly is true we are in the second mark phase phase so we allocate black. +// gcmarknewobject marks a newly allocated object black. obj must +// not contain any non-nil pointers. +// +// This is nosplit so it can manipulate a gcWork without preemption. +// //go:nowritebarrier -func gcmarknewobject_m(obj, size, scanSize uintptr) { +//go:nosplit +func gcmarknewobject(obj, size, scanSize uintptr) { if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } heapBitsForAddr(obj).setMarked() - atomic.Xadd64(&work.bytesMarked, int64(size)) gcw := &getg().m.p.ptr().gcw + gcw.bytesMarked += uint64(size) gcw.scanWork += int64(scanSize) } -- cgit v1.3 From 6002e01e34f5b847eb4c49ca84e9623d7204f5f2 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 30 Mar 2016 17:02:23 -0400 Subject: runtime: allocate black during GC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we allocate white for most of concurrent marking. This is based on the classical argument that it produces less floating garbage, since allocations during GC may not get linked into the heap and allocating white lets us reclaim these. However, it's not clear how often this actually happens, especially since our write barrier shades any pointer as soon as it's installed in the heap regardless of the color of the slot. On the other hand, allocating black has several advantages that seem to significantly outweigh this downside. 1) It naturally bounds the total scan work to the live heap size at the start of a GC cycle. Allocating white does not, and thus depends entirely on assists to prevent the heap from growing faster than it can be scanned. 2) It reduces the total amount of scan work per GC cycle by the size of newly allocated objects that are linked into the heap graph, since objects allocated black never need to be scanned. 3) It reduces total write barrier work since more objects will already be black when they are linked into the heap graph. This gives a slight overall improvement in benchmarks. name old time/op new time/op delta XBenchGarbage-12 2.24ms ± 0% 2.21ms ± 1% -1.32% (p=0.000 n=18+17) name old time/op new time/op delta BinaryTree17-12 2.60s ± 3% 2.53s ± 3% -2.56% (p=0.000 n=20+20) Fannkuch11-12 2.08s ± 1% 2.08s ± 0% ~ (p=0.452 n=19+19) FmtFprintfEmpty-12 45.1ns ± 2% 45.3ns ± 2% ~ (p=0.367 n=19+20) FmtFprintfString-12 131ns ± 3% 129ns ± 0% -1.60% (p=0.000 n=20+16) FmtFprintfInt-12 122ns ± 0% 121ns ± 2% -0.86% (p=0.000 n=16+19) FmtFprintfIntInt-12 187ns ± 1% 186ns ± 1% ~ (p=0.514 n=18+19) FmtFprintfPrefixedInt-12 189ns ± 0% 188ns ± 1% -0.54% (p=0.000 n=16+18) FmtFprintfFloat-12 256ns ± 0% 254ns ± 1% -0.43% (p=0.000 n=17+19) FmtManyArgs-12 769ns ± 0% 763ns ± 0% -0.72% (p=0.000 n=18+18) GobDecode-12 7.08ms ± 2% 7.00ms ± 1% -1.22% (p=0.000 n=20+20) GobEncode-12 5.88ms ± 0% 5.88ms ± 1% ~ (p=0.406 n=18+18) Gzip-12 214ms ± 0% 214ms ± 1% ~ (p=0.103 n=17+18) Gunzip-12 37.6ms ± 0% 37.6ms ± 0% ~ (p=0.563 n=17+17) HTTPClientServer-12 77.2µs ± 3% 76.9µs ± 2% ~ (p=0.606 n=20+20) JSONEncode-12 15.1ms ± 1% 15.2ms ± 2% ~ (p=0.138 n=19+19) JSONDecode-12 53.3ms ± 1% 53.1ms ± 1% -0.33% (p=0.000 n=19+18) Mandelbrot200-12 4.04ms ± 1% 4.04ms ± 1% ~ (p=0.075 n=19+18) GoParse-12 3.30ms ± 1% 3.29ms ± 1% -0.57% (p=0.000 n=18+16) RegexpMatchEasy0_32-12 69.5ns ± 1% 69.9ns ± 3% ~ (p=0.822 n=18+20) RegexpMatchEasy0_1K-12 237ns ± 1% 237ns ± 0% ~ (p=0.398 n=19+18) RegexpMatchEasy1_32-12 69.8ns ± 2% 69.5ns ± 1% ~ (p=0.090 n=20+16) RegexpMatchEasy1_1K-12 371ns ± 1% 372ns ± 1% ~ (p=0.178 n=19+20) RegexpMatchMedium_32-12 108ns ± 2% 108ns ± 3% ~ (p=0.124 n=20+19) RegexpMatchMedium_1K-12 33.9µs ± 2% 34.2µs ± 4% ~ (p=0.309 n=20+19) RegexpMatchHard_32-12 1.75µs ± 2% 1.77µs ± 4% +1.28% (p=0.018 n=19+18) RegexpMatchHard_1K-12 52.7µs ± 1% 53.4µs ± 4% +1.23% (p=0.013 n=15+18) Revcomp-12 354ms ± 1% 359ms ± 4% +1.27% (p=0.043 n=20+20) Template-12 63.6ms ± 2% 63.7ms ± 2% ~ (p=0.654 n=20+18) TimeParse-12 313ns ± 1% 316ns ± 2% +0.80% (p=0.014 n=17+20) TimeFormat-12 332ns ± 0% 329ns ± 0% -0.66% (p=0.000 n=16+16) [Geo mean] 51.7µs 51.6µs -0.09% Change-Id: I2214a6a0e4f544699ea166073249a8efdf080dc0 Reviewed-on: https://go-review.googlesource.com/21323 Reviewed-by: Rick Hudson Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot --- src/runtime/malloc.go | 4 ++-- src/runtime/mgc.go | 20 ++++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) (limited to 'src/runtime/malloc.go') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 9e1f47e1e6..081d1419cb 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -690,11 +690,11 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { publicationBarrier() } - // GCmarkterminate allocates black + // Allocate black during GC. // All slots hold nil so no scanning is needed. // This may be racing with GC so do it atomically if there can be // a race marking the bit. - if gcphase == _GCmarktermination || gcBlackenPromptly { + if gcphase != _GCoff { gcmarknewobject(uintptr(x), size, scanSize) } diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index d120dae05a..194439337b 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -24,6 +24,10 @@ // Hudson, R., and Moss, J.E.B. Copying Garbage Collection without stopping the world. // Concurrency and Computation: Practice and Experience 15(3-5), 2003. // +// TODO(austin): The rest of this comment is woefully out of date and +// needs to be rewritten. There is no distinct scan phase any more and +// we allocate black during GC. +// // 0. Set phase = GCscan from GCoff. // 1. Wait for all P's to acknowledge phase change. // At this point all goroutines have passed through a GC safepoint and @@ -244,7 +248,7 @@ var gcBlackenPromptly bool const ( _GCoff = iota // GC not running; sweeping in background, write barrier disabled - _GCmark // GC marking roots and workbufs, write barrier ENABLED + _GCmark // GC marking roots and workbufs: allocate black, write barrier ENABLED _GCmarktermination // GC mark termination: allocate black, P's help GC, write barrier ENABLED ) @@ -467,14 +471,18 @@ func (c *gcControllerState) startCycle() { // It should only be called when gcBlackenEnabled != 0 (because this // is when assists are enabled and the necessary statistics are // available). +// +// TODO: Consider removing the periodic controller update altogether. +// Since we switched to allocating black, in theory we shouldn't have +// to change the assist ratio. However, this is still a useful hook +// that we've found many uses for when experimenting. func (c *gcControllerState) revise() { // Compute the expected scan work remaining. // - // Note that the scannable heap size is likely to increase - // during the GC cycle. This is why it's important to revise - // the assist ratio throughout the cycle: if the scannable - // heap size increases, the assist ratio based on the initial - // scannable heap size may target too little scan work. + // Note that we currently count allocations during GC as both + // scannable heap (heap_scan) and scan work completed + // (scanWork), so this difference won't be changed by + // allocations during GC. // // This particular estimate is a strict upper bound on the // possible remaining scan work for the current heap. -- cgit v1.3