From 6b33b0e98e9be77d98b026ae2adf10dd71be5a1b Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Sun, 10 Apr 2016 09:08:00 -0700 Subject: cmd/compile: avoid a spill in append fast path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of spilling newlen, recalculate it. This removes a spill from the fast path, at the cost of a cheap recalculation on the (rare) growth path. This uses 8 bytes less of stack space. It generates two more bytes of code, but that is due to suboptimal register allocation; see far below. Runtime append microbenchmarks are all over the map, presumably due to incidental code movement. Sample code: func s(b []byte) []byte { b = append(b, 1, 2, 3) return b } Before: "".s t=1 size=160 args=0x30 locals=0x48 0x0000 00000 (append.go:8) TEXT "".s(SB), $72-48 0x0000 00000 (append.go:8) MOVQ (TLS), CX 0x0009 00009 (append.go:8) CMPQ SP, 16(CX) 0x000d 00013 (append.go:8) JLS 149 0x0013 00019 (append.go:8) SUBQ $72, SP 0x0017 00023 (append.go:8) FUNCDATA $0, gclocals·6432f8c6a0d23fa7bee6c5d96f21a92a(SB) 0x0017 00023 (append.go:8) FUNCDATA $1, gclocals·33cdeccccebe80329f1fdbee7f5874cb(SB) 0x0017 00023 (append.go:9) MOVQ "".b+88(FP), CX 0x001c 00028 (append.go:9) LEAQ 3(CX), DX 0x0020 00032 (append.go:9) MOVQ DX, "".autotmp_0+64(SP) 0x0025 00037 (append.go:9) MOVQ "".b+96(FP), BX 0x002a 00042 (append.go:9) CMPQ DX, BX 0x002d 00045 (append.go:9) JGT $0, 86 0x002f 00047 (append.go:8) MOVQ "".b+80(FP), AX 0x0034 00052 (append.go:9) MOVB $1, (AX)(CX*1) 0x0038 00056 (append.go:9) MOVB $2, 1(AX)(CX*1) 0x003d 00061 (append.go:9) MOVB $3, 2(AX)(CX*1) 0x0042 00066 (append.go:10) MOVQ AX, "".~r1+104(FP) 0x0047 00071 (append.go:10) MOVQ DX, "".~r1+112(FP) 0x004c 00076 (append.go:10) MOVQ BX, "".~r1+120(FP) 0x0051 00081 (append.go:10) ADDQ $72, SP 0x0055 00085 (append.go:10) RET 0x0056 00086 (append.go:9) LEAQ type.[]uint8(SB), AX 0x005d 00093 (append.go:9) MOVQ AX, (SP) 0x0061 00097 (append.go:9) MOVQ "".b+80(FP), BP 0x0066 00102 (append.go:9) MOVQ BP, 8(SP) 0x006b 00107 (append.go:9) MOVQ CX, 16(SP) 0x0070 00112 (append.go:9) MOVQ BX, 24(SP) 0x0075 00117 (append.go:9) MOVQ DX, 32(SP) 0x007a 00122 (append.go:9) PCDATA $0, $0 0x007a 00122 (append.go:9) CALL runtime.growslice(SB) 0x007f 00127 (append.go:9) MOVQ 40(SP), AX 0x0084 00132 (append.go:9) MOVQ 56(SP), BX 0x0089 00137 (append.go:8) MOVQ "".b+88(FP), CX 0x008e 00142 (append.go:9) MOVQ "".autotmp_0+64(SP), DX 0x0093 00147 (append.go:9) JMP 52 0x0095 00149 (append.go:9) NOP 0x0095 00149 (append.go:8) CALL runtime.morestack_noctxt(SB) 0x009a 00154 (append.go:8) JMP 0 After: "".s t=1 size=176 args=0x30 locals=0x40 0x0000 00000 (append.go:8) TEXT "".s(SB), $64-48 0x0000 00000 (append.go:8) MOVQ (TLS), CX 0x0009 00009 (append.go:8) CMPQ SP, 16(CX) 0x000d 00013 (append.go:8) JLS 151 0x0013 00019 (append.go:8) SUBQ $64, SP 0x0017 00023 (append.go:8) FUNCDATA $0, gclocals·6432f8c6a0d23fa7bee6c5d96f21a92a(SB) 0x0017 00023 (append.go:8) FUNCDATA $1, gclocals·33cdeccccebe80329f1fdbee7f5874cb(SB) 0x0017 00023 (append.go:9) MOVQ "".b+80(FP), CX 0x001c 00028 (append.go:9) LEAQ 3(CX), DX 0x0020 00032 (append.go:9) MOVQ "".b+88(FP), BX 0x0025 00037 (append.go:9) CMPQ DX, BX 0x0028 00040 (append.go:9) JGT $0, 81 0x002a 00042 (append.go:8) MOVQ "".b+72(FP), AX 0x002f 00047 (append.go:9) MOVB $1, (AX)(CX*1) 0x0033 00051 (append.go:9) MOVB $2, 1(AX)(CX*1) 0x0038 00056 (append.go:9) MOVB $3, 2(AX)(CX*1) 0x003d 00061 (append.go:10) MOVQ AX, "".~r1+96(FP) 0x0042 00066 (append.go:10) MOVQ DX, "".~r1+104(FP) 0x0047 00071 (append.go:10) MOVQ BX, "".~r1+112(FP) 0x004c 00076 (append.go:10) ADDQ $64, SP 0x0050 00080 (append.go:10) RET 0x0051 00081 (append.go:9) LEAQ type.[]uint8(SB), AX 0x0058 00088 (append.go:9) MOVQ AX, (SP) 0x005c 00092 (append.go:9) MOVQ "".b+72(FP), BP 0x0061 00097 (append.go:9) MOVQ BP, 8(SP) 0x0066 00102 (append.go:9) MOVQ CX, 16(SP) 0x006b 00107 (append.go:9) MOVQ BX, 24(SP) 0x0070 00112 (append.go:9) MOVQ DX, 32(SP) 0x0075 00117 (append.go:9) PCDATA $0, $0 0x0075 00117 (append.go:9) CALL runtime.growslice(SB) 0x007a 00122 (append.go:9) MOVQ 40(SP), AX 0x007f 00127 (append.go:9) MOVQ 48(SP), CX 0x0084 00132 (append.go:9) MOVQ 56(SP), BX 0x0089 00137 (append.go:9) ADDQ $3, CX 0x008d 00141 (append.go:9) MOVQ CX, DX 0x0090 00144 (append.go:8) MOVQ "".b+80(FP), CX 0x0095 00149 (append.go:9) JMP 47 0x0097 00151 (append.go:9) NOP 0x0097 00151 (append.go:8) CALL runtime.morestack_noctxt(SB) 0x009c 00156 (append.go:8) JMP 0 Observe that in the following sequence, we should use DX directly instead of using CX as a temporary register, which would make the new code a strict improvement on the old: 0x007f 00127 (append.go:9) MOVQ 48(SP), CX 0x0084 00132 (append.go:9) MOVQ 56(SP), BX 0x0089 00137 (append.go:9) ADDQ $3, CX 0x008d 00141 (append.go:9) MOVQ CX, DX 0x0090 00144 (append.go:8) MOVQ "".b+80(FP), CX Change-Id: I4ee50b18fa53865901d2d7f86c2cbb54c6fa6924 Reviewed-on: https://go-review.googlesource.com/21812 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/slice.go | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/runtime/slice.go') diff --git a/src/runtime/slice.go b/src/runtime/slice.go index 0bc0299f72..4ab221056c 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -37,6 +37,12 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { // It is passed the slice type, the old slice, and the desired new minimum capacity, // and it returns a new slice with at least that capacity, with the old data // copied into it. +// The new slice's length is set to the old slice's length, +// NOT to the new requested capacity. +// This is for codegen convenience. The old slice's length is used immediately +// to calculate where to write new values during an append. +// TODO: When the old backend is gone, reconsider this decision. +// The SSA backend might prefer the new length or to return only ptr/cap and save stack space. func growslice(t *slicetype, old slice, cap int) slice { if raceenabled { callerpc := getcallerpc(unsafe.Pointer(&t)) -- cgit v1.3 From ad7448fe982d83de15deec9c55c56d0cd9261c6c Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Sun, 10 Apr 2016 17:32:35 +0200 Subject: runtime: speed up makeslice by avoiding divisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only compute the number of maximum allowed elements per slice once. name old time/op new time/op delta MakeSlice-2 55.5ns ± 1% 45.6ns ± 2% -17.88% (p=0.000 n=99+100) Change-Id: I951feffda5d11910a75e55d7e978d306d14da2c5 Reviewed-on: https://go-review.googlesource.com/21801 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/append_test.go | 8 ++++++++ src/runtime/slice.go | 14 +++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'src/runtime/slice.go') diff --git a/src/runtime/append_test.go b/src/runtime/append_test.go index 3170870b0e..6d7836a351 100644 --- a/src/runtime/append_test.go +++ b/src/runtime/append_test.go @@ -7,6 +7,14 @@ import "testing" const N = 20 +func BenchmarkMakeSlice(b *testing.B) { + var x []byte + for i := 0; i < b.N; i++ { + x = make([]byte, 32) + _ = x + } +} + func BenchmarkGrowSliceBytes(b *testing.B) { b.StopTimer() var x = make([]byte, 9) diff --git a/src/runtime/slice.go b/src/runtime/slice.go index 4ab221056c..f36ec0b466 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -16,19 +16,27 @@ type slice struct { // TODO: take uintptrs instead of int64s? func makeslice(t *slicetype, len64, cap64 int64) slice { - // NOTE: The len > MaxMem/elemsize check here is not strictly necessary, + // NOTE: The len > maxElements check here is not strictly necessary, // but it produces a 'len out of range' error instead of a 'cap out of range' error // when someone does make([]T, bignumber). 'cap out of range' is true too, // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. + + maxElements := ^uintptr(0) + if t.elem.size > 0 { + maxElements = _MaxMem / t.elem.size + } + len := int(len64) - if len64 < 0 || int64(len) != len64 || t.elem.size > 0 && uintptr(len) > _MaxMem/t.elem.size { + if len64 < 0 || int64(len) != len64 || uintptr(len) > maxElements { panic(errorString("makeslice: len out of range")) } + cap := int(cap64) - if cap < len || int64(cap) != cap64 || t.elem.size > 0 && uintptr(cap) > _MaxMem/t.elem.size { + if cap < len || int64(cap) != cap64 || uintptr(cap) > maxElements { panic(errorString("makeslice: cap out of range")) } + p := newarray(t.elem, uintptr(cap)) return slice{p, len, cap} } -- cgit v1.3 From a4dd6ea1524901fab5deac60772345babd058ae7 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Tue, 19 Apr 2016 09:18:59 -0700 Subject: runtime: add maxSliceCap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids expensive division calculations for many common slice element sizes. name old time/op new time/op delta MakeSlice-8 51.9ns ± 3% 35.1ns ± 2% -32.41% (p=0.000 n=10+10) GrowSliceBytes-8 44.1ns ± 2% 44.1ns ± 1% ~ (p=0.984 n=10+10) GrowSliceInts-8 60.9ns ± 3% 60.9ns ± 3% ~ (p=0.698 n=10+10) GrowSlicePtr-8 131ns ± 1% 120ns ± 2% -8.41% (p=0.000 n=8+10) GrowSliceStruct24Bytes-8 111ns ± 2% 103ns ± 3% -7.23% (p=0.000 n=8+8) Change-Id: I2630eb3d73c814db030cad16e620ea7fecbbd312 Reviewed-on: https://go-review.googlesource.com/22223 Reviewed-by: Keith Randall --- src/runtime/malloc.go | 2 +- src/runtime/slice.go | 35 +++++++++++++++++++++++++---------- 2 files changed, 26 insertions(+), 11 deletions(-) (limited to 'src/runtime/slice.go') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index ee4728c9a5..a3e55ec2fb 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -792,7 +792,7 @@ func newarray(typ *_type, n uintptr) unsafe.Pointer { if typ.kind&kindNoPointers != 0 { flags |= flagNoScan } - if int(n) < 0 || (typ.size > 0 && n > _MaxMem/typ.size) { + if int(n) < 0 || n > maxSliceCap(typ.size) { panic(plainError("runtime: allocation size out of range")) } return mallocgc(typ.size*n, typ, flags) diff --git a/src/runtime/slice.go b/src/runtime/slice.go index f36ec0b466..f9414d7658 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -14,6 +14,28 @@ type slice struct { cap int } +// maxElems is a lookup table containing the maximum capacity for a slice. +// The index is the size of the slice element. +var maxElems = [...]uintptr{ + ^uintptr(0), + _MaxMem / 1, _MaxMem / 2, _MaxMem / 3, _MaxMem / 4, + _MaxMem / 5, _MaxMem / 6, _MaxMem / 7, _MaxMem / 8, + _MaxMem / 9, _MaxMem / 10, _MaxMem / 11, _MaxMem / 12, + _MaxMem / 13, _MaxMem / 14, _MaxMem / 15, _MaxMem / 16, + _MaxMem / 17, _MaxMem / 18, _MaxMem / 19, _MaxMem / 20, + _MaxMem / 21, _MaxMem / 22, _MaxMem / 23, _MaxMem / 24, + _MaxMem / 25, _MaxMem / 26, _MaxMem / 27, _MaxMem / 28, + _MaxMem / 29, _MaxMem / 30, _MaxMem / 31, _MaxMem / 32, +} + +// maxSliceCap returns the maximum capacity for a slice. +func maxSliceCap(elemsize uintptr) uintptr { + if elemsize < uintptr(len(maxElems)) { + return maxElems[elemsize] + } + return _MaxMem / elemsize +} + // TODO: take uintptrs instead of int64s? func makeslice(t *slicetype, len64, cap64 int64) slice { // NOTE: The len > maxElements check here is not strictly necessary, @@ -22,11 +44,7 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. - maxElements := ^uintptr(0) - if t.elem.size > 0 { - maxElements = _MaxMem / t.elem.size - } - + maxElements := maxSliceCap(t.elem.size) len := int(len64) if len64 < 0 || int64(len) != len64 || uintptr(len) > maxElements { panic(errorString("makeslice: len out of range")) @@ -84,27 +102,24 @@ func growslice(t *slicetype, old slice, cap int) slice { } } - var lenmem, capmem, maxcap uintptr + var lenmem, capmem uintptr const ptrSize = unsafe.Sizeof((*byte)(nil)) switch et.size { case 1: lenmem = uintptr(old.len) capmem = roundupsize(uintptr(newcap)) newcap = int(capmem) - maxcap = _MaxMem case ptrSize: lenmem = uintptr(old.len) * ptrSize capmem = roundupsize(uintptr(newcap) * ptrSize) newcap = int(capmem / ptrSize) - maxcap = _MaxMem / ptrSize default: lenmem = uintptr(old.len) * et.size capmem = roundupsize(uintptr(newcap) * et.size) newcap = int(capmem / et.size) - maxcap = _MaxMem / et.size } - if cap < old.cap || uintptr(newcap) > maxcap { + if cap < old.cap || uintptr(newcap) > maxSliceCap(et.size) { panic(errorString("growslice: cap out of range")) } -- cgit v1.3 From 0150f15a924a7b4ac0c794012f6b12c8aa406b54 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Tue, 19 Apr 2016 15:14:26 -0700 Subject: runtime: call mallocgc directly from makeslice and growslice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extra checks provided by newarray are redundant in these cases. This shrinks by one frame the call stack expected by the pprof test. name old time/op new time/op delta MakeSlice-8 34.3ns ± 2% 30.5ns ± 3% -11.03% (p=0.000 n=24+22) GrowSlicePtr-8 134ns ± 2% 129ns ± 3% -3.25% (p=0.000 n=25+24) Change-Id: Icd828655906b921c732701fd9d61da3fa217b0af Reviewed-on: https://go-review.googlesource.com/22276 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/pprof/mprof_test.go | 2 +- src/runtime/slice.go | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'src/runtime/slice.go') diff --git a/src/runtime/pprof/mprof_test.go b/src/runtime/pprof/mprof_test.go index d15102c703..0fff9d46d9 100644 --- a/src/runtime/pprof/mprof_test.go +++ b/src/runtime/pprof/mprof_test.go @@ -82,7 +82,7 @@ func TestMemoryProfiler(t *testing.T) { # 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:61 `, (1<<10)*memoryProfilerRun, (1<<20)*memoryProfilerRun), - fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ + fmt.Sprintf(`0: 0 \[%v: %v\] @ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ 0x[0-9,a-f]+ # 0x[0-9,a-f]+ runtime/pprof_test\.allocateTransient2M\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:27 # 0x[0-9,a-f]+ runtime/pprof_test\.TestMemoryProfiler\+0x[0-9,a-f]+ .*/runtime/pprof/mprof_test.go:62 `, memoryProfilerRun, (2<<20)*memoryProfilerRun), diff --git a/src/runtime/slice.go b/src/runtime/slice.go index f9414d7658..873e97ebff 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -55,7 +55,12 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { panic(errorString("makeslice: cap out of range")) } - p := newarray(t.elem, uintptr(cap)) + et := t.elem + var flags uint32 + if et.kind&kindNoPointers != 0 { + flags = flagNoScan + } + p := mallocgc(et.size*uintptr(cap), et, flags) return slice{p, len, cap} } @@ -130,7 +135,7 @@ func growslice(t *slicetype, old slice, cap int) slice { memclr(add(p, lenmem), capmem-lenmem) } else { // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory. - p = newarray(et, uintptr(newcap)) + p = mallocgc(capmem, et, 0) if !writeBarrier.enabled { memmove(p, old.array, lenmem) } else { -- cgit v1.3 From bfe0cbdc50cbc6a632d1e5ebbdcc625d69451935 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 19 Apr 2016 15:38:59 -0700 Subject: cmd/compile,runtime: pass elem type to {make,grow}slice No point in passing the slice type to these functions. All they need is the element type. One less indirection, maybe a few less []T type descriptors in the binary. Change-Id: Ib0b83b5f14ca21d995ecc199ce8ac00c4eb375e6 Reviewed-on: https://go-review.googlesource.com/22275 Reviewed-by: Josh Bleecher Snyder --- src/cmd/compile/internal/gc/cgen.go | 2 +- src/cmd/compile/internal/gc/ssa.go | 2 +- src/cmd/compile/internal/gc/walk.go | 8 ++++---- src/runtime/slice.go | 16 +++++++--------- 4 files changed, 13 insertions(+), 15 deletions(-) (limited to 'src/runtime/slice.go') diff --git a/src/cmd/compile/internal/gc/cgen.go b/src/cmd/compile/internal/gc/cgen.go index 658cc8a50e..5c5bedaa31 100644 --- a/src/cmd/compile/internal/gc/cgen.go +++ b/src/cmd/compile/internal/gc/cgen.go @@ -2876,7 +2876,7 @@ func cgen_append(n, res *Node) { arg.Addable = true arg.Xoffset = Ctxt.FixedFrameSize() arg.Type = Ptrto(Types[TUINT8]) - Cgen(typename(res.Type), &arg) + Cgen(typename(res.Type.Elem()), &arg) arg.Xoffset += int64(Widthptr) arg.Type = Types[Tptr] diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index c4008c9ce1..11e362c116 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -2185,7 +2185,7 @@ func (s *state) append(n *Node, inplace bool) *ssa.Value { // Call growslice s.startBlock(grow) - taddr := s.newValue1A(ssa.OpAddr, Types[TUINTPTR], &ssa.ExternSymbol{Types[TUINTPTR], typenamesym(n.Type)}, s.sb) + taddr := s.newValue1A(ssa.OpAddr, Types[TUINTPTR], &ssa.ExternSymbol{Types[TUINTPTR], typenamesym(n.Type.Elem())}, s.sb) r := s.rtcall(growslice, true, []*Type{pt, Types[TINT], Types[TINT]}, taddr, p, l, c, nl) diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index e4d93339a9..82ac74ae33 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -1420,11 +1420,11 @@ opswitch: r = walkexpr(r, init) n = r } else { - // makeslice(t *Type, nel int64, max int64) (ary []any) + // makeslice(et *Type, nel int64, max int64) (ary []any) fn := syslook("makeslice") fn = substArgTypes(fn, t.Elem()) // any-1 - n = mkcall1(fn, n.Type, init, typename(n.Type), conv(l, Types[TINT64]), conv(r, Types[TINT64])) + n = mkcall1(fn, n.Type, init, typename(t.Elem()), conv(l, Types[TINT64]), conv(r, Types[TINT64])) } case ORUNESTR: @@ -2799,7 +2799,7 @@ func appendslice(n *Node, init *Nodes) *Node { fn = substArgTypes(fn, s.Type.Elem(), s.Type.Elem()) // s = growslice(T, s, n) - nif.Nbody.Set1(Nod(OAS, s, mkcall1(fn, s.Type, &nif.Ninit, typename(s.Type), s, nn))) + nif.Nbody.Set1(Nod(OAS, s, mkcall1(fn, s.Type, &nif.Ninit, typename(s.Type.Elem()), s, nn))) l = append(l, nif) // s = s[:n] @@ -2929,7 +2929,7 @@ func walkappend(n *Node, init *Nodes, dst *Node) *Node { fn = substArgTypes(fn, ns.Type.Elem(), ns.Type.Elem()) nx.Nbody.Set1(Nod(OAS, ns, - mkcall1(fn, ns.Type, &nx.Ninit, typename(ns.Type), ns, + mkcall1(fn, ns.Type, &nx.Ninit, typename(ns.Type.Elem()), ns, Nod(OADD, Nod(OLEN, ns, nil), na)))) l = append(l, nx) diff --git a/src/runtime/slice.go b/src/runtime/slice.go index 873e97ebff..e86c1ce2c8 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -37,14 +37,14 @@ func maxSliceCap(elemsize uintptr) uintptr { } // TODO: take uintptrs instead of int64s? -func makeslice(t *slicetype, len64, cap64 int64) slice { +func makeslice(et *_type, len64, cap64 int64) slice { // NOTE: The len > maxElements check here is not strictly necessary, // but it produces a 'len out of range' error instead of a 'cap out of range' error // when someone does make([]T, bignumber). 'cap out of range' is true too, // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. - maxElements := maxSliceCap(t.elem.size) + maxElements := maxSliceCap(et.size) len := int(len64) if len64 < 0 || int64(len) != len64 || uintptr(len) > maxElements { panic(errorString("makeslice: len out of range")) @@ -55,7 +55,6 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { panic(errorString("makeslice: cap out of range")) } - et := t.elem var flags uint32 if et.kind&kindNoPointers != 0 { flags = flagNoScan @@ -65,7 +64,7 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { } // growslice handles slice growth during append. -// It is passed the slice type, the old slice, and the desired new minimum capacity, +// It is passed the slice element type, the old slice, and the desired new minimum capacity, // and it returns a new slice with at least that capacity, with the old data // copied into it. // The new slice's length is set to the old slice's length, @@ -74,16 +73,15 @@ func makeslice(t *slicetype, len64, cap64 int64) slice { // to calculate where to write new values during an append. // TODO: When the old backend is gone, reconsider this decision. // The SSA backend might prefer the new length or to return only ptr/cap and save stack space. -func growslice(t *slicetype, old slice, cap int) slice { +func growslice(et *_type, old slice, cap int) slice { if raceenabled { - callerpc := getcallerpc(unsafe.Pointer(&t)) - racereadrangepc(old.array, uintptr(old.len*int(t.elem.size)), callerpc, funcPC(growslice)) + callerpc := getcallerpc(unsafe.Pointer(&et)) + racereadrangepc(old.array, uintptr(old.len*int(et.size)), callerpc, funcPC(growslice)) } if msanenabled { - msanread(old.array, uintptr(old.len*int(t.elem.size))) + msanread(old.array, uintptr(old.len*int(et.size))) } - et := t.elem if et.size == 0 { if cap < old.cap { panic(errorString("growslice: cap out of range")) -- cgit v1.3 From 001e8e8070e8ed3a578dbad93cc3f70257e965bd Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 19 Apr 2016 19:35:10 -0700 Subject: runtime: simplify mallocgc flag argument mallocgc can calculate noscan itself. The only remaining flag argument is needzero, so we just make that a boolean arg. Fixes #15379 Change-Id: I839a70790b2a0c9dbcee2600052bfbd6c8148e20 Reviewed-on: https://go-review.googlesource.com/22290 Reviewed-by: Josh Bleecher Snyder Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot --- src/runtime/chan.go | 2 +- src/runtime/malloc.go | 48 +++++++++++------------------------------------- src/runtime/mfinal.go | 2 +- src/runtime/os1_plan9.go | 4 ++-- src/runtime/panic.go | 2 +- src/runtime/select.go | 2 +- src/runtime/slice.go | 11 +++-------- src/runtime/string.go | 6 +++--- 8 files changed, 23 insertions(+), 54 deletions(-) (limited to 'src/runtime/slice.go') diff --git a/src/runtime/chan.go b/src/runtime/chan.go index 8543cb4c9c..3fb0236785 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -74,7 +74,7 @@ func makechan(t *chantype, size int64) *hchan { // buf points into the same allocation, elemtype is persistent. // SudoG's are referenced from their owning thread so they can't be collected. // TODO(dvyukov,rlh): Rethink when collector can move allocated objects. - c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, flagNoScan)) + c = (*hchan)(mallocgc(hchanSize+uintptr(size)*elem.size, nil, true)) if size > 0 && elem.size != 0 { c.buf = add(unsafe.Pointer(c), hchanSize) } else { diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index a3e55ec2fb..95d24a467a 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -87,9 +87,6 @@ import ( const ( debugMalloc = false - flagNoScan = _FlagNoScan - flagNoZero = _FlagNoZero - maxTinySize = _TinySize tinySizeClass = _TinySizeClass maxSmallSize = _MaxSmallSize @@ -487,16 +484,10 @@ func (h *mheap) sysAlloc(n uintptr) unsafe.Pointer { // base address for all 0-byte allocations var zerobase uintptr -const ( - // flags to malloc - _FlagNoScan = 1 << 0 // GC doesn't have to scan object - _FlagNoZero = 1 << 1 // don't zero memory -) - // Allocate an object of size bytes. // Small objects are allocated from the per-P cache's free lists. // Large objects (> 32 kB) are allocated straight from the heap. -func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { +func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") } @@ -505,10 +496,6 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { return unsafe.Pointer(&zerobase) } - if flags&flagNoScan == 0 && typ == nil { - throw("malloc missing type") - } - if debug.sbrk != 0 { align := uintptr(16) if typ != nil { @@ -553,14 +540,15 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { c := gomcache() var s *mspan var x unsafe.Pointer + noscan := typ == nil || typ.kind&kindNoPointers != 0 if size <= maxSmallSize { - if flags&flagNoScan != 0 && size < maxTinySize { + if noscan && size < maxTinySize { // Tiny allocator. // // Tiny allocator combines several tiny allocation requests // into a single memory block. The resulting memory block // is freed when all subobjects are unreachable. The subobjects - // must be FlagNoScan (don't have pointers), this ensures that + // must be noscan (don't have pointers), this ensures that // the amount of potentially wasted memory is bounded. // // Size of the memory block used for combining (maxTinySize) is tunable. @@ -650,7 +638,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { // prefetchnta offers best performance, see change list message. prefetchnta(uintptr(v.ptr().next)) x = unsafe.Pointer(v) - if flags&flagNoZero == 0 { + if needzero { v.ptr().next = 0 if size > 2*sys.PtrSize && ((*[2]uintptr)(x))[1] != 0 { memclr(unsafe.Pointer(v), size) @@ -661,13 +649,13 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { var s *mspan shouldhelpgc = true systemstack(func() { - s = largeAlloc(size, flags) + s = largeAlloc(size, needzero) }) x = unsafe.Pointer(uintptr(s.start << pageShift)) size = s.elemsize } - if flags&flagNoScan != 0 { + if noscan { // All objects are pre-marked as noscan. Nothing to do. } else { // If allocating a defer+arg block, now that we've picked a malloc size @@ -747,7 +735,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer { return x } -func largeAlloc(size uintptr, flag uint32) *mspan { +func largeAlloc(size uintptr, needzero bool) *mspan { // print("largeAlloc size=", size, "\n") if size+_PageSize < size { @@ -763,7 +751,7 @@ func largeAlloc(size uintptr, flag uint32) *mspan { // pays the debt down to npage pages. deductSweepCredit(npages*_PageSize, npages) - s := mheap_.alloc(npages, 0, true, flag&_FlagNoZero == 0) + s := mheap_.alloc(npages, 0, true, needzero) if s == nil { throw("out of memory") } @@ -774,11 +762,7 @@ func largeAlloc(size uintptr, flag uint32) *mspan { // implementation of new builtin func newobject(typ *_type) unsafe.Pointer { - flags := uint32(0) - if typ.kind&kindNoPointers != 0 { - flags |= flagNoScan - } - return mallocgc(typ.size, typ, flags) + return mallocgc(typ.size, typ, true) } //go:linkname reflect_unsafe_New reflect.unsafe_New @@ -788,14 +772,10 @@ func reflect_unsafe_New(typ *_type) unsafe.Pointer { // implementation of make builtin for slices func newarray(typ *_type, n uintptr) unsafe.Pointer { - flags := uint32(0) - if typ.kind&kindNoPointers != 0 { - flags |= flagNoScan - } if int(n) < 0 || n > maxSliceCap(typ.size) { panic(plainError("runtime: allocation size out of range")) } - return mallocgc(typ.size*n, typ, flags) + return mallocgc(typ.size*n, typ, true) } //go:linkname reflect_unsafe_NewArray reflect.unsafe_NewArray @@ -803,12 +783,6 @@ func reflect_unsafe_NewArray(typ *_type, n uintptr) unsafe.Pointer { return newarray(typ, n) } -// rawmem returns a chunk of pointerless memory. It is -// not zeroed. -func rawmem(size uintptr) unsafe.Pointer { - return mallocgc(size, nil, flagNoScan|flagNoZero) -} - func profilealloc(mp *m, x unsafe.Pointer, size uintptr) { mp.mcache.next_sample = nextSample() mProf_Malloc(x, size) diff --git a/src/runtime/mfinal.go b/src/runtime/mfinal.go index b862f019b6..f698e72709 100644 --- a/src/runtime/mfinal.go +++ b/src/runtime/mfinal.go @@ -172,7 +172,7 @@ func runfinq() { // all not yet finalized objects are stored in finq. // If we do not mark it as FlagNoScan, // the last finalized object is not collected. - frame = mallocgc(framesz, nil, flagNoScan) + frame = mallocgc(framesz, nil, true) framecap = framesz } diff --git a/src/runtime/os1_plan9.go b/src/runtime/os1_plan9.go index 2c257442ba..eb7a0c6481 100644 --- a/src/runtime/os1_plan9.go +++ b/src/runtime/os1_plan9.go @@ -17,10 +17,10 @@ func mpreinit(mp *m) { // Initialize stack and goroutine for note handling. mp.gsignal = malg(32 * 1024) mp.gsignal.m = mp - mp.notesig = (*int8)(mallocgc(_ERRMAX, nil, _FlagNoScan)) + mp.notesig = (*int8)(mallocgc(_ERRMAX, nil, true)) // Initialize stack for handling strings from the // errstr system call, as used in package syscall. - mp.errstr = (*byte)(mallocgc(_ERRMAX, nil, _FlagNoScan)) + mp.errstr = (*byte)(mallocgc(_ERRMAX, nil, true)) } func msigsave(mp *m) { diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 59fbc83369..382a20e4e7 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -205,7 +205,7 @@ func newdefer(siz int32) *_defer { if d == nil { // Allocate new defer+args. total := roundupsize(totaldefersize(uintptr(siz))) - d = (*_defer)(mallocgc(total, deferType, 0)) + d = (*_defer)(mallocgc(total, deferType, true)) } d.siz = siz gp := mp.curg diff --git a/src/runtime/select.go b/src/runtime/select.go index 9810db5453..433048fb79 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -626,7 +626,7 @@ const ( func reflect_rselect(cases []runtimeSelect) (chosen int, recvOK bool) { // flagNoScan is safe here, because all objects are also referenced from cases. size := selectsize(uintptr(len(cases))) - sel := (*hselect)(mallocgc(size, nil, flagNoScan)) + sel := (*hselect)(mallocgc(size, nil, true)) newselect(sel, int64(size), int32(len(cases))) r := new(bool) for i := range cases { diff --git a/src/runtime/slice.go b/src/runtime/slice.go index e86c1ce2c8..e15e6c4dc6 100644 --- a/src/runtime/slice.go +++ b/src/runtime/slice.go @@ -43,7 +43,6 @@ func makeslice(et *_type, len64, cap64 int64) slice { // when someone does make([]T, bignumber). 'cap out of range' is true too, // but since the cap is only being supplied implicitly, saying len is clearer. // See issue 4085. - maxElements := maxSliceCap(et.size) len := int(len64) if len64 < 0 || int64(len) != len64 || uintptr(len) > maxElements { @@ -55,11 +54,7 @@ func makeslice(et *_type, len64, cap64 int64) slice { panic(errorString("makeslice: cap out of range")) } - var flags uint32 - if et.kind&kindNoPointers != 0 { - flags = flagNoScan - } - p := mallocgc(et.size*uintptr(cap), et, flags) + p := mallocgc(et.size*uintptr(cap), et, true) return slice{p, len, cap} } @@ -128,12 +123,12 @@ func growslice(et *_type, old slice, cap int) slice { var p unsafe.Pointer if et.kind&kindNoPointers != 0 { - p = rawmem(capmem) + p = mallocgc(capmem, nil, false) memmove(p, old.array, lenmem) memclr(add(p, lenmem), capmem-lenmem) } else { // Note: can't use rawmem (which avoids zeroing of memory), because then GC can scan uninitialized memory. - p = mallocgc(capmem, et, 0) + p = mallocgc(capmem, et, true) if !writeBarrier.enabled { memmove(p, old.array, lenmem) } else { diff --git a/src/runtime/string.go b/src/runtime/string.go index 2d20e0a9c3..112ce5d588 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -284,7 +284,7 @@ func stringiter2(s string, k int) (int, rune) { // The storage is not zeroed. Callers should use // b to set the string contents and then drop b. func rawstring(size int) (s string, b []byte) { - p := mallocgc(uintptr(size), nil, flagNoScan|flagNoZero) + p := mallocgc(uintptr(size), nil, false) stringStructOf(&s).str = p stringStructOf(&s).len = size @@ -302,7 +302,7 @@ func rawstring(size int) (s string, b []byte) { // rawbyteslice allocates a new byte slice. The byte slice is not zeroed. func rawbyteslice(size int) (b []byte) { cap := roundupsize(uintptr(size)) - p := mallocgc(cap, nil, flagNoScan|flagNoZero) + p := mallocgc(cap, nil, false) if cap != uintptr(size) { memclr(add(p, uintptr(size)), cap-uintptr(size)) } @@ -317,7 +317,7 @@ func rawruneslice(size int) (b []rune) { throw("out of memory") } mem := roundupsize(uintptr(size) * 4) - p := mallocgc(mem, nil, flagNoScan|flagNoZero) + p := mallocgc(mem, nil, false) if mem != uintptr(size)*4 { memclr(add(p, uintptr(size)*4), mem-uintptr(size)*4) } -- cgit v1.3