From e3869a6b65bb0f95dac7eca3d86055160b12589f Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 7 Sep 2015 23:18:02 -0700 Subject: [dev.ssa] cmd/compile/internal/ssa: implement write barriers For now, we only use typedmemmove. This can be optimized in future CLs. Also add a feature to help with binary searching bad compilations. Together with GOSSAPKG, GOSSAHASH specifies the last few binary digits of the hash of function names that should be compiled. So GOSSAHASH=0110 means compile only those functions whose last 4 bits of hash are 0110. By adding digits to the front we can binary search for the function whose SSA-generated code is causing a test to fail. Change-Id: I5a8b6b70c6f034f59e5753965234cd42ea36d524 Reviewed-on: https://go-review.googlesource.com/14530 Reviewed-by: Keith Randall --- src/runtime/mbarrier.go | 8 ++++++++ src/runtime/stack2.go | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index 0dbe1ffc9d..c94e44f142 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -185,6 +185,14 @@ func typedmemmove(typ *_type, dst, src unsafe.Pointer) { heapBitsBulkBarrier(uintptr(dst), typ.size) } +//go:nosplit +func typedmemmove_nostore(typ *_type, dst unsafe.Pointer) { + if typ.kind&kindNoPointers != 0 { + return + } + heapBitsBulkBarrier(uintptr(dst), typ.size) +} + //go:linkname reflect_typedmemmove reflect.typedmemmove func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { typedmemmove(typ, dst, src) diff --git a/src/runtime/stack2.go b/src/runtime/stack2.go index 5ec8d8d060..02b82ebe13 100644 --- a/src/runtime/stack2.go +++ b/src/runtime/stack2.go @@ -54,6 +54,8 @@ The linkers explore all possible call traces involving non-splitting functions to make sure that this limit cannot be violated. */ +// Constants here match those in cmd/internal/obj/stack.go. + const ( // StackSystem is a number of additional bytes to add // to each stack below the usual guard area for OS-specific @@ -84,7 +86,7 @@ const ( // The stack guard is a pointer this many bytes above the // bottom of the stack. - _StackGuard = 640*stackGuardMultiplier + _StackSystem + _StackGuard = 960*stackGuardMultiplier + _StackSystem // After a stack split check the SP is allowed to be this // many bytes below the stack guard. This saves an instruction -- cgit v1.3 From d29e92be523efd8270c0e7ca0eaa6afa86bbedca Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 19 Sep 2015 12:01:39 -0700 Subject: [dev.ssa] cmd/compile: Use varkill only for non-SSAable vars For variables which get SSA'd, SSA keeps track of all the def/kill. It is only for on-stack variables that we need them. This reduces stack frame sizes significantly because often the only use of a variable was a varkill, and without that last use the variable doesn't get allocated in the frame at all. Fixes #12602 Change-Id: I3f00a768aa5ddd8d7772f375b25f846086a3e689 Reviewed-on: https://go-review.googlesource.com/14758 Reviewed-by: Brad Fitzpatrick --- src/cmd/compile/internal/gc/ssa.go | 4 +++- src/cmd/internal/obj/stack.go | 2 +- src/runtime/stack2.go | 2 +- test/nosplit.go | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 8e0f0dcc9b..6cb5c571c2 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -794,7 +794,9 @@ func (s *state) stmt(n *Node) { // We only care about liveness info at call sites, so putting the // varkill in the store chain is enough to keep it correctly ordered // with respect to call ops. - s.vars[&memVar] = s.newValue1A(ssa.OpVarKill, ssa.TypeMem, n.Left, s.mem()) + if !canSSA(n.Left) { + s.vars[&memVar] = s.newValue1A(ssa.OpVarKill, ssa.TypeMem, n.Left, s.mem()) + } case OCHECKNIL: p := s.expr(n.Left) diff --git a/src/cmd/internal/obj/stack.go b/src/cmd/internal/obj/stack.go index b1630b55fc..87698b3eeb 100644 --- a/src/cmd/internal/obj/stack.go +++ b/src/cmd/internal/obj/stack.go @@ -41,7 +41,7 @@ const ( STACKSYSTEM = 0 StackSystem = STACKSYSTEM StackBig = 4096 - StackGuard = 960*stackGuardMultiplier + StackSystem + StackGuard = 640*stackGuardMultiplier + StackSystem StackSmall = 128 StackLimit = StackGuard - StackSystem - StackSmall ) diff --git a/src/runtime/stack2.go b/src/runtime/stack2.go index 02b82ebe13..59d4ef694d 100644 --- a/src/runtime/stack2.go +++ b/src/runtime/stack2.go @@ -86,7 +86,7 @@ const ( // The stack guard is a pointer this many bytes above the // bottom of the stack. - _StackGuard = 960*stackGuardMultiplier + _StackSystem + _StackGuard = 640*stackGuardMultiplier + _StackSystem // After a stack split check the SP is allowed to be this // many bytes below the stack guard. This saves an instruction diff --git a/test/nosplit.go b/test/nosplit.go index e7c00f5783..e5c2a9f30e 100644 --- a/test/nosplit.go +++ b/test/nosplit.go @@ -285,12 +285,12 @@ TestCases: // Instead of rewriting the test cases above, adjust // the first stack frame to use up the extra bytes. if i == 0 { - size += 832 - 128 + size += 512 - 128 // Noopt builds have a larger stackguard. // See ../cmd/dist/buildruntime.go:stackGuardMultiplier for _, s := range strings.Split(os.Getenv("GO_GCFLAGS"), " ") { if s == "-N" { - size += 960 + size += 640 } } } -- cgit v1.3 From 57670ad8b29fb62dc87e970fde95e3263f6948ff Mon Sep 17 00:00:00 2001 From: David Chase Date: Fri, 9 Oct 2015 16:48:30 -0400 Subject: [dev.ssa] cmd/compile: fill remaining SSA gaps Changed racewalk/race detector to use FP in a more sensible way. Relaxed checks for CONVNOP when race detecting. Modified tighten to ensure that GetClosurePtr cannot float out of entry block (turns out this cannot be relaxed, DX is sometimes stomped by other code accompanying race detection). Added case for addr(CONVNOP) Modified addr to take "bounded" flag to suppress nilchecks where it is set (usually, by race detector). Cannot leave unimplemented-complainer enabled because it turns out we are optimistically running SSA on every platform. Change-Id: Ife021654ee4065b3ffac62326d09b4b317b9f2e0 Reviewed-on: https://go-review.googlesource.com/15710 Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/builtin.go | 1 + src/cmd/compile/internal/gc/builtin/runtime.go | 1 + src/cmd/compile/internal/gc/racewalk.go | 14 ++------ src/cmd/compile/internal/gc/ssa.go | 47 ++++++++++++++++---------- src/cmd/compile/internal/ssa/schedule.go | 2 +- src/cmd/compile/internal/ssa/tighten.go | 3 +- src/runtime/race_amd64.s | 16 ++++++++- 7 files changed, 53 insertions(+), 31 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index 0e5fe2ab60..66f66a7690 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -151,6 +151,7 @@ const runtimeimport = "" + "func @\"\".uint64tofloat64 (? uint64) (? float64)\n" + "func @\"\".complex128div (@\"\".num·2 complex128, @\"\".den·3 complex128) (@\"\".quo·1 complex128)\n" + "func @\"\".racefuncenter (? uintptr)\n" + + "func @\"\".racefuncenterfp (? *int32)\n" + "func @\"\".racefuncexit ()\n" + "func @\"\".raceread (? uintptr)\n" + "func @\"\".racewrite (? uintptr)\n" + diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index f8487de45b..43c35ca850 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -189,6 +189,7 @@ func complex128div(num complex128, den complex128) (quo complex128) // race detection func racefuncenter(uintptr) +func racefuncenterfp(*int32) func racefuncexit() func raceread(uintptr) func racewrite(uintptr) diff --git a/src/cmd/compile/internal/gc/racewalk.go b/src/cmd/compile/internal/gc/racewalk.go index 9301d87d2e..852ae98ec1 100644 --- a/src/cmd/compile/internal/gc/racewalk.go +++ b/src/cmd/compile/internal/gc/racewalk.go @@ -11,7 +11,7 @@ import ( // The racewalk pass modifies the code tree for the function as follows: // -// 1. It inserts a call to racefuncenter at the beginning of each function. +// 1. It inserts a call to racefuncenterfp at the beginning of each function. // 2. It inserts a call to racefuncexit at the end of each function. // 3. It inserts a call to raceread before each memory read. // 4. It inserts a call to racewrite before each memory write. @@ -26,7 +26,7 @@ import ( // at best instrumentation would cause infinite recursion. var omit_pkgs = []string{"runtime", "runtime/race"} -// Only insert racefuncenter/racefuncexit into the following packages. +// Only insert racefuncenterfp/racefuncexit into the following packages. // Memory accesses in the packages are either uninteresting or will cause false positives. var noinst_pkgs = []string{"sync", "sync/atomic"} @@ -64,15 +64,7 @@ func racewalk(fn *Node) { racewalklist(fn.Func.Exit, nil) } - // nodpc is the PC of the caller as extracted by - // getcallerpc. We use -widthptr(FP) for x86. - // BUG: this will not work on arm. - nodpc := Nod(OXXX, nil, nil) - - *nodpc = *nodfp - nodpc.Type = Types[TUINTPTR] - nodpc.Xoffset = int64(-Widthptr) - nd := mkcall("racefuncenter", nil, nil, nodpc) + nd := mkcall("racefuncenterfp", nil, nil, Nod(OADDR, nodfp, nil)) fn.Func.Enter = concat(list1(nd), fn.Func.Enter) nd = mkcall("racefuncexit", nil, nil) fn.Func.Exit = list(fn.Func.Exit, nd) diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index b568c58fba..312d494f5d 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -1250,7 +1250,7 @@ func (s *state) expr(n *Node) *ssa.Value { aux := &ssa.ExternSymbol{n.Type, n.Left.Sym} return s.entryNewValue1A(ssa.OpAddr, n.Type, aux, s.sb) case OPARAM: - addr := s.addr(n) + addr := s.addr(n, false) return s.newValue2(ssa.OpLoad, n.Left.Type, addr, s.mem()) case ONAME: if n.Class == PFUNC { @@ -1262,10 +1262,10 @@ func (s *state) expr(n *Node) *ssa.Value { if canSSA(n) { return s.variable(n, n.Type) } - addr := s.addr(n) + addr := s.addr(n, false) return s.newValue2(ssa.OpLoad, n.Type, addr, s.mem()) case OCLOSUREVAR: - addr := s.addr(n) + addr := s.addr(n, false) return s.newValue2(ssa.OpLoad, n.Type, addr, s.mem()) case OLITERAL: switch n.Val().Ctype() { @@ -1376,8 +1376,10 @@ func (s *state) expr(n *Node) *ssa.Value { } if flag_race != 0 { - s.Unimplementedf("questionable CONVNOP from race detector %v -> %v\n", from, to) - return nil + // These appear to be fine, but they fail the + // integer constraint below, so okay them here. + // Sample non-integer conversion: map[string]string -> *uint8 + return v } if etypesign(from.Etype) == 0 { @@ -1716,7 +1718,7 @@ func (s *state) expr(n *Node) *ssa.Value { return s.expr(n.Left) case OADDR: - return s.addr(n.Left) + return s.addr(n.Left, n.Bounded) case OINDREG: if int(n.Reg) != Thearch.REGSP { @@ -1733,7 +1735,7 @@ func (s *state) expr(n *Node) *ssa.Value { case ODOT: // TODO: fix when we can SSA struct types. - p := s.addr(n) + p := s.addr(n, false) return s.newValue2(ssa.OpLoad, n.Type, p, s.mem()) case ODOTPTR: @@ -1757,11 +1759,11 @@ func (s *state) expr(n *Node) *ssa.Value { ptr = s.newValue2(ssa.OpAddPtr, ptrtyp, ptr, i) return s.newValue2(ssa.OpLoad, Types[TUINT8], ptr, s.mem()) case n.Left.Type.IsSlice(): - p := s.addr(n) + p := s.addr(n, false) return s.newValue2(ssa.OpLoad, n.Left.Type.Type, p, s.mem()) case n.Left.Type.IsArray(): // TODO: fix when we can SSA arrays of length 1. - p := s.addr(n) + p := s.addr(n, false) return s.newValue2(ssa.OpLoad, n.Left.Type.Type, p, s.mem()) default: s.Fatalf("bad type for index %v", n.Left.Type) @@ -1927,7 +1929,7 @@ func (s *state) expr(n *Node) *ssa.Value { args = append(args, s.expr(l.N)) store = append(store, true) } else { - args = append(args, s.addr(l.N)) + args = append(args, s.addr(l.N, false)) store = append(store, false) } } @@ -1970,7 +1972,7 @@ func (s *state) assign(left *Node, right *ssa.Value, wb bool) { // right == nil means use the zero value of the assigned type. if !canSSA(left) { // if we can't ssa this memory, treat it as just zeroing out the backing memory - addr := s.addr(left) + addr := s.addr(left, false) if left.Op == ONAME { s.vars[&memVar] = s.newValue1A(ssa.OpVarDef, ssa.TypeMem, left, s.mem()) } @@ -1985,7 +1987,7 @@ func (s *state) assign(left *Node, right *ssa.Value, wb bool) { return } // not ssa-able. Treat as a store. - addr := s.addr(left) + addr := s.addr(left, false) if left.Op == ONAME { s.vars[&memVar] = s.newValue1A(ssa.OpVarDef, ssa.TypeMem, left, s.mem()) } @@ -2187,7 +2189,9 @@ func etypesign(e uint8) int8 { // addr converts the address of the expression n to SSA, adds it to s and returns the SSA result. // The value that the returned Value represents is guaranteed to be non-nil. -func (s *state) addr(n *Node) *ssa.Value { +// If bounded is true then this address does not require a nil check for its operand +// even if that would otherwise be implied. +func (s *state) addr(n *Node, bounded bool) *ssa.Value { switch n.Op { case ONAME: switch n.Class { @@ -2250,7 +2254,7 @@ func (s *state) addr(n *Node) *ssa.Value { p := s.newValue1(ssa.OpSlicePtr, Ptrto(n.Left.Type.Type), a) return s.newValue2(ssa.OpPtrIndex, Ptrto(n.Left.Type.Type), p, i) } else { // array - a := s.addr(n.Left) + a := s.addr(n.Left, bounded) i := s.expr(n.Right) i = s.extendIndex(i) len := s.constInt(Types[TINT], n.Left.Type.Bound) @@ -2261,14 +2265,18 @@ func (s *state) addr(n *Node) *ssa.Value { } case OIND: p := s.expr(n.Left) - s.nilCheck(p) + if !bounded { + s.nilCheck(p) + } return p case ODOT: - p := s.addr(n.Left) + p := s.addr(n.Left, bounded) return s.newValue2(ssa.OpAddPtr, p.Type, p, s.constIntPtr(Types[TUINTPTR], n.Xoffset)) case ODOTPTR: p := s.expr(n.Left) - s.nilCheck(p) + if !bounded { + s.nilCheck(p) + } return s.newValue2(ssa.OpAddPtr, p.Type, p, s.constIntPtr(Types[TUINTPTR], n.Xoffset)) case OCLOSUREVAR: return s.newValue2(ssa.OpAddPtr, Ptrto(n.Type), @@ -2285,6 +2293,11 @@ func (s *state) addr(n *Node) *ssa.Value { original_p.Xoffset = n.Xoffset aux := &ssa.ArgSymbol{Typ: n.Type, Node: &original_p} return s.entryNewValue1A(ssa.OpAddr, Ptrto(n.Type), aux, s.sp) + case OCONVNOP: + addr := s.addr(n.Left, bounded) + to := Ptrto(n.Type) + return s.newValue1(ssa.OpCopy, to, addr) // ensure that addr has the right type + default: s.Unimplementedf("unhandled addr %v", Oconv(int(n.Op), 0)) return nil diff --git a/src/cmd/compile/internal/ssa/schedule.go b/src/cmd/compile/internal/ssa/schedule.go index 949de31afc..dd0a42a5dd 100644 --- a/src/cmd/compile/internal/ssa/schedule.go +++ b/src/cmd/compile/internal/ssa/schedule.go @@ -86,7 +86,7 @@ func schedule(f *Func) { // in the entry block where there are no phi functions, so there is no // conflict or ambiguity here. if b != f.Entry { - f.Fatalf("LoweredGetClosurePtr appeared outside of entry block.") + f.Fatalf("LoweredGetClosurePtr appeared outside of entry block, b=%s", b.String()) } score[v.ID] = ScorePhi case v.Op == OpPhi: diff --git a/src/cmd/compile/internal/ssa/tighten.go b/src/cmd/compile/internal/ssa/tighten.go index a43218095e..05c349cc17 100644 --- a/src/cmd/compile/internal/ssa/tighten.go +++ b/src/cmd/compile/internal/ssa/tighten.go @@ -54,7 +54,8 @@ func tighten(f *Func) { for _, b := range f.Blocks { for i := 0; i < len(b.Values); i++ { v := b.Values[i] - if v.Op == OpPhi { + if v.Op == OpPhi || v.Op == OpGetClosurePtr { + // GetClosurePtr must stay in entry block continue } if uses[v.ID] == 1 && !phi[v.ID] && home[v.ID] != b && len(v.Args) < 2 { diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s index d9e674b61f..80c4d79a7d 100644 --- a/src/runtime/race_amd64.s +++ b/src/runtime/race_amd64.s @@ -159,14 +159,28 @@ call: ret: RET +// func runtime·racefuncenterfp(fp uintptr) +// Called from instrumented code. +// Like racefuncenter but passes FP, not PC +TEXT runtime·racefuncenterfp(SB), NOSPLIT, $0-8 + MOVQ fp+0(FP), R11 + MOVQ -8(R11), R11 + JMP racefuncenter<>(SB) + // func runtime·racefuncenter(pc uintptr) // Called from instrumented code. TEXT runtime·racefuncenter(SB), NOSPLIT, $0-8 + MOVQ callpc+0(FP), R11 + JMP racefuncenter<>(SB) + +// Common code for racefuncenter/racefuncenterfp +// R11 = caller's return address +TEXT racefuncenter<>(SB), NOSPLIT, $0-0 MOVQ DX, R15 // save function entry context (for closures) get_tls(R12) MOVQ g(R12), R14 MOVQ g_racectx(R14), RARG0 // goroutine context - MOVQ callpc+0(FP), RARG1 + MOVQ R11, RARG1 // void __tsan_func_enter(ThreadState *thr, void *pc); MOVQ $__tsan_func_enter(SB), AX // racecall<> preserves R15 -- cgit v1.3 From e99dd520665000dfeb848fb4ecd381314b8fe61b Mon Sep 17 00:00:00 2001 From: David Chase Date: Mon, 19 Oct 2015 11:36:07 -0400 Subject: [dev.ssa] cmd/compile: enhance SSA filtering, add OpConvert Modified GOSSA{HASH.PKG} environment variable filters to make it easier to make/run with all SSA for testing. Disable attempts at SSA for architectures that are not amd64 (avoid spurious errors/unimplementeds.) Removed easy out for unimplemented features. Add convert op for proper liveness in presence of uintptr to/from unsafe.Pointer conversions. Tweaked stack sizes to get a pass on windows; 1024 instead 768, was observed to pass at least once. Change-Id: Ida3800afcda67d529e3b1cf48ca4a3f0fa48b2c5 Reviewed-on: https://go-review.googlesource.com/16201 Reviewed-by: Keith Randall Run-TryBot: David Chase --- src/cmd/compile/internal/gc/pgen.go | 4 +- src/cmd/compile/internal/gc/ssa.go | 85 +++++++++++++++++--------- src/cmd/compile/internal/ssa/gen/AMD64.rules | 3 + src/cmd/compile/internal/ssa/gen/genericOps.go | 5 +- src/cmd/compile/internal/ssa/opGen.go | 5 ++ src/cmd/compile/internal/ssa/rewriteAMD64.go | 18 ++++++ src/cmd/compile/internal/ssa/tighten.go | 8 ++- src/cmd/dist/test.go | 5 -- src/cmd/internal/obj/stack.go | 2 +- src/cmd/internal/obj/util.go | 3 + src/runtime/stack.go | 2 +- test/nosplit.go | 8 ++- 12 files changed, 105 insertions(+), 43 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go index a5010a31b4..b3ba2fbb46 100644 --- a/src/cmd/compile/internal/gc/pgen.go +++ b/src/cmd/compile/internal/gc/pgen.go @@ -414,7 +414,9 @@ func compile(fn *Node) { // Build an SSA backend function. // TODO: get rid of usessa. - ssafn, usessa = buildssa(Curfn) + if Thearch.Thestring == "amd64" { + ssafn, usessa = buildssa(Curfn) + } continpc = nil breakpc = nil diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 64391b0fca..8939f14136 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -24,8 +24,32 @@ import ( // it will never return nil, and the bool can be removed. func buildssa(fn *Node) (ssafn *ssa.Func, usessa bool) { name := fn.Func.Nname.Sym.Name + gossahash := os.Getenv("GOSSAHASH") usessa = strings.HasSuffix(name, "_ssa") || strings.Contains(name, "_ssa.") || name == os.Getenv("GOSSAFUNC") + // Environment variable control of SSA CG + // 1. IF GOSSAFUNC == current function name THEN + // compile this function with SSA and log output to ssa.html + + // 2. IF GOSSAHASH == "y" or "Y" THEN + // compile this function (and everything else) with SSA + + // 3. IF GOSSAHASH == "" THEN + // IF GOSSAPKG == current package name THEN + // compile this function (and everything in this package) with SSA + // ELSE + // use the old back end for this function. + // This is for compatibility with existing test harness and should go away. + + // 4. IF GOSSAHASH is a suffix of the binary-rendered SHA1 hash of the function name THEN + // compile this function with SSA + // ELSE + // compile this function with the old back end. + + // Plan is for 3 to be remove, and the 2) dependence on GOSSAHASH changes + // from "y"/"Y" to empty -- then SSA is default, and is disabled by setting + // GOSSAHASH to a value that is neither 0 nor 1 (e.g., "N" or "X") + if usessa { fmt.Println("generating SSA for", name) dumplist("buildssa-enter", fn.Func.Enter) @@ -58,17 +82,6 @@ func buildssa(fn *Node) (ssafn *ssa.Func, usessa bool) { } }() - // If SSA support for the function is incomplete, - // assume that any panics are due to violated - // invariants. Swallow them silently. - defer func() { - if err := recover(); err != nil { - if !e.unimplemented { - panic(err) - } - } - }() - // We construct SSA using an algorithm similar to // Brau, Buchwald, Hack, Leißa, Mallon, and Zwinkau // http://pp.info.uni-karlsruhe.de/uploads/publikationen/braun13cc.pdf @@ -167,27 +180,17 @@ func buildssa(fn *Node) (ssafn *ssa.Func, usessa bool) { // Main call to ssa package to compile function ssa.Compile(s.f) - // Calculate stats about what percentage of functions SSA handles. - if false { - fmt.Printf("SSA implemented: %t\n", !e.unimplemented) - } - - if e.unimplemented { - return nil, false - } - - // TODO: enable codegen more broadly once the codegen stabilizes - // and runtime support is in (gc maps, write barriers, etc.) - if usessa { + if usessa || gossahash == "y" || gossahash == "Y" { return s.f, true } - if localpkg.Name != os.Getenv("GOSSAPKG") { - return s.f, false - } - if os.Getenv("GOSSAHASH") == "" { + if gossahash == "" { + if localpkg.Name != os.Getenv("GOSSAPKG") { + return s.f, false + } // Use everything in the package return s.f, true } + // Check the hash of the name against a partial input hash. // We use this feature to do a binary search within a package to // find a function that is incorrectly compiled. @@ -195,10 +198,26 @@ func buildssa(fn *Node) (ssafn *ssa.Func, usessa bool) { for _, b := range sha1.Sum([]byte(name)) { hstr += fmt.Sprintf("%08b", b) } - if strings.HasSuffix(hstr, os.Getenv("GOSSAHASH")) { + + if strings.HasSuffix(hstr, gossahash) { fmt.Printf("GOSSAHASH triggered %s\n", name) return s.f, true } + + // Iteratively try additional hashes to allow tests for multi-point + // failure. + for i := 0; true; i++ { + ev := fmt.Sprintf("GOSSAHASH%d", i) + evv := os.Getenv(ev) + if evv == "" { + break + } + if strings.HasSuffix(hstr, evv) { + fmt.Printf("%s triggered %s\n", ev, name) + return s.f, true + } + } + return s.f, false } @@ -1353,6 +1372,15 @@ func (s *state) expr(n *Node) *ssa.Value { // Assume everything will work out, so set up our return value. // Anything interesting that happens from here is a fatal. x := s.expr(n.Left) + + // Special case for not confusing GC and liveness. + // We don't want pointers accidentally classified + // as not-pointers or vice-versa because of copy + // elision. + if to.IsPtr() != from.IsPtr() { + return s.newValue1(ssa.OpConvert, to, x) + } + v := s.newValue1(ssa.OpCopy, to, x) // ensure that v has the right type // CONVNOP closure @@ -1364,6 +1392,7 @@ func (s *state) expr(n *Node) *ssa.Value { if from.Etype == to.Etype { return v } + // unsafe.Pointer <--> *T if to.Etype == TUNSAFEPTR && from.IsPtr() || from.Etype == TUNSAFEPTR && to.IsPtr() { return v diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules index dd50dd2d27..abe103571d 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules @@ -281,6 +281,9 @@ (Store [2] ptr val mem) -> (MOVWstore ptr val mem) (Store [1] ptr val mem) -> (MOVBstore ptr val mem) +// We want this to stick out so the to/from ptr conversion is obvious +(Convert x) -> (LEAQ x) + // checks (IsNonNil p) -> (SETNE (TESTQ p p)) (IsInBounds idx len) -> (SETB (CMPQ idx len)) diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go index 5881596441..8a8837c0e9 100644 --- a/src/cmd/compile/internal/ssa/gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/gen/genericOps.go @@ -237,8 +237,9 @@ var genericOps = []opData{ {name: "Sqrt"}, // sqrt(arg0), float64 only // Data movement - {name: "Phi"}, // select an argument based on which predecessor block we came from - {name: "Copy"}, // output = arg0 + {name: "Phi"}, // select an argument based on which predecessor block we came from + {name: "Copy"}, // output = arg0 + {name: "Convert"}, // output = arg0 -- a copy that converts to/from a pointer // constants. Constant values are stored in the aux field. // booleans have a bool aux field, strings have a string aux diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index d86dce354b..4c191807ba 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -455,6 +455,7 @@ const ( OpSqrt OpPhi OpCopy + OpConvert OpConstBool OpConstString OpConstNil @@ -3866,6 +3867,10 @@ var opcodeTable = [...]opInfo{ name: "Copy", generic: true, }, + { + name: "Convert", + generic: true, + }, { name: "ConstBool", generic: true, diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index 2fd9a08d5b..3fe272c204 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -1670,6 +1670,24 @@ func rewriteValueAMD64(v *Value, config *Config) bool { goto endc395c0a53eeccf597e225a07b53047d1 endc395c0a53eeccf597e225a07b53047d1: ; + case OpConvert: + // match: (Convert x) + // cond: + // result: (LEAQ x) + { + t := v.Type + x := v.Args[0] + v.Op = OpAMD64LEAQ + v.AuxInt = 0 + v.Aux = nil + v.resetArgs() + v.Type = t + v.AddArg(x) + return true + } + goto end1cac40a6074914d6ae3d4aa039a625ed + end1cac40a6074914d6ae3d4aa039a625ed: + ; case OpCvt32Fto32: // match: (Cvt32Fto32 x) // cond: diff --git a/src/cmd/compile/internal/ssa/tighten.go b/src/cmd/compile/internal/ssa/tighten.go index 1da5071a2a..4fa26d2d18 100644 --- a/src/cmd/compile/internal/ssa/tighten.go +++ b/src/cmd/compile/internal/ssa/tighten.go @@ -54,8 +54,12 @@ func tighten(f *Func) { for _, b := range f.Blocks { for i := 0; i < len(b.Values); i++ { v := b.Values[i] - if v.Op == OpPhi || v.Op == OpGetClosurePtr { - // GetClosurePtr must stay in entry block + if v.Op == OpPhi || v.Op == OpGetClosurePtr || v.Op == OpConvert { + // GetClosurePtr must stay in entry block. + // OpConvert must not float over call sites. + // TODO do we instead need a dependence edge of some sort for OpConvert? + // Would memory do the trick, or do we need something else that relates + // to safe point operations? continue } if len(v.Args) > 0 && v.Args[len(v.Args)-1].Type.IsMemory() { diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index c92109afa5..be6cdb5c0b 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -278,11 +278,6 @@ func (t *tester) registerStdTest(pkg string) { // TODO: Remove when SSA codegen is used by default. func (t *tester) registerSSATest(pkg string) { - switch pkg { - // known failures - case "runtime": - return - } t.tests = append(t.tests, distTest{ name: "go_test_ssa:" + pkg, heading: "Testing packages with SSA codegen.", diff --git a/src/cmd/internal/obj/stack.go b/src/cmd/internal/obj/stack.go index 87698b3eeb..1ca673285a 100644 --- a/src/cmd/internal/obj/stack.go +++ b/src/cmd/internal/obj/stack.go @@ -41,7 +41,7 @@ const ( STACKSYSTEM = 0 StackSystem = STACKSYSTEM StackBig = 4096 - StackGuard = 640*stackGuardMultiplier + StackSystem + StackGuard = 1024*stackGuardMultiplier + StackSystem StackSmall = 128 StackLimit = StackGuard - StackSystem - StackSmall ) diff --git a/src/cmd/internal/obj/util.go b/src/cmd/internal/obj/util.go index 73d33666e2..a71d69edfc 100644 --- a/src/cmd/internal/obj/util.go +++ b/src/cmd/internal/obj/util.go @@ -385,6 +385,9 @@ func Dconv(p *Prog, a *Addr) string { if a.Index != REG_NONE { str += fmt.Sprintf("(%v*%d)", Rconv(int(a.Index)), int(a.Scale)) } + if p.As == ATYPE && a.Gotype != nil { + str += fmt.Sprintf("%s", a.Gotype.Name) + } case TYPE_CONST: if a.Reg != 0 { diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 1809a4d9ac..128278ebdc 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -86,7 +86,7 @@ const ( // The stack guard is a pointer this many bytes above the // bottom of the stack. - _StackGuard = 640*stackGuardMultiplier + _StackSystem + _StackGuard = 1024*stackGuardMultiplier + _StackSystem // After a stack split check the SP is allowed to be this // many bytes below the stack guard. This saves an instruction diff --git a/test/nosplit.go b/test/nosplit.go index e5c2a9f30e..70e8fced86 100644 --- a/test/nosplit.go +++ b/test/nosplit.go @@ -9,6 +9,7 @@ package main import ( "bytes" + "cmd/internal/obj" "fmt" "io/ioutil" "log" @@ -285,12 +286,13 @@ TestCases: // Instead of rewriting the test cases above, adjust // the first stack frame to use up the extra bytes. if i == 0 { - size += 512 - 128 + size += (obj.StackGuard - 128) - 128 // Noopt builds have a larger stackguard. - // See ../cmd/dist/buildruntime.go:stackGuardMultiplier + // See ../src/cmd/dist/buildruntime.go:stackGuardMultiplier + // This increase is included in obj.StackGuard for _, s := range strings.Split(os.Getenv("GO_GCFLAGS"), " ") { if s == "-N" { - size += 640 + size += obj.StackGuard } } } -- cgit v1.3 From 02f4d0a130ba95d7a03418c3ef308d7d21b34af3 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 2 Nov 2015 08:10:26 -0800 Subject: [dev.ssa] cmd/compile: start arguments as spilled Declare a function's arguments as having already been spilled so their use just requires a restore. Allow spill locations to be portions of larger objects the stack. Required to load portions of compound input arguments. Rename the memory input to InputMem. Use Arg for the pre-spilled argument values. Change-Id: I8fe2a03ffbba1022d98bfae2052b376b96d32dda Reviewed-on: https://go-review.googlesource.com/16536 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: David Chase --- src/cmd/compile/internal/gc/pgen.go | 3 + src/cmd/compile/internal/gc/ssa.go | 54 +++++-- src/cmd/compile/internal/ssa/compile.go | 2 +- src/cmd/compile/internal/ssa/config.go | 2 +- src/cmd/compile/internal/ssa/deadcode.go | 34 ++-- src/cmd/compile/internal/ssa/deadcode_test.go | 8 +- src/cmd/compile/internal/ssa/deadstore_test.go | 6 +- src/cmd/compile/internal/ssa/decompose.go | 69 ++++++++- src/cmd/compile/internal/ssa/dom_test.go | 26 ++-- src/cmd/compile/internal/ssa/func.go | 7 +- src/cmd/compile/internal/ssa/func_test.go | 32 ++-- src/cmd/compile/internal/ssa/gen/generic.rules | 45 ++++-- src/cmd/compile/internal/ssa/gen/genericOps.go | 3 +- src/cmd/compile/internal/ssa/html.go | 4 + src/cmd/compile/internal/ssa/location.go | 14 +- src/cmd/compile/internal/ssa/lower.go | 2 +- src/cmd/compile/internal/ssa/nilcheck_test.go | 20 +-- src/cmd/compile/internal/ssa/opGen.go | 5 + src/cmd/compile/internal/ssa/print.go | 8 + src/cmd/compile/internal/ssa/regalloc.go | 10 ++ src/cmd/compile/internal/ssa/regalloc_test.go | 2 +- src/cmd/compile/internal/ssa/rewritegeneric.go | 206 +++++++++++++++++++++---- src/cmd/compile/internal/ssa/schedule_test.go | 2 +- src/cmd/compile/internal/ssa/shift_test.go | 2 +- src/cmd/compile/internal/ssa/stackalloc.go | 34 ++-- src/cmd/compile/internal/ssa/tighten.go | 4 +- src/runtime/runtime-gdb_test.go | 7 - 27 files changed, 470 insertions(+), 141 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go index 87e99df2e6..c8f2059543 100644 --- a/src/cmd/compile/internal/gc/pgen.go +++ b/src/cmd/compile/internal/gc/pgen.go @@ -484,6 +484,9 @@ func compile(fn *Node) { if ssafn != nil && usessa { genssa(ssafn, ptxt, gcargs, gclocals) + if Curfn.Func.Endlineno != 0 { + lineno = Curfn.Func.Endlineno + } return } Genlist(Curfn.Func.Enter) diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 5a8e43dedb..2c935b7247 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -97,7 +97,7 @@ func buildssa(fn *Node) (ssafn *ssa.Func, usessa bool) { // Allocate starting values s.labels = map[string]*ssaLabel{} s.labeledNodes = map[*Node]*ssaLabel{} - s.startmem = s.entryNewValue0(ssa.OpArg, ssa.TypeMem) + s.startmem = s.entryNewValue0(ssa.OpInitMem, ssa.TypeMem) s.sp = s.entryNewValue0(ssa.OpSP, Types[TUINTPTR]) // TODO: use generic pointer type (unsafe.Pointer?) instead s.sb = s.entryNewValue0(ssa.OpSB, Types[TUINTPTR]) @@ -3168,6 +3168,12 @@ func (s *state) lookupVarIncoming(b *ssa.Block, t ssa.Type, name *Node) *ssa.Val if name == &memVar { return s.startmem } + if canSSA(name) { + v := s.entryNewValue0A(ssa.OpArg, t, name) + // v starts with AuxInt == 0. + s.addNamedValue(name, v) + return v + } // variable is live at the entry block. Load it. addr := s.decladdrs[name] if addr == nil { @@ -3239,18 +3245,21 @@ func (s *state) addNamedValue(n *Node, v *ssa.Value) { // Don't track autotmp_ variables. return } - if n.Class == PPARAM || n.Class == PPARAMOUT { - // TODO: Remove this + if n.Class == PAUTO && (v.Type.IsString() || v.Type.IsSlice() || v.Type.IsInterface()) { + // TODO: can't handle auto compound objects with pointers yet. + // The live variable analysis barfs because we don't put VARDEF + // pseudos in the right place when we spill to these nodes. return } if n.Class == PAUTO && n.Xoffset != 0 { s.Fatalf("AUTO var with offset %s %d", n, n.Xoffset) } - values, ok := s.f.NamedValues[n] + loc := ssa.LocalSlot{N: n, Type: n.Type, Off: 0} + values, ok := s.f.NamedValues[loc] if !ok { - s.f.Names = append(s.f.Names, n) + s.f.Names = append(s.f.Names, loc) } - s.f.NamedValues[n] = append(values, v) + s.f.NamedValues[loc] = append(values, v) } // an unresolved branch @@ -3873,11 +3882,17 @@ func (s *genState) genValue(v *ssa.Value) { return } p := Prog(movSizeByType(v.Type)) - n := autoVar(v.Args[0]) + n, off := autoVar(v.Args[0]) p.From.Type = obj.TYPE_MEM - p.From.Name = obj.NAME_AUTO p.From.Node = n p.From.Sym = Linksym(n.Sym) + p.From.Offset = off + if n.Class == PPARAM { + p.From.Name = obj.NAME_PARAM + p.From.Offset += n.Xoffset + } else { + p.From.Name = obj.NAME_AUTO + } p.To.Type = obj.TYPE_REG p.To.Reg = regnum(v) @@ -3889,11 +3904,17 @@ func (s *genState) genValue(v *ssa.Value) { p := Prog(movSizeByType(v.Type)) p.From.Type = obj.TYPE_REG p.From.Reg = regnum(v.Args[0]) - n := autoVar(v) + n, off := autoVar(v) p.To.Type = obj.TYPE_MEM - p.To.Name = obj.NAME_AUTO p.To.Node = n p.To.Sym = Linksym(n.Sym) + p.To.Offset = off + if n.Class == PPARAM { + p.To.Name = obj.NAME_PARAM + p.To.Offset += n.Xoffset + } else { + p.To.Name = obj.NAME_AUTO + } case ssa.OpPhi: // just check to make sure regalloc and stackalloc did it right if v.Type.IsMemory() { @@ -3912,9 +3933,10 @@ func (s *genState) genValue(v *ssa.Value) { v.Fatalf("const value %v shouldn't have a location", v) } - case ssa.OpArg: + case ssa.OpInitMem: // memory arg needs no code - // TODO: check that only mem arg goes here. + case ssa.OpArg: + // input args need no code case ssa.OpAMD64LoweredGetClosurePtr: // Output is hardwired to DX only, // and DX contains the closure pointer on @@ -4476,9 +4498,11 @@ func regnum(v *ssa.Value) int16 { return ssaRegToReg[reg.(*ssa.Register).Num] } -// autoVar returns a *Node representing the auto variable assigned to v. -func autoVar(v *ssa.Value) *Node { - return v.Block.Func.RegAlloc[v.ID].(*ssa.LocalSlot).N.(*Node) +// autoVar returns a *Node and int64 representing the auto variable and offset within it +// where v should be spilled. +func autoVar(v *ssa.Value) (*Node, int64) { + loc := v.Block.Func.RegAlloc[v.ID].(ssa.LocalSlot) + return loc.N.(*Node), loc.Off } // ssaExport exports a bunch of compiler services for the ssa backend. diff --git a/src/cmd/compile/internal/ssa/compile.go b/src/cmd/compile/internal/ssa/compile.go index af672eea99..01238f24ca 100644 --- a/src/cmd/compile/internal/ssa/compile.go +++ b/src/cmd/compile/internal/ssa/compile.go @@ -83,8 +83,8 @@ type pass struct { var passes = [...]pass{ {"phielim", phielim}, {"copyelim", copyelim}, - {"decompose", decompose}, {"early deadcode", deadcode}, // remove generated dead code to avoid doing pointless work during opt + {"decompose", decompose}, {"opt", opt}, {"opt deadcode", deadcode}, // remove any blocks orphaned during opt {"generic cse", cse}, diff --git a/src/cmd/compile/internal/ssa/config.go b/src/cmd/compile/internal/ssa/config.go index 014c960267..6d3a949a6a 100644 --- a/src/cmd/compile/internal/ssa/config.go +++ b/src/cmd/compile/internal/ssa/config.go @@ -103,7 +103,7 @@ func (c *Config) Frontend() Frontend { return c.fe } // NewFunc returns a new, empty function object func (c *Config) NewFunc() *Func { // TODO(khr): should this function take name, type, etc. as arguments? - return &Func{Config: c, NamedValues: map[GCNode][]*Value{}} + return &Func{Config: c, NamedValues: map[LocalSlot][]*Value{}} } func (c *Config) Logf(msg string, args ...interface{}) { c.fe.Logf(msg, args...) } diff --git a/src/cmd/compile/internal/ssa/deadcode.go b/src/cmd/compile/internal/ssa/deadcode.go index 3351589fda..e9d6525701 100644 --- a/src/cmd/compile/internal/ssa/deadcode.go +++ b/src/cmd/compile/internal/ssa/deadcode.go @@ -162,24 +162,38 @@ func deadcode(f *Func) { } f.Blocks = f.Blocks[:i] - // Remove dead entries from namedValues map. - for name, values := range f.NamedValues { - i := 0 + // Remove dead & duplicate entries from namedValues map. + s := newSparseSet(f.NumValues()) + i = 0 + for _, name := range f.Names { + j := 0 + s.clear() + values := f.NamedValues[name] for _, v := range values { for v.Op == OpCopy { v = v.Args[0] } - if live[v.ID] { - values[i] = v - i++ + if live[v.ID] && !s.contains(v.ID) { + values[j] = v + j++ + s.add(v.ID) } } - f.NamedValues[name] = values[:i] - tail := values[i:] - for j := range tail { - tail[j] = nil + if j == 0 { + delete(f.NamedValues, name) + } else { + f.Names[i] = name + i++ + for k := len(values) - 1; k >= j; k-- { + values[k] = nil + } + f.NamedValues[name] = values[:j] } } + for k := len(f.Names) - 1; k >= i; k-- { + f.Names[k] = LocalSlot{} + } + f.Names = f.Names[:i] // TODO: renumber Blocks and Values densely? // TODO: save dead Values and Blocks for reuse? Or should we just let GC handle it? diff --git a/src/cmd/compile/internal/ssa/deadcode_test.go b/src/cmd/compile/internal/ssa/deadcode_test.go index 7f491c77f9..c59d77ea60 100644 --- a/src/cmd/compile/internal/ssa/deadcode_test.go +++ b/src/cmd/compile/internal/ssa/deadcode_test.go @@ -10,7 +10,7 @@ func TestDeadLoop(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit")), Bloc("exit", Exit("mem")), @@ -40,7 +40,7 @@ func TestDeadValue(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("deadval", OpConst64, TypeInt64, 37, nil), Goto("exit")), Bloc("exit", @@ -64,7 +64,7 @@ func TestNeverTaken(t *testing.T) { fun := Fun(c, "entry", Bloc("entry", Valu("cond", OpConstBool, TypeBool, 0, nil), - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), If("cond", "then", "else")), Bloc("then", Goto("exit")), @@ -98,7 +98,7 @@ func TestNestedDeadBlocks(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("cond", OpConstBool, TypeBool, 0, nil), If("cond", "b2", "b4")), Bloc("b2", diff --git a/src/cmd/compile/internal/ssa/deadstore_test.go b/src/cmd/compile/internal/ssa/deadstore_test.go index 159ac4e439..4514c99004 100644 --- a/src/cmd/compile/internal/ssa/deadstore_test.go +++ b/src/cmd/compile/internal/ssa/deadstore_test.go @@ -12,7 +12,7 @@ func TestDeadStore(t *testing.T) { ptrType := &TypeImpl{Size_: 8, Ptr: true, Name: "testptr", Elem_: elemType} // dummy for testing fun := Fun(c, "entry", Bloc("entry", - Valu("start", OpArg, TypeMem, 0, ".mem"), + Valu("start", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Valu("v", OpConstBool, TypeBool, 1, nil), Valu("addr1", OpAddr, ptrType, 0, nil, "sb"), @@ -47,7 +47,7 @@ func TestDeadStorePhi(t *testing.T) { ptrType := &TypeImpl{Size_: 8, Ptr: true, Name: "testptr"} // dummy for testing fun := Fun(c, "entry", Bloc("entry", - Valu("start", OpArg, TypeMem, 0, ".mem"), + Valu("start", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Valu("v", OpConstBool, TypeBool, 1, nil), Valu("addr", OpAddr, ptrType, 0, nil, "sb"), @@ -74,7 +74,7 @@ func TestDeadStoreTypes(t *testing.T) { t2 := &TypeImpl{Size_: 4, Ptr: true, Name: "t2"} fun := Fun(c, "entry", Bloc("entry", - Valu("start", OpArg, TypeMem, 0, ".mem"), + Valu("start", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Valu("v", OpConstBool, TypeBool, 1, nil), Valu("addr1", OpAddr, t1, 0, nil, "sb"), diff --git a/src/cmd/compile/internal/ssa/decompose.go b/src/cmd/compile/internal/ssa/decompose.go index 2057d8ea5c..c8a1df281a 100644 --- a/src/cmd/compile/internal/ssa/decompose.go +++ b/src/cmd/compile/internal/ssa/decompose.go @@ -29,8 +29,75 @@ func decompose(f *Func) { } } } - // TODO: decompose complex? // TODO: decompose 64-bit ops on 32-bit archs? + + // Split up named values into their components. + // NOTE: the component values we are making are dead at this point. + // We must do the opt pass before any deadcode elimination or we will + // lose the name->value correspondence. + for _, name := range f.Names { + t := name.Type + switch { + case t.IsComplex(): + var elemType Type + if t.Size() == 16 { + elemType = f.Config.fe.TypeFloat64() + } else { + elemType = f.Config.fe.TypeFloat32() + } + rName := LocalSlot{name.N, elemType, name.Off} + iName := LocalSlot{name.N, elemType, name.Off + elemType.Size()} + f.Names = append(f.Names, rName, iName) + for _, v := range f.NamedValues[name] { + r := v.Block.NewValue1(v.Line, OpComplexReal, elemType, v) + i := v.Block.NewValue1(v.Line, OpComplexImag, elemType, v) + f.NamedValues[rName] = append(f.NamedValues[rName], r) + f.NamedValues[iName] = append(f.NamedValues[iName], i) + } + case t.IsString(): + ptrType := f.Config.fe.TypeBytePtr() + lenType := f.Config.fe.TypeInt() + ptrName := LocalSlot{name.N, ptrType, name.Off} + lenName := LocalSlot{name.N, lenType, name.Off + f.Config.PtrSize} + f.Names = append(f.Names, ptrName, lenName) + for _, v := range f.NamedValues[name] { + ptr := v.Block.NewValue1(v.Line, OpStringPtr, ptrType, v) + len := v.Block.NewValue1(v.Line, OpStringLen, lenType, v) + f.NamedValues[ptrName] = append(f.NamedValues[ptrName], ptr) + f.NamedValues[lenName] = append(f.NamedValues[lenName], len) + } + case t.IsSlice(): + ptrType := f.Config.fe.TypeBytePtr() + lenType := f.Config.fe.TypeInt() + ptrName := LocalSlot{name.N, ptrType, name.Off} + lenName := LocalSlot{name.N, lenType, name.Off + f.Config.PtrSize} + capName := LocalSlot{name.N, lenType, name.Off + 2*f.Config.PtrSize} + f.Names = append(f.Names, ptrName, lenName, capName) + for _, v := range f.NamedValues[name] { + ptr := v.Block.NewValue1(v.Line, OpSlicePtr, ptrType, v) + len := v.Block.NewValue1(v.Line, OpSliceLen, lenType, v) + cap := v.Block.NewValue1(v.Line, OpSliceCap, lenType, v) + f.NamedValues[ptrName] = append(f.NamedValues[ptrName], ptr) + f.NamedValues[lenName] = append(f.NamedValues[lenName], len) + f.NamedValues[capName] = append(f.NamedValues[capName], cap) + } + case t.IsInterface(): + ptrType := f.Config.fe.TypeBytePtr() + typeName := LocalSlot{name.N, ptrType, name.Off} + dataName := LocalSlot{name.N, ptrType, name.Off + f.Config.PtrSize} + f.Names = append(f.Names, typeName, dataName) + for _, v := range f.NamedValues[name] { + typ := v.Block.NewValue1(v.Line, OpITab, ptrType, v) + data := v.Block.NewValue1(v.Line, OpIData, ptrType, v) + f.NamedValues[typeName] = append(f.NamedValues[typeName], typ) + f.NamedValues[dataName] = append(f.NamedValues[dataName], data) + } + //case t.IsStruct(): + // TODO + case t.Size() > f.Config.IntSize: + f.Unimplementedf("undecomposed type %s", t) + } + } } func decomposeStringPhi(v *Value) { diff --git a/src/cmd/compile/internal/ssa/dom_test.go b/src/cmd/compile/internal/ssa/dom_test.go index eff7205fa3..84e0093799 100644 --- a/src/cmd/compile/internal/ssa/dom_test.go +++ b/src/cmd/compile/internal/ssa/dom_test.go @@ -20,7 +20,7 @@ func genLinear(size int) []bloc { var blocs []bloc blocs = append(blocs, Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto(blockn(0)), ), ) @@ -43,7 +43,7 @@ func genFwdBack(size int) []bloc { var blocs []bloc blocs = append(blocs, Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), Goto(blockn(0)), ), @@ -73,7 +73,7 @@ func genManyPred(size int) []bloc { var blocs []bloc blocs = append(blocs, Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), Goto(blockn(0)), ), @@ -111,7 +111,7 @@ func genMaxPred(size int) []bloc { var blocs []bloc blocs = append(blocs, Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), Goto(blockn(0)), ), @@ -136,7 +136,7 @@ func genMaxPredValue(size int) []bloc { var blocs []bloc blocs = append(blocs, Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), Goto(blockn(0)), ), @@ -223,7 +223,7 @@ func TestDominatorsSingleBlock(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Exit("mem"))) doms := map[string]string{} @@ -238,7 +238,7 @@ func TestDominatorsSimple(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("a")), Bloc("a", Goto("b")), @@ -266,7 +266,7 @@ func TestDominatorsMultPredFwd(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), If("p", "a", "c")), Bloc("a", @@ -294,7 +294,7 @@ func TestDominatorsDeadCode(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 0, nil), If("p", "b3", "b5")), Bloc("b2", Exit("mem")), @@ -319,7 +319,7 @@ func TestDominatorsMultPredRev(t *testing.T) { Bloc("entry", Goto("first")), Bloc("first", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), Goto("a")), Bloc("a", @@ -348,7 +348,7 @@ func TestDominatorsMultPred(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), If("p", "a", "c")), Bloc("a", @@ -376,7 +376,7 @@ func TestPostDominators(t *testing.T) { c := testConfig(t) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), If("p", "a", "c")), Bloc("a", @@ -403,7 +403,7 @@ func TestInfiniteLoop(t *testing.T) { // note lack of an exit block fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("p", OpConstBool, TypeBool, 1, nil), Goto("a")), Bloc("a", diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index ce11b184f6..e5fbfdb5ff 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -26,12 +26,11 @@ type Func struct { // when register allocation is done, maps value ids to locations RegAlloc []Location - // map from *gc.Node to set of Values that represent that Node. - // The Node must be an ONAME with PPARAM, PPARAMOUT, or PAUTO class. - NamedValues map[GCNode][]*Value + // map from LocalSlot to set of Values that we want to store in that slot. + NamedValues map[LocalSlot][]*Value // Names is a copy of NamedValues.Keys. We keep a separate list // of keys to make iteration order deterministic. - Names []GCNode + Names []LocalSlot } // NumBlocks returns an integer larger than the id of any Block in the Func. diff --git a/src/cmd/compile/internal/ssa/func_test.go b/src/cmd/compile/internal/ssa/func_test.go index dc5d220db8..d35690a30c 100644 --- a/src/cmd/compile/internal/ssa/func_test.go +++ b/src/cmd/compile/internal/ssa/func_test.go @@ -18,7 +18,7 @@ // // fun := Fun("entry", // Bloc("entry", -// Valu("mem", OpArg, TypeMem, 0, ".mem"), +// Valu("mem", OpInitMem, TypeMem, 0, ".mem"), // Goto("exit")), // Bloc("exit", // Exit("mem")), @@ -263,7 +263,7 @@ func TestArgs(t *testing.T) { Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "a", "b"), - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit")), Bloc("exit", Exit("mem"))) @@ -286,7 +286,7 @@ func TestEquiv(t *testing.T) { Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "a", "b"), - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit")), Bloc("exit", Exit("mem"))), @@ -295,7 +295,7 @@ func TestEquiv(t *testing.T) { Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "a", "b"), - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit")), Bloc("exit", Exit("mem"))), @@ -307,7 +307,7 @@ func TestEquiv(t *testing.T) { Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "a", "b"), - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit")), Bloc("exit", Exit("mem"))), @@ -318,7 +318,7 @@ func TestEquiv(t *testing.T) { Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "a", "b"), - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit"))), }, } @@ -335,26 +335,26 @@ func TestEquiv(t *testing.T) { { Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Goto("exit")), Bloc("exit", Exit("mem"))), Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Exit("mem"))), }, // value order changed { Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("a", OpConst64, TypeInt64, 14, nil), Exit("mem"))), Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Exit("mem"))), @@ -363,12 +363,12 @@ func TestEquiv(t *testing.T) { { Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 14, nil), Exit("mem"))), Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 26, nil), Exit("mem"))), }, @@ -376,12 +376,12 @@ func TestEquiv(t *testing.T) { { Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 0, 14), Exit("mem"))), Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 0, 26), Exit("mem"))), }, @@ -389,14 +389,14 @@ func TestEquiv(t *testing.T) { { Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 14, nil), Valu("b", OpConst64, TypeInt64, 26, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "a", "b"), Exit("mem"))), Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("a", OpConst64, TypeInt64, 0, nil), Valu("b", OpConst64, TypeInt64, 14, nil), Valu("sum", OpAdd64, TypeInt64, 0, nil, "b", "a"), diff --git a/src/cmd/compile/internal/ssa/gen/generic.rules b/src/cmd/compile/internal/ssa/gen/generic.rules index bb347aea8b..9c1da92b7e 100644 --- a/src/cmd/compile/internal/ssa/gen/generic.rules +++ b/src/cmd/compile/internal/ssa/gen/generic.rules @@ -188,12 +188,12 @@ (Load ptr mem) && t.IsString() -> (StringMake (Load ptr mem) - (Load - (OffPtr [config.PtrSize] ptr) + (Load + (OffPtr [config.PtrSize] ptr) mem)) (Store [2*config.PtrSize] dst (StringMake ptr len) mem) -> (Store [config.PtrSize] - (OffPtr [config.PtrSize] dst) + (OffPtr [config.PtrSize] dst) len (Store [config.PtrSize] dst ptr mem)) @@ -215,18 +215,18 @@ (Load ptr mem) && t.IsSlice() -> (SliceMake (Load ptr mem) - (Load - (OffPtr [config.PtrSize] ptr) + (Load + (OffPtr [config.PtrSize] ptr) mem) - (Load - (OffPtr [2*config.PtrSize] ptr) + (Load + (OffPtr [2*config.PtrSize] ptr) mem)) (Store [3*config.PtrSize] dst (SliceMake ptr len cap) mem) -> (Store [config.PtrSize] - (OffPtr [2*config.PtrSize] dst) + (OffPtr [2*config.PtrSize] dst) cap (Store [config.PtrSize] - (OffPtr [config.PtrSize] dst) + (OffPtr [config.PtrSize] dst) len (Store [config.PtrSize] dst ptr mem))) @@ -261,3 +261,30 @@ // Get rid of Convert ops for pointer arithmetic on unsafe.Pointer. (Convert (Add64 (Convert ptr) off)) -> (Add64 ptr off) + +// Decompose compound argument values +(Arg {n} [off]) && v.Type.IsString() -> + (StringMake + (Arg {n} [off]) + (Arg {n} [off+config.PtrSize])) + +(Arg {n} [off]) && v.Type.IsSlice() -> + (SliceMake + (Arg {n} [off]) + (Arg {n} [off+config.PtrSize]) + (Arg {n} [off+2*config.PtrSize])) + +(Arg {n} [off]) && v.Type.IsInterface() -> + (IMake + (Arg {n} [off]) + (Arg {n} [off+config.PtrSize])) + +(Arg {n} [off]) && v.Type.IsComplex() && v.Type.Size() == 16 -> + (ComplexMake + (Arg {n} [off]) + (Arg {n} [off+8])) + +(Arg {n} [off]) && v.Type.IsComplex() && v.Type.Size() == 8 -> + (ComplexMake + (Arg {n} [off]) + (Arg {n} [off+4])) diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go index 162ee0dab4..8eb10a7d9b 100644 --- a/src/cmd/compile/internal/ssa/gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/gen/genericOps.go @@ -260,7 +260,8 @@ var genericOps = []opData{ // TODO: Const32F, ... // Constant-like things - {name: "Arg"}, // memory input to the function. + {name: "InitMem"}, // memory input to the function. + {name: "Arg"}, // argument to the function. aux=GCNode of arg, off = offset in that arg. // The address of a variable. arg0 is the base pointer (SB or SP, depending // on whether it is a global or stack variable). The Aux field identifies the diff --git a/src/cmd/compile/internal/ssa/html.go b/src/cmd/compile/internal/ssa/html.go index 68a432c676..9b8fc3750b 100644 --- a/src/cmd/compile/internal/ssa/html.go +++ b/src/cmd/compile/internal/ssa/html.go @@ -472,3 +472,7 @@ func (p htmlFuncPrinter) startDepCycle() { func (p htmlFuncPrinter) endDepCycle() { fmt.Fprintln(p.w, "") } + +func (p htmlFuncPrinter) named(n LocalSlot, vals []*Value) { + // TODO +} diff --git a/src/cmd/compile/internal/ssa/location.go b/src/cmd/compile/internal/ssa/location.go index 0f9fb33eeb..85f525565b 100644 --- a/src/cmd/compile/internal/ssa/location.go +++ b/src/cmd/compile/internal/ssa/location.go @@ -4,6 +4,8 @@ package ssa +import "fmt" + // A place that an ssa variable can reside. type Location interface { Name() string // name to use in assembly templates: %rax, 16(%rsp), ... @@ -21,10 +23,16 @@ func (r *Register) Name() string { } // A LocalSlot is a location in the stack frame. +// It is (possibly a subpiece of) a PPARAM, PPARAMOUT, or PAUTO ONAME node. type LocalSlot struct { - N GCNode // a *gc.Node for an auto variable + N GCNode // an ONAME *gc.Node representing a variable on the stack + Type Type // type of slot + Off int64 // offset of slot in N } -func (s *LocalSlot) Name() string { - return s.N.String() +func (s LocalSlot) Name() string { + if s.Off == 0 { + return fmt.Sprintf("%s[%s]", s.N, s.Type) + } + return fmt.Sprintf("%s+%d[%s]", s.N, s.Off, s.Type) } diff --git a/src/cmd/compile/internal/ssa/lower.go b/src/cmd/compile/internal/ssa/lower.go index 9c28bd10a5..bf3c15f78b 100644 --- a/src/cmd/compile/internal/ssa/lower.go +++ b/src/cmd/compile/internal/ssa/lower.go @@ -21,7 +21,7 @@ func checkLower(f *Func) { continue // lowered } switch v.Op { - case OpSP, OpSB, OpArg, OpCopy, OpPhi, OpVarDef, OpVarKill: + case OpSP, OpSB, OpInitMem, OpArg, OpCopy, OpPhi, OpVarDef, OpVarKill: continue // ok not to lower } s := "not lowered: " + v.Op.String() + " " + v.Type.SimpleString() diff --git a/src/cmd/compile/internal/ssa/nilcheck_test.go b/src/cmd/compile/internal/ssa/nilcheck_test.go index 8f32f32b1d..d4a55c0855 100644 --- a/src/cmd/compile/internal/ssa/nilcheck_test.go +++ b/src/cmd/compile/internal/ssa/nilcheck_test.go @@ -21,7 +21,7 @@ func benchmarkNilCheckDeep(b *testing.B, depth int) { var blocs []bloc blocs = append(blocs, Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto(blockn(0)), ), @@ -67,7 +67,7 @@ func TestNilcheckSimple(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -104,7 +104,7 @@ func TestNilcheckDomOrder(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -140,7 +140,7 @@ func TestNilcheckAddr(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -173,7 +173,7 @@ func TestNilcheckAddPtr(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -207,7 +207,7 @@ func TestNilcheckPhi(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Valu("sp", OpSP, TypeInvalid, 0, nil), Valu("baddr", OpAddr, TypeBool, 0, "b", "sp"), @@ -251,7 +251,7 @@ func TestNilcheckKeepRemove(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -299,7 +299,7 @@ func TestNilcheckInFalseBranch(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -350,7 +350,7 @@ func TestNilcheckUser(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", @@ -389,7 +389,7 @@ func TestNilcheckBug(t *testing.T) { c := NewConfig("amd64", DummyFrontend{t}, nil) fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("sb", OpSB, TypeInvalid, 0, nil), Goto("checkPtr")), Bloc("checkPtr", diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 400f59e174..d043e076ea 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -475,6 +475,7 @@ const ( OpConst64F OpConstInterface OpConstSlice + OpInitMem OpArg OpAddr OpSP @@ -3987,6 +3988,10 @@ var opcodeTable = [...]opInfo{ name: "ConstSlice", generic: true, }, + { + name: "InitMem", + generic: true, + }, { name: "Arg", generic: true, diff --git a/src/cmd/compile/internal/ssa/print.go b/src/cmd/compile/internal/ssa/print.go index 192dc83b39..b61e6f1cc7 100644 --- a/src/cmd/compile/internal/ssa/print.go +++ b/src/cmd/compile/internal/ssa/print.go @@ -28,6 +28,7 @@ type funcPrinter interface { value(v *Value, live bool) startDepCycle() endDepCycle() + named(n LocalSlot, vals []*Value) } type stringFuncPrinter struct { @@ -73,6 +74,10 @@ func (p stringFuncPrinter) startDepCycle() { func (p stringFuncPrinter) endDepCycle() {} +func (p stringFuncPrinter) named(n LocalSlot, vals []*Value) { + fmt.Fprintf(p.w, "name %s: %v\n", n.Name(), vals) +} + func fprintFunc(p funcPrinter, f *Func) { reachable, live := findlive(f) p.header(f) @@ -136,4 +141,7 @@ func fprintFunc(p funcPrinter, f *Func) { p.endBlock(b) } + for name, vals := range f.NamedValues { + p.named(name, vals) + } } diff --git a/src/cmd/compile/internal/ssa/regalloc.go b/src/cmd/compile/internal/ssa/regalloc.go index 89deb14a4a..a751d66988 100644 --- a/src/cmd/compile/internal/ssa/regalloc.go +++ b/src/cmd/compile/internal/ssa/regalloc.go @@ -759,6 +759,16 @@ func (s *regAllocState) regalloc(f *Func) { pc++ continue } + if v.Op == OpArg { + // Args are "pre-spilled" values. We don't allocate + // any register here. We just set up the spill pointer to + // point at itself and any later user will restore it to use it. + s.values[v.ID].spill = v + s.values[v.ID].spillUsed = true // use is guaranteed + b.Values = append(b.Values, v) + pc++ + continue + } s.clearUses(pc*2 - 1) regspec := opcodeTable[v.Op].reg if regDebug { diff --git a/src/cmd/compile/internal/ssa/regalloc_test.go b/src/cmd/compile/internal/ssa/regalloc_test.go index dcd253ea14..08260fbbbb 100644 --- a/src/cmd/compile/internal/ssa/regalloc_test.go +++ b/src/cmd/compile/internal/ssa/regalloc_test.go @@ -10,7 +10,7 @@ func TestLiveControlOps(t *testing.T) { c := testConfig(t) f := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("x", OpAMD64MOVBconst, TypeInt8, 0, 1), Valu("y", OpAMD64MOVBconst, TypeInt8, 0, 2), Valu("a", OpAMD64TESTB, TypeBool, 0, nil, "x", "y"), diff --git a/src/cmd/compile/internal/ssa/rewritegeneric.go b/src/cmd/compile/internal/ssa/rewritegeneric.go index 2448b43547..c349603583 100644 --- a/src/cmd/compile/internal/ssa/rewritegeneric.go +++ b/src/cmd/compile/internal/ssa/rewritegeneric.go @@ -23,6 +23,8 @@ func rewriteValuegeneric(v *Value, config *Config) bool { return rewriteValuegeneric_OpAnd64(v, config) case OpAnd8: return rewriteValuegeneric_OpAnd8(v, config) + case OpArg: + return rewriteValuegeneric_OpArg(v, config) case OpArrayIndex: return rewriteValuegeneric_OpArrayIndex(v, config) case OpCom16: @@ -402,6 +404,156 @@ endeaf127389bd0d4b0e0e297830f8f463b: ; return false } +func rewriteValuegeneric_OpArg(v *Value, config *Config) bool { + b := v.Block + _ = b + // match: (Arg {n} [off]) + // cond: v.Type.IsString() + // result: (StringMake (Arg {n} [off]) (Arg {n} [off+config.PtrSize])) + { + n := v.Aux + off := v.AuxInt + if !(v.Type.IsString()) { + goto end939d3f946bf61eb85b46b374e7afa9e9 + } + v.Op = OpStringMake + v.AuxInt = 0 + v.Aux = nil + v.resetArgs() + v0 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v0.Type = config.fe.TypeBytePtr() + v0.Aux = n + v0.AuxInt = off + v.AddArg(v0) + v1 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v1.Type = config.fe.TypeInt() + v1.Aux = n + v1.AuxInt = off + config.PtrSize + v.AddArg(v1) + return true + } + goto end939d3f946bf61eb85b46b374e7afa9e9 +end939d3f946bf61eb85b46b374e7afa9e9: + ; + // match: (Arg {n} [off]) + // cond: v.Type.IsSlice() + // result: (SliceMake (Arg {n} [off]) (Arg {n} [off+config.PtrSize]) (Arg {n} [off+2*config.PtrSize])) + { + n := v.Aux + off := v.AuxInt + if !(v.Type.IsSlice()) { + goto endab4b93ad3b1cf55e5bf25d1fd9cd498e + } + v.Op = OpSliceMake + v.AuxInt = 0 + v.Aux = nil + v.resetArgs() + v0 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v0.Type = config.fe.TypeBytePtr() + v0.Aux = n + v0.AuxInt = off + v.AddArg(v0) + v1 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v1.Type = config.fe.TypeInt() + v1.Aux = n + v1.AuxInt = off + config.PtrSize + v.AddArg(v1) + v2 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v2.Type = config.fe.TypeInt() + v2.Aux = n + v2.AuxInt = off + 2*config.PtrSize + v.AddArg(v2) + return true + } + goto endab4b93ad3b1cf55e5bf25d1fd9cd498e +endab4b93ad3b1cf55e5bf25d1fd9cd498e: + ; + // match: (Arg {n} [off]) + // cond: v.Type.IsInterface() + // result: (IMake (Arg {n} [off]) (Arg {n} [off+config.PtrSize])) + { + n := v.Aux + off := v.AuxInt + if !(v.Type.IsInterface()) { + goto end851de8e588a39e81b4e2aef06566bf3e + } + v.Op = OpIMake + v.AuxInt = 0 + v.Aux = nil + v.resetArgs() + v0 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v0.Type = config.fe.TypeBytePtr() + v0.Aux = n + v0.AuxInt = off + v.AddArg(v0) + v1 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v1.Type = config.fe.TypeBytePtr() + v1.Aux = n + v1.AuxInt = off + config.PtrSize + v.AddArg(v1) + return true + } + goto end851de8e588a39e81b4e2aef06566bf3e +end851de8e588a39e81b4e2aef06566bf3e: + ; + // match: (Arg {n} [off]) + // cond: v.Type.IsComplex() && v.Type.Size() == 16 + // result: (ComplexMake (Arg {n} [off]) (Arg {n} [off+8])) + { + n := v.Aux + off := v.AuxInt + if !(v.Type.IsComplex() && v.Type.Size() == 16) { + goto end0988fc6a62c810b2f4976cb6cf44387f + } + v.Op = OpComplexMake + v.AuxInt = 0 + v.Aux = nil + v.resetArgs() + v0 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v0.Type = config.fe.TypeFloat64() + v0.Aux = n + v0.AuxInt = off + v.AddArg(v0) + v1 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v1.Type = config.fe.TypeFloat64() + v1.Aux = n + v1.AuxInt = off + 8 + v.AddArg(v1) + return true + } + goto end0988fc6a62c810b2f4976cb6cf44387f +end0988fc6a62c810b2f4976cb6cf44387f: + ; + // match: (Arg {n} [off]) + // cond: v.Type.IsComplex() && v.Type.Size() == 8 + // result: (ComplexMake (Arg {n} [off]) (Arg {n} [off+4])) + { + n := v.Aux + off := v.AuxInt + if !(v.Type.IsComplex() && v.Type.Size() == 8) { + goto enda348e93e0036873dd7089a2939c22e3e + } + v.Op = OpComplexMake + v.AuxInt = 0 + v.Aux = nil + v.resetArgs() + v0 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v0.Type = config.fe.TypeFloat32() + v0.Aux = n + v0.AuxInt = off + v.AddArg(v0) + v1 := b.NewValue0(v.Line, OpArg, TypeInvalid) + v1.Type = config.fe.TypeFloat32() + v1.Aux = n + v1.AuxInt = off + 4 + v.AddArg(v1) + return true + } + goto enda348e93e0036873dd7089a2939c22e3e +enda348e93e0036873dd7089a2939c22e3e: + ; + return false +} func rewriteValuegeneric_OpArrayIndex(v *Value, config *Config) bool { b := v.Block _ = b @@ -2115,13 +2267,13 @@ end1b106f89e0e3e26c613b957a7c98d8ad: ; // match: (Load ptr mem) // cond: t.IsString() - // result: (StringMake (Load ptr mem) (Load (OffPtr [config.PtrSize] ptr) mem)) + // result: (StringMake (Load ptr mem) (Load (OffPtr [config.PtrSize] ptr) mem)) { t := v.Type ptr := v.Args[0] mem := v.Args[1] if !(t.IsString()) { - goto end7c75255555bf9dd796298d9f6eaf9cf2 + goto enddd15a6f3d53a6ce7a19d4e181dd1c13a } v.Op = OpStringMake v.AuxInt = 0 @@ -2133,9 +2285,9 @@ end1b106f89e0e3e26c613b957a7c98d8ad: v0.AddArg(mem) v.AddArg(v0) v1 := b.NewValue0(v.Line, OpLoad, TypeInvalid) - v1.Type = config.fe.TypeUintptr() + v1.Type = config.fe.TypeInt() v2 := b.NewValue0(v.Line, OpOffPtr, TypeInvalid) - v2.Type = config.fe.TypeUintptr().PtrTo() + v2.Type = config.fe.TypeInt().PtrTo() v2.AuxInt = config.PtrSize v2.AddArg(ptr) v1.AddArg(v2) @@ -2143,18 +2295,18 @@ end1b106f89e0e3e26c613b957a7c98d8ad: v.AddArg(v1) return true } - goto end7c75255555bf9dd796298d9f6eaf9cf2 -end7c75255555bf9dd796298d9f6eaf9cf2: + goto enddd15a6f3d53a6ce7a19d4e181dd1c13a +enddd15a6f3d53a6ce7a19d4e181dd1c13a: ; // match: (Load ptr mem) // cond: t.IsSlice() - // result: (SliceMake (Load ptr mem) (Load (OffPtr [config.PtrSize] ptr) mem) (Load (OffPtr [2*config.PtrSize] ptr) mem)) + // result: (SliceMake (Load ptr mem) (Load (OffPtr [config.PtrSize] ptr) mem) (Load (OffPtr [2*config.PtrSize] ptr) mem)) { t := v.Type ptr := v.Args[0] mem := v.Args[1] if !(t.IsSlice()) { - goto end12c46556d962198680eb3238859e3016 + goto end65e8b0055aa7491b9b6066d9fe1b2c13 } v.Op = OpSliceMake v.AuxInt = 0 @@ -2166,18 +2318,18 @@ end7c75255555bf9dd796298d9f6eaf9cf2: v0.AddArg(mem) v.AddArg(v0) v1 := b.NewValue0(v.Line, OpLoad, TypeInvalid) - v1.Type = config.fe.TypeUintptr() + v1.Type = config.fe.TypeInt() v2 := b.NewValue0(v.Line, OpOffPtr, TypeInvalid) - v2.Type = config.fe.TypeUintptr().PtrTo() + v2.Type = config.fe.TypeInt().PtrTo() v2.AuxInt = config.PtrSize v2.AddArg(ptr) v1.AddArg(v2) v1.AddArg(mem) v.AddArg(v1) v3 := b.NewValue0(v.Line, OpLoad, TypeInvalid) - v3.Type = config.fe.TypeUintptr() + v3.Type = config.fe.TypeInt() v4 := b.NewValue0(v.Line, OpOffPtr, TypeInvalid) - v4.Type = config.fe.TypeUintptr().PtrTo() + v4.Type = config.fe.TypeInt().PtrTo() v4.AuxInt = 2 * config.PtrSize v4.AddArg(ptr) v3.AddArg(v4) @@ -2185,8 +2337,8 @@ end7c75255555bf9dd796298d9f6eaf9cf2: v.AddArg(v3) return true } - goto end12c46556d962198680eb3238859e3016 -end12c46556d962198680eb3238859e3016: + goto end65e8b0055aa7491b9b6066d9fe1b2c13 +end65e8b0055aa7491b9b6066d9fe1b2c13: ; // match: (Load ptr mem) // cond: t.IsInterface() @@ -2916,14 +3068,14 @@ end3851a482d7bd37a93c4d81581e85b3ab: ; // match: (Store [2*config.PtrSize] dst (StringMake ptr len) mem) // cond: - // result: (Store [config.PtrSize] (OffPtr [config.PtrSize] dst) len (Store [config.PtrSize] dst ptr mem)) + // result: (Store [config.PtrSize] (OffPtr [config.PtrSize] dst) len (Store [config.PtrSize] dst ptr mem)) { if v.AuxInt != 2*config.PtrSize { - goto end12abe4021d24e76ed56d64b18730bffb + goto endd3a6ecebdad5899570a79fe5c62f34f1 } dst := v.Args[0] if v.Args[1].Op != OpStringMake { - goto end12abe4021d24e76ed56d64b18730bffb + goto endd3a6ecebdad5899570a79fe5c62f34f1 } ptr := v.Args[1].Args[0] len := v.Args[1].Args[1] @@ -2934,7 +3086,7 @@ end3851a482d7bd37a93c4d81581e85b3ab: v.resetArgs() v.AuxInt = config.PtrSize v0 := b.NewValue0(v.Line, OpOffPtr, TypeInvalid) - v0.Type = config.fe.TypeUintptr().PtrTo() + v0.Type = config.fe.TypeInt().PtrTo() v0.AuxInt = config.PtrSize v0.AddArg(dst) v.AddArg(v0) @@ -2948,19 +3100,19 @@ end3851a482d7bd37a93c4d81581e85b3ab: v.AddArg(v1) return true } - goto end12abe4021d24e76ed56d64b18730bffb -end12abe4021d24e76ed56d64b18730bffb: + goto endd3a6ecebdad5899570a79fe5c62f34f1 +endd3a6ecebdad5899570a79fe5c62f34f1: ; // match: (Store [3*config.PtrSize] dst (SliceMake ptr len cap) mem) // cond: - // result: (Store [config.PtrSize] (OffPtr [2*config.PtrSize] dst) cap (Store [config.PtrSize] (OffPtr [config.PtrSize] dst) len (Store [config.PtrSize] dst ptr mem))) + // result: (Store [config.PtrSize] (OffPtr [2*config.PtrSize] dst) cap (Store [config.PtrSize] (OffPtr [config.PtrSize] dst) len (Store [config.PtrSize] dst ptr mem))) { if v.AuxInt != 3*config.PtrSize { - goto end7498d25e17db5398cf073a8590e35cc2 + goto endd5cc8c3dad7d24c845b0b88fc51487ae } dst := v.Args[0] if v.Args[1].Op != OpSliceMake { - goto end7498d25e17db5398cf073a8590e35cc2 + goto endd5cc8c3dad7d24c845b0b88fc51487ae } ptr := v.Args[1].Args[0] len := v.Args[1].Args[1] @@ -2972,7 +3124,7 @@ end12abe4021d24e76ed56d64b18730bffb: v.resetArgs() v.AuxInt = config.PtrSize v0 := b.NewValue0(v.Line, OpOffPtr, TypeInvalid) - v0.Type = config.fe.TypeUintptr().PtrTo() + v0.Type = config.fe.TypeInt().PtrTo() v0.AuxInt = 2 * config.PtrSize v0.AddArg(dst) v.AddArg(v0) @@ -2980,7 +3132,7 @@ end12abe4021d24e76ed56d64b18730bffb: v1 := b.NewValue0(v.Line, OpStore, TypeInvalid) v1.AuxInt = config.PtrSize v2 := b.NewValue0(v.Line, OpOffPtr, TypeInvalid) - v2.Type = config.fe.TypeUintptr().PtrTo() + v2.Type = config.fe.TypeInt().PtrTo() v2.AuxInt = config.PtrSize v2.AddArg(dst) v1.AddArg(v2) @@ -2996,8 +3148,8 @@ end12abe4021d24e76ed56d64b18730bffb: v.AddArg(v1) return true } - goto end7498d25e17db5398cf073a8590e35cc2 -end7498d25e17db5398cf073a8590e35cc2: + goto endd5cc8c3dad7d24c845b0b88fc51487ae +endd5cc8c3dad7d24c845b0b88fc51487ae: ; // match: (Store [2*config.PtrSize] dst (IMake itab data) mem) // cond: diff --git a/src/cmd/compile/internal/ssa/schedule_test.go b/src/cmd/compile/internal/ssa/schedule_test.go index 7f62ab9e3b..30c029ef7c 100644 --- a/src/cmd/compile/internal/ssa/schedule_test.go +++ b/src/cmd/compile/internal/ssa/schedule_test.go @@ -11,7 +11,7 @@ func TestSchedule(t *testing.T) { cases := []fun{ Fun(c, "entry", Bloc("entry", - Valu("mem0", OpArg, TypeMem, 0, ".mem"), + Valu("mem0", OpInitMem, TypeMem, 0, ".mem"), Valu("ptr", OpConst64, TypeInt64, 0xABCD, nil), Valu("v", OpConst64, TypeInt64, 12, nil), Valu("mem1", OpStore, TypeMem, 8, nil, "ptr", "v", "mem0"), diff --git a/src/cmd/compile/internal/ssa/shift_test.go b/src/cmd/compile/internal/ssa/shift_test.go index 611b418b6d..9e7f0585a6 100644 --- a/src/cmd/compile/internal/ssa/shift_test.go +++ b/src/cmd/compile/internal/ssa/shift_test.go @@ -28,7 +28,7 @@ func makeConstShiftFunc(c *Config, amount int64, op Op, typ Type) fun { ptyp := &TypeImpl{Size_: 8, Ptr: true, Name: "ptr"} fun := Fun(c, "entry", Bloc("entry", - Valu("mem", OpArg, TypeMem, 0, ".mem"), + Valu("mem", OpInitMem, TypeMem, 0, ".mem"), Valu("SP", OpSP, TypeUInt64, 0, nil), Valu("argptr", OpOffPtr, ptyp, 8, nil, "SP"), Valu("resptr", OpOffPtr, ptyp, 16, nil, "SP"), diff --git a/src/cmd/compile/internal/ssa/stackalloc.go b/src/cmd/compile/internal/ssa/stackalloc.go index 793162a797..3eb5c3cf4a 100644 --- a/src/cmd/compile/internal/ssa/stackalloc.go +++ b/src/cmd/compile/internal/ssa/stackalloc.go @@ -44,6 +44,13 @@ func stackalloc(f *Func) { } case v.Op == OpLoadReg: s.add(v.Args[0].ID) + case v.Op == OpArg: + // This is an input argument which is pre-spilled. It is kind of + // like a StoreReg, but we don't remove v.ID here because we want + // this value to appear live even before this point. Being live + // all the way to the start of the entry block prevents other + // values from being allocated to the same slot and clobbering + // the input value before we have a chance to load it. } } } @@ -51,7 +58,7 @@ func stackalloc(f *Func) { // Build map from values to their names, if any. // A value may be associated with more than one name (e.g. after // the assignment i=j). This step picks one name per value arbitrarily. - names := make([]GCNode, f.NumValues()) + names := make([]LocalSlot, f.NumValues()) for _, name := range f.Names { // Note: not "range f.NamedValues" above, because // that would be nondeterministic. @@ -74,9 +81,17 @@ func stackalloc(f *Func) { } } + // Allocate args to their assigned locations. + for _, v := range f.Entry.Values { + if v.Op != OpArg { + continue + } + f.setHome(v, LocalSlot{v.Aux.(GCNode), v.Type, v.AuxInt}) + } + // For each type, we keep track of all the stack slots we // have allocated for that type. - locations := map[Type][]*LocalSlot{} + locations := map[Type][]LocalSlot{} // Each time we assign a stack slot to a value v, we remember // the slot we used via an index into locations[v.Type]. @@ -99,16 +114,16 @@ func stackalloc(f *Func) { // If this is a named value, try to use the name as // the spill location. - var name GCNode + var name LocalSlot if v.Op == OpStoreReg { name = names[v.Args[0].ID] } else { name = names[v.ID] } - if name != nil && v.Type.Equal(name.Typ()) { + if name.N != nil && v.Type.Equal(name.Type) { for _, id := range interfere[v.ID] { h := f.getHome(id) - if h != nil && h.(*LocalSlot).N == name { + if h != nil && h.(LocalSlot) == name { // A variable can interfere with itself. // It is rare, but but it can happen. goto noname @@ -118,17 +133,16 @@ func stackalloc(f *Func) { for _, a := range v.Args { for _, id := range interfere[a.ID] { h := f.getHome(id) - if h != nil && h.(*LocalSlot).N == name { + if h != nil && h.(LocalSlot) == name { goto noname } } } } - loc := &LocalSlot{name} - f.setHome(v, loc) + f.setHome(v, name) if v.Op == OpPhi { for _, a := range v.Args { - f.setHome(a, loc) + f.setHome(a, name) } } continue @@ -169,7 +183,7 @@ func stackalloc(f *Func) { } // If there is no unused stack slot, allocate a new one. if i == len(locs) { - locs = append(locs, &LocalSlot{f.Config.fe.Auto(v.Type)}) + locs = append(locs, LocalSlot{N: f.Config.fe.Auto(v.Type), Type: v.Type, Off: 0}) locations[v.Type] = locs } // Use the stack variable at that index for v. diff --git a/src/cmd/compile/internal/ssa/tighten.go b/src/cmd/compile/internal/ssa/tighten.go index 4fa26d2d18..6726c06e76 100644 --- a/src/cmd/compile/internal/ssa/tighten.go +++ b/src/cmd/compile/internal/ssa/tighten.go @@ -54,8 +54,8 @@ func tighten(f *Func) { for _, b := range f.Blocks { for i := 0; i < len(b.Values); i++ { v := b.Values[i] - if v.Op == OpPhi || v.Op == OpGetClosurePtr || v.Op == OpConvert { - // GetClosurePtr must stay in entry block. + if v.Op == OpPhi || v.Op == OpGetClosurePtr || v.Op == OpConvert || v.Op == OpArg { + // GetClosurePtr & Arg must stay in entry block. // OpConvert must not float over call sites. // TODO do we instead need a dependence edge of some sort for OpConvert? // Would memory do the trick, or do we need something else that relates diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 2843633ee1..48567f1b9c 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -94,9 +94,6 @@ func TestGdbPython(t *testing.T) { "-ex", "echo END\n", "-ex", "echo BEGIN print strvar\n", "-ex", "print strvar", - "-ex", "echo END\n", - "-ex", "echo BEGIN print ptrvar\n", - "-ex", "print ptrvar", "-ex", "echo END\n"} // without framepointer, gdb cannot backtrace our non-standard @@ -151,10 +148,6 @@ func TestGdbPython(t *testing.T) { t.Fatalf("print strvar failed: %s", bl) } - if bl := blocks["print ptrvar"]; !strVarRe.MatchString(bl) { - t.Fatalf("print ptrvar failed: %s", bl) - } - btGoroutineRe := regexp.MustCompile(`^#0\s+runtime.+at`) if bl := blocks["goroutine 2 bt"]; canBackTrace && !btGoroutineRe.MatchString(bl) { t.Fatalf("goroutine 2 bt failed: %s", bl) -- cgit v1.3 From 594c3aa063befecb6bd51a191781e44d1cd1b566 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 14 Jan 2016 15:28:26 -0800 Subject: [dev.ssa] runtime: fix windows build Pull the relevant part of https://go-review.googlesource.com/#/c/18304/ into the SSA branch. This fixes the windows SSA build. Change-Id: Iee3834d8e7019cd90307e32029f3d1032ceb46c1 Reviewed-on: https://go-review.googlesource.com/18673 Reviewed-by: Austin Clements --- src/runtime/sys_windows_amd64.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index b15eacbf32..1ca1dbb591 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -243,7 +243,7 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT,$0 LEAQ g__size(SP), BX MOVQ BX, g_m(SP) - LEAQ -8192(SP), CX + LEAQ -32768(SP), CX MOVQ CX, (g_stack+stack_lo)(SP) ADDQ $const__StackGuard, CX MOVQ CX, g_stackguard0(SP) -- cgit v1.3 From 5ba31940dc2934dc76a47f0e614d441225ea3a95 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 25 Jan 2016 17:06:54 -0800 Subject: [dev.ssa] cmd/compile: fix write barriers for SSA The old write barriers used _nostore versions, which don't work for Ian's cgo checker. Instead, we adopt the same write barrier pattern as the default compiler. It's a bit trickier to code up but should be more efficient. Change-Id: I6696c3656cf179e28f800b0e096b7259bd5f3bb7 Reviewed-on: https://go-review.googlesource.com/18941 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: David Chase --- misc/cgo/errors/ptr.go | 208 ++++++++++--------- src/cmd/compile/internal/gc/builtin.go | 1 - src/cmd/compile/internal/gc/builtin/runtime.go | 1 - src/cmd/compile/internal/gc/go.go | 3 +- src/cmd/compile/internal/gc/pgen.go | 3 +- src/cmd/compile/internal/gc/ssa.go | 268 +++++++++++++++++++------ src/runtime/mbarrier.go | 8 - 7 files changed, 318 insertions(+), 174 deletions(-) (limited to 'src/runtime') diff --git a/misc/cgo/errors/ptr.go b/misc/cgo/errors/ptr.go index a0e3e066b4..0dd291f5ed 100644 --- a/misc/cgo/errors/ptr.go +++ b/misc/cgo/errors/ptr.go @@ -134,111 +134,109 @@ var ptrTests = []ptrTest{ body: `parg := [1]**C.char{&hello[0]}; C.f(&parg[0])`, fail: true, }, - /* - { - // Storing a Go pointer into C memory should fail. - name: "barrier", - c: `#include - char **f1() { return malloc(sizeof(char*)); } - void f2(char **p) {}`, - body: `p := C.f1(); *p = new(C.char); C.f2(p)`, - fail: true, - expensive: true, - }, - { - // Storing a Go pointer into C memory by assigning a - // large value should fail. - name: "barrier-struct", - c: `#include - struct s { char *a[10]; }; - struct s *f1() { return malloc(sizeof(struct s)); } - void f2(struct s *p) {}`, - body: `p := C.f1(); p.a = [10]*C.char{new(C.char)}; C.f2(p)`, - fail: true, - expensive: true, - }, - { - // Storing a Go pointer into C memory using a slice - // copy should fail. - name: "barrier-slice", - c: `#include - struct s { char *a[10]; }; - struct s *f1() { return malloc(sizeof(struct s)); } - void f2(struct s *p) {}`, - body: `p := C.f1(); copy(p.a[:], []*C.char{new(C.char)}); C.f2(p)`, - fail: true, - expensive: true, - }, - { - // A very large value uses a GC program, which is a - // different code path. - name: "barrier-gcprog-array", - c: `#include - struct s { char *a[32769]; }; - struct s *f1() { return malloc(sizeof(struct s)); } - void f2(struct s *p) {}`, - body: `p := C.f1(); p.a = [32769]*C.char{new(C.char)}; C.f2(p)`, - fail: true, - expensive: true, - }, - { - // Similar case, with a source on the heap. - name: "barrier-gcprog-array-heap", - c: `#include - struct s { char *a[32769]; }; - struct s *f1() { return malloc(sizeof(struct s)); } - void f2(struct s *p) {} - void f3(void *p) {}`, - imports: []string{"unsafe"}, - body: `p := C.f1(); n := &[32769]*C.char{new(C.char)}; p.a = *n; C.f2(p); n[0] = nil; C.f3(unsafe.Pointer(n))`, - fail: true, - expensive: true, - }, - { - // A GC program with a struct. - name: "barrier-gcprog-struct", - c: `#include - struct s { char *a[32769]; }; - struct s2 { struct s f; }; - struct s2 *f1() { return malloc(sizeof(struct s2)); } - void f2(struct s2 *p) {}`, - body: `p := C.f1(); p.f = C.struct_s{[32769]*C.char{new(C.char)}}; C.f2(p)`, - fail: true, - expensive: true, - }, - { - // Similar case, with a source on the heap. - name: "barrier-gcprog-struct-heap", - c: `#include - struct s { char *a[32769]; }; - struct s2 { struct s f; }; - struct s2 *f1() { return malloc(sizeof(struct s2)); } - void f2(struct s2 *p) {} - void f3(void *p) {}`, - imports: []string{"unsafe"}, - body: `p := C.f1(); n := &C.struct_s{[32769]*C.char{new(C.char)}}; p.f = *n; C.f2(p); n.a[0] = nil; C.f3(unsafe.Pointer(n))`, - fail: true, - expensive: true, - }, - { - // Exported functions may not return Go pointers. - name: "export1", - c: `extern unsigned char *GoFn();`, - support: `//export GoFn - func GoFn() *byte { return new(byte) }`, - body: `C.GoFn()`, - fail: true, - }, - { - // Returning a C pointer is fine. - name: "exportok", - c: `#include - extern unsigned char *GoFn();`, - support: `//export GoFn - func GoFn() *byte { return (*byte)(C.malloc(1)) }`, - body: `C.GoFn()`, - }, - */ + { + // Storing a Go pointer into C memory should fail. + name: "barrier", + c: `#include + char **f1() { return malloc(sizeof(char*)); } + void f2(char **p) {}`, + body: `p := C.f1(); *p = new(C.char); C.f2(p)`, + fail: true, + expensive: true, + }, + { + // Storing a Go pointer into C memory by assigning a + // large value should fail. + name: "barrier-struct", + c: `#include + struct s { char *a[10]; }; + struct s *f1() { return malloc(sizeof(struct s)); } + void f2(struct s *p) {}`, + body: `p := C.f1(); p.a = [10]*C.char{new(C.char)}; C.f2(p)`, + fail: true, + expensive: true, + }, + { + // Storing a Go pointer into C memory using a slice + // copy should fail. + name: "barrier-slice", + c: `#include + struct s { char *a[10]; }; + struct s *f1() { return malloc(sizeof(struct s)); } + void f2(struct s *p) {}`, + body: `p := C.f1(); copy(p.a[:], []*C.char{new(C.char)}); C.f2(p)`, + fail: true, + expensive: true, + }, + { + // A very large value uses a GC program, which is a + // different code path. + name: "barrier-gcprog-array", + c: `#include + struct s { char *a[32769]; }; + struct s *f1() { return malloc(sizeof(struct s)); } + void f2(struct s *p) {}`, + body: `p := C.f1(); p.a = [32769]*C.char{new(C.char)}; C.f2(p)`, + fail: true, + expensive: true, + }, + { + // Similar case, with a source on the heap. + name: "barrier-gcprog-array-heap", + c: `#include + struct s { char *a[32769]; }; + struct s *f1() { return malloc(sizeof(struct s)); } + void f2(struct s *p) {} + void f3(void *p) {}`, + imports: []string{"unsafe"}, + body: `p := C.f1(); n := &[32769]*C.char{new(C.char)}; p.a = *n; C.f2(p); n[0] = nil; C.f3(unsafe.Pointer(n))`, + fail: true, + expensive: true, + }, + { + // A GC program with a struct. + name: "barrier-gcprog-struct", + c: `#include + struct s { char *a[32769]; }; + struct s2 { struct s f; }; + struct s2 *f1() { return malloc(sizeof(struct s2)); } + void f2(struct s2 *p) {}`, + body: `p := C.f1(); p.f = C.struct_s{[32769]*C.char{new(C.char)}}; C.f2(p)`, + fail: true, + expensive: true, + }, + { + // Similar case, with a source on the heap. + name: "barrier-gcprog-struct-heap", + c: `#include + struct s { char *a[32769]; }; + struct s2 { struct s f; }; + struct s2 *f1() { return malloc(sizeof(struct s2)); } + void f2(struct s2 *p) {} + void f3(void *p) {}`, + imports: []string{"unsafe"}, + body: `p := C.f1(); n := &C.struct_s{[32769]*C.char{new(C.char)}}; p.f = *n; C.f2(p); n.a[0] = nil; C.f3(unsafe.Pointer(n))`, + fail: true, + expensive: true, + }, + { + // Exported functions may not return Go pointers. + name: "export1", + c: `extern unsigned char *GoFn();`, + support: `//export GoFn + func GoFn() *byte { return new(byte) }`, + body: `C.GoFn()`, + fail: true, + }, + { + // Returning a C pointer is fine. + name: "exportok", + c: `#include + extern unsigned char *GoFn();`, + support: `//export GoFn + func GoFn() *byte { return (*byte)(C.malloc(1)) }`, + body: `C.GoFn()`, + }, } func main() { diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index 568ffdd4fd..7f2e80b52f 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -117,7 +117,6 @@ const runtimeimport = "" + "func @\"\".writebarrierfat1110 (@\"\".dst·1 *any, _ uintptr, @\"\".src·3 any)\n" + "func @\"\".writebarrierfat1111 (@\"\".dst·1 *any, _ uintptr, @\"\".src·3 any)\n" + "func @\"\".typedmemmove (@\"\".typ·1 *byte, @\"\".dst·2 *any, @\"\".src·3 *any)\n" + - "func @\"\".typedmemmove_nostore (@\"\".typ·1 *byte, @\"\".dst·2 *any)\n" + "func @\"\".typedslicecopy (@\"\".typ·2 *byte, @\"\".dst·3 any, @\"\".src·4 any) (? int)\n" + "func @\"\".selectnbsend (@\"\".chanType·2 *byte, @\"\".hchan·3 chan<- any, @\"\".elem·4 *any) (? bool)\n" + "func @\"\".selectnbrecv (@\"\".chanType·2 *byte, @\"\".elem·3 *any, @\"\".hchan·4 <-chan any) (? bool)\n" + diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index 07a0c31650..70663eeee4 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -151,7 +151,6 @@ func writebarrierfat1111(dst *any, _ uintptr, src any) // *byte is really *runtime.Type func typedmemmove(typ *byte, dst *any, src *any) -func typedmemmove_nostore(typ *byte, dst *any) func typedslicecopy(typ *byte, dst any, src any) int func selectnbsend(chanType *byte, hchan chan<- any, elem *any) bool diff --git a/src/cmd/compile/internal/gc/go.go b/src/cmd/compile/internal/gc/go.go index fc7a78a87c..08442a415b 100644 --- a/src/cmd/compile/internal/gc/go.go +++ b/src/cmd/compile/internal/gc/go.go @@ -868,6 +868,7 @@ var throwreturn *Node var growslice *Node -var typedmemmove_nostore *Node +var writebarrierptr *Node +var typedmemmove *Node var panicdottype *Node diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go index 6f5913406e..f90f89a805 100644 --- a/src/cmd/compile/internal/gc/pgen.go +++ b/src/cmd/compile/internal/gc/pgen.go @@ -353,7 +353,8 @@ func compile(fn *Node) { panicdivide = Sysfunc("panicdivide") throwreturn = Sysfunc("throwreturn") growslice = Sysfunc("growslice") - typedmemmove_nostore = Sysfunc("typedmemmove_nostore") + writebarrierptr = Sysfunc("writebarrierptr") + typedmemmove = Sysfunc("typedmemmove") panicdottype = Sysfunc("panicdottype") } diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 6bdf0c69e0..a05e33196a 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -550,8 +550,8 @@ func (s *state) stmt(n *Node) { case OAS2DOTTYPE: res, resok := s.dottype(n.Rlist.N, true) - s.assign(n.List.N, res, false, n.Lineno) - s.assign(n.List.Next.N, resok, false, n.Lineno) + s.assign(n.List.N, res, false, false, n.Lineno) + s.assign(n.List.Next.N, resok, false, false, n.Lineno) return case ODCL: @@ -572,7 +572,7 @@ func (s *state) stmt(n *Node) { prealloc[n.Left] = palloc } r := s.expr(palloc) - s.assign(n.Left.Name.Heapaddr, r, false, n.Lineno) + s.assign(n.Left.Name.Heapaddr, r, false, false, n.Lineno) case OLABEL: sym := n.Left.Sym @@ -641,30 +641,52 @@ func (s *state) stmt(n *Node) { s.f.StaticData = append(data, n) return } - var r *ssa.Value + + var t *Type if n.Right != nil { - if n.Right.Op == OSTRUCTLIT || n.Right.Op == OARRAYLIT { - // All literals with nonzero fields have already been - // rewritten during walk. Any that remain are just T{} - // or equivalents. Leave r = nil to get zeroing behavior. - if !iszero(n.Right) { - Fatalf("literal with nonzero value in SSA: %v", n.Right) - } + t = n.Right.Type + } else { + t = n.Left.Type + } + + // Evaluate RHS. + rhs := n.Right + if rhs != nil && (rhs.Op == OSTRUCTLIT || rhs.Op == OARRAYLIT) { + // All literals with nonzero fields have already been + // rewritten during walk. Any that remain are just T{} + // or equivalents. Use the zero value. + if !iszero(rhs) { + Fatalf("literal with nonzero value in SSA: %v", rhs) + } + rhs = nil + } + var r *ssa.Value + needwb := n.Op == OASWB && rhs != nil + deref := !canSSAType(t) + if deref { + if rhs == nil { + r = nil // Signal assign to use OpZero. + } else { + r = s.addr(rhs, false) + } + } else { + if rhs == nil { + r = s.zeroVal(t) } else { - r = s.expr(n.Right) + r = s.expr(rhs) } } - if n.Right != nil && n.Right.Op == OAPPEND { + if rhs != nil && rhs.Op == OAPPEND { // Yuck! The frontend gets rid of the write barrier, but we need it! // At least, we need it in the case where growslice is called. // TODO: Do the write barrier on just the growslice branch. // TODO: just add a ptr graying to the end of growslice? // TODO: check whether we need to do this for ODOTTYPE and ORECV also. // They get similar wb-removal treatment in walk.go:OAS. - s.assign(n.Left, r, true, n.Lineno) - return + needwb = true } - s.assign(n.Left, r, n.Op == OASWB, n.Lineno) + + s.assign(n.Left, r, needwb, deref, n.Lineno) case OIF: bThen := s.f.NewBlock(ssa.BlockPlain) @@ -1939,7 +1961,8 @@ func (s *state) expr(n *Node) *ssa.Value { return s.newValue3(ssa.OpSliceMake, n.Type, p, l, c) case OCALLFUNC, OCALLINTER, OCALLMETH: - return s.call(n, callNormal) + a := s.call(n, callNormal) + return s.newValue2(ssa.OpLoad, n.Type, a, s.mem()) case OGETG: return s.newValue1(ssa.OpGetG, n.Type, s.mem()) @@ -2014,17 +2037,22 @@ func (s *state) expr(n *Node) *ssa.Value { p = s.variable(&ptrVar, pt) // generates phi for ptr c = s.variable(&capVar, Types[TINT]) // generates phi for cap p2 := s.newValue2(ssa.OpPtrIndex, pt, p, l) + // TODO: just one write barrier call for all of these writes? + // TODO: maybe just one writeBarrier.enabled check? for i, arg := range args { addr := s.newValue2(ssa.OpPtrIndex, pt, p2, s.constInt(Types[TINT], int64(i))) if store[i] { - s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, et.Size(), addr, arg, s.mem()) + if haspointers(et) { + s.insertWBstore(et, addr, arg, n.Lineno) + } else { + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, et.Size(), addr, arg, s.mem()) + } } else { - s.vars[&memVar] = s.newValue3I(ssa.OpMove, ssa.TypeMem, et.Size(), addr, arg, s.mem()) - } - if haspointers(et) { - // TODO: just one write barrier call for all of these writes? - // TODO: maybe just one writeBarrier.enabled check? - s.insertWB(et, addr, n.Lineno) + if haspointers(et) { + s.insertWBmove(et, addr, arg, n.Lineno) + } else { + s.vars[&memVar] = s.newValue3I(ssa.OpMove, ssa.TypeMem, et.Size(), addr, arg, s.mem()) + } } } @@ -2083,26 +2111,21 @@ func (s *state) condBranch(cond *Node, yes, no *ssa.Block, likely int8) { b.AddEdgeTo(no) } -func (s *state) assign(left *Node, right *ssa.Value, wb bool, line int32) { +// assign does left = right. +// Right has already been evaluated to ssa, left has not. +// If deref is true, then we do left = *right instead (and right has already been nil-checked). +// If deref is true and right == nil, just do left = 0. +// Include a write barrier if wb is true. +func (s *state) assign(left *Node, right *ssa.Value, wb, deref bool, line int32) { if left.Op == ONAME && isblank(left) { return } t := left.Type dowidth(t) - if right == nil { - // right == nil means use the zero value of the assigned type. - if !canSSA(left) { - // if we can't ssa this memory, treat it as just zeroing out the backing memory - addr := s.addr(left, false) - if left.Op == ONAME { - s.vars[&memVar] = s.newValue1A(ssa.OpVarDef, ssa.TypeMem, left, s.mem()) - } - s.vars[&memVar] = s.newValue2I(ssa.OpZero, ssa.TypeMem, t.Size(), addr, s.mem()) - return - } - right = s.zeroVal(t) - } if canSSA(left) { + if deref { + s.Fatalf("can SSA LHS %s but not RHS %s", left, right) + } if left.Op == ODOT { // We're assigning to a field of an ssa-able value. // We need to build a new structure with the new value for the @@ -2134,7 +2157,7 @@ func (s *state) assign(left *Node, right *ssa.Value, wb bool, line int32) { } // Recursively assign the new value we've made to the base of the dot op. - s.assign(left.Left, new, false, line) + s.assign(left.Left, new, false, false, line) // TODO: do we need to update named values here? return } @@ -2143,15 +2166,30 @@ func (s *state) assign(left *Node, right *ssa.Value, wb bool, line int32) { s.addNamedValue(left, right) return } - // not ssa-able. Treat as a store. + // Left is not ssa-able. Compute its address. addr := s.addr(left, false) if left.Op == ONAME { s.vars[&memVar] = s.newValue1A(ssa.OpVarDef, ssa.TypeMem, left, s.mem()) } - s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, t.Size(), addr, right, s.mem()) + if deref { + // Treat as a mem->mem move. + if right == nil { + s.vars[&memVar] = s.newValue2I(ssa.OpZero, ssa.TypeMem, t.Size(), addr, s.mem()) + return + } + if wb { + s.insertWBmove(t, addr, right, line) + return + } + s.vars[&memVar] = s.newValue3I(ssa.OpMove, ssa.TypeMem, t.Size(), addr, right, s.mem()) + return + } + // Treat as a store. if wb { - s.insertWB(left.Type, addr, line) + s.insertWBstore(t, addr, right, line) + return } + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, t.Size(), addr, right, s.mem()) } // zeroVal returns the zero value for type t. @@ -2221,6 +2259,8 @@ const ( callGo ) +// Calls the function n using the specified call type. +// Returns the address of the return value (or nil if none). func (s *state) call(n *Node, k callKind) *ssa.Value { var sym *Sym // target symbol (if static) var closure *ssa.Value // ptr to closure to run (if dynamic) @@ -2234,9 +2274,6 @@ func (s *state) call(n *Node, k callKind) *ssa.Value { break } closure = s.expr(fn) - if closure == nil { - return nil // TODO: remove when expr always returns non-nil - } case OCALLMETH: if fn.Op != ODOTMETH { Fatalf("OCALLMETH: n.Left not an ODOTMETH: %v", fn) @@ -2324,7 +2361,7 @@ func (s *state) call(n *Node, k callKind) *ssa.Value { b.Control = call b.AddEdgeTo(bNext) - // Read result from stack at the start of the fallthrough block + // Start exit block, find address of result. s.startBlock(bNext) var titer Iter fp := Structfirst(&titer, Getoutarg(n.Left.Type)) @@ -2332,8 +2369,7 @@ func (s *state) call(n *Node, k callKind) *ssa.Value { // call has no return value. Continue with the next statement. return nil } - a := s.entryNewValue1I(ssa.OpOffPtr, Ptrto(fp.Type), fp.Width, s.sp) - return s.newValue2(ssa.OpLoad, fp.Type, a, call) + return s.entryNewValue1I(ssa.OpOffPtr, Ptrto(fp.Type), fp.Width, s.sp) } // etypesign returns the signed-ness of e, for integer/pointer etypes. @@ -2483,6 +2519,8 @@ func (s *state) addr(n *Node, bounded bool) *ssa.Value { case OCONVNOP: addr := s.addr(n.Left, bounded) return s.newValue1(ssa.OpCopy, t, addr) // ensure that addr has the right type + case OCALLFUNC, OCALLINTER, OCALLMETH: + return s.call(n, callNormal) default: s.Unimplementedf("unhandled addr %v", Oconv(int(n.Op), 0)) @@ -2682,15 +2720,17 @@ func (s *state) rtcall(fn *Node, returns bool, results []*Type, args ...*ssa.Val return res } -// insertWB inserts a write barrier. A value of type t has already -// been stored at location p. Tell the runtime about this write. -// Note: there must be no GC suspension points between the write and -// the call that this function inserts. -func (s *state) insertWB(t *Type, p *ssa.Value, line int32) { +// insertWBmove inserts the assignment *left = *right including a write barrier. +// t is the type being assigned. +func (s *state) insertWBmove(t *Type, left, right *ssa.Value, line int32) { // if writeBarrier.enabled { - // typedmemmove_nostore(&t, p) + // typedmemmove(&t, left, right) + // } else { + // *left = *right // } bThen := s.f.NewBlock(ssa.BlockPlain) + bElse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) aux := &ssa.ExternSymbol{Types[TBOOL], syslook("writeBarrier", 0).Sym} flagaddr := s.newValue1A(ssa.OpAddr, Ptrto(Types[TBOOL]), aux, s.sb) @@ -2701,17 +2741,131 @@ func (s *state) insertWB(t *Type, p *ssa.Value, line int32) { b.Likely = ssa.BranchUnlikely b.Control = flag b.AddEdgeTo(bThen) + b.AddEdgeTo(bElse) s.startBlock(bThen) - // TODO: writebarrierptr_nostore if just one pointer word (or a few?) taddr := s.newValue1A(ssa.OpAddr, Types[TUINTPTR], &ssa.ExternSymbol{Types[TUINTPTR], typenamesym(t)}, s.sb) - s.rtcall(typedmemmove_nostore, true, nil, taddr, p) + s.rtcall(typedmemmove, true, nil, taddr, left, right) + s.endBlock().AddEdgeTo(bEnd) + + s.startBlock(bElse) + s.vars[&memVar] = s.newValue3I(ssa.OpMove, ssa.TypeMem, t.Size(), left, right, s.mem()) + s.endBlock().AddEdgeTo(bEnd) + + s.startBlock(bEnd) if Debug_wb > 0 { Warnl(int(line), "write barrier") } +} + +// insertWBstore inserts the assignment *left = right including a write barrier. +// t is the type being assigned. +func (s *state) insertWBstore(t *Type, left, right *ssa.Value, line int32) { + // store scalar fields + // if writeBarrier.enabled { + // writebarrierptr for pointer fields + // } else { + // store pointer fields + // } - b.AddEdgeTo(s.curBlock) + if t.IsStruct() { + n := t.NumFields() + for i := int64(0); i < n; i++ { + ft := t.FieldType(i) + addr := s.newValue1I(ssa.OpOffPtr, ft.PtrTo(), t.FieldOff(i), left) + val := s.newValue1I(ssa.OpStructSelect, ft, i, right) + if haspointers(ft.(*Type)) { + s.insertWBstore(ft.(*Type), addr, val, line) + } else { + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, ft.Size(), addr, val, s.mem()) + } + } + return + } + + switch { + case t.IsPtr() || t.IsMap() || t.IsChan(): + // no scalar fields. + case t.IsString(): + len := s.newValue1(ssa.OpStringLen, Types[TINT], right) + lenAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), s.config.IntSize, left) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, lenAddr, len, s.mem()) + case t.IsSlice(): + len := s.newValue1(ssa.OpSliceLen, Types[TINT], right) + cap := s.newValue1(ssa.OpSliceCap, Types[TINT], right) + lenAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), s.config.IntSize, left) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, lenAddr, len, s.mem()) + capAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TINT]), 2*s.config.IntSize, left) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, capAddr, cap, s.mem()) + case t.IsInterface(): + // itab field doesn't need a write barrier (even though it is a pointer). + itab := s.newValue1(ssa.OpITab, Ptrto(Types[TUINT8]), right) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.IntSize, left, itab, s.mem()) + default: + s.Fatalf("bad write barrier type %s", t) + } + + bThen := s.f.NewBlock(ssa.BlockPlain) + bElse := s.f.NewBlock(ssa.BlockPlain) + bEnd := s.f.NewBlock(ssa.BlockPlain) + + aux := &ssa.ExternSymbol{Types[TBOOL], syslook("writeBarrier", 0).Sym} + flagaddr := s.newValue1A(ssa.OpAddr, Ptrto(Types[TBOOL]), aux, s.sb) + // TODO: select the .enabled field. It is currently first, so not needed for now. + flag := s.newValue2(ssa.OpLoad, Types[TBOOL], flagaddr, s.mem()) + b := s.endBlock() + b.Kind = ssa.BlockIf + b.Likely = ssa.BranchUnlikely + b.Control = flag + b.AddEdgeTo(bThen) + b.AddEdgeTo(bElse) + + // Issue write barriers for pointer writes. + s.startBlock(bThen) + switch { + case t.IsPtr() || t.IsMap() || t.IsChan(): + s.rtcall(writebarrierptr, true, nil, left, right) + case t.IsString(): + ptr := s.newValue1(ssa.OpStringPtr, Ptrto(Types[TUINT8]), right) + s.rtcall(writebarrierptr, true, nil, left, ptr) + case t.IsSlice(): + ptr := s.newValue1(ssa.OpSlicePtr, Ptrto(Types[TUINT8]), right) + s.rtcall(writebarrierptr, true, nil, left, ptr) + case t.IsInterface(): + idata := s.newValue1(ssa.OpIData, Ptrto(Types[TUINT8]), right) + idataAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TUINT8]), s.config.PtrSize, left) + s.rtcall(writebarrierptr, true, nil, idataAddr, idata) + default: + s.Fatalf("bad write barrier type %s", t) + } + s.endBlock().AddEdgeTo(bEnd) + + // Issue regular stores for pointer writes. + s.startBlock(bElse) + switch { + case t.IsPtr() || t.IsMap() || t.IsChan(): + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, right, s.mem()) + case t.IsString(): + ptr := s.newValue1(ssa.OpStringPtr, Ptrto(Types[TUINT8]), right) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, ptr, s.mem()) + case t.IsSlice(): + ptr := s.newValue1(ssa.OpSlicePtr, Ptrto(Types[TUINT8]), right) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, left, ptr, s.mem()) + case t.IsInterface(): + idata := s.newValue1(ssa.OpIData, Ptrto(Types[TUINT8]), right) + idataAddr := s.newValue1I(ssa.OpOffPtr, Ptrto(Types[TUINT8]), s.config.PtrSize, left) + s.vars[&memVar] = s.newValue3I(ssa.OpStore, ssa.TypeMem, s.config.PtrSize, idataAddr, idata, s.mem()) + default: + s.Fatalf("bad write barrier type %s", t) + } + s.endBlock().AddEdgeTo(bEnd) + + s.startBlock(bEnd) + + if Debug_wb > 0 { + Warnl(int(line), "write barrier") + } } // slice computes the slice v[i:j:k] and returns ptr, len, and cap of result. diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index 03011d2400..45086c43cd 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -197,14 +197,6 @@ func typedmemmove(typ *_type, dst, src unsafe.Pointer) { heapBitsBulkBarrier(uintptr(dst), typ.size) } -//go:nosplit -func typedmemmove_nostore(typ *_type, dst unsafe.Pointer) { - if typ.kind&kindNoPointers != 0 { - return - } - heapBitsBulkBarrier(uintptr(dst), typ.size) -} - //go:linkname reflect_typedmemmove reflect.typedmemmove func reflect_typedmemmove(typ *_type, dst, src unsafe.Pointer) { typedmemmove(typ, dst, src) -- cgit v1.3 From ae276d8c2342aff7b9bdf6563ffac5d21da70db6 Mon Sep 17 00:00:00 2001 From: David Chase Date: Thu, 18 Feb 2016 17:49:45 -0500 Subject: [dev.ssa] cmd/compile: reenable TestStackBarrierProfiling Tested it 1000x on OS X and Linux amd64, no failures. Updated TODO. Change-Id: Ia60c8d90962f6e5f7c3ed1ded6ba1b25eee983e1 Reviewed-on: https://go-review.googlesource.com/19662 Reviewed-by: Todd Neal --- src/cmd/compile/internal/ssa/TODO | 1 - src/runtime/pprof/pprof_test.go | 5 ----- 2 files changed, 6 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/ssa/TODO b/src/cmd/compile/internal/ssa/TODO index 5fa14ee44b..5e5cb4b865 100644 --- a/src/cmd/compile/internal/ssa/TODO +++ b/src/cmd/compile/internal/ssa/TODO @@ -7,7 +7,6 @@ Coverage Correctness ----------- - Debugging info (check & fix as much as we can) -- Re-enable TestStackBarrierProfiling (src/runtime/pprof/pprof_test.go) - @ directive in rewrites might read overwritten data. Save @loc in variable before modifying v. diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index f7c1a46805..ab6b1835c5 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -375,11 +375,6 @@ func TestStackBarrierProfiling(t *testing.T) { t.Skipf("low resolution timers inhibit profiling signals (golang.org/issue/13405)") return } - if true { - // TODO(khr): remove - t.Skipf("skipping for SSA branch, flaky") - return - } if !strings.Contains(os.Getenv("GODEBUG"), "gcstackbarrierall=1") { // Re-execute this test with constant GC and stack -- cgit v1.3 From d3f15ff6bc353d94b7249f33bb030ee1f7ee887e Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 25 Feb 2016 11:40:51 -0800 Subject: [dev.ssa] cmd/compile: shrink stack guard Our stack frame sizes look pretty good now. Lower the stack guard from 1024 to 720. Tip is currently using 720. We could go lower (to 640 at least) except PPC doesn't like that. Change-Id: Ie5f96c0e822435638223f1e8a2bd1a1eed68e6aa Reviewed-on: https://go-review.googlesource.com/19922 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: David Chase --- src/cmd/internal/obj/stack.go | 2 +- src/runtime/stack.go | 2 +- test/nosplit.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/internal/obj/stack.go b/src/cmd/internal/obj/stack.go index 1a2ee12291..80f6c6c164 100644 --- a/src/cmd/internal/obj/stack.go +++ b/src/cmd/internal/obj/stack.go @@ -11,7 +11,7 @@ const ( STACKSYSTEM = 0 StackSystem = STACKSYSTEM StackBig = 4096 - StackGuard = 1024*stackGuardMultiplier + StackSystem + StackGuard = 720*stackGuardMultiplier + StackSystem StackSmall = 128 StackLimit = StackGuard - StackSystem - StackSmall ) diff --git a/src/runtime/stack.go b/src/runtime/stack.go index ba1a1bb143..81059965d9 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -90,7 +90,7 @@ const ( // The stack guard is a pointer this many bytes above the // bottom of the stack. - _StackGuard = 1024*sys.StackGuardMultiplier + _StackSystem + _StackGuard = 720*sys.StackGuardMultiplier + _StackSystem // After a stack split check the SP is allowed to be this // many bytes below the stack guard. This saves an instruction diff --git a/test/nosplit.go b/test/nosplit.go index 2bf7077808..082fc3b0e6 100644 --- a/test/nosplit.go +++ b/test/nosplit.go @@ -302,13 +302,13 @@ TestCases: // Instead of rewriting the test cases above, adjust // the first stack frame to use up the extra bytes. if i == 0 { - size += (1024 - 128) - 128 + size += (720 - 128) - 128 // Noopt builds have a larger stackguard. // See ../src/cmd/dist/buildruntime.go:stackGuardMultiplier // This increase is included in obj.StackGuard for _, s := range strings.Split(os.Getenv("GO_GCFLAGS"), " ") { if s == "-N" { - size += 1024 + size += 720 } } } -- cgit v1.3 From 8107b0012f1d5f808e33f812c456e20554c383c8 Mon Sep 17 00:00:00 2001 From: David Chase Date: Sun, 28 Feb 2016 11:15:22 -0500 Subject: [dev.ssa] cmd/compile: use 32-bit load to read writebarrier Avoid targeting a partial register with load; ensure source of load (writebarrier) is aligned. Better yet would be "CMPB $1,writebarrier" but that requires wrestling with flagalloc (mem operand complicates moving instruction around). Didn't see a change in time for benchcmd -n 10 Build go build net/http Verified that we clean the code up properly: 0x20a8 : mov 0xc30a2(%rip),%eax # 0xc5150 0x20ae : test %al,%al Change-Id: Id5fb8c260eaec27bd727cb0ae1476c60343b0986 Reviewed-on: https://go-review.googlesource.com/19998 Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/ssa.go | 12 ++++++++---- src/runtime/mgc.go | 7 ++++--- 2 files changed, 12 insertions(+), 7 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 9847806110..8e68c20fb4 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -2718,9 +2718,11 @@ func (s *state) insertWBmove(t *Type, left, right *ssa.Value, line int32) { bEnd := s.f.NewBlock(ssa.BlockPlain) aux := &ssa.ExternSymbol{Types[TBOOL], syslook("writeBarrier", 0).Sym} - flagaddr := s.newValue1A(ssa.OpAddr, Ptrto(Types[TBOOL]), aux, s.sb) + flagaddr := s.newValue1A(ssa.OpAddr, Ptrto(Types[TUINT32]), aux, s.sb) // TODO: select the .enabled field. It is currently first, so not needed for now. - flag := s.newValue2(ssa.OpLoad, Types[TBOOL], flagaddr, s.mem()) + // Load word, test byte, avoiding partial register write from load byte. + flag := s.newValue2(ssa.OpLoad, Types[TUINT32], flagaddr, s.mem()) + flag = s.newValue1(ssa.OpTrunc64to8, Types[TBOOL], flag) b := s.endBlock() b.Kind = ssa.BlockIf b.Likely = ssa.BranchUnlikely @@ -2761,9 +2763,11 @@ func (s *state) insertWBstore(t *Type, left, right *ssa.Value, line int32) { bEnd := s.f.NewBlock(ssa.BlockPlain) aux := &ssa.ExternSymbol{Types[TBOOL], syslook("writeBarrier", 0).Sym} - flagaddr := s.newValue1A(ssa.OpAddr, Ptrto(Types[TBOOL]), aux, s.sb) + flagaddr := s.newValue1A(ssa.OpAddr, Ptrto(Types[TUINT32]), aux, s.sb) // TODO: select the .enabled field. It is currently first, so not needed for now. - flag := s.newValue2(ssa.OpLoad, Types[TBOOL], flagaddr, s.mem()) + // Load word, test byte, avoiding partial register write from load byte. + flag := s.newValue2(ssa.OpLoad, Types[TUINT32], flagaddr, s.mem()) + flag = s.newValue1(ssa.OpTrunc64to8, Types[TBOOL], flag) b := s.endBlock() b.Kind = ssa.BlockIf b.Likely = ssa.BranchUnlikely diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 94301c6dc7..102d44160e 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -216,9 +216,10 @@ var gcphase uint32 // The compiler knows about this variable. // If you change it, you must change the compiler too. var writeBarrier struct { - enabled bool // compiler emits a check of this before calling write barrier - needed bool // whether we need a write barrier for current GC phase - cgo bool // whether we need a write barrier for a cgo check + enabled bool // compiler emits a check of this before calling write barrier + needed bool // whether we need a write barrier for current GC phase + cgo bool // whether we need a write barrier for a cgo check + alignme uint64 // guarantee alignment so that compiler can use a 32 or 64-bit load } // gcBlackenEnabled is 1 if mutator assists and background mark -- cgit v1.3