From 9ae8f71c9431d287893443fa2b7fbdb72a9b56a2 Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Mon, 3 Aug 2020 13:19:46 -0400 Subject: [dev.link] cmd/link: stop renumbering files for pclntab generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creates two new symbols: runtime.cutab, and runtime.filetab, and strips the filenames out of runtime.pclntab_old. All stats are for cmd/compile. Time: Pclntab_GC 48.2ms ± 3% 45.5ms ± 9% -5.47% (p=0.004 n=9+9) Alloc/op: Pclntab_GC 30.0MB ± 0% 29.5MB ± 0% -1.88% (p=0.000 n=10+10) Allocs/op: Pclntab_GC 90.4k ± 0% 73.1k ± 0% -19.11% (p=0.000 n=10+10) live-B: Pclntab_GC 29.1M ± 0% 29.2M ± 0% +0.10% (p=0.000 n=10+10) binary sizes: NEW: 18565600 OLD: 18532768 The size differences in the binary are caused by the increased size of the Func objects, and (less likely) some extra alignment padding needed as a result. This is probably the maximum increase in size we'll size from the pclntab reworking. Change-Id: Idd95a9b159fea46f7701cfe6506813b88257fbea Reviewed-on: https://go-review.googlesource.com/c/go/+/246497 Run-TryBot: Jeremy Faller TryBot-Result: Gobot Gobot Reviewed-by: Than McIntosh Reviewed-by: Austin Clements --- src/runtime/symtab.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'src/runtime/symtab.go') diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index ddb5ea82b4..fbd9315522 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -334,14 +334,17 @@ const ( funcID_wrapper // any autogenerated code (hash/eq algorithms, method wrappers, etc.) ) -// PCHeader holds data used by the pclntab lookups. +// pcHeader holds data used by the pclntab lookups. type pcHeader struct { magic uint32 // 0xFFFFFFFA pad1, pad2 uint8 // 0,0 minLC uint8 // min instruction size ptrSize uint8 // size of a ptr in bytes nfunc int // number of functions in the module + nfiles uint // number of entries in the file tab. funcnameOffset uintptr // offset to the funcnametab variable from pcHeader + cuOffset uintptr // offset to the cutab variable from pcHeader + filetabOffset uintptr // offset to the filetab variable from pcHeader pclnOffset uintptr // offset to the pclntab variable from pcHeader } @@ -353,9 +356,10 @@ type pcHeader struct { type moduledata struct { pcHeader *pcHeader funcnametab []byte + cutab []uint32 + filetab []byte pclntable []byte ftab []functab - filetab []uint32 findfunctab uintptr minpc, maxpc uintptr @@ -851,7 +855,12 @@ func funcfile(f funcInfo, fileno int32) string { if !f.valid() { return "?" } - return gostringnocopy(&datap.pclntable[datap.filetab[fileno]]) + // Make sure the cu index and file offset are valid + if fileoff := datap.cutab[f.cuOffset+uint32(fileno)]; fileoff != ^uint32(0) { + return gostringnocopy(&datap.filetab[fileoff]) + } + // pcln section is corrupt. + return "?" } func funcline1(f funcInfo, targetpc uintptr, strict bool) (file string, line int32) { @@ -865,7 +874,7 @@ func funcline1(f funcInfo, targetpc uintptr, strict bool) (file string, line int // print("looking for ", hex(targetpc), " in ", funcname(f), " got file=", fileno, " line=", lineno, "\n") return "?", 0 } - file = gostringnocopy(&datap.pclntable[datap.filetab[fileno]]) + file = funcfile(f, fileno) return } @@ -1005,7 +1014,7 @@ type inlinedCall struct { parent int16 // index of parent in the inltree, or < 0 funcID funcID // type of the called function _ byte - file int32 // fileno index into filetab + file int32 // perCU file index for inlined call. See cmd/link:pcln.go line int32 // line number of the call site func_ int32 // offset into pclntab for name of called function parentPc int32 // position of an instruction whose source position is the call site (offset from entry) -- cgit v1.3 From 26407b22129e2e54db269c1a92826521addd8d56 Mon Sep 17 00:00:00 2001 From: Jeremy Faller Date: Wed, 12 Aug 2020 19:26:53 -0400 Subject: [dev.link] cmd/{compile,link}: remove pcdata tables from pclntab_old MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the pctables out of pclntab_old. Creates a new generator symbol, runtime.pctab, which holds all the deduplicated pctables. Also, tightens up some of the types in runtime. Darwin, cmd/compile statistics: alloc/op Pclntab_GC 26.4MB ± 0% 13.8MB ± 0% allocs/op Pclntab_GC 89.9k ± 0% 86.4k ± 0% liveB Pclntab_GC 25.5M ± 0% 24.2M ± 0% No significant change in binary size. Change-Id: I1560fd4421f8a210f8d4b508fbc54e1780e338f9 Reviewed-on: https://go-review.googlesource.com/c/go/+/248332 Run-TryBot: Jeremy Faller TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/data.go | 2 + src/cmd/link/internal/ld/pcln.go | 143 +++++++++++++++++--------- src/cmd/link/internal/ld/symtab.go | 4 + src/cmd/link/internal/loader/symbolbuilder.go | 9 ++ src/cmd/link/internal/sym/symbol.go | 4 - src/debug/gosym/pclntab.go | 10 +- src/runtime/runtime2.go | 8 +- src/runtime/symtab.go | 28 ++--- 8 files changed, 136 insertions(+), 72 deletions(-) (limited to 'src/runtime/symtab.go') diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go index a551d46403..2aecbfbeb5 100644 --- a/src/cmd/link/internal/ld/data.go +++ b/src/cmd/link/internal/ld/data.go @@ -1925,6 +1925,7 @@ func (state *dodataState) allocateDataSections(ctxt *Link) { ldr.SetSymSect(ldr.LookupOrCreateSym("runtime.funcnametab", 0), sect) ldr.SetSymSect(ldr.LookupOrCreateSym("runtime.cutab", 0), sect) ldr.SetSymSect(ldr.LookupOrCreateSym("runtime.filetab", 0), sect) + ldr.SetSymSect(ldr.LookupOrCreateSym("runtime.pctab", 0), sect) ldr.SetSymSect(ldr.LookupOrCreateSym("runtime.pclntab_old", 0), sect) ldr.SetSymSect(ldr.LookupOrCreateSym("runtime.epclntab", 0), sect) if ctxt.HeadType == objabi.Haix { @@ -2511,6 +2512,7 @@ func (ctxt *Link) address() []*sym.Segment { ctxt.defineInternal("runtime.funcnametab", sym.SRODATA) ctxt.defineInternal("runtime.cutab", sym.SRODATA) ctxt.defineInternal("runtime.filetab", sym.SRODATA) + ctxt.defineInternal("runtime.pctab", sym.SRODATA) ctxt.defineInternal("runtime.pclntab_old", sym.SRODATA) ctxt.xdefine("runtime.epclntab", sym.SRODATA, int64(pclntab.Vaddr+pclntab.Length)) ctxt.xdefine("runtime.noptrdata", sym.SNOPTRDATA, int64(noptr.Vaddr)) diff --git a/src/cmd/link/internal/ld/pcln.go b/src/cmd/link/internal/ld/pcln.go index e9fd5937e7..576f1c3780 100644 --- a/src/cmd/link/internal/ld/pcln.go +++ b/src/cmd/link/internal/ld/pcln.go @@ -43,6 +43,7 @@ type pclntab struct { findfunctab loader.Sym cutab loader.Sym filetab loader.Sym + pctab loader.Sym // The number of functions + number of TEXT sections - 1. This is such an // unexpected value because platforms that have more than one TEXT section @@ -273,10 +274,11 @@ func (state *pclntab) generatePCHeader(ctxt *Link) { off = writeSymOffset(off, state.funcnametab) off = writeSymOffset(off, state.cutab) off = writeSymOffset(off, state.filetab) + off = writeSymOffset(off, state.pctab) off = writeSymOffset(off, state.pclntab) } - size := int64(8 + 6*ctxt.Arch.PtrSize) + size := int64(8 + 7*ctxt.Arch.PtrSize) state.pcheader = state.addGeneratedSym(ctxt, "runtime.pcheader", size, writeHeader) } @@ -463,6 +465,68 @@ func (state *pclntab) generateFilenameTabs(ctxt *Link, compUnits []*sym.Compilat return cuOffsets } +// generatePctab creates the runtime.pctab variable, holding all the +// deduplicated pcdata. +func (state *pclntab) generatePctab(ctxt *Link, container loader.Bitmap) { + ldr := ctxt.loader + + // Pctab offsets of 0 are considered invalid in the runtime. We respect + // that by just padding a single byte at the beginning of runtime.pctab, + // that way no real offsets can be zero. + size := int64(1) + + // Walk the functions, finding offset to store each pcdata. + seen := make(map[loader.Sym]struct{}) + saveOffset := func(pcSym loader.Sym) { + if _, ok := seen[pcSym]; !ok { + datSize := ldr.SymSize(pcSym) + if datSize != 0 { + ldr.SetSymValue(pcSym, size) + } else { + // Invalid PC data, record as zero. + ldr.SetSymValue(pcSym, 0) + } + size += datSize + seen[pcSym] = struct{}{} + } + } + for _, s := range ctxt.Textp { + if !emitPcln(ctxt, s, container) { + continue + } + fi := ldr.FuncInfo(s) + if !fi.Valid() { + continue + } + fi.Preload() + + pcSyms := []loader.Sym{fi.Pcsp(), fi.Pcfile(), fi.Pcline()} + for _, pcSym := range pcSyms { + saveOffset(pcSym) + } + for _, pcSym := range fi.Pcdata() { + saveOffset(pcSym) + } + if fi.NumInlTree() > 0 { + saveOffset(fi.Pcinline()) + } + } + + // TODO: There is no reason we need a generator for this variable, and it + // could be moved to a carrier symbol. However, carrier symbols containing + // carrier symbols don't work yet (as of Aug 2020). Once this is fixed, + // runtime.pctab could just be a carrier sym. + writePctab := func(ctxt *Link, s loader.Sym) { + ldr := ctxt.loader + sb := ldr.MakeSymbolUpdater(s) + for sym := range seen { + sb.SetBytesAt(ldr.SymValue(sym), ldr.Data(sym)) + } + } + + state.pctab = state.addGeneratedSym(ctxt, "runtime.pctab", size, writePctab) +} + // pclntab initializes the pclntab symbol with // runtime function and file name information. @@ -494,6 +558,9 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { // runtime.filetab // []null terminated filename strings // + // runtime.pctab + // []byte of deduplicated pc data. + // // runtime.pclntab_old // function table, alternating PC and offset to func struct [each entry thearch.ptrsize bytes] // end PC [thearch.ptrsize bytes] @@ -514,6 +581,7 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { state.generatePCHeader(ctxt) state.generateFuncnametab(ctxt, container) cuOffsets := state.generateFilenameTabs(ctxt, compUnits, container) + state.generatePctab(ctxt, container) funcdataBytes := int64(0) ldr.SetCarrierSym(state.pclntab, state.carrier) @@ -525,21 +593,6 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { ftab.Grow(int64(state.nfunc)*2*int64(ctxt.Arch.PtrSize) + int64(ctxt.Arch.PtrSize) + 4) - szHint := len(ctxt.Textp) * 2 - pctaboff := make(map[string]uint32, szHint) - writepctab := func(off int32, p []byte) int32 { - start, ok := pctaboff[string(p)] - if !ok { - if len(p) > 0 { - start = uint32(len(ftab.Data())) - ftab.AddBytes(p) - } - pctaboff[string(p)] = start - } - newoff := int32(ftab.SetUint32(ctxt.Arch, int64(off), start)) - return newoff - } - setAddr := (*loader.SymbolBuilder).SetAddrPlus if ctxt.IsExe() && ctxt.IsInternal() { // Internal linking static executable. At this point the function @@ -555,10 +608,6 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { } } - pcsp := sym.Pcdata{} - pcfile := sym.Pcdata{} - pcline := sym.Pcdata{} - pcdata := []sym.Pcdata{} funcdata := []loader.Sym{} funcdataoff := []int64{} @@ -583,18 +632,13 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { } prevFunc = s - pcsp.P = pcsp.P[:0] - pcline.P = pcline.P[:0] - pcfile.P = pcfile.P[:0] - pcdata = pcdata[:0] + var numPCData int32 funcdataoff = funcdataoff[:0] funcdata = funcdata[:0] fi := ldr.FuncInfo(s) if fi.Valid() { fi.Preload() - for _, dataSym := range fi.Pcdata() { - pcdata = append(pcdata, sym.Pcdata{P: ldr.Data(dataSym)}) - } + numPCData = int32(len(fi.Pcdata())) nfd := fi.NumFuncdataoff() for i := uint32(0); i < nfd; i++ { funcdataoff = append(funcdataoff, fi.Funcdataoff(int(i))) @@ -602,15 +646,12 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { funcdata = fi.Funcdata(funcdata) } + writeInlPCData := false if fi.Valid() && fi.NumInlTree() > 0 { - - if len(pcdata) <= objabi.PCDATA_InlTreeIndex { - // Create inlining pcdata table. - newpcdata := make([]sym.Pcdata, objabi.PCDATA_InlTreeIndex+1) - copy(newpcdata, pcdata) - pcdata = newpcdata + writeInlPCData = true + if numPCData <= objabi.PCDATA_InlTreeIndex { + numPCData = objabi.PCDATA_InlTreeIndex + 1 } - if len(funcdataoff) <= objabi.FUNCDATA_InlTree { // Create inline tree funcdata. newfuncdata := make([]loader.Sym, objabi.FUNCDATA_InlTree+1) @@ -635,7 +676,7 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { // fixed size of struct, checked below off := funcstart - end := funcstart + int32(ctxt.Arch.PtrSize) + 3*4 + 6*4 + int32(len(pcdata))*4 + int32(len(funcdata))*int32(ctxt.Arch.PtrSize) + end := funcstart + int32(ctxt.Arch.PtrSize) + 3*4 + 6*4 + numPCData*4 + int32(len(funcdata))*int32(ctxt.Arch.PtrSize) if len(funcdata) > 0 && (end&int32(ctxt.Arch.PtrSize-1) != 0) { end += 4 } @@ -664,23 +705,21 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { off = int32(ftab.SetUint32(ctxt.Arch, int64(off), deferreturn)) cu := ldr.SymUnit(s) - if fi.Valid() { - pcsp = sym.Pcdata{P: ldr.Data(fi.Pcsp())} - pcfile = sym.Pcdata{P: ldr.Data(fi.Pcfile())} - pcline = sym.Pcdata{P: ldr.Data(fi.Pcline())} - } if fi.Valid() && fi.NumInlTree() > 0 { its := oldState.genInlTreeSym(cu, fi, ctxt.Arch, state) funcdata[objabi.FUNCDATA_InlTree] = its - pcdata[objabi.PCDATA_InlTreeIndex] = sym.Pcdata{P: ldr.Data(fi.Pcinline())} } // pcdata - off = writepctab(off, pcsp.P) - off = writepctab(off, pcfile.P) - off = writepctab(off, pcline.P) - off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(len(pcdata)))) + if fi.Valid() { + off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(ldr.SymValue(fi.Pcsp())))) + off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(ldr.SymValue(fi.Pcfile())))) + off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(ldr.SymValue(fi.Pcline())))) + } else { + off += 12 + } + off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(numPCData))) // Store the offset to compilation unit's file table. cuIdx := ^uint32(0) @@ -700,9 +739,17 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { // nfuncdata must be the final entry. off = int32(ftab.SetUint8(ctxt.Arch, int64(off), uint8(len(funcdata)))) - for i := range pcdata { - off = writepctab(off, pcdata[i].P) + + // Output the pcdata. + if fi.Valid() { + for i, pcSym := range fi.Pcdata() { + ftab.SetUint32(ctxt.Arch, int64(off+int32(i*4)), uint32(ldr.SymValue(pcSym))) + } + if writeInlPCData { + ftab.SetUint32(ctxt.Arch, int64(off+objabi.PCDATA_InlTreeIndex*4), uint32(ldr.SymValue(fi.Pcinline()))) + } } + off += numPCData * 4 // funcdata, must be pointer-aligned and we're only int32-aligned. // Missing funcdata will be 0 (nil pointer). @@ -724,7 +771,7 @@ func (ctxt *Link) pclntab(container loader.Bitmap) *pclntab { } if off != end { - ctxt.Errorf(s, "bad math in functab: funcstart=%d off=%d but end=%d (npcdata=%d nfuncdata=%d ptrsize=%d)", funcstart, off, end, len(pcdata), len(funcdata), ctxt.Arch.PtrSize) + ctxt.Errorf(s, "bad math in functab: funcstart=%d off=%d but end=%d (npcdata=%d nfuncdata=%d ptrsize=%d)", funcstart, off, end, numPCData, len(funcdata), ctxt.Arch.PtrSize) errorexit() } diff --git a/src/cmd/link/internal/ld/symtab.go b/src/cmd/link/internal/ld/symtab.go index d05b98f04a..520aaa44c2 100644 --- a/src/cmd/link/internal/ld/symtab.go +++ b/src/cmd/link/internal/ld/symtab.go @@ -627,6 +627,10 @@ func (ctxt *Link) symtab(pcln *pclntab) []sym.SymKind { moduledata.AddAddr(ctxt.Arch, pcln.filetab) moduledata.AddUint(ctxt.Arch, uint64(ldr.SymSize(pcln.filetab))) moduledata.AddUint(ctxt.Arch, uint64(ldr.SymSize(pcln.filetab))) + // The pctab slice + moduledata.AddAddr(ctxt.Arch, pcln.pctab) + moduledata.AddUint(ctxt.Arch, uint64(ldr.SymSize(pcln.pctab))) + moduledata.AddUint(ctxt.Arch, uint64(ldr.SymSize(pcln.pctab))) // The pclntab slice moduledata.AddAddr(ctxt.Arch, pcln.pclntab) moduledata.AddUint(ctxt.Arch, uint64(ldr.SymSize(pcln.pclntab))) diff --git a/src/cmd/link/internal/loader/symbolbuilder.go b/src/cmd/link/internal/loader/symbolbuilder.go index e14d89a927..c0c723d7f0 100644 --- a/src/cmd/link/internal/loader/symbolbuilder.go +++ b/src/cmd/link/internal/loader/symbolbuilder.go @@ -336,6 +336,15 @@ func (sb *SymbolBuilder) Addstring(str string) int64 { return r } +func (sb *SymbolBuilder) SetBytesAt(off int64, b []byte) int64 { + datLen := int64(len(b)) + if off+datLen > int64(len(sb.data)) { + panic("attempt to write past end of buffer") + } + copy(sb.data[off:off+datLen], b) + return off + datLen +} + func (sb *SymbolBuilder) addSymRef(tgt Sym, add int64, typ objabi.RelocType, rsize int) int64 { if sb.kind == 0 { sb.kind = sym.SDATA diff --git a/src/cmd/link/internal/sym/symbol.go b/src/cmd/link/internal/sym/symbol.go index 1a4165ebf7..70cf36a87e 100644 --- a/src/cmd/link/internal/sym/symbol.go +++ b/src/cmd/link/internal/sym/symbol.go @@ -33,7 +33,3 @@ func VersionToABI(v int) (obj.ABI, bool) { } return ^obj.ABI(0), false } - -type Pcdata struct { - P []byte -} diff --git a/src/debug/gosym/pclntab.go b/src/debug/gosym/pclntab.go index 21edddda20..a72f9847d7 100644 --- a/src/debug/gosym/pclntab.go +++ b/src/debug/gosym/pclntab.go @@ -58,6 +58,7 @@ type LineTable struct { functab []byte nfunctab uint32 filetab []byte + pctab []byte // points to the pctables. nfiletab uint32 funcNames map[uint32]string // cache the function names strings map[uint32]string // interned substrings of Data, keyed by offset @@ -235,6 +236,8 @@ func (t *LineTable) parsePclnTab() { offset = t.uintptr(t.Data[8+4*t.ptrsize:]) t.filetab = t.Data[offset:] offset = t.uintptr(t.Data[8+5*t.ptrsize:]) + t.pctab = t.Data[offset:] + offset = t.uintptr(t.Data[8+6*t.ptrsize:]) t.funcdata = t.Data[offset:] t.functab = t.Data[offset:] functabsize := t.nfunctab*2*t.ptrsize + t.ptrsize @@ -244,6 +247,7 @@ func (t *LineTable) parsePclnTab() { t.funcdata = t.Data t.funcnametab = t.Data t.functab = t.Data[8+t.ptrsize:] + t.pctab = t.Data functabsize := t.nfunctab*2*t.ptrsize + t.ptrsize fileoff := t.binary.Uint32(t.functab[functabsize:]) t.functab = t.functab[:functabsize] @@ -373,7 +377,7 @@ func (t *LineTable) step(p *[]byte, pc *uint64, val *int32, first bool) bool { // off is the offset to the beginning of the pc-value table, // and entry is the start PC for the corresponding function. func (t *LineTable) pcvalue(off uint32, entry, targetpc uint64) int32 { - p := t.funcdata[off:] + p := t.pctab[off:] val := int32(-1) pc := entry @@ -396,8 +400,8 @@ func (t *LineTable) findFileLine(entry uint64, filetab, linetab uint32, filenum, return 0 } - fp := t.funcdata[filetab:] - fl := t.funcdata[linetab:] + fp := t.pctab[filetab:] + fl := t.pctab[linetab:] fileVal := int32(-1) filePC := entry lineVal := int32(-1) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 5a79c7e6ec..755c409078 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -800,10 +800,10 @@ type _func struct { args int32 // in/out args size deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. - pcsp int32 - pcfile int32 - pcln int32 - npcdata int32 + pcsp uint32 + pcfile uint32 + pcln uint32 + npcdata uint32 cuOffset uint32 // runtime.cutab offset of this function's CU funcID funcID // set for certain special runtime functions _ [2]byte // pad diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index fbd9315522..0610f75179 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -345,6 +345,7 @@ type pcHeader struct { funcnameOffset uintptr // offset to the funcnametab variable from pcHeader cuOffset uintptr // offset to the cutab variable from pcHeader filetabOffset uintptr // offset to the filetab variable from pcHeader + pctabOffset uintptr // offset to the pctab varible from pcHeader pclnOffset uintptr // offset to the pclntab variable from pcHeader } @@ -358,6 +359,7 @@ type moduledata struct { funcnametab []byte cutab []uint32 filetab []byte + pctab []byte pclntable []byte ftab []functab findfunctab uintptr @@ -721,7 +723,7 @@ type pcvalueCache struct { type pcvalueCacheEnt struct { // targetpc and off together are the key of this cache entry. targetpc uintptr - off int32 + off uint32 // val is the value of this cached pcvalue entry. val int32 } @@ -736,7 +738,7 @@ func pcvalueCacheKey(targetpc uintptr) uintptr { // Returns the PCData value, and the PC where this value starts. // TODO: the start PC is returned only when cache is nil. -func pcvalue(f funcInfo, off int32, targetpc uintptr, cache *pcvalueCache, strict bool) (int32, uintptr) { +func pcvalue(f funcInfo, off uint32, targetpc uintptr, cache *pcvalueCache, strict bool) (int32, uintptr) { if off == 0 { return -1, 0 } @@ -770,7 +772,7 @@ func pcvalue(f funcInfo, off int32, targetpc uintptr, cache *pcvalueCache, stric return -1, 0 } datap := f.datap - p := datap.pclntable[off:] + p := datap.pctab[off:] pc := f.entry prevpc := pc val := int32(-1) @@ -812,7 +814,7 @@ func pcvalue(f funcInfo, off int32, targetpc uintptr, cache *pcvalueCache, stric print("runtime: invalid pc-encoded table f=", funcname(f), " pc=", hex(pc), " targetpc=", hex(targetpc), " tab=", p, "\n") - p = datap.pclntable[off:] + p = datap.pctab[off:] pc = f.entry val = -1 for { @@ -893,7 +895,7 @@ func funcspdelta(f funcInfo, targetpc uintptr, cache *pcvalueCache) int32 { // funcMaxSPDelta returns the maximum spdelta at any point in f. func funcMaxSPDelta(f funcInfo) int32 { datap := f.datap - p := datap.pclntable[f.pcsp:] + p := datap.pctab[f.pcsp:] pc := f.entry val := int32(-1) max := int32(0) @@ -909,20 +911,20 @@ func funcMaxSPDelta(f funcInfo) int32 { } } -func pcdatastart(f funcInfo, table int32) int32 { - return *(*int32)(add(unsafe.Pointer(&f.nfuncdata), unsafe.Sizeof(f.nfuncdata)+uintptr(table)*4)) +func pcdatastart(f funcInfo, table uint32) uint32 { + return *(*uint32)(add(unsafe.Pointer(&f.nfuncdata), unsafe.Sizeof(f.nfuncdata)+uintptr(table)*4)) } -func pcdatavalue(f funcInfo, table int32, targetpc uintptr, cache *pcvalueCache) int32 { - if table < 0 || table >= f.npcdata { +func pcdatavalue(f funcInfo, table uint32, targetpc uintptr, cache *pcvalueCache) int32 { + if table >= f.npcdata { return -1 } r, _ := pcvalue(f, pcdatastart(f, table), targetpc, cache, true) return r } -func pcdatavalue1(f funcInfo, table int32, targetpc uintptr, cache *pcvalueCache, strict bool) int32 { - if table < 0 || table >= f.npcdata { +func pcdatavalue1(f funcInfo, table uint32, targetpc uintptr, cache *pcvalueCache, strict bool) int32 { + if table >= f.npcdata { return -1 } r, _ := pcvalue(f, pcdatastart(f, table), targetpc, cache, strict) @@ -931,8 +933,8 @@ func pcdatavalue1(f funcInfo, table int32, targetpc uintptr, cache *pcvalueCache // Like pcdatavalue, but also return the start PC of this PCData value. // It doesn't take a cache. -func pcdatavalue2(f funcInfo, table int32, targetpc uintptr) (int32, uintptr) { - if table < 0 || table >= f.npcdata { +func pcdatavalue2(f funcInfo, table uint32, targetpc uintptr) (int32, uintptr) { + if table >= f.npcdata { return -1, 0 } return pcvalue(f, pcdatastart(f, table), targetpc, nil, true) -- cgit v1.3 From 7c58ef732efd9bf0d0882bb95371ce1909924a75 Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Mon, 14 Sep 2020 16:55:34 +0200 Subject: runtime: implement GODEBUG=inittrace=1 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting inittrace=1 causes the runtime to emit a single line to standard error for each package with init work, summarizing the execution time and memory allocation. The emitted debug information for init functions can be used to find bottlenecks or regressions in Go startup performance. Packages with no init function work (user defined or compiler generated) are omitted. Tracing plugin inits is not supported as they can execute concurrently. This would make the implementation of tracing more complex while adding support for a very rare use case. Plugin inits can be traced separately by testing a main package importing the plugins package imports explicitly. $ GODEBUG=inittrace=1 go test init internal/bytealg @0.008 ms, 0 ms clock, 0 bytes, 0 allocs init runtime @0.059 ms, 0.026 ms clock, 0 bytes, 0 allocs init math @0.19 ms, 0.001 ms clock, 0 bytes, 0 allocs init errors @0.22 ms, 0.004 ms clock, 0 bytes, 0 allocs init strconv @0.24 ms, 0.002 ms clock, 32 bytes, 2 allocs init sync @0.28 ms, 0.003 ms clock, 16 bytes, 1 allocs init unicode @0.44 ms, 0.11 ms clock, 23328 bytes, 24 allocs ... Inspired by stapelberg@google.com who instrumented doInit in a prototype to measure init times with GDB. Fixes #41378 Change-Id: Ic37c6a0cfc95488de9e737f5e346b8dbb39174e1 Reviewed-on: https://go-review.googlesource.com/c/go/+/254659 Trust: Martin Möhrmann Run-TryBot: Martin Möhrmann TryBot-Result: Go Bot Reviewed-by: Keith Randall --- doc/diagnostics.html | 2 ++ src/runtime/extern.go | 13 ++++++++++ src/runtime/malloc.go | 56 ++++++++++++++++++++++++--------------- src/runtime/proc.go | 69 ++++++++++++++++++++++++++++++++++++++++++++----- src/runtime/runtime1.go | 13 ++++++++-- src/runtime/symtab.go | 16 ++++++++++++ 6 files changed, 140 insertions(+), 29 deletions(-) (limited to 'src/runtime/symtab.go') diff --git a/doc/diagnostics.html b/doc/diagnostics.html index 478611c15c..f9368886c4 100644 --- a/doc/diagnostics.html +++ b/doc/diagnostics.html @@ -454,6 +454,8 @@ environmental variable is set accordingly.

  • GODEBUG=gctrace=1 prints garbage collector events at each collection, summarizing the amount of memory collected and the length of the pause.
    • +
  • GODEBUG=inittrace=1 prints a summary of execution time and memory allocation +information for completed package initilization work.
  • GODEBUG=schedtrace=X prints scheduling events every X milliseconds.
    • diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 7316503ed2..b75507b8f8 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -78,6 +78,19 @@ It is a comma-separated list of name=val pairs setting these named variables: If the line ends with "(forced)", this GC was forced by a runtime.GC() call. + inittrace: setting inittrace=1 causes the runtime to emit a single line to standard + error for each package with init work, summarizing the execution time and memory + allocation. No information is printed for inits executed as part of plugin loading + and for packages without both user defined and compiler generated init work. + The format of this line is subject to change. Currently, it is: + init # @#ms, # ms clock, # bytes, # allocs + where the fields are as follows: + init # the package name + @# ms time in milliseconds when the init started since program start + # clock wall-clock time for package initialization work + # bytes memory allocated on the heap + # allocs number of heap allocations + madvdontneed: setting madvdontneed=1 will use MADV_DONTNEED instead of MADV_FREE on Linux when returning memory to the kernel. This is less efficient, but causes RSS numbers to drop diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index f7e9b7c4b4..b19d1f2671 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -909,27 +909,34 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { return unsafe.Pointer(&zerobase) } - if debug.sbrk != 0 { - align := uintptr(16) - if typ != nil { - // TODO(austin): This should be just - // align = uintptr(typ.align) - // but that's only 4 on 32-bit platforms, - // even if there's a uint64 field in typ (see #599). - // This causes 64-bit atomic accesses to panic. - // Hence, we use stricter alignment that matches - // the normal allocator better. - if size&7 == 0 { - align = 8 - } else if size&3 == 0 { - align = 4 - } else if size&1 == 0 { - align = 2 - } else { - align = 1 + if debug.malloc { + if debug.sbrk != 0 { + align := uintptr(16) + if typ != nil { + // TODO(austin): This should be just + // align = uintptr(typ.align) + // but that's only 4 on 32-bit platforms, + // even if there's a uint64 field in typ (see #599). + // This causes 64-bit atomic accesses to panic. + // Hence, we use stricter alignment that matches + // the normal allocator better. + if size&7 == 0 { + align = 8 + } else if size&3 == 0 { + align = 4 + } else if size&1 == 0 { + align = 2 + } else { + align = 1 + } } + return persistentalloc(size, align, &memstats.other_sys) + } + + if inittrace.active && inittrace.id == getg().goid { + // Init functions are executed sequentially in a single Go routine. + inittrace.allocs += 1 } - return persistentalloc(size, align, &memstats.other_sys) } // assistG is the G to charge for this allocation, or nil if @@ -1136,8 +1143,15 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { mp.mallocing = 0 releasem(mp) - if debug.allocfreetrace != 0 { - tracealloc(x, size, typ) + if debug.malloc { + if debug.allocfreetrace != 0 { + tracealloc(x, size, typ) + } + + if inittrace.active && inittrace.id == getg().goid { + // Init functions are executed sequentially in a single Go routine. + inittrace.bytes += uint64(size) + } } if rate := MemProfileRate; rate > 0 { diff --git a/src/runtime/proc.go b/src/runtime/proc.go index a1e2ed0680..4872480314 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -154,11 +154,20 @@ func main() { throw("runtime.main not on m0") } - doInit(&runtime_inittask) // must be before defer - if nanotime() == 0 { + // Record when the world started. + // Must be before doInit for tracing init. + runtimeInitTime = nanotime() + if runtimeInitTime == 0 { throw("nanotime returning zero") } + if debug.inittrace != 0 { + inittrace.id = getg().goid + inittrace.active = true + } + + doInit(&runtime_inittask) // Must be before defer. + // Defer unlock so that runtime.Goexit during init does the unlock too. needUnlock := true defer func() { @@ -167,9 +176,6 @@ func main() { } }() - // Record when the world started. - runtimeInitTime = nanotime() - gcenable() main_init_done = make(chan bool) @@ -196,6 +202,10 @@ func main() { doInit(&main_inittask) + // Disable init tracing after main init done to avoid overhead + // of collecting statistics in malloc and newproc + inittrace.active = false + close(main_init_done) needUnlock = false @@ -5665,6 +5675,17 @@ type initTask struct { // followed by nfns pcs, one per init function to run } +// inittrace stores statistics for init functions which are +// updated by malloc and newproc when active is true. +var inittrace tracestat + +type tracestat struct { + active bool // init tracing activation status + id int64 // init go routine id + allocs uint64 // heap allocations + bytes uint64 // heap allocated bytes +} + func doInit(t *initTask) { switch t.state { case 2: // fully initialized @@ -5673,16 +5694,52 @@ func doInit(t *initTask) { throw("recursive call during initialization - linker skew") default: // not initialized yet t.state = 1 // initialization in progress + for i := uintptr(0); i < t.ndeps; i++ { p := add(unsafe.Pointer(t), (3+i)*sys.PtrSize) t2 := *(**initTask)(p) doInit(t2) } + + if t.nfns == 0 { + t.state = 2 // initialization done + return + } + + var ( + start int64 + before tracestat + ) + + if inittrace.active { + start = nanotime() + // Load stats non-atomically since tracinit is updated only by this init go routine. + before = inittrace + } + + firstFunc := add(unsafe.Pointer(t), (3+t.ndeps)*sys.PtrSize) for i := uintptr(0); i < t.nfns; i++ { - p := add(unsafe.Pointer(t), (3+t.ndeps+i)*sys.PtrSize) + p := add(firstFunc, i*sys.PtrSize) f := *(*func())(unsafe.Pointer(&p)) f() } + + if inittrace.active { + end := nanotime() + // Load stats non-atomically since tracinit is updated only by this init go routine. + after := inittrace + + pkg := funcpkgpath(findfunc(funcPC(firstFunc))) + + var sbuf [24]byte + print("init ", pkg, " @") + print(string(fmtNSAsMS(sbuf[:], uint64(start-runtimeInitTime))), " ms, ") + print(string(fmtNSAsMS(sbuf[:], uint64(end-start))), " ms clock, ") + print(string(itoa(sbuf[:], after.bytes-before.bytes)), " bytes, ") + print(string(itoa(sbuf[:], after.allocs-before.allocs)), " allocs") + print("\n") + } + t.state = 2 // initialization done } } diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index 7c893aa25c..0f182ac58e 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -300,7 +300,6 @@ type dbgVar struct { // existing int var for that value, which may // already have an initial value. var debug struct { - allocfreetrace int32 cgocheck int32 clobberfree int32 efence int32 @@ -311,13 +310,20 @@ var debug struct { gctrace int32 invalidptr int32 madvdontneed int32 // for Linux; issue 28466 - sbrk int32 scavenge int32 scavtrace int32 scheddetail int32 schedtrace int32 tracebackancestors int32 asyncpreemptoff int32 + + // debug.malloc is used as a combined debug check + // in the malloc function and should be set + // if any of the below debug options is != 0. + malloc bool + allocfreetrace int32 + inittrace int32 + sbrk int32 } var dbgvars = []dbgVar{ @@ -339,6 +345,7 @@ var dbgvars = []dbgVar{ {"schedtrace", &debug.schedtrace}, {"tracebackancestors", &debug.tracebackancestors}, {"asyncpreemptoff", &debug.asyncpreemptoff}, + {"inittrace", &debug.inittrace}, } func parsedebugvars() { @@ -378,6 +385,8 @@ func parsedebugvars() { } } + debug.malloc = (debug.allocfreetrace | debug.inittrace | debug.sbrk) != 0 + setTraceback(gogetenv("GOTRACEBACK")) traceback_env = traceback_cache } diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index a14f5c13d9..84637376bf 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -844,6 +844,22 @@ func funcname(f funcInfo) string { return gostringnocopy(cfuncname(f)) } +func funcpkgpath(f funcInfo) string { + name := funcname(f) + i := len(name) - 1 + for ; i > 0; i-- { + if name[i] == '/' { + break + } + } + for ; i < len(name); i++ { + if name[i] == '.' { + break + } + } + return name[:i] +} + func cfuncnameFromNameoff(f funcInfo, nameoff int32) *byte { if !f.valid() { return nil -- cgit v1.3 From 30c18878730434027dbefd343aad74963a1fdc48 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 1 Oct 2020 17:22:38 -0400 Subject: runtime,cmd/cgo: simplify C -> Go call path This redesigns the way calls work from C to exported Go functions. It removes several steps from the call path, makes cmd/cgo no longer sensitive to the Go calling convention, and eliminates the use of reflectcall from cgo. In order to avoid generating a large amount of FFI glue between the C and Go ABIs, the cgo tool has long depended on generating a C function that marshals the arguments into a struct, and then the actual ABI switch happens in functions with fixed signatures that simply take a pointer to this struct. In a way, this CL simply pushes this idea further. Currently, the cgo tool generates this argument struct in the exact layout of the Go stack frame and depends on reflectcall to unpack it into the appropriate Go call (even though it's actually reflectcall'ing a function generated by cgo). In this CL, we decouple this struct from the Go stack layout. Instead, cgo generates a Go function that takes the struct, unpacks it, and calls the exported function. Since this generated function has a generic signature (like the rest of the call path), we don't need reflectcall and can instead depend on the Go compiler itself to implement the call to the exported Go function. One complication is that syscall.NewCallback on Windows, which converts a Go function into a C function pointer, depends on cgocallback's current dynamic calling approach since the signatures of the callbacks aren't known statically. For this specific case, we continue to depend on reflectcall. Really, the current approach makes some overly simplistic assumptions about translating the C ABI to the Go ABI. Now we're at least in a much better position to do a proper ABI translation. For comparison, the current cgo call path looks like: GoF (generated C function) -> crosscall2 (in cgo/asm_*.s) -> _cgoexp_GoF (generated Go function) -> cgocallback (in asm_*.s) -> cgocallback_gofunc (in asm_*.s) -> cgocallbackg (in cgocall.go) -> cgocallbackg1 (in cgocall.go) -> reflectcall (in asm_*.s) -> _cgoexpwrap_GoF (generated Go function) -> p.GoF Now the call path looks like: GoF (generated C function) -> crosscall2 (in cgo/asm_*.s) -> cgocallback (in asm_*.s) -> cgocallbackg (in cgocall.go) -> cgocallbackg1 (in cgocall.go) -> _cgoexp_GoF (generated Go function) -> p.GoF Notably: 1. We combine _cgoexp_GoF and _cgoexpwrap_GoF and move the combined operation to the end of the sequence. This combined function also handles reflectcall's previous role. 2. We combined cgocallback and cgocallback_gofunc since the only purpose of having both was to convert a raw PC into a Go function value. We instead construct the Go function value in cgocallbackg1. 3. cgocallbackg1 no longer reaches backwards through the stack to get the arguments to cgocallback_gofunc. Instead, we just pass the arguments down. 4. Currently, we need an explicit msanwrite to mark the results struct as written because reflectcall doesn't do this. Now, the results are written by regular Go assignments, so the Go compiler generates the necessary MSAN annotations. This also means we no longer need to track the size of the arguments frame. Updates #40724, since now we don't need to teach cgo about the register ABI or change how it uses reflectcall. Change-Id: I7840489a2597962aeb670e0c1798a16a7359c94f Reviewed-on: https://go-review.googlesource.com/c/go/+/258938 Trust: Austin Clements Run-TryBot: Austin Clements TryBot-Result: Go Bot Reviewed-by: Cherry Zhang --- misc/cgo/test/callback.go | 2 +- src/cmd/cgo/doc.go | 2 +- src/cmd/cgo/out.go | 162 +++++++++++++------------------------- src/cmd/internal/objabi/funcid.go | 6 +- src/runtime/asm_386.s | 56 +++++-------- src/runtime/asm_amd64.s | 55 +++++-------- src/runtime/asm_arm.s | 58 +++++--------- src/runtime/asm_arm64.s | 48 ++++------- src/runtime/asm_mips64x.s | 54 +++++-------- src/runtime/asm_mipsx.s | 54 +++++-------- src/runtime/asm_ppc64x.s | 55 +++++-------- src/runtime/asm_riscv64.s | 5 +- src/runtime/asm_s390x.s | 54 +++++-------- src/runtime/asm_wasm.s | 5 +- src/runtime/cgo/asm_386.s | 12 +-- src/runtime/cgo/asm_amd64.s | 20 +++-- src/runtime/cgo/asm_arm.s | 61 +++++++------- src/runtime/cgo/asm_arm64.s | 19 ++--- src/runtime/cgo/asm_mips64x.s | 19 ++--- src/runtime/cgo/asm_mipsx.s | 19 ++--- src/runtime/cgo/asm_ppc64x.s | 20 ++--- src/runtime/cgo/asm_s390x.s | 14 ++-- src/runtime/cgo/callbacks.go | 29 +++---- src/runtime/cgocall.go | 140 +++++++++----------------------- src/runtime/proc.go | 2 +- src/runtime/race/output_test.go | 2 +- src/runtime/stubs.go | 25 +++--- src/runtime/symtab.go | 2 +- src/runtime/sys_windows_386.s | 67 ++++++---------- src/runtime/sys_windows_amd64.s | 53 ++++++------- src/runtime/sys_windows_arm.s | 3 + src/runtime/syscall_windows.go | 42 +++++++++- src/runtime/traceback.go | 2 +- 33 files changed, 471 insertions(+), 696 deletions(-) (limited to 'src/runtime/symtab.go') diff --git a/misc/cgo/test/callback.go b/misc/cgo/test/callback.go index e749650293..814888e3ac 100644 --- a/misc/cgo/test/callback.go +++ b/misc/cgo/test/callback.go @@ -181,7 +181,7 @@ func testCallbackCallers(t *testing.T) { name := []string{ "runtime.cgocallbackg1", "runtime.cgocallbackg", - "runtime.cgocallback_gofunc", + "runtime.cgocallback", "runtime.asmcgocall", "runtime.cgocall", "test._Cfunc_callback", diff --git a/src/cmd/cgo/doc.go b/src/cmd/cgo/doc.go index b3f371b08c..e782c866ac 100644 --- a/src/cmd/cgo/doc.go +++ b/src/cmd/cgo/doc.go @@ -721,7 +721,7 @@ linkage to the desired libraries. The main function is provided by _cgo_main.c: int main() { return 0; } - void crosscall2(void(*fn)(void*, int, uintptr_t), void *a, int c, uintptr_t ctxt) { } + void crosscall2(void(*fn)(void*), void *a, int c, uintptr_t ctxt) { } uintptr_t _cgo_wait_runtime_init_done(void) { return 0; } void _cgo_release_context(uintptr_t ctxt) { } char* _cgo_topofstack(void) { return (char*)0; } diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 82316a300b..eef54f2d0f 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -59,14 +59,14 @@ func (p *Package) writeDefs() { // Write C main file for using gcc to resolve imports. fmt.Fprintf(fm, "int main() { return 0; }\n") if *importRuntimeCgo { - fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int, __SIZE_TYPE__), void *a, int c, __SIZE_TYPE__ ctxt) { }\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*), void *a, int c, __SIZE_TYPE__ ctxt) { }\n") fmt.Fprintf(fm, "__SIZE_TYPE__ _cgo_wait_runtime_init_done(void) { return 0; }\n") fmt.Fprintf(fm, "void _cgo_release_context(__SIZE_TYPE__ ctxt) { }\n") fmt.Fprintf(fm, "char* _cgo_topofstack(void) { return (char*)0; }\n") } else { // If we're not importing runtime/cgo, we *are* runtime/cgo, // which provides these functions. We just need a prototype. - fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*, int, __SIZE_TYPE__), void *a, int c, __SIZE_TYPE__ ctxt);\n") + fmt.Fprintf(fm, "void crosscall2(void(*fn)(void*), void *a, int c, __SIZE_TYPE__ ctxt);\n") fmt.Fprintf(fm, "__SIZE_TYPE__ _cgo_wait_runtime_init_done(void);\n") fmt.Fprintf(fm, "void _cgo_release_context(__SIZE_TYPE__);\n") } @@ -852,7 +852,7 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "#pragma GCC diagnostic ignored \"-Wpragmas\"\n") fmt.Fprintf(fgcc, "#pragma GCC diagnostic ignored \"-Waddress-of-packed-member\"\n") - fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *, int, __SIZE_TYPE__), void *, int, __SIZE_TYPE__);\n") + fmt.Fprintf(fgcc, "extern void crosscall2(void (*fn)(void *), void *, int, __SIZE_TYPE__);\n") fmt.Fprintf(fgcc, "extern __SIZE_TYPE__ _cgo_wait_runtime_init_done(void);\n") fmt.Fprintf(fgcc, "extern void _cgo_release_context(__SIZE_TYPE__);\n\n") fmt.Fprintf(fgcc, "extern char* _cgo_topofstack(void);") @@ -862,59 +862,48 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { for _, exp := range p.ExpFunc { fn := exp.Func - // Construct a gcc struct matching the gc argument and - // result frame. The gcc struct will be compiled with - // __attribute__((packed)) so all padding must be accounted - // for explicitly. + // Construct a struct that will be used to communicate + // arguments from C to Go. The C and Go definitions + // just have to agree. The gcc struct will be compiled + // with __attribute__((packed)) so all padding must be + // accounted for explicitly. ctype := "struct {\n" + gotype := new(bytes.Buffer) + fmt.Fprintf(gotype, "struct {\n") off := int64(0) npad := 0 - if fn.Recv != nil { - t := p.cgoType(fn.Recv.List[0].Type) - ctype += fmt.Sprintf("\t\t%s recv;\n", t.C) + argField := func(typ ast.Expr, namePat string, args ...interface{}) { + name := fmt.Sprintf(namePat, args...) + t := p.cgoType(typ) + if off%t.Align != 0 { + pad := t.Align - off%t.Align + ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) + off += pad + npad++ + } + ctype += fmt.Sprintf("\t\t%s %s;\n", t.C, name) + fmt.Fprintf(gotype, "\t\t%s ", name) + noSourceConf.Fprint(gotype, fset, typ) + fmt.Fprintf(gotype, "\n") off += t.Size } + if fn.Recv != nil { + argField(fn.Recv.List[0].Type, "recv") + } fntype := fn.Type forFieldList(fntype.Params, func(i int, aname string, atype ast.Expr) { - t := p.cgoType(atype) - if off%t.Align != 0 { - pad := t.Align - off%t.Align - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } - ctype += fmt.Sprintf("\t\t%s p%d;\n", t.C, i) - off += t.Size + argField(atype, "p%d", i) }) - if off%p.PtrSize != 0 { - pad := p.PtrSize - off%p.PtrSize - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } forFieldList(fntype.Results, func(i int, aname string, atype ast.Expr) { - t := p.cgoType(atype) - if off%t.Align != 0 { - pad := t.Align - off%t.Align - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } - ctype += fmt.Sprintf("\t\t%s r%d;\n", t.C, i) - off += t.Size + argField(atype, "r%d", i) }) - if off%p.PtrSize != 0 { - pad := p.PtrSize - off%p.PtrSize - ctype += fmt.Sprintf("\t\tchar __pad%d[%d];\n", npad, pad) - off += pad - npad++ - } if ctype == "struct {\n" { ctype += "\t\tchar unused;\n" // avoid empty struct } ctype += "\t}" + fmt.Fprintf(gotype, "\t}") // Get the return type of the wrapper function // compiled by gcc. @@ -965,12 +954,15 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { } fmt.Fprintf(fgcch, "extern %s;\n", s) - fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *, int, __SIZE_TYPE__);\n", cPrefix, exp.ExpName) + fmt.Fprintf(fgcc, "extern void _cgoexp%s_%s(void *);\n", cPrefix, exp.ExpName) fmt.Fprintf(fgcc, "\nCGO_NO_SANITIZE_THREAD") fmt.Fprintf(fgcc, "\n%s\n", s) fmt.Fprintf(fgcc, "{\n") fmt.Fprintf(fgcc, "\t__SIZE_TYPE__ _cgo_ctxt = _cgo_wait_runtime_init_done();\n") - fmt.Fprintf(fgcc, "\t%s %v _cgo_a;\n", ctype, p.packedAttribute()) + // The results part of the argument structure must be + // initialized to 0 so the write barriers generated by + // the assignments to these fields in Go are safe. + fmt.Fprintf(fgcc, "\t%s %v _cgo_a = {0};\n", ctype, p.packedAttribute()) if gccResult != "void" && (len(fntype.Results.List) > 1 || len(fntype.Results.List[0].Names) > 1) { fmt.Fprintf(fgcc, "\t%s r;\n", gccResult) } @@ -999,82 +991,28 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { fmt.Fprintf(fgcc, "}\n") // Build the wrapper function compiled by cmd/compile. - goname := "_cgoexpwrap" + cPrefix + "_" - if fn.Recv != nil { - goname += fn.Recv.List[0].Names[0].Name + "_" - } - goname += exp.Func.Name.Name + // This unpacks the argument struct above and calls the Go function. fmt.Fprintf(fgo2, "//go:cgo_export_dynamic %s\n", exp.ExpName) fmt.Fprintf(fgo2, "//go:linkname _cgoexp%s_%s _cgoexp%s_%s\n", cPrefix, exp.ExpName, cPrefix, exp.ExpName) fmt.Fprintf(fgo2, "//go:cgo_export_static _cgoexp%s_%s\n", cPrefix, exp.ExpName) - fmt.Fprintf(fgo2, "//go:nosplit\n") // no split stack, so no use of m or g - fmt.Fprintf(fgo2, "//go:norace\n") // must not have race detector calls inserted - fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a unsafe.Pointer, n int32, ctxt uintptr) {\n", cPrefix, exp.ExpName) - fmt.Fprintf(fgo2, "\tfn := %s\n", goname) - // The indirect here is converting from a Go function pointer to a C function pointer. - fmt.Fprintf(fgo2, "\t_cgo_runtime_cgocallback(**(**unsafe.Pointer)(unsafe.Pointer(&fn)), a, uintptr(n), ctxt);\n") - fmt.Fprintf(fgo2, "}\n") + fmt.Fprintf(fgo2, "func _cgoexp%s_%s(a *%s) {\n", cPrefix, exp.ExpName, gotype) fmt.Fprintf(fm, "int _cgoexp%s_%s;\n", cPrefix, exp.ExpName) - // This code uses printer.Fprint, not conf.Fprint, - // because we don't want //line comments in the middle - // of the function types. - fmt.Fprintf(fgo2, "\n") - fmt.Fprintf(fgo2, "func %s(", goname) - comma := false - if fn.Recv != nil { - fmt.Fprintf(fgo2, "recv ") - printer.Fprint(fgo2, fset, fn.Recv.List[0].Type) - comma = true - } - forFieldList(fntype.Params, - func(i int, aname string, atype ast.Expr) { - if comma { - fmt.Fprintf(fgo2, ", ") - } - fmt.Fprintf(fgo2, "p%d ", i) - printer.Fprint(fgo2, fset, atype) - comma = true - }) - fmt.Fprintf(fgo2, ")") if gccResult != "void" { - fmt.Fprint(fgo2, " (") + // Write results back to frame. + fmt.Fprintf(fgo2, "\t") forFieldList(fntype.Results, func(i int, aname string, atype ast.Expr) { if i > 0 { - fmt.Fprint(fgo2, ", ") + fmt.Fprintf(fgo2, ", ") } - fmt.Fprintf(fgo2, "r%d ", i) - printer.Fprint(fgo2, fset, atype) + fmt.Fprintf(fgo2, "a.r%d", i) }) - fmt.Fprint(fgo2, ")") - } - fmt.Fprint(fgo2, " {\n") - if gccResult == "void" { - fmt.Fprint(fgo2, "\t") - } else { - // Verify that any results don't contain any - // Go pointers. - addedDefer := false - forFieldList(fntype.Results, - func(i int, aname string, atype ast.Expr) { - if !p.hasPointer(nil, atype, false) { - return - } - if !addedDefer { - fmt.Fprint(fgo2, "\tdefer func() {\n") - addedDefer = true - } - fmt.Fprintf(fgo2, "\t\t_cgoCheckResult(r%d)\n", i) - }) - if addedDefer { - fmt.Fprint(fgo2, "\t}()\n") - } - fmt.Fprint(fgo2, "\treturn ") + fmt.Fprintf(fgo2, " = ") } if fn.Recv != nil { - fmt.Fprintf(fgo2, "recv.") + fmt.Fprintf(fgo2, "a.recv.") } fmt.Fprintf(fgo2, "%s(", exp.Func.Name) forFieldList(fntype.Params, @@ -1082,9 +1020,20 @@ func (p *Package) writeExports(fgo2, fm, fgcc, fgcch io.Writer) { if i > 0 { fmt.Fprint(fgo2, ", ") } - fmt.Fprintf(fgo2, "p%d", i) + fmt.Fprintf(fgo2, "a.p%d", i) }) fmt.Fprint(fgo2, ")\n") + if gccResult != "void" { + // Verify that any results don't contain any + // Go pointers. + forFieldList(fntype.Results, + func(i int, aname string, atype ast.Expr) { + if !p.hasPointer(nil, atype, false) { + return + } + fmt.Fprintf(fgo2, "\t_cgoCheckResult(a.r%d)\n", i) + }) + } fmt.Fprint(fgo2, "}\n") } @@ -1582,9 +1531,6 @@ const goProlog = ` //go:linkname _cgo_runtime_cgocall runtime.cgocall func _cgo_runtime_cgocall(unsafe.Pointer, uintptr) int32 -//go:linkname _cgo_runtime_cgocallback runtime.cgocallback -func _cgo_runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) - //go:linkname _cgoCheckPointer runtime.cgoCheckPointer func _cgoCheckPointer(interface{}, interface{}) diff --git a/src/cmd/internal/objabi/funcid.go b/src/cmd/internal/objabi/funcid.go index 6c9336f31c..1d098ee172 100644 --- a/src/cmd/internal/objabi/funcid.go +++ b/src/cmd/internal/objabi/funcid.go @@ -26,7 +26,7 @@ const ( FuncID_gcBgMarkWorker FuncID_systemstack_switch FuncID_systemstack - FuncID_cgocallback_gofunc + FuncID_cgocallback FuncID_gogo FuncID_externalthreadhandler FuncID_debugCallV1 @@ -70,8 +70,8 @@ func GetFuncID(name string, isWrapper bool) FuncID { return FuncID_systemstack_switch case "runtime.systemstack": return FuncID_systemstack - case "runtime.cgocallback_gofunc": - return FuncID_cgocallback_gofunc + case "runtime.cgocallback": + return FuncID_cgocallback case "runtime.gogo": return FuncID_gogo case "runtime.externalthreadhandler": diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 11863fba39..a54b68e03d 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -702,25 +702,9 @@ nosave: MOVL AX, ret+8(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 - LEAL fn+0(FP), AX - MOVL AX, 0(SP) - MOVL frame+4(FP), AX - MOVL AX, 4(SP) - MOVL framesize+8(FP), AX - MOVL AX, 8(SP) - MOVL ctxt+12(FP), AX - MOVL AX, 12(SP) - MOVL $runtime·cgocallback_gofunc(SB), AX - CALL AX - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$12-16 +TEXT ·cgocallback(SB),NOSPLIT,$16-12 // Frame size must match commented places below NO_LOCAL_POINTERS // If g is nil, Go did not create the current thread. @@ -780,34 +764,36 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. // - // In the new goroutine, 4(SP) holds the saved oldm (DX) register. - // 8(SP) is unused. + // In the new goroutine, 12(SP) holds the saved oldm (DX) register. MOVL m_curg(BP), SI MOVL SI, g(CX) MOVL (g_sched+gobuf_sp)(SI), DI // prepare stack as DI MOVL (g_sched+gobuf_pc)(SI), BP - MOVL BP, -4(DI) - MOVL ctxt+12(FP), CX - LEAL -(4+12)(DI), SP - MOVL DX, 4(SP) - MOVL CX, 0(SP) + MOVL BP, -4(DI) // "push" return PC on the g stack + // Gather our arguments into registers. + MOVL fn+0(FP), AX + MOVL frame+4(FP), BX + MOVL ctxt+8(FP), CX + LEAL -(4+16)(DI), SP // Must match declared frame size + MOVL DX, 12(SP) + MOVL AX, 0(SP) + MOVL BX, 4(SP) + MOVL CX, 8(SP) CALL runtime·cgocallbackg(SB) - MOVL 4(SP), DX + MOVL 12(SP), DX // Restore g->sched (== m->curg->sched) from saved values. get_tls(CX) MOVL g(CX), SI - MOVL 12(SP), BP + MOVL 16(SP), BP // Must match declared frame size MOVL BP, (g_sched+gobuf_pc)(SI) - LEAL (12+4)(SP), DI + LEAL (16+4)(SP), DI // Must match declared frame size MOVL DI, (g_sched+gobuf_sp)(SI) // Switch back to m->g0's stack and restore m->g0->sched.sp. diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 256f4112cd..3d5d9c4d58 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -691,25 +691,9 @@ nosave: MOVL AX, ret+16(FP) RET -// func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - LEAQ fn+0(FP), AX - MOVQ AX, 0(SP) - MOVQ frame+8(FP), AX - MOVQ AX, 8(SP) - MOVQ framesize+16(FP), AX - MOVQ AX, 16(SP) - MOVQ ctxt+24(FP), AX - MOVQ AX, 24(SP) - MOVQ $runtime·cgocallback_gofunc(SB), AX - CALL AX - RET - -// func cgocallback_gofunc(fn, frame, framesize, ctxt uintptr) +// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$32-24 NO_LOCAL_POINTERS // If g is nil, Go did not create the current thread. @@ -769,37 +753,40 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. // - // In the new goroutine, 8(SP) holds the saved R8. + // In the new goroutine, 24(SP) holds the saved R8. MOVQ m_curg(BX), SI MOVQ SI, g(CX) MOVQ (g_sched+gobuf_sp)(SI), DI // prepare stack as DI MOVQ (g_sched+gobuf_pc)(SI), BX - MOVQ BX, -8(DI) + MOVQ BX, -8(DI) // "push" return PC on the g stack + // Gather our arguments into registers. + MOVQ fn+0(FP), BX + MOVQ frame+8(FP), CX + MOVQ ctxt+16(FP), DX // Compute the size of the frame, including return PC and, if // GOEXPERIMENT=framepointer, the saved base pointer - MOVQ ctxt+24(FP), BX - LEAQ fv+0(FP), AX - SUBQ SP, AX - SUBQ AX, DI + LEAQ fn+0(FP), AX + SUBQ SP, AX // AX is our actual frame size + SUBQ AX, DI // Allocate the same frame size on the g stack MOVQ DI, SP - MOVQ R8, 8(SP) + MOVQ R8, 24(SP) MOVQ BX, 0(SP) + MOVQ CX, 8(SP) + MOVQ DX, 16(SP) CALL runtime·cgocallbackg(SB) - MOVQ 8(SP), R8 + MOVQ 24(SP), R8 // Compute the size of the frame again. FP and SP have // completely different values here than they did above, // but only their difference matters. - LEAQ fv+0(FP), AX + LEAQ fn+0(FP), AX SUBQ SP, AX // Restore g->sched (== m->curg->sched) from saved values. diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 51a50c604c..c54b4eb006 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -643,25 +643,9 @@ nosave: MOVW R0, ret+8(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 - MOVW $fn+0(FP), R0 - MOVW R0, 4(R13) - MOVW frame+4(FP), R0 - MOVW R0, 8(R13) - MOVW framesize+8(FP), R0 - MOVW R0, 12(R13) - MOVW ctxt+12(FP), R0 - MOVW R0, 16(R13) - MOVW $runtime·cgocallback_gofunc(SB), R0 - BL (R0) - RET - -// cgocallback_gofunc(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16 +TEXT ·cgocallback(SB),NOSPLIT,$12-12 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -686,7 +670,7 @@ needm: MOVW $runtime·needm(SB), R0 BL (R0) - // Set m->sched.sp = SP, so that if a panic happens + // Set m->g0->sched.sp = SP, so that if a panic happens // during the function we are about to execute, it will // have a valid SP to run on the g0 stack. // The next few lines (after the havem label) @@ -706,10 +690,10 @@ havem: // Save current m->g0->sched.sp on stack and then set it to SP. // Save current sp in m->g0->sched.sp in preparation for // switch back to m->curg stack. - // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-8(SP). + // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-12(SP). MOVW m_g0(R8), R3 MOVW (g_sched+gobuf_sp)(R3), R4 - MOVW R4, savedsp-8(SP) + MOVW R4, savedsp-12(SP) // must match frame size MOVW R13, (g_sched+gobuf_sp)(R3) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -718,30 +702,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -4(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVW m_curg(R8), R0 BL setg<>(SB) MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 MOVW (g_sched+gobuf_pc)(g), R5 - MOVW R5, -12(R4) - MOVW ctxt+12(FP), R0 - MOVW R0, -8(R4) - MOVW $-12(R4), R13 + MOVW R5, -(12+4)(R4) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVW fn+0(FP), R1 + MOVW frame+4(FP), R2 + MOVW ctxt+8(FP), R3 + MOVW $-(12+4)(R4), R13 // switch stack; must match frame size + MOVW R1, 4(R13) + MOVW R2, 8(R13) + MOVW R3, 12(R13) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVW 0(R13), R5 MOVW R5, (g_sched+gobuf_pc)(g) - MOVW $12(R13), R4 + MOVW $(12+4)(R13), R4 // must match frame size MOVW R4, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -751,7 +735,7 @@ havem: MOVW m_g0(R8), R0 BL setg<>(SB) MOVW (g_sched+gobuf_sp)(g), R13 - MOVW savedsp-8(SP), R4 + MOVW savedsp-12(SP), R4 // must match frame size MOVW R4, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_arm64.s b/src/runtime/asm_arm64.s index 1f46d1962c..a45e342478 100644 --- a/src/runtime/asm_arm64.s +++ b/src/runtime/asm_arm64.s @@ -958,25 +958,9 @@ nosave: MOVD R0, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$40-32 - MOVD $fn+0(FP), R0 - MOVD R0, 8(RSP) - MOVD frame+8(FP), R0 - MOVD R0, 16(RSP) - MOVD framesize+16(FP), R0 - MOVD R0, 24(RSP) - MOVD ctxt+24(FP), R0 - MOVD R0, 32(RSP) - MOVD $runtime·cgocallback_gofunc(SB), R0 - BL (R0) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load g from thread-local storage. @@ -1001,7 +985,7 @@ needm: MOVD $runtime·needm(SB), R0 BL (R0) - // Set m->sched.sp = SP, so that if a panic happens + // Set m->g0->sched.sp = SP, so that if a panic happens // during the function we are about to execute, it will // have a valid SP to run on the g0 stack. // The next few lines (after the havem label) @@ -1037,16 +1021,11 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVD m_curg(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 @@ -1054,10 +1033,15 @@ havem: MOVD R5, -48(R4) MOVD (g_sched+gobuf_bp)(g), R5 MOVD R5, -56(R4) - MOVD ctxt+24(FP), R0 - MOVD R0, -40(R4) + // Gather our arguments into registers. + MOVD fn+0(FP), R1 + MOVD frame+8(FP), R2 + MOVD ctxt+16(FP), R3 MOVD $-48(R4), R0 // maintain 16-byte SP alignment - MOVD R0, RSP + MOVD R0, RSP // switch stack + MOVD R1, 8(RSP) + MOVD R2, 16(RSP) + MOVD R3, 24(RSP) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. diff --git a/src/runtime/asm_mips64x.s b/src/runtime/asm_mips64x.s index 0ff1b24225..19781f7885 100644 --- a/src/runtime/asm_mips64x.s +++ b/src/runtime/asm_mips64x.s @@ -471,25 +471,9 @@ g0: MOVW R2, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - MOVV $fn+0(FP), R1 - MOVV R1, 8(R29) - MOVV frame+8(FP), R1 - MOVV R1, 16(R29) - MOVV framesize+16(FP), R1 - MOVV R1, 24(R29) - MOVV ctxt+24(FP), R1 - MOVV R1, 32(R29) - MOVV $runtime·cgocallback_gofunc(SB), R1 - JAL (R1) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -537,7 +521,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 8(R29) aka savedsp-16(SP). MOVV m_g0(R3), R1 MOVV (g_sched+gobuf_sp)(R1), R2 - MOVV R2, savedsp-16(SP) + MOVV R2, savedsp-24(SP) // must match frame size MOVV R29, (g_sched+gobuf_sp)(R1) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -546,30 +530,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVV m_curg(R3), g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R2 // prepare stack as R2 MOVV (g_sched+gobuf_pc)(g), R4 - MOVV R4, -24(R2) - MOVV ctxt+24(FP), R1 - MOVV R1, -16(R2) - MOVV $-24(R2), R29 + MOVV R4, -(24+8)(R2) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVV fn+0(FP), R5 + MOVV frame+8(FP), R6 + MOVV ctxt+16(FP), R7 + MOVV $-(24+8)(R2), R29 // switch stack; must match frame size + MOVV R5, 8(R29) + MOVV R6, 16(R29) + MOVV R7, 24(R29) JAL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVV 0(R29), R4 MOVV R4, (g_sched+gobuf_pc)(g) - MOVV $24(R29), R2 + MOVV $(24+8)(R29), R2 // must match frame size MOVV R2, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -579,7 +563,7 @@ havem: MOVV m_g0(R3), g JAL runtime·save_g(SB) MOVV (g_sched+gobuf_sp)(g), R29 - MOVV savedsp-16(SP), R2 + MOVV savedsp-24(SP), R2 // must match frame size MOVV R2, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_mipsx.s b/src/runtime/asm_mipsx.s index aca0510b69..ee87d81436 100644 --- a/src/runtime/asm_mipsx.s +++ b/src/runtime/asm_mipsx.s @@ -472,25 +472,9 @@ g0: MOVW R2, ret+8(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 - MOVW $fn+0(FP), R1 - MOVW R1, 4(R29) - MOVW frame+4(FP), R1 - MOVW R1, 8(R29) - MOVW framesize+8(FP), R1 - MOVW R1, 12(R29) - MOVW ctxt+12(FP), R1 - MOVW R1, 16(R29) - MOVW $runtime·cgocallback_gofunc(SB), R1 - JAL (R1) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16 +TEXT ·cgocallback(SB),NOSPLIT,$12-12 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -538,7 +522,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 4(R29) aka savedsp-8(SP). MOVW m_g0(R3), R1 MOVW (g_sched+gobuf_sp)(R1), R2 - MOVW R2, savedsp-8(SP) + MOVW R2, savedsp-12(SP) // must match frame size MOVW R29, (g_sched+gobuf_sp)(R1) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -547,30 +531,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -4(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVW m_curg(R3), g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R2 // prepare stack as R2 MOVW (g_sched+gobuf_pc)(g), R4 - MOVW R4, -12(R2) - MOVW ctxt+12(FP), R1 - MOVW R1, -8(R2) - MOVW $-12(R2), R29 + MOVW R4, -(12+4)(R2) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVW fn+0(FP), R5 + MOVW frame+4(FP), R6 + MOVW ctxt+8(FP), R7 + MOVW $-(12+4)(R2), R29 // switch stack; must match frame size + MOVW R5, 4(R29) + MOVW R6, 8(R29) + MOVW R7, 12(R29) JAL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVW 0(R29), R4 MOVW R4, (g_sched+gobuf_pc)(g) - MOVW $12(R29), R2 + MOVW $(12+4)(R29), R2 // must match frame size MOVW R2, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -580,7 +564,7 @@ havem: MOVW m_g0(R3), g JAL runtime·save_g(SB) MOVW (g_sched+gobuf_sp)(g), R29 - MOVW savedsp-8(SP), R2 + MOVW savedsp-12(SP), R2 // must match frame size MOVW R2, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 603058a61b..dc34c0e4c8 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -651,26 +651,9 @@ g0: MOVW R3, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - MOVD $fn+0(FP), R3 - MOVD R3, FIXED_FRAME+0(R1) - MOVD frame+8(FP), R3 - MOVD R3, FIXED_FRAME+8(R1) - MOVD framesize+16(FP), R3 - MOVD R3, FIXED_FRAME+16(R1) - MOVD ctxt+24(FP), R3 - MOVD R3, FIXED_FRAME+24(R1) - MOVD $runtime·cgocallback_gofunc(SB), R12 - MOVD R12, CTR - BL (CTR) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -721,7 +704,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). MOVD m_g0(R8), R3 MOVD (g_sched+gobuf_sp)(R3), R4 - MOVD R4, savedsp-16(SP) + MOVD R4, savedsp-24(SP) // must match frame size MOVD R1, (g_sched+gobuf_sp)(R3) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -730,30 +713,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVD m_curg(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 MOVD (g_sched+gobuf_pc)(g), R5 - MOVD R5, -(FIXED_FRAME+16)(R4) - MOVD ctxt+24(FP), R3 - MOVD R3, -16(R4) - MOVD $-(FIXED_FRAME+16)(R4), R1 + MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVD fn+0(FP), R5 + MOVD frame+8(FP), R6 + MOVD ctxt+16(FP), R7 + MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size + MOVD R5, FIXED_FRAME+0(R1) + MOVD R6, FIXED_FRAME+8(R1) + MOVD R7, FIXED_FRAME+16(R1) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVD 0(R1), R5 MOVD R5, (g_sched+gobuf_pc)(g) - MOVD $(FIXED_FRAME+16)(R1), R4 + MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -763,7 +746,7 @@ havem: MOVD m_g0(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R1 - MOVD savedsp-16(SP), R4 + MOVD savedsp-24(SP), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index a136085084..fd01fd6f07 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -453,8 +453,9 @@ TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 // traceback from goexit1 must hit code range of goexit MOV ZERO, ZERO // NOP -// func cgocallback_gofunc(fv uintptr, frame uintptr, framesize, ctxt uintptr) -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$24-32 +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) +// See cgocall.go for more details. +TEXT ·cgocallback(SB),NOSPLIT,$0-24 // TODO(jsing): Add support for cgo - issue #36641. WORD $0 // crash diff --git a/src/runtime/asm_s390x.s b/src/runtime/asm_s390x.s index 46a434119b..7baef37324 100644 --- a/src/runtime/asm_s390x.s +++ b/src/runtime/asm_s390x.s @@ -575,25 +575,9 @@ g0: MOVW R2, ret+16(FP) RET -// cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt) -// Turn the fn into a Go func (by taking its address) and call -// cgocallback_gofunc. -TEXT runtime·cgocallback(SB),NOSPLIT,$32-32 - MOVD $fn+0(FP), R3 - MOVD R3, 8(R15) - MOVD frame+8(FP), R3 - MOVD R3, 16(R15) - MOVD framesize+16(FP), R3 - MOVD R3, 24(R15) - MOVD ctxt+24(FP), R3 - MOVD R3, 32(R15) - MOVD $runtime·cgocallback_gofunc(SB), R3 - BL (R3) - RET - -// cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt) +// cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) // See cgocall.go for more details. -TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32 +TEXT ·cgocallback(SB),NOSPLIT,$24-24 NO_LOCAL_POINTERS // Load m and g from thread-local storage. @@ -641,7 +625,7 @@ havem: // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP). MOVD m_g0(R8), R3 MOVD (g_sched+gobuf_sp)(R3), R4 - MOVD R4, savedsp-16(SP) + MOVD R4, savedsp-24(SP) // must match frame size MOVD R15, (g_sched+gobuf_sp)(R3) // Switch to m->curg stack and call runtime.cgocallbackg. @@ -650,30 +634,30 @@ havem: // save that information (m->curg->sched) so we can restore it. // We can restore m->curg->sched.sp easily, because calling // runtime.cgocallbackg leaves SP unchanged upon return. - // To save m->curg->sched.pc, we push it onto the stack. - // This has the added benefit that it looks to the traceback - // routine like cgocallbackg is going to return to that - // PC (because the frame we allocate below has the same - // size as cgocallback_gofunc's frame declared above) - // so that the traceback will seamlessly trace back into - // the earlier calls. - // - // In the new goroutine, -8(SP) is unused (where SP refers to - // m->curg's SP while we're setting it up, before we've adjusted it). + // To save m->curg->sched.pc, we push it onto the curg stack and + // open a frame the same size as cgocallback's g0 frame. + // Once we switch to the curg stack, the pushed PC will appear + // to be the return PC of cgocallback, so that the traceback + // will seamlessly trace back into the earlier calls. MOVD m_curg(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4 MOVD (g_sched+gobuf_pc)(g), R5 - MOVD R5, -24(R4) - MOVD ctxt+24(FP), R5 - MOVD R5, -16(R4) - MOVD $-24(R4), R15 + MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size + // Gather our arguments into registers. + MOVD fn+0(FP), R1 + MOVD frame+8(FP), R2 + MOVD ctxt+16(FP), R3 + MOVD $-(24+8)(R4), R15 // switch stack; must match frame size + MOVD R1, 8(R15) + MOVD R2, 16(R15) + MOVD R3, 24(R15) BL runtime·cgocallbackg(SB) // Restore g->sched (== m->curg->sched) from saved values. MOVD 0(R15), R5 MOVD R5, (g_sched+gobuf_pc)(g) - MOVD $24(R15), R4 + MOVD $(24+8)(R15), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // Switch back to m->g0's stack and restore m->g0->sched.sp. @@ -683,7 +667,7 @@ havem: MOVD m_g0(R8), g BL runtime·save_g(SB) MOVD (g_sched+gobuf_sp)(g), R15 - MOVD savedsp-16(SP), R4 + MOVD savedsp-24(SP), R4 // must match frame size MOVD R4, (g_sched+gobuf_sp)(g) // If the m on entry was nil, we called needm above to borrow an m diff --git a/src/runtime/asm_wasm.s b/src/runtime/asm_wasm.s index 1275af136b..67e81adf0b 100644 --- a/src/runtime/asm_wasm.s +++ b/src/runtime/asm_wasm.s @@ -288,9 +288,6 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0 TEXT ·asmcgocall(SB), NOSPLIT, $0-0 UNDEF -TEXT ·cgocallback_gofunc(SB), NOSPLIT, $16-32 - UNDEF - #define DISPATCH(NAME, MAXSIZE) \ Get R0; \ I64Const $MAXSIZE; \ @@ -432,7 +429,7 @@ TEXT runtime·goexit(SB), NOSPLIT, $0-0 CALL runtime·goexit1(SB) // does not return UNDEF -TEXT runtime·cgocallback(SB), NOSPLIT, $32-32 +TEXT runtime·cgocallback(SB), NOSPLIT, $0-24 UNDEF // gcWriteBarrier performs a heap pointer write and informs the GC. diff --git a/src/runtime/cgo/asm_386.s b/src/runtime/cgo/asm_386.s index 7293c20bf8..2e7e9512e2 100644 --- a/src/runtime/cgo/asm_386.s +++ b/src/runtime/cgo/asm_386.s @@ -5,8 +5,9 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT,$28-16 MOVL BP, 24(SP) MOVL BX, 20(SP) @@ -15,12 +16,11 @@ TEXT crosscall2(SB),NOSPLIT,$28-16 MOVL ctxt+12(FP), AX MOVL AX, 8(SP) - MOVL n+8(FP), AX - MOVL AX, 4(SP) MOVL a+4(FP), AX - MOVL AX, 0(SP) + MOVL AX, 4(SP) MOVL fn+0(FP), AX - CALL AX + MOVL AX, 0(SP) + CALL runtime·cgocallback(SB) MOVL 12(SP), DI MOVL 16(SP), SI diff --git a/src/runtime/cgo/asm_amd64.s b/src/runtime/cgo/asm_amd64.s index 06c538b9bc..5dc8e2d235 100644 --- a/src/runtime/cgo/asm_amd64.s +++ b/src/runtime/cgo/asm_amd64.s @@ -5,8 +5,10 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. +// This signature is known to SWIG, so we can't change it. #ifndef GOOS_windows TEXT crosscall2(SB),NOSPLIT,$0x50-0 /* keeps stack pointer 32-byte aligned */ #else @@ -33,11 +35,12 @@ TEXT crosscall2(SB),NOSPLIT,$0x110-0 /* also need to save xmm6 - xmm15 */ MOVUPS X14, 0xe0(SP) MOVUPS X15, 0xf0(SP) - MOVQ DX, 0x0(SP) /* arg */ - MOVQ R8, 0x8(SP) /* argsize (includes padding) */ + MOVQ CX, 0x0(SP) /* fn */ + MOVQ DX, 0x8(SP) /* arg */ + // Skip n in R8. MOVQ R9, 0x10(SP) /* ctxt */ - CALL CX /* fn */ + CALL runtime·cgocallback(SB) MOVQ 0x48(SP), DI MOVQ 0x50(SP), SI @@ -52,11 +55,12 @@ TEXT crosscall2(SB),NOSPLIT,$0x110-0 /* also need to save xmm6 - xmm15 */ MOVUPS 0xe0(SP), X14 MOVUPS 0xf0(SP), X15 #else - MOVQ SI, 0x0(SP) /* arg */ - MOVQ DX, 0x8(SP) /* argsize (includes padding) */ + MOVQ DI, 0x0(SP) /* fn */ + MOVQ SI, 0x8(SP) /* arg */ + // Skip n in DX. MOVQ CX, 0x10(SP) /* ctxt */ - CALL DI /* fn */ + CALL runtime·cgocallback(SB) #endif MOVQ 0x18(SP), BX diff --git a/src/runtime/cgo/asm_arm.s b/src/runtime/cgo/asm_arm.s index 60132c14a8..ea55e173c1 100644 --- a/src/runtime/cgo/asm_arm.s +++ b/src/runtime/cgo/asm_arm.s @@ -5,51 +5,52 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 - /* - * We still need to save all callee save register as before, and then - * push 3 args for fn (R1, R2, R3). - * Also note that at procedure entry in gc world, 4(R13) will be the - * first arg, so we must push another dummy reg (R0) for 0(R13). - * Additionally, runtime·load_g will clobber R0, so we need to save R0 - * nevertheless. - */ SUB $(8*9), R13 // Reserve space for the floating point registers. - MOVM.WP [R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, g, R11, R12, R14], (R13) + // The C arguments arrive in R0, R1, R2, and R3. We want to + // pass R0, R1, and R3 to Go, so we push those on the stack. + // Also, save C callee-save registers R4-R12. + MOVM.WP [R0, R1, R3, R4, R5, R6, R7, R8, R9, g, R11, R12], (R13) + // Finally, save the link register R14. This also puts the + // arguments we pushed for cgocallback where they need to be, + // starting at 4(R13). + MOVW.W R14, -4(R13) // Skip floating point registers on GOARM < 6. MOVB runtime·goarm(SB), R11 CMP $6, R11 BLT skipfpsave - MOVD F8, (14*4+8*1)(R13) - MOVD F9, (14*4+8*2)(R13) - MOVD F10, (14*4+8*3)(R13) - MOVD F11, (14*4+8*4)(R13) - MOVD F12, (14*4+8*5)(R13) - MOVD F13, (14*4+8*6)(R13) - MOVD F14, (14*4+8*7)(R13) - MOVD F15, (14*4+8*8)(R13) + MOVD F8, (13*4+8*1)(R13) + MOVD F9, (13*4+8*2)(R13) + MOVD F10, (13*4+8*3)(R13) + MOVD F11, (13*4+8*4)(R13) + MOVD F12, (13*4+8*5)(R13) + MOVD F13, (13*4+8*6)(R13) + MOVD F14, (13*4+8*7)(R13) + MOVD F15, (13*4+8*8)(R13) skipfpsave: BL runtime·load_g(SB) - MOVW R15, R14 // R15 is PC. - MOVW 0(R13), R15 + // We set up the arguments to cgocallback when saving registers above. + BL runtime·cgocallback(SB) MOVB runtime·goarm(SB), R11 CMP $6, R11 BLT skipfprest - MOVD (14*4+8*1)(R13), F8 - MOVD (14*4+8*2)(R13), F9 - MOVD (14*4+8*3)(R13), F10 - MOVD (14*4+8*4)(R13), F11 - MOVD (14*4+8*5)(R13), F12 - MOVD (14*4+8*6)(R13), F13 - MOVD (14*4+8*7)(R13), F14 - MOVD (14*4+8*8)(R13), F15 + MOVD (13*4+8*1)(R13), F8 + MOVD (13*4+8*2)(R13), F9 + MOVD (13*4+8*3)(R13), F10 + MOVD (13*4+8*4)(R13), F11 + MOVD (13*4+8*5)(R13), F12 + MOVD (13*4+8*6)(R13), F13 + MOVD (13*4+8*7)(R13), F14 + MOVD (13*4+8*8)(R13), F15 skipfprest: - MOVM.IAW (R13), [R0, R1, R2, R3, R4, R5, R6, R7, R8, R9, g, R11, R12, R14] + MOVW.P 4(R13), R14 + MOVM.IAW (R13), [R0, R1, R3, R4, R5, R6, R7, R8, R9, g, R11, R12] ADD $(8*9), R13 MOVW R14, R15 diff --git a/src/runtime/cgo/asm_arm64.s b/src/runtime/cgo/asm_arm64.s index ce56f9b1c7..1cb25cf89e 100644 --- a/src/runtime/cgo/asm_arm64.s +++ b/src/runtime/cgo/asm_arm64.s @@ -5,19 +5,20 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 /* * We still need to save all callee save register as before, and then - * push 3 args for fn (R1, R2, R3). + * push 3 args for fn (R0, R1, R3), skipping R2. * Also note that at procedure entry in gc world, 8(RSP) will be the * first arg. * TODO(minux): use LDP/STP here if it matters. */ SUB $(8*24), RSP - MOVD R1, (8*1)(RSP) - MOVD R2, (8*2)(RSP) + MOVD R0, (8*1)(RSP) + MOVD R1, (8*2)(RSP) MOVD R3, (8*3)(RSP) MOVD R19, (8*4)(RSP) MOVD R20, (8*5)(RSP) @@ -40,15 +41,11 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 FMOVD F14, (8*22)(RSP) FMOVD F15, (8*23)(RSP) - MOVD R0, R19 - // Initialize Go ABI environment BL runtime·load_g(SB) - BL (R19) - MOVD (8*1)(RSP), R1 - MOVD (8*2)(RSP), R2 - MOVD (8*3)(RSP), R3 + BL runtime·cgocallback(SB) + MOVD (8*4)(RSP), R19 MOVD (8*5)(RSP), R20 MOVD (8*6)(RSP), R21 diff --git a/src/runtime/cgo/asm_mips64x.s b/src/runtime/cgo/asm_mips64x.s index 1235852dbe..e51cdf3d12 100644 --- a/src/runtime/cgo/asm_mips64x.s +++ b/src/runtime/cgo/asm_mips64x.s @@ -6,14 +6,14 @@ #include "textflag.h" -/* - * void crosscall2(void (*fn)(void*, int32, uintptr), void*, int32, uintptr) - * Save registers and call fn with two arguments. - */ +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 /* * We still need to save all callee save register as before, and then - * push 3 args for fn (R5, R6, R7). + * push 3 args for fn (R4, R5, R7), skipping R6. * Also note that at procedure entry in gc world, 8(R29) will be the * first arg. */ @@ -22,9 +22,9 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 #else ADDV $(-8*15), R29 #endif - MOVV R5, (8*1)(R29) // void* - MOVW R6, (8*2)(R29) // int32 - MOVV R7, (8*3)(R29) // uintptr + MOVV R4, (8*1)(R29) // fn unsafe.Pointer + MOVV R5, (8*2)(R29) // a unsafe.Pointer + MOVV R7, (8*3)(R29) // ctxt uintptr MOVV R16, (8*4)(R29) MOVV R17, (8*5)(R29) MOVV R18, (8*6)(R29) @@ -52,7 +52,8 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 SRLV $32, R31, RSB SLLV $32, RSB JAL runtime·load_g(SB) - JAL (R4) + + JAL runtime·cgocallback(SB) MOVV (8*4)(R29), R16 MOVV (8*5)(R29), R17 diff --git a/src/runtime/cgo/asm_mipsx.s b/src/runtime/cgo/asm_mipsx.s index e3090da223..1127c8beb4 100644 --- a/src/runtime/cgo/asm_mipsx.s +++ b/src/runtime/cgo/asm_mipsx.s @@ -6,14 +6,14 @@ #include "textflag.h" -/* - * void crosscall2(void (*fn)(void*, int32, uintptr), void*, int32, uintptr) - * Save registers and call fn with two arguments. - */ +// Called by C code generated by cmd/cgo. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 /* * We still need to save all callee save register as before, and then - * push 3 args for fn (R5, R6, R7). + * push 3 args for fn (R4, R5, R7), skipping R6. * Also note that at procedure entry in gc world, 4(R29) will be the * first arg. */ @@ -25,9 +25,9 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 #else SUBU $(4*14-16), R29 // For soft-float, no FPR. #endif - MOVW R5, (4*1)(R29) - MOVW R6, (4*2)(R29) - MOVW R7, (4*3)(R29) + MOVW R4, (4*1)(R29) // fn unsafe.Pointer + MOVW R5, (4*2)(R29) // a unsafe.Pointer + MOVW R7, (4*3)(R29) // ctxt uintptr MOVW R16, (4*4)(R29) MOVW R17, (4*5)(R29) MOVW R18, (4*6)(R29) @@ -47,7 +47,8 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 MOVD F30, (4*14+8*5)(R29) #endif JAL runtime·load_g(SB) - JAL (R4) + + JAL runtime·cgocallback(SB) MOVW (4*4)(R29), R16 MOVW (4*5)(R29), R17 diff --git a/src/runtime/cgo/asm_ppc64x.s b/src/runtime/cgo/asm_ppc64x.s index 3876f9389c..f4efc1e67d 100644 --- a/src/runtime/cgo/asm_ppc64x.s +++ b/src/runtime/cgo/asm_ppc64x.s @@ -8,8 +8,9 @@ #include "asm_ppc64x.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 // Start with standard C stack frame layout and linkage MOVD LR, R0 @@ -26,19 +27,18 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 BL runtime·reginit(SB) BL runtime·load_g(SB) - MOVD R3, R12 #ifdef GOARCH_ppc64 // ppc64 use elf ABI v1. we must get the real entry address from // first slot of the function descriptor before call. // Same for AIX. - MOVD 8(R12), R2 - MOVD (R12), R12 + MOVD 8(R3), R2 + MOVD (R3), R3 #endif - MOVD R12, CTR - MOVD R4, FIXED_FRAME+0(R1) - MOVW R5, FIXED_FRAME+8(R1) - MOVD R6, FIXED_FRAME+16(R1) - BL (CTR) + MOVD R3, FIXED_FRAME+0(R1) // fn unsafe.Pointer + MOVD R4, FIXED_FRAME+8(R1) // a unsafe.Pointer + // Skip R5 = n uint32 + MOVD R6, FIXED_FRAME+16(R1) // ctxt uintptr + BL runtime·cgocallback(SB) ADD $(288+3*8+FIXED_FRAME), R1 diff --git a/src/runtime/cgo/asm_s390x.s b/src/runtime/cgo/asm_s390x.s index 7eab8f652a..8bf16e75e2 100644 --- a/src/runtime/cgo/asm_s390x.s +++ b/src/runtime/cgo/asm_s390x.s @@ -5,8 +5,9 @@ #include "textflag.h" // Called by C code generated by cmd/cgo. -// func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) -// Saves C callee-saved registers and calls fn with three arguments. +// func crosscall2(fn, a unsafe.Pointer, n int32, ctxt uintptr) +// Saves C callee-saved registers and calls cgocallback with three arguments. +// fn is the PC of a func(a unsafe.Pointer) function. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 // Start with standard C stack frame layout and linkage. @@ -29,10 +30,11 @@ TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 // Initialize Go ABI environment. BL runtime·load_g(SB) - MOVD R3, 8(R15) // arg1 - MOVW R4, 16(R15) // arg2 - MOVD R5, 24(R15) // arg3 - BL (R2) // fn(arg1, arg2, arg3) + MOVD R2, 8(R15) // fn unsafe.Pointer + MOVD R3, 16(R15) // a unsafe.Pointer + // Skip R4 = n uint32 + MOVD R5, 24(R15) // ctxt uintptr + BL runtime·cgocallback(SB) FMOVD 32(R15), F8 FMOVD 40(R15), F9 diff --git a/src/runtime/cgo/callbacks.go b/src/runtime/cgo/callbacks.go index 14a218ec92..cd8b795387 100644 --- a/src/runtime/cgo/callbacks.go +++ b/src/runtime/cgo/callbacks.go @@ -9,20 +9,18 @@ import "unsafe" // These utility functions are available to be called from code // compiled with gcc via crosscall2. -// cgocallback is defined in runtime -//go:linkname _runtime_cgocallback runtime.cgocallback -func _runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) - // The declaration of crosscall2 is: -// void crosscall2(void (*fn)(void *, int), void *, int); +// void crosscall2(void (*fn)(void *), void *, int); // // We need to export the symbol crosscall2 in order to support // callbacks from shared libraries. This applies regardless of // linking mode. // -// Compatibility note: crosscall2 actually takes four arguments, but -// it works to call it with three arguments when calling _cgo_panic. -// That is supported for backward compatibility. +// Compatibility note: SWIG uses crosscall2 in exactly one situation: +// to call _cgo_panic using the pattern shown below. We need to keep +// that pattern working. In particular, crosscall2 actually takes four +// arguments, but it works to call it with three arguments when +// calling _cgo_panic. //go:cgo_export_static crosscall2 //go:cgo_export_dynamic crosscall2 @@ -34,21 +32,18 @@ func _runtime_cgocallback(unsafe.Pointer, unsafe.Pointer, uintptr, uintptr) // crosscall2(_cgo_panic, &a, sizeof a); // /* The function call will not return. */ +// TODO: We should export a regular C function to panic, change SWIG +// to use that instead of the above pattern, and then we can drop +// backwards-compatibility from crosscall2 and stop exporting it. + //go:linkname _runtime_cgo_panic_internal runtime._cgo_panic_internal func _runtime_cgo_panic_internal(p *byte) //go:linkname _cgo_panic _cgo_panic //go:cgo_export_static _cgo_panic //go:cgo_export_dynamic _cgo_panic -//go:nosplit -//go:norace -func _cgo_panic(a unsafe.Pointer, n int32) { - f := _runtime_cgo_panic_internal - type funcval struct { - pc unsafe.Pointer - } - fv := *(**funcval)(unsafe.Pointer(&f)) - _runtime_cgocallback(fv.pc, a, uintptr(n), 0) +func _cgo_panic(a *struct{ cstr *byte }) { + _runtime_cgo_panic_internal(a.cstr) } //go:cgo_import_static x_cgo_init diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index 7ab42a0ed0..9bca279318 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -35,31 +35,25 @@ // cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't // know about packages). The gcc-compiled C function f calls GoF. // -// GoF calls crosscall2(_cgoexp_GoF, frame, framesize, ctxt). -// Crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from -// the gcc function call ABI to the gc function call ABI. -// It is called from gcc to call gc functions. In this case it calls -// _cgoexp_GoF(frame, framesize), still running on m.g0's stack -// and outside the $GOMAXPROCS limit. Thus, this code cannot yet -// call arbitrary Go code directly and must be careful not to allocate -// memory or use up m.g0's stack. +// GoF initializes "frame", a structure containing all of its +// arguments and slots for p.GoF's results. It calls +// crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI. // -// _cgoexp_GoF (generated by cmd/cgo) calls -// runtime.cgocallback(funcPC(p.GoF), frame, framesize, ctxt). -// (The reason for having _cgoexp_GoF instead of writing a crosscall3 -// to make this call directly is that _cgoexp_GoF, because it is compiled -// with gc instead of gcc, can refer to dotted names like -// runtime.cgocallback and p.GoF.) +// crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from +// the gcc function call ABI to the gc function call ABI. At this +// point we're in the Go runtime, but we're still running on m.g0's +// stack and outside the $GOMAXPROCS limit. crosscall2 calls +// runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI. +// (crosscall2's framesize argument is no longer used, but there's one +// case where SWIG calls crosscall2 directly and expects to pass this +// argument. See _cgo_panic.) // -// runtime.cgocallback (in asm_$GOARCH.s) turns the raw PC of p.GoF -// into a Go function value and calls runtime.cgocallback_gofunc. -// -// runtime.cgocallback_gofunc (in asm_$GOARCH.s) switches from m.g0's -// stack to the original g (m.curg)'s stack, on which it calls -// runtime.cgocallbackg(p.GoF, frame, framesize). -// As part of the stack switch, runtime.cgocallback saves the current -// SP as m.g0.sched.sp, so that any use of m.g0's stack during the -// execution of the callback will be done below the existing stack frames. +// runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack +// to the original g (m.curg)'s stack, on which it calls +// runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the +// stack switch, runtime.cgocallback saves the current SP as +// m.g0.sched.sp, so that any use of m.g0's stack during the execution +// of the callback will be done below the existing stack frames. // Before overwriting m.g0.sched.sp, it pushes the old value on the // m.g0 stack, so that it can be restored later. // @@ -67,19 +61,26 @@ // stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will // block until the $GOMAXPROCS limit allows running this goroutine. // Once exitsyscall has returned, it is safe to do things like call the memory -// allocator or invoke the Go callback function p.GoF. runtime.cgocallbackg +// allocator or invoke the Go callback function. runtime.cgocallbackg // first defers a function to unwind m.g0.sched.sp, so that if p.GoF // panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack // and the m.curg stack will be unwound in lock step. -// Then it calls p.GoF. Finally it pops but does not execute the deferred -// function, calls runtime.entersyscall, and returns to runtime.cgocallback. +// Then it calls _cgoexp_GoF(frame). +// +// _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments +// from frame, calls p.GoF, writes the results back to frame, and +// returns. Now we start unwinding this whole process. +// +// runtime.cgocallbackg pops but does not execute the deferred +// function to unwind m.g0.sched.sp, calls runtime.entersyscall, and +// returns to runtime.cgocallback. // // After it regains control, runtime.cgocallback switches back to // m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old -// m.g0.sched.sp value from the stack, and returns to _cgoexp_GoF. +// m.g0.sched.sp value from the stack, and returns to crosscall2. // -// _cgoexp_GoF immediately returns to crosscall2, which restores the -// callee-save registers for gcc and returns to GoF, which returns to f. +// crosscall2 restores the callee-save registers for gcc and returns +// to GoF, which unpacks any result values and returns to f. package runtime @@ -196,7 +197,7 @@ func cgocall(fn, arg unsafe.Pointer) int32 { // Call from C back to Go. //go:nosplit -func cgocallbackg(ctxt uintptr) { +func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { gp := getg() if gp != gp.m.curg { println("runtime: bad g in cgocallback") @@ -224,7 +225,7 @@ func cgocallbackg(ctxt uintptr) { osPreemptExtExit(gp.m) - cgocallbackg1(ctxt) + cgocallbackg1(fn, frame, ctxt) // At this point unlockOSThread has been called. // The following code must not change to a different m. @@ -239,7 +240,7 @@ func cgocallbackg(ctxt uintptr) { gp.m.syscall = syscall } -func cgocallbackg1(ctxt uintptr) { +func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { gp := getg() if gp.m.needextram || atomic.Load(&extraMWaiters) > 0 { gp.m.needextram = false @@ -283,79 +284,16 @@ func cgocallbackg1(ctxt uintptr) { raceacquire(unsafe.Pointer(&racecgosync)) } - type args struct { - fn *funcval - arg unsafe.Pointer - argsize uintptr - } - var cb *args - - // Location of callback arguments depends on stack frame layout - // and size of stack frame of cgocallback_gofunc. - sp := gp.m.g0.sched.sp - switch GOARCH { - default: - throw("cgocallbackg is unimplemented on arch") - case "arm": - // On arm, stack frame is two words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "arm64": - // On arm64, stack frame is four words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - // Additional two words (16-byte alignment) are for saving FP. - cb = (*args)(unsafe.Pointer(sp + 7*sys.PtrSize)) - case "amd64": - // On amd64, stack frame is two words, plus caller PC and BP. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "386": - // On 386, stack frame is three words, plus caller PC. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "ppc64", "ppc64le", "s390x": - // On ppc64 and s390x, the callback arguments are in the arguments area of - // cgocallback's stack frame. The stack looks like this: - // +--------------------+------------------------------+ - // | | ... | - // | cgoexp_$fn +------------------------------+ - // | | fixed frame area | - // +--------------------+------------------------------+ - // | | arguments area | - // | cgocallback +------------------------------+ <- sp + 2*minFrameSize + 2*ptrSize - // | | fixed frame area | - // +--------------------+------------------------------+ <- sp + minFrameSize + 2*ptrSize - // | | local variables (2 pointers) | - // | cgocallback_gofunc +------------------------------+ <- sp + minFrameSize - // | | fixed frame area | - // +--------------------+------------------------------+ <- sp - cb = (*args)(unsafe.Pointer(sp + 2*sys.MinFrameSize + 2*sys.PtrSize)) - case "mips64", "mips64le": - // On mips64x, stack frame is two words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - case "mips", "mipsle": - // On mipsx, stack frame is two words and there's a saved LR between - // SP and the stack frame and between the stack frame and the arguments. - cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize)) - } - - // Invoke callback. - // NOTE(rsc): passing nil for argtype means that the copying of the - // results back into cb.arg happens without any corresponding write barriers. - // For cgo, cb.arg points into a C stack frame and therefore doesn't - // hold any pointers that the GC can find anyway - the write barrier - // would be a no-op. - reflectcall(nil, unsafe.Pointer(cb.fn), cb.arg, uint32(cb.argsize), 0) + // Invoke callback. This function is generated by cmd/cgo and + // will unpack the argument frame and call the Go function. + var cb func(frame unsafe.Pointer) + cbFV := funcval{uintptr(fn)} + *(*unsafe.Pointer)(unsafe.Pointer(&cb)) = noescape(unsafe.Pointer(&cbFV)) + cb(frame) if raceenabled { racereleasemerge(unsafe.Pointer(&racecgosync)) } - if msanenabled { - // Tell msan that we wrote to the entire argument block. - // This tells msan that we set the results. - // Since we have already called the function it doesn't - // matter that we are writing to the non-result parameters. - msanwrite(cb.arg, cb.argsize) - } // Do not unwind m->g0->sched.sp. // Our caller, cgocallback, will do that. diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 83d2a524e0..c629fd45f0 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -4243,7 +4243,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { // First, it may be that the g switch has no PC update, because the SP // either corresponds to a user g throughout (as in asmcgocall) // or because it has been arranged to look like a user g frame - // (as in cgocallback_gofunc). In this case, since the entire + // (as in cgocallback). In this case, since the entire // transition is a g+SP update, a partial transition updating just one of // those will be detected by the stack bounds check. // diff --git a/src/runtime/race/output_test.go b/src/runtime/race/output_test.go index d3e7762175..b4b8936c7c 100644 --- a/src/runtime/race/output_test.go +++ b/src/runtime/race/output_test.go @@ -309,7 +309,7 @@ Read at 0x[0-9,a-f]+ by main goroutine: Previous write at 0x[0-9,a-f]+ by goroutine [0-9]: main\.goCallback\(\) .*/main\.go:27 \+0x[0-9,a-f]+ - main._cgoexpwrap_[0-9a-z]+_goCallback\(\) + _cgoexp_[0-9a-z]+_goCallback\(\) .*_cgo_gotypes\.go:[0-9]+ \+0x[0-9,a-f]+ Goroutine [0-9] \(running\) created at: diff --git a/src/runtime/stubs.go b/src/runtime/stubs.go index 6290142a41..d77cb4d460 100644 --- a/src/runtime/stubs.go +++ b/src/runtime/stubs.go @@ -148,7 +148,13 @@ func noescape(p unsafe.Pointer) unsafe.Pointer { return unsafe.Pointer(x ^ 0) } -func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr) +// Not all cgocallback frames are actually cgocallback, +// so not all have these arguments. Mark them uintptr so that the GC +// does not misinterpret memory when the arguments are not present. +// cgocallback is not called from Go, only from crosscall2. +// This in turn calls cgocallbackg, which is where we'll find +// pointer-declared arguments. +func cgocallback(fn, frame, ctxt uintptr) func gogo(buf *gobuf) func gosave(buf *gobuf) @@ -163,10 +169,11 @@ func breakpoint() // back into arg+retoffset before returning. If copying result bytes back, // the caller should pass the argument frame type as argtype, so that // call can execute appropriate write barriers during the copy. -// Package reflect passes a frame type. In package runtime, there is only -// one call that copies results back, in cgocallbackg1, and it does NOT pass a -// frame type, meaning there are no write barriers invoked. See that call -// site for justification. +// +// Package reflect always passes a frame type. In package runtime, +// Windows callbacks are the only use of this that copies results +// back, and those cannot have pointers in their results, so runtime +// passes nil for the frame type. // // Package reflect accesses this symbol through a linkname. func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32) @@ -187,14 +194,6 @@ type neverCallThisFunction struct{} // prematurely and if there is leftover state it may panic. func goexit(neverCallThisFunction) -// Not all cgocallback_gofunc frames are actually cgocallback_gofunc, -// so not all have these arguments. Mark them uintptr so that the GC -// does not misinterpret memory when the arguments are not present. -// cgocallback_gofunc is not called from go, only from cgocallback, -// so the arguments will be found via cgocallback's pointer-declared arguments. -// See the assembly implementations for more details. -func cgocallback_gofunc(fv, frame, framesize, ctxt uintptr) - // publicationBarrier performs a store/store barrier (a "publication" // or "export" barrier). Some form of synchronization is required // between initializing an object and making that object accessible to diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 84637376bf..932fba3de0 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -326,7 +326,7 @@ const ( funcID_gcBgMarkWorker funcID_systemstack_switch funcID_systemstack - funcID_cgocallback_gofunc + funcID_cgocallback funcID_gogo funcID_externalthreadhandler funcID_debugCallV1 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index 4ac1527ab1..2e5e82879c 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -239,7 +239,7 @@ GLOBL runtime·cbctxts(SB), NOPTR, $4 TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVL 0(SP), AX // will use to find our callback context - // remove return address from stack, we are not returning there + // remove return address from stack, we are not returning to callbackasm, but to its caller. ADDL $4, SP // address to callback parameters into CX @@ -251,50 +251,35 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 PUSHL BP PUSHL BX - // determine index into runtime·cbctxts table + // Go ABI requires DF flag to be cleared. + CLD + + // determine index into runtime·cbs table SUBL $runtime·callbackasm(SB), AX MOVL $0, DX MOVL $5, BX // divide by 5 because each call instruction in runtime·callbacks is 5 bytes long DIVL BX - - // find correspondent runtime·cbctxts table entry - MOVL runtime·cbctxts(SB), BX - MOVL -4(BX)(AX*4), BX - - // extract callback context - MOVL wincallbackcontext_gobody(BX), AX - MOVL wincallbackcontext_argsize(BX), DX - - // preserve whatever's at the memory location that - // the callback will use to store the return value - PUSHL 0(CX)(DX*1) - - // extend argsize by size of return value - ADDL $4, DX - - // remember how to restore stack on return - MOVL wincallbackcontext_restorestack(BX), BX - PUSHL BX - - // call target Go function - PUSHL DX // argsize (including return value) - PUSHL CX // callback parameters - PUSHL AX // address of target Go function - CLD - CALL runtime·cgocallback_gofunc(SB) - POPL AX - POPL CX - POPL DX - - // how to restore stack on return - POPL BX - - // return value into AX (as per Windows spec) - // and restore previously preserved value - MOVL -4(CX)(DX*1), AX - POPL -4(CX)(DX*1) - - MOVL BX, CX // cannot use BX anymore + SUBL $1, AX // subtract 1 because return PC is to the next slot + + // Create a struct callbackArgs on our stack. + SUBL $(12+callbackArgs__size), SP + MOVL AX, (12+callbackArgs_index)(SP) // callback index + MOVL CX, (12+callbackArgs_args)(SP) // address of args vector + MOVL $0, (12+callbackArgs_result)(SP) // result + LEAL 12(SP), AX // AX = &callbackArgs{...} + + // Call cgocallback, which will call callbackWrap(frame). + MOVL $0, 8(SP) // context + MOVL AX, 4(SP) // frame (address of callbackArgs) + LEAL ·callbackWrap(SB), AX + MOVL AX, 0(SP) // PC of function to call + CALL runtime·cgocallback(SB) + + // Get callback result. + MOVL (12+callbackArgs_result)(SP), AX + // Get popRet. + MOVL (12+callbackArgs_retPop)(SP), CX // Can't use a callee-save register + ADDL $(12+callbackArgs__size), SP // restore registers as required for windows callback POPL BX diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index 847542592b..e9ec99a51d 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -291,31 +291,20 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVQ DX, (16+8)(SP) MOVQ R8, (16+16)(SP) MOVQ R9, (16+24)(SP) + // R8 = address of args vector + LEAQ (16+0)(SP), R8 - // remove return address from stack, we are not returning there + // remove return address from stack, we are not returning to callbackasm, but to its caller. MOVQ 0(SP), AX ADDQ $8, SP - // determine index into runtime·cbctxts table + // determine index into runtime·cbs table MOVQ $runtime·callbackasm(SB), DX SUBQ DX, AX MOVQ $0, DX MOVQ $5, CX // divide by 5 because each call instruction in runtime·callbacks is 5 bytes long DIVL CX - - // find correspondent runtime·cbctxts table entry - MOVQ runtime·cbctxts(SB), CX - MOVQ -8(CX)(AX*8), AX - - // extract callback context - MOVQ wincallbackcontext_argsize(AX), DX - MOVQ wincallbackcontext_gobody(AX), AX - - // preserve whatever's at the memory location that - // the callback will use to store the return value - LEAQ 8(SP), CX // args vector, skip return address - PUSHQ 0(CX)(DX*1) // store 8 bytes from just after the args array - ADDQ $8, DX // extend argsize by size of return value + SUBQ $1, AX // subtract 1 because return PC is to the next slot // DI SI BP BX R12 R13 R14 R15 registers and DF flag are preserved // as required by windows callback convention. @@ -330,18 +319,25 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 MOVQ R14, 8(SP) MOVQ R15, 0(SP) - // prepare call stack. use SUBQ to hide from stack frame checks - // cgocallback(Go func, void *frame, uintptr framesize) - SUBQ $24, SP - MOVQ DX, 16(SP) // argsize (including return value) - MOVQ CX, 8(SP) // callback parameters - MOVQ AX, 0(SP) // address of target Go function + // Go ABI requires DF flag to be cleared. CLD - CALL runtime·cgocallback_gofunc(SB) - MOVQ 0(SP), AX - MOVQ 8(SP), CX - MOVQ 16(SP), DX - ADDQ $24, SP + + // Create a struct callbackArgs on our stack to be passed as + // the "frame" to cgocallback and on to callbackWrap. + SUBQ $(24+callbackArgs__size), SP + MOVQ AX, (24+callbackArgs_index)(SP) // callback index + MOVQ R8, (24+callbackArgs_args)(SP) // address of args vector + MOVQ $0, (24+callbackArgs_result)(SP) // result + LEAQ 24(SP), AX + // Call cgocallback, which will call callbackWrap(frame). + MOVQ $0, 16(SP) // context + MOVQ AX, 8(SP) // frame (address of callbackArgs) + LEAQ ·callbackWrap(SB), BX + MOVQ BX, 0(SP) // PC of function value to call (callbackWrap) + CALL ·cgocallback(SB) + // Get callback result. + MOVQ (24+callbackArgs_result)(SP), AX + ADDQ $(24+callbackArgs__size), SP // restore registers as required for windows callback MOVQ 0(SP), R15 @@ -355,8 +351,7 @@ TEXT runtime·callbackasm1(SB),NOSPLIT,$0 ADDQ $64, SP POPFQ - MOVQ -8(CX)(DX*1), AX // return value - POPQ -8(CX)(DX*1) // restore bytes just after the args + // The return value was placed in AX above. RET // uint32 tstart_stdcall(M *newm); diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s index 57415e1306..3fc6d27cb0 100644 --- a/src/runtime/sys_windows_arm.s +++ b/src/runtime/sys_windows_arm.s @@ -314,6 +314,9 @@ TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME,$0 GLOBL runtime·cbctxts(SB), NOPTR, $4 TEXT runtime·callbackasm1(SB),NOSPLIT|NOFRAME,$0 + // TODO(austin): This needs to be converted to match changes + // in cgocallback, but I have no way to test. See CL 258938, + // and callbackasm1 on amd64 and 386. MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} SUB $36, R13 // space for locals diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 0e2fcfb02d..ff43e7cbed 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -5,6 +5,7 @@ package runtime import ( + "runtime/internal/sys" "unsafe" ) @@ -22,10 +23,7 @@ func (c *wincallbackcontext) setCleanstack(cleanstack bool) { c.cleanstack = cleanstack } -var ( - cbs callbacks - cbctxts **wincallbackcontext = &cbs.ctxt[0] // to simplify access to cbs.ctxt in sys_windows_*.s -) +var cbs callbacks func callbackasm() @@ -53,6 +51,8 @@ func callbackasmAddr(i int) uintptr { return funcPC(callbackasm) + uintptr(i*entrySize) } +const callbackMaxArgs = 64 + //go:linkname compileCallback syscall.compileCallback func compileCallback(fn eface, cleanstack bool) (code uintptr) { if fn._type == nil || (fn._type.kind&kindMask) != kindFunc { @@ -66,6 +66,9 @@ func compileCallback(fn eface, cleanstack bool) (code uintptr) { if ft.out()[0].size != uintptrSize { panic("compileCallback: expected function with one uintptr-sized result") } + if len(ft.in()) > callbackMaxArgs { + panic("compileCallback: too many function arguments") + } argsize := uintptr(0) for _, t := range ft.in() { if t.size > uintptrSize { @@ -106,6 +109,37 @@ func compileCallback(fn eface, cleanstack bool) (code uintptr) { return r } +type callbackArgs struct { + index uintptr + args *uintptr // Arguments in stdcall/cdecl convention, with registers spilled + // Below are out-args from callbackWrap + result uintptr + retPop uintptr // For 386 cdecl, how many bytes to pop on return +} + +// callbackWrap is called by callbackasm to invoke a registered C callback. +func callbackWrap(a *callbackArgs) { + c := cbs.ctxt[a.index] + a.retPop = c.restorestack + + // Convert from stdcall to Go ABI. We assume the stack layout + // is the same, and we just need to make room for the result. + // + // TODO: This isn't a good assumption. For example, a function + // that takes two uint16 arguments will be laid out + // differently by the stdcall and Go ABIs. We should implement + // proper ABI conversion. + var frame [callbackMaxArgs + 1]uintptr + memmove(unsafe.Pointer(&frame), unsafe.Pointer(a.args), c.argsize) + + // Even though this is copying back results, we can pass a nil + // type because those results must not require write barriers. + reflectcall(nil, c.gobody, noescape(unsafe.Pointer(&frame)), sys.PtrSize+uint32(c.argsize), uint32(c.argsize)) + + // Extract the result. + a.result = frame[c.argsize/sys.PtrSize] +} + const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 // When available, this function will use LoadLibraryEx with the filename diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 94f4a44976..f3df152535 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -450,7 +450,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in } n++ - if f.funcID == funcID_cgocallback_gofunc && len(cgoCtxt) > 0 { + if f.funcID == funcID_cgocallback && len(cgoCtxt) > 0 { ctxt := cgoCtxt[len(cgoCtxt)-1] cgoCtxt = cgoCtxt[:len(cgoCtxt)-1] -- cgit v1.3 From 8414b1a5a40e5ef19508e4895b4c12a91fa498e7 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 21 Oct 2020 20:43:16 -0400 Subject: runtime: remove go115ReduceLiveness and go115RestartSeq Make them always true. Delete code that are only executed when they are false. Change-Id: I6194fa00de23486c2b0a0c9075fe3a09d9c52762 Reviewed-on: https://go-review.googlesource.com/c/go/+/264339 Trust: Cherry Zhang Reviewed-by: Austin Clements --- src/runtime/debugcall.go | 34 +++++++---------------------- src/runtime/funcdata.h | 3 +-- src/runtime/preempt.go | 57 ++++++++++++++---------------------------------- src/runtime/symtab.go | 7 +----- 4 files changed, 26 insertions(+), 75 deletions(-) (limited to 'src/runtime/symtab.go') diff --git a/src/runtime/debugcall.go b/src/runtime/debugcall.go index b5480c73ae..efc68a767d 100644 --- a/src/runtime/debugcall.go +++ b/src/runtime/debugcall.go @@ -76,32 +76,14 @@ func debugCallCheck(pc uintptr) string { return } - if !go115ReduceLiveness { - // Look up PC's register map. - pcdata := int32(-1) - if pc != f.entry { - pc-- - pcdata = pcdatavalue(f, _PCDATA_RegMapIndex, pc, nil) - } - if pcdata == -1 { - pcdata = 0 // in prologue - } - stkmap := (*stackmap)(funcdata(f, _FUNCDATA_RegPointerMaps)) - if pcdata == _PCDATA_RegMapUnsafe || stkmap == nil { - // Not at a safe point. - ret = debugCallUnsafePoint - return - } - } else { - // Check that this isn't an unsafe-point. - if pc != f.entry { - pc-- - } - up := pcdatavalue(f, _PCDATA_UnsafePoint, pc, nil) - if up != _PCDATA_UnsafePointSafe { - // Not at a safe point. - ret = debugCallUnsafePoint - } + // Check that this isn't an unsafe-point. + if pc != f.entry { + pc-- + } + up := pcdatavalue(f, _PCDATA_UnsafePoint, pc, nil) + if up != _PCDATA_UnsafePointSafe { + // Not at a safe point. + ret = debugCallUnsafePoint } }) return ret diff --git a/src/runtime/funcdata.h b/src/runtime/funcdata.h index cd76c06992..e9b6a7ed06 100644 --- a/src/runtime/funcdata.h +++ b/src/runtime/funcdata.h @@ -8,13 +8,12 @@ // // These must agree with symtab.go and ../cmd/internal/objabi/funcdata.go. -#define PCDATA_RegMapIndex 0 +#define PCDATA_UnsafePoint 0 #define PCDATA_StackMapIndex 1 #define PCDATA_InlTreeIndex 2 #define FUNCDATA_ArgsPointerMaps 0 /* garbage collector blocks */ #define FUNCDATA_LocalsPointerMaps 1 -#define FUNCDATA_RegPointerMaps 2 #define FUNCDATA_StackObjects 3 #define FUNCDATA_InlTree 4 #define FUNCDATA_OpenCodedDeferInfo 5 /* info for func with open-coded defers */ diff --git a/src/runtime/preempt.go b/src/runtime/preempt.go index 17ef2c90d3..372185266f 100644 --- a/src/runtime/preempt.go +++ b/src/runtime/preempt.go @@ -58,11 +58,6 @@ import ( "unsafe" ) -// Keep in sync with cmd/compile/internal/gc/plive.go:go115ReduceLiveness. -const go115ReduceLiveness = true - -const go115RestartSeq = go115ReduceLiveness && true // enable restartable sequences - type suspendGState struct { g *g @@ -402,24 +397,12 @@ func isAsyncSafePoint(gp *g, pc, sp, lr uintptr) (bool, uintptr) { // use the LR for unwinding, which will be bad. return false, 0 } - var up int32 - var startpc uintptr - if !go115ReduceLiveness { - smi := pcdatavalue(f, _PCDATA_RegMapIndex, pc, nil) - if smi == _PCDATA_RegMapUnsafe { - // Unsafe-point marked by compiler. This includes - // atomic sequences (e.g., write barrier) and nosplit - // functions (except at calls). - return false, 0 - } - } else { - up, startpc = pcdatavalue2(f, _PCDATA_UnsafePoint, pc) - if up != _PCDATA_UnsafePointSafe { - // Unsafe-point marked by compiler. This includes - // atomic sequences (e.g., write barrier) and nosplit - // functions (except at calls). - return false, 0 - } + up, startpc := pcdatavalue2(f, _PCDATA_UnsafePoint, pc) + if up != _PCDATA_UnsafePointSafe { + // Unsafe-point marked by compiler. This includes + // atomic sequences (e.g., write barrier) and nosplit + // functions (except at calls). + return false, 0 } if fd := funcdata(f, _FUNCDATA_LocalsPointerMaps); fd == nil || fd == unsafe.Pointer(&no_pointers_stackmap) { // This is assembly code. Don't assume it's @@ -455,25 +438,17 @@ func isAsyncSafePoint(gp *g, pc, sp, lr uintptr) (bool, uintptr) { // in incrementally. return false, 0 } - if go115RestartSeq { - switch up { - case _PCDATA_Restart1, _PCDATA_Restart2: - // Restartable instruction sequence. Back off PC to - // the start PC. - if startpc == 0 || startpc > pc || pc-startpc > 20 { - throw("bad restart PC") - } - return true, startpc - case _PCDATA_RestartAtEntry: - // Restart from the function entry at resumption. - return true, f.entry - } - } else { - switch up { - case _PCDATA_Restart1, _PCDATA_Restart2, _PCDATA_RestartAtEntry: - // go115RestartSeq is not enabled. Treat it as unsafe point. - return false, 0 + switch up { + case _PCDATA_Restart1, _PCDATA_Restart2: + // Restartable instruction sequence. Back off PC to + // the start PC. + if startpc == 0 || startpc > pc || pc-startpc > 20 { + throw("bad restart PC") } + return true, startpc + case _PCDATA_RestartAtEntry: + // Restart from the function entry at resumption. + return true, f.entry } return true, pc } diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 932fba3de0..a4b4344ea7 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -268,14 +268,12 @@ func (f *Func) funcInfo() funcInfo { // // See funcdata.h and ../cmd/internal/objabi/funcdata.go. const ( - _PCDATA_RegMapIndex = 0 // if !go115ReduceLiveness - _PCDATA_UnsafePoint = 0 // if go115ReduceLiveness + _PCDATA_UnsafePoint = 0 _PCDATA_StackMapIndex = 1 _PCDATA_InlTreeIndex = 2 _FUNCDATA_ArgsPointerMaps = 0 _FUNCDATA_LocalsPointerMaps = 1 - _FUNCDATA_RegPointerMaps = 2 // if !go115ReduceLiveness _FUNCDATA_StackObjects = 3 _FUNCDATA_InlTree = 4 _FUNCDATA_OpenCodedDeferInfo = 5 @@ -284,9 +282,6 @@ const ( ) const ( - // Only if !go115ReduceLiveness. - _PCDATA_RegMapUnsafe = _PCDATA_UnsafePointUnsafe // Unsafe for async preemption - // PCDATA_UnsafePoint values. _PCDATA_UnsafePointSafe = -1 // Safe for async preemption _PCDATA_UnsafePointUnsafe = -2 // Unsafe for async preemption -- cgit v1.3 From f96b62be2edd8acc08b79777d692937e8ed79b4a Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 27 Oct 2020 19:47:29 -0400 Subject: cmd/internal/objabi, runtime: compact FUNCDATA indices As we deleted register maps, move FUNCDATA indices of stack objects, inline trees, and open-coded defers earlier. Change-Id: If73797b8c11fd207655c9498802fca9f6f9ac338 Reviewed-on: https://go-review.googlesource.com/c/go/+/265761 Trust: Cherry Zhang Reviewed-by: Austin Clements --- src/cmd/internal/objabi/funcdata.go | 6 +++--- src/runtime/funcdata.h | 6 +++--- src/runtime/symtab.go | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'src/runtime/symtab.go') diff --git a/src/cmd/internal/objabi/funcdata.go b/src/cmd/internal/objabi/funcdata.go index 1c5e5e1c8c..faa2863325 100644 --- a/src/cmd/internal/objabi/funcdata.go +++ b/src/cmd/internal/objabi/funcdata.go @@ -17,9 +17,9 @@ const ( FUNCDATA_ArgsPointerMaps = 0 FUNCDATA_LocalsPointerMaps = 1 - FUNCDATA_StackObjects = 3 - FUNCDATA_InlTree = 4 - FUNCDATA_OpenCodedDeferInfo = 5 + FUNCDATA_StackObjects = 2 + FUNCDATA_InlTree = 3 + FUNCDATA_OpenCodedDeferInfo = 4 // ArgsSizeUnknown is set in Func.argsize to mark all functions // whose argument size is unknown (C vararg functions, and diff --git a/src/runtime/funcdata.h b/src/runtime/funcdata.h index e9b6a7ed06..798dbaceab 100644 --- a/src/runtime/funcdata.h +++ b/src/runtime/funcdata.h @@ -14,9 +14,9 @@ #define FUNCDATA_ArgsPointerMaps 0 /* garbage collector blocks */ #define FUNCDATA_LocalsPointerMaps 1 -#define FUNCDATA_StackObjects 3 -#define FUNCDATA_InlTree 4 -#define FUNCDATA_OpenCodedDeferInfo 5 /* info for func with open-coded defers */ +#define FUNCDATA_StackObjects 2 +#define FUNCDATA_InlTree 3 +#define FUNCDATA_OpenCodedDeferInfo 4 /* info for func with open-coded defers */ // Pseudo-assembly statements. diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index a4b4344ea7..7667f23f1d 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -274,9 +274,9 @@ const ( _FUNCDATA_ArgsPointerMaps = 0 _FUNCDATA_LocalsPointerMaps = 1 - _FUNCDATA_StackObjects = 3 - _FUNCDATA_InlTree = 4 - _FUNCDATA_OpenCodedDeferInfo = 5 + _FUNCDATA_StackObjects = 2 + _FUNCDATA_InlTree = 3 + _FUNCDATA_OpenCodedDeferInfo = 4 _ArgsSizeUnknown = -0x80000000 ) -- cgit v1.3