From 5387cdcb24a07f5d0d49d5105ced2b69e6aafde9 Mon Sep 17 00:00:00 2001
From: Jeremy Faller <jeremy@golang.org>
Date: Fri, 7 Aug 2020 11:31:20 -0400
Subject: [dev.link] cmd/link, cmd/compile: create content addressable pcdata
 syms

Switch pcdata over to content addressable symbols. This is the last
step before removing these from pclntab_old.

No meaningful benchmarks changes come from this work.

Change-Id: I3f74f3d6026a278babe437c8010e22992c92bd89
Reviewed-on: https://go-review.googlesource.com/c/go/+/247399
Reviewed-by: Austin Clements <austin@google.com>
Reviewed-by: Than McIntosh <thanm@google.com>
---
 src/cmd/link/internal/loader/loader.go | 46 +++++++++++++++-------------------
 1 file changed, 20 insertions(+), 26 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index 8fd10b0848..f149e3c831 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -1878,19 +1878,24 @@ func (fi *FuncInfo) FuncID() objabi.FuncID {
 	return objabi.FuncID((*goobj.FuncInfo)(nil).ReadFuncID(fi.data))
 }
 
-func (fi *FuncInfo) Pcsp() []byte {
-	pcsp, end := (*goobj.FuncInfo)(nil).ReadPcsp(fi.data)
-	return fi.r.BytesAt(fi.r.PcdataBase()+pcsp, int(end-pcsp))
+func (fi *FuncInfo) Pcsp() Sym {
+	sym := (*goobj.FuncInfo)(nil).ReadPcsp(fi.data)
+	return fi.l.resolve(fi.r, sym)
 }
 
-func (fi *FuncInfo) Pcfile() []byte {
-	pcf, end := (*goobj.FuncInfo)(nil).ReadPcfile(fi.data)
-	return fi.r.BytesAt(fi.r.PcdataBase()+pcf, int(end-pcf))
+func (fi *FuncInfo) Pcfile() Sym {
+	sym := (*goobj.FuncInfo)(nil).ReadPcfile(fi.data)
+	return fi.l.resolve(fi.r, sym)
 }
 
-func (fi *FuncInfo) Pcline() []byte {
-	pcln, end := (*goobj.FuncInfo)(nil).ReadPcline(fi.data)
-	return fi.r.BytesAt(fi.r.PcdataBase()+pcln, int(end-pcln))
+func (fi *FuncInfo) Pcline() Sym {
+	sym := (*goobj.FuncInfo)(nil).ReadPcline(fi.data)
+	return fi.l.resolve(fi.r, sym)
+}
+
+func (fi *FuncInfo) Pcinline() Sym {
+	sym := (*goobj.FuncInfo)(nil).ReadPcinline(fi.data)
+	return fi.l.resolve(fi.r, sym)
 }
 
 // Preload has to be called prior to invoking the various methods
@@ -1899,27 +1904,16 @@ func (fi *FuncInfo) Preload() {
 	fi.lengths = (*goobj.FuncInfo)(nil).ReadFuncInfoLengths(fi.data)
 }
 
-func (fi *FuncInfo) Pcinline() []byte {
+func (fi *FuncInfo) Pcdata() []Sym {
 	if !fi.lengths.Initialized {
 		panic("need to call Preload first")
 	}
-	pcinl, end := (*goobj.FuncInfo)(nil).ReadPcinline(fi.data, fi.lengths.PcdataOff)
-	return fi.r.BytesAt(fi.r.PcdataBase()+pcinl, int(end-pcinl))
-}
-
-func (fi *FuncInfo) NumPcdata() uint32 {
-	if !fi.lengths.Initialized {
-		panic("need to call Preload first")
-	}
-	return fi.lengths.NumPcdata
-}
-
-func (fi *FuncInfo) Pcdata(k int) []byte {
-	if !fi.lengths.Initialized {
-		panic("need to call Preload first")
+	syms := (*goobj.FuncInfo)(nil).ReadPcdata(fi.data)
+	ret := make([]Sym, len(syms))
+	for i := range ret {
+		ret[i] = fi.l.resolve(fi.r, syms[i])
 	}
-	pcdat, end := (*goobj.FuncInfo)(nil).ReadPcdata(fi.data, fi.lengths.PcdataOff, uint32(k))
-	return fi.r.BytesAt(fi.r.PcdataBase()+pcdat, int(end-pcdat))
+	return ret
 }
 
 func (fi *FuncInfo) NumFuncdataoff() uint32 {
-- 
cgit v1.3


From 5402d40d5b041399392b29e4543f5fc4506197bd Mon Sep 17 00:00:00 2001
From: Jeremy Faller <jeremy@golang.org>
Date: Tue, 18 Aug 2020 16:35:26 -0400
Subject: [dev.link] cmd/link: fix memory growth on dev.link
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CL 247399 caused memory growth in the linker. Fix this by adjusting how
we preallocate the number of symbols we'll need.

cmd/compile (Darwin), alloc/op:
Loadlib_GC                   33.5MB ± 0%    27.3MB ± 0%

Change-Id: I34997329ea4412716114df97fc9dad6ad0c171ee
Reviewed-on: https://go-review.googlesource.com/c/go/+/249024
Run-TryBot: Jeremy Faller <jeremy@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Austin Clements <austin@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
---
 src/cmd/link/internal/ld/lib.go        |  2 +-
 src/cmd/link/internal/loader/loader.go | 36 ++++++++++++++++++++++------------
 2 files changed, 25 insertions(+), 13 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go
index caa4566190..a01bdefa37 100644
--- a/src/cmd/link/internal/ld/lib.go
+++ b/src/cmd/link/internal/ld/lib.go
@@ -543,7 +543,7 @@ func (ctxt *Link) loadlib() {
 	}
 
 	// Add non-package symbols and references of externally defined symbols.
-	ctxt.loader.LoadNonpkgSyms(ctxt.Arch)
+	ctxt.loader.LoadSyms(ctxt.Arch)
 
 	// Load symbols from shared libraries, after all Go object symbols are loaded.
 	for _, lib := range ctxt.Library {
diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index f149e3c831..ea9cd1bd2e 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -328,7 +328,7 @@ func NewLoader(flags uint32, elfsetstring elfsetstringFunc, reporter *ErrorRepor
 	ldr := &Loader{
 		start:                make(map[*oReader]Sym),
 		objs:                 []objIdx{{}, {extReader, 0}}, // reserve index 0 for nil symbol, 1 for external symbols
-		objSyms:              make([]objSym, 1, 100000),    // reserve index 0 for nil symbol
+		objSyms:              make([]objSym, 1, 1),         // This will get overwritten later.
 		extReader:            extReader,
 		symsByName:           [2]map[string]Sym{make(map[string]Sym, 80000), make(map[string]Sym, 50000)}, // preallocate ~2MB for ABI0 and ~1MB for ABI1 symbols
 		objByPkg:             make(map[string]*oReader),
@@ -2016,8 +2016,9 @@ func (l *Loader) FuncInfo(i Sym) FuncInfo {
 	return FuncInfo{}
 }
 
-// Preload a package: add autolibs, add defined package symbols to the symbol table.
-// Does not add non-package symbols yet, which will be done in LoadNonpkgSyms.
+// Preload a package: adds autolib.
+// Does not add defined package or non-packaged symbols to the symbol table.
+// These are done in LoadSyms.
 // Does not read symbol data.
 // Returns the fingerprint of the object.
 func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, unit *sym.CompilationUnit, length int64) goobj.FingerprintType {
@@ -2060,8 +2061,6 @@ func (l *Loader) Preload(localSymVersion int, f *bio.Reader, lib *sym.Library, u
 	}
 
 	l.addObj(lib.Pkg, or)
-	st := loadState{l: l}
-	st.preloadSyms(or, pkgDef)
 
 	// The caller expects us consuming all the data
 	f.MustSeek(length, os.SEEK_CUR)
@@ -2144,17 +2143,30 @@ func (st *loadState) preloadSyms(r *oReader, kind int) {
 	}
 }
 
-// Add hashed (content-addressable) symbols, non-package symbols, and
+// Add syms, hashed (content-addressable) symbols, non-package symbols, and
 // references to external symbols (which are always named).
-func (l *Loader) LoadNonpkgSyms(arch *sys.Arch) {
+func (l *Loader) LoadSyms(arch *sys.Arch) {
+	// Allocate space for symbols, making a guess as to how much space we need.
+	// This function was determined empirically by looking at the cmd/compile on
+	// Darwin, and picking factors for hashed and hashed64 syms.
+	var symSize, hashedSize, hashed64Size int
+	for _, o := range l.objs[goObjStart:] {
+		symSize += o.r.ndef + o.r.nhasheddef/2 + o.r.nhashed64def/2 + o.r.NNonpkgdef()
+		hashedSize += o.r.nhasheddef / 2
+		hashed64Size += o.r.nhashed64def / 2
+	}
+	// Index 0 is invalid for symbols.
+	l.objSyms = make([]objSym, 1, symSize)
+
 	l.npkgsyms = l.NSym()
-	// Preallocate some space (a few hundreds KB) for some symbols.
-	// As of Go 1.15, linking cmd/compile has ~8000 hashed64 symbols and
-	// ~13000 hashed symbols.
 	st := loadState{
 		l:            l,
-		hashed64Syms: make(map[uint64]symAndSize, 10000),
-		hashedSyms:   make(map[goobj.HashType]symAndSize, 15000),
+		hashed64Syms: make(map[uint64]symAndSize, hashed64Size),
+		hashedSyms:   make(map[goobj.HashType]symAndSize, hashedSize),
+	}
+
+	for _, o := range l.objs[goObjStart:] {
+		st.preloadSyms(o.r, pkgDef)
 	}
 	for _, o := range l.objs[goObjStart:] {
 		st.preloadSyms(o.r, hashed64Def)
-- 
cgit v1.3


From 83ed734df03fd11d71f06bc02906a723afaf3936 Mon Sep 17 00:00:00 2001
From: Cherry Zhang <cherryyz@google.com>
Date: Tue, 11 Aug 2020 19:57:50 -0400
Subject: cmd/link: pre-resolve package reference

Pre-resolve package index references, so it doesn't need to do a
map lookup in every cross-package symbol reference resolution. It
increases the memory usage very slightly (O(# imported packages)).

Change-Id: Ia76c97ac51f1c2c2d5ea7ae34853850ec69ef0a8
Reviewed-on: https://go-review.googlesource.com/c/go/+/253604
Run-TryBot: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Than McIntosh <thanm@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
---
 src/cmd/link/internal/loader/loader.go | 36 +++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index 8fd10b0848..43a0352e0b 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -93,11 +93,12 @@ type oReader struct {
 	version      int    // version of static symbol
 	flags        uint32 // read from object file
 	pkgprefix    string
-	syms         []Sym  // Sym's global index, indexed by local index
-	ndef         int    // cache goobj.Reader.NSym()
-	nhashed64def int    // cache goobj.Reader.NHashed64Def()
-	nhasheddef   int    // cache goobj.Reader.NHashedDef()
-	objidx       uint32 // index of this reader in the objs slice
+	syms         []Sym    // Sym's global index, indexed by local index
+	pkg          []uint32 // indices of referenced package by PkgIdx (index into loader.objs array)
+	ndef         int      // cache goobj.Reader.NSym()
+	nhashed64def int      // cache goobj.Reader.NHashed64Def()
+	nhasheddef   int      // cache goobj.Reader.NHashedDef()
+	objidx       uint32   // index of this reader in the objs slice
 }
 
 // Total number of defined symbols (package symbols, hashed symbols, and
@@ -219,7 +220,7 @@ type Loader struct {
 
 	deferReturnTramp map[Sym]bool // whether the symbol is a trampoline of a deferreturn call
 
-	objByPkg map[string]*oReader // map package path to its Go object reader
+	objByPkg map[string]uint32 // map package path to the index of its Go object reader
 
 	anonVersion int // most recently assigned ext static sym pseudo-version
 
@@ -331,7 +332,7 @@ func NewLoader(flags uint32, elfsetstring elfsetstringFunc, reporter *ErrorRepor
 		objSyms:              make([]objSym, 1, 100000),    // reserve index 0 for nil symbol
 		extReader:            extReader,
 		symsByName:           [2]map[string]Sym{make(map[string]Sym, 80000), make(map[string]Sym, 50000)}, // preallocate ~2MB for ABI0 and ~1MB for ABI1 symbols
-		objByPkg:             make(map[string]*oReader),
+		objByPkg:             make(map[string]uint32),
 		outer:                make(map[Sym]Sym),
 		sub:                  make(map[Sym]Sym),
 		dynimplib:            make(map[Sym]string),
@@ -370,7 +371,7 @@ func (l *Loader) addObj(pkg string, r *oReader) Sym {
 	}
 	pkg = objabi.PathToPrefix(pkg) // the object file contains escaped package path
 	if _, ok := l.objByPkg[pkg]; !ok {
-		l.objByPkg[pkg] = r
+		l.objByPkg[pkg] = r.objidx
 	}
 	i := Sym(len(l.objSyms))
 	l.start[r] = i
@@ -635,12 +636,7 @@ func (l *Loader) resolve(r *oReader, s goobj.SymRef) Sym {
 	case goobj.PkgIdxSelf:
 		rr = r
 	default:
-		pkg := r.Pkg(int(p))
-		var ok bool
-		rr, ok = l.objByPkg[pkg]
-		if !ok {
-			log.Fatalf("reference of nonexisted package %s, from %v", pkg, r.unit.Lib)
-		}
+		rr = l.objs[r.pkg[p]].r
 	}
 	return l.toGlobal(rr, s.SymIdx)
 }
@@ -2195,6 +2191,18 @@ func loadObjRefs(l *Loader, r *oReader, arch *sys.Arch) {
 		}
 	}
 
+	// referenced packages
+	npkg := r.NPkg()
+	r.pkg = make([]uint32, npkg)
+	for i := 1; i < npkg; i++ { // PkgIdx 0 is a dummy invalid package
+		pkg := r.Pkg(i)
+		objidx, ok := l.objByPkg[pkg]
+		if !ok {
+			log.Fatalf("reference of nonexisted package %s, from %v", pkg, r.unit.Lib)
+		}
+		r.pkg[i] = objidx
+	}
+
 	// load flags of package refs
 	for i, n := 0, r.NRefFlags(); i < n; i++ {
 		rf := r.RefFlags(i)
-- 
cgit v1.3


From af18bce87cc7ee1ffc68f91abefa241ab209539e Mon Sep 17 00:00:00 2001
From: Cherry Zhang <cherryyz@google.com>
Date: Sun, 20 Sep 2020 23:29:20 -0400
Subject: cmd/link: consider interface conversions only in reachable code

The linker prunes methods that are not directly reachable if the
receiver type is never converted to interface. Currently, this
"never" is too strong: it is invalidated even if the interface
conversion is in an unreachable function. This CL improves it by
only considering interface conversions in reachable code. To do
that, we introduce a marker relocation R_USEIFACE, which marks
the target symbol as UsedInIface if the source symbol is reached.

binary size    before      after
cmd/compile   18897528   18887400
cmd/go        13607372   13470652

Change-Id: I66c6b69eeff9ae02d84d2e6f2bc7f1b29dd53910
Reviewed-on: https://go-review.googlesource.com/c/go/+/256797
Trust: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Jeremy Faller <jeremy@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>
---
 src/cmd/compile/internal/gc/pgen.go                |  8 ++-
 src/cmd/compile/internal/gc/sinit.go               |  2 +-
 src/cmd/compile/internal/gc/walk.go                | 15 +++--
 src/cmd/internal/obj/s390x/asmz.go                 |  6 +-
 src/cmd/internal/obj/wasm/wasmobj.go               |  2 +
 src/cmd/internal/obj/x86/asm6.go                   |  7 ++-
 src/cmd/internal/objabi/reloctype.go               |  5 ++
 src/cmd/internal/objabi/reloctype_string.go        | 66 +++++++++++++++++++++-
 src/cmd/link/internal/ld/data.go                   |  6 ++
 src/cmd/link/internal/ld/deadcode.go               | 14 +++++
 .../internal/ld/testdata/deadcode/ifacemethod.go   |  9 ++-
 src/cmd/link/internal/loader/loader.go             |  1 +
 src/cmd/link/internal/wasm/asm.go                  |  3 +
 13 files changed, 129 insertions(+), 15 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go
index 74262595b0..52b1ed351d 100644
--- a/src/cmd/compile/internal/gc/pgen.go
+++ b/src/cmd/compile/internal/gc/pgen.go
@@ -231,6 +231,11 @@ func compile(fn *Node) {
 		return
 	}
 
+	// Set up the function's LSym early to avoid data races with the assemblers.
+	// Do this before walk, as walk needs the LSym to set attributes/relocations
+	// (e.g. in markTypeUsedInInterface).
+	fn.Func.initLSym(true)
+
 	walk(fn)
 	if nerrors != 0 {
 		return
@@ -250,9 +255,6 @@ func compile(fn *Node) {
 		return
 	}
 
-	// Set up the function's LSym early to avoid data races with the assemblers.
-	fn.Func.initLSym(true)
-
 	// Make sure type syms are declared for all types that might
 	// be types of stack objects. We need to do this here
 	// because symbols must be allocated before the parallel
diff --git a/src/cmd/compile/internal/gc/sinit.go b/src/cmd/compile/internal/gc/sinit.go
index 71ed558461..af19a96bbc 100644
--- a/src/cmd/compile/internal/gc/sinit.go
+++ b/src/cmd/compile/internal/gc/sinit.go
@@ -278,7 +278,7 @@ func (s *InitSchedule) staticassign(l *Node, r *Node) bool {
 			return Isconst(val, CTNIL)
 		}
 
-		markTypeUsedInInterface(val.Type)
+		markTypeUsedInInterface(val.Type, l.Sym.Linksym())
 
 		var itab *Node
 		if l.Type.IsEmptyInterface() {
diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go
index 933f16d9a0..d238cc2f45 100644
--- a/src/cmd/compile/internal/gc/walk.go
+++ b/src/cmd/compile/internal/gc/walk.go
@@ -805,8 +805,8 @@ opswitch:
 		fromType := n.Left.Type
 		toType := n.Type
 
-		if !fromType.IsInterface() {
-			markTypeUsedInInterface(fromType)
+		if !fromType.IsInterface() && !Curfn.Func.Nname.isBlank() { // skip unnamed functions (func _())
+			markTypeUsedInInterface(fromType, Curfn.Func.lsym)
 		}
 
 		// typeword generates the type word of the interface value.
@@ -1621,8 +1621,13 @@ opswitch:
 
 // markTypeUsedInInterface marks that type t is converted to an interface.
 // This information is used in the linker in dead method elimination.
-func markTypeUsedInInterface(t *types.Type) {
-	typenamesym(t).Linksym().Set(obj.AttrUsedInIface, true)
+func markTypeUsedInInterface(t *types.Type, from *obj.LSym) {
+	tsym := typenamesym(t).Linksym()
+	// Emit a marker relocation. The linker will know the type is converted
+	// to an interface if "from" is reachable.
+	r := obj.Addrel(from)
+	r.Sym = tsym
+	r.Type = objabi.R_USEIFACE
 }
 
 // rtconvfn returns the parameter and result types that will be used by a
@@ -3687,6 +3692,8 @@ func usemethod(n *Node) {
 	// Also need to check for reflect package itself (see Issue #38515).
 	if s := res0.Type.Sym; s != nil && s.Name == "Method" && isReflectPkg(s.Pkg) {
 		Curfn.Func.SetReflectMethod(true)
+		// The LSym is initialized at this point. We need to set the attribute on the LSym.
+		Curfn.Func.lsym.Set(obj.AttrReflectMethod, true)
 	}
 }
 
diff --git a/src/cmd/internal/obj/s390x/asmz.go b/src/cmd/internal/obj/s390x/asmz.go
index 68f01f1c5d..cb3a2c3196 100644
--- a/src/cmd/internal/obj/s390x/asmz.go
+++ b/src/cmd/internal/obj/s390x/asmz.go
@@ -461,6 +461,7 @@ func spanz(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
 	buffer := make([]byte, 0)
 	changed := true
 	loop := 0
+	nrelocs0 := len(c.cursym.R)
 	for changed {
 		if loop > 100 {
 			c.ctxt.Diag("stuck in spanz loop")
@@ -468,7 +469,10 @@ func spanz(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) {
 		}
 		changed = false
 		buffer = buffer[:0]
-		c.cursym.R = make([]obj.Reloc, 0)
+		for i := range c.cursym.R[nrelocs0:] {
+			c.cursym.R[nrelocs0+i] = obj.Reloc{}
+		}
+		c.cursym.R = c.cursym.R[:nrelocs0] // preserve marker relocations generated by the compiler
 		for p := c.cursym.Func.Text; p != nil; p = p.Link {
 			pc := int64(len(buffer))
 			if pc != p.Pc {
diff --git a/src/cmd/internal/obj/wasm/wasmobj.go b/src/cmd/internal/obj/wasm/wasmobj.go
index 70e8e51e65..a9e093a8ad 100644
--- a/src/cmd/internal/obj/wasm/wasmobj.go
+++ b/src/cmd/internal/obj/wasm/wasmobj.go
@@ -1007,6 +1007,7 @@ func assemble(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) {
 					panic("bad name for Call")
 				}
 				r := obj.Addrel(s)
+				r.Siz = 1 // actually variable sized
 				r.Off = int32(w.Len())
 				r.Type = objabi.R_CALL
 				if p.Mark&WasmImport != 0 {
@@ -1033,6 +1034,7 @@ func assemble(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) {
 		case AI32Const, AI64Const:
 			if p.From.Name == obj.NAME_EXTERN {
 				r := obj.Addrel(s)
+				r.Siz = 1 // actually variable sized
 				r.Off = int32(w.Len())
 				r.Type = objabi.R_ADDR
 				r.Sym = p.From.Sym
diff --git a/src/cmd/internal/obj/x86/asm6.go b/src/cmd/internal/obj/x86/asm6.go
index fb99c620ad..4940c79eaa 100644
--- a/src/cmd/internal/obj/x86/asm6.go
+++ b/src/cmd/internal/obj/x86/asm6.go
@@ -2100,14 +2100,15 @@ func span6(ctxt *obj.Link, s *obj.LSym, newprog obj.ProgAlloc) {
 	var c int32
 	errors := ctxt.Errors
 	var nops []nopPad // Padding for a particular assembly (reuse slice storage if multiple assemblies)
+	nrelocs0 := len(s.R)
 	for {
 		// This loop continues while there are reasons to re-assemble
 		// whole block, like the presence of long forward jumps.
 		reAssemble := false
-		for i := range s.R {
-			s.R[i] = obj.Reloc{}
+		for i := range s.R[nrelocs0:] {
+			s.R[nrelocs0+i] = obj.Reloc{}
 		}
-		s.R = s.R[:0]
+		s.R = s.R[:nrelocs0] // preserve marker relocations generated by the compiler
 		s.P = s.P[:0]
 		c = 0
 		var pPrev *obj.Prog
diff --git a/src/cmd/internal/objabi/reloctype.go b/src/cmd/internal/objabi/reloctype.go
index f029a3c396..1e328d659f 100644
--- a/src/cmd/internal/objabi/reloctype.go
+++ b/src/cmd/internal/objabi/reloctype.go
@@ -89,6 +89,11 @@ const (
 	// should be linked into the final binary, even if there are no other
 	// direct references. (This is used for types reachable by reflection.)
 	R_USETYPE
+	// R_USEIFACE marks a type is converted to an interface in the function this
+	// relocation is applied to. The target is a type descriptor.
+	// This is a marker relocation (0-sized), for the linker's reachabililty
+	// analysis.
+	R_USEIFACE
 	// R_METHODOFF resolves to a 32-bit offset from the beginning of the section
 	// holding the data being relocated to the referenced symbol.
 	// It is a variant of R_ADDROFF used when linking from the uncommonType of a
diff --git a/src/cmd/internal/objabi/reloctype_string.go b/src/cmd/internal/objabi/reloctype_string.go
index 83dfe71e07..caf24eea58 100644
--- a/src/cmd/internal/objabi/reloctype_string.go
+++ b/src/cmd/internal/objabi/reloctype_string.go
@@ -4,9 +4,71 @@ package objabi
 
 import "strconv"
 
-const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF"
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[R_ADDR-1]
+	_ = x[R_ADDRPOWER-2]
+	_ = x[R_ADDRARM64-3]
+	_ = x[R_ADDRMIPS-4]
+	_ = x[R_ADDROFF-5]
+	_ = x[R_WEAKADDROFF-6]
+	_ = x[R_SIZE-7]
+	_ = x[R_CALL-8]
+	_ = x[R_CALLARM-9]
+	_ = x[R_CALLARM64-10]
+	_ = x[R_CALLIND-11]
+	_ = x[R_CALLPOWER-12]
+	_ = x[R_CALLMIPS-13]
+	_ = x[R_CALLRISCV-14]
+	_ = x[R_CONST-15]
+	_ = x[R_PCREL-16]
+	_ = x[R_TLS_LE-17]
+	_ = x[R_TLS_IE-18]
+	_ = x[R_GOTOFF-19]
+	_ = x[R_PLT0-20]
+	_ = x[R_PLT1-21]
+	_ = x[R_PLT2-22]
+	_ = x[R_USEFIELD-23]
+	_ = x[R_USETYPE-24]
+	_ = x[R_USEIFACE-25]
+	_ = x[R_METHODOFF-26]
+	_ = x[R_POWER_TOC-27]
+	_ = x[R_GOTPCREL-28]
+	_ = x[R_JMPMIPS-29]
+	_ = x[R_DWARFSECREF-30]
+	_ = x[R_DWARFFILEREF-31]
+	_ = x[R_ARM64_TLS_LE-32]
+	_ = x[R_ARM64_TLS_IE-33]
+	_ = x[R_ARM64_GOTPCREL-34]
+	_ = x[R_ARM64_GOT-35]
+	_ = x[R_ARM64_PCREL-36]
+	_ = x[R_ARM64_LDST8-37]
+	_ = x[R_ARM64_LDST32-38]
+	_ = x[R_ARM64_LDST64-39]
+	_ = x[R_ARM64_LDST128-40]
+	_ = x[R_POWER_TLS_LE-41]
+	_ = x[R_POWER_TLS_IE-42]
+	_ = x[R_POWER_TLS-43]
+	_ = x[R_ADDRPOWER_DS-44]
+	_ = x[R_ADDRPOWER_GOT-45]
+	_ = x[R_ADDRPOWER_PCREL-46]
+	_ = x[R_ADDRPOWER_TOCREL-47]
+	_ = x[R_ADDRPOWER_TOCREL_DS-48]
+	_ = x[R_RISCV_PCREL_ITYPE-49]
+	_ = x[R_RISCV_PCREL_STYPE-50]
+	_ = x[R_PCRELDBL-51]
+	_ = x[R_ADDRMIPSU-52]
+	_ = x[R_ADDRMIPSTLS-53]
+	_ = x[R_ADDRCUOFF-54]
+	_ = x[R_WASMIMPORT-55]
+	_ = x[R_XCOFFREF-56]
+}
+
+const _RelocType_name = "R_ADDRR_ADDRPOWERR_ADDRARM64R_ADDRMIPSR_ADDROFFR_WEAKADDROFFR_SIZER_CALLR_CALLARMR_CALLARM64R_CALLINDR_CALLPOWERR_CALLMIPSR_CALLRISCVR_CONSTR_PCRELR_TLS_LER_TLS_IER_GOTOFFR_PLT0R_PLT1R_PLT2R_USEFIELDR_USETYPER_USEIFACER_METHODOFFR_POWER_TOCR_GOTPCRELR_JMPMIPSR_DWARFSECREFR_DWARFFILEREFR_ARM64_TLS_LER_ARM64_TLS_IER_ARM64_GOTPCRELR_ARM64_GOTR_ARM64_PCRELR_ARM64_LDST8R_ARM64_LDST32R_ARM64_LDST64R_ARM64_LDST128R_POWER_TLS_LER_POWER_TLS_IER_POWER_TLSR_ADDRPOWER_DSR_ADDRPOWER_GOTR_ADDRPOWER_PCRELR_ADDRPOWER_TOCRELR_ADDRPOWER_TOCREL_DSR_RISCV_PCREL_ITYPER_RISCV_PCREL_STYPER_PCRELDBLR_ADDRMIPSUR_ADDRMIPSTLSR_ADDRCUOFFR_WASMIMPORTR_XCOFFREF"
 
-var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 219, 230, 240, 249, 262, 276, 290, 304, 320, 331, 344, 357, 371, 385, 400, 414, 428, 439, 453, 468, 485, 503, 524, 543, 562, 572, 583, 596, 607, 619, 629}
+var _RelocType_index = [...]uint16{0, 6, 17, 28, 38, 47, 60, 66, 72, 81, 92, 101, 112, 122, 133, 140, 147, 155, 163, 171, 177, 183, 189, 199, 208, 218, 229, 240, 250, 259, 272, 286, 300, 314, 330, 341, 354, 367, 381, 395, 410, 424, 438, 449, 463, 478, 495, 513, 534, 553, 572, 582, 593, 606, 617, 629, 639}
 
 func (i RelocType) String() string {
 	i -= 1
diff --git a/src/cmd/link/internal/ld/data.go b/src/cmd/link/internal/ld/data.go
index a730125cf2..0a3418bfc9 100644
--- a/src/cmd/link/internal/ld/data.go
+++ b/src/cmd/link/internal/ld/data.go
@@ -698,6 +698,9 @@ func windynrelocsym(ctxt *Link, rel *loader.SymbolBuilder, s loader.Sym) {
 	relocs := ctxt.loader.Relocs(s)
 	for ri := 0; ri < relocs.Count(); ri++ {
 		r := relocs.At(ri)
+		if r.IsMarker() {
+			continue // skip marker relocations
+		}
 		targ := r.Sym()
 		if targ == 0 {
 			continue
@@ -775,6 +778,9 @@ func dynrelocsym(ctxt *Link, s loader.Sym) {
 	relocs := ldr.Relocs(s)
 	for ri := 0; ri < relocs.Count(); ri++ {
 		r := relocs.At(ri)
+		if r.IsMarker() {
+			continue // skip marker relocations
+		}
 		if ctxt.BuildMode == BuildModePIE && ctxt.LinkMode == LinkInternal {
 			// It's expected that some relocations will be done
 			// later by relocsym (R_TLS_LE, R_ADDROFF), so
diff --git a/src/cmd/link/internal/ld/deadcode.go b/src/cmd/link/internal/ld/deadcode.go
index 7f14aa3d27..816a23b9a7 100644
--- a/src/cmd/link/internal/ld/deadcode.go
+++ b/src/cmd/link/internal/ld/deadcode.go
@@ -153,6 +153,20 @@ func (d *deadcodePass) flood() {
 				// do nothing for now as we still load all type symbols.
 				continue
 			}
+			if t == objabi.R_USEIFACE {
+				// R_USEIFACE is a marker relocation that tells the linker the type is
+				// converted to an interface, i.e. should have UsedInIface set. See the
+				// comment below for why we need to unset the Reachable bit and re-mark it.
+				rs := r.Sym()
+				if !d.ldr.AttrUsedInIface(rs) {
+					d.ldr.SetAttrUsedInIface(rs, true)
+					if d.ldr.AttrReachable(rs) {
+						d.ldr.SetAttrReachable(rs, false)
+						d.mark(rs, symIdx)
+					}
+				}
+				continue
+			}
 			rs := r.Sym()
 			if isgotype && usedInIface && d.ldr.IsGoType(rs) && !d.ldr.AttrUsedInIface(rs) {
 				// If a type is converted to an interface, it is possible to obtain an
diff --git a/src/cmd/link/internal/ld/testdata/deadcode/ifacemethod.go b/src/cmd/link/internal/ld/testdata/deadcode/ifacemethod.go
index b62f18c342..32a24cf6f0 100644
--- a/src/cmd/link/internal/ld/testdata/deadcode/ifacemethod.go
+++ b/src/cmd/link/internal/ld/testdata/deadcode/ifacemethod.go
@@ -18,6 +18,13 @@ var p *T
 var e interface{}
 
 func main() {
-	p = new(T) // used T, but never converted to interface
+	p = new(T) // used T, but never converted to interface in any reachable code
 	e.(I).M()  // used I and I.M
 }
+
+func Unused() { // convert T to interface, but this function is not reachable
+	var i I = T(0)
+	i.M()
+}
+
+var Unused2 interface{} = T(1) // convert T to interface, in an unreachable global initializer
diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index 43a0352e0b..ea99233f67 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -63,6 +63,7 @@ type Reloc struct {
 func (rel Reloc) Type() objabi.RelocType { return objabi.RelocType(rel.Reloc.Type()) + rel.typ }
 func (rel Reloc) Sym() Sym               { return rel.l.resolve(rel.r, rel.Reloc.Sym()) }
 func (rel Reloc) SetSym(s Sym)           { rel.Reloc.SetSym(goobj.SymRef{PkgIdx: 0, SymIdx: uint32(s)}) }
+func (rel Reloc) IsMarker() bool         { return rel.Siz() == 0 }
 
 func (rel Reloc) SetType(t objabi.RelocType) {
 	if t != objabi.RelocType(uint8(t)) {
diff --git a/src/cmd/link/internal/wasm/asm.go b/src/cmd/link/internal/wasm/asm.go
index 3bd56a6e3a..31851fbb56 100644
--- a/src/cmd/link/internal/wasm/asm.go
+++ b/src/cmd/link/internal/wasm/asm.go
@@ -167,6 +167,9 @@ func asmb2(ctxt *ld.Link, ldr *loader.Loader) {
 			off := int32(0)
 			for ri := 0; ri < relocs.Count(); ri++ {
 				r := relocs.At(ri)
+				if r.Siz() == 0 {
+					continue // skip marker relocations
+				}
 				wfn.Write(P[off:r.Off()])
 				off = r.Off()
 				rs := ldr.ResolveABIAlias(r.Sym())
-- 
cgit v1.3


From 66770f4b1de37d9c5c962edb2980a70102e09ec3 Mon Sep 17 00:00:00 2001
From: Cherry Zhang <cherryyz@google.com>
Date: Mon, 28 Sep 2020 13:10:30 -0400
Subject: cmd/compile: mark type namedata symbols content-addressable

Type namedata symbols are for type/field/method names and package
paths. We can use content-addressable symbol mechanism for them.

Change-Id: I923fda17b7094c7a0e46aad7c450622eb3826294
Reviewed-on: https://go-review.googlesource.com/c/go/+/257960
Trust: Cherry Zhang <cherryyz@google.com>
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>
Reviewed-by: Jeremy Faller <jeremy@golang.org>
---
 src/cmd/compile/internal/gc/reflect.go | 2 ++
 src/cmd/internal/obj/objfile.go        | 7 +++++++
 src/cmd/internal/obj/sym.go            | 5 ++++-
 src/cmd/link/internal/loader/loader.go | 4 ++--
 4 files changed, 15 insertions(+), 3 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/compile/internal/gc/reflect.go b/src/cmd/compile/internal/gc/reflect.go
index ae3e2f8e65..21429af782 100644
--- a/src/cmd/compile/internal/gc/reflect.go
+++ b/src/cmd/compile/internal/gc/reflect.go
@@ -511,6 +511,7 @@ func dimportpath(p *types.Pkg) {
 	s := Ctxt.Lookup("type..importpath." + p.Prefix + ".")
 	ot := dnameData(s, 0, str, "", nil, false)
 	ggloblsym(s, int32(ot), obj.DUPOK|obj.RODATA)
+	s.Set(obj.AttrContentAddressable, true)
 	p.Pathsym = s
 }
 
@@ -638,6 +639,7 @@ func dname(name, tag string, pkg *types.Pkg, exported bool) *obj.LSym {
 	}
 	ot := dnameData(s, 0, name, tag, pkg, exported)
 	ggloblsym(s, int32(ot), obj.DUPOK|obj.RODATA)
+	s.Set(obj.AttrContentAddressable, true)
 	return s
 }
 
diff --git a/src/cmd/internal/obj/objfile.go b/src/cmd/internal/obj/objfile.go
index e4b9620568..186016c970 100644
--- a/src/cmd/internal/obj/objfile.go
+++ b/src/cmd/internal/obj/objfile.go
@@ -372,6 +372,13 @@ func contentHash64(s *LSym) goobj.Hash64Type {
 // hashed symbols.
 func (w *writer) contentHash(s *LSym) goobj.HashType {
 	h := sha1.New()
+	// Don't dedup type symbols with others, as they are in a different
+	// section.
+	if strings.HasPrefix(s.Name, "type.") {
+		h.Write([]byte{'T'})
+	} else {
+		h.Write([]byte{0})
+	}
 	// The compiler trims trailing zeros _sometimes_. We just do
 	// it always.
 	h.Write(bytes.TrimRight(s.P, "\x00"))
diff --git a/src/cmd/internal/obj/sym.go b/src/cmd/internal/obj/sym.go
index d58877ee15..e5d7b2cbfd 100644
--- a/src/cmd/internal/obj/sym.go
+++ b/src/cmd/internal/obj/sym.go
@@ -38,6 +38,7 @@ import (
 	"log"
 	"math"
 	"sort"
+	"strings"
 )
 
 func Linknew(arch *LinkArch) *Link {
@@ -204,7 +205,9 @@ func (ctxt *Link) NumberSyms() {
 		// if Pkgpath is unknown, cannot hash symbols with relocations, as it
 		// may reference named symbols whose names are not fully expanded.
 		if s.ContentAddressable() && (ctxt.Pkgpath != "" || len(s.R) == 0) {
-			if len(s.P) <= 8 && len(s.R) == 0 { // we can use short hash only for symbols without relocations
+			if len(s.P) <= 8 && len(s.R) == 0 && !strings.HasPrefix(s.Name, "type.") {
+				// We can use short hash only for symbols without relocations.
+				// Don't use short hash for type symbols, as they need special handling.
 				s.PkgIdx = goobj.PkgIdxHashed64
 				s.SymIdx = hashed64idx
 				if hashed64idx != int32(len(ctxt.hashed64defs)) {
diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index ea99233f67..4025edc23f 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -2153,11 +2153,11 @@ func (l *Loader) LoadNonpkgSyms(arch *sys.Arch) {
 	l.npkgsyms = l.NSym()
 	// Preallocate some space (a few hundreds KB) for some symbols.
 	// As of Go 1.15, linking cmd/compile has ~8000 hashed64 symbols and
-	// ~13000 hashed symbols.
+	// ~27000 hashed symbols.
 	st := loadState{
 		l:            l,
 		hashed64Syms: make(map[uint64]symAndSize, 10000),
-		hashedSyms:   make(map[goobj.HashType]symAndSize, 15000),
+		hashedSyms:   make(map[goobj.HashType]symAndSize, 30000),
 	}
 	for _, o := range l.objs[goObjStart:] {
 		st.preloadSyms(o.r, hashed64Def)
-- 
cgit v1.3


From 15eaa870e14195c17dbb4be0d11bc40dba94ff22 Mon Sep 17 00:00:00 2001
From: Joel Sing <joel@sing.id.au>
Date: Fri, 14 Aug 2020 00:13:28 +1000
Subject: cmd/link: add support for external linking on linux/riscv64

Fixes #36739

Change-Id: Id7573b343786360c72524f9f27d2a8f08d379cf3
Reviewed-on: https://go-review.googlesource.com/c/go/+/243517
Trust: Joel Sing <joel@sing.id.au>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Than McIntosh <thanm@google.com>
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
---
 src/cmd/dist/test.go                   |   2 +-
 src/cmd/link/internal/ld/config.go     |   4 +-
 src/cmd/link/internal/ld/elf.go        |   3 +
 src/cmd/link/internal/ld/lib.go        |   6 +-
 src/cmd/link/internal/ld/main.go       |   8 ++
 src/cmd/link/internal/loader/loader.go |   5 ++
 src/cmd/link/internal/riscv64/asm.go   | 148 +++++++++++++++++++++++++++++++--
 src/cmd/link/internal/riscv64/obj.go   |   3 +
 8 files changed, 169 insertions(+), 10 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go
index 94b81613a9..bcb12f29fb 100644
--- a/src/cmd/dist/test.go
+++ b/src/cmd/dist/test.go
@@ -921,7 +921,7 @@ func (t *tester) extLink() bool {
 		"darwin-amd64", "darwin-arm64",
 		"dragonfly-amd64",
 		"freebsd-386", "freebsd-amd64", "freebsd-arm",
-		"linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-s390x",
+		"linux-386", "linux-amd64", "linux-arm", "linux-arm64", "linux-ppc64le", "linux-mips64", "linux-mips64le", "linux-mips", "linux-mipsle", "linux-riscv64", "linux-s390x",
 		"netbsd-386", "netbsd-amd64",
 		"openbsd-386", "openbsd-amd64",
 		"windows-386", "windows-amd64":
diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go
index f55e4fc027..834c87d06b 100644
--- a/src/cmd/link/internal/ld/config.go
+++ b/src/cmd/link/internal/ld/config.go
@@ -202,7 +202,7 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) {
 	// Internally linking cgo is incomplete on some architectures.
 	// https://golang.org/issue/14449
 	// https://golang.org/issue/21961
-	if iscgo && ctxt.Arch.InFamily(sys.MIPS64, sys.MIPS, sys.PPC64) {
+	if iscgo && ctxt.Arch.InFamily(sys.MIPS64, sys.MIPS, sys.PPC64, sys.RISCV64) {
 		return true, objabi.GOARCH + " does not support internal cgo"
 	}
 	if iscgo && objabi.GOOS == "android" {
@@ -285,8 +285,6 @@ func determineLinkMode(ctxt *Link) {
 		}
 	case LinkExternal:
 		switch {
-		case objabi.GOARCH == "riscv64":
-			Exitf("external linking not supported for %s/riscv64", objabi.GOOS)
 		case objabi.GOARCH == "ppc64" && objabi.GOOS != "aix":
 			Exitf("external linking not supported for %s/ppc64", objabi.GOOS)
 		}
diff --git a/src/cmd/link/internal/ld/elf.go b/src/cmd/link/internal/ld/elf.go
index f5a2f899fc..f44e16583d 100644
--- a/src/cmd/link/internal/ld/elf.go
+++ b/src/cmd/link/internal/ld/elf.go
@@ -506,6 +506,9 @@ func Elfinit(ctxt *Link) {
 		if ctxt.Arch.Family == sys.MIPS64 {
 			ehdr.flags = 0x20000004 /* MIPS 3 CPIC */
 		}
+		if ctxt.Arch.Family == sys.RISCV64 {
+			ehdr.flags = 0x4 /* RISCV Float ABI Double */
+		}
 		elf64 = true
 
 		ehdr.phoff = ELF64HDRSIZE      /* Must be ELF64HDRSIZE: first PHdr must follow ELF header */
diff --git a/src/cmd/link/internal/ld/lib.go b/src/cmd/link/internal/ld/lib.go
index 8d04973fcf..a68725bef9 100644
--- a/src/cmd/link/internal/ld/lib.go
+++ b/src/cmd/link/internal/ld/lib.go
@@ -247,12 +247,16 @@ type Arch struct {
 	Elfreloc1      func(*Link, *OutBuf, *loader.Loader, loader.Sym, loader.ExtReloc, int, int64) bool
 	ElfrelocSize   uint32 // size of an ELF relocation record, must match Elfreloc1.
 	Elfsetupplt    func(ctxt *Link, plt, gotplt *loader.SymbolBuilder, dynamic loader.Sym)
-	Gentext        func(*Link, *loader.Loader)
+	Gentext        func(*Link, *loader.Loader) // Generate text before addressing has been performed.
 	Machoreloc1    func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtReloc, int64) bool
 	MachorelocSize uint32 // size of an Mach-O relocation record, must match Machoreloc1.
 	PEreloc1       func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtReloc, int64) bool
 	Xcoffreloc1    func(*sys.Arch, *OutBuf, *loader.Loader, loader.Sym, loader.ExtReloc, int64) bool
 
+	// Generate additional symbols for the native symbol table just prior to
+	// code generation.
+	GenSymsLate func(*Link, *loader.Loader)
+
 	// TLSIEtoLE converts a TLS Initial Executable relocation to
 	// a TLS Local Executable relocation.
 	//
diff --git a/src/cmd/link/internal/ld/main.go b/src/cmd/link/internal/ld/main.go
index 3f7370b636..5ae57d1992 100644
--- a/src/cmd/link/internal/ld/main.go
+++ b/src/cmd/link/internal/ld/main.go
@@ -341,6 +341,14 @@ func Main(arch *sys.Arch, theArch Arch) {
 		}(f, s)
 	}
 	wg.Wait()
+
+	// Generate additional symbols for the native symbol table just prior
+	// to code generation.
+	bench.Start("GenSymsLate")
+	if thearch.GenSymsLate != nil {
+		thearch.GenSymsLate(ctxt, ctxt.loader)
+	}
+
 	bench.Start("Asmb2")
 	asmb2(ctxt)
 
diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index 19d8d98b1e..47cac0441b 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -1791,6 +1791,11 @@ func (l *Loader) SortSub(s Sym) Sym {
 	return sl[0].s
 }
 
+// SortSyms sorts a list of symbols by their value.
+func (l *Loader) SortSyms(ss []Sym) {
+	sort.SliceStable(ss, func(i, j int) bool { return l.SymValue(ss[i]) < l.SymValue(ss[j]) })
+}
+
 // Insure that reachable bitmap and its siblings have enough size.
 func (l *Loader) growAttrBitmaps(reqLen int) {
 	if reqLen > l.attrReachable.Len() {
diff --git a/src/cmd/link/internal/riscv64/asm.go b/src/cmd/link/internal/riscv64/asm.go
index 1236145fb1..66c47c69f8 100644
--- a/src/cmd/link/internal/riscv64/asm.go
+++ b/src/cmd/link/internal/riscv64/asm.go
@@ -11,20 +11,138 @@ import (
 	"cmd/link/internal/ld"
 	"cmd/link/internal/loader"
 	"cmd/link/internal/sym"
+	"debug/elf"
 	"fmt"
 	"log"
+	"sort"
 )
 
+// fakeLabelName matches the RISCV_FAKE_LABEL_NAME from binutils.
+const fakeLabelName = ".L0 "
+
 func gentext(ctxt *ld.Link, ldr *loader.Loader) {
 }
 
+func genSymsLate(ctxt *ld.Link, ldr *loader.Loader) {
+	if ctxt.LinkMode != ld.LinkExternal {
+		return
+	}
+
+	// Generate a local text symbol for each relocation target, as the
+	// R_RISCV_PCREL_LO12_* relocations generated by elfreloc1 need it.
+	if ctxt.Textp == nil {
+		log.Fatal("genSymsLate called before Textp has been assigned")
+	}
+	var hi20Syms []loader.Sym
+	for _, s := range ctxt.Textp {
+		relocs := ldr.Relocs(s)
+		for ri := 0; ri < relocs.Count(); ri++ {
+			r := relocs.At(ri)
+			if r.Type() != objabi.R_RISCV_PCREL_ITYPE && r.Type() != objabi.R_RISCV_PCREL_STYPE {
+				continue
+			}
+			if r.Off() == 0 && ldr.SymType(s) == sym.STEXT {
+				// Use the symbol for the function instead of creating
+				// an overlapping symbol.
+				continue
+			}
+
+			// TODO(jsing): Consider generating ELF symbols without needing
+			// loader symbols, in order to reduce memory consumption. This
+			// would require changes to genelfsym so that it called
+			// putelfsym and putelfsyment as appropriate.
+			sb := ldr.MakeSymbolBuilder(fakeLabelName)
+			sb.SetType(sym.STEXT)
+			sb.SetValue(ldr.SymValue(s) + int64(r.Off()))
+			sb.SetLocal(true)
+			sb.SetReachable(true)
+			sb.SetVisibilityHidden(true)
+			sb.SetSect(ldr.SymSect(s))
+			if outer := ldr.OuterSym(s); outer != 0 {
+				ldr.AddInteriorSym(outer, sb.Sym())
+			}
+			hi20Syms = append(hi20Syms, sb.Sym())
+		}
+	}
+	ctxt.Textp = append(ctxt.Textp, hi20Syms...)
+	ldr.SortSyms(ctxt.Textp)
+}
+
+func findHI20Symbol(ctxt *ld.Link, ldr *loader.Loader, val int64) loader.Sym {
+	idx := sort.Search(len(ctxt.Textp), func(i int) bool { return ldr.SymValue(ctxt.Textp[i]) >= val })
+	if idx >= len(ctxt.Textp) {
+		return 0
+	}
+	if s := ctxt.Textp[idx]; ldr.SymValue(s) == val && ldr.SymType(s) == sym.STEXT {
+		return s
+	}
+	return 0
+}
+
 func elfreloc1(ctxt *ld.Link, out *ld.OutBuf, ldr *loader.Loader, s loader.Sym, r loader.ExtReloc, ri int, sectoff int64) bool {
-	log.Fatalf("elfreloc1")
-	return false
+	elfsym := ld.ElfSymForReloc(ctxt, r.Xsym)
+	switch r.Type {
+	case objabi.R_ADDR, objabi.R_DWARFSECREF:
+		out.Write64(uint64(sectoff))
+		switch r.Size {
+		case 4:
+			out.Write64(uint64(elf.R_RISCV_32) | uint64(elfsym)<<32)
+		case 8:
+			out.Write64(uint64(elf.R_RISCV_64) | uint64(elfsym)<<32)
+		default:
+			ld.Errorf(nil, "unknown size %d for %v relocation", r.Size, r.Type)
+			return false
+		}
+		out.Write64(uint64(r.Xadd))
+
+	case objabi.R_CALLRISCV:
+		// Call relocations are currently handled via R_RISCV_PCREL_ITYPE.
+		// TODO(jsing): Consider generating elf.R_RISCV_CALL instead of a
+		// HI20/LO12_I pair.
+
+	case objabi.R_RISCV_PCREL_ITYPE, objabi.R_RISCV_PCREL_STYPE:
+		// Find the text symbol for the AUIPC instruction targeted
+		// by this relocation.
+		relocs := ldr.Relocs(s)
+		offset := int64(relocs.At(ri).Off())
+		hi20Sym := findHI20Symbol(ctxt, ldr, ldr.SymValue(s)+offset)
+		if hi20Sym == 0 {
+			ld.Errorf(nil, "failed to find text symbol for HI20 relocation at %d (%x)", sectoff, ldr.SymValue(s)+offset)
+			return false
+		}
+		hi20ElfSym := ld.ElfSymForReloc(ctxt, hi20Sym)
+
+		// Emit two relocations - a R_RISCV_PCREL_HI20 relocation and a
+		// corresponding R_RISCV_PCREL_LO12_I or R_RISCV_PCREL_LO12_S relocation.
+		// Note that the LO12 relocation must point to a target that has a valid
+		// HI20 PC-relative relocation text symbol, which in turn points to the
+		// given symbol. For further details see the ELF specification for RISC-V:
+		//
+		//   https://github.com/riscv/riscv-elf-psabi-doc/blob/master/riscv-elf.md#pc-relative-symbol-addresses
+		//
+		var hiRel, loRel elf.R_RISCV
+		switch r.Type {
+		case objabi.R_RISCV_PCREL_ITYPE:
+			hiRel, loRel = elf.R_RISCV_PCREL_HI20, elf.R_RISCV_PCREL_LO12_I
+		case objabi.R_RISCV_PCREL_STYPE:
+			hiRel, loRel = elf.R_RISCV_PCREL_HI20, elf.R_RISCV_PCREL_LO12_S
+		}
+		out.Write64(uint64(sectoff))
+		out.Write64(uint64(hiRel) | uint64(elfsym)<<32)
+		out.Write64(uint64(r.Xadd))
+		out.Write64(uint64(sectoff + 4))
+		out.Write64(uint64(loRel) | uint64(hi20ElfSym)<<32)
+		out.Write64(uint64(0))
+
+	default:
+		return false
+	}
+
+	return true
 }
 
 func elfsetupplt(ctxt *ld.Link, plt, gotplt *loader.SymbolBuilder, dynamic loader.Sym) {
-	log.Fatalf("elfsetuplt")
+	log.Fatalf("elfsetupplt")
 }
 
 func machoreloc1(*sys.Arch, *ld.OutBuf, *loader.Loader, loader.Sym, loader.ExtReloc, int64) bool {
@@ -33,8 +151,20 @@ func machoreloc1(*sys.Arch, *ld.OutBuf, *loader.Loader, loader.Sym, loader.ExtRe
 }
 
 func archreloc(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, r loader.Reloc, s loader.Sym, val int64) (o int64, nExtReloc int, ok bool) {
-	rs := r.Sym()
-	rs = ldr.ResolveABIAlias(rs)
+	if target.IsExternal() {
+		switch r.Type() {
+		case objabi.R_CALLRISCV:
+			return val, 0, true
+
+		case objabi.R_RISCV_PCREL_ITYPE, objabi.R_RISCV_PCREL_STYPE:
+			return val, 2, true
+		}
+
+		return val, 0, false
+	}
+
+	rs := ldr.ResolveABIAlias(r.Sym())
+
 	switch r.Type() {
 	case objabi.R_CALLRISCV:
 		// Nothing to do.
@@ -89,3 +219,11 @@ func archrelocvariant(*ld.Target, *loader.Loader, loader.Reloc, sym.RelocVariant
 	log.Fatalf("archrelocvariant")
 	return -1
 }
+
+func extreloc(target *ld.Target, ldr *loader.Loader, r loader.Reloc, s loader.Sym) (loader.ExtReloc, bool) {
+	switch r.Type() {
+	case objabi.R_RISCV_PCREL_ITYPE, objabi.R_RISCV_PCREL_STYPE:
+		return ld.ExtrelocViaOuterSym(ldr, r, s), true
+	}
+	return loader.ExtReloc{}, false
+}
diff --git a/src/cmd/link/internal/riscv64/obj.go b/src/cmd/link/internal/riscv64/obj.go
index e66d3cd856..917324d922 100644
--- a/src/cmd/link/internal/riscv64/obj.go
+++ b/src/cmd/link/internal/riscv64/obj.go
@@ -23,9 +23,12 @@ func Init() (*sys.Arch, ld.Arch) {
 		Archinit:         archinit,
 		Archreloc:        archreloc,
 		Archrelocvariant: archrelocvariant,
+		Extreloc:         extreloc,
 		Elfreloc1:        elfreloc1,
+		ElfrelocSize:     24,
 		Elfsetupplt:      elfsetupplt,
 		Gentext:          gentext,
+		GenSymsLate:      genSymsLate,
 		Machoreloc1:      machoreloc1,
 
 		Linuxdynld: "/lib/ld.so.1",
-- 
cgit v1.3


From 627959eb04ee0edc4a985a7526ed7fe838ad2573 Mon Sep 17 00:00:00 2001
From: Cherry Zhang <cherryyz@google.com>
Date: Wed, 14 Oct 2020 21:15:37 -0400
Subject: cmd/link: support cgo internal/linking on darwin/arm64

Cgo programs work as well. Still not enabled by default for now.

Enable internal linking tests.

Updates #38485.

Change-Id: I8324a5c263fba221eb4e67d71207ca84fa241e6c
Reviewed-on: https://go-review.googlesource.com/c/go/+/263637
Trust: Cherry Zhang <cherryyz@google.com>
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Than McIntosh <thanm@google.com>
---
 misc/cgo/test/issue4029.c                  |  2 +-
 misc/cgo/test/issue4029.go                 |  3 +-
 misc/cgo/test/issue4029w.go                |  2 +-
 src/cmd/dist/test.go                       |  7 +--
 src/cmd/link/internal/arm64/asm.go         | 91 ++++++++++++++++++++++++++++--
 src/cmd/link/internal/ld/config.go         |  3 -
 src/cmd/link/internal/loader/loader.go     |  8 ++-
 src/cmd/link/internal/loadmacho/ldmacho.go | 57 +++++++++++--------
 8 files changed, 131 insertions(+), 42 deletions(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/misc/cgo/test/issue4029.c b/misc/cgo/test/issue4029.c
index e6a777fe64..e79c5a709c 100644
--- a/misc/cgo/test/issue4029.c
+++ b/misc/cgo/test/issue4029.c
@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // +build !windows,!static
-// +build !darwin !internal_pie
+// +build !darwin !internal_pie,!arm64
 
 #include <stdint.h>
 #include <dlfcn.h>
diff --git a/misc/cgo/test/issue4029.go b/misc/cgo/test/issue4029.go
index 8602ce19e2..b2d131833a 100644
--- a/misc/cgo/test/issue4029.go
+++ b/misc/cgo/test/issue4029.go
@@ -3,10 +3,11 @@
 // license that can be found in the LICENSE file.
 
 // +build !windows,!static
-// +build !darwin !internal_pie
+// +build !darwin !internal_pie,!arm64
 
 // Excluded in darwin internal linking PIE mode, as dynamic export is not
 // supported.
+// Excluded in internal linking mode on darwin/arm64, as it is always PIE.
 
 package cgotest
 
diff --git a/misc/cgo/test/issue4029w.go b/misc/cgo/test/issue4029w.go
index de0cf2138a..b969bdd0fe 100644
--- a/misc/cgo/test/issue4029w.go
+++ b/misc/cgo/test/issue4029w.go
@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build windows static darwin,internal_pie
+// +build windows static darwin,internal_pie darwin,arm64
 
 package cgotest
 
diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go
index bcb12f29fb..09d69f72ed 100644
--- a/src/cmd/dist/test.go
+++ b/src/cmd/dist/test.go
@@ -946,9 +946,6 @@ func (t *tester) internalLink() bool {
 	if goos == "ios" {
 		return false
 	}
-	if goos == "darwin" && goarch == "arm64" {
-		return false
-	}
 	// Internally linking cgo is incomplete on some architectures.
 	// https://golang.org/issue/10373
 	// https://golang.org/issue/14449
@@ -964,7 +961,7 @@ func (t *tester) internalLink() bool {
 
 func (t *tester) internalLinkPIE() bool {
 	switch goos + "-" + goarch {
-	case "darwin-amd64",
+	case "darwin-amd64", "darwin-arm64",
 		"linux-amd64", "linux-arm64",
 		"android-arm64",
 		"windows-amd64", "windows-386", "windows-arm":
@@ -1088,7 +1085,7 @@ func (t *tester) cgoTest(dt *distTest) error {
 
 	pair := gohostos + "-" + goarch
 	switch pair {
-	case "darwin-amd64",
+	case "darwin-amd64", "darwin-arm64",
 		"openbsd-386", "openbsd-amd64",
 		"windows-386", "windows-amd64":
 		// test linkmode=external, but __thread not supported, so skip testtls.
diff --git a/src/cmd/link/internal/arm64/asm.go b/src/cmd/link/internal/arm64/asm.go
index 585c96852f..e456411155 100644
--- a/src/cmd/link/internal/arm64/asm.go
+++ b/src/cmd/link/internal/arm64/asm.go
@@ -71,13 +71,13 @@ func gentext(ctxt *ld.Link, ldr *loader.Loader) {
 }
 
 func adddynrel(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, s loader.Sym, r loader.Reloc, rIdx int) bool {
-
 	targ := r.Sym()
 	var targType sym.SymKind
 	if targ != 0 {
 		targType = ldr.SymType(targ)
 	}
 
+	const pcrel = 1
 	switch r.Type() {
 	default:
 		if r.Type() >= objabi.ElfRelocOffset {
@@ -201,6 +201,75 @@ func adddynrel(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, s loade
 		su := ldr.MakeSymbolUpdater(s)
 		su.SetRelocType(rIdx, objabi.R_ARM64_LDST128)
 		return true
+
+	// Handle relocations found in Mach-O object files.
+	case objabi.MachoRelocOffset + ld.MACHO_ARM64_RELOC_UNSIGNED*2:
+		if targType == sym.SDYNIMPORT {
+			ldr.Errorf(s, "unexpected reloc for dynamic symbol %s", ldr.SymName(targ))
+		}
+		su := ldr.MakeSymbolUpdater(s)
+		su.SetRelocType(rIdx, objabi.R_ADDR)
+		if target.IsPIE() && target.IsInternal() {
+			// For internal linking PIE, this R_ADDR relocation cannot
+			// be resolved statically. We need to generate a dynamic
+			// relocation. Let the code below handle it.
+			break
+		}
+		return true
+
+	case objabi.MachoRelocOffset + ld.MACHO_ARM64_RELOC_BRANCH26*2 + pcrel:
+		su := ldr.MakeSymbolUpdater(s)
+		su.SetRelocType(rIdx, objabi.R_CALLARM64)
+		if targType == sym.SDYNIMPORT {
+			addpltsym(target, ldr, syms, targ)
+			su.SetRelocSym(rIdx, syms.PLT)
+			su.SetRelocAdd(rIdx, int64(ldr.SymPlt(targ)))
+		}
+		return true
+
+	case objabi.MachoRelocOffset + ld.MACHO_ARM64_RELOC_PAGE21*2 + pcrel,
+		objabi.MachoRelocOffset + ld.MACHO_ARM64_RELOC_PAGEOFF12*2:
+		if targType == sym.SDYNIMPORT {
+			ldr.Errorf(s, "unexpected relocation for dynamic symbol %s", ldr.SymName(targ))
+		}
+		su := ldr.MakeSymbolUpdater(s)
+		su.SetRelocType(rIdx, objabi.R_ARM64_PCREL)
+		return true
+
+	case objabi.MachoRelocOffset + ld.MACHO_ARM64_RELOC_GOT_LOAD_PAGE21*2 + pcrel,
+		objabi.MachoRelocOffset + ld.MACHO_ARM64_RELOC_GOT_LOAD_PAGEOFF12*2:
+		if targType != sym.SDYNIMPORT {
+			// have symbol
+			// turn MOVD sym@GOT (adrp+ldr) into MOVD $sym (adrp+add)
+			data := ldr.Data(s)
+			off := r.Off()
+			if int(off+3) >= len(data) {
+				ldr.Errorf(s, "unexpected GOT_LOAD reloc for non-dynamic symbol %s", ldr.SymName(targ))
+				return false
+			}
+			o := target.Arch.ByteOrder.Uint32(data[off:])
+			su := ldr.MakeSymbolUpdater(s)
+			switch {
+			case (o>>24)&0x9f == 0x90: // adrp
+				// keep instruction unchanged, change relocation type below
+			case o>>24 == 0xf9: // ldr
+				// rewrite to add
+				o = (0x91 << 24) | (o & (1<<22 - 1))
+				su.MakeWritable()
+				su.SetUint32(target.Arch, int64(off), o)
+			default:
+				ldr.Errorf(s, "unexpected GOT_LOAD reloc for non-dynamic symbol %s", ldr.SymName(targ))
+				return false
+			}
+			su.SetRelocType(rIdx, objabi.R_ARM64_PCREL)
+			return true
+		}
+		ld.AddGotSym(target, ldr, syms, targ, 0)
+		su := ldr.MakeSymbolUpdater(s)
+		su.SetRelocType(rIdx, objabi.R_ARM64_GOT)
+		su.SetRelocSym(rIdx, syms.GOT)
+		su.SetRelocAdd(rIdx, int64(ldr.SymGot(targ)))
+		return true
 	}
 
 	// Reread the reloc to incorporate any changes in type above.
@@ -671,14 +740,28 @@ func archreloc(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, r loade
 			}
 			o0 := (uint32((t>>12)&3) << 29) | (uint32((t>>12>>2)&0x7ffff) << 5)
 			return val | int64(o0), noExtReloc, isOk
-		} else if (val>>24)&0x91 == 0x91 {
-			// R_AARCH64_ADD_ABS_LO12_NC
+		} else if (val>>24)&0x9f == 0x91 {
+			// ELF R_AARCH64_ADD_ABS_LO12_NC or Mach-O ARM64_RELOC_PAGEOFF12
 			// patch instruction: add
 			t := ldr.SymAddr(rs) + r.Add() - ((ldr.SymValue(s) + int64(r.Off())) &^ 0xfff)
 			o1 := uint32(t&0xfff) << 10
 			return val | int64(o1), noExtReloc, isOk
+		} else if (val>>24)&0x3b == 0x39 {
+			// Mach-O ARM64_RELOC_PAGEOFF12
+			// patch ldr/str(b/h/w/d/q) (integer or vector) instructions, which have different scaling factors.
+			// Mach-O uses same relocation type for them.
+			shift := uint32(val) >> 30
+			if shift == 0 && (val>>20)&0x048 == 0x048 { // 128-bit vector load
+				shift = 4
+			}
+			t := ldr.SymAddr(rs) + r.Add() - ((ldr.SymValue(s) + int64(r.Off())) &^ 0xfff)
+			if t&(1<<shift-1) != 0 {
+				ldr.Errorf(s, "invalid address: %x for relocation type: ARM64_RELOC_PAGEOFF12", t)
+			}
+			o1 := (uint32(t&0xfff) >> shift) << 10
+			return val | int64(o1), noExtReloc, isOk
 		} else {
-			ldr.Errorf(s, "unsupported instruction for %x R_PCRELARM64", val)
+			ldr.Errorf(s, "unsupported instruction for %x R_ARM64_PCREL", val)
 		}
 
 	case objabi.R_ARM64_LDST8:
diff --git a/src/cmd/link/internal/ld/config.go b/src/cmd/link/internal/ld/config.go
index 834c87d06b..a54b96dd5d 100644
--- a/src/cmd/link/internal/ld/config.go
+++ b/src/cmd/link/internal/ld/config.go
@@ -208,9 +208,6 @@ func mustLinkExternal(ctxt *Link) (res bool, reason string) {
 	if iscgo && objabi.GOOS == "android" {
 		return true, objabi.GOOS + " does not support internal cgo"
 	}
-	if iscgo && objabi.GOOS == "darwin" && objabi.GOARCH == "arm64" {
-		return true, objabi.GOOS + "/" + objabi.GOARCH + " does not support internal cgo"
-	}
 
 	// When the race flag is set, the LLVM tsan relocatable file is linked
 	// into the final binary, which means external linking is required because
diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index 47cac0441b..d861efcb13 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -2622,11 +2622,15 @@ func (l *Loader) Dump() {
 	fmt.Println("Nsyms:", len(l.objSyms))
 	fmt.Println("syms")
 	for i := Sym(1); i < Sym(len(l.objSyms)); i++ {
-		pi := interface{}("")
+		pi := ""
 		if l.IsExternal(i) {
 			pi = fmt.Sprintf("<ext %d>", l.extIndex(i))
 		}
-		fmt.Println(i, l.SymName(i), l.SymType(i), pi)
+		sect := ""
+		if l.SymSect(i) != nil {
+			sect = l.SymSect(i).Name
+		}
+		fmt.Printf("%v %v %v %v %x %v\n", i, l.SymName(i), l.SymType(i), pi, l.SymValue(i), sect)
 	}
 	fmt.Println("symsByName")
 	for name, i := range l.symsByName[0] {
diff --git a/src/cmd/link/internal/loadmacho/ldmacho.go b/src/cmd/link/internal/loadmacho/ldmacho.go
index 864d80835b..d12f2bc2ac 100644
--- a/src/cmd/link/internal/loadmacho/ldmacho.go
+++ b/src/cmd/link/internal/loadmacho/ldmacho.go
@@ -43,7 +43,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
 
-// TODO(crawshaw): de-duplicate these symbols with cmd/internal/ld
+// TODO(crawshaw): de-duplicate these symbols with cmd/link/internal/ld
 const (
 	MACHO_X86_64_RELOC_UNSIGNED = 0
 	MACHO_X86_64_RELOC_SIGNED   = 1
@@ -172,11 +172,12 @@ const (
 	LdMachoCpuVax         = 1
 	LdMachoCpu68000       = 6
 	LdMachoCpu386         = 7
-	LdMachoCpuAmd64       = 0x1000007
+	LdMachoCpuAmd64       = 1<<24 | 7
 	LdMachoCpuMips        = 8
 	LdMachoCpu98000       = 10
 	LdMachoCpuHppa        = 11
 	LdMachoCpuArm         = 12
+	LdMachoCpuArm64       = 1<<24 | 12
 	LdMachoCpu88000       = 13
 	LdMachoCpuSparc       = 14
 	LdMachoCpu860         = 15
@@ -471,11 +472,14 @@ func Load(l *loader.Loader, arch *sys.Arch, localSymVersion int, f *bio.Reader,
 	switch arch.Family {
 	default:
 		return errorf("mach-o %s unimplemented", arch.Name)
-
 	case sys.AMD64:
 		if e != binary.LittleEndian || m.cputype != LdMachoCpuAmd64 {
 			return errorf("mach-o object but not amd64")
 		}
+	case sys.ARM64:
+		if e != binary.LittleEndian || m.cputype != LdMachoCpuArm64 {
+			return errorf("mach-o object but not arm64")
+		}
 	}
 
 	m.cmd = make([]ldMachoCmd, ncmd)
@@ -633,7 +637,9 @@ func Load(l *loader.Loader, arch *sys.Arch, localSymVersion int, f *bio.Reader,
 		}
 
 		bld.SetType(l.SymType(outer))
-		l.AddInteriorSym(outer, s)
+		if l.SymSize(outer) != 0 { // skip empty section (0-sized symbol)
+			l.AddInteriorSym(outer, s)
+		}
 
 		bld.SetValue(int64(machsym.value - sect.addr))
 		if !l.AttrCgoExportDynamic(s) {
@@ -722,27 +728,28 @@ func Load(l *loader.Loader, arch *sys.Arch, localSymVersion int, f *bio.Reader,
 
 			// Handle X86_64_RELOC_SIGNED referencing a section (rel.extrn == 0).
 			p := l.Data(s)
-			if arch.Family == sys.AMD64 && rel.extrn == 0 && rel.type_ == MACHO_X86_64_RELOC_SIGNED {
-				// Calculate the addend as the offset into the section.
-				//
-				// The rip-relative offset stored in the object file is encoded
-				// as follows:
-				//
-				//    movsd	0x00000360(%rip),%xmm0
-				//
-				// To get the absolute address of the value this rip-relative address is pointing
-				// to, we must add the address of the next instruction to it. This is done by
-				// taking the address of the relocation and adding 4 to it (since the rip-relative
-				// offset can at most be 32 bits long).  To calculate the offset into the section the
-				// relocation is referencing, we subtract the vaddr of the start of the referenced
-				// section found in the original object file.
-				//
-				// [For future reference, see Darwin's /usr/include/mach-o/x86_64/reloc.h]
-				secaddr := c.seg.sect[rel.symnum-1].addr
-
-				rAdd = int64(uint64(int64(int32(e.Uint32(p[rOff:])))+int64(rOff)+4) - secaddr)
-			} else {
-				rAdd = int64(int32(e.Uint32(p[rOff:])))
+			if arch.Family == sys.AMD64 {
+				if rel.extrn == 0 && rel.type_ == MACHO_X86_64_RELOC_SIGNED {
+					// Calculate the addend as the offset into the section.
+					//
+					// The rip-relative offset stored in the object file is encoded
+					// as follows:
+					//
+					//    movsd	0x00000360(%rip),%xmm0
+					//
+					// To get the absolute address of the value this rip-relative address is pointing
+					// to, we must add the address of the next instruction to it. This is done by
+					// taking the address of the relocation and adding 4 to it (since the rip-relative
+					// offset can at most be 32 bits long).  To calculate the offset into the section the
+					// relocation is referencing, we subtract the vaddr of the start of the referenced
+					// section found in the original object file.
+					//
+					// [For future reference, see Darwin's /usr/include/mach-o/x86_64/reloc.h]
+					secaddr := c.seg.sect[rel.symnum-1].addr
+					rAdd = int64(uint64(int64(int32(e.Uint32(p[rOff:])))+int64(rOff)+4) - secaddr)
+				} else {
+					rAdd = int64(int32(e.Uint32(p[rOff:])))
+				}
 			}
 
 			// An unsigned internal relocation has a value offset
-- 
cgit v1.3


From 8e5778ed70ec3d371615a663520a586745fb7bee Mon Sep 17 00:00:00 2001
From: Than McIntosh <thanm@google.com>
Date: Thu, 5 Nov 2020 14:19:47 -0500
Subject: cmd/link: report error if builtin referenced but not defined

When the compiler refers to a runtime builtin, it emits an indexed
symbol reference in the object file via predetermined/preassigned ID
within the PkgIdxBuiltin pseudo-package. At link time when the loader
encounters these references, it redirects them to the corresponding
defined symbol in the runtime package. This redirection process
currently assumes that if a runtime builtin is referenced, we'll
always have a definition for it. This assumption holds in most cases,
however for the builtins "runtime.racefuncenter" and
"runtime.racefuncexit", we'll only see definitions if the runtime
package we're linking against was built with "-race".

In the bug in question, build passes "-gcflags=-race" during
compilation of the main package, but doesn't pass "-race" directly to
'go build', and as a result the final link combines a
race-instrumented main with a non-race runtime; this results in R_CALL
relocations with zero-valued target symbols, resulting in a panic
during stack checking.

This patch changes the loader's resolve method to detect situations
where we're asking for builtin "runtime.X", but the runtime package
read in doesn't contain a definition for X.

Fixes #42396.

Change-Id: Iafd38bd3b0f7f462868d120ccd4d7d1b88b27436
Reviewed-on: https://go-review.googlesource.com/c/go/+/267881
Trust: Than McIntosh <thanm@google.com>
Run-TryBot: Than McIntosh <thanm@google.com>
Reviewed-by: Jeremy Faller <jeremy@golang.org>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
---
 src/cmd/link/internal/loader/loader.go | 29 ++++++++++++++++++-
 src/cmd/link/link_test.go              | 52 ++++++++++++++++++++++++++++++++++
 2 files changed, 80 insertions(+), 1 deletion(-)

(limited to 'src/cmd/link/internal/loader/loader.go')

diff --git a/src/cmd/link/internal/loader/loader.go b/src/cmd/link/internal/loader/loader.go
index d861efcb13..971cc432ff 100644
--- a/src/cmd/link/internal/loader/loader.go
+++ b/src/cmd/link/internal/loader/loader.go
@@ -633,7 +633,11 @@ func (l *Loader) resolve(r *oReader, s goobj.SymRef) Sym {
 		i := int(s.SymIdx) + r.ndef + r.nhashed64def + r.nhasheddef
 		return r.syms[i]
 	case goobj.PkgIdxBuiltin:
-		return l.builtinSyms[s.SymIdx]
+		if bi := l.builtinSyms[s.SymIdx]; bi != 0 {
+			return bi
+		}
+		l.reportMissingBuiltin(int(s.SymIdx), r.unit.Lib.Pkg)
+		return 0
 	case goobj.PkgIdxSelf:
 		rr = r
 	default:
@@ -642,6 +646,29 @@ func (l *Loader) resolve(r *oReader, s goobj.SymRef) Sym {
 	return l.toGlobal(rr, s.SymIdx)
 }
 
+// reportMissingBuiltin issues an error in the case where we have a
+// relocation against a runtime builtin whose definition is not found
+// when the runtime package is built. The canonical example is
+// "runtime.racefuncenter" -- currently if you do something like
+//
+//    go build -gcflags=-race myprogram.go
+//
+// the compiler will insert calls to the builtin runtime.racefuncenter,
+// but the version of the runtime used for linkage won't actually contain
+// definitions of that symbol. See issue #42396 for details.
+//
+// As currently implemented, this is a fatal error. This has drawbacks
+// in that if there are multiple missing builtins, the error will only
+// cite the first one. On the plus side, terminating the link here has
+// advantages in that we won't run the risk of panics or crashes later
+// on in the linker due to R_CALL relocations with 0-valued target
+// symbols.
+func (l *Loader) reportMissingBuiltin(bsym int, reflib string) {
+	bname, _ := goobj.BuiltinName(bsym)
+	log.Fatalf("reference to undefined builtin %q from package %q",
+		bname, reflib)
+}
+
 // Look up a symbol by name, return global index, or 0 if not found.
 // This is more like Syms.ROLookup than Lookup -- it doesn't create
 // new symbol.
diff --git a/src/cmd/link/link_test.go b/src/cmd/link/link_test.go
index 204410e976..158c670739 100644
--- a/src/cmd/link/link_test.go
+++ b/src/cmd/link/link_test.go
@@ -7,6 +7,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"cmd/internal/sys"
 	"debug/macho"
 	"internal/testenv"
 	"io/ioutil"
@@ -873,3 +874,54 @@ func TestIssue38554(t *testing.T) {
 		t.Errorf("binary too big: got %d, want < %d", got, want)
 	}
 }
+
+const testIssue42396src = `
+package main
+
+//go:noinline
+//go:nosplit
+func callee(x int) {
+}
+
+func main() {
+	callee(9)
+}
+`
+
+func TestIssue42396(t *testing.T) {
+	testenv.MustHaveGoBuild(t)
+
+	if !sys.RaceDetectorSupported(runtime.GOOS, runtime.GOARCH) {
+		t.Skip("no race detector support")
+	}
+
+	t.Parallel()
+
+	tmpdir, err := ioutil.TempDir("", "TestIssue42396")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	src := filepath.Join(tmpdir, "main.go")
+	err = ioutil.WriteFile(src, []byte(testIssue42396src), 0666)
+	if err != nil {
+		t.Fatalf("failed to write source file: %v", err)
+	}
+	exe := filepath.Join(tmpdir, "main.exe")
+	cmd := exec.Command(testenv.GoToolPath(t), "build", "-gcflags=-race", "-o", exe, src)
+	out, err := cmd.CombinedOutput()
+	if err == nil {
+		t.Fatalf("build unexpectedly succeeded")
+	}
+
+	// Check to make sure that we see a reasonable error message
+	// and not a panic.
+	if strings.Contains(string(out), "panic:") {
+		t.Fatalf("build should not fail with panic:\n%s", out)
+	}
+	const want = "reference to undefined builtin"
+	if !strings.Contains(string(out), want) {
+		t.Fatalf("error message incorrect: expected it to contain %q but instead got:\n%s\n", want, out)
+	}
+}
-- 
cgit v1.3