[dev.regabi] cmd/compile: split out package staticdata [generated]

[git-generate]
cd src/cmd/compile/internal/gc

rf '
	# Export API and move to its own files.
	mv addrsym InitAddr
	mv pfuncsym InitFunc
	mv slicesym InitSlice
	mv slicebytes InitSliceBytes
	mv stringsym StringSym
	mv funcsym FuncSym
	mv makefuncsym NeedFuncSym
	mv dumpfuncsyms WriteFuncSyms
	mv InitAddr InitFunc InitSlice InitSliceBytes stringSymPrefix \
		StringSym fileStringSym slicedataGen slicedata dstringdata \
		funcsyms FuncSym NeedFuncSym WriteFuncSyms \
		data.go

	mv initEmbed WriteEmbed
	mv dumpembeds obj.go

	mv data.go embed.go cmd/compile/internal/staticdata
'

Change-Id: I209c5e597c8acfa29a48527695a9ddc1e9ea8e6a
Reviewed-on: https://go-review.googlesource.com/c/go/+/279474
Trust: Russ Cox <rsc@golang.org>
Run-TryBot: Russ Cox <rsc@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>

2 files changed, 489 insertions, 0 deletions

diff --git a/src/cmd/compile/internal/staticdata/data.go b/src/cmd/compile/internal/staticdata/data.go
new file mode 100644
index 0000000000..7627aaa11a
--- /dev/null
+++ b/src/cmd/compile/internal/staticdata/data.go
@@ -0,0 +1,296 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package staticdata
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strconv"
+	"sync"
+
+	"cmd/compile/internal/base"
+	"cmd/compile/internal/ir"
+	"cmd/compile/internal/objw"
+	"cmd/compile/internal/typecheck"
+	"cmd/compile/internal/types"
+	"cmd/internal/obj"
+	"cmd/internal/objabi"
+	"cmd/internal/src"
+)
+
+// InitAddr writes the static address of a to n. a must be an ONAME.
+// Neither n nor a is modified.
+func InitAddr(n *ir.Name, noff int64, a *ir.Name, aoff int64) {
+	if n.Op() != ir.ONAME {
+		base.Fatalf("addrsym n op %v", n.Op())
+	}
+	if n.Sym() == nil {
+		base.Fatalf("addrsym nil n sym")
+	}
+	if a.Op() != ir.ONAME {
+		base.Fatalf("addrsym a op %v", a.Op())
+	}
+	s := n.Sym().Linksym()
+	s.WriteAddr(base.Ctxt, noff, types.PtrSize, a.Sym().Linksym(), aoff)
+}
+
+// InitFunc writes the static address of f to n. f must be a global function.
+// Neither n nor f is modified.
+func InitFunc(n *ir.Name, noff int64, f *ir.Name) {
+	if n.Op() != ir.ONAME {
+		base.Fatalf("pfuncsym n op %v", n.Op())
+	}
+	if n.Sym() == nil {
+		base.Fatalf("pfuncsym nil n sym")
+	}
+	if f.Class_ != ir.PFUNC {
+		base.Fatalf("pfuncsym class not PFUNC %d", f.Class_)
+	}
+	s := n.Sym().Linksym()
+	s.WriteAddr(base.Ctxt, noff, types.PtrSize, FuncSym(f.Sym()).Linksym(), 0)
+}
+
+// InitSlice writes a static slice symbol {&arr, lencap, lencap} to n+noff.
+// InitSlice does not modify n.
+func InitSlice(n *ir.Name, noff int64, arr *ir.Name, lencap int64) {
+	s := n.Sym().Linksym()
+	if arr.Op() != ir.ONAME {
+		base.Fatalf("slicesym non-name arr %v", arr)
+	}
+	s.WriteAddr(base.Ctxt, noff, types.PtrSize, arr.Sym().Linksym(), 0)
+	s.WriteInt(base.Ctxt, noff+types.SliceLenOffset, types.PtrSize, lencap)
+	s.WriteInt(base.Ctxt, noff+types.SliceCapOffset, types.PtrSize, lencap)
+}
+
+func InitSliceBytes(nam *ir.Name, off int64, s string) {
+	if nam.Op() != ir.ONAME {
+		base.Fatalf("slicebytes %v", nam)
+	}
+	InitSlice(nam, off, slicedata(nam.Pos(), s), int64(len(s)))
+}
+
+const (
+	stringSymPrefix  = "go.string."
+	stringSymPattern = ".gostring.%d.%x"
+)
+
+// StringSym returns a symbol containing the string s.
+// The symbol contains the string data, not a string header.
+func StringSym(pos src.XPos, s string) (data *obj.LSym) {
+	var symname string
+	if len(s) > 100 {
+		// Huge strings are hashed to avoid long names in object files.
+		// Indulge in some paranoia by writing the length of s, too,
+		// as protection against length extension attacks.
+		// Same pattern is known to fileStringSym below.
+		h := sha256.New()
+		io.WriteString(h, s)
+		symname = fmt.Sprintf(stringSymPattern, len(s), h.Sum(nil))
+	} else {
+		// Small strings get named directly by their contents.
+		symname = strconv.Quote(s)
+	}
+
+	symdata := base.Ctxt.Lookup(stringSymPrefix + symname)
+	if !symdata.OnList() {
+		off := dstringdata(symdata, 0, s, pos, "string")
+		objw.Global(symdata, int32(off), obj.DUPOK|obj.RODATA|obj.LOCAL)
+		symdata.Set(obj.AttrContentAddressable, true)
+	}
+
+	return symdata
+}
+
+// fileStringSym returns a symbol for the contents and the size of file.
+// If readonly is true, the symbol shares storage with any literal string
+// or other file with the same content and is placed in a read-only section.
+// If readonly is false, the symbol is a read-write copy separate from any other,
+// for use as the backing store of a []byte.
+// The content hash of file is copied into hash. (If hash is nil, nothing is copied.)
+// The returned symbol contains the data itself, not a string header.
+func fileStringSym(pos src.XPos, file string, readonly bool, hash []byte) (*obj.LSym, int64, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer f.Close()
+	info, err := f.Stat()
+	if err != nil {
+		return nil, 0, err
+	}
+	if !info.Mode().IsRegular() {
+		return nil, 0, fmt.Errorf("not a regular file")
+	}
+	size := info.Size()
+	if size <= 1*1024 {
+		data, err := ioutil.ReadAll(f)
+		if err != nil {
+			return nil, 0, err
+		}
+		if int64(len(data)) != size {
+			return nil, 0, fmt.Errorf("file changed between reads")
+		}
+		var sym *obj.LSym
+		if readonly {
+			sym = StringSym(pos, string(data))
+		} else {
+			sym = slicedata(pos, string(data)).Sym().Linksym()
+		}
+		if len(hash) > 0 {
+			sum := sha256.Sum256(data)
+			copy(hash, sum[:])
+		}
+		return sym, size, nil
+	}
+	if size > 2e9 {
+		// ggloblsym takes an int32,
+		// and probably the rest of the toolchain
+		// can't handle such big symbols either.
+		// See golang.org/issue/9862.
+		return nil, 0, fmt.Errorf("file too large")
+	}
+
+	// File is too big to read and keep in memory.
+	// Compute hash if needed for read-only content hashing or if the caller wants it.
+	var sum []byte
+	if readonly || len(hash) > 0 {
+		h := sha256.New()
+		n, err := io.Copy(h, f)
+		if err != nil {
+			return nil, 0, err
+		}
+		if n != size {
+			return nil, 0, fmt.Errorf("file changed between reads")
+		}
+		sum = h.Sum(nil)
+		copy(hash, sum)
+	}
+
+	var symdata *obj.LSym
+	if readonly {
+		symname := fmt.Sprintf(stringSymPattern, size, sum)
+		symdata = base.Ctxt.Lookup(stringSymPrefix + symname)
+		if !symdata.OnList() {
+			info := symdata.NewFileInfo()
+			info.Name = file
+			info.Size = size
+			objw.Global(symdata, int32(size), obj.DUPOK|obj.RODATA|obj.LOCAL)
+			// Note: AttrContentAddressable cannot be set here,
+			// because the content-addressable-handling code
+			// does not know about file symbols.
+		}
+	} else {
+		// Emit a zero-length data symbol
+		// and then fix up length and content to use file.
+		symdata = slicedata(pos, "").Sym().Linksym()
+		symdata.Size = size
+		symdata.Type = objabi.SNOPTRDATA
+		info := symdata.NewFileInfo()
+		info.Name = file
+		info.Size = size
+	}
+
+	return symdata, size, nil
+}
+
+var slicedataGen int
+
+func slicedata(pos src.XPos, s string) *ir.Name {
+	slicedataGen++
+	symname := fmt.Sprintf(".gobytes.%d", slicedataGen)
+	sym := types.LocalPkg.Lookup(symname)
+	symnode := typecheck.NewName(sym)
+	sym.Def = symnode
+
+	lsym := sym.Linksym()
+	off := dstringdata(lsym, 0, s, pos, "slice")
+	objw.Global(lsym, int32(off), obj.NOPTR|obj.LOCAL)
+
+	return symnode
+}
+
+func dstringdata(s *obj.LSym, off int, t string, pos src.XPos, what string) int {
+	// Objects that are too large will cause the data section to overflow right away,
+	// causing a cryptic error message by the linker. Check for oversize objects here
+	// and provide a useful error message instead.
+	if int64(len(t)) > 2e9 {
+		base.ErrorfAt(pos, "%v with length %v is too big", what, len(t))
+		return 0
+	}
+
+	s.WriteString(base.Ctxt, int64(off), len(t), t)
+	return off + len(t)
+}
+
+var (
+	funcsymsmu sync.Mutex // protects funcsyms and associated package lookups (see func funcsym)
+	funcsyms   []*types.Sym
+)
+
+// FuncSym returns s·f.
+func FuncSym(s *types.Sym) *types.Sym {
+	// funcsymsmu here serves to protect not just mutations of funcsyms (below),
+	// but also the package lookup of the func sym name,
+	// since this function gets called concurrently from the backend.
+	// There are no other concurrent package lookups in the backend,
+	// except for the types package, which is protected separately.
+	// Reusing funcsymsmu to also cover this package lookup
+	// avoids a general, broader, expensive package lookup mutex.
+	// Note makefuncsym also does package look-up of func sym names,
+	// but that it is only called serially, from the front end.
+	funcsymsmu.Lock()
+	sf, existed := s.Pkg.LookupOK(ir.FuncSymName(s))
+	// Don't export s·f when compiling for dynamic linking.
+	// When dynamically linking, the necessary function
+	// symbols will be created explicitly with makefuncsym.
+	// See the makefuncsym comment for details.
+	if !base.Ctxt.Flag_dynlink && !existed {
+		funcsyms = append(funcsyms, s)
+	}
+	funcsymsmu.Unlock()
+	return sf
+}
+
+// NeedFuncSym ensures that s·f is exported.
+// It is only used with -dynlink.
+// When not compiling for dynamic linking,
+// the funcsyms are created as needed by
+// the packages that use them.
+// Normally we emit the s·f stubs as DUPOK syms,
+// but DUPOK doesn't work across shared library boundaries.
+// So instead, when dynamic linking, we only create
+// the s·f stubs in s's package.
+func NeedFuncSym(s *types.Sym) {
+	if !base.Ctxt.Flag_dynlink {
+		base.Fatalf("makefuncsym dynlink")
+	}
+	if s.IsBlank() {
+		return
+	}
+	if base.Flag.CompilingRuntime && (s.Name == "getg" || s.Name == "getclosureptr" || s.Name == "getcallerpc" || s.Name == "getcallersp") {
+		// runtime.getg(), getclosureptr(), getcallerpc(), and
+		// getcallersp() are not real functions and so do not
+		// get funcsyms.
+		return
+	}
+	if _, existed := s.Pkg.LookupOK(ir.FuncSymName(s)); !existed {
+		funcsyms = append(funcsyms, s)
+	}
+}
+
+func WriteFuncSyms() {
+	sort.Slice(funcsyms, func(i, j int) bool {
+		return funcsyms[i].LinksymName() < funcsyms[j].LinksymName()
+	})
+	for _, s := range funcsyms {
+		sf := s.Pkg.Lookup(ir.FuncSymName(s)).Linksym()
+		objw.SymPtr(sf, 0, s.Linksym(), 0)
+		objw.Global(sf, int32(types.PtrSize), obj.DUPOK|obj.RODATA)
+	}
+}
diff --git a/src/cmd/compile/internal/staticdata/embed.go b/src/cmd/compile/internal/staticdata/embed.go
new file mode 100644
index 0000000000..55c9a3356e
--- /dev/null
+++ b/src/cmd/compile/internal/staticdata/embed.go
@@ -0,0 +1,193 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package staticdata
+
+import (
+	"path"
+	"sort"
+	"strings"
+
+	"cmd/compile/internal/base"
+	"cmd/compile/internal/ir"
+	"cmd/compile/internal/objw"
+	"cmd/compile/internal/types"
+	"cmd/internal/obj"
+)
+
+const (
+	embedUnknown = iota
+	embedBytes
+	embedString
+	embedFiles
+)
+
+func embedFileList(v *ir.Name) []string {
+	kind := embedKind(v.Type())
+	if kind == embedUnknown {
+		base.ErrorfAt(v.Pos(), "go:embed cannot apply to var of type %v", v.Type())
+		return nil
+	}
+
+	// Build list of files to store.
+	have := make(map[string]bool)
+	var list []string
+	for _, e := range *v.Embed {
+		for _, pattern := range e.Patterns {
+			files, ok := base.Flag.Cfg.Embed.Patterns[pattern]
+			if !ok {
+				base.ErrorfAt(e.Pos, "invalid go:embed: build system did not map pattern: %s", pattern)
+			}
+			for _, file := range files {
+				if base.Flag.Cfg.Embed.Files[file] == "" {
+					base.ErrorfAt(e.Pos, "invalid go:embed: build system did not map file: %s", file)
+					continue
+				}
+				if !have[file] {
+					have[file] = true
+					list = append(list, file)
+				}
+				if kind == embedFiles {
+					for dir := path.Dir(file); dir != "." && !have[dir]; dir = path.Dir(dir) {
+						have[dir] = true
+						list = append(list, dir+"/")
+					}
+				}
+			}
+		}
+	}
+	sort.Slice(list, func(i, j int) bool {
+		return embedFileLess(list[i], list[j])
+	})
+
+	if kind == embedString || kind == embedBytes {
+		if len(list) > 1 {
+			base.ErrorfAt(v.Pos(), "invalid go:embed: multiple files for type %v", v.Type())
+			return nil
+		}
+	}
+
+	return list
+}
+
+// embedKindApprox determines the kind of embedding variable, approximately.
+// The match is approximate because we haven't done scope resolution yet and
+// can't tell whether "string" and "byte" really mean "string" and "byte".
+// The result must be confirmed later, after type checking, using embedKind.
+func embedKindApprox(typ ir.Node) int {
+	if typ.Sym() != nil && typ.Sym().Name == "FS" && (typ.Sym().Pkg.Path == "embed" || (typ.Sym().Pkg == types.LocalPkg && base.Ctxt.Pkgpath == "embed")) {
+		return embedFiles
+	}
+	// These are not guaranteed to match only string and []byte -
+	// maybe the local package has redefined one of those words.
+	// But it's the best we can do now during the noder.
+	// The stricter check happens later, in initEmbed calling embedKind.
+	if typ.Sym() != nil && typ.Sym().Name == "string" && typ.Sym().Pkg == types.LocalPkg {
+		return embedString
+	}
+	if typ, ok := typ.(*ir.SliceType); ok {
+		if sym := typ.Elem.Sym(); sym != nil && sym.Name == "byte" && sym.Pkg == types.LocalPkg {
+			return embedBytes
+		}
+	}
+	return embedUnknown
+}
+
+// embedKind determines the kind of embedding variable.
+func embedKind(typ *types.Type) int {
+	if typ.Sym() != nil && typ.Sym().Name == "FS" && (typ.Sym().Pkg.Path == "embed" || (typ.Sym().Pkg == types.LocalPkg && base.Ctxt.Pkgpath == "embed")) {
+		return embedFiles
+	}
+	if typ == types.Types[types.TSTRING] {
+		return embedString
+	}
+	if typ.Sym() == nil && typ.IsSlice() && typ.Elem() == types.ByteType {
+		return embedBytes
+	}
+	return embedUnknown
+}
+
+func embedFileNameSplit(name string) (dir, elem string, isDir bool) {
+	if name[len(name)-1] == '/' {
+		isDir = true
+		name = name[:len(name)-1]
+	}
+	i := len(name) - 1
+	for i >= 0 && name[i] != '/' {
+		i--
+	}
+	if i < 0 {
+		return ".", name, isDir
+	}
+	return name[:i], name[i+1:], isDir
+}
+
+// embedFileLess implements the sort order for a list of embedded files.
+// See the comment inside ../../../../embed/embed.go's Files struct for rationale.
+func embedFileLess(x, y string) bool {
+	xdir, xelem, _ := embedFileNameSplit(x)
+	ydir, yelem, _ := embedFileNameSplit(y)
+	return xdir < ydir || xdir == ydir && xelem < yelem
+}
+
+// WriteEmbed emits the init data for a //go:embed variable,
+// which is either a string, a []byte, or an embed.FS.
+func WriteEmbed(v *ir.Name) {
+	files := embedFileList(v)
+	switch kind := embedKind(v.Type()); kind {
+	case embedUnknown:
+		base.ErrorfAt(v.Pos(), "go:embed cannot apply to var of type %v", v.Type())
+
+	case embedString, embedBytes:
+		file := files[0]
+		fsym, size, err := fileStringSym(v.Pos(), base.Flag.Cfg.Embed.Files[file], kind == embedString, nil)
+		if err != nil {
+			base.ErrorfAt(v.Pos(), "embed %s: %v", file, err)
+		}
+		sym := v.Sym().Linksym()
+		off := 0
+		off = objw.SymPtr(sym, off, fsym, 0)       // data string
+		off = objw.Uintptr(sym, off, uint64(size)) // len
+		if kind == embedBytes {
+			objw.Uintptr(sym, off, uint64(size)) // cap for slice
+		}
+
+	case embedFiles:
+		slicedata := base.Ctxt.Lookup(`"".` + v.Sym().Name + `.files`)
+		off := 0
+		// []files pointed at by Files
+		off = objw.SymPtr(slicedata, off, slicedata, 3*types.PtrSize) // []file, pointing just past slice
+		off = objw.Uintptr(slicedata, off, uint64(len(files)))
+		off = objw.Uintptr(slicedata, off, uint64(len(files)))
+
+		// embed/embed.go type file is:
+		//	name string
+		//	data string
+		//	hash [16]byte
+		// Emit one of these per file in the set.
+		const hashSize = 16
+		hash := make([]byte, hashSize)
+		for _, file := range files {
+			off = objw.SymPtr(slicedata, off, StringSym(v.Pos(), file), 0) // file string
+			off = objw.Uintptr(slicedata, off, uint64(len(file)))
+			if strings.HasSuffix(file, "/") {
+				// entry for directory - no data
+				off = objw.Uintptr(slicedata, off, 0)
+				off = objw.Uintptr(slicedata, off, 0)
+				off += hashSize
+			} else {
+				fsym, size, err := fileStringSym(v.Pos(), base.Flag.Cfg.Embed.Files[file], true, hash)
+				if err != nil {
+					base.ErrorfAt(v.Pos(), "embed %s: %v", file, err)
+				}
+				off = objw.SymPtr(slicedata, off, fsym, 0) // data string
+				off = objw.Uintptr(slicedata, off, uint64(size))
+				off = int(slicedata.WriteBytes(base.Ctxt, int64(off), hash))
+			}
+		}
+		objw.Global(slicedata, int32(off), obj.RODATA|obj.LOCAL)
+		sym := v.Sym().Linksym()
+		objw.SymPtr(sym, 0, slicedata, 0)
+	}
+}