[dev.fuzz] all: merge master into dev.fuzz

Change-Id: I5d8c8329ccc9d747bd81ade6b1cb7cb8ae2e94b2

4 files changed, 136 insertions, 11 deletions

diff --git a/src/cmd/internal/buildid/buildid.go b/src/cmd/internal/buildid/buildid.go
index ac238d70ea..1e8855d3ac 100644
--- a/src/cmd/internal/buildid/buildid.go
+++ b/src/cmd/internal/buildid/buildid.go
@@ -10,18 +10,15 @@ import (
 	"fmt"
 	"internal/xcoff"
 	"io"
+	"io/fs"
 	"os"
 	"strconv"
 	"strings"
 )
 
 var (
-	errBuildIDToolchain = fmt.Errorf("build ID only supported in gc toolchain")
 	errBuildIDMalformed = fmt.Errorf("malformed object file")
-	errBuildIDUnknown   = fmt.Errorf("lost build ID")
-)
 
-var (
 	bangArch = []byte("!<arch>")
 	pkgdef   = []byte("__.PKGDEF")
 	goobject = []byte("go object ")
@@ -109,7 +106,7 @@ func ReadFile(name string) (id string, err error) {
 // in cmd/go/internal/work/exec.go.
 func readGccgoArchive(name string, f *os.File) (string, error) {
 	bad := func() (string, error) {
-		return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
+		return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
 	}
 
 	off := int64(8)
@@ -167,7 +164,7 @@ func readGccgoArchive(name string, f *os.File) (string, error) {
 // in cmd/go/internal/work/exec.go.
 func readGccgoBigArchive(name string, f *os.File) (string, error) {
 	bad := func() (string, error) {
-		return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
+		return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
 	}
 
 	// Read fixed-length header.
@@ -309,13 +306,38 @@ func readRaw(name string, data []byte) (id string, err error) {
 
 	j := bytes.Index(data[i+len(goBuildPrefix):], goBuildEnd)
 	if j < 0 {
-		return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
+		return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
 	}
 
 	quoted := data[i+len(goBuildPrefix)-1 : i+len(goBuildPrefix)+j+1]
 	id, err = strconv.Unquote(string(quoted))
 	if err != nil {
-		return "", &os.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
+		return "", &fs.PathError{Op: "parse", Path: name, Err: errBuildIDMalformed}
 	}
 	return id, nil
 }
+
+// HashToString converts the hash h to a string to be recorded
+// in package archives and binaries as part of the build ID.
+// We use the first 120 bits of the hash (5 chunks of 24 bits each) and encode
+// it in base64, resulting in a 20-byte string. Because this is only used for
+// detecting the need to rebuild installed files (not for lookups
+// in the object file cache), 120 bits are sufficient to drive the
+// probability of a false "do not need to rebuild" decision to effectively zero.
+// We embed two different hashes in archives and four in binaries,
+// so cutting to 20 bytes is a significant savings when build IDs are displayed.
+// (20*4+3 = 83 bytes compared to 64*4+3 = 259 bytes for the
+// more straightforward option of printing the entire h in base64).
+func HashToString(h [32]byte) string {
+	const b64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+	const chunks = 5
+	var dst [chunks * 4]byte
+	for i := 0; i < chunks; i++ {
+		v := uint32(h[3*i])<<16 | uint32(h[3*i+1])<<8 | uint32(h[3*i+2])
+		dst[4*i+0] = b64[(v>>18)&0x3F]
+		dst[4*i+1] = b64[(v>>12)&0x3F]
+		dst[4*i+2] = b64[(v>>6)&0x3F]
+		dst[4*i+3] = b64[v&0x3F]
+	}
+	return string(dst[:])
+}
diff --git a/src/cmd/internal/buildid/buildid_test.go b/src/cmd/internal/buildid/buildid_test.go
index 904c2c6f37..e832f9987e 100644
--- a/src/cmd/internal/buildid/buildid_test.go
+++ b/src/cmd/internal/buildid/buildid_test.go
@@ -11,6 +11,7 @@ import (
 	"io/ioutil"
 	"os"
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -146,3 +147,33 @@ func TestFindAndHash(t *testing.T) {
 		}
 	}
 }
+
+func TestExcludedReader(t *testing.T) {
+	const s = "0123456789abcdefghijklmn"
+	tests := []struct {
+		start, end int64    // excluded range
+		results    []string // expected results of reads
+	}{
+		{12, 15, []string{"0123456789", "ab\x00\x00\x00fghij", "klmn"}},                              // within one read
+		{8, 21, []string{"01234567\x00\x00", "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "\x00lmn"}}, // across multiple reads
+		{10, 20, []string{"0123456789", "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", "klmn"}},         // a whole read
+		{0, 5, []string{"\x00\x00\x00\x00\x0056789", "abcdefghij", "klmn"}},                          // start
+		{12, 24, []string{"0123456789", "ab\x00\x00\x00\x00\x00\x00\x00\x00", "\x00\x00\x00\x00"}},   // end
+	}
+	p := make([]byte, 10)
+	for _, test := range tests {
+		r := &excludedReader{strings.NewReader(s), 0, test.start, test.end}
+		for _, res := range test.results {
+			n, err := r.Read(p)
+			if err != nil {
+				t.Errorf("read failed: %v", err)
+			}
+			if n != len(res) {
+				t.Errorf("unexpected number of bytes read: want %d, got %d", len(res), n)
+			}
+			if string(p[:n]) != res {
+				t.Errorf("unexpected bytes: want %q, got %q", res, p[:n])
+			}
+		}
+	}
+}
diff --git a/src/cmd/internal/buildid/note.go b/src/cmd/internal/buildid/note.go
index 2d26ea9961..f5b6fc565f 100644
--- a/src/cmd/internal/buildid/note.go
+++ b/src/cmd/internal/buildid/note.go
@@ -11,6 +11,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 )
 
@@ -96,7 +97,7 @@ func readELF(name string, f *os.File, data []byte) (buildid string, err error) {
 
 	ef, err := elf.NewFile(bytes.NewReader(data))
 	if err != nil {
-		return "", &os.PathError{Path: name, Op: "parse", Err: err}
+		return "", &fs.PathError{Path: name, Op: "parse", Err: err}
 	}
 	var gnu string
 	for _, p := range ef.Progs {
@@ -181,13 +182,13 @@ func readMacho(name string, f *os.File, data []byte) (buildid string, err error)
 
 	mf, err := macho.NewFile(f)
 	if err != nil {
-		return "", &os.PathError{Path: name, Op: "parse", Err: err}
+		return "", &fs.PathError{Path: name, Op: "parse", Err: err}
 	}
 
 	sect := mf.Section("__text")
 	if sect == nil {
 		// Every binary has a __text section. Something is wrong.
-		return "", &os.PathError{Path: name, Op: "parse", Err: fmt.Errorf("cannot find __text section")}
+		return "", &fs.PathError{Path: name, Op: "parse", Err: fmt.Errorf("cannot find __text section")}
 	}
 
 	// It should be in the first few bytes, but read a lot just in case,
diff --git a/src/cmd/internal/buildid/rewrite.go b/src/cmd/internal/buildid/rewrite.go
index 5be54552a6..a7928959c4 100644
--- a/src/cmd/internal/buildid/rewrite.go
+++ b/src/cmd/internal/buildid/rewrite.go
@@ -6,7 +6,9 @@ package buildid
 
 import (
 	"bytes"
+	"cmd/internal/codesign"
 	"crypto/sha256"
+	"debug/macho"
 	"fmt"
 	"io"
 )
@@ -26,6 +28,11 @@ func FindAndHash(r io.Reader, id string, bufSize int) (matches []int64, hash [32
 	zeros := make([]byte, len(id))
 	idBytes := []byte(id)
 
+	// For Mach-O files, we want to exclude the code signature.
+	// The code signature contains hashes of the whole file (except the signature
+	// itself), including the buildid. So the buildid cannot contain the signature.
+	r = excludeMachoCodeSignature(r)
+
 	// The strategy is to read the file through buf, looking for id,
 	// but we need to worry about what happens if id is broken up
 	// and returned in parts by two different reads.
@@ -87,5 +94,69 @@ func Rewrite(w io.WriterAt, pos []int64, id string) error {
 			return err
 		}
 	}
+
+	// Update Mach-O code signature, if any.
+	if f, cmd, ok := findMachoCodeSignature(w); ok {
+		if codesign.Size(int64(cmd.Dataoff), "a.out") == int64(cmd.Datasize) {
+			// Update the signature if the size matches, so we don't need to
+			// fix up headers. Binaries generated by the Go linker should have
+			// the expected size. Otherwise skip.
+			text := f.Segment("__TEXT")
+			cs := make([]byte, cmd.Datasize)
+			codesign.Sign(cs, w.(io.Reader), "a.out", int64(cmd.Dataoff), int64(text.Offset), int64(text.Filesz), f.Type == macho.TypeExec)
+			if _, err := w.WriteAt(cs, int64(cmd.Dataoff)); err != nil {
+				return err
+			}
+		}
+	}
+
 	return nil
 }
+
+func excludeMachoCodeSignature(r io.Reader) io.Reader {
+	_, cmd, ok := findMachoCodeSignature(r)
+	if !ok {
+		return r
+	}
+	return &excludedReader{r, 0, int64(cmd.Dataoff), int64(cmd.Dataoff + cmd.Datasize)}
+}
+
+// excludedReader wraps an io.Reader. Reading from it returns the bytes from
+// the underlying reader, except that when the byte offset is within the
+// range between start and end, it returns zero bytes.
+type excludedReader struct {
+	r          io.Reader
+	off        int64 // current offset
+	start, end int64 // the range to be excluded (read as zero)
+}
+
+func (r *excludedReader) Read(p []byte) (int, error) {
+	n, err := r.r.Read(p)
+	if n > 0 && r.off+int64(n) > r.start && r.off < r.end {
+		cstart := r.start - r.off
+		if cstart < 0 {
+			cstart = 0
+		}
+		cend := r.end - r.off
+		if cend > int64(n) {
+			cend = int64(n)
+		}
+		zeros := make([]byte, cend-cstart)
+		copy(p[cstart:cend], zeros)
+	}
+	r.off += int64(n)
+	return n, err
+}
+
+func findMachoCodeSignature(r interface{}) (*macho.File, codesign.CodeSigCmd, bool) {
+	ra, ok := r.(io.ReaderAt)
+	if !ok {
+		return nil, codesign.CodeSigCmd{}, false
+	}
+	f, err := macho.NewFile(ra)
+	if err != nil {
+		return nil, codesign.CodeSigCmd{}, false
+	}
+	cmd, ok := codesign.FindCodeSigCmd(f)
+	return f, cmd, ok
+}