cmd/ld, runtime: adjust symbol table representation

This CL changes the encoding used for the Go symbol table,
stored in the binary and used at run time. It does not change
any of the semantics or structure: the bits are just packed
a little differently.

The comment at the top of runtime/symtab.c describes the new format.

Compared to the Go 1.0 format, the main changes are:

* Store symbol addresses as full-pointer-sized host-endian values.
  (For 6g, this means addresses are 64-bit little-endian.)

* Store other values (frame sizes and so on) varint-encoded.

The second change more than compensates for the first:
for the godoc binary on OS X/amd64, the new symbol table
is 8% smaller than the old symbol table (1,425,668 down from 1,546,276).

This is a required step for allowing the host linker (gcc) to write
the final Go binary, since it will have to fill in the symbol address slots
(so the slots must be host-endian) and on 64-bit systems it may
choose addresses above 4 GB.

R=golang-dev, iant
CC=golang-dev
https://golang.org/cl/7403054

1 files changed, 108 insertions, 15 deletions

diff --git a/src/pkg/debug/gosym/symtab.go b/src/pkg/debug/gosym/symtab.go
index cc01e0b9d6..81ed4fb27d 100644
--- a/src/pkg/debug/gosym/symtab.go
+++ b/src/pkg/debug/gosym/symtab.go
@@ -99,31 +99,116 @@ type Table struct {
 }
 
 type sym struct {
-	value  uint32
-	gotype uint32
+	value  uint64
+	gotype uint64
 	typ    byte
 	name   []byte
 }
 
-var littleEndianSymtab = []byte{0xFE, 0xFF, 0xFF, 0xFF, 0x00, 0x00}
+var littleEndianSymtab = []byte{0xFD, 0xFF, 0xFF, 0xFF, 0x00, 0x00, 0x00}
+var bigEndianSymtab = []byte{0xFF, 0xFF, 0xFF, 0xFD, 0x00, 0x00, 0x00}
+
+var oldLittleEndianSymtab = []byte{0xFE, 0xFF, 0xFF, 0xFF, 0x00, 0x00}
 
 func walksymtab(data []byte, fn func(sym) error) error {
 	var order binary.ByteOrder = binary.BigEndian
-	if bytes.HasPrefix(data, littleEndianSymtab) {
+	newTable := false
+	switch {
+	case bytes.HasPrefix(data, oldLittleEndianSymtab):
+		// Same as Go 1.0, but little endian.
+		// Format was used during interim development between Go 1.0 and Go 1.1.
+		// Should not be widespread, but easy to support.
 		data = data[6:]
 		order = binary.LittleEndian
+	case bytes.HasPrefix(data, bigEndianSymtab):
+		newTable = true
+	case bytes.HasPrefix(data, littleEndianSymtab):
+		newTable = true
+		order = binary.LittleEndian
+	}
+	var ptrsz int
+	if newTable {
+		if len(data) < 8 {
+			return &DecodingError{len(data), "unexpected EOF", nil}
+		}
+		ptrsz = int(data[7])
+		if ptrsz != 4 && ptrsz != 8 {
+			return &DecodingError{7, "invalid pointer size", ptrsz}
+		}
+		data = data[8:]
 	}
 	var s sym
 	p := data
-	for len(p) >= 6 {
-		s.value = order.Uint32(p[0:4])
-		typ := p[4]
-		if typ&0x80 == 0 {
-			return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ}
+	for len(p) >= 4 {
+		var typ byte
+		if newTable {
+			// Symbol type, value, Go type.
+			typ = p[0] & 0x3F
+			wideValue := p[0]&0x40 != 0
+			goType := p[0]&0x80 != 0
+			if typ < 26 {
+				typ += 'A'
+			} else {
+				typ += 'a' - 26
+			}
+			s.typ = typ
+			p = p[1:]
+			if wideValue {
+				if len(p) < ptrsz {
+					return &DecodingError{len(data), "unexpected EOF", nil}
+				}
+				// fixed-width value
+				if ptrsz == 8 {
+					s.value = order.Uint64(p[0:8])
+					p = p[8:]
+				} else {
+					s.value = uint64(order.Uint32(p[0:4]))
+					p = p[4:]
+				}
+			} else {
+				// varint value
+				s.value = 0
+				shift := uint(0)
+				for len(p) > 0 && p[0]&0x80 != 0 {
+					s.value |= uint64(p[0]&0x7F) << shift
+					shift += 7
+					p = p[1:]
+				}
+				if len(p) == 0 {
+					return &DecodingError{len(data), "unexpected EOF", nil}
+				}
+				s.value |= uint64(p[0]) << shift
+				p = p[1:]
+			}
+			if goType {
+				if len(p) < ptrsz {
+					return &DecodingError{len(data), "unexpected EOF", nil}
+				}
+				// fixed-width go type
+				if ptrsz == 8 {
+					s.gotype = order.Uint64(p[0:8])
+					p = p[8:]
+				} else {
+					s.gotype = uint64(order.Uint32(p[0:4]))
+					p = p[4:]
+				}
+			}
+		} else {
+			// Value, symbol type.
+			s.value = uint64(order.Uint32(p[0:4]))
+			if len(p) < 5 {
+				return &DecodingError{len(data), "unexpected EOF", nil}
+			}
+			typ = p[4]
+			if typ&0x80 == 0 {
+				return &DecodingError{len(data) - len(p) + 4, "bad symbol type", typ}
+			}
+			typ &^= 0x80
+			s.typ = typ
+			p = p[5:]
 		}
-		typ &^= 0x80
-		s.typ = typ
-		p = p[5:]
+
+		// Name.
 		var i int
 		var nnul int
 		for i = 0; i < len(p); i++ {
@@ -142,13 +227,21 @@ func walksymtab(data []byte, fn func(sym) error) error {
 				}
 			}
 		}
-		if i+nnul+4 > len(p) {
+		if len(p) < i+nnul {
 			return &DecodingError{len(data), "unexpected EOF", nil}
 		}
 		s.name = p[0:i]
 		i += nnul
-		s.gotype = order.Uint32(p[i : i+4])
-		p = p[i+4:]
+		p = p[i:]
+
+		if !newTable {
+			if len(p) < 4 {
+				return &DecodingError{len(data), "unexpected EOF", nil}
+			}
+			// Go type.
+			s.gotype = uint64(order.Uint32(p[:4]))
+			p = p[4:]
+		}
 		fn(s)
 	}
 	return nil