cmd/cgo: fix cgo bad typedefs

Two fixes:

1) Typedefs of the bad typedefs should also not be rewritten to the
   underlying type.  They shouldn't just be uintptr, though, they should
   retain the C naming structure.  For example, in C:

   typedef const __CFString * CFStringRef;
   typedef CFStringRef SecKeyAlgorithm;

   we want the Go:

   type _Ctype_CFStringRef uintptr
   type _Ctype_SecKeyAlgorithm = _Ctype_CFStringRef

2) We need more types than just function arguments/return values.
   At least we need types of global variables, so when we see a reference to:

   extern const SecKeyAlgorithm kSecKeyAlgorithmECDSASignatureDigestX962SHA1;

   we know that we need to investigate the type SecKeyAlgorithm.
   Might as well just find every typedef and check the badness of all of them.
   This requires looping until a fixed point of known types is reached.
   Usually it takes just 2 iterations, sometimes 3.

Fixes #24161

Change-Id: I32ca7e48eb4d4133c6242e91d1879636f5224ea9
Reviewed-on: https://go-review.googlesource.com/123177
Run-TryBot: Keith Randall <khr@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Ian Lance Taylor <iant@golang.org>

2 files changed, 71 insertions, 31 deletions

diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go
index 9f4a611f19..251cb18f5f 100644
--- a/src/cmd/cgo/gcc.go
+++ b/src/cmd/cgo/gcc.go
@@ -164,24 +164,22 @@ func (p *Package) Translate(f *File) {
 		cref.Name.C = cname(cref.Name.Go)
 	}
 	p.loadDefines(f)
-	needType := p.guessKinds(f)
-	if len(needType) > 0 {
-		p.loadDWARF(f, needType)
-		// If there are typedefs used as arguments, add those
-		// types to the list of types we're interested in, and
-		// try again.
-		if len(p.ArgTypedefs) > 0 {
-			for _, a := range p.ArgTypedefs {
-				f.Name[a] = &Name{
-					Go: a,
-					C:  a,
-				}
-			}
-			needType := p.guessKinds(f)
-			if len(needType) > 0 {
-				p.loadDWARF(f, needType)
+	p.typedefs = map[string]bool{}
+	p.typedefList = nil
+	numTypedefs := -1
+	for len(p.typedefs) > numTypedefs {
+		numTypedefs = len(p.typedefs)
+		// Also ask about any typedefs we've seen so far.
+		for _, a := range p.typedefList {
+			f.Name[a] = &Name{
+				Go: a,
+				C:  a,
 			}
 		}
+		needType := p.guessKinds(f)
+		if len(needType) > 0 {
+			p.loadDWARF(f, needType)
+		}
 	}
 	if p.rewriteCalls(f) {
 		// Add `import _cgo_unsafe "unsafe"` after the package statement.
@@ -566,6 +564,7 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 				fatalf("malformed __cgo__ name: %s", name)
 			}
 			types[i] = t.Type
+			p.recordTypedefs(t.Type)
 		}
 		if e.Tag != dwarf.TagCompileUnit {
 			r.SkipChildren()
@@ -630,7 +629,43 @@ func (p *Package) loadDWARF(f *File, names []*Name) {
 		}
 		conv.FinishType(pos)
 	}
-	p.ArgTypedefs = conv.argTypedefs
+}
+
+// recordTypedefs remembers in p.typedefs all the typedefs used in dtypes and its children.
+func (p *Package) recordTypedefs(dtype dwarf.Type) {
+	p.recordTypedefs1(dtype, map[dwarf.Type]bool{})
+}
+func (p *Package) recordTypedefs1(dtype dwarf.Type, visited map[dwarf.Type]bool) {
+	if dtype == nil {
+		return
+	}
+	if visited[dtype] {
+		return
+	}
+	visited[dtype] = true
+	switch dt := dtype.(type) {
+	case *dwarf.TypedefType:
+		if !p.typedefs[dt.Name] {
+			p.typedefs[dt.Name] = true
+			p.typedefList = append(p.typedefList, dt.Name)
+			p.recordTypedefs1(dt.Type, visited)
+		}
+	case *dwarf.PtrType:
+		p.recordTypedefs1(dt.Type, visited)
+	case *dwarf.ArrayType:
+		p.recordTypedefs1(dt.Type, visited)
+	case *dwarf.QualType:
+		p.recordTypedefs1(dt.Type, visited)
+	case *dwarf.FuncType:
+		p.recordTypedefs1(dt.ReturnType, visited)
+		for _, a := range dt.ParamType {
+			p.recordTypedefs1(a, visited)
+		}
+	case *dwarf.StructType:
+		for _, f := range dt.Field {
+			p.recordTypedefs1(f.Type, visited)
+		}
+	}
 }
 
 // mangleName does name mangling to translate names
@@ -1712,9 +1747,6 @@ type typeConv struct {
 
 	ptrSize int64
 	intSize int64
-
-	// Typedefs used as argument types for C calls.
-	argTypedefs []string
 }
 
 var tagGen int
@@ -2275,9 +2307,6 @@ func (c *typeConv) FuncArg(dtype dwarf.Type, pos token.Pos) *Type {
 			C:     tr,
 		}
 	case *dwarf.TypedefType:
-		// Keep track of all the typedefs used as arguments.
-		c.argTypedefs = append(c.argTypedefs, dt.Name)
-
 		// C has much more relaxed rules than Go for
 		// implicit type conversions. When the parameter
 		// is type T defined as *X, simulate a little of the
@@ -2290,7 +2319,7 @@ func (c *typeConv) FuncArg(dtype dwarf.Type, pos token.Pos) *Type {
 			}
 			// ...or the typedef is one in which we expect bad pointers.
 			// It will be a uintptr instead of *X.
-			if c.badPointerTypedef(dt) {
+			if c.baseBadPointerTypedef(dt) {
 				break
 			}
 
@@ -2334,9 +2363,6 @@ func (c *typeConv) FuncType(dtype *dwarf.FuncType, pos token.Pos) *FuncType {
 		gr = []*ast.Field{{Type: c.goVoid}}
 	} else if dtype.ReturnType != nil {
 		r = c.Type(unqual(dtype.ReturnType), pos)
-		if dt, ok := dtype.ReturnType.(*dwarf.TypedefType); ok {
-			c.argTypedefs = append(c.argTypedefs, dt.Name)
-		}
 		gr = []*ast.Field{{Type: r.Go}}
 	}
 	return &FuncType{
@@ -2645,6 +2671,19 @@ func (c *typeConv) badPointerTypedef(dt *dwarf.TypedefType) bool {
 	return false
 }
 
+// baseBadPointerTypedef reports whether the base of a chain of typedefs is a bad typedef
+// as badPointerTypedef reports.
+func (c *typeConv) baseBadPointerTypedef(dt *dwarf.TypedefType) bool {
+	for {
+		if t, ok := dt.Type.(*dwarf.TypedefType); ok {
+			dt = t
+			continue
+		}
+		break
+	}
+	return c.badPointerTypedef(dt)
+}
+
 func (c *typeConv) badCFType(dt *dwarf.TypedefType) bool {
 	// The real bad types are CFNumberRef and CFDateRef.
 	// Sometimes non-pointers are stored in these types.
diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go
index 84b1daadbe..540fe7499a 100644
--- a/src/cmd/cgo/main.go
+++ b/src/cmd/cgo/main.go
@@ -43,10 +43,11 @@ type Package struct {
 	Name        map[string]*Name // accumulated Name from Files
 	ExpFunc     []*ExpFunc       // accumulated ExpFunc from Files
 	Decl        []ast.Decl
-	GoFiles     []string // list of Go files
-	GccFiles    []string // list of gcc output files
-	Preamble    string   // collected preamble for _cgo_export.h
-	ArgTypedefs []string // typedefs used as arguments to or results of C functions
+	GoFiles     []string        // list of Go files
+	GccFiles    []string        // list of gcc output files
+	Preamble    string          // collected preamble for _cgo_export.h
+	typedefs    map[string]bool // type names that appear in the types of the objects we're interested in
+	typedefList []string
 }
 
 // A File collects information about a single Go input file.