cmd/compile: some fixes in type substituter for Instantiate

In the case in (*TSubster).Type() that we were running into an
incomplete underlying type (TFORW), we should just be immediately
returning the type returned by ts.SubstForwFunc(forw), since that call
returns a proper type node, and has set up any remaining work that has
to be done when we get done with the current top-level type definition.
(For import, that function is doInst, which does an Instantiate of the
new substituted type, with the delayed part via deferredInstStack.) We
should not continue doing the later parts of (*TSubster).Type(), since
the underlying type may not yet have its methods filled in, etc.

Also, in Instantiate(), we need to put the desired new type on
deferredInstStack, even if the base type node already exists, if the
type node is in TFORW state. This is now exactly the case when
Instantiate is called from (*TSubster).Type via doInst, since
(*TSubster).Type has already called NewIncompleteNamedType().

Fixes #48716
Fixes #48889

Change-Id: Icd6be5721c4ac75bf8869b8bbdeca50069d632ec
Reviewed-on: https://go-review.googlesource.com/c/go/+/355250
Trust: Dan Scales <danscales@google.com>
Run-TryBot: Dan Scales <danscales@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Keith Randall <khr@golang.org>

2 files changed, 22 insertions, 4 deletions

diff --git a/src/cmd/compile/internal/typecheck/iimport.go b/src/cmd/compile/internal/typecheck/iimport.go
index 08850079eb..df49d74a40 100644
--- a/src/cmd/compile/internal/typecheck/iimport.go
+++ b/src/cmd/compile/internal/typecheck/iimport.go
@@ -1822,9 +1822,25 @@ func Instantiate(pos src.XPos, baseType *types.Type, targs []*types.Type) *types
 	instSym := baseSym.Pkg.Lookup(name)
 	if instSym.Def != nil {
 		// May match existing type from previous import or
-		// types2-to-types1 conversion, or from in-progress instantiation
-		// in the current type import stack.
-		return instSym.Def.Type()
+		// types2-to-types1 conversion.
+		t := instSym.Def.Type()
+		if t.Kind() != types.TFORW {
+			return t
+		}
+		// Or, we have started creating this type in (*TSubster).Typ, but its
+		// underlying type was not completed yet, so we need to add this type
+		// to deferredInstStack, if not already there.
+		found := false
+		for _, t2 := range deferredInstStack {
+			if t2 == t {
+				found = true
+				break
+			}
+		}
+		if !found {
+			deferredInstStack = append(deferredInstStack, t)
+		}
+		return t
 	}
 
 	t := NewIncompleteNamedType(baseType.Pos(), instSym)
@@ -1865,6 +1881,7 @@ func resumeDoInst() {
 // during a type substitution for an instantiation. This is needed for
 // instantiations of mutually recursive types.
 func doInst(t *types.Type) *types.Type {
+	assert(t.Kind() == types.TFORW)
 	return Instantiate(t.Pos(), t.OrigSym().Def.(*ir.Name).Type(), t.RParams())
 }
 
@@ -1873,6 +1890,7 @@ func doInst(t *types.Type) *types.Type {
 // instantiation being created, baseType is the base generic type, and targs are
 // the type arguments that baseType is being instantiated with.
 func substInstType(t *types.Type, baseType *types.Type, targs []*types.Type) {
+	assert(t.Kind() == types.TFORW)
 	subst := Tsubster{
 		Tparams:       baseType.RParams(),
 		Targs:         targs,
diff --git a/src/cmd/compile/internal/typecheck/subr.go b/src/cmd/compile/internal/typecheck/subr.go
index 68240329f5..6288d15a01 100644
--- a/src/cmd/compile/internal/typecheck/subr.go
+++ b/src/cmd/compile/internal/typecheck/subr.go
@@ -1217,7 +1217,7 @@ func (ts *Tsubster) typ1(t *types.Type) *types.Type {
 		}
 	case types.TFORW:
 		if ts.SubstForwFunc != nil {
-			newt = ts.SubstForwFunc(t)
+			return ts.SubstForwFunc(forw)
 		} else {
 			assert(false)
 		}