go/types, types2: replace setDefType with pending type check

Given a type definition of the form:

  type T RHS

The setDefType function would set T.fromRHS as soon as we knew its
top-level type. For instance, in:

  type S struct { ... }

S.fromRHS is set to a struct type before type-checking anything inside
the struct.

This permit access to the (incomplete) RHS type in a cyclic type
declaration. Accessing this information is fraught (as it's incomplete),
but was used for reporting certain types of cycles.

This CL replaces setDefType with a check that ensures no value of type
T is used before its RHS is set up.

This CL is strictly more complete than what setDefType achieved. For
instance, it enables correct reporting for the below cycles:

  type A [unsafe.Sizeof(A{})]int

  var v any = 42
  type B [v.(B)]int

  func f() C {
    return C{}
  }
  type C [unsafe.Sizeof(f())]int

Fixes #76383
Fixes #76384

Change-Id: I9dfab5b708013b418fa66e43362bb4d8483fedec
Reviewed-on: https://go-review.googlesource.com/c/go/+/724140
Auto-Submit: Mark Freeman <markfreeman@google.com>
Reviewed-by: Robert Griesemer <gri@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

2 files changed, 25 insertions, 14 deletions

diff --git a/src/cmd/compile/internal/types2/expr.go b/src/cmd/compile/internal/types2/expr.go
index 39bf4055a3..9d7580cb01 100644
--- a/src/cmd/compile/internal/types2/expr.go
+++ b/src/cmd/compile/internal/types2/expr.go
@@ -993,6 +993,13 @@ func (check *Checker) rawExpr(T *target, x *operand, e syntax.Expr, hint Type, a
 		check.nonGeneric(T, x)
 	}
 
+	// Here, x is a value, meaning it has a type. If that type is pending, then we have
+	// a cycle. As an example:
+	//
+	//  type T [unsafe.Sizeof(T{})]int
+	//
+	// has a cycle T->T which is deemed valid (by decl.go), but which is in fact invalid.
+	check.pendingType(x)
 	check.record(x)
 
 	return kind
@@ -1027,6 +1034,22 @@ func (check *Checker) nonGeneric(T *target, x *operand) {
 	}
 }
 
+// If x has a pending type (i.e. its declaring object is on the object path), pendingType
+// reports an error and invalidates x.mode and x.typ.
+// Otherwise it leaves x alone.
+func (check *Checker) pendingType(x *operand) {
+	if x.mode == invalid || x.mode == novalue {
+		return
+	}
+	if n, ok := Unalias(x.typ).(*Named); ok {
+		if i, ok := check.objPathIdx[n.obj]; ok {
+			check.cycleError(check.objPath, i)
+			x.mode = invalid
+			x.typ = Typ[Invalid]
+		}
+	}
+}
+
 // exprInternal contains the core of type checking of expressions.
 // Must only be called by rawExpr.
 // (See rawExpr for an explanation of the parameters.)
diff --git a/src/cmd/compile/internal/types2/typexpr.go b/src/cmd/compile/internal/types2/typexpr.go
index 303f782ac4..c3e40184f5 100644
--- a/src/cmd/compile/internal/types2/typexpr.go
+++ b/src/cmd/compile/internal/types2/typexpr.go
@@ -417,20 +417,8 @@ func (check *Checker) typInternal(e0 syntax.Expr, def *TypeName) (T Type) {
 	return typ
 }
 
-func setDefType(def *TypeName, typ Type) {
-	if def != nil {
-		switch t := def.typ.(type) {
-		case *Alias:
-			t.fromRHS = typ
-		case *Basic:
-			assert(t == Typ[Invalid])
-		case *Named:
-			t.fromRHS = typ
-		default:
-			panic(fmt.Sprintf("unexpected type %T", t))
-		}
-	}
-}
+// TODO(markfreeman): Remove this function.
+func setDefType(def *TypeName, typ Type) {}
 
 func (check *Checker) instantiatedType(x syntax.Expr, xlist []syntax.Expr, def *TypeName) (res Type) {
 	if check.conf.Trace {