go/types, types2: remove need for coreString in signature.go

Also, add additional test cases for NewSignatureType
to check expected panic behavior.

Change-Id: If26cd81a2af384bf2084dd09119483c0584715c8
Reviewed-on: https://go-review.googlesource.com/c/go/+/655695
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Robert Griesemer <gri@google.com>
Auto-Submit: Robert Griesemer <gri@google.com>
Reviewed-by: Robert Findley <rfindley@google.com>

4 files changed, 118 insertions, 30 deletions

diff --git a/src/cmd/compile/internal/types2/issues_test.go b/src/cmd/compile/internal/types2/issues_test.go
index 51014d8d82..8ddb39987a 100644
--- a/src/cmd/compile/internal/types2/issues_test.go
+++ b/src/cmd/compile/internal/types2/issues_test.go
@@ -627,7 +627,15 @@ func TestIssue50646(t *testing.T) {
 
 func TestIssue55030(t *testing.T) {
 	// makeSig makes the signature func(typ...)
-	makeSig := func(typ Type) {
+	// If valid is not set, making that signature is expected to panic.
+	makeSig := func(typ Type, valid bool) {
+		if !valid {
+			defer func() {
+				if recover() == nil {
+					panic("NewSignatureType panic expected")
+				}
+			}()
+		}
 		par := NewParam(nopos, nil, "", typ)
 		params := NewTuple(par)
 		NewSignatureType(nil, nil, nil, params, nil, true)
@@ -635,30 +643,46 @@ func TestIssue55030(t *testing.T) {
 
 	// makeSig must not panic for the following (example) types:
 	// []int
-	makeSig(NewSlice(Typ[Int]))
+	makeSig(NewSlice(Typ[Int]), true)
 
 	// string
-	makeSig(Typ[String])
+	makeSig(Typ[String], true)
 
-	// P where P's core type is string
+	// P where P's common underlying type is string
 	{
 		P := NewTypeName(nopos, nil, "P", nil) // [P string]
-		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[String]})))
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[String]})), true)
 	}
 
-	// P where P's core type is an (unnamed) slice
+	// P where P's common underlying type is an (unnamed) slice
 	{
 		P := NewTypeName(nopos, nil, "P", nil) // [P []int]
-		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{NewSlice(Typ[Int])})))
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{NewSlice(Typ[Int])})), true)
 	}
 
-	// P where P's core type is bytestring (i.e., string or []byte)
+	// P where P's type set contains strings and []byte
 	{
 		t1 := NewTerm(true, Typ[String])          // ~string
 		t2 := NewTerm(false, NewSlice(Typ[Byte])) // []byte
 		u := NewUnion([]*Term{t1, t2})            // ~string | []byte
 		P := NewTypeName(nopos, nil, "P", nil)    // [P ~string | []byte]
-		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{u})))
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{u})), true)
+	}
+
+	// makeSig must panic for the following (example) types:
+	// int
+	makeSig(Typ[Int], false)
+
+	// P where P's type set doesn't have any specific types
+	{
+		P := NewTypeName(nopos, nil, "P", nil) // [P any]
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Universe.Lookup("any").Type()})), false)
+	}
+
+	// P where P's type set doesn't have any slice or string types
+	{
+		P := NewTypeName(nopos, nil, "P", nil) // [P any]
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[Int]})), false)
 	}
 }
 
diff --git a/src/cmd/compile/internal/types2/signature.go b/src/cmd/compile/internal/types2/signature.go
index 505997110b..eaecb77af5 100644
--- a/src/cmd/compile/internal/types2/signature.go
+++ b/src/cmd/compile/internal/types2/signature.go
@@ -32,9 +32,13 @@ type Signature struct {
 }
 
 // NewSignatureType creates a new function type for the given receiver,
-// receiver type parameters, type parameters, parameters, and results. If
-// variadic is set, params must hold at least one parameter and the last
-// parameter's core type must be of unnamed slice or bytestring type.
+// receiver type parameters, type parameters, parameters, and results.
+// If variadic is set, params must hold at least one parameter and the
+// last parameter must be an unnamed slice or a type parameter whose
+// type set has an unnamed slice as common underlying type.
+// As a special case, for variadic signatures the last parameter may
+// also be a string type, or a type parameter containing a mix of byte
+// slices and string types in its type set.
 // If recv is non-nil, typeParams must be empty. If recvTypeParams is
 // non-empty, recv must be non-nil.
 func NewSignatureType(recv *Var, recvTypeParams, typeParams []*TypeParam, params, results *Tuple, variadic bool) *Signature {
@@ -43,9 +47,25 @@ func NewSignatureType(recv *Var, recvTypeParams, typeParams []*TypeParam, params
 		if n == 0 {
 			panic("variadic function must have at least one parameter")
 		}
-		core := coreString(params.At(n - 1).typ)
-		if _, ok := core.(*Slice); !ok && !isString(core) {
-			panic(fmt.Sprintf("got %s, want variadic parameter with unnamed slice type or string as common underlying type", core.String()))
+		last := params.At(n - 1).typ
+		var S *Slice
+		typeset(last, func(t, _ Type) bool {
+			var s *Slice
+			if isString(t) {
+				s = NewSlice(universeByte)
+			} else {
+				s, _ = Unalias(t).(*Slice) // don't accept a named slice type
+			}
+			if S == nil {
+				S = s
+			} else if !Identical(S, s) {
+				S = nil
+				return false
+			}
+			return true
+		})
+		if S == nil {
+			panic(fmt.Sprintf("got %s, want variadic parameter of unnamed slice or string type", last))
 		}
 	}
 	sig := &Signature{recv: recv, params: params, results: results, variadic: variadic}
diff --git a/src/go/types/issues_test.go b/src/go/types/issues_test.go
index 2c1cfb8bfa..6388dcf687 100644
--- a/src/go/types/issues_test.go
+++ b/src/go/types/issues_test.go
@@ -637,7 +637,15 @@ func TestIssue50646(t *testing.T) {
 
 func TestIssue55030(t *testing.T) {
 	// makeSig makes the signature func(typ...)
-	makeSig := func(typ Type) {
+	// If valid is not set, making that signature is expected to panic.
+	makeSig := func(typ Type, valid bool) {
+		if !valid {
+			defer func() {
+				if recover() == nil {
+					panic("NewSignatureType panic expected")
+				}
+			}()
+		}
 		par := NewParam(nopos, nil, "", typ)
 		params := NewTuple(par)
 		NewSignatureType(nil, nil, nil, params, nil, true)
@@ -645,30 +653,46 @@ func TestIssue55030(t *testing.T) {
 
 	// makeSig must not panic for the following (example) types:
 	// []int
-	makeSig(NewSlice(Typ[Int]))
+	makeSig(NewSlice(Typ[Int]), true)
 
 	// string
-	makeSig(Typ[String])
+	makeSig(Typ[String], true)
 
-	// P where P's core type is string
+	// P where P's common underlying type is string
 	{
 		P := NewTypeName(nopos, nil, "P", nil) // [P string]
-		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[String]})))
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[String]})), true)
 	}
 
-	// P where P's core type is an (unnamed) slice
+	// P where P's common underlying type is an (unnamed) slice
 	{
 		P := NewTypeName(nopos, nil, "P", nil) // [P []int]
-		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{NewSlice(Typ[Int])})))
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{NewSlice(Typ[Int])})), true)
 	}
 
-	// P where P's core type is bytestring (i.e., string or []byte)
+	// P where P's type set contains strings and []byte
 	{
 		t1 := NewTerm(true, Typ[String])          // ~string
 		t2 := NewTerm(false, NewSlice(Typ[Byte])) // []byte
 		u := NewUnion([]*Term{t1, t2})            // ~string | []byte
 		P := NewTypeName(nopos, nil, "P", nil)    // [P ~string | []byte]
-		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{u})))
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{u})), true)
+	}
+
+	// makeSig must panic for the following (example) types:
+	// int
+	makeSig(Typ[Int], false)
+
+	// P where P's type set doesn't have any specific types
+	{
+		P := NewTypeName(nopos, nil, "P", nil) // [P any]
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Universe.Lookup("any").Type()})), false)
+	}
+
+	// P where P's type set doesn't have any slice or string types
+	{
+		P := NewTypeName(nopos, nil, "P", nil) // [P any]
+		makeSig(NewTypeParam(P, NewInterfaceType(nil, []Type{Typ[Int]})), false)
 	}
 }
 
diff --git a/src/go/types/signature.go b/src/go/types/signature.go
index 0bf28f8947..f059ecb183 100644
--- a/src/go/types/signature.go
+++ b/src/go/types/signature.go
@@ -45,9 +45,13 @@ func NewSignature(recv *Var, params, results *Tuple, variadic bool) *Signature {
 }
 
 // NewSignatureType creates a new function type for the given receiver,
-// receiver type parameters, type parameters, parameters, and results. If
-// variadic is set, params must hold at least one parameter and the last
-// parameter's core type must be of unnamed slice or bytestring type.
+// receiver type parameters, type parameters, parameters, and results.
+// If variadic is set, params must hold at least one parameter and the
+// last parameter must be an unnamed slice or a type parameter whose
+// type set has an unnamed slice as common underlying type.
+// As a special case, for variadic signatures the last parameter may
+// also be a string type, or a type parameter containing a mix of byte
+// slices and string types in its type set.
 // If recv is non-nil, typeParams must be empty. If recvTypeParams is
 // non-empty, recv must be non-nil.
 func NewSignatureType(recv *Var, recvTypeParams, typeParams []*TypeParam, params, results *Tuple, variadic bool) *Signature {
@@ -56,9 +60,25 @@ func NewSignatureType(recv *Var, recvTypeParams, typeParams []*TypeParam, params
 		if n == 0 {
 			panic("variadic function must have at least one parameter")
 		}
-		core := coreString(params.At(n - 1).typ)
-		if _, ok := core.(*Slice); !ok && !isString(core) {
-			panic(fmt.Sprintf("got %s, want variadic parameter with unnamed slice type or string as common underlying type", core.String()))
+		last := params.At(n - 1).typ
+		var S *Slice
+		typeset(last, func(t, _ Type) bool {
+			var s *Slice
+			if isString(t) {
+				s = NewSlice(universeByte)
+			} else {
+				s, _ = Unalias(t).(*Slice) // don't accept a named slice type
+			}
+			if S == nil {
+				S = s
+			} else if !Identical(S, s) {
+				S = nil
+				return false
+			}
+			return true
+		})
+		if S == nil {
+			panic(fmt.Sprintf("got %s, want variadic parameter of unnamed slice or string type", last))
 		}
 	}
 	sig := &Signature{recv: recv, params: params, results: results, variadic: variadic}