runtime: fix sub-uintptr-sized Windows callback arguments

The Windows callback support accepts Go functions with arguments that
are uintptr-sized or smaller. However, it doesn't implement smaller
arguments correctly. It assumes the Windows arguments layout is
equivalent to the Go argument layout. This is often true, but because
Windows C ABIs pad arguments to word size, while Go packs arguments,
the layout is different if there are multiple sub-word-size arguments
in a row. For example, a function with two uint16 arguments will have
a two-word C argument frame, but only a 4 byte Go argument frame.
There are also subtleties surrounding floating-point register
arguments that it doesn't handle correctly.

To fix this, when constructing a callback, we examine the Go
function's signature to construct a mapping between the C argument
frame and the Go argument frame. When the callback is invoked, we use
this mapping to build the Go argument frame and copy the result back.

This adds several test cases to TestStdcallAndCDeclCallbacks that
exercise more complex function signatures. These all fail with the
current code, but work with this CL.

In addition to fixing these callback types, this is also a step toward
the Go register ABI (#40724), which is going to make the ABI
translation more complex.

Change-Id: I19fb1681b659d9fd528ffd5e88912bebb95da052
Reviewed-on: https://go-review.googlesource.com/c/go/+/263271
Trust: Austin Clements <austin@google.com>
Trust: Alex Brainman <alex.brainman@gmail.com>
Run-TryBot: Austin Clements <austin@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>

1 files changed, 29 insertions, 3 deletions

diff --git a/src/runtime/syscall_windows_test.go b/src/runtime/syscall_windows_test.go
index cb942beb3e..7705d2a017 100644
--- a/src/runtime/syscall_windows_test.go
+++ b/src/runtime/syscall_windows_test.go
@@ -317,9 +317,13 @@ func (f cbFunc) cSrc(w io.Writer, cdecl bool) {
 	cArgs := make([]string, t.NumIn())
 	for i := range cTypes {
 		// We included stdint.h, so this works for all sized
-		// integer types.
+		// integer types, and uint8Pair_t.
 		cTypes[i] = t.In(i).Name() + "_t"
-		cArgs[i] = fmt.Sprintf("%d", i+1)
+		if t.In(i).Name() == "uint8Pair" {
+			cArgs[i] = fmt.Sprintf("(uint8Pair_t){%d,1}", i)
+		} else {
+			cArgs[i] = fmt.Sprintf("%d", i+1)
+		}
 	}
 	fmt.Fprintf(w, `
 typedef uintptr_t %s (*%s)(%s);
@@ -341,6 +345,8 @@ func (f cbFunc) testOne(t *testing.T, dll *syscall.DLL, cdecl bool, cb uintptr)
 	}
 }
 
+type uint8Pair struct{ x, y uint8 }
+
 var cbFuncs = []cbFunc{
 	{func(i1, i2 uintptr) uintptr {
 		return i1 + i2
@@ -366,6 +372,23 @@ var cbFuncs = []cbFunc{
 	{func(i1, i2, i3, i4, i5, i6, i7, i8, i9 uintptr) uintptr {
 		return i1 + i2 + i3 + i4 + i5 + i6 + i7 + i8 + i9
 	}},
+
+	// Non-uintptr parameters.
+	{func(i1, i2, i3, i4, i5, i6, i7, i8, i9 uint8) uintptr {
+		return uintptr(i1 + i2 + i3 + i4 + i5 + i6 + i7 + i8 + i9)
+	}},
+	{func(i1, i2, i3, i4, i5, i6, i7, i8, i9 uint16) uintptr {
+		return uintptr(i1 + i2 + i3 + i4 + i5 + i6 + i7 + i8 + i9)
+	}},
+	{func(i1, i2, i3, i4, i5, i6, i7, i8, i9 int8) uintptr {
+		return uintptr(i1 + i2 + i3 + i4 + i5 + i6 + i7 + i8 + i9)
+	}},
+	{func(i1 int8, i2 int16, i3 int32, i4, i5 uintptr) uintptr {
+		return uintptr(i1) + uintptr(i2) + uintptr(i3) + i4 + i5
+	}},
+	{func(i1, i2, i3, i4, i5 uint8Pair) uintptr {
+		return uintptr(i1.x + i1.y + i2.x + i2.y + i3.x + i3.y + i4.x + i4.y + i5.x + i5.y)
+	}},
 }
 
 type cbDLL struct {
@@ -380,7 +403,10 @@ func (d *cbDLL) makeSrc(t *testing.T, path string) {
 	}
 	defer f.Close()
 
-	fmt.Fprintf(f, "#include <stdint.h>\n\n")
+	fmt.Fprint(f, `
+#include <stdint.h>
+typedef struct { uint8_t x, y; } uint8Pair_t;
+`)
 	for _, cbf := range cbFuncs {
 		cbf.cSrc(f, false)
 		cbf.cSrc(f, true)