runtime/trace: use regular unwinding for cgo callbacks

Introduce a new m.incgocallback field that is true while C code calls
into Go code. Use it in the tracer in order to fallback to the default
unwinder instead of frame pointer unwinding for this scenario. The
existing fields (incgo, ncgo) were not sufficient to detect the case
where a thread created in C calls into Go code.

Motivation:

1. Take advantage of a cgo symbolizer, if registered, to unwind through
   C stacks without frame pointers.
2. Reduce the chance of crashes. It seems unsafe to follow frame
   pointers when there could be C code that was compiled without frame
   pointers.

Removing the curgp.m.incgocallback check in traceStackID shows the
following minor differences between frame pointer unwinding and the
default unwinder when there is no cgo symbolizer involved.

    trace_test.go:60: "goCalledFromCThread": got stack:
        main.goCalledFromCThread
        	/src/runtime/testdata/testprogcgo/trace.go:58
        _cgoexp_45c15a3efb3a_goCalledFromCThread
        	_cgo_gotypes.go:694
        runtime.cgocallbackg1
        	/src/runtime/cgocall.go:318
        runtime.cgocallbackg
        	/src/runtime/cgocall.go:236
        runtime.cgocallback
        	/src/runtime/asm_amd64.s:998
        crosscall2
        	/src/runtime/cgo/asm_amd64.s:30

        want stack:
        main.goCalledFromCThread
        	/src/runtime/testdata/testprogcgo/trace.go:58
        _cgoexp_45c15a3efb3a_goCalledFromCThread
        	_cgo_gotypes.go:694
        runtime.cgocallbackg1
        	/src/runtime/cgocall.go:318
        runtime.cgocallbackg
        	/src/runtime/cgocall.go:236
        runtime.cgocallback
        	/src/runtime/asm_amd64.s:998

    trace_test.go:60: "goCalledFromC": got stack:
        main.goCalledFromC
        	/src/runtime/testdata/testprogcgo/trace.go:51
        _cgoexp_45c15a3efb3a_goCalledFromC
        	_cgo_gotypes.go:687
        runtime.cgocallbackg1
        	/src/runtime/cgocall.go:318
        runtime.cgocallbackg
        	/src/runtime/cgocall.go:236
        runtime.cgocallback
        	/src/runtime/asm_amd64.s:998
        crosscall2
        	/src/runtime/cgo/asm_amd64.s:30
        runtime.asmcgocall
        	/src/runtime/asm_amd64.s:848
        main._Cfunc_cCalledFromGo
        	_cgo_gotypes.go:263
        main.goCalledFromGo
        	/src/runtime/testdata/testprogcgo/trace.go:46
        main.Trace
        	/src/runtime/testdata/testprogcgo/trace.go:37
        main.main
        	/src/runtime/testdata/testprogcgo/main.go:34

        want stack:
        main.goCalledFromC
        	/src/runtime/testdata/testprogcgo/trace.go:51
        _cgoexp_45c15a3efb3a_goCalledFromC
        	_cgo_gotypes.go:687
        runtime.cgocallbackg1
        	/src/runtime/cgocall.go:318
        runtime.cgocallbackg
        	/src/runtime/cgocall.go:236
        runtime.cgocallback
        	/src/runtime/asm_amd64.s:998
        runtime.systemstack_switch
        	/src/runtime/asm_amd64.s:463
        runtime.cgocall
        	/src/runtime/cgocall.go:168
        main._Cfunc_cCalledFromGo
        	_cgo_gotypes.go:263
        main.goCalledFromGo
        	/src/runtime/testdata/testprogcgo/trace.go:46
        main.Trace
        	/src/runtime/testdata/testprogcgo/trace.go:37
        main.main
        	/src/runtime/testdata/testprogcgo/main.go:34

For #16638

Change-Id: I95fa27a3170c5abd923afc6eadab4eae777ced31
Reviewed-on: https://go-review.googlesource.com/c/go/+/474916
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Michael Pratt <mpratt@google.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Felix Geisendörfer <felix.geisendoerfer@datadoghq.com>

3 files changed, 116 insertions, 0 deletions

diff --git a/src/runtime/testdata/testprogcgo/trace.go b/src/runtime/testdata/testprogcgo/trace.go
new file mode 100644
index 0000000000..875434b1f1
--- /dev/null
+++ b/src/runtime/testdata/testprogcgo/trace.go
@@ -0,0 +1,60 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+/*
+// Defined in trace_*.c.
+void cCalledFromGo(void);
+*/
+import "C"
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"runtime/trace"
+)
+
+func init() {
+	register("Trace", Trace)
+}
+
+// Trace is used by TestTraceUnwindCGO.
+func Trace() {
+	file, err := os.CreateTemp("", "testprogcgo_trace")
+	if err != nil {
+		log.Fatalf("failed to create temp file: %s", err)
+	}
+	defer file.Close()
+
+	if err := trace.Start(file); err != nil {
+		log.Fatal(err)
+	}
+	defer trace.Stop()
+
+	goCalledFromGo()
+	<-goCalledFromCThreadChan
+
+	fmt.Printf("trace path:%s", file.Name())
+}
+
+// goCalledFromGo calls cCalledFromGo which calls back into goCalledFromC and
+// goCalledFromCThread.
+func goCalledFromGo() {
+	C.cCalledFromGo()
+}
+
+//export goCalledFromC
+func goCalledFromC() {
+	trace.Log(context.Background(), "goCalledFromC", "")
+}
+
+var goCalledFromCThreadChan = make(chan struct{})
+
+//export goCalledFromCThread
+func goCalledFromCThread() {
+	trace.Log(context.Background(), "goCalledFromCThread", "")
+	close(goCalledFromCThreadChan)
+}
diff --git a/src/runtime/testdata/testprogcgo/trace_unix.c b/src/runtime/testdata/testprogcgo/trace_unix.c
new file mode 100644
index 0000000000..0fa55c7215
--- /dev/null
+++ b/src/runtime/testdata/testprogcgo/trace_unix.c
@@ -0,0 +1,27 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build unix
+
+// The unix C definitions for trace.go. That file uses //export so
+// it can't put function definitions in the "C" import comment.
+
+#include <pthread.h>
+#include <assert.h>
+
+extern void goCalledFromC(void);
+extern void goCalledFromCThread(void);
+
+static void* cCalledFromCThread(void *p) {
+	goCalledFromCThread();
+	return NULL;
+}
+
+void cCalledFromGo(void) {
+	goCalledFromC();
+
+	pthread_t thread;
+	assert(pthread_create(&thread, NULL, cCalledFromCThread, NULL) == 0);
+	assert(pthread_join(thread, NULL) == 0);
+}
diff --git a/src/runtime/testdata/testprogcgo/trace_windows.c b/src/runtime/testdata/testprogcgo/trace_windows.c
new file mode 100644
index 0000000000..77580547ab
--- /dev/null
+++ b/src/runtime/testdata/testprogcgo/trace_windows.c
@@ -0,0 +1,29 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The windows C definitions for trace.go. That file uses //export so
+// it can't put function definitions in the "C" import comment.
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <process.h>
+#include "_cgo_export.h"
+
+extern void goCalledFromC(void);
+extern void goCalledFromCThread(void);
+
+__stdcall
+static unsigned int cCalledFromCThread(void *p) {
+	goCalledFromCThread();
+	return 0;
+}
+
+void cCalledFromGo(void) {
+	goCalledFromC();
+
+	uintptr_t thread;
+	thread = _beginthreadex(NULL, 0, cCalledFromCThread, NULL, 0, NULL);
+	WaitForSingleObject((HANDLE)thread, INFINITE);
+	CloseHandle((HANDLE)thread);
+}