runtime/metrics: add metric for total goroutines created

For #15490.

Change-Id: Ic587dda1f42d613ea131a6b53ce6ba6e6cadf4c7
Reviewed-on: https://go-review.googlesource.com/c/go/+/690398
Reviewed-by: Michael Pratt <mpratt@google.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

6 files changed, 49 insertions, 5 deletions

diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go
index 47b1b891e1..028de6b2e1 100644
--- a/src/runtime/metrics.go
+++ b/src/runtime/metrics.go
@@ -500,6 +500,13 @@ func initMetrics() {
 				out.scalar = uint64(in.schedStats.gWaiting)
 			},
 		},
+		"/sched/goroutines-created:goroutines": {
+			deps: makeStatDepSet(schedStatsDep),
+			compute: func(in *statAggregate, out *metricValue) {
+				out.kind = metricKindUint64
+				out.scalar = uint64(in.schedStats.gCreated)
+			},
+		},
 		"/sched/latencies:seconds": {
 			compute: func(_ *statAggregate, out *metricValue) {
 				sched.timeToRun.write(out)
@@ -779,6 +786,7 @@ type schedStatsAggregate struct {
 	gRunnable uint64
 	gNonGo    uint64
 	gWaiting  uint64
+	gCreated  uint64
 }
 
 // compute populates the schedStatsAggregate with values from the runtime.
@@ -790,10 +798,12 @@ func (a *schedStatsAggregate) compute() {
 	lock(&sched.lock)
 
 	// Collect running/runnable from per-P run queues.
+	a.gCreated += sched.goroutinesCreated.Load()
 	for _, p := range allp {
 		if p == nil || p.status == _Pdead {
 			break
 		}
+		a.gCreated += p.goroutinesCreated
 		switch p.status {
 		case _Prunning:
 			a.gRunning++
diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go
index cf22bb73ad..dd0b485cf8 100644
--- a/src/runtime/metrics/description.go
+++ b/src/runtime/metrics/description.go
@@ -438,6 +438,12 @@ var allDesc = []Description{
 		Kind:        KindUint64,
 	},
 	{
+		Name:        "/sched/goroutines-created:goroutines",
+		Description: "Count of goroutines created since program start.",
+		Cumulative:  true,
+		Kind:        KindUint64,
+	},
+	{
 		Name:        "/sched/goroutines/not-in-go:goroutines",
 		Description: "Approximate count of goroutines running or blocked in a system call or cgo call. Not guaranteed to add up to /sched/goroutines:goroutines with other goroutine metrics.",
 		Kind:        KindUint64,
diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go
index c379b201b4..2d3b716a3c 100644
--- a/src/runtime/metrics/doc.go
+++ b/src/runtime/metrics/doc.go
@@ -509,6 +509,9 @@ Below is the full list of supported metrics, ordered lexicographically.
 		operating system threads that can execute user-level Go code
 		simultaneously.
 
+	/sched/goroutines-created:goroutines
+		Count of goroutines created since program start.
+
 	/sched/goroutines/not-in-go:goroutines
 		Approximate count of goroutines running or blocked in
 		a system call or cgo call. Not guaranteed to add up to
diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go
index 5b16cbcb22..77223a37a7 100644
--- a/src/runtime/metrics_test.go
+++ b/src/runtime/metrics_test.go
@@ -1583,12 +1583,14 @@ func TestReadMetricsSched(t *testing.T) {
 		runnable
 		running
 		waiting
+		created
 	)
-	var s [4]metrics.Sample
-	s[0].Name = "/sched/goroutines/not-in-go:goroutines"
-	s[1].Name = "/sched/goroutines/runnable:goroutines"
-	s[2].Name = "/sched/goroutines/running:goroutines"
-	s[3].Name = "/sched/goroutines/waiting:goroutines"
+	var s [5]metrics.Sample
+	s[notInGo].Name = "/sched/goroutines/not-in-go:goroutines"
+	s[runnable].Name = "/sched/goroutines/runnable:goroutines"
+	s[running].Name = "/sched/goroutines/running:goroutines"
+	s[waiting].Name = "/sched/goroutines/waiting:goroutines"
+	s[created].Name = "/sched/goroutines-created:goroutines"
 
 	logMetrics := func(t *testing.T, s []metrics.Sample) {
 		for i := range s {
@@ -1645,6 +1647,9 @@ func TestReadMetricsSched(t *testing.T) {
 		check(t, &s[waiting], 0, waitingSlack)
 	})
 
+	metrics.Read(s[:])
+	createdAfterBase := s[created].Value.Uint64()
+
 	// Force Running count to be high. We'll use these goroutines
 	// for Runnable, too.
 	const count = 10
@@ -1673,6 +1678,11 @@ func TestReadMetricsSched(t *testing.T) {
 	// of runnable goroutines all spinning. We cannot write anything
 	// out.
 	if testenv.HasParallelism() {
+		t.Run("created", func(t *testing.T) {
+			metrics.Read(s[:])
+			logMetrics(t, s[:])
+			checkEq(t, &s[created], createdAfterBase+count)
+		})
 		t.Run("running", func(t *testing.T) {
 			defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(count + 4))
 			// It can take a little bit for the scheduler to
@@ -1706,6 +1716,11 @@ func TestReadMetricsSched(t *testing.T) {
 		exit.Store(1)
 
 		// Now we can check our invariants.
+		t.Run("created", func(t *testing.T) {
+			// Look for count-1 goroutines because we read metrics
+			// *before* t.Run goroutine was created for this sub-test.
+			checkEq(t, &s[created], createdAfterBase+count-1)
+		})
 		t.Run("running", func(t *testing.T) {
 			logMetrics(t, s[:])
 			checkEq(t, &s[running], 1)
diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 15f10f087e..329d1b2624 100644
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -5262,6 +5262,7 @@ func newproc1(fn *funcval, callergp *g, callerpc uintptr, parked bool, waitreaso
 			racereleasemergeg(newg, unsafe.Pointer(&labelSync))
 		}
 	}
+	pp.goroutinesCreated++
 	releasem(mp)
 
 	return newg
@@ -5841,6 +5842,8 @@ func (pp *p) destroy() {
 	pp.gcAssistTime = 0
 	gcCleanups.queued += pp.cleanupsQueued
 	pp.cleanupsQueued = 0
+	sched.goroutinesCreated.Add(int64(pp.goroutinesCreated))
+	pp.goroutinesCreated = 0
 	pp.xRegs.free()
 	pp.status = _Pdead
 }
diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go
index c5d15754ec..a80a34a18e 100644
--- a/src/runtime/runtime2.go
+++ b/src/runtime/runtime2.go
@@ -764,6 +764,9 @@ type p struct {
 	// gcStopTime is the nanotime timestamp that this P last entered _Pgcstop.
 	gcStopTime int64
 
+	// goroutinesCreated is the total count of goroutines created by this P.
+	goroutinesCreated uint64
+
 	// xRegs is the per-P extended register state used by asynchronous
 	// preemption. This is an empty struct on platforms that don't use extended
 	// register state.
@@ -892,6 +895,10 @@ type schedt struct {
 	// M, but waiting for locks within the runtime. This field stores the value
 	// for Ms that have exited.
 	totalRuntimeLockWaitTime atomic.Int64
+
+	// goroutinesCreated (plus the value of goroutinesCreated on each P in allp)
+	// is the sum of all goroutines created by the program.
+	goroutinesCreated atomic.Uint64
 }
 
 // Values for the flags field of a sigTabT.