From 2159c26ceb32bbfa86036431750c0752fca84ef6 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Tue, 14 Apr 2020 21:06:26 +0000 Subject: runtime/metrics: add package interface This change creates the runtime/metrics package and adds the initial interface as laid out in the design document. For #37112. Change-Id: I202dcee08ab008dd63bf96f7a4162f5b5f813637 Reviewed-on: https://go-review.googlesource.com/c/go/+/247040 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics/description.go | 52 ++++++++++++++++++++++++++++ src/runtime/metrics/doc.go | 49 +++++++++++++++++++++++++++ src/runtime/metrics/histogram.go | 30 +++++++++++++++++ src/runtime/metrics/sample.go | 29 ++++++++++++++++ src/runtime/metrics/value.go | 69 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 229 insertions(+) create mode 100644 src/runtime/metrics/description.go create mode 100644 src/runtime/metrics/doc.go create mode 100644 src/runtime/metrics/histogram.go create mode 100644 src/runtime/metrics/sample.go create mode 100644 src/runtime/metrics/value.go (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go new file mode 100644 index 0000000000..32bb950a72 --- /dev/null +++ b/src/runtime/metrics/description.go @@ -0,0 +1,52 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +// Description describes a runtime metric. +type Description struct { + // Name is the full name of the metric which includes the unit. + // + // The format of the metric may be described by the following regular expression. + // + // ^(?P/[^:]+):(?P[^:*\/]+(?:[*\/][^:*\/]+)*)$ + // + // The format splits the name into two components, separated by a colon: a path which always + // starts with a /, and a machine-parseable unit. The name may contain any valid Unicode + // codepoint in between / characters, but by convention will try to stick to lowercase + // characters and hyphens. An example of such a path might be "/memory/heap/free". + // + // The unit is by convention a series of lowercase English unit names (singular or plural) + // without prefixes delimited by '*' or '/'. The unit names may contain any valid Unicode + // codepoint that is not a delimiter. + // Examples of units might be "seconds", "bytes", "bytes/second", "cpu-seconds", + // "byte*cpu-seconds", and "bytes/second/second". + // + // A complete name might look like "/memory/heap/free:bytes". + Name string + + // Kind is the kind of value for this metric. + // + // The purpose of this field is to allow users to filter out metrics whose values are + // types which their application may not understand. + Kind ValueKind + + // Cumulative is whether or not the metric is cumulative. If a cumulative metric is just + // a single number, then it increases monotonically. If the metric is a distribution, + // then each bucket count increases monotonically. + // + // This flag thus indicates whether or not it's useful to compute a rate from this value. + Cumulative bool + + // StopTheWorld is whether or not the metric requires a stop-the-world + // event in order to collect it. + StopTheWorld bool +} + +var allDesc = []Description{} + +// All returns a slice of containing metric descriptions for all supported metrics. +func All() []Description { + return allDesc +} diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go new file mode 100644 index 0000000000..b48c22ba30 --- /dev/null +++ b/src/runtime/metrics/doc.go @@ -0,0 +1,49 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +/* +Package metrics provides a stable interface to access implementation-defined +metrics exported by the Go runtime. This package is similar to existing functions +like runtime.ReadMemStats and debug.ReadGCStats, but significantly more general. + +The set of metrics defined by this package may evolve as the runtime itself +evolves, and also enables variation across Go implementations, whose relevant +metric sets may not intersect. + +Interface + +Metrics are designated by a string key, rather than, for example, a field name in +a struct. The full list of supported metrics is always available in the slice of +Descriptions returned by All. Each Description also includes useful information +about the metric, such as how to display it (e.g. gauge vs. counter) and how difficult +or disruptive it is to obtain it (e.g. do you need to stop the world?). + +Thus, users of this API are encouraged to sample supported metrics defined by the +slice returned by All to remain compatible across Go versions. Of course, situations +arise where reading specific metrics is critical. For these cases, users are +encouranged to use build tags, and although metrics may be deprecated and removed, +users should consider this to be an exceptional and rare event, coinciding with a +very large change in a particular Go implementation. + +Each metric key also has a "kind" that describes the format of the metric's value. +In the interest of not breaking users of this package, the "kind" for a given metric +is guaranteed not to change. If it must change, then a new metric will be introduced +with a new key and a new "kind." + +Metric key format + +As mentioned earlier, metric keys are strings. Their format is simple and well-defined, +designed to be both human and machine readable. It is split into two components, +separated by a colon: a rooted path and a unit. The choice to include the unit in +the key is motivated by compatibility: if a metric's unit changes, its semantics likely +did also, and a new key should be introduced. + +For more details on the precise definition of the metric key's path and unit formats, see +the documentation of the Name field of the Description struct. + +Supported metrics + +TODO(mknyszek): List them here as they're added. +*/ +package metrics diff --git a/src/runtime/metrics/histogram.go b/src/runtime/metrics/histogram.go new file mode 100644 index 0000000000..e1364e1e26 --- /dev/null +++ b/src/runtime/metrics/histogram.go @@ -0,0 +1,30 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +// Float64Histogram represents a distribution of float64 values. +type Float64Histogram struct { + // Counts contains the weights for each histogram bucket. The length of + // Counts is equal to the length of Buckets (in the metric description) + // plus one to account for the implicit minimum bucket. + // + // Given N buckets, the following is the mathematical relationship between + // Counts and Buckets. + // count[0] is the weight of the range (-inf, bucket[0]) + // count[n] is the weight of the range [bucket[n], bucket[n+1]), for 0 < n < N-1 + // count[N-1] is the weight of the range [bucket[N-1], inf) + Counts []uint64 + + // Buckets contains the boundaries between histogram buckets, in increasing order. + // + // Because this slice contains boundaries, there are len(Buckets)+1 counts: + // a count for all values less than the first boundary, a count covering each + // [slice[i], slice[i+1]) interval, and a count for all values greater than or + // equal to the last boundary. + // + // For a given metric name, the value of Buckets is guaranteed not to change + // between calls until program exit. + Buckets []float64 +} diff --git a/src/runtime/metrics/sample.go b/src/runtime/metrics/sample.go new file mode 100644 index 0000000000..c7a3fc424a --- /dev/null +++ b/src/runtime/metrics/sample.go @@ -0,0 +1,29 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +// Sample captures a single metric sample. +type Sample struct { + // Name is the name of the metric sampled. + // + // It must correspond to a name in one of the metric descriptions + // returned by Descriptions. + Name string + + // Value is the value of the metric sample. + Value Value +} + +// Read populates each Value field in the given slice of metric samples. +// +// Desired metrics should be present in the slice with the appropriate name. +// The user of this API is encouraged to re-use the same slice between calls. +// +// Metric values with names not appearing in the value returned by Descriptions +// will have the value populated as KindBad to indicate that the name is +// unknown. +func Read(m []Sample) { + panic("unimplemented") +} diff --git a/src/runtime/metrics/value.go b/src/runtime/metrics/value.go new file mode 100644 index 0000000000..0b056b4ea8 --- /dev/null +++ b/src/runtime/metrics/value.go @@ -0,0 +1,69 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics + +import ( + "math" + "unsafe" +) + +// ValueKind is a tag for a metric Value which indicates its type. +type ValueKind int + +const ( + // KindBad indicates that the Value has no type and should not be used. + KindBad ValueKind = iota + + // KindUint64 indicates that the type of the Value is a uint64. + KindUint64 + + // KindFloat64 indicates that the type of the Value is a float64. + KindFloat64 + + // KindFloat64Histogram indicates that the type of the Value is a *Float64Histogram. + KindFloat64Histogram +) + +// Value represents a metric value returned by the runtime. +type Value struct { + kind ValueKind + scalar uint64 // contains scalar values for scalar Kinds. + pointer unsafe.Pointer // contains non-scalar values. +} + +// Kind returns the a tag representing the kind of value this is. +func (v Value) Kind() ValueKind { + return v.kind +} + +// Uint64 returns the internal uint64 value for the metric. +// +// If v.Kind() != KindUint64, this method panics. +func (v Value) Uint64() uint64 { + if v.kind != KindUint64 { + panic("called Uint64 on non-uint64 metric value") + } + return v.scalar +} + +// Float64 returns the internal float64 value for the metric. +// +// If v.Kind() != KindFloat64, this method panics. +func (v Value) Float64() float64 { + if v.kind != KindFloat64 { + panic("called Float64 on non-float64 metric value") + } + return math.Float64frombits(v.scalar) +} + +// Float64Histogram returns the internal *Float64Histogram value for the metric. +// +// If v.Kind() != KindFloat64Histogram, this method panics. +func (v Value) Float64Histogram() *Float64Histogram { + if v.kind != KindFloat64Histogram { + panic("called Float64 on non-float64 metric value") + } + return (*Float64Histogram)(v.pointer) +} -- cgit v1.3 From b08dfbaa439e4e396b979e02ea2e7d36972e8b7a Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 1 Jul 2020 16:02:42 +0000 Subject: runtime,runtime/metrics: add memory metrics This change adds support for a variety of runtime memory metrics and contains the base implementation of Read for the runtime/metrics package, which lives in the runtime. It also adds testing infrastructure for the metrics package, and a bunch of format and documentation tests. For #37112. Change-Id: I16a2c4781eeeb2de0abcb045c15105f1210e2d8a Reviewed-on: https://go-review.googlesource.com/c/go/+/247041 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Reviewed-by: Michael Pratt Trust: Michael Knyszek --- src/cmd/go/internal/work/gc.go | 6 +- src/runtime/export_test.go | 26 +++ src/runtime/metrics.go | 367 ++++++++++++++++++++++++++++++++ src/runtime/metrics/description.go | 80 ++++++- src/runtime/metrics/description_test.go | 125 +++++++++++ src/runtime/metrics/doc.go | 56 ++++- src/runtime/metrics/sample.go | 10 +- src/runtime/metrics_test.go | 114 ++++++++++ src/runtime/mstats.go | 3 +- 9 files changed, 781 insertions(+), 6 deletions(-) create mode 100644 src/runtime/metrics.go create mode 100644 src/runtime/metrics/description_test.go create mode 100644 src/runtime/metrics_test.go (limited to 'src/runtime/metrics') diff --git a/src/cmd/go/internal/work/gc.go b/src/cmd/go/internal/work/gc.go index e93031431c..0c4a7fa6e3 100644 --- a/src/cmd/go/internal/work/gc.go +++ b/src/cmd/go/internal/work/gc.go @@ -89,7 +89,11 @@ func (gcToolchain) gc(b *Builder, a *Action, archive string, importcfg []byte, s extFiles := len(p.CgoFiles) + len(p.CFiles) + len(p.CXXFiles) + len(p.MFiles) + len(p.FFiles) + len(p.SFiles) + len(p.SysoFiles) + len(p.SwigFiles) + len(p.SwigCXXFiles) if p.Standard { switch p.ImportPath { - case "bytes", "internal/poll", "net", "os", "runtime/pprof", "runtime/trace", "sync", "syscall", "time": + case "bytes", "internal/poll", "net", "os": + fallthrough + case "runtime/metrics", "runtime/pprof", "runtime/trace": + fallthrough + case "sync", "syscall", "time": extFiles++ } } diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index ff901fd7be..d043fe3ee5 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -298,6 +298,32 @@ func (p *ProfBuf) Close() { (*profBuf)(p).close() } +func ReadMetricsSlow(memStats *MemStats, samplesp unsafe.Pointer, len, cap int) { + stopTheWorld("ReadMetricsSlow") + + // Initialize the metrics beforehand because this could + // allocate and skew the stats. + semacquire(&metricsSema) + initMetrics() + semrelease(&metricsSema) + + systemstack(func() { + // Read memstats first. It's going to flush + // the mcaches which readMetrics does not do, so + // going the other way around may result in + // inconsistent statistics. + readmemstats_m(memStats) + }) + + // Read metrics off the system stack. + // + // The only part of readMetrics that could allocate + // and skew the stats is initMetrics. + readMetrics(samplesp, len, cap) + + startTheWorld() +} + // ReadMemStatsSlow returns both the runtime-computed MemStats and // MemStats accumulated by scanning the heap. func ReadMemStatsSlow() (base, slow MemStats) { diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go new file mode 100644 index 0000000000..44b5a29751 --- /dev/null +++ b/src/runtime/metrics.go @@ -0,0 +1,367 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +// Metrics implementation exported to runtime/metrics. + +import ( + "unsafe" +) + +var ( + // metrics is a map of runtime/metrics keys to + // data used by the runtime to sample each metric's + // value. + metricsSema uint32 = 1 + metricsInit bool + metrics map[string]metricData +) + +type metricData struct { + // deps is the set of runtime statistics that this metric + // depends on. Before compute is called, the statAggregate + // which will be passed must ensure() these dependencies. + deps statDepSet + + // compute is a function that populates a metricValue + // given a populated statAggregate structure. + compute func(in *statAggregate, out *metricValue) +} + +// initMetrics initializes the metrics map if it hasn't been yet. +// +// metricsSema must be held. +func initMetrics() { + if metricsInit { + return + } + metrics = map[string]metricData{ + "/memory/classes/heap/free:bytes": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(in.heapStats.committed - in.heapStats.inHeap - + in.heapStats.inStacks - in.heapStats.inWorkBufs - + in.heapStats.inPtrScalarBits) + }, + }, + "/memory/classes/heap/objects:bytes": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.heapStats.inObjects + }, + }, + "/memory/classes/heap/released:bytes": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(in.heapStats.released) + }, + }, + "/memory/classes/heap/stacks:bytes": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(in.heapStats.inStacks) + }, + }, + "/memory/classes/heap/unused:bytes": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(in.heapStats.inHeap) - in.heapStats.inObjects + }, + }, + "/memory/classes/metadata/mcache/free:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.mCacheSys - in.sysStats.mCacheInUse + }, + }, + "/memory/classes/metadata/mcache/inuse:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.mCacheInUse + }, + }, + "/memory/classes/metadata/mspan/free:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.mSpanSys - in.sysStats.mSpanInUse + }, + }, + "/memory/classes/metadata/mspan/inuse:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.mSpanInUse + }, + }, + "/memory/classes/metadata/other:bytes": { + deps: makeStatDepSet(heapStatsDep, sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(in.heapStats.inWorkBufs+in.heapStats.inPtrScalarBits) + in.sysStats.gcMiscSys + }, + }, + "/memory/classes/os-stacks:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.stacksSys + }, + }, + "/memory/classes/other:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.otherSys + }, + }, + "/memory/classes/profiling/buckets:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.buckHashSys + }, + }, + "/memory/classes/total:bytes": { + deps: makeStatDepSet(heapStatsDep, sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(in.heapStats.committed+in.heapStats.released) + + in.sysStats.stacksSys + in.sysStats.mSpanSys + + in.sysStats.mCacheSys + in.sysStats.buckHashSys + + in.sysStats.gcMiscSys + in.sysStats.otherSys + }, + }, + } + metricsInit = true +} + +// statDep is a dependency on a group of statistics +// that a metric might have. +type statDep uint + +const ( + heapStatsDep statDep = iota // corresponds to heapStatsAggregate + sysStatsDep // corresponds to sysStatsAggregate + numStatsDeps +) + +// statDepSet represents a set of statDeps. +// +// Under the hood, it's a bitmap. +type statDepSet [1]uint64 + +// makeStatDepSet creates a new statDepSet from a list of statDeps. +func makeStatDepSet(deps ...statDep) statDepSet { + var s statDepSet + for _, d := range deps { + s[d/64] |= 1 << (d % 64) + } + return s +} + +// differennce returns set difference of s from b as a new set. +func (s statDepSet) difference(b statDepSet) statDepSet { + var c statDepSet + for i := range s { + c[i] = s[i] &^ b[i] + } + return c +} + +// union returns the union of the two sets as a new set. +func (s statDepSet) union(b statDepSet) statDepSet { + var c statDepSet + for i := range s { + c[i] = s[i] | b[i] + } + return c +} + +// empty returns true if there are no dependencies in the set. +func (s *statDepSet) empty() bool { + for _, c := range s { + if c != 0 { + return false + } + } + return true +} + +// has returns true if the set contains a given statDep. +func (s *statDepSet) has(d statDep) bool { + return s[d/64]&(1<<(d%64)) != 0 +} + +// heapStatsAggregate represents memory stats obtained from the +// runtime. This set of stats is grouped together because they +// depend on each other in some way to make sense of the runtime's +// current heap memory use. They're also sharded across Ps, so it +// makes sense to grab them all at once. +type heapStatsAggregate struct { + heapStatsDelta + + // inObjects is the bytes of memory occupied by objects, + // derived from other values in heapStats. + inObjects uint64 +} + +// compute populates the heapStatsAggregate with values from the runtime. +func (a *heapStatsAggregate) compute() { + memstats.heapStats.read(&a.heapStatsDelta) + + // Calculate derived stats. + a.inObjects = uint64(a.largeAlloc - a.largeFree) + for i := range a.smallAllocCount { + a.inObjects += uint64(a.smallAllocCount[i]-a.smallFreeCount[i]) * uint64(class_to_size[i]) + } +} + +// sysStatsAggregate represents system memory stats obtained +// from the runtime. This set of stats is grouped together because +// they're all relatively cheap to acquire and generally independent +// of one another and other runtime memory stats. The fact that they +// may be acquired at different times, especially with respect to +// heapStatsAggregate, means there could be some skew, but because of +// these stats are independent, there's no real consistency issue here. +type sysStatsAggregate struct { + stacksSys uint64 + mSpanSys uint64 + mSpanInUse uint64 + mCacheSys uint64 + mCacheInUse uint64 + buckHashSys uint64 + gcMiscSys uint64 + otherSys uint64 +} + +// compute populates the sysStatsAggregate with values from the runtime. +func (a *sysStatsAggregate) compute() { + a.stacksSys = memstats.stacks_sys.load() + a.buckHashSys = memstats.buckhash_sys.load() + a.gcMiscSys = memstats.gcMiscSys.load() + a.otherSys = memstats.other_sys.load() + + systemstack(func() { + lock(&mheap_.lock) + a.mSpanSys = memstats.mspan_sys.load() + a.mSpanInUse = uint64(mheap_.spanalloc.inuse) + a.mCacheSys = memstats.mcache_sys.load() + a.mCacheInUse = uint64(mheap_.cachealloc.inuse) + unlock(&mheap_.lock) + }) +} + +// statAggregate is the main driver of the metrics implementation. +// +// It contains multiple aggregates of runtime statistics, as well +// as a set of these aggregates that it has populated. The aggergates +// are populated lazily by its ensure method. +type statAggregate struct { + ensured statDepSet + heapStats heapStatsAggregate + sysStats sysStatsAggregate +} + +// ensure populates statistics aggregates determined by deps if they +// haven't yet been populated. +func (a *statAggregate) ensure(deps *statDepSet) { + missing := deps.difference(a.ensured) + if missing.empty() { + return + } + for i := statDep(0); i < numStatsDeps; i++ { + if !missing.has(i) { + continue + } + switch i { + case heapStatsDep: + a.heapStats.compute() + case sysStatsDep: + a.sysStats.compute() + } + } + a.ensured = a.ensured.union(missing) +} + +// metricValidKind is a runtime copy of runtime/metrics.ValueKind and +// must be kept structurally identical to that type. +type metricKind int + +const ( + // These values must be kept identical to their corresponding Kind* values + // in the runtime/metrics package. + metricKindBad metricKind = iota + metricKindUint64 + metricKindFloat64 + metricKindFloat64Histogram +) + +// metricSample is a runtime copy of runtime/metrics.Sample and +// must be kept structurally identical to that type. +type metricSample struct { + name string + value metricValue +} + +// metricValue is a runtime copy of runtime/metrics.Sample and +// must be kept structurally identical to that type. +type metricValue struct { + kind metricKind + scalar uint64 // contains scalar values for scalar Kinds. + pointer unsafe.Pointer // contains non-scalar values. +} + +// agg is used by readMetrics, and is protected by metricsSema. +// +// Managed as a global variable because its pointer will be +// an argument to a dynamically-defined function, and we'd +// like to avoid it escaping to the heap. +var agg statAggregate + +// readMetrics is the implementation of runtime/metrics.Read. +// +//go:linkname readMetrics runtime/metrics.runtime_readMetrics +func readMetrics(samplesp unsafe.Pointer, len int, cap int) { + // Construct a slice from the args. + sl := slice{samplesp, len, cap} + samples := *(*[]metricSample)(unsafe.Pointer(&sl)) + + // Acquire the metricsSema but with handoff. This operation + // is expensive enough that queueing up goroutines and handing + // off between them will be noticably better-behaved. + semacquire1(&metricsSema, true, 0, 0) + + // Ensure the map is initialized. + initMetrics() + + // Clear agg defensively. + agg = statAggregate{} + + // Sample. + for i := range samples { + sample := &samples[i] + data, ok := metrics[sample.name] + if !ok { + sample.value.kind = metricKindBad + continue + } + // Ensure we have all the stats we need. + // agg is populated lazily. + agg.ensure(&data.deps) + + // Compute the value based on the stats we have. + data.compute(&agg, &sample.value) + } + + semrelease(&metricsSema) +} diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 32bb950a72..2e7df7e09f 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -10,7 +10,7 @@ type Description struct { // // The format of the metric may be described by the following regular expression. // - // ^(?P/[^:]+):(?P[^:*\/]+(?:[*\/][^:*\/]+)*)$ + // ^(?P/[^:]+):(?P[^:*/]+(?:[*/][^:*/]+)*)$ // // The format splits the name into two components, separated by a colon: a path which always // starts with a /, and a machine-parseable unit. The name may contain any valid Unicode @@ -26,6 +26,9 @@ type Description struct { // A complete name might look like "/memory/heap/free:bytes". Name string + // Description is an English language sentence describing the metric. + Description string + // Kind is the kind of value for this metric. // // The purpose of this field is to allow users to filter out metrics whose values are @@ -44,7 +47,80 @@ type Description struct { StopTheWorld bool } -var allDesc = []Description{} +// The English language descriptions below must be kept in sync with the +// descriptions of each metric in doc.go. +var allDesc = []Description{ + { + Name: "/memory/classes/heap/free:bytes", + Description: "Memory that is available for allocation, and may be returned to the underlying system.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/objects:bytes", + Description: "Memory occupied by live objects and dead objects that have not yet been collected.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/released:bytes", + Description: "Memory that has been returned to the underlying system.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/stacks:bytes", + Description: "Memory allocated from the heap that is occupied by stacks.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/heap/unused:bytes", + Description: "Memory that is unavailable for allocation, but cannot be returned to the underlying system.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mcache/free:bytes", + Description: "Memory that is reserved for runtime mcache structures, but not in-use.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mcache/inuse:bytes", + Description: "Memory that is occupied by runtime mcache structures that are currently being used.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mspan/free:bytes", + Description: "Memory that is reserved for runtime mspan structures, but not in-use.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/mspan/inuse:bytes", + Description: "Memory that is occupied by runtime mspan structures that are currently being used.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/metadata/other:bytes", + Description: "Memory that is reserved for or used to hold runtime metadata.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/os-stacks:bytes", + Description: "Stack memory allocated by the underlying operating system.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/other:bytes", + Description: "Memory used by execution trace buffers, structures for debugging the runtime, finalizer and profiler specials, and more.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/profiling/buckets:bytes", + Description: "Memory that is used by the stack trace hash map used for profiling.", + Kind: KindUint64, + }, + { + Name: "/memory/classes/total:bytes", + Description: "All memory mapped by the Go runtime into the current process as read-write. Note that this does not include memory mapped by code called via cgo or via the syscall package. Sum of all metrics in /memory/classes.", + Kind: KindUint64, + }, +} // All returns a slice of containing metric descriptions for all supported metrics. func All() []Description { diff --git a/src/runtime/metrics/description_test.go b/src/runtime/metrics/description_test.go new file mode 100644 index 0000000000..e966a281a1 --- /dev/null +++ b/src/runtime/metrics/description_test.go @@ -0,0 +1,125 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package metrics_test + +import ( + "bufio" + "os" + "path/filepath" + "regexp" + "runtime" + "runtime/metrics" + "strings" + "testing" +) + +func TestDescriptionNameFormat(t *testing.T) { + r := regexp.MustCompile("^(?P/[^:]+):(?P[^:*/]+(?:[*/][^:*/]+)*)$") + descriptions := metrics.All() + for _, desc := range descriptions { + if !r.MatchString(desc.Name) { + t.Errorf("metrics %q does not match regexp %s", desc.Name, r) + } + } +} + +func extractMetricDocs(t *testing.T) map[string]string { + if runtime.GOOS == "android" { + t.Skip("no access to Go source on android") + } + + // Get doc.go. + _, filename, _, _ := runtime.Caller(0) + filename = filepath.Join(filepath.Dir(filename), "doc.go") + + f, err := os.Open(filename) + if err != nil { + t.Fatal(err) + } + const ( + stateSearch = iota // look for list of metrics + stateNextMetric // look for next metric + stateNextDescription // build description + ) + state := stateSearch + s := bufio.NewScanner(f) + result := make(map[string]string) + var metric string + var prevMetric string + var desc strings.Builder + for s.Scan() { + line := strings.TrimSpace(s.Text()) + switch state { + case stateSearch: + if line == "Supported metrics" { + state = stateNextMetric + } + case stateNextMetric: + // Ignore empty lines until we find a non-empty + // one. This will be our metric name. + if len(line) != 0 { + prevMetric = metric + metric = line + if prevMetric > metric { + t.Errorf("metrics %s and %s are out of lexicographical order", prevMetric, metric) + } + state = stateNextDescription + } + case stateNextDescription: + if len(line) == 0 || line == `*/` { + // An empty line means we're done. + // Write down the description and look + // for a new metric. + result[metric] = desc.String() + desc.Reset() + state = stateNextMetric + } else { + // As long as we're seeing data, assume that's + // part of the description and append it. + if desc.Len() != 0 { + // Turn previous newlines into spaces. + desc.WriteString(" ") + } + desc.WriteString(line) + } + } + if line == `*/` { + break + } + } + if state == stateSearch { + t.Fatalf("failed to find supported metrics docs in %s", filename) + } + return result +} + +func TestDescriptionDocs(t *testing.T) { + docs := extractMetricDocs(t) + descriptions := metrics.All() + for _, d := range descriptions { + want := d.Description + got, ok := docs[d.Name] + if !ok { + t.Errorf("no docs found for metric %s", d.Name) + continue + } + if got != want { + t.Errorf("mismatched description and docs for metric %s", d.Name) + t.Errorf("want: %q, got %q", want, got) + continue + } + } + if len(docs) > len(descriptions) { + docsLoop: + for name, _ := range docs { + for _, d := range descriptions { + if name == d.Name { + continue docsLoop + } + } + t.Errorf("stale documentation for non-existent metric: %s", name) + } + } +} diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index b48c22ba30..fb4e23a2b5 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -44,6 +44,60 @@ the documentation of the Name field of the Description struct. Supported metrics -TODO(mknyszek): List them here as they're added. + /memory/classes/heap/free:bytes + Memory that is available for allocation, and may be returned + to the underlying system. + + /memory/classes/heap/objects:bytes + Memory occupied by live objects and dead objects that have + not yet been collected. + + /memory/classes/heap/released:bytes + Memory that has been returned to the underlying system. + + /memory/classes/heap/stacks:bytes + Memory allocated from the heap that is occupied by stacks. + + /memory/classes/heap/unused:bytes + Memory that is unavailable for allocation, but cannot be + returned to the underlying system. + + /memory/classes/metadata/mcache/free:bytes + Memory that is reserved for runtime mcache structures, but + not in-use. + + /memory/classes/metadata/mcache/inuse:bytes + Memory that is occupied by runtime mcache structures that + are currently being used. + + /memory/classes/metadata/mspan/free:bytes + Memory that is reserved for runtime mspan structures, but + not in-use. + + /memory/classes/metadata/mspan/inuse:bytes + Memory that is occupied by runtime mspan structures that are + currently being used. + + /memory/classes/metadata/other:bytes + Memory that is reserved for or used to hold runtime + metadata. + + /memory/classes/os-stacks:bytes + Stack memory allocated by the underlying operating system. + + /memory/classes/other:bytes + Memory used by execution trace buffers, structures for + debugging the runtime, finalizer and profiler specials, and + more. + + /memory/classes/profiling/buckets:bytes + Memory that is used by the stack trace hash map used for + profiling. + + /memory/classes/total:bytes + All memory mapped by the Go runtime into the current process + as read-write. Note that this does not include memory mapped + by code called via cgo or via the syscall package. + Sum of all metrics in /memory/classes. */ package metrics diff --git a/src/runtime/metrics/sample.go b/src/runtime/metrics/sample.go index c7a3fc424a..b4b0979aa6 100644 --- a/src/runtime/metrics/sample.go +++ b/src/runtime/metrics/sample.go @@ -4,6 +4,11 @@ package metrics +import ( + _ "runtime" // depends on the runtime via a linkname'd function + "unsafe" +) + // Sample captures a single metric sample. type Sample struct { // Name is the name of the metric sampled. @@ -16,6 +21,9 @@ type Sample struct { Value Value } +// Implemented in the runtime. +func runtime_readMetrics(unsafe.Pointer, int, int) + // Read populates each Value field in the given slice of metric samples. // // Desired metrics should be present in the slice with the appropriate name. @@ -25,5 +33,5 @@ type Sample struct { // will have the value populated as KindBad to indicate that the name is // unknown. func Read(m []Sample) { - panic("unimplemented") + runtime_readMetrics(unsafe.Pointer(&m[0]), len(m), cap(m)) } diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go new file mode 100644 index 0000000000..f00aad07c4 --- /dev/null +++ b/src/runtime/metrics_test.go @@ -0,0 +1,114 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime_test + +import ( + "runtime" + "runtime/metrics" + "strings" + "testing" + "unsafe" +) + +func prepareAllMetricsSamples() (map[string]metrics.Description, []metrics.Sample) { + all := metrics.All() + samples := make([]metrics.Sample, len(all)) + descs := make(map[string]metrics.Description) + for i := range all { + samples[i].Name = all[i].Name + descs[all[i].Name] = all[i] + } + return descs, samples +} + +func TestReadMetrics(t *testing.T) { + // Tests whether readMetrics produces values aligning + // with ReadMemStats while the world is stopped. + var mstats runtime.MemStats + _, samples := prepareAllMetricsSamples() + runtime.ReadMetricsSlow(&mstats, unsafe.Pointer(&samples[0]), len(samples), cap(samples)) + + checkUint64 := func(t *testing.T, m string, got, want uint64) { + t.Helper() + if got != want { + t.Errorf("metric %q: got %d, want %d", m, got, want) + } + } + + // Check to make sure the values we read line up with other values we read. + for i := range samples { + switch name := samples[i].Name; name { + case "/memory/classes/heap/free:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapIdle-mstats.HeapReleased) + case "/memory/classes/heap/released:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapReleased) + case "/memory/classes/heap/objects:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapAlloc) + case "/memory/classes/heap/unused:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapInuse-mstats.HeapAlloc) + case "/memory/classes/heap/stacks:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.StackInuse) + case "/memory/classes/metadata/mcache/free:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.MCacheSys-mstats.MCacheInuse) + case "/memory/classes/metadata/mcache/inuse:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.MCacheInuse) + case "/memory/classes/metadata/mspan/free:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.MSpanSys-mstats.MSpanInuse) + case "/memory/classes/metadata/mspan/inuse:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.MSpanInuse) + case "/memory/classes/metadata/other:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.GCSys) + case "/memory/classes/os-stacks:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.StackSys-mstats.StackInuse) + case "/memory/classes/other:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.OtherSys) + case "/memory/classes/profiling/buckets:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.BuckHashSys) + case "/memory/classes/total:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.Sys) + } + } +} + +func TestReadMetricsConsistency(t *testing.T) { + // Tests whether readMetrics produces consistent, sensible values. + // The values are read concurrently with the runtime doing other + // things (e.g. allocating) so what we read can't reasonably compared + // to runtime values. + + // Read all the supported metrics through the metrics package. + descs, samples := prepareAllMetricsSamples() + metrics.Read(samples) + + // Check to make sure the values we read make sense. + var totalVirtual struct { + got, want uint64 + } + for i := range samples { + kind := samples[i].Value.Kind() + if want := descs[samples[i].Name].Kind; kind != want { + t.Errorf("supported metric %q has unexpected kind: got %d, want %d", samples[i].Name, kind, want) + continue + } + if samples[i].Name != "/memory/classes/total:bytes" && strings.HasPrefix(samples[i].Name, "/memory/classes") { + v := samples[i].Value.Uint64() + totalVirtual.want += v + + // None of these stats should ever get this big. + // If they do, there's probably overflow involved, + // usually due to bad accounting. + if int64(v) < 0 { + t.Errorf("%q has high/negative value: %d", samples[i].Name, v) + } + } + switch samples[i].Name { + case "/memory/classes/total:bytes": + totalVirtual.got = samples[i].Value.Uint64() + } + } + if totalVirtual.got != totalVirtual.want { + t.Errorf(`"/memory/classes/total:bytes" does not match sum of /memory/classes/**: got %d, want %d`, totalVirtual.got, totalVirtual.want) + } +} diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index a8eca85fe6..512a06cffa 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -882,7 +882,8 @@ func (m *consistentHeapStats) unsafeClear() { // heapStatsDelta, the resulting values should be complete and // valid statistic values. // -// Not safe to call concurrently. +// Not safe to call concurrently. The world must be stopped +// or metricsSema must be held. func (m *consistentHeapStats) read(out *heapStatsDelta) { // Getting preempted after this point is not safe because // we read allp. We need to make sure a STW can't happen -- cgit v1.3 From 07c3f65d53df7bb9f84bdbd2ab64c0ae12337e3e Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 6 Aug 2020 15:44:27 +0000 Subject: runtime,runtime/metrics: add heap object count metric For #37112. Change-Id: Idd3dd5c84215ddd1ab05c2e76e848aa0a4d40fb0 Reviewed-on: https://go-review.googlesource.com/c/go/+/247043 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics.go | 18 ++++++++++++++++-- src/runtime/metrics/description.go | 5 +++++ src/runtime/metrics/doc.go | 3 +++ src/runtime/metrics_test.go | 2 ++ 4 files changed, 26 insertions(+), 2 deletions(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index 44b5a29751..cf619cca4b 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -38,6 +38,13 @@ func initMetrics() { return } metrics = map[string]metricData{ + "/gc/heap/objects:objects": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.heapStats.numObjects + }, + }, "/memory/classes/heap/free:bytes": { deps: makeStatDepSet(heapStatsDep), compute: func(in *statAggregate, out *metricValue) { @@ -210,9 +217,13 @@ func (s *statDepSet) has(d statDep) bool { type heapStatsAggregate struct { heapStatsDelta + // Derived from values in heapStatsDelta. + // inObjects is the bytes of memory occupied by objects, - // derived from other values in heapStats. inObjects uint64 + + // numObjects is the number of live objects in the heap. + numObjects uint64 } // compute populates the heapStatsAggregate with values from the runtime. @@ -221,8 +232,11 @@ func (a *heapStatsAggregate) compute() { // Calculate derived stats. a.inObjects = uint64(a.largeAlloc - a.largeFree) + a.numObjects = uint64(a.largeAllocCount - a.largeFreeCount) for i := range a.smallAllocCount { - a.inObjects += uint64(a.smallAllocCount[i]-a.smallFreeCount[i]) * uint64(class_to_size[i]) + n := uint64(a.smallAllocCount[i] - a.smallFreeCount[i]) + a.inObjects += n * uint64(class_to_size[i]) + a.numObjects += n } } diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 2e7df7e09f..47013e1451 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -50,6 +50,11 @@ type Description struct { // The English language descriptions below must be kept in sync with the // descriptions of each metric in doc.go. var allDesc = []Description{ + { + Name: "/gc/heap/objects:objects", + Description: "Number of objects, live or unswept, occupying heap memory.", + Kind: KindUint64, + }, { Name: "/memory/classes/heap/free:bytes", Description: "Memory that is available for allocation, and may be returned to the underlying system.", diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index fb4e23a2b5..4ac44bb19c 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -44,6 +44,9 @@ the documentation of the Name field of the Description struct. Supported metrics + /gc/heap/objects:objects + Number of objects, live or unswept, occupying heap memory. + /memory/classes/heap/free:bytes Memory that is available for allocation, and may be returned to the underlying system. diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index d925b057b0..6c0be7dc0b 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -70,6 +70,8 @@ func TestReadMetrics(t *testing.T) { checkUint64(t, name, samples[i].Value.Uint64(), mstats.BuckHashSys) case "/memory/classes/total:bytes": checkUint64(t, name, samples[i].Value.Uint64(), mstats.Sys) + case "/gc/heap/objects:objects": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapObjects) } } } -- cgit v1.3 From a8b28ebc87854fb6f2ba99f415f046dc2ff63604 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 6 Aug 2020 16:47:58 +0000 Subject: runtime,runtime/metrics: add heap goal and GC cycle metrics This change adds three new metrics: the heap goal, GC cycle count, and forced GC count. These metrics are identical to their MemStats counterparts. For #37112. Change-Id: I5a5e8dd550c0d646e5dcdbdf38274895e27cdd88 Reviewed-on: https://go-review.googlesource.com/c/go/+/247044 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics.go | 51 ++++++++++++++++++++++++++++++++------ src/runtime/metrics/description.go | 23 +++++++++++++++++ src/runtime/metrics/doc.go | 12 +++++++++ src/runtime/metrics_test.go | 8 ++++++ 4 files changed, 86 insertions(+), 8 deletions(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index cf619cca4b..6595a4342c 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -7,6 +7,7 @@ package runtime // Metrics implementation exported to runtime/metrics. import ( + "runtime/internal/atomic" "unsafe" ) @@ -38,6 +39,34 @@ func initMetrics() { return } metrics = map[string]metricData{ + "/gc/cycles/automatic:gc-cycles": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.gcCyclesDone - in.sysStats.gcCyclesForced + }, + }, + "/gc/cycles/forced:gc-cycles": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.gcCyclesForced + }, + }, + "/gc/cycles/total:gc-cycles": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.gcCyclesDone + }, + }, + "/gc/heap/goal:bytes": { + deps: makeStatDepSet(sysStatsDep), + compute: func(in *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = in.sysStats.heapGoal + }, + }, "/gc/heap/objects:objects": { deps: makeStatDepSet(heapStatsDep), compute: func(in *statAggregate, out *metricValue) { @@ -248,14 +277,17 @@ func (a *heapStatsAggregate) compute() { // heapStatsAggregate, means there could be some skew, but because of // these stats are independent, there's no real consistency issue here. type sysStatsAggregate struct { - stacksSys uint64 - mSpanSys uint64 - mSpanInUse uint64 - mCacheSys uint64 - mCacheInUse uint64 - buckHashSys uint64 - gcMiscSys uint64 - otherSys uint64 + stacksSys uint64 + mSpanSys uint64 + mSpanInUse uint64 + mCacheSys uint64 + mCacheInUse uint64 + buckHashSys uint64 + gcMiscSys uint64 + otherSys uint64 + heapGoal uint64 + gcCyclesDone uint64 + gcCyclesForced uint64 } // compute populates the sysStatsAggregate with values from the runtime. @@ -264,6 +296,9 @@ func (a *sysStatsAggregate) compute() { a.buckHashSys = memstats.buckhash_sys.load() a.gcMiscSys = memstats.gcMiscSys.load() a.otherSys = memstats.other_sys.load() + a.heapGoal = atomic.Load64(&memstats.next_gc) + a.gcCyclesDone = uint64(memstats.numgc) + a.gcCyclesForced = uint64(memstats.numforcedgc) systemstack(func() { lock(&mheap_.lock) diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 47013e1451..66d229c270 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -50,6 +50,29 @@ type Description struct { // The English language descriptions below must be kept in sync with the // descriptions of each metric in doc.go. var allDesc = []Description{ + { + Name: "/gc/cycles/automatic:gc-cycles", + Description: "Count of completed GC cycles generated by the Go runtime.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/cycles/forced:gc-cycles", + Description: "Count of completed forced GC cycles.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/cycles/total:gc-cycles", + Description: "Count of all completed GC cycles.", + Kind: KindUint64, + Cumulative: true, + }, + { + Name: "/gc/heap/goal:bytes", + Description: "Heap size target for the end of the GC cycle.", + Kind: KindUint64, + }, { Name: "/gc/heap/objects:objects", Description: "Number of objects, live or unswept, occupying heap memory.", diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index 4ac44bb19c..9b44e73ee6 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -44,6 +44,18 @@ the documentation of the Name field of the Description struct. Supported metrics + /gc/cycles/automatic:gc-cycles + Count of completed GC cycles generated by the Go runtime. + + /gc/cycles/forced:gc-cycles + Count of completed forced GC cycles. + + /gc/cycles/total:gc-cycles + Count of all completed GC cycles. + + /gc/heap/goal:bytes + Heap size target for the end of the GC cycle. + /gc/heap/objects:objects Number of objects, live or unswept, occupying heap memory. diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 6c0be7dc0b..3724760294 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -72,6 +72,14 @@ func TestReadMetrics(t *testing.T) { checkUint64(t, name, samples[i].Value.Uint64(), mstats.Sys) case "/gc/heap/objects:objects": checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapObjects) + case "/gc/heap/goal:bytes": + checkUint64(t, name, samples[i].Value.Uint64(), mstats.NextGC) + case "/gc/cycles/automatic:gc-cycles": + checkUint64(t, name, samples[i].Value.Uint64(), uint64(mstats.NumGC-mstats.NumForcedGC)) + case "/gc/cycles/forced:gc-cycles": + checkUint64(t, name, samples[i].Value.Uint64(), uint64(mstats.NumForcedGC)) + case "/gc/cycles/total:gc-cycles": + checkUint64(t, name, samples[i].Value.Uint64(), uint64(mstats.NumGC)) } } } -- cgit v1.3 From 8e2370bf7f0c992ce1ea5dc54b43551cea71a485 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 6 Aug 2020 19:04:46 +0000 Subject: runtime,runtime/metrics: add object size distribution metrics This change adds metrics for the distribution of objects allocated and freed by size, mirroring MemStats' BySize field. For #37112. Change-Id: Ibaf1812da93598b37265ec97abc6669c1a5efcbf Reviewed-on: https://go-review.googlesource.com/c/go/+/247045 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics.go | 52 ++++++++++++++++++++++++++++++++++++++ src/runtime/metrics/description.go | 10 ++++++++ src/runtime/metrics/doc.go | 6 +++++ src/runtime/metrics_test.go | 36 ++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index 6595a4342c..32d8ab461c 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -18,6 +18,8 @@ var ( metricsSema uint32 = 1 metricsInit bool metrics map[string]metricData + + sizeClassBuckets []float64 ) type metricData struct { @@ -38,6 +40,10 @@ func initMetrics() { if metricsInit { return } + sizeClassBuckets = make([]float64, _NumSizeClasses) + for i := range sizeClassBuckets { + sizeClassBuckets[i] = float64(class_to_size[i]) + } metrics = map[string]metricData{ "/gc/cycles/automatic:gc-cycles": { deps: makeStatDepSet(sysStatsDep), @@ -60,6 +66,26 @@ func initMetrics() { out.scalar = in.sysStats.gcCyclesDone }, }, + "/gc/heap/allocs-by-size:objects": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + hist := out.float64HistOrInit(sizeClassBuckets) + hist.counts[len(hist.counts)-1] = uint64(in.heapStats.largeAllocCount) + for i := range hist.buckets { + hist.counts[i] = uint64(in.heapStats.smallAllocCount[i]) + } + }, + }, + "/gc/heap/frees-by-size:objects": { + deps: makeStatDepSet(heapStatsDep), + compute: func(in *statAggregate, out *metricValue) { + hist := out.float64HistOrInit(sizeClassBuckets) + hist.counts[len(hist.counts)-1] = uint64(in.heapStats.largeFreeCount) + for i := range hist.buckets { + hist.counts[i] = uint64(in.heapStats.smallFreeCount[i]) + } + }, + }, "/gc/heap/goal:bytes": { deps: makeStatDepSet(sysStatsDep), compute: func(in *statAggregate, out *metricValue) { @@ -370,6 +396,32 @@ type metricValue struct { pointer unsafe.Pointer // contains non-scalar values. } +// float64HistOrInit tries to pull out an existing float64Histogram +// from the value, but if none exists, then it allocates one with +// the given buckets. +func (v *metricValue) float64HistOrInit(buckets []float64) *metricFloat64Histogram { + var hist *metricFloat64Histogram + if v.kind == metricKindFloat64Histogram && v.pointer != nil { + hist = (*metricFloat64Histogram)(v.pointer) + } else { + v.kind = metricKindFloat64Histogram + hist = new(metricFloat64Histogram) + v.pointer = unsafe.Pointer(hist) + } + hist.buckets = buckets + if len(hist.counts) != len(hist.buckets)+1 { + hist.counts = make([]uint64, len(buckets)+1) + } + return hist +} + +// metricFloat64Histogram is a runtime copy of runtime/metrics.Float64Histogram +// and must be kept structurally identical to that type. +type metricFloat64Histogram struct { + counts []uint64 + buckets []float64 +} + // agg is used by readMetrics, and is protected by metricsSema. // // Managed as a global variable because its pointer will be diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 66d229c270..e43904fc7d 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -68,6 +68,16 @@ var allDesc = []Description{ Kind: KindUint64, Cumulative: true, }, + { + Name: "/gc/heap/allocs-by-size:objects", + Description: "Distribution of all objects allocated by approximate size.", + Kind: KindFloat64Histogram, + }, + { + Name: "/gc/heap/frees-by-size:objects", + Description: "Distribution of all objects freed by approximate size.", + Kind: KindFloat64Histogram, + }, { Name: "/gc/heap/goal:bytes", Description: "Heap size target for the end of the GC cycle.", diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index 9b44e73ee6..5045a5b4c1 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -53,6 +53,12 @@ Supported metrics /gc/cycles/total:gc-cycles Count of all completed GC cycles. + /gc/heap/allocs-by-size:objects + Distribution of all objects allocated by approximate size. + + /gc/heap/frees-by-size:objects + Distribution of all objects freed by approximate size. + /gc/heap/goal:bytes Heap size target for the end of the GC cycle. diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 3724760294..1a30810544 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -98,6 +98,10 @@ func TestReadMetricsConsistency(t *testing.T) { var totalVirtual struct { got, want uint64 } + var objects struct { + alloc, free *metrics.Float64Histogram + total uint64 + } for i := range samples { kind := samples[i].Value.Kind() if want := descs[samples[i].Name].Kind; kind != want { @@ -118,11 +122,43 @@ func TestReadMetricsConsistency(t *testing.T) { switch samples[i].Name { case "/memory/classes/total:bytes": totalVirtual.got = samples[i].Value.Uint64() + case "/gc/heap/objects:objects": + objects.total = samples[i].Value.Uint64() + case "/gc/heap/allocs-by-size:objects": + objects.alloc = samples[i].Value.Float64Histogram() + case "/gc/heap/frees-by-size:objects": + objects.free = samples[i].Value.Float64Histogram() } } if totalVirtual.got != totalVirtual.want { t.Errorf(`"/memory/classes/total:bytes" does not match sum of /memory/classes/**: got %d, want %d`, totalVirtual.got, totalVirtual.want) } + if len(objects.alloc.Buckets) != len(objects.free.Buckets) { + t.Error("allocs-by-size and frees-by-size buckets don't match in length") + } else if len(objects.alloc.Counts) != len(objects.free.Counts) { + t.Error("allocs-by-size and frees-by-size counts don't match in length") + } else { + for i := range objects.alloc.Buckets { + ba := objects.alloc.Buckets[i] + bf := objects.free.Buckets[i] + if ba != bf { + t.Errorf("bucket %d is different for alloc and free hists: %f != %f", i, ba, bf) + } + } + if !t.Failed() { + got, want := uint64(0), objects.total + for i := range objects.alloc.Counts { + if objects.alloc.Counts[i] < objects.free.Counts[i] { + t.Errorf("found more allocs than frees in object dist bucket %d", i) + continue + } + got += objects.alloc.Counts[i] - objects.free.Counts[i] + } + if got != want { + t.Errorf("object distribution counts don't match count of live objects: got %d, want %d", got, want) + } + } + } } func BenchmarkReadMetricsLatency(b *testing.B) { -- cgit v1.3 From d39a89fd5843f535d634620d27110b320431f584 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 6 Aug 2020 21:59:13 +0000 Subject: runtime,runtime/metrics: add metric for distribution of GC pauses For #37112. Change-Id: Ibb0425c9c582ae3da3b2662d5bbe830d7df9079c Reviewed-on: https://go-review.googlesource.com/c/go/+/247047 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics.go | 9 +++++++++ src/runtime/metrics/description.go | 5 +++++ src/runtime/metrics/doc.go | 3 +++ src/runtime/metrics_test.go | 22 ++++++++++++++++++++++ src/runtime/mgc.go | 3 +++ src/runtime/mstats.go | 12 ++++++++++++ 6 files changed, 54 insertions(+) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index 2be38ccaaa..0e391472b2 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -102,6 +102,15 @@ func initMetrics() { out.scalar = in.heapStats.numObjects }, }, + "/gc/pauses:seconds": { + compute: func(_ *statAggregate, out *metricValue) { + hist := out.float64HistOrInit(timeHistBuckets) + hist.counts[len(hist.counts)-1] = atomic.Load64(&memstats.gcPauseDist.overflow) + for i := range hist.buckets { + hist.counts[i] = atomic.Load64(&memstats.gcPauseDist.counts[i]) + } + }, + }, "/memory/classes/heap/free:bytes": { deps: makeStatDepSet(heapStatsDep), compute: func(in *statAggregate, out *metricValue) { diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index e43904fc7d..47959e467c 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -88,6 +88,11 @@ var allDesc = []Description{ Description: "Number of objects, live or unswept, occupying heap memory.", Kind: KindUint64, }, + { + Name: "/gc/pauses:seconds", + Description: "Distribution individual GC-related stop-the-world pause latencies.", + Kind: KindFloat64Histogram, + }, { Name: "/memory/classes/heap/free:bytes", Description: "Memory that is available for allocation, and may be returned to the underlying system.", diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index 5045a5b4c1..1e12ade5a1 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -65,6 +65,9 @@ Supported metrics /gc/heap/objects:objects Number of objects, live or unswept, occupying heap memory. + /gc/pauses:seconds + Distribution individual GC-related stop-the-world pause latencies. + /memory/classes/heap/free:bytes Memory that is available for allocation, and may be returned to the underlying system. diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 1a30810544..7b3132bc30 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -90,6 +90,11 @@ func TestReadMetricsConsistency(t *testing.T) { // things (e.g. allocating) so what we read can't reasonably compared // to runtime values. + // Run a few GC cycles to get some of the stats to be non-zero. + runtime.GC() + runtime.GC() + runtime.GC() + // Read all the supported metrics through the metrics package. descs, samples := prepareAllMetricsSamples() metrics.Read(samples) @@ -102,6 +107,10 @@ func TestReadMetricsConsistency(t *testing.T) { alloc, free *metrics.Float64Histogram total uint64 } + var gc struct { + numGC uint64 + pauses uint64 + } for i := range samples { kind := samples[i].Value.Kind() if want := descs[samples[i].Name].Kind; kind != want { @@ -128,6 +137,14 @@ func TestReadMetricsConsistency(t *testing.T) { objects.alloc = samples[i].Value.Float64Histogram() case "/gc/heap/frees-by-size:objects": objects.free = samples[i].Value.Float64Histogram() + case "/gc/cycles:gc-cycles": + gc.numGC = samples[i].Value.Uint64() + case "/gc/pauses:seconds": + h := samples[i].Value.Float64Histogram() + gc.pauses = 0 + for i := range h.Counts { + gc.pauses += h.Counts[i] + } } } if totalVirtual.got != totalVirtual.want { @@ -159,6 +176,11 @@ func TestReadMetricsConsistency(t *testing.T) { } } } + // The current GC has at least 2 pauses per GC. + // Check to see if that value makes sense. + if gc.pauses < gc.numGC*2 { + t.Errorf("fewer pauses than expected: got %d, want at least %d", gc.pauses, gc.numGC*2) + } } func BenchmarkReadMetricsLatency(b *testing.B) { diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 540c376f1c..b0ab0ae6bb 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1418,6 +1418,7 @@ func gcStart(trigger gcTrigger) { now = startTheWorldWithSema(trace.enabled) work.pauseNS += now - work.pauseStart work.tMark = now + memstats.gcPauseDist.record(now - work.pauseStart) }) // Release the world sema before Gosched() in STW mode @@ -1565,6 +1566,7 @@ top: systemstack(func() { now := startTheWorldWithSema(true) work.pauseNS += now - work.pauseStart + memstats.gcPauseDist.record(now - work.pauseStart) }) semrelease(&worldsema) goto top @@ -1677,6 +1679,7 @@ func gcMarkTermination(nextTriggerRatio float64) { unixNow := sec*1e9 + int64(nsec) work.pauseNS += now - work.pauseStart work.tEnd = now + memstats.gcPauseDist.record(now - work.pauseStart) atomic.Store64(&memstats.last_gc_unix, uint64(unixNow)) // must be Unix time to make sense to user atomic.Store64(&memstats.last_gc_nanotime, uint64(now)) // monotonic time for us memstats.pause_ns[memstats.numgc%uint32(len(memstats.pause_ns))] = uint64(work.pauseNS) diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index 07f466ec49..e0a417d213 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -157,6 +157,14 @@ type mstats struct { // heapStats is a set of statistics heapStats consistentHeapStats + + _ uint32 // ensure gcPauseDist is aligned + + // gcPauseDist represents the distribution of all GC-related + // application pauses in the runtime. + // + // Each individual pause is counted separately, unlike pause_ns. + gcPauseDist timeHistogram } var memstats mstats @@ -443,6 +451,10 @@ func init() { println(offset) throw("memstats.heapStats not aligned to 8 bytes") } + if offset := unsafe.Offsetof(memstats.gcPauseDist); offset%8 != 0 { + println(offset) + throw("memstats.gcPauseDist not aligned to 8 bytes") + } // Ensure the size of heapStatsDelta causes adjacent fields/slots (e.g. // [3]heapStatsDelta) to be 8-byte aligned. if size := unsafe.Sizeof(heapStatsDelta{}); size%8 != 0 { -- cgit v1.3 From 80c6b92ecb911409f57d06793a1213395b75ebe2 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Fri, 7 Aug 2020 16:37:29 +0000 Subject: runtime,runtime/metrics: export goroutine count as a metric For #37112. Change-Id: I994dfe848605b95ef6aec24f53869e929247e987 Reviewed-on: https://go-review.googlesource.com/c/go/+/247049 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics.go | 6 ++++++ src/runtime/metrics/description.go | 5 +++++ src/runtime/metrics/doc.go | 3 +++ src/runtime/metrics_test.go | 4 ++++ 4 files changed, 18 insertions(+) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index 0e391472b2..d3c0341aee 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -214,6 +214,12 @@ func initMetrics() { in.sysStats.gcMiscSys + in.sysStats.otherSys }, }, + "/sched/goroutines:goroutines": { + compute: func(_ *statAggregate, out *metricValue) { + out.kind = metricKindUint64 + out.scalar = uint64(gcount()) + }, + }, } metricsInit = true } diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 47959e467c..bc2e0882db 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -163,6 +163,11 @@ var allDesc = []Description{ Description: "All memory mapped by the Go runtime into the current process as read-write. Note that this does not include memory mapped by code called via cgo or via the syscall package. Sum of all metrics in /memory/classes.", Kind: KindUint64, }, + { + Name: "/sched/goroutines:goroutines", + Description: "Count of live goroutines.", + Kind: KindUint64, + }, } // All returns a slice of containing metric descriptions for all supported metrics. diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index 1e12ade5a1..e340f3d0dd 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -123,5 +123,8 @@ Supported metrics as read-write. Note that this does not include memory mapped by code called via cgo or via the syscall package. Sum of all metrics in /memory/classes. + + /sched/goroutines:goroutines + Count of live goroutines. */ package metrics diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 7b3132bc30..167edd57fd 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -145,6 +145,10 @@ func TestReadMetricsConsistency(t *testing.T) { for i := range h.Counts { gc.pauses += h.Counts[i] } + case "/sched/goroutines:goroutines": + if samples[i].Value.Uint64() < 1 { + t.Error("number of goroutines is less than one") + } } } if totalVirtual.got != totalVirtual.want { -- cgit v1.3 From add45938b53114656980ecc47021b4463d9f2507 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 19 Nov 2020 15:01:34 +0000 Subject: runtime/metrics: clarify memory and GC metrics documentation Change-Id: I8940990a591a808ddd4b8613531f52453f85bde1 Reviewed-on: https://go-review.googlesource.com/c/go/+/271557 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics/description.go | 29 +++++++++++++++++------------ src/runtime/metrics/doc.go | 23 +++++++++++++++-------- 2 files changed, 32 insertions(+), 20 deletions(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index bc2e0882db..9d3611b64c 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -58,7 +58,7 @@ var allDesc = []Description{ }, { Name: "/gc/cycles/forced:gc-cycles", - Description: "Count of completed forced GC cycles.", + Description: "Count of completed GC cycles forced by the application.", Kind: KindUint64, Cumulative: true, }, @@ -94,28 +94,33 @@ var allDesc = []Description{ Kind: KindFloat64Histogram, }, { - Name: "/memory/classes/heap/free:bytes", - Description: "Memory that is available for allocation, and may be returned to the underlying system.", - Kind: KindUint64, + Name: "/memory/classes/heap/free:bytes", + Description: "Memory that is completely free and eligible to be returned to the underlying system, " + + "but has not been. This metric is the runtime's estimate of free address space that is backed by " + + "physical memory.", + Kind: KindUint64, }, { Name: "/memory/classes/heap/objects:bytes", - Description: "Memory occupied by live objects and dead objects that have not yet been collected.", + Description: "Memory occupied by live objects and dead objects that have not yet been marked free by the garbage collector.", Kind: KindUint64, }, { - Name: "/memory/classes/heap/released:bytes", - Description: "Memory that has been returned to the underlying system.", - Kind: KindUint64, + Name: "/memory/classes/heap/released:bytes", + Description: "Memory that is completely free and has been returned to the underlying system. This " + + "metric is the runtime's estimate of free address space that is still mapped into the process, " + + "but is not backed by physical memory.", + Kind: KindUint64, }, { - Name: "/memory/classes/heap/stacks:bytes", - Description: "Memory allocated from the heap that is occupied by stacks.", - Kind: KindUint64, + Name: "/memory/classes/heap/stacks:bytes", + Description: "Memory allocated from the heap that is reserved for stack space. Not all of it is necessarily " + + "simultaneously in use, but it may not be used for any other purpose.", + Kind: KindUint64, }, { Name: "/memory/classes/heap/unused:bytes", - Description: "Memory that is unavailable for allocation, but cannot be returned to the underlying system.", + Description: "Memory that is reserved for heap objects but is otherwise not currently used to hold heap objects.", Kind: KindUint64, }, { diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index e340f3d0dd..f58cdcdd03 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -48,7 +48,7 @@ Supported metrics Count of completed GC cycles generated by the Go runtime. /gc/cycles/forced:gc-cycles - Count of completed forced GC cycles. + Count of completed GC cycles forced by the application. /gc/cycles/total:gc-cycles Count of all completed GC cycles. @@ -69,22 +69,29 @@ Supported metrics Distribution individual GC-related stop-the-world pause latencies. /memory/classes/heap/free:bytes - Memory that is available for allocation, and may be returned - to the underlying system. + Memory that is completely free and eligible to be returned to + the underlying system, but has not been. This metric is the + runtime's estimate of free address space that is backed by + physical memory. /memory/classes/heap/objects:bytes Memory occupied by live objects and dead objects that have - not yet been collected. + not yet been marked free by the garbage collector. /memory/classes/heap/released:bytes - Memory that has been returned to the underlying system. + Memory that is completely free and has been returned to + the underlying system. This metric is the runtime's estimate of + free address space that is still mapped into the process, but + is not backed by physical memory. /memory/classes/heap/stacks:bytes - Memory allocated from the heap that is occupied by stacks. + Memory allocated from the heap that is reserved for stack + space. Not all of it is necessarily simultaneously in use, but + it may not be used for any other purpose. /memory/classes/heap/unused:bytes - Memory that is unavailable for allocation, but cannot be - returned to the underlying system. + Memory that is reserved for heap objects but is otherwise not + currently used to hold heap objects. /memory/classes/metadata/mcache/free:bytes Memory that is reserved for runtime mcache structures, but -- cgit v1.3 From 59f5fdac5de669ef534cc744c94d225445a3c193 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Thu, 19 Nov 2020 15:07:06 +0000 Subject: runtime/metrics: clarify Read's documentation Change-Id: Idbcbc304f1568399a82af9dcd51e511393ed5ee0 Reviewed-on: https://go-review.googlesource.com/c/go/+/271558 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics/sample.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/sample.go b/src/runtime/metrics/sample.go index b4b0979aa6..60189cb334 100644 --- a/src/runtime/metrics/sample.go +++ b/src/runtime/metrics/sample.go @@ -27,11 +27,11 @@ func runtime_readMetrics(unsafe.Pointer, int, int) // Read populates each Value field in the given slice of metric samples. // // Desired metrics should be present in the slice with the appropriate name. -// The user of this API is encouraged to re-use the same slice between calls. +// The user of this API is encouraged to re-use the same slice between calls for +// efficiency, but is not required to do so. // -// Metric values with names not appearing in the value returned by Descriptions -// will have the value populated as KindBad to indicate that the name is -// unknown. +// Sample values with names not appearing in All will have their Value populated +// as KindBad to indicate that the name is unknown. func Read(m []Sample) { runtime_readMetrics(unsafe.Pointer(&m[0]), len(m), cap(m)) } -- cgit v1.3 From 48a1a5189843571a08461a5756e5fe553f966c94 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Mon, 23 Nov 2020 19:25:32 +0000 Subject: runtime/metrics: tweak wording of stack and unused memory metrics This change tweaks and simplifies the descriptions of a couple metrics to make them easier to parse (for humans). Change-Id: I852654c7e7042c662ebdfa6334e3baf49ca4b33c Reviewed-on: https://go-review.googlesource.com/c/go/+/272566 Trust: Michael Knyszek Run-TryBot: Michael Knyszek Reviewed-by: Robert Findley TryBot-Result: Go Bot --- src/runtime/metrics/description.go | 9 ++++----- src/runtime/metrics/doc.go | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 9d3611b64c..32af5d1727 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -113,14 +113,13 @@ var allDesc = []Description{ Kind: KindUint64, }, { - Name: "/memory/classes/heap/stacks:bytes", - Description: "Memory allocated from the heap that is reserved for stack space. Not all of it is necessarily " + - "simultaneously in use, but it may not be used for any other purpose.", - Kind: KindUint64, + Name: "/memory/classes/heap/stacks:bytes", + Description: "Memory allocated from the heap that is reserved for stack space, whether or not it is currently in-use.", + Kind: KindUint64, }, { Name: "/memory/classes/heap/unused:bytes", - Description: "Memory that is reserved for heap objects but is otherwise not currently used to hold heap objects.", + Description: "Memory that is reserved for heap objects but is not currently used to hold heap objects.", Kind: KindUint64, }, { diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index f58cdcdd03..42b5bc3724 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -86,12 +86,11 @@ Supported metrics /memory/classes/heap/stacks:bytes Memory allocated from the heap that is reserved for stack - space. Not all of it is necessarily simultaneously in use, but - it may not be used for any other purpose. + space, whether or not it is currently in-use. /memory/classes/heap/unused:bytes - Memory that is reserved for heap objects but is otherwise not - currently used to hold heap objects. + Memory that is reserved for heap objects but is not currently + used to hold heap objects. /memory/classes/metadata/mcache/free:bytes Memory that is reserved for runtime mcache structures, but -- cgit v1.3 From 5627a4dc3013fed02c4b8097413643b682cac276 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 8 Dec 2020 19:44:33 -0500 Subject: runtime/metrics: simplify test to support more environments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit go test sets the working directory to that of the package being tested, so opening one of the package source files can be done in a simpler way. This also allows the test to run in more environments, for example when GOROOT_FINAL¹ is set. Also remove the testenv.HasSrc-like check for Go source. The doc.go file is a part of the package being built and tested, so it's expected to be available. If it's important for this test to handle when a test binary is built with go test -c and executed elsewhere without package source files, something more than testenv.HasSrc would be needed. ¹ https://golang.org/cmd/go/#hdr-Environment_variables Fixes #43085. Change-Id: Ie6ade395a8fc7beebdadbad6f4873800138dfc26 Reviewed-on: https://go-review.googlesource.com/c/go/+/276452 Reviewed-by: Bryan C. Mills Reviewed-by: Michael Knyszek Trust: Dmitri Shuralyov --- src/runtime/metrics/description_test.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/description_test.go b/src/runtime/metrics/description_test.go index e966a281a1..448639ee77 100644 --- a/src/runtime/metrics/description_test.go +++ b/src/runtime/metrics/description_test.go @@ -7,9 +7,7 @@ package metrics_test import ( "bufio" "os" - "path/filepath" "regexp" - "runtime" "runtime/metrics" "strings" "testing" @@ -26,17 +24,9 @@ func TestDescriptionNameFormat(t *testing.T) { } func extractMetricDocs(t *testing.T) map[string]string { - if runtime.GOOS == "android" { - t.Skip("no access to Go source on android") - } - - // Get doc.go. - _, filename, _, _ := runtime.Caller(0) - filename = filepath.Join(filepath.Dir(filename), "doc.go") - - f, err := os.Open(filename) + f, err := os.Open("doc.go") if err != nil { - t.Fatal(err) + t.Fatalf("failed to open doc.go in runtime/metrics package: %v", err) } const ( stateSearch = iota // look for list of metrics @@ -90,7 +80,7 @@ func extractMetricDocs(t *testing.T) map[string]string { } } if state == stateSearch { - t.Fatalf("failed to find supported metrics docs in %s", filename) + t.Fatalf("failed to find supported metrics docs in %s", f.Name()) } return result } -- cgit v1.3 From d0f40d29223a2aef58475f81a559b2d27396134a Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Fri, 4 Dec 2020 16:04:53 +0000 Subject: runtime/metrics: add ordering line to supported metrics docs This change adds an additional line explaining the ordering of the supported metrics list. It's also necessary to ensure "Supported metrics" is displayed by godoc as a proper header. This modification does mean the description test, that ensures descriptions line up with documentation, needs to change slightly so it it doesn't read this new line as documentation. Make this new line the line the test uses to decide when to begin. Change-Id: I654c1c20e97a80ea79c8eb864445153ce91950bf Reviewed-on: https://go-review.googlesource.com/c/go/+/275852 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics/description_test.go | 2 +- src/runtime/metrics/doc.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/description_test.go b/src/runtime/metrics/description_test.go index 448639ee77..fd1fd46efc 100644 --- a/src/runtime/metrics/description_test.go +++ b/src/runtime/metrics/description_test.go @@ -43,7 +43,7 @@ func extractMetricDocs(t *testing.T) map[string]string { line := strings.TrimSpace(s.Text()) switch state { case stateSearch: - if line == "Supported metrics" { + if line == "Below is the full list of supported metrics, ordered lexicographically." { state = stateNextMetric } case stateNextMetric: diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index 42b5bc3724..05a887e4f4 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -44,6 +44,8 @@ the documentation of the Name field of the Description struct. Supported metrics +Below is the full list of supported metrics, ordered lexicographically. + /gc/cycles/automatic:gc-cycles Count of completed GC cycles generated by the Go runtime. -- cgit v1.3 From e0d20e52ee00fdf197f359d98526ff7ca0842e6b Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Mon, 7 Dec 2020 15:03:51 +0000 Subject: runtime/metrics: expand Read documention with caveats This change modifies the documentation of Read with some caveats about reusing the slice passed in to Read as well as with what concurrent situations are safe. Change-Id: I76fd31acc67ae384546a8442dfbf9d16b7445cff Reviewed-on: https://go-review.googlesource.com/c/go/+/275853 Run-TryBot: Michael Knyszek TryBot-Result: Go Bot Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics/sample.go | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/sample.go b/src/runtime/metrics/sample.go index 60189cb334..35534dd70d 100644 --- a/src/runtime/metrics/sample.go +++ b/src/runtime/metrics/sample.go @@ -30,6 +30,16 @@ func runtime_readMetrics(unsafe.Pointer, int, int) // The user of this API is encouraged to re-use the same slice between calls for // efficiency, but is not required to do so. // +// Note that re-use has some caveats. Notably, Values should not be read or +// manipulated while a Read with that value is outstanding; that is a data race. +// This property includes pointer-typed Values (e.g. Float64Histogram) whose +// underlying storage will be reused by Read when possible. To safely use such +// values in a concurrent setting, all data must be deep-copied. +// +// It is safe to execute multiple Read calls concurrently, but their arguments +// must share no underlying memory. When in doubt, create a new []Sample from +// scratch, which is always safe, though may be inefficient. +// // Sample values with names not appearing in All will have their Value populated // as KindBad to indicate that the name is unknown. func Read(m []Sample) { -- cgit v1.3 From 985d91666cebbd0aef36034cc28596da280ead37 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Mon, 7 Dec 2020 15:10:43 +0000 Subject: runtime/metrics: add a note about floating-point values to package docs This change adds a note to the package documentation that the package will never produce a NaN or infinity, to help ease usability. Change-Id: I72ff6ab636ca23722a68ef11e707c68b0724ac04 Reviewed-on: https://go-review.googlesource.com/c/go/+/275854 Run-TryBot: Michael Knyszek Trust: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/metrics/doc.go | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/runtime/metrics') diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index 05a887e4f4..a68184ee82 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -42,6 +42,12 @@ did also, and a new key should be introduced. For more details on the precise definition of the metric key's path and unit formats, see the documentation of the Name field of the Description struct. +A note about floats + +This package supports metrics whose values have a floating-point representation. In +order to improve ease-of-use, this package promises to never produce the following +classes of floating-point values: NaN, infinity. + Supported metrics Below is the full list of supported metrics, ordered lexicographically. -- cgit v1.3