From 682ffae6db749ba63df4b8bc1739974346bb14d7 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 31 Oct 2016 12:07:13 -0400 Subject: internal/pprof/profile: new package, moved from cmd/internal/pprof/profile This allows both the runtime and the cmd/pprof code to use the package, just like we do for internal/trace. Change-Id: I7606977284e1def36c9647354c58e7c1e93dba6b Reviewed-on: https://go-review.googlesource.com/32452 Run-TryBot: Russ Cox Reviewed-by: Brad Fitzpatrick --- src/cmd/internal/pprof/driver/driver.go | 2 +- src/cmd/internal/pprof/driver/interactive.go | 2 +- src/cmd/internal/pprof/fetch/fetch.go | 2 +- src/cmd/internal/pprof/plugin/plugin.go | 2 +- src/cmd/internal/pprof/profile/encode.go | 470 -------- src/cmd/internal/pprof/profile/filter.go | 158 --- src/cmd/internal/pprof/profile/legacy_profile.go | 1250 ---------------------- src/cmd/internal/pprof/profile/profile.go | 572 ---------- src/cmd/internal/pprof/profile/profile_test.go | 24 - src/cmd/internal/pprof/profile/proto.go | 360 ------- src/cmd/internal/pprof/profile/proto_test.go | 67 -- src/cmd/internal/pprof/profile/prune.go | 97 -- src/cmd/internal/pprof/report/report.go | 2 +- src/cmd/internal/pprof/symbolizer/symbolizer.go | 2 +- src/cmd/internal/pprof/symbolz/symbolz.go | 2 +- src/cmd/pprof/pprof.go | 2 +- src/cmd/trace/pprof.go | 2 +- src/go/build/deps_test.go | 1 + src/internal/pprof/profile/encode.go | 470 ++++++++ src/internal/pprof/profile/filter.go | 158 +++ src/internal/pprof/profile/legacy_profile.go | 1250 ++++++++++++++++++++++ src/internal/pprof/profile/profile.go | 572 ++++++++++ src/internal/pprof/profile/profile_test.go | 24 + src/internal/pprof/profile/proto.go | 360 +++++++ src/internal/pprof/profile/proto_test.go | 67 ++ src/internal/pprof/profile/prune.go | 97 ++ 26 files changed, 3008 insertions(+), 3007 deletions(-) delete mode 100644 src/cmd/internal/pprof/profile/encode.go delete mode 100644 src/cmd/internal/pprof/profile/filter.go delete mode 100644 src/cmd/internal/pprof/profile/legacy_profile.go delete mode 100644 src/cmd/internal/pprof/profile/profile.go delete mode 100644 src/cmd/internal/pprof/profile/profile_test.go delete mode 100644 src/cmd/internal/pprof/profile/proto.go delete mode 100644 src/cmd/internal/pprof/profile/proto_test.go delete mode 100644 src/cmd/internal/pprof/profile/prune.go create mode 100644 src/internal/pprof/profile/encode.go create mode 100644 src/internal/pprof/profile/filter.go create mode 100644 src/internal/pprof/profile/legacy_profile.go create mode 100644 src/internal/pprof/profile/profile.go create mode 100644 src/internal/pprof/profile/profile_test.go create mode 100644 src/internal/pprof/profile/proto.go create mode 100644 src/internal/pprof/profile/proto_test.go create mode 100644 src/internal/pprof/profile/prune.go (limited to 'src') diff --git a/src/cmd/internal/pprof/driver/driver.go b/src/cmd/internal/pprof/driver/driver.go index 8f6c7e1a9c..f3210723cd 100644 --- a/src/cmd/internal/pprof/driver/driver.go +++ b/src/cmd/internal/pprof/driver/driver.go @@ -23,9 +23,9 @@ import ( "cmd/internal/pprof/commands" "cmd/internal/pprof/plugin" - "cmd/internal/pprof/profile" "cmd/internal/pprof/report" "cmd/internal/pprof/tempfile" + "internal/pprof/profile" ) // PProf acquires a profile, and symbolizes it using a profile diff --git a/src/cmd/internal/pprof/driver/interactive.go b/src/cmd/internal/pprof/driver/interactive.go index 1b08226527..81df976fa1 100644 --- a/src/cmd/internal/pprof/driver/interactive.go +++ b/src/cmd/internal/pprof/driver/interactive.go @@ -14,7 +14,7 @@ import ( "cmd/internal/pprof/commands" "cmd/internal/pprof/plugin" - "cmd/internal/pprof/profile" + "internal/pprof/profile" ) var profileFunctionNames = []string{} diff --git a/src/cmd/internal/pprof/fetch/fetch.go b/src/cmd/internal/pprof/fetch/fetch.go index ffd282e74d..95d9be6aa2 100644 --- a/src/cmd/internal/pprof/fetch/fetch.go +++ b/src/cmd/internal/pprof/fetch/fetch.go @@ -17,7 +17,7 @@ import ( "time" "cmd/internal/pprof/plugin" - "cmd/internal/pprof/profile" + "internal/pprof/profile" ) // FetchProfile reads from a data source (network, file) and generates a diff --git a/src/cmd/internal/pprof/plugin/plugin.go b/src/cmd/internal/pprof/plugin/plugin.go index d5025d5517..ff1e8adfaf 100644 --- a/src/cmd/internal/pprof/plugin/plugin.go +++ b/src/cmd/internal/pprof/plugin/plugin.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "cmd/internal/pprof/profile" + "internal/pprof/profile" ) // A FlagSet creates and parses command-line flags. diff --git a/src/cmd/internal/pprof/profile/encode.go b/src/cmd/internal/pprof/profile/encode.go deleted file mode 100644 index 6b879a84ac..0000000000 --- a/src/cmd/internal/pprof/profile/encode.go +++ /dev/null @@ -1,470 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package profile - -import ( - "errors" - "fmt" - "sort" -) - -func (p *Profile) decoder() []decoder { - return profileDecoder -} - -// preEncode populates the unexported fields to be used by encode -// (with suffix X) from the corresponding exported fields. The -// exported fields are cleared up to facilitate testing. -func (p *Profile) preEncode() { - strings := make(map[string]int) - addString(strings, "") - - for _, st := range p.SampleType { - st.typeX = addString(strings, st.Type) - st.unitX = addString(strings, st.Unit) - } - - for _, s := range p.Sample { - s.labelX = nil - var keys []string - for k := range s.Label { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - vs := s.Label[k] - for _, v := range vs { - s.labelX = append(s.labelX, - Label{ - keyX: addString(strings, k), - strX: addString(strings, v), - }, - ) - } - } - var numKeys []string - for k := range s.NumLabel { - numKeys = append(numKeys, k) - } - sort.Strings(numKeys) - for _, k := range numKeys { - vs := s.NumLabel[k] - for _, v := range vs { - s.labelX = append(s.labelX, - Label{ - keyX: addString(strings, k), - numX: v, - }, - ) - } - } - s.locationIDX = nil - for _, l := range s.Location { - s.locationIDX = append(s.locationIDX, l.ID) - } - } - - for _, m := range p.Mapping { - m.fileX = addString(strings, m.File) - m.buildIDX = addString(strings, m.BuildID) - } - - for _, l := range p.Location { - for i, ln := range l.Line { - if ln.Function != nil { - l.Line[i].functionIDX = ln.Function.ID - } else { - l.Line[i].functionIDX = 0 - } - } - if l.Mapping != nil { - l.mappingIDX = l.Mapping.ID - } else { - l.mappingIDX = 0 - } - } - for _, f := range p.Function { - f.nameX = addString(strings, f.Name) - f.systemNameX = addString(strings, f.SystemName) - f.filenameX = addString(strings, f.Filename) - } - - p.dropFramesX = addString(strings, p.DropFrames) - p.keepFramesX = addString(strings, p.KeepFrames) - - if pt := p.PeriodType; pt != nil { - pt.typeX = addString(strings, pt.Type) - pt.unitX = addString(strings, pt.Unit) - } - - p.stringTable = make([]string, len(strings)) - for s, i := range strings { - p.stringTable[i] = s - } -} - -func (p *Profile) encode(b *buffer) { - for _, x := range p.SampleType { - encodeMessage(b, 1, x) - } - for _, x := range p.Sample { - encodeMessage(b, 2, x) - } - for _, x := range p.Mapping { - encodeMessage(b, 3, x) - } - for _, x := range p.Location { - encodeMessage(b, 4, x) - } - for _, x := range p.Function { - encodeMessage(b, 5, x) - } - encodeStrings(b, 6, p.stringTable) - encodeInt64Opt(b, 7, p.dropFramesX) - encodeInt64Opt(b, 8, p.keepFramesX) - encodeInt64Opt(b, 9, p.TimeNanos) - encodeInt64Opt(b, 10, p.DurationNanos) - if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) { - encodeMessage(b, 11, p.PeriodType) - } - encodeInt64Opt(b, 12, p.Period) -} - -var profileDecoder = []decoder{ - nil, // 0 - // repeated ValueType sample_type = 1 - func(b *buffer, m message) error { - x := new(ValueType) - pp := m.(*Profile) - pp.SampleType = append(pp.SampleType, x) - return decodeMessage(b, x) - }, - // repeated Sample sample = 2 - func(b *buffer, m message) error { - x := new(Sample) - pp := m.(*Profile) - pp.Sample = append(pp.Sample, x) - return decodeMessage(b, x) - }, - // repeated Mapping mapping = 3 - func(b *buffer, m message) error { - x := new(Mapping) - pp := m.(*Profile) - pp.Mapping = append(pp.Mapping, x) - return decodeMessage(b, x) - }, - // repeated Location location = 4 - func(b *buffer, m message) error { - x := new(Location) - pp := m.(*Profile) - pp.Location = append(pp.Location, x) - return decodeMessage(b, x) - }, - // repeated Function function = 5 - func(b *buffer, m message) error { - x := new(Function) - pp := m.(*Profile) - pp.Function = append(pp.Function, x) - return decodeMessage(b, x) - }, - // repeated string string_table = 6 - func(b *buffer, m message) error { - err := decodeStrings(b, &m.(*Profile).stringTable) - if err != nil { - return err - } - if *&m.(*Profile).stringTable[0] != "" { - return errors.New("string_table[0] must be ''") - } - return nil - }, - // repeated int64 drop_frames = 7 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).dropFramesX) }, - // repeated int64 keep_frames = 8 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).keepFramesX) }, - // repeated int64 time_nanos = 9 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).TimeNanos) }, - // repeated int64 duration_nanos = 10 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).DurationNanos) }, - // optional string period_type = 11 - func(b *buffer, m message) error { - x := new(ValueType) - pp := m.(*Profile) - pp.PeriodType = x - return decodeMessage(b, x) - }, - // repeated int64 period = 12 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).Period) }, -} - -// postDecode takes the unexported fields populated by decode (with -// suffix X) and populates the corresponding exported fields. -// The unexported fields are cleared up to facilitate testing. -func (p *Profile) postDecode() error { - var err error - - mappings := make(map[uint64]*Mapping) - for _, m := range p.Mapping { - m.File, err = getString(p.stringTable, &m.fileX, err) - m.BuildID, err = getString(p.stringTable, &m.buildIDX, err) - mappings[m.ID] = m - } - - functions := make(map[uint64]*Function) - for _, f := range p.Function { - f.Name, err = getString(p.stringTable, &f.nameX, err) - f.SystemName, err = getString(p.stringTable, &f.systemNameX, err) - f.Filename, err = getString(p.stringTable, &f.filenameX, err) - functions[f.ID] = f - } - - locations := make(map[uint64]*Location) - for _, l := range p.Location { - l.Mapping = mappings[l.mappingIDX] - l.mappingIDX = 0 - for i, ln := range l.Line { - if id := ln.functionIDX; id != 0 { - l.Line[i].Function = functions[id] - if l.Line[i].Function == nil { - return fmt.Errorf("Function ID %d not found", id) - } - l.Line[i].functionIDX = 0 - } - } - locations[l.ID] = l - } - - for _, st := range p.SampleType { - st.Type, err = getString(p.stringTable, &st.typeX, err) - st.Unit, err = getString(p.stringTable, &st.unitX, err) - } - - for _, s := range p.Sample { - labels := make(map[string][]string) - numLabels := make(map[string][]int64) - for _, l := range s.labelX { - var key, value string - key, err = getString(p.stringTable, &l.keyX, err) - if l.strX != 0 { - value, err = getString(p.stringTable, &l.strX, err) - labels[key] = append(labels[key], value) - } else { - numLabels[key] = append(numLabels[key], l.numX) - } - } - if len(labels) > 0 { - s.Label = labels - } - if len(numLabels) > 0 { - s.NumLabel = numLabels - } - s.Location = nil - for _, lid := range s.locationIDX { - s.Location = append(s.Location, locations[lid]) - } - s.locationIDX = nil - } - - p.DropFrames, err = getString(p.stringTable, &p.dropFramesX, err) - p.KeepFrames, err = getString(p.stringTable, &p.keepFramesX, err) - - if pt := p.PeriodType; pt == nil { - p.PeriodType = &ValueType{} - } - - if pt := p.PeriodType; pt != nil { - pt.Type, err = getString(p.stringTable, &pt.typeX, err) - pt.Unit, err = getString(p.stringTable, &pt.unitX, err) - } - p.stringTable = nil - return nil -} - -func (p *ValueType) decoder() []decoder { - return valueTypeDecoder -} - -func (p *ValueType) encode(b *buffer) { - encodeInt64Opt(b, 1, p.typeX) - encodeInt64Opt(b, 2, p.unitX) -} - -var valueTypeDecoder = []decoder{ - nil, // 0 - // optional int64 type = 1 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).typeX) }, - // optional int64 unit = 2 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).unitX) }, -} - -func (p *Sample) decoder() []decoder { - return sampleDecoder -} - -func (p *Sample) encode(b *buffer) { - encodeUint64s(b, 1, p.locationIDX) - for _, x := range p.Value { - encodeInt64(b, 2, x) - } - for _, x := range p.labelX { - encodeMessage(b, 3, x) - } -} - -var sampleDecoder = []decoder{ - nil, // 0 - // repeated uint64 location = 1 - func(b *buffer, m message) error { return decodeUint64s(b, &m.(*Sample).locationIDX) }, - // repeated int64 value = 2 - func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Sample).Value) }, - // repeated Label label = 3 - func(b *buffer, m message) error { - s := m.(*Sample) - n := len(s.labelX) - s.labelX = append(s.labelX, Label{}) - return decodeMessage(b, &s.labelX[n]) - }, -} - -func (p Label) decoder() []decoder { - return labelDecoder -} - -func (p Label) encode(b *buffer) { - encodeInt64Opt(b, 1, p.keyX) - encodeInt64Opt(b, 2, p.strX) - encodeInt64Opt(b, 3, p.numX) -} - -var labelDecoder = []decoder{ - nil, // 0 - // optional int64 key = 1 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).keyX) }, - // optional int64 str = 2 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).strX) }, - // optional int64 num = 3 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).numX) }, -} - -func (p *Mapping) decoder() []decoder { - return mappingDecoder -} - -func (p *Mapping) encode(b *buffer) { - encodeUint64Opt(b, 1, p.ID) - encodeUint64Opt(b, 2, p.Start) - encodeUint64Opt(b, 3, p.Limit) - encodeUint64Opt(b, 4, p.Offset) - encodeInt64Opt(b, 5, p.fileX) - encodeInt64Opt(b, 6, p.buildIDX) - encodeBoolOpt(b, 7, p.HasFunctions) - encodeBoolOpt(b, 8, p.HasFilenames) - encodeBoolOpt(b, 9, p.HasLineNumbers) - encodeBoolOpt(b, 10, p.HasInlineFrames) -} - -var mappingDecoder = []decoder{ - nil, // 0 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).ID) }, // optional uint64 id = 1 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Start) }, // optional uint64 memory_offset = 2 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Limit) }, // optional uint64 memory_limit = 3 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Offset) }, // optional uint64 file_offset = 4 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).fileX) }, // optional int64 filename = 5 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).buildIDX) }, // optional int64 build_id = 6 - func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFunctions) }, // optional bool has_functions = 7 - func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFilenames) }, // optional bool has_filenames = 8 - func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasLineNumbers) }, // optional bool has_line_numbers = 9 - func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasInlineFrames) }, // optional bool has_inline_frames = 10 -} - -func (p *Location) decoder() []decoder { - return locationDecoder -} - -func (p *Location) encode(b *buffer) { - encodeUint64Opt(b, 1, p.ID) - encodeUint64Opt(b, 2, p.mappingIDX) - encodeUint64Opt(b, 3, p.Address) - for i := range p.Line { - encodeMessage(b, 4, &p.Line[i]) - } -} - -var locationDecoder = []decoder{ - nil, // 0 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).ID) }, // optional uint64 id = 1; - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).mappingIDX) }, // optional uint64 mapping_id = 2; - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).Address) }, // optional uint64 address = 3; - func(b *buffer, m message) error { // repeated Line line = 4 - pp := m.(*Location) - n := len(pp.Line) - pp.Line = append(pp.Line, Line{}) - return decodeMessage(b, &pp.Line[n]) - }, -} - -func (p *Line) decoder() []decoder { - return lineDecoder -} - -func (p *Line) encode(b *buffer) { - encodeUint64Opt(b, 1, p.functionIDX) - encodeInt64Opt(b, 2, p.Line) -} - -var lineDecoder = []decoder{ - nil, // 0 - // optional uint64 function_id = 1 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Line).functionIDX) }, - // optional int64 line = 2 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Line).Line) }, -} - -func (p *Function) decoder() []decoder { - return functionDecoder -} - -func (p *Function) encode(b *buffer) { - encodeUint64Opt(b, 1, p.ID) - encodeInt64Opt(b, 2, p.nameX) - encodeInt64Opt(b, 3, p.systemNameX) - encodeInt64Opt(b, 4, p.filenameX) - encodeInt64Opt(b, 5, p.StartLine) -} - -var functionDecoder = []decoder{ - nil, // 0 - // optional uint64 id = 1 - func(b *buffer, m message) error { return decodeUint64(b, &m.(*Function).ID) }, - // optional int64 function_name = 2 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).nameX) }, - // optional int64 function_system_name = 3 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).systemNameX) }, - // repeated int64 filename = 4 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).filenameX) }, - // optional int64 start_line = 5 - func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).StartLine) }, -} - -func addString(strings map[string]int, s string) int64 { - i, ok := strings[s] - if !ok { - i = len(strings) - strings[s] = i - } - return int64(i) -} - -func getString(strings []string, strng *int64, err error) (string, error) { - if err != nil { - return "", err - } - s := int(*strng) - if s < 0 || s >= len(strings) { - return "", errMalformed - } - *strng = 0 - return strings[s], nil -} diff --git a/src/cmd/internal/pprof/profile/filter.go b/src/cmd/internal/pprof/profile/filter.go deleted file mode 100644 index 1baa096a49..0000000000 --- a/src/cmd/internal/pprof/profile/filter.go +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Implements methods to filter samples from profiles. - -package profile - -import "regexp" - -// FilterSamplesByName filters the samples in a profile and only keeps -// samples where at least one frame matches focus but none match ignore. -// Returns true is the corresponding regexp matched at least one sample. -func (p *Profile) FilterSamplesByName(focus, ignore, hide *regexp.Regexp) (fm, im, hm bool) { - focusOrIgnore := make(map[uint64]bool) - hidden := make(map[uint64]bool) - for _, l := range p.Location { - if ignore != nil && l.matchesName(ignore) { - im = true - focusOrIgnore[l.ID] = false - } else if focus == nil || l.matchesName(focus) { - fm = true - focusOrIgnore[l.ID] = true - } - if hide != nil && l.matchesName(hide) { - hm = true - l.Line = l.unmatchedLines(hide) - if len(l.Line) == 0 { - hidden[l.ID] = true - } - } - } - - s := make([]*Sample, 0, len(p.Sample)) - for _, sample := range p.Sample { - if focusedAndNotIgnored(sample.Location, focusOrIgnore) { - if len(hidden) > 0 { - var locs []*Location - for _, loc := range sample.Location { - if !hidden[loc.ID] { - locs = append(locs, loc) - } - } - if len(locs) == 0 { - // Remove sample with no locations (by not adding it to s). - continue - } - sample.Location = locs - } - s = append(s, sample) - } - } - p.Sample = s - - return -} - -// matchesName returns whether the function name or file in the -// location matches the regular expression. -func (loc *Location) matchesName(re *regexp.Regexp) bool { - for _, ln := range loc.Line { - if fn := ln.Function; fn != nil { - if re.MatchString(fn.Name) { - return true - } - if re.MatchString(fn.Filename) { - return true - } - } - } - return false -} - -// unmatchedLines returns the lines in the location that do not match -// the regular expression. -func (loc *Location) unmatchedLines(re *regexp.Regexp) []Line { - var lines []Line - for _, ln := range loc.Line { - if fn := ln.Function; fn != nil { - if re.MatchString(fn.Name) { - continue - } - if re.MatchString(fn.Filename) { - continue - } - } - lines = append(lines, ln) - } - return lines -} - -// focusedAndNotIgnored looks up a slice of ids against a map of -// focused/ignored locations. The map only contains locations that are -// explicitly focused or ignored. Returns whether there is at least -// one focused location but no ignored locations. -func focusedAndNotIgnored(locs []*Location, m map[uint64]bool) bool { - var f bool - for _, loc := range locs { - if focus, focusOrIgnore := m[loc.ID]; focusOrIgnore { - if focus { - // Found focused location. Must keep searching in case there - // is an ignored one as well. - f = true - } else { - // Found ignored location. Can return false right away. - return false - } - } - } - return f -} - -// TagMatch selects tags for filtering -type TagMatch func(key, val string, nval int64) bool - -// FilterSamplesByTag removes all samples from the profile, except -// those that match focus and do not match the ignore regular -// expression. -func (p *Profile) FilterSamplesByTag(focus, ignore TagMatch) (fm, im bool) { - samples := make([]*Sample, 0, len(p.Sample)) - for _, s := range p.Sample { - focused, ignored := focusedSample(s, focus, ignore) - fm = fm || focused - im = im || ignored - if focused && !ignored { - samples = append(samples, s) - } - } - p.Sample = samples - return -} - -// focusedTag checks a sample against focus and ignore regexps. -// Returns whether the focus/ignore regexps match any tags -func focusedSample(s *Sample, focus, ignore TagMatch) (fm, im bool) { - fm = focus == nil - for key, vals := range s.Label { - for _, val := range vals { - if ignore != nil && ignore(key, val, 0) { - im = true - } - if !fm && focus(key, val, 0) { - fm = true - } - } - } - for key, vals := range s.NumLabel { - for _, val := range vals { - if ignore != nil && ignore(key, "", val) { - im = true - } - if !fm && focus(key, "", val) { - fm = true - } - } - } - return fm, im -} diff --git a/src/cmd/internal/pprof/profile/legacy_profile.go b/src/cmd/internal/pprof/profile/legacy_profile.go deleted file mode 100644 index 5ad3e25640..0000000000 --- a/src/cmd/internal/pprof/profile/legacy_profile.go +++ /dev/null @@ -1,1250 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file implements parsers to convert legacy profiles into the -// profile.proto format. - -package profile - -import ( - "bufio" - "bytes" - "fmt" - "io" - "math" - "regexp" - "strconv" - "strings" -) - -var ( - countStartRE = regexp.MustCompile(`\A(\w+) profile: total \d+\n\z`) - countRE = regexp.MustCompile(`\A(\d+) @(( 0x[0-9a-f]+)+)\n\z`) - - heapHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`) - heapSampleRE = regexp.MustCompile(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`) - - contentionSampleRE = regexp.MustCompile(`(\d+) *(\d+) @([ x0-9a-f]*)`) - - hexNumberRE = regexp.MustCompile(`0x[0-9a-f]+`) - - growthHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ growthz`) - - fragmentationHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz`) - - threadzStartRE = regexp.MustCompile(`--- threadz \d+ ---`) - threadStartRE = regexp.MustCompile(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`) - - procMapsRE = regexp.MustCompile(`([[:xdigit:]]+)-([[:xdigit:]]+)\s+([-rwxp]+)\s+([[:xdigit:]]+)\s+([[:xdigit:]]+):([[:xdigit:]]+)\s+([[:digit:]]+)\s*(\S+)?`) - - briefMapsRE = regexp.MustCompile(`\s*([[:xdigit:]]+)-([[:xdigit:]]+):\s*(\S+)(\s.*@)?([[:xdigit:]]+)?`) - - // LegacyHeapAllocated instructs the heapz parsers to use the - // allocated memory stats instead of the default in-use memory. Note - // that tcmalloc doesn't provide all allocated memory, only in-use - // stats. - LegacyHeapAllocated bool -) - -func isSpaceOrComment(line string) bool { - trimmed := strings.TrimSpace(line) - return len(trimmed) == 0 || trimmed[0] == '#' -} - -// parseGoCount parses a Go count profile (e.g., threadcreate or -// goroutine) and returns a new Profile. -func parseGoCount(b []byte) (*Profile, error) { - r := bytes.NewBuffer(b) - - var line string - var err error - for { - // Skip past comments and empty lines seeking a real header. - line, err = r.ReadString('\n') - if err != nil { - return nil, err - } - if !isSpaceOrComment(line) { - break - } - } - - m := countStartRE.FindStringSubmatch(line) - if m == nil { - return nil, errUnrecognized - } - profileType := m[1] - p := &Profile{ - PeriodType: &ValueType{Type: profileType, Unit: "count"}, - Period: 1, - SampleType: []*ValueType{{Type: profileType, Unit: "count"}}, - } - locations := make(map[uint64]*Location) - for { - line, err = r.ReadString('\n') - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - if isSpaceOrComment(line) { - continue - } - if strings.HasPrefix(line, "---") { - break - } - m := countRE.FindStringSubmatch(line) - if m == nil { - return nil, errMalformed - } - n, err := strconv.ParseInt(m[1], 0, 64) - if err != nil { - return nil, errMalformed - } - fields := strings.Fields(m[2]) - locs := make([]*Location, 0, len(fields)) - for _, stk := range fields { - addr, err := strconv.ParseUint(stk, 0, 64) - if err != nil { - return nil, errMalformed - } - // Adjust all frames by -1 to land on the call instruction. - addr-- - loc := locations[addr] - if loc == nil { - loc = &Location{ - Address: addr, - } - locations[addr] = loc - p.Location = append(p.Location, loc) - } - locs = append(locs, loc) - } - p.Sample = append(p.Sample, &Sample{ - Location: locs, - Value: []int64{n}, - }) - } - - if err = parseAdditionalSections(strings.TrimSpace(line), r, p); err != nil { - return nil, err - } - return p, nil -} - -// remapLocationIDs ensures there is a location for each address -// referenced by a sample, and remaps the samples to point to the new -// location ids. -func (p *Profile) remapLocationIDs() { - seen := make(map[*Location]bool, len(p.Location)) - var locs []*Location - - for _, s := range p.Sample { - for _, l := range s.Location { - if seen[l] { - continue - } - l.ID = uint64(len(locs) + 1) - locs = append(locs, l) - seen[l] = true - } - } - p.Location = locs -} - -func (p *Profile) remapFunctionIDs() { - seen := make(map[*Function]bool, len(p.Function)) - var fns []*Function - - for _, l := range p.Location { - for _, ln := range l.Line { - fn := ln.Function - if fn == nil || seen[fn] { - continue - } - fn.ID = uint64(len(fns) + 1) - fns = append(fns, fn) - seen[fn] = true - } - } - p.Function = fns -} - -// remapMappingIDs matches location addresses with existing mappings -// and updates them appropriately. This is O(N*M), if this ever shows -// up as a bottleneck, evaluate sorting the mappings and doing a -// binary search, which would make it O(N*log(M)). -func (p *Profile) remapMappingIDs() { - if len(p.Mapping) == 0 { - return - } - - // Some profile handlers will incorrectly set regions for the main - // executable if its section is remapped. Fix them through heuristics. - - // Remove the initial mapping if named '/anon_hugepage' and has a - // consecutive adjacent mapping. - if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") { - if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start { - p.Mapping = p.Mapping[1:] - } - } - - // Subtract the offset from the start of the main mapping if it - // ends up at a recognizable start address. - const expectedStart = 0x400000 - if m := p.Mapping[0]; m.Start-m.Offset == expectedStart { - m.Start = expectedStart - m.Offset = 0 - } - - for _, l := range p.Location { - if a := l.Address; a != 0 { - for _, m := range p.Mapping { - if m.Start <= a && a < m.Limit { - l.Mapping = m - break - } - } - } - } - - // Reset all mapping IDs. - for i, m := range p.Mapping { - m.ID = uint64(i + 1) - } -} - -var cpuInts = []func([]byte) (uint64, []byte){ - get32l, - get32b, - get64l, - get64b, -} - -func get32l(b []byte) (uint64, []byte) { - if len(b) < 4 { - return 0, nil - } - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24, b[4:] -} - -func get32b(b []byte) (uint64, []byte) { - if len(b) < 4 { - return 0, nil - } - return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24, b[4:] -} - -func get64l(b []byte) (uint64, []byte) { - if len(b) < 8 { - return 0, nil - } - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:] -} - -func get64b(b []byte) (uint64, []byte) { - if len(b) < 8 { - return 0, nil - } - return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:] -} - -// ParseTracebacks parses a set of tracebacks and returns a newly -// populated profile. It will accept any text file and generate a -// Profile out of it with any hex addresses it can identify, including -// a process map if it can recognize one. Each sample will include a -// tag "source" with the addresses recognized in string format. -func ParseTracebacks(b []byte) (*Profile, error) { - r := bytes.NewBuffer(b) - - p := &Profile{ - PeriodType: &ValueType{Type: "trace", Unit: "count"}, - Period: 1, - SampleType: []*ValueType{ - {Type: "trace", Unit: "count"}, - }, - } - - var sources []string - var sloc []*Location - - locs := make(map[uint64]*Location) - for { - l, err := r.ReadString('\n') - if err != nil { - if err != io.EOF { - return nil, err - } - if l == "" { - break - } - } - if sectionTrigger(l) == memoryMapSection { - break - } - if s, addrs := extractHexAddresses(l); len(s) > 0 { - for _, addr := range addrs { - // Addresses from stack traces point to the next instruction after - // each call. Adjust by -1 to land somewhere on the actual call. - addr-- - loc := locs[addr] - if locs[addr] == nil { - loc = &Location{ - Address: addr, - } - p.Location = append(p.Location, loc) - locs[addr] = loc - } - sloc = append(sloc, loc) - } - - sources = append(sources, s...) - } else { - if len(sources) > 0 || len(sloc) > 0 { - addTracebackSample(sloc, sources, p) - sloc, sources = nil, nil - } - } - } - - // Add final sample to save any leftover data. - if len(sources) > 0 || len(sloc) > 0 { - addTracebackSample(sloc, sources, p) - } - - if err := p.ParseMemoryMap(r); err != nil { - return nil, err - } - return p, nil -} - -func addTracebackSample(l []*Location, s []string, p *Profile) { - p.Sample = append(p.Sample, - &Sample{ - Value: []int64{1}, - Location: l, - Label: map[string][]string{"source": s}, - }) -} - -// parseCPU parses a profilez legacy profile and returns a newly -// populated Profile. -// -// The general format for profilez samples is a sequence of words in -// binary format. The first words are a header with the following data: -// 1st word -- 0 -// 2nd word -- 3 -// 3rd word -- 0 if a c++ application, 1 if a java application. -// 4th word -- Sampling period (in microseconds). -// 5th word -- Padding. -func parseCPU(b []byte) (*Profile, error) { - var parse func([]byte) (uint64, []byte) - var n1, n2, n3, n4, n5 uint64 - for _, parse = range cpuInts { - var tmp []byte - n1, tmp = parse(b) - n2, tmp = parse(tmp) - n3, tmp = parse(tmp) - n4, tmp = parse(tmp) - n5, tmp = parse(tmp) - - if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 == 0 { - b = tmp - return cpuProfile(b, int64(n4), parse) - } - } - return nil, errUnrecognized -} - -// cpuProfile returns a new Profile from C++ profilez data. -// b is the profile bytes after the header, period is the profiling -// period, and parse is a function to parse 8-byte chunks from the -// profile in its native endianness. -func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) { - p := &Profile{ - Period: period * 1000, - PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"}, - SampleType: []*ValueType{ - {Type: "samples", Unit: "count"}, - {Type: "cpu", Unit: "nanoseconds"}, - }, - } - var err error - if b, _, err = parseCPUSamples(b, parse, true, p); err != nil { - return nil, err - } - - // If all samples have the same second-to-the-bottom frame, it - // strongly suggests that it is an uninteresting artifact of - // measurement -- a stack frame pushed by the signal handler. The - // bottom frame is always correct as it is picked up from the signal - // structure, not the stack. Check if this is the case and if so, - // remove. - if len(p.Sample) > 1 && len(p.Sample[0].Location) > 1 { - allSame := true - id1 := p.Sample[0].Location[1].Address - for _, s := range p.Sample { - if len(s.Location) < 2 || id1 != s.Location[1].Address { - allSame = false - break - } - } - if allSame { - for _, s := range p.Sample { - s.Location = append(s.Location[:1], s.Location[2:]...) - } - } - } - - if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil { - return nil, err - } - return p, nil -} - -// parseCPUSamples parses a collection of profilez samples from a -// profile. -// -// profilez samples are a repeated sequence of stack frames of the -// form: -// 1st word -- The number of times this stack was encountered. -// 2nd word -- The size of the stack (StackSize). -// 3rd word -- The first address on the stack. -// ... -// StackSize + 2 -- The last address on the stack -// The last stack trace is of the form: -// 1st word -- 0 -// 2nd word -- 1 -// 3rd word -- 0 -// -// Addresses from stack traces may point to the next instruction after -// each call. Optionally adjust by -1 to land somewhere on the actual -// call (except for the leaf, which is not a call). -func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust bool, p *Profile) ([]byte, map[uint64]*Location, error) { - locs := make(map[uint64]*Location) - for len(b) > 0 { - var count, nstk uint64 - count, b = parse(b) - nstk, b = parse(b) - if b == nil || nstk > uint64(len(b)/4) { - return nil, nil, errUnrecognized - } - var sloc []*Location - addrs := make([]uint64, nstk) - for i := 0; i < int(nstk); i++ { - addrs[i], b = parse(b) - } - - if count == 0 && nstk == 1 && addrs[0] == 0 { - // End of data marker - break - } - for i, addr := range addrs { - if adjust && i > 0 { - addr-- - } - loc := locs[addr] - if loc == nil { - loc = &Location{ - Address: addr, - } - locs[addr] = loc - p.Location = append(p.Location, loc) - } - sloc = append(sloc, loc) - } - p.Sample = append(p.Sample, - &Sample{ - Value: []int64{int64(count), int64(count) * p.Period}, - Location: sloc, - }) - } - // Reached the end without finding the EOD marker. - return b, locs, nil -} - -// parseHeap parses a heapz legacy or a growthz profile and -// returns a newly populated Profile. -func parseHeap(b []byte) (p *Profile, err error) { - r := bytes.NewBuffer(b) - l, err := r.ReadString('\n') - if err != nil { - return nil, errUnrecognized - } - - sampling := "" - - if header := heapHeaderRE.FindStringSubmatch(l); header != nil { - p = &Profile{ - SampleType: []*ValueType{ - {Type: "objects", Unit: "count"}, - {Type: "space", Unit: "bytes"}, - }, - PeriodType: &ValueType{Type: "objects", Unit: "bytes"}, - } - - var period int64 - if len(header[6]) > 0 { - if period, err = strconv.ParseInt(header[6], 10, 64); err != nil { - return nil, errUnrecognized - } - } - - switch header[5] { - case "heapz_v2", "heap_v2": - sampling, p.Period = "v2", period - case "heapprofile": - sampling, p.Period = "", 1 - case "heap": - sampling, p.Period = "v2", period/2 - default: - return nil, errUnrecognized - } - } else if header = growthHeaderRE.FindStringSubmatch(l); header != nil { - p = &Profile{ - SampleType: []*ValueType{ - {Type: "objects", Unit: "count"}, - {Type: "space", Unit: "bytes"}, - }, - PeriodType: &ValueType{Type: "heapgrowth", Unit: "count"}, - Period: 1, - } - } else if header = fragmentationHeaderRE.FindStringSubmatch(l); header != nil { - p = &Profile{ - SampleType: []*ValueType{ - {Type: "objects", Unit: "count"}, - {Type: "space", Unit: "bytes"}, - }, - PeriodType: &ValueType{Type: "allocations", Unit: "count"}, - Period: 1, - } - } else { - return nil, errUnrecognized - } - - if LegacyHeapAllocated { - for _, st := range p.SampleType { - st.Type = "alloc_" + st.Type - } - } else { - for _, st := range p.SampleType { - st.Type = "inuse_" + st.Type - } - } - - locs := make(map[uint64]*Location) - for { - l, err = r.ReadString('\n') - if err != nil { - if err != io.EOF { - return nil, err - } - - if l == "" { - break - } - } - - if isSpaceOrComment(l) { - continue - } - l = strings.TrimSpace(l) - - if sectionTrigger(l) != unrecognizedSection { - break - } - - value, blocksize, addrs, err := parseHeapSample(l, p.Period, sampling) - if err != nil { - return nil, err - } - var sloc []*Location - for _, addr := range addrs { - // Addresses from stack traces point to the next instruction after - // each call. Adjust by -1 to land somewhere on the actual call. - addr-- - loc := locs[addr] - if locs[addr] == nil { - loc = &Location{ - Address: addr, - } - p.Location = append(p.Location, loc) - locs[addr] = loc - } - sloc = append(sloc, loc) - } - - p.Sample = append(p.Sample, &Sample{ - Value: value, - Location: sloc, - NumLabel: map[string][]int64{"bytes": {blocksize}}, - }) - } - - if err = parseAdditionalSections(l, r, p); err != nil { - return nil, err - } - return p, nil -} - -// parseHeapSample parses a single row from a heap profile into a new Sample. -func parseHeapSample(line string, rate int64, sampling string) (value []int64, blocksize int64, addrs []uint64, err error) { - sampleData := heapSampleRE.FindStringSubmatch(line) - if len(sampleData) != 6 { - return value, blocksize, addrs, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData)) - } - - // Use first two values by default; tcmalloc sampling generates the - // same value for both, only the older heap-profile collect separate - // stats for in-use and allocated objects. - valueIndex := 1 - if LegacyHeapAllocated { - valueIndex = 3 - } - - var v1, v2 int64 - if v1, err = strconv.ParseInt(sampleData[valueIndex], 10, 64); err != nil { - return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) - } - if v2, err = strconv.ParseInt(sampleData[valueIndex+1], 10, 64); err != nil { - return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) - } - - if v1 == 0 { - if v2 != 0 { - return value, blocksize, addrs, fmt.Errorf("allocation count was 0 but allocation bytes was %d", v2) - } - } else { - blocksize = v2 / v1 - if sampling == "v2" { - v1, v2 = scaleHeapSample(v1, v2, rate) - } - } - - value = []int64{v1, v2} - addrs = parseHexAddresses(sampleData[5]) - - return value, blocksize, addrs, nil -} - -// extractHexAddresses extracts hex numbers from a string and returns -// them, together with their numeric value, in a slice. -func extractHexAddresses(s string) ([]string, []uint64) { - hexStrings := hexNumberRE.FindAllString(s, -1) - var ids []uint64 - for _, s := range hexStrings { - if id, err := strconv.ParseUint(s, 0, 64); err == nil { - ids = append(ids, id) - } else { - // Do not expect any parsing failures due to the regexp matching. - panic("failed to parse hex value:" + s) - } - } - return hexStrings, ids -} - -// parseHexAddresses parses hex numbers from a string and returns them -// in a slice. -func parseHexAddresses(s string) []uint64 { - _, ids := extractHexAddresses(s) - return ids -} - -// scaleHeapSample adjusts the data from a heapz Sample to -// account for its probability of appearing in the collected -// data. heapz profiles are a sampling of the memory allocations -// requests in a program. We estimate the unsampled value by dividing -// each collected sample by its probability of appearing in the -// profile. heapz v2 profiles rely on a poisson process to determine -// which samples to collect, based on the desired average collection -// rate R. The probability of a sample of size S to appear in that -// profile is 1-exp(-S/R). -func scaleHeapSample(count, size, rate int64) (int64, int64) { - if count == 0 || size == 0 { - return 0, 0 - } - - if rate <= 1 { - // if rate==1 all samples were collected so no adjustment is needed. - // if rate<1 treat as unknown and skip scaling. - return count, size - } - - avgSize := float64(size) / float64(count) - scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) - - return int64(float64(count) * scale), int64(float64(size) * scale) -} - -// parseContention parses a mutex or contention profile. There are 2 cases: -// "--- contentionz " for legacy C++ profiles (and backwards compatibility) -// "--- mutex:" or "--- contention:" for profiles generated by the Go runtime. -// This code converts the text output from runtime into a *Profile. (In the future -// the runtime might write a serialized Profile directly making this unnecessary.) -func parseContention(b []byte) (*Profile, error) { - r := bytes.NewBuffer(b) - l, err := r.ReadString('\n') - if err != nil { - return nil, errUnrecognized - } - if strings.HasPrefix(l, "--- contentionz ") { - return parseCppContention(r) - } else if strings.HasPrefix(l, "--- mutex:") { - return parseCppContention(r) - } else if strings.HasPrefix(l, "--- contention:") { - return parseCppContention(r) - } - return nil, errUnrecognized -} - -// parseCppContention parses the output from synchronization_profiling.cc -// for backward compatibility, and the compatible (non-debug) block profile -// output from the Go runtime. -func parseCppContention(r *bytes.Buffer) (*Profile, error) { - p := &Profile{ - PeriodType: &ValueType{Type: "contentions", Unit: "count"}, - Period: 1, - SampleType: []*ValueType{ - {Type: "contentions", Unit: "count"}, - {Type: "delay", Unit: "nanoseconds"}, - }, - } - - var cpuHz int64 - var l string - var err error - // Parse text of the form "attribute = value" before the samples. - const delimiter = "=" - for { - l, err = r.ReadString('\n') - if err != nil { - if err != io.EOF { - return nil, err - } - - if l == "" { - break - } - } - - if l = strings.TrimSpace(l); l == "" { - continue - } - - if strings.HasPrefix(l, "---") { - break - } - - attr := strings.SplitN(l, delimiter, 2) - if len(attr) != 2 { - break - } - key, val := strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]) - var err error - switch key { - case "cycles/second": - if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil { - return nil, errUnrecognized - } - case "sampling period": - if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil { - return nil, errUnrecognized - } - case "ms since reset": - ms, err := strconv.ParseInt(val, 0, 64) - if err != nil { - return nil, errUnrecognized - } - p.DurationNanos = ms * 1000 * 1000 - case "format": - // CPP contentionz profiles don't have format. - return nil, errUnrecognized - case "resolution": - // CPP contentionz profiles don't have resolution. - return nil, errUnrecognized - case "discarded samples": - default: - return nil, errUnrecognized - } - } - - locs := make(map[uint64]*Location) - for { - if l = strings.TrimSpace(l); strings.HasPrefix(l, "---") { - break - } - value, addrs, err := parseContentionSample(l, p.Period, cpuHz) - if err != nil { - return nil, err - } - var sloc []*Location - for _, addr := range addrs { - // Addresses from stack traces point to the next instruction after - // each call. Adjust by -1 to land somewhere on the actual call. - addr-- - loc := locs[addr] - if locs[addr] == nil { - loc = &Location{ - Address: addr, - } - p.Location = append(p.Location, loc) - locs[addr] = loc - } - sloc = append(sloc, loc) - } - p.Sample = append(p.Sample, &Sample{ - Value: value, - Location: sloc, - }) - - if l, err = r.ReadString('\n'); err != nil { - if err != io.EOF { - return nil, err - } - if l == "" { - break - } - } - } - - if err = parseAdditionalSections(l, r, p); err != nil { - return nil, err - } - - return p, nil -} - -// parseContentionSample parses a single row from a contention profile -// into a new Sample. -func parseContentionSample(line string, period, cpuHz int64) (value []int64, addrs []uint64, err error) { - sampleData := contentionSampleRE.FindStringSubmatch(line) - if sampleData == nil { - return value, addrs, errUnrecognized - } - - v1, err := strconv.ParseInt(sampleData[1], 10, 64) - if err != nil { - return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) - } - v2, err := strconv.ParseInt(sampleData[2], 10, 64) - if err != nil { - return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) - } - - // Unsample values if period and cpuHz are available. - // - Delays are scaled to cycles and then to nanoseconds. - // - Contentions are scaled to cycles. - if period > 0 { - if cpuHz > 0 { - cpuGHz := float64(cpuHz) / 1e9 - v1 = int64(float64(v1) * float64(period) / cpuGHz) - } - v2 = v2 * period - } - - value = []int64{v2, v1} - addrs = parseHexAddresses(sampleData[3]) - - return value, addrs, nil -} - -// parseThread parses a Threadz profile and returns a new Profile. -func parseThread(b []byte) (*Profile, error) { - r := bytes.NewBuffer(b) - - var line string - var err error - for { - // Skip past comments and empty lines seeking a real header. - line, err = r.ReadString('\n') - if err != nil { - return nil, err - } - if !isSpaceOrComment(line) { - break - } - } - - if m := threadzStartRE.FindStringSubmatch(line); m != nil { - // Advance over initial comments until first stack trace. - for { - line, err = r.ReadString('\n') - if err != nil { - if err != io.EOF { - return nil, err - } - - if line == "" { - break - } - } - if sectionTrigger(line) != unrecognizedSection || line[0] == '-' { - break - } - } - } else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { - return nil, errUnrecognized - } - - p := &Profile{ - SampleType: []*ValueType{{Type: "thread", Unit: "count"}}, - PeriodType: &ValueType{Type: "thread", Unit: "count"}, - Period: 1, - } - - locs := make(map[uint64]*Location) - // Recognize each thread and populate profile samples. - for sectionTrigger(line) == unrecognizedSection { - if strings.HasPrefix(line, "---- no stack trace for") { - line = "" - break - } - if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { - return nil, errUnrecognized - } - - var addrs []uint64 - line, addrs, err = parseThreadSample(r) - if err != nil { - return nil, errUnrecognized - } - if len(addrs) == 0 { - // We got a --same as previous threads--. Bump counters. - if len(p.Sample) > 0 { - s := p.Sample[len(p.Sample)-1] - s.Value[0]++ - } - continue - } - - var sloc []*Location - for _, addr := range addrs { - // Addresses from stack traces point to the next instruction after - // each call. Adjust by -1 to land somewhere on the actual call. - addr-- - loc := locs[addr] - if locs[addr] == nil { - loc = &Location{ - Address: addr, - } - p.Location = append(p.Location, loc) - locs[addr] = loc - } - sloc = append(sloc, loc) - } - - p.Sample = append(p.Sample, &Sample{ - Value: []int64{1}, - Location: sloc, - }) - } - - if err = parseAdditionalSections(line, r, p); err != nil { - return nil, err - } - - return p, nil -} - -// parseThreadSample parses a symbolized or unsymbolized stack trace. -// Returns the first line after the traceback, the sample (or nil if -// it hits a 'same-as-previous' marker) and an error. -func parseThreadSample(b *bytes.Buffer) (nextl string, addrs []uint64, err error) { - var l string - sameAsPrevious := false - for { - if l, err = b.ReadString('\n'); err != nil { - if err != io.EOF { - return "", nil, err - } - if l == "" { - break - } - } - if l = strings.TrimSpace(l); l == "" { - continue - } - - if strings.HasPrefix(l, "---") { - break - } - if strings.Contains(l, "same as previous thread") { - sameAsPrevious = true - continue - } - - addrs = append(addrs, parseHexAddresses(l)...) - } - - if sameAsPrevious { - return l, nil, nil - } - return l, addrs, nil -} - -// parseAdditionalSections parses any additional sections in the -// profile, ignoring any unrecognized sections. -func parseAdditionalSections(l string, b *bytes.Buffer, p *Profile) (err error) { - for { - if sectionTrigger(l) == memoryMapSection { - break - } - // Ignore any unrecognized sections. - if l, err := b.ReadString('\n'); err != nil { - if err != io.EOF { - return err - } - if l == "" { - break - } - } - } - return p.ParseMemoryMap(b) -} - -// ParseMemoryMap parses a memory map in the format of -// /proc/self/maps, and overrides the mappings in the current profile. -// It renumbers the samples and locations in the profile correspondingly. -func (p *Profile) ParseMemoryMap(rd io.Reader) error { - b := bufio.NewReader(rd) - - var attrs []string - var r *strings.Replacer - const delimiter = "=" - for { - l, err := b.ReadString('\n') - if err != nil { - if err != io.EOF { - return err - } - if l == "" { - break - } - } - if l = strings.TrimSpace(l); l == "" { - continue - } - - if r != nil { - l = r.Replace(l) - } - m, err := parseMappingEntry(l) - if err != nil { - if err == errUnrecognized { - // Recognize assignments of the form: attr=value, and replace - // $attr with value on subsequent mappings. - if attr := strings.SplitN(l, delimiter, 2); len(attr) == 2 { - attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1])) - r = strings.NewReplacer(attrs...) - } - // Ignore any unrecognized entries - continue - } - return err - } - if m == nil || (m.File == "" && len(p.Mapping) != 0) { - // In some cases the first entry may include the address range - // but not the name of the file. It should be followed by - // another entry with the name. - continue - } - if len(p.Mapping) == 1 && p.Mapping[0].File == "" { - // Update the name if this is the entry following that empty one. - p.Mapping[0].File = m.File - continue - } - p.Mapping = append(p.Mapping, m) - } - p.remapLocationIDs() - p.remapFunctionIDs() - p.remapMappingIDs() - return nil -} - -func parseMappingEntry(l string) (*Mapping, error) { - mapping := &Mapping{} - var err error - if me := procMapsRE.FindStringSubmatch(l); len(me) == 9 { - if !strings.Contains(me[3], "x") { - // Skip non-executable entries. - return nil, nil - } - if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil { - return nil, errUnrecognized - } - if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil { - return nil, errUnrecognized - } - if me[4] != "" { - if mapping.Offset, err = strconv.ParseUint(me[4], 16, 64); err != nil { - return nil, errUnrecognized - } - } - mapping.File = me[8] - return mapping, nil - } - - if me := briefMapsRE.FindStringSubmatch(l); len(me) == 6 { - if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil { - return nil, errUnrecognized - } - if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil { - return nil, errUnrecognized - } - mapping.File = me[3] - if me[5] != "" { - if mapping.Offset, err = strconv.ParseUint(me[5], 16, 64); err != nil { - return nil, errUnrecognized - } - } - return mapping, nil - } - - return nil, errUnrecognized -} - -type sectionType int - -const ( - unrecognizedSection sectionType = iota - memoryMapSection -) - -var memoryMapTriggers = []string{ - "--- Memory map: ---", - "MAPPED_LIBRARIES:", -} - -func sectionTrigger(line string) sectionType { - for _, trigger := range memoryMapTriggers { - if strings.Contains(line, trigger) { - return memoryMapSection - } - } - return unrecognizedSection -} - -func (p *Profile) addLegacyFrameInfo() { - switch { - case isProfileType(p, heapzSampleTypes) || - isProfileType(p, heapzInUseSampleTypes) || - isProfileType(p, heapzAllocSampleTypes): - p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr - case isProfileType(p, contentionzSampleTypes): - p.DropFrames, p.KeepFrames = lockRxStr, "" - default: - p.DropFrames, p.KeepFrames = cpuProfilerRxStr, "" - } -} - -var heapzSampleTypes = []string{"allocations", "size"} // early Go pprof profiles -var heapzInUseSampleTypes = []string{"inuse_objects", "inuse_space"} -var heapzAllocSampleTypes = []string{"alloc_objects", "alloc_space"} -var contentionzSampleTypes = []string{"contentions", "delay"} - -func isProfileType(p *Profile, t []string) bool { - st := p.SampleType - if len(st) != len(t) { - return false - } - - for i := range st { - if st[i].Type != t[i] { - return false - } - } - return true -} - -var allocRxStr = strings.Join([]string{ - // POSIX entry points. - `calloc`, - `cfree`, - `malloc`, - `free`, - `memalign`, - `do_memalign`, - `(__)?posix_memalign`, - `pvalloc`, - `valloc`, - `realloc`, - - // TC malloc. - `tcmalloc::.*`, - `tc_calloc`, - `tc_cfree`, - `tc_malloc`, - `tc_free`, - `tc_memalign`, - `tc_posix_memalign`, - `tc_pvalloc`, - `tc_valloc`, - `tc_realloc`, - `tc_new`, - `tc_delete`, - `tc_newarray`, - `tc_deletearray`, - `tc_new_nothrow`, - `tc_newarray_nothrow`, - - // Memory-allocation routines on OS X. - `malloc_zone_malloc`, - `malloc_zone_calloc`, - `malloc_zone_valloc`, - `malloc_zone_realloc`, - `malloc_zone_memalign`, - `malloc_zone_free`, - - // Go runtime - `runtime\..*`, - - // Other misc. memory allocation routines - `BaseArena::.*`, - `(::)?do_malloc_no_errno`, - `(::)?do_malloc_pages`, - `(::)?do_malloc`, - `DoSampledAllocation`, - `MallocedMemBlock::MallocedMemBlock`, - `_M_allocate`, - `__builtin_(vec_)?delete`, - `__builtin_(vec_)?new`, - `__gnu_cxx::new_allocator::allocate`, - `__libc_malloc`, - `__malloc_alloc_template::allocate`, - `allocate`, - `cpp_alloc`, - `operator new(\[\])?`, - `simple_alloc::allocate`, -}, `|`) - -var allocSkipRxStr = strings.Join([]string{ - // Preserve Go runtime frames that appear in the middle/bottom of - // the stack. - `runtime\.panic`, -}, `|`) - -var cpuProfilerRxStr = strings.Join([]string{ - `ProfileData::Add`, - `ProfileData::prof_handler`, - `CpuProfiler::prof_handler`, - `__pthread_sighandler`, - `__restore`, -}, `|`) - -var lockRxStr = strings.Join([]string{ - `RecordLockProfileData`, - `(base::)?RecordLockProfileData.*`, - `(base::)?SubmitMutexProfileData.*`, - `(base::)?SubmitSpinLockProfileData.*`, - `(Mutex::)?AwaitCommon.*`, - `(Mutex::)?Unlock.*`, - `(Mutex::)?UnlockSlow.*`, - `(Mutex::)?ReaderUnlock.*`, - `(MutexLock::)?~MutexLock.*`, - `(SpinLock::)?Unlock.*`, - `(SpinLock::)?SlowUnlock.*`, - `(SpinLockHolder::)?~SpinLockHolder.*`, -}, `|`) diff --git a/src/cmd/internal/pprof/profile/profile.go b/src/cmd/internal/pprof/profile/profile.go deleted file mode 100644 index 28e713d7be..0000000000 --- a/src/cmd/internal/pprof/profile/profile.go +++ /dev/null @@ -1,572 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package profile provides a representation of profile.proto and -// methods to encode/decode profiles in this format. -package profile - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "regexp" - "strings" - "time" -) - -// Profile is an in-memory representation of profile.proto. -type Profile struct { - SampleType []*ValueType - Sample []*Sample - Mapping []*Mapping - Location []*Location - Function []*Function - - DropFrames string - KeepFrames string - - TimeNanos int64 - DurationNanos int64 - PeriodType *ValueType - Period int64 - - dropFramesX int64 - keepFramesX int64 - stringTable []string -} - -// ValueType corresponds to Profile.ValueType -type ValueType struct { - Type string // cpu, wall, inuse_space, etc - Unit string // seconds, nanoseconds, bytes, etc - - typeX int64 - unitX int64 -} - -// Sample corresponds to Profile.Sample -type Sample struct { - Location []*Location - Value []int64 - Label map[string][]string - NumLabel map[string][]int64 - - locationIDX []uint64 - labelX []Label -} - -// Label corresponds to Profile.Label -type Label struct { - keyX int64 - // Exactly one of the two following values must be set - strX int64 - numX int64 // Integer value for this label -} - -// Mapping corresponds to Profile.Mapping -type Mapping struct { - ID uint64 - Start uint64 - Limit uint64 - Offset uint64 - File string - BuildID string - HasFunctions bool - HasFilenames bool - HasLineNumbers bool - HasInlineFrames bool - - fileX int64 - buildIDX int64 -} - -// Location corresponds to Profile.Location -type Location struct { - ID uint64 - Mapping *Mapping - Address uint64 - Line []Line - - mappingIDX uint64 -} - -// Line corresponds to Profile.Line -type Line struct { - Function *Function - Line int64 - - functionIDX uint64 -} - -// Function corresponds to Profile.Function -type Function struct { - ID uint64 - Name string - SystemName string - Filename string - StartLine int64 - - nameX int64 - systemNameX int64 - filenameX int64 -} - -// Parse parses a profile and checks for its validity. The input -// may be a gzip-compressed encoded protobuf or one of many legacy -// profile formats which may be unsupported in the future. -func Parse(r io.Reader) (*Profile, error) { - orig, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - - var p *Profile - if len(orig) >= 2 && orig[0] == 0x1f && orig[1] == 0x8b { - gz, err := gzip.NewReader(bytes.NewBuffer(orig)) - if err != nil { - return nil, fmt.Errorf("decompressing profile: %v", err) - } - data, err := ioutil.ReadAll(gz) - if err != nil { - return nil, fmt.Errorf("decompressing profile: %v", err) - } - orig = data - } - if p, err = parseUncompressed(orig); err != nil { - if p, err = parseLegacy(orig); err != nil { - return nil, fmt.Errorf("parsing profile: %v", err) - } - } - - if err := p.CheckValid(); err != nil { - return nil, fmt.Errorf("malformed profile: %v", err) - } - return p, nil -} - -var errUnrecognized = fmt.Errorf("unrecognized profile format") -var errMalformed = fmt.Errorf("malformed profile format") - -func parseLegacy(data []byte) (*Profile, error) { - parsers := []func([]byte) (*Profile, error){ - parseCPU, - parseHeap, - parseGoCount, // goroutine, threadcreate - parseThread, - parseContention, - } - - for _, parser := range parsers { - p, err := parser(data) - if err == nil { - p.setMain() - p.addLegacyFrameInfo() - return p, nil - } - if err != errUnrecognized { - return nil, err - } - } - return nil, errUnrecognized -} - -func parseUncompressed(data []byte) (*Profile, error) { - p := &Profile{} - if err := unmarshal(data, p); err != nil { - return nil, err - } - - if err := p.postDecode(); err != nil { - return nil, err - } - - return p, nil -} - -var libRx = regexp.MustCompile(`([.]so$|[.]so[._][0-9]+)`) - -// setMain scans Mapping entries and guesses which entry is main -// because legacy profiles don't obey the convention of putting main -// first. -func (p *Profile) setMain() { - for i := 0; i < len(p.Mapping); i++ { - file := strings.TrimSpace(strings.Replace(p.Mapping[i].File, "(deleted)", "", -1)) - if len(file) == 0 { - continue - } - if len(libRx.FindStringSubmatch(file)) > 0 { - continue - } - if strings.HasPrefix(file, "[") { - continue - } - // Swap what we guess is main to position 0. - tmp := p.Mapping[i] - p.Mapping[i] = p.Mapping[0] - p.Mapping[0] = tmp - break - } -} - -// Write writes the profile as a gzip-compressed marshaled protobuf. -func (p *Profile) Write(w io.Writer) error { - p.preEncode() - b := marshal(p) - zw := gzip.NewWriter(w) - defer zw.Close() - _, err := zw.Write(b) - return err -} - -// CheckValid tests whether the profile is valid. Checks include, but are -// not limited to: -// - len(Profile.Sample[n].value) == len(Profile.value_unit) -// - Sample.id has a corresponding Profile.Location -func (p *Profile) CheckValid() error { - // Check that sample values are consistent - sampleLen := len(p.SampleType) - if sampleLen == 0 && len(p.Sample) != 0 { - return fmt.Errorf("missing sample type information") - } - for _, s := range p.Sample { - if len(s.Value) != sampleLen { - return fmt.Errorf("mismatch: sample has: %d values vs. %d types", len(s.Value), len(p.SampleType)) - } - } - - // Check that all mappings/locations/functions are in the tables - // Check that there are no duplicate ids - mappings := make(map[uint64]*Mapping, len(p.Mapping)) - for _, m := range p.Mapping { - if m.ID == 0 { - return fmt.Errorf("found mapping with reserved ID=0") - } - if mappings[m.ID] != nil { - return fmt.Errorf("multiple mappings with same id: %d", m.ID) - } - mappings[m.ID] = m - } - functions := make(map[uint64]*Function, len(p.Function)) - for _, f := range p.Function { - if f.ID == 0 { - return fmt.Errorf("found function with reserved ID=0") - } - if functions[f.ID] != nil { - return fmt.Errorf("multiple functions with same id: %d", f.ID) - } - functions[f.ID] = f - } - locations := make(map[uint64]*Location, len(p.Location)) - for _, l := range p.Location { - if l.ID == 0 { - return fmt.Errorf("found location with reserved id=0") - } - if locations[l.ID] != nil { - return fmt.Errorf("multiple locations with same id: %d", l.ID) - } - locations[l.ID] = l - if m := l.Mapping; m != nil { - if m.ID == 0 || mappings[m.ID] != m { - return fmt.Errorf("inconsistent mapping %p: %d", m, m.ID) - } - } - for _, ln := range l.Line { - if f := ln.Function; f != nil { - if f.ID == 0 || functions[f.ID] != f { - return fmt.Errorf("inconsistent function %p: %d", f, f.ID) - } - } - } - } - return nil -} - -// Aggregate merges the locations in the profile into equivalence -// classes preserving the request attributes. It also updates the -// samples to point to the merged locations. -func (p *Profile) Aggregate(inlineFrame, function, filename, linenumber, address bool) error { - for _, m := range p.Mapping { - m.HasInlineFrames = m.HasInlineFrames && inlineFrame - m.HasFunctions = m.HasFunctions && function - m.HasFilenames = m.HasFilenames && filename - m.HasLineNumbers = m.HasLineNumbers && linenumber - } - - // Aggregate functions - if !function || !filename { - for _, f := range p.Function { - if !function { - f.Name = "" - f.SystemName = "" - } - if !filename { - f.Filename = "" - } - } - } - - // Aggregate locations - if !inlineFrame || !address || !linenumber { - for _, l := range p.Location { - if !inlineFrame && len(l.Line) > 1 { - l.Line = l.Line[len(l.Line)-1:] - } - if !linenumber { - for i := range l.Line { - l.Line[i].Line = 0 - } - } - if !address { - l.Address = 0 - } - } - } - - return p.CheckValid() -} - -// Print dumps a text representation of a profile. Intended mainly -// for debugging purposes. -func (p *Profile) String() string { - - ss := make([]string, 0, len(p.Sample)+len(p.Mapping)+len(p.Location)) - if pt := p.PeriodType; pt != nil { - ss = append(ss, fmt.Sprintf("PeriodType: %s %s", pt.Type, pt.Unit)) - } - ss = append(ss, fmt.Sprintf("Period: %d", p.Period)) - if p.TimeNanos != 0 { - ss = append(ss, fmt.Sprintf("Time: %v", time.Unix(0, p.TimeNanos))) - } - if p.DurationNanos != 0 { - ss = append(ss, fmt.Sprintf("Duration: %v", time.Duration(p.DurationNanos))) - } - - ss = append(ss, "Samples:") - var sh1 string - for _, s := range p.SampleType { - sh1 = sh1 + fmt.Sprintf("%s/%s ", s.Type, s.Unit) - } - ss = append(ss, strings.TrimSpace(sh1)) - for _, s := range p.Sample { - var sv string - for _, v := range s.Value { - sv = fmt.Sprintf("%s %10d", sv, v) - } - sv = sv + ": " - for _, l := range s.Location { - sv = sv + fmt.Sprintf("%d ", l.ID) - } - ss = append(ss, sv) - const labelHeader = " " - if len(s.Label) > 0 { - ls := labelHeader - for k, v := range s.Label { - ls = ls + fmt.Sprintf("%s:%v ", k, v) - } - ss = append(ss, ls) - } - if len(s.NumLabel) > 0 { - ls := labelHeader - for k, v := range s.NumLabel { - ls = ls + fmt.Sprintf("%s:%v ", k, v) - } - ss = append(ss, ls) - } - } - - ss = append(ss, "Locations") - for _, l := range p.Location { - locStr := fmt.Sprintf("%6d: %#x ", l.ID, l.Address) - if m := l.Mapping; m != nil { - locStr = locStr + fmt.Sprintf("M=%d ", m.ID) - } - if len(l.Line) == 0 { - ss = append(ss, locStr) - } - for li := range l.Line { - lnStr := "??" - if fn := l.Line[li].Function; fn != nil { - lnStr = fmt.Sprintf("%s %s:%d s=%d", - fn.Name, - fn.Filename, - l.Line[li].Line, - fn.StartLine) - if fn.Name != fn.SystemName { - lnStr = lnStr + "(" + fn.SystemName + ")" - } - } - ss = append(ss, locStr+lnStr) - // Do not print location details past the first line - locStr = " " - } - } - - ss = append(ss, "Mappings") - for _, m := range p.Mapping { - bits := "" - if m.HasFunctions { - bits = bits + "[FN]" - } - if m.HasFilenames { - bits = bits + "[FL]" - } - if m.HasLineNumbers { - bits = bits + "[LN]" - } - if m.HasInlineFrames { - bits = bits + "[IN]" - } - ss = append(ss, fmt.Sprintf("%d: %#x/%#x/%#x %s %s %s", - m.ID, - m.Start, m.Limit, m.Offset, - m.File, - m.BuildID, - bits)) - } - - return strings.Join(ss, "\n") + "\n" -} - -// Merge adds profile p adjusted by ratio r into profile p. Profiles -// must be compatible (same Type and SampleType). -// TODO(rsilvera): consider normalizing the profiles based on the -// total samples collected. -func (p *Profile) Merge(pb *Profile, r float64) error { - if err := p.Compatible(pb); err != nil { - return err - } - - pb = pb.Copy() - - // Keep the largest of the two periods. - if pb.Period > p.Period { - p.Period = pb.Period - } - - p.DurationNanos += pb.DurationNanos - - p.Mapping = append(p.Mapping, pb.Mapping...) - for i, m := range p.Mapping { - m.ID = uint64(i + 1) - } - p.Location = append(p.Location, pb.Location...) - for i, l := range p.Location { - l.ID = uint64(i + 1) - } - p.Function = append(p.Function, pb.Function...) - for i, f := range p.Function { - f.ID = uint64(i + 1) - } - - if r != 1.0 { - for _, s := range pb.Sample { - for i, v := range s.Value { - s.Value[i] = int64((float64(v) * r)) - } - } - } - p.Sample = append(p.Sample, pb.Sample...) - return p.CheckValid() -} - -// Compatible determines if two profiles can be compared/merged. -// returns nil if the profiles are compatible; otherwise an error with -// details on the incompatibility. -func (p *Profile) Compatible(pb *Profile) error { - if !compatibleValueTypes(p.PeriodType, pb.PeriodType) { - return fmt.Errorf("incompatible period types %v and %v", p.PeriodType, pb.PeriodType) - } - - if len(p.SampleType) != len(pb.SampleType) { - return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) - } - - for i := range p.SampleType { - if !compatibleValueTypes(p.SampleType[i], pb.SampleType[i]) { - return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) - } - } - - return nil -} - -// HasFunctions determines if all locations in this profile have -// symbolized function information. -func (p *Profile) HasFunctions() bool { - for _, l := range p.Location { - if l.Mapping == nil || !l.Mapping.HasFunctions { - return false - } - } - return true -} - -// HasFileLines determines if all locations in this profile have -// symbolized file and line number information. -func (p *Profile) HasFileLines() bool { - for _, l := range p.Location { - if l.Mapping == nil || (!l.Mapping.HasFilenames || !l.Mapping.HasLineNumbers) { - return false - } - } - return true -} - -func compatibleValueTypes(v1, v2 *ValueType) bool { - if v1 == nil || v2 == nil { - return true // No grounds to disqualify. - } - return v1.Type == v2.Type && v1.Unit == v2.Unit -} - -// Copy makes a fully independent copy of a profile. -func (p *Profile) Copy() *Profile { - p.preEncode() - b := marshal(p) - - pp := &Profile{} - if err := unmarshal(b, pp); err != nil { - panic(err) - } - if err := pp.postDecode(); err != nil { - panic(err) - } - - return pp -} - -// Demangler maps symbol names to a human-readable form. This may -// include C++ demangling and additional simplification. Names that -// are not demangled may be missing from the resulting map. -type Demangler func(name []string) (map[string]string, error) - -// Demangle attempts to demangle and optionally simplify any function -// names referenced in the profile. It works on a best-effort basis: -// it will silently preserve the original names in case of any errors. -func (p *Profile) Demangle(d Demangler) error { - // Collect names to demangle. - var names []string - for _, fn := range p.Function { - names = append(names, fn.SystemName) - } - - // Update profile with demangled names. - demangled, err := d(names) - if err != nil { - return err - } - for _, fn := range p.Function { - if dd, ok := demangled[fn.SystemName]; ok { - fn.Name = dd - } - } - return nil -} - -// Empty returns true if the profile contains no samples. -func (p *Profile) Empty() bool { - return len(p.Sample) == 0 -} diff --git a/src/cmd/internal/pprof/profile/profile_test.go b/src/cmd/internal/pprof/profile/profile_test.go deleted file mode 100644 index 09b11a456f..0000000000 --- a/src/cmd/internal/pprof/profile/profile_test.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package profile - -import ( - "bytes" - "testing" -) - -func TestEmptyProfile(t *testing.T) { - var buf bytes.Buffer - p, err := Parse(&buf) - if err != nil { - t.Error("Want no error, got", err) - } - if p == nil { - t.Fatal("Want a valid profile, got ") - } - if !p.Empty() { - t.Errorf("Profile should be empty, got %#v", p) - } -} diff --git a/src/cmd/internal/pprof/profile/proto.go b/src/cmd/internal/pprof/profile/proto.go deleted file mode 100644 index 11d7f9ff9b..0000000000 --- a/src/cmd/internal/pprof/profile/proto.go +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// This file is a simple protocol buffer encoder and decoder. -// -// A protocol message must implement the message interface: -// decoder() []decoder -// encode(*buffer) -// -// The decode method returns a slice indexed by field number that gives the -// function to decode that field. -// The encode method encodes its receiver into the given buffer. -// -// The two methods are simple enough to be implemented by hand rather than -// by using a protocol compiler. -// -// See profile.go for examples of messages implementing this interface. -// -// There is no support for groups, message sets, or "has" bits. - -package profile - -import "errors" - -type buffer struct { - field int - typ int - u64 uint64 - data []byte - tmp [16]byte -} - -type decoder func(*buffer, message) error - -type message interface { - decoder() []decoder - encode(*buffer) -} - -func marshal(m message) []byte { - var b buffer - m.encode(&b) - return b.data -} - -func encodeVarint(b *buffer, x uint64) { - for x >= 128 { - b.data = append(b.data, byte(x)|0x80) - x >>= 7 - } - b.data = append(b.data, byte(x)) -} - -func encodeLength(b *buffer, tag int, len int) { - encodeVarint(b, uint64(tag)<<3|2) - encodeVarint(b, uint64(len)) -} - -func encodeUint64(b *buffer, tag int, x uint64) { - // append varint to b.data - encodeVarint(b, uint64(tag)<<3|0) - encodeVarint(b, x) -} - -func encodeUint64s(b *buffer, tag int, x []uint64) { - if len(x) > 2 { - // Use packed encoding - n1 := len(b.data) - for _, u := range x { - encodeVarint(b, u) - } - n2 := len(b.data) - encodeLength(b, tag, n2-n1) - n3 := len(b.data) - copy(b.tmp[:], b.data[n2:n3]) - copy(b.data[n1+(n3-n2):], b.data[n1:n2]) - copy(b.data[n1:], b.tmp[:n3-n2]) - return - } - for _, u := range x { - encodeUint64(b, tag, u) - } -} - -func encodeUint64Opt(b *buffer, tag int, x uint64) { - if x == 0 { - return - } - encodeUint64(b, tag, x) -} - -func encodeInt64(b *buffer, tag int, x int64) { - u := uint64(x) - encodeUint64(b, tag, u) -} - -func encodeInt64Opt(b *buffer, tag int, x int64) { - if x == 0 { - return - } - encodeInt64(b, tag, x) -} - -func encodeInt64s(b *buffer, tag int, x []int64) { - if len(x) > 2 { - // Use packed encoding - n1 := len(b.data) - for _, u := range x { - encodeVarint(b, uint64(u)) - } - n2 := len(b.data) - encodeLength(b, tag, n2-n1) - n3 := len(b.data) - copy(b.tmp[:], b.data[n2:n3]) - copy(b.data[n1+(n3-n2):], b.data[n1:n2]) - copy(b.data[n1:], b.tmp[:n3-n2]) - return - } - for _, u := range x { - encodeInt64(b, tag, u) - } -} - -func encodeString(b *buffer, tag int, x string) { - encodeLength(b, tag, len(x)) - b.data = append(b.data, x...) -} - -func encodeStrings(b *buffer, tag int, x []string) { - for _, s := range x { - encodeString(b, tag, s) - } -} - -func encodeStringOpt(b *buffer, tag int, x string) { - if x == "" { - return - } - encodeString(b, tag, x) -} - -func encodeBool(b *buffer, tag int, x bool) { - if x { - encodeUint64(b, tag, 1) - } else { - encodeUint64(b, tag, 0) - } -} - -func encodeBoolOpt(b *buffer, tag int, x bool) { - if x == false { - return - } - encodeBool(b, tag, x) -} - -func encodeMessage(b *buffer, tag int, m message) { - n1 := len(b.data) - m.encode(b) - n2 := len(b.data) - encodeLength(b, tag, n2-n1) - n3 := len(b.data) - copy(b.tmp[:], b.data[n2:n3]) - copy(b.data[n1+(n3-n2):], b.data[n1:n2]) - copy(b.data[n1:], b.tmp[:n3-n2]) -} - -func unmarshal(data []byte, m message) (err error) { - b := buffer{data: data, typ: 2} - return decodeMessage(&b, m) -} - -func le64(p []byte) uint64 { - return uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 -} - -func le32(p []byte) uint32 { - return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 -} - -func decodeVarint(data []byte) (uint64, []byte, error) { - var i int - var u uint64 - for i = 0; ; i++ { - if i >= 10 || i >= len(data) { - return 0, nil, errors.New("bad varint") - } - u |= uint64(data[i]&0x7F) << uint(7*i) - if data[i]&0x80 == 0 { - return u, data[i+1:], nil - } - } -} - -func decodeField(b *buffer, data []byte) ([]byte, error) { - x, data, err := decodeVarint(data) - if err != nil { - return nil, err - } - b.field = int(x >> 3) - b.typ = int(x & 7) - b.data = nil - b.u64 = 0 - switch b.typ { - case 0: - b.u64, data, err = decodeVarint(data) - if err != nil { - return nil, err - } - case 1: - if len(data) < 8 { - return nil, errors.New("not enough data") - } - b.u64 = le64(data[:8]) - data = data[8:] - case 2: - var n uint64 - n, data, err = decodeVarint(data) - if err != nil { - return nil, err - } - if n > uint64(len(data)) { - return nil, errors.New("too much data") - } - b.data = data[:n] - data = data[n:] - case 5: - if len(data) < 4 { - return nil, errors.New("not enough data") - } - b.u64 = uint64(le32(data[:4])) - data = data[4:] - default: - return nil, errors.New("unknown type: " + string(b.typ)) - } - - return data, nil -} - -func checkType(b *buffer, typ int) error { - if b.typ != typ { - return errors.New("type mismatch") - } - return nil -} - -func decodeMessage(b *buffer, m message) error { - if err := checkType(b, 2); err != nil { - return err - } - dec := m.decoder() - data := b.data - for len(data) > 0 { - // pull varint field# + type - var err error - data, err = decodeField(b, data) - if err != nil { - return err - } - if b.field >= len(dec) || dec[b.field] == nil { - continue - } - if err := dec[b.field](b, m); err != nil { - return err - } - } - return nil -} - -func decodeInt64(b *buffer, x *int64) error { - if err := checkType(b, 0); err != nil { - return err - } - *x = int64(b.u64) - return nil -} - -func decodeInt64s(b *buffer, x *[]int64) error { - if b.typ == 2 { - // Packed encoding - data := b.data - for len(data) > 0 { - var u uint64 - var err error - - if u, data, err = decodeVarint(data); err != nil { - return err - } - *x = append(*x, int64(u)) - } - return nil - } - var i int64 - if err := decodeInt64(b, &i); err != nil { - return err - } - *x = append(*x, i) - return nil -} - -func decodeUint64(b *buffer, x *uint64) error { - if err := checkType(b, 0); err != nil { - return err - } - *x = b.u64 - return nil -} - -func decodeUint64s(b *buffer, x *[]uint64) error { - if b.typ == 2 { - data := b.data - // Packed encoding - for len(data) > 0 { - var u uint64 - var err error - - if u, data, err = decodeVarint(data); err != nil { - return err - } - *x = append(*x, u) - } - return nil - } - var u uint64 - if err := decodeUint64(b, &u); err != nil { - return err - } - *x = append(*x, u) - return nil -} - -func decodeString(b *buffer, x *string) error { - if err := checkType(b, 2); err != nil { - return err - } - *x = string(b.data) - return nil -} - -func decodeStrings(b *buffer, x *[]string) error { - var s string - if err := decodeString(b, &s); err != nil { - return err - } - *x = append(*x, s) - return nil -} - -func decodeBool(b *buffer, x *bool) error { - if err := checkType(b, 0); err != nil { - return err - } - if int64(b.u64) == 0 { - *x = false - } else { - *x = true - } - return nil -} diff --git a/src/cmd/internal/pprof/profile/proto_test.go b/src/cmd/internal/pprof/profile/proto_test.go deleted file mode 100644 index c2613fc375..0000000000 --- a/src/cmd/internal/pprof/profile/proto_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package profile - -import ( - "reflect" - "testing" -) - -func TestPackedEncoding(t *testing.T) { - - type testcase struct { - uint64s []uint64 - int64s []int64 - encoded []byte - } - for i, tc := range []testcase{ - { - []uint64{0, 1, 10, 100, 1000, 10000}, - []int64{1000, 0, 1000}, - []byte{10, 8, 0, 1, 10, 100, 232, 7, 144, 78, 18, 5, 232, 7, 0, 232, 7}, - }, - { - []uint64{10000}, - nil, - []byte{8, 144, 78}, - }, - { - nil, - []int64{-10000}, - []byte{16, 240, 177, 255, 255, 255, 255, 255, 255, 255, 1}, - }, - } { - source := &packedInts{tc.uint64s, tc.int64s} - if got, want := marshal(source), tc.encoded; !reflect.DeepEqual(got, want) { - t.Errorf("failed encode %d, got %v, want %v", i, got, want) - } - - dest := new(packedInts) - if err := unmarshal(tc.encoded, dest); err != nil { - t.Errorf("failed decode %d: %v", i, err) - continue - } - if got, want := dest.uint64s, tc.uint64s; !reflect.DeepEqual(got, want) { - t.Errorf("failed decode uint64s %d, got %v, want %v", i, got, want) - } - if got, want := dest.int64s, tc.int64s; !reflect.DeepEqual(got, want) { - t.Errorf("failed decode int64s %d, got %v, want %v", i, got, want) - } - } -} - -type packedInts struct { - uint64s []uint64 - int64s []int64 -} - -func (u *packedInts) decoder() []decoder { - return []decoder{ - nil, - func(b *buffer, m message) error { return decodeUint64s(b, &m.(*packedInts).uint64s) }, - func(b *buffer, m message) error { return decodeInt64s(b, &m.(*packedInts).int64s) }, - } -} - -func (u *packedInts) encode(b *buffer) { - encodeUint64s(b, 1, u.uint64s) - encodeInt64s(b, 2, u.int64s) -} diff --git a/src/cmd/internal/pprof/profile/prune.go b/src/cmd/internal/pprof/profile/prune.go deleted file mode 100644 index 1924fada7a..0000000000 --- a/src/cmd/internal/pprof/profile/prune.go +++ /dev/null @@ -1,97 +0,0 @@ -// Copyright 2014 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Implements methods to remove frames from profiles. - -package profile - -import ( - "fmt" - "regexp" -) - -// Prune removes all nodes beneath a node matching dropRx, and not -// matching keepRx. If the root node of a Sample matches, the sample -// will have an empty stack. -func (p *Profile) Prune(dropRx, keepRx *regexp.Regexp) { - prune := make(map[uint64]bool) - pruneBeneath := make(map[uint64]bool) - - for _, loc := range p.Location { - var i int - for i = len(loc.Line) - 1; i >= 0; i-- { - if fn := loc.Line[i].Function; fn != nil && fn.Name != "" { - funcName := fn.Name - // Account for leading '.' on the PPC ELF v1 ABI. - if funcName[0] == '.' { - funcName = funcName[1:] - } - if dropRx.MatchString(funcName) { - if keepRx == nil || !keepRx.MatchString(funcName) { - break - } - } - } - } - - if i >= 0 { - // Found matching entry to prune. - pruneBeneath[loc.ID] = true - - // Remove the matching location. - if i == len(loc.Line)-1 { - // Matched the top entry: prune the whole location. - prune[loc.ID] = true - } else { - loc.Line = loc.Line[i+1:] - } - } - } - - // Prune locs from each Sample - for _, sample := range p.Sample { - // Scan from the root to the leaves to find the prune location. - // Do not prune frames before the first user frame, to avoid - // pruning everything. - foundUser := false - for i := len(sample.Location) - 1; i >= 0; i-- { - id := sample.Location[i].ID - if !prune[id] && !pruneBeneath[id] { - foundUser = true - continue - } - if !foundUser { - continue - } - if prune[id] { - sample.Location = sample.Location[i+1:] - break - } - if pruneBeneath[id] { - sample.Location = sample.Location[i:] - break - } - } - } -} - -// RemoveUninteresting prunes and elides profiles using built-in -// tables of uninteresting function names. -func (p *Profile) RemoveUninteresting() error { - var keep, drop *regexp.Regexp - var err error - - if p.DropFrames != "" { - if drop, err = regexp.Compile("^(" + p.DropFrames + ")$"); err != nil { - return fmt.Errorf("failed to compile regexp %s: %v", p.DropFrames, err) - } - if p.KeepFrames != "" { - if keep, err = regexp.Compile("^(" + p.KeepFrames + ")$"); err != nil { - return fmt.Errorf("failed to compile regexp %s: %v", p.KeepFrames, err) - } - } - p.Prune(drop, keep) - } - return nil -} diff --git a/src/cmd/internal/pprof/report/report.go b/src/cmd/internal/pprof/report/report.go index 989665301f..4f5252b28e 100644 --- a/src/cmd/internal/pprof/report/report.go +++ b/src/cmd/internal/pprof/report/report.go @@ -18,7 +18,7 @@ import ( "time" "cmd/internal/pprof/plugin" - "cmd/internal/pprof/profile" + "internal/pprof/profile" ) // Generate generates a report as directed by the Report. diff --git a/src/cmd/internal/pprof/symbolizer/symbolizer.go b/src/cmd/internal/pprof/symbolizer/symbolizer.go index bc22800530..d81f3eafaf 100644 --- a/src/cmd/internal/pprof/symbolizer/symbolizer.go +++ b/src/cmd/internal/pprof/symbolizer/symbolizer.go @@ -14,7 +14,7 @@ import ( "strings" "cmd/internal/pprof/plugin" - "cmd/internal/pprof/profile" + "internal/pprof/profile" ) // Symbolize adds symbol and line number information to all locations diff --git a/src/cmd/internal/pprof/symbolz/symbolz.go b/src/cmd/internal/pprof/symbolz/symbolz.go index 2f2850afeb..6e58001962 100644 --- a/src/cmd/internal/pprof/symbolz/symbolz.go +++ b/src/cmd/internal/pprof/symbolz/symbolz.go @@ -15,7 +15,7 @@ import ( "strconv" "strings" - "cmd/internal/pprof/profile" + "internal/pprof/profile" ) var ( diff --git a/src/cmd/pprof/pprof.go b/src/cmd/pprof/pprof.go index 0c979b1831..01f44566ba 100644 --- a/src/cmd/pprof/pprof.go +++ b/src/cmd/pprof/pprof.go @@ -19,9 +19,9 @@ import ( "cmd/internal/pprof/driver" "cmd/internal/pprof/fetch" "cmd/internal/pprof/plugin" - "cmd/internal/pprof/profile" "cmd/internal/pprof/symbolizer" "cmd/internal/pprof/symbolz" + "internal/pprof/profile" ) func main() { diff --git a/src/cmd/trace/pprof.go b/src/cmd/trace/pprof.go index 3bae15c608..dea3a749fc 100644 --- a/src/cmd/trace/pprof.go +++ b/src/cmd/trace/pprof.go @@ -8,8 +8,8 @@ package main import ( "bufio" - "cmd/internal/pprof/profile" "fmt" + "internal/pprof/profile" "internal/trace" "io" "io/ioutil" diff --git a/src/go/build/deps_test.go b/src/go/build/deps_test.go index cb101abe7c..1314e551ea 100644 --- a/src/go/build/deps_test.go +++ b/src/go/build/deps_test.go @@ -257,6 +257,7 @@ var pkgDeps = map[string][]string{ "index/suffixarray": {"L4", "regexp"}, "internal/singleflight": {"sync"}, "internal/trace": {"L4", "OS"}, + "internal/pprof/profile": {"L4", "OS", "compress/gzip", "regexp"}, "math/big": {"L4"}, "mime": {"L4", "OS", "syscall", "internal/syscall/windows/registry"}, "mime/quotedprintable": {"L4"}, diff --git a/src/internal/pprof/profile/encode.go b/src/internal/pprof/profile/encode.go new file mode 100644 index 0000000000..6b879a84ac --- /dev/null +++ b/src/internal/pprof/profile/encode.go @@ -0,0 +1,470 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package profile + +import ( + "errors" + "fmt" + "sort" +) + +func (p *Profile) decoder() []decoder { + return profileDecoder +} + +// preEncode populates the unexported fields to be used by encode +// (with suffix X) from the corresponding exported fields. The +// exported fields are cleared up to facilitate testing. +func (p *Profile) preEncode() { + strings := make(map[string]int) + addString(strings, "") + + for _, st := range p.SampleType { + st.typeX = addString(strings, st.Type) + st.unitX = addString(strings, st.Unit) + } + + for _, s := range p.Sample { + s.labelX = nil + var keys []string + for k := range s.Label { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := s.Label[k] + for _, v := range vs { + s.labelX = append(s.labelX, + Label{ + keyX: addString(strings, k), + strX: addString(strings, v), + }, + ) + } + } + var numKeys []string + for k := range s.NumLabel { + numKeys = append(numKeys, k) + } + sort.Strings(numKeys) + for _, k := range numKeys { + vs := s.NumLabel[k] + for _, v := range vs { + s.labelX = append(s.labelX, + Label{ + keyX: addString(strings, k), + numX: v, + }, + ) + } + } + s.locationIDX = nil + for _, l := range s.Location { + s.locationIDX = append(s.locationIDX, l.ID) + } + } + + for _, m := range p.Mapping { + m.fileX = addString(strings, m.File) + m.buildIDX = addString(strings, m.BuildID) + } + + for _, l := range p.Location { + for i, ln := range l.Line { + if ln.Function != nil { + l.Line[i].functionIDX = ln.Function.ID + } else { + l.Line[i].functionIDX = 0 + } + } + if l.Mapping != nil { + l.mappingIDX = l.Mapping.ID + } else { + l.mappingIDX = 0 + } + } + for _, f := range p.Function { + f.nameX = addString(strings, f.Name) + f.systemNameX = addString(strings, f.SystemName) + f.filenameX = addString(strings, f.Filename) + } + + p.dropFramesX = addString(strings, p.DropFrames) + p.keepFramesX = addString(strings, p.KeepFrames) + + if pt := p.PeriodType; pt != nil { + pt.typeX = addString(strings, pt.Type) + pt.unitX = addString(strings, pt.Unit) + } + + p.stringTable = make([]string, len(strings)) + for s, i := range strings { + p.stringTable[i] = s + } +} + +func (p *Profile) encode(b *buffer) { + for _, x := range p.SampleType { + encodeMessage(b, 1, x) + } + for _, x := range p.Sample { + encodeMessage(b, 2, x) + } + for _, x := range p.Mapping { + encodeMessage(b, 3, x) + } + for _, x := range p.Location { + encodeMessage(b, 4, x) + } + for _, x := range p.Function { + encodeMessage(b, 5, x) + } + encodeStrings(b, 6, p.stringTable) + encodeInt64Opt(b, 7, p.dropFramesX) + encodeInt64Opt(b, 8, p.keepFramesX) + encodeInt64Opt(b, 9, p.TimeNanos) + encodeInt64Opt(b, 10, p.DurationNanos) + if pt := p.PeriodType; pt != nil && (pt.typeX != 0 || pt.unitX != 0) { + encodeMessage(b, 11, p.PeriodType) + } + encodeInt64Opt(b, 12, p.Period) +} + +var profileDecoder = []decoder{ + nil, // 0 + // repeated ValueType sample_type = 1 + func(b *buffer, m message) error { + x := new(ValueType) + pp := m.(*Profile) + pp.SampleType = append(pp.SampleType, x) + return decodeMessage(b, x) + }, + // repeated Sample sample = 2 + func(b *buffer, m message) error { + x := new(Sample) + pp := m.(*Profile) + pp.Sample = append(pp.Sample, x) + return decodeMessage(b, x) + }, + // repeated Mapping mapping = 3 + func(b *buffer, m message) error { + x := new(Mapping) + pp := m.(*Profile) + pp.Mapping = append(pp.Mapping, x) + return decodeMessage(b, x) + }, + // repeated Location location = 4 + func(b *buffer, m message) error { + x := new(Location) + pp := m.(*Profile) + pp.Location = append(pp.Location, x) + return decodeMessage(b, x) + }, + // repeated Function function = 5 + func(b *buffer, m message) error { + x := new(Function) + pp := m.(*Profile) + pp.Function = append(pp.Function, x) + return decodeMessage(b, x) + }, + // repeated string string_table = 6 + func(b *buffer, m message) error { + err := decodeStrings(b, &m.(*Profile).stringTable) + if err != nil { + return err + } + if *&m.(*Profile).stringTable[0] != "" { + return errors.New("string_table[0] must be ''") + } + return nil + }, + // repeated int64 drop_frames = 7 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).dropFramesX) }, + // repeated int64 keep_frames = 8 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).keepFramesX) }, + // repeated int64 time_nanos = 9 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).TimeNanos) }, + // repeated int64 duration_nanos = 10 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).DurationNanos) }, + // optional string period_type = 11 + func(b *buffer, m message) error { + x := new(ValueType) + pp := m.(*Profile) + pp.PeriodType = x + return decodeMessage(b, x) + }, + // repeated int64 period = 12 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Profile).Period) }, +} + +// postDecode takes the unexported fields populated by decode (with +// suffix X) and populates the corresponding exported fields. +// The unexported fields are cleared up to facilitate testing. +func (p *Profile) postDecode() error { + var err error + + mappings := make(map[uint64]*Mapping) + for _, m := range p.Mapping { + m.File, err = getString(p.stringTable, &m.fileX, err) + m.BuildID, err = getString(p.stringTable, &m.buildIDX, err) + mappings[m.ID] = m + } + + functions := make(map[uint64]*Function) + for _, f := range p.Function { + f.Name, err = getString(p.stringTable, &f.nameX, err) + f.SystemName, err = getString(p.stringTable, &f.systemNameX, err) + f.Filename, err = getString(p.stringTable, &f.filenameX, err) + functions[f.ID] = f + } + + locations := make(map[uint64]*Location) + for _, l := range p.Location { + l.Mapping = mappings[l.mappingIDX] + l.mappingIDX = 0 + for i, ln := range l.Line { + if id := ln.functionIDX; id != 0 { + l.Line[i].Function = functions[id] + if l.Line[i].Function == nil { + return fmt.Errorf("Function ID %d not found", id) + } + l.Line[i].functionIDX = 0 + } + } + locations[l.ID] = l + } + + for _, st := range p.SampleType { + st.Type, err = getString(p.stringTable, &st.typeX, err) + st.Unit, err = getString(p.stringTable, &st.unitX, err) + } + + for _, s := range p.Sample { + labels := make(map[string][]string) + numLabels := make(map[string][]int64) + for _, l := range s.labelX { + var key, value string + key, err = getString(p.stringTable, &l.keyX, err) + if l.strX != 0 { + value, err = getString(p.stringTable, &l.strX, err) + labels[key] = append(labels[key], value) + } else { + numLabels[key] = append(numLabels[key], l.numX) + } + } + if len(labels) > 0 { + s.Label = labels + } + if len(numLabels) > 0 { + s.NumLabel = numLabels + } + s.Location = nil + for _, lid := range s.locationIDX { + s.Location = append(s.Location, locations[lid]) + } + s.locationIDX = nil + } + + p.DropFrames, err = getString(p.stringTable, &p.dropFramesX, err) + p.KeepFrames, err = getString(p.stringTable, &p.keepFramesX, err) + + if pt := p.PeriodType; pt == nil { + p.PeriodType = &ValueType{} + } + + if pt := p.PeriodType; pt != nil { + pt.Type, err = getString(p.stringTable, &pt.typeX, err) + pt.Unit, err = getString(p.stringTable, &pt.unitX, err) + } + p.stringTable = nil + return nil +} + +func (p *ValueType) decoder() []decoder { + return valueTypeDecoder +} + +func (p *ValueType) encode(b *buffer) { + encodeInt64Opt(b, 1, p.typeX) + encodeInt64Opt(b, 2, p.unitX) +} + +var valueTypeDecoder = []decoder{ + nil, // 0 + // optional int64 type = 1 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).typeX) }, + // optional int64 unit = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*ValueType).unitX) }, +} + +func (p *Sample) decoder() []decoder { + return sampleDecoder +} + +func (p *Sample) encode(b *buffer) { + encodeUint64s(b, 1, p.locationIDX) + for _, x := range p.Value { + encodeInt64(b, 2, x) + } + for _, x := range p.labelX { + encodeMessage(b, 3, x) + } +} + +var sampleDecoder = []decoder{ + nil, // 0 + // repeated uint64 location = 1 + func(b *buffer, m message) error { return decodeUint64s(b, &m.(*Sample).locationIDX) }, + // repeated int64 value = 2 + func(b *buffer, m message) error { return decodeInt64s(b, &m.(*Sample).Value) }, + // repeated Label label = 3 + func(b *buffer, m message) error { + s := m.(*Sample) + n := len(s.labelX) + s.labelX = append(s.labelX, Label{}) + return decodeMessage(b, &s.labelX[n]) + }, +} + +func (p Label) decoder() []decoder { + return labelDecoder +} + +func (p Label) encode(b *buffer) { + encodeInt64Opt(b, 1, p.keyX) + encodeInt64Opt(b, 2, p.strX) + encodeInt64Opt(b, 3, p.numX) +} + +var labelDecoder = []decoder{ + nil, // 0 + // optional int64 key = 1 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).keyX) }, + // optional int64 str = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).strX) }, + // optional int64 num = 3 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Label).numX) }, +} + +func (p *Mapping) decoder() []decoder { + return mappingDecoder +} + +func (p *Mapping) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeUint64Opt(b, 2, p.Start) + encodeUint64Opt(b, 3, p.Limit) + encodeUint64Opt(b, 4, p.Offset) + encodeInt64Opt(b, 5, p.fileX) + encodeInt64Opt(b, 6, p.buildIDX) + encodeBoolOpt(b, 7, p.HasFunctions) + encodeBoolOpt(b, 8, p.HasFilenames) + encodeBoolOpt(b, 9, p.HasLineNumbers) + encodeBoolOpt(b, 10, p.HasInlineFrames) +} + +var mappingDecoder = []decoder{ + nil, // 0 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).ID) }, // optional uint64 id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Start) }, // optional uint64 memory_offset = 2 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Limit) }, // optional uint64 memory_limit = 3 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Mapping).Offset) }, // optional uint64 file_offset = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).fileX) }, // optional int64 filename = 5 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Mapping).buildIDX) }, // optional int64 build_id = 6 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFunctions) }, // optional bool has_functions = 7 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasFilenames) }, // optional bool has_filenames = 8 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasLineNumbers) }, // optional bool has_line_numbers = 9 + func(b *buffer, m message) error { return decodeBool(b, &m.(*Mapping).HasInlineFrames) }, // optional bool has_inline_frames = 10 +} + +func (p *Location) decoder() []decoder { + return locationDecoder +} + +func (p *Location) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeUint64Opt(b, 2, p.mappingIDX) + encodeUint64Opt(b, 3, p.Address) + for i := range p.Line { + encodeMessage(b, 4, &p.Line[i]) + } +} + +var locationDecoder = []decoder{ + nil, // 0 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).ID) }, // optional uint64 id = 1; + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).mappingIDX) }, // optional uint64 mapping_id = 2; + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Location).Address) }, // optional uint64 address = 3; + func(b *buffer, m message) error { // repeated Line line = 4 + pp := m.(*Location) + n := len(pp.Line) + pp.Line = append(pp.Line, Line{}) + return decodeMessage(b, &pp.Line[n]) + }, +} + +func (p *Line) decoder() []decoder { + return lineDecoder +} + +func (p *Line) encode(b *buffer) { + encodeUint64Opt(b, 1, p.functionIDX) + encodeInt64Opt(b, 2, p.Line) +} + +var lineDecoder = []decoder{ + nil, // 0 + // optional uint64 function_id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Line).functionIDX) }, + // optional int64 line = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Line).Line) }, +} + +func (p *Function) decoder() []decoder { + return functionDecoder +} + +func (p *Function) encode(b *buffer) { + encodeUint64Opt(b, 1, p.ID) + encodeInt64Opt(b, 2, p.nameX) + encodeInt64Opt(b, 3, p.systemNameX) + encodeInt64Opt(b, 4, p.filenameX) + encodeInt64Opt(b, 5, p.StartLine) +} + +var functionDecoder = []decoder{ + nil, // 0 + // optional uint64 id = 1 + func(b *buffer, m message) error { return decodeUint64(b, &m.(*Function).ID) }, + // optional int64 function_name = 2 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).nameX) }, + // optional int64 function_system_name = 3 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).systemNameX) }, + // repeated int64 filename = 4 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).filenameX) }, + // optional int64 start_line = 5 + func(b *buffer, m message) error { return decodeInt64(b, &m.(*Function).StartLine) }, +} + +func addString(strings map[string]int, s string) int64 { + i, ok := strings[s] + if !ok { + i = len(strings) + strings[s] = i + } + return int64(i) +} + +func getString(strings []string, strng *int64, err error) (string, error) { + if err != nil { + return "", err + } + s := int(*strng) + if s < 0 || s >= len(strings) { + return "", errMalformed + } + *strng = 0 + return strings[s], nil +} diff --git a/src/internal/pprof/profile/filter.go b/src/internal/pprof/profile/filter.go new file mode 100644 index 0000000000..1baa096a49 --- /dev/null +++ b/src/internal/pprof/profile/filter.go @@ -0,0 +1,158 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implements methods to filter samples from profiles. + +package profile + +import "regexp" + +// FilterSamplesByName filters the samples in a profile and only keeps +// samples where at least one frame matches focus but none match ignore. +// Returns true is the corresponding regexp matched at least one sample. +func (p *Profile) FilterSamplesByName(focus, ignore, hide *regexp.Regexp) (fm, im, hm bool) { + focusOrIgnore := make(map[uint64]bool) + hidden := make(map[uint64]bool) + for _, l := range p.Location { + if ignore != nil && l.matchesName(ignore) { + im = true + focusOrIgnore[l.ID] = false + } else if focus == nil || l.matchesName(focus) { + fm = true + focusOrIgnore[l.ID] = true + } + if hide != nil && l.matchesName(hide) { + hm = true + l.Line = l.unmatchedLines(hide) + if len(l.Line) == 0 { + hidden[l.ID] = true + } + } + } + + s := make([]*Sample, 0, len(p.Sample)) + for _, sample := range p.Sample { + if focusedAndNotIgnored(sample.Location, focusOrIgnore) { + if len(hidden) > 0 { + var locs []*Location + for _, loc := range sample.Location { + if !hidden[loc.ID] { + locs = append(locs, loc) + } + } + if len(locs) == 0 { + // Remove sample with no locations (by not adding it to s). + continue + } + sample.Location = locs + } + s = append(s, sample) + } + } + p.Sample = s + + return +} + +// matchesName returns whether the function name or file in the +// location matches the regular expression. +func (loc *Location) matchesName(re *regexp.Regexp) bool { + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if re.MatchString(fn.Name) { + return true + } + if re.MatchString(fn.Filename) { + return true + } + } + } + return false +} + +// unmatchedLines returns the lines in the location that do not match +// the regular expression. +func (loc *Location) unmatchedLines(re *regexp.Regexp) []Line { + var lines []Line + for _, ln := range loc.Line { + if fn := ln.Function; fn != nil { + if re.MatchString(fn.Name) { + continue + } + if re.MatchString(fn.Filename) { + continue + } + } + lines = append(lines, ln) + } + return lines +} + +// focusedAndNotIgnored looks up a slice of ids against a map of +// focused/ignored locations. The map only contains locations that are +// explicitly focused or ignored. Returns whether there is at least +// one focused location but no ignored locations. +func focusedAndNotIgnored(locs []*Location, m map[uint64]bool) bool { + var f bool + for _, loc := range locs { + if focus, focusOrIgnore := m[loc.ID]; focusOrIgnore { + if focus { + // Found focused location. Must keep searching in case there + // is an ignored one as well. + f = true + } else { + // Found ignored location. Can return false right away. + return false + } + } + } + return f +} + +// TagMatch selects tags for filtering +type TagMatch func(key, val string, nval int64) bool + +// FilterSamplesByTag removes all samples from the profile, except +// those that match focus and do not match the ignore regular +// expression. +func (p *Profile) FilterSamplesByTag(focus, ignore TagMatch) (fm, im bool) { + samples := make([]*Sample, 0, len(p.Sample)) + for _, s := range p.Sample { + focused, ignored := focusedSample(s, focus, ignore) + fm = fm || focused + im = im || ignored + if focused && !ignored { + samples = append(samples, s) + } + } + p.Sample = samples + return +} + +// focusedTag checks a sample against focus and ignore regexps. +// Returns whether the focus/ignore regexps match any tags +func focusedSample(s *Sample, focus, ignore TagMatch) (fm, im bool) { + fm = focus == nil + for key, vals := range s.Label { + for _, val := range vals { + if ignore != nil && ignore(key, val, 0) { + im = true + } + if !fm && focus(key, val, 0) { + fm = true + } + } + } + for key, vals := range s.NumLabel { + for _, val := range vals { + if ignore != nil && ignore(key, "", val) { + im = true + } + if !fm && focus(key, "", val) { + fm = true + } + } + } + return fm, im +} diff --git a/src/internal/pprof/profile/legacy_profile.go b/src/internal/pprof/profile/legacy_profile.go new file mode 100644 index 0000000000..5ad3e25640 --- /dev/null +++ b/src/internal/pprof/profile/legacy_profile.go @@ -0,0 +1,1250 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements parsers to convert legacy profiles into the +// profile.proto format. + +package profile + +import ( + "bufio" + "bytes" + "fmt" + "io" + "math" + "regexp" + "strconv" + "strings" +) + +var ( + countStartRE = regexp.MustCompile(`\A(\w+) profile: total \d+\n\z`) + countRE = regexp.MustCompile(`\A(\d+) @(( 0x[0-9a-f]+)+)\n\z`) + + heapHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`) + heapSampleRE = regexp.MustCompile(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`) + + contentionSampleRE = regexp.MustCompile(`(\d+) *(\d+) @([ x0-9a-f]*)`) + + hexNumberRE = regexp.MustCompile(`0x[0-9a-f]+`) + + growthHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ growthz`) + + fragmentationHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz`) + + threadzStartRE = regexp.MustCompile(`--- threadz \d+ ---`) + threadStartRE = regexp.MustCompile(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`) + + procMapsRE = regexp.MustCompile(`([[:xdigit:]]+)-([[:xdigit:]]+)\s+([-rwxp]+)\s+([[:xdigit:]]+)\s+([[:xdigit:]]+):([[:xdigit:]]+)\s+([[:digit:]]+)\s*(\S+)?`) + + briefMapsRE = regexp.MustCompile(`\s*([[:xdigit:]]+)-([[:xdigit:]]+):\s*(\S+)(\s.*@)?([[:xdigit:]]+)?`) + + // LegacyHeapAllocated instructs the heapz parsers to use the + // allocated memory stats instead of the default in-use memory. Note + // that tcmalloc doesn't provide all allocated memory, only in-use + // stats. + LegacyHeapAllocated bool +) + +func isSpaceOrComment(line string) bool { + trimmed := strings.TrimSpace(line) + return len(trimmed) == 0 || trimmed[0] == '#' +} + +// parseGoCount parses a Go count profile (e.g., threadcreate or +// goroutine) and returns a new Profile. +func parseGoCount(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + + var line string + var err error + for { + // Skip past comments and empty lines seeking a real header. + line, err = r.ReadString('\n') + if err != nil { + return nil, err + } + if !isSpaceOrComment(line) { + break + } + } + + m := countStartRE.FindStringSubmatch(line) + if m == nil { + return nil, errUnrecognized + } + profileType := m[1] + p := &Profile{ + PeriodType: &ValueType{Type: profileType, Unit: "count"}, + Period: 1, + SampleType: []*ValueType{{Type: profileType, Unit: "count"}}, + } + locations := make(map[uint64]*Location) + for { + line, err = r.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + if isSpaceOrComment(line) { + continue + } + if strings.HasPrefix(line, "---") { + break + } + m := countRE.FindStringSubmatch(line) + if m == nil { + return nil, errMalformed + } + n, err := strconv.ParseInt(m[1], 0, 64) + if err != nil { + return nil, errMalformed + } + fields := strings.Fields(m[2]) + locs := make([]*Location, 0, len(fields)) + for _, stk := range fields { + addr, err := strconv.ParseUint(stk, 0, 64) + if err != nil { + return nil, errMalformed + } + // Adjust all frames by -1 to land on the call instruction. + addr-- + loc := locations[addr] + if loc == nil { + loc = &Location{ + Address: addr, + } + locations[addr] = loc + p.Location = append(p.Location, loc) + } + locs = append(locs, loc) + } + p.Sample = append(p.Sample, &Sample{ + Location: locs, + Value: []int64{n}, + }) + } + + if err = parseAdditionalSections(strings.TrimSpace(line), r, p); err != nil { + return nil, err + } + return p, nil +} + +// remapLocationIDs ensures there is a location for each address +// referenced by a sample, and remaps the samples to point to the new +// location ids. +func (p *Profile) remapLocationIDs() { + seen := make(map[*Location]bool, len(p.Location)) + var locs []*Location + + for _, s := range p.Sample { + for _, l := range s.Location { + if seen[l] { + continue + } + l.ID = uint64(len(locs) + 1) + locs = append(locs, l) + seen[l] = true + } + } + p.Location = locs +} + +func (p *Profile) remapFunctionIDs() { + seen := make(map[*Function]bool, len(p.Function)) + var fns []*Function + + for _, l := range p.Location { + for _, ln := range l.Line { + fn := ln.Function + if fn == nil || seen[fn] { + continue + } + fn.ID = uint64(len(fns) + 1) + fns = append(fns, fn) + seen[fn] = true + } + } + p.Function = fns +} + +// remapMappingIDs matches location addresses with existing mappings +// and updates them appropriately. This is O(N*M), if this ever shows +// up as a bottleneck, evaluate sorting the mappings and doing a +// binary search, which would make it O(N*log(M)). +func (p *Profile) remapMappingIDs() { + if len(p.Mapping) == 0 { + return + } + + // Some profile handlers will incorrectly set regions for the main + // executable if its section is remapped. Fix them through heuristics. + + // Remove the initial mapping if named '/anon_hugepage' and has a + // consecutive adjacent mapping. + if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") { + if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start { + p.Mapping = p.Mapping[1:] + } + } + + // Subtract the offset from the start of the main mapping if it + // ends up at a recognizable start address. + const expectedStart = 0x400000 + if m := p.Mapping[0]; m.Start-m.Offset == expectedStart { + m.Start = expectedStart + m.Offset = 0 + } + + for _, l := range p.Location { + if a := l.Address; a != 0 { + for _, m := range p.Mapping { + if m.Start <= a && a < m.Limit { + l.Mapping = m + break + } + } + } + } + + // Reset all mapping IDs. + for i, m := range p.Mapping { + m.ID = uint64(i + 1) + } +} + +var cpuInts = []func([]byte) (uint64, []byte){ + get32l, + get32b, + get64l, + get64b, +} + +func get32l(b []byte) (uint64, []byte) { + if len(b) < 4 { + return 0, nil + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24, b[4:] +} + +func get32b(b []byte) (uint64, []byte) { + if len(b) < 4 { + return 0, nil + } + return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24, b[4:] +} + +func get64l(b []byte) (uint64, []byte) { + if len(b) < 8 { + return 0, nil + } + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:] +} + +func get64b(b []byte) (uint64, []byte) { + if len(b) < 8 { + return 0, nil + } + return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:] +} + +// ParseTracebacks parses a set of tracebacks and returns a newly +// populated profile. It will accept any text file and generate a +// Profile out of it with any hex addresses it can identify, including +// a process map if it can recognize one. Each sample will include a +// tag "source" with the addresses recognized in string format. +func ParseTracebacks(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + + p := &Profile{ + PeriodType: &ValueType{Type: "trace", Unit: "count"}, + Period: 1, + SampleType: []*ValueType{ + {Type: "trace", Unit: "count"}, + }, + } + + var sources []string + var sloc []*Location + + locs := make(map[uint64]*Location) + for { + l, err := r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + if l == "" { + break + } + } + if sectionTrigger(l) == memoryMapSection { + break + } + if s, addrs := extractHexAddresses(l); len(s) > 0 { + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + sources = append(sources, s...) + } else { + if len(sources) > 0 || len(sloc) > 0 { + addTracebackSample(sloc, sources, p) + sloc, sources = nil, nil + } + } + } + + // Add final sample to save any leftover data. + if len(sources) > 0 || len(sloc) > 0 { + addTracebackSample(sloc, sources, p) + } + + if err := p.ParseMemoryMap(r); err != nil { + return nil, err + } + return p, nil +} + +func addTracebackSample(l []*Location, s []string, p *Profile) { + p.Sample = append(p.Sample, + &Sample{ + Value: []int64{1}, + Location: l, + Label: map[string][]string{"source": s}, + }) +} + +// parseCPU parses a profilez legacy profile and returns a newly +// populated Profile. +// +// The general format for profilez samples is a sequence of words in +// binary format. The first words are a header with the following data: +// 1st word -- 0 +// 2nd word -- 3 +// 3rd word -- 0 if a c++ application, 1 if a java application. +// 4th word -- Sampling period (in microseconds). +// 5th word -- Padding. +func parseCPU(b []byte) (*Profile, error) { + var parse func([]byte) (uint64, []byte) + var n1, n2, n3, n4, n5 uint64 + for _, parse = range cpuInts { + var tmp []byte + n1, tmp = parse(b) + n2, tmp = parse(tmp) + n3, tmp = parse(tmp) + n4, tmp = parse(tmp) + n5, tmp = parse(tmp) + + if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 == 0 { + b = tmp + return cpuProfile(b, int64(n4), parse) + } + } + return nil, errUnrecognized +} + +// cpuProfile returns a new Profile from C++ profilez data. +// b is the profile bytes after the header, period is the profiling +// period, and parse is a function to parse 8-byte chunks from the +// profile in its native endianness. +func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) { + p := &Profile{ + Period: period * 1000, + PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"}, + SampleType: []*ValueType{ + {Type: "samples", Unit: "count"}, + {Type: "cpu", Unit: "nanoseconds"}, + }, + } + var err error + if b, _, err = parseCPUSamples(b, parse, true, p); err != nil { + return nil, err + } + + // If all samples have the same second-to-the-bottom frame, it + // strongly suggests that it is an uninteresting artifact of + // measurement -- a stack frame pushed by the signal handler. The + // bottom frame is always correct as it is picked up from the signal + // structure, not the stack. Check if this is the case and if so, + // remove. + if len(p.Sample) > 1 && len(p.Sample[0].Location) > 1 { + allSame := true + id1 := p.Sample[0].Location[1].Address + for _, s := range p.Sample { + if len(s.Location) < 2 || id1 != s.Location[1].Address { + allSame = false + break + } + } + if allSame { + for _, s := range p.Sample { + s.Location = append(s.Location[:1], s.Location[2:]...) + } + } + } + + if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil { + return nil, err + } + return p, nil +} + +// parseCPUSamples parses a collection of profilez samples from a +// profile. +// +// profilez samples are a repeated sequence of stack frames of the +// form: +// 1st word -- The number of times this stack was encountered. +// 2nd word -- The size of the stack (StackSize). +// 3rd word -- The first address on the stack. +// ... +// StackSize + 2 -- The last address on the stack +// The last stack trace is of the form: +// 1st word -- 0 +// 2nd word -- 1 +// 3rd word -- 0 +// +// Addresses from stack traces may point to the next instruction after +// each call. Optionally adjust by -1 to land somewhere on the actual +// call (except for the leaf, which is not a call). +func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust bool, p *Profile) ([]byte, map[uint64]*Location, error) { + locs := make(map[uint64]*Location) + for len(b) > 0 { + var count, nstk uint64 + count, b = parse(b) + nstk, b = parse(b) + if b == nil || nstk > uint64(len(b)/4) { + return nil, nil, errUnrecognized + } + var sloc []*Location + addrs := make([]uint64, nstk) + for i := 0; i < int(nstk); i++ { + addrs[i], b = parse(b) + } + + if count == 0 && nstk == 1 && addrs[0] == 0 { + // End of data marker + break + } + for i, addr := range addrs { + if adjust && i > 0 { + addr-- + } + loc := locs[addr] + if loc == nil { + loc = &Location{ + Address: addr, + } + locs[addr] = loc + p.Location = append(p.Location, loc) + } + sloc = append(sloc, loc) + } + p.Sample = append(p.Sample, + &Sample{ + Value: []int64{int64(count), int64(count) * p.Period}, + Location: sloc, + }) + } + // Reached the end without finding the EOD marker. + return b, locs, nil +} + +// parseHeap parses a heapz legacy or a growthz profile and +// returns a newly populated Profile. +func parseHeap(b []byte) (p *Profile, err error) { + r := bytes.NewBuffer(b) + l, err := r.ReadString('\n') + if err != nil { + return nil, errUnrecognized + } + + sampling := "" + + if header := heapHeaderRE.FindStringSubmatch(l); header != nil { + p = &Profile{ + SampleType: []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + }, + PeriodType: &ValueType{Type: "objects", Unit: "bytes"}, + } + + var period int64 + if len(header[6]) > 0 { + if period, err = strconv.ParseInt(header[6], 10, 64); err != nil { + return nil, errUnrecognized + } + } + + switch header[5] { + case "heapz_v2", "heap_v2": + sampling, p.Period = "v2", period + case "heapprofile": + sampling, p.Period = "", 1 + case "heap": + sampling, p.Period = "v2", period/2 + default: + return nil, errUnrecognized + } + } else if header = growthHeaderRE.FindStringSubmatch(l); header != nil { + p = &Profile{ + SampleType: []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + }, + PeriodType: &ValueType{Type: "heapgrowth", Unit: "count"}, + Period: 1, + } + } else if header = fragmentationHeaderRE.FindStringSubmatch(l); header != nil { + p = &Profile{ + SampleType: []*ValueType{ + {Type: "objects", Unit: "count"}, + {Type: "space", Unit: "bytes"}, + }, + PeriodType: &ValueType{Type: "allocations", Unit: "count"}, + Period: 1, + } + } else { + return nil, errUnrecognized + } + + if LegacyHeapAllocated { + for _, st := range p.SampleType { + st.Type = "alloc_" + st.Type + } + } else { + for _, st := range p.SampleType { + st.Type = "inuse_" + st.Type + } + } + + locs := make(map[uint64]*Location) + for { + l, err = r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + + if l == "" { + break + } + } + + if isSpaceOrComment(l) { + continue + } + l = strings.TrimSpace(l) + + if sectionTrigger(l) != unrecognizedSection { + break + } + + value, blocksize, addrs, err := parseHeapSample(l, p.Period, sampling) + if err != nil { + return nil, err + } + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + p.Sample = append(p.Sample, &Sample{ + Value: value, + Location: sloc, + NumLabel: map[string][]int64{"bytes": {blocksize}}, + }) + } + + if err = parseAdditionalSections(l, r, p); err != nil { + return nil, err + } + return p, nil +} + +// parseHeapSample parses a single row from a heap profile into a new Sample. +func parseHeapSample(line string, rate int64, sampling string) (value []int64, blocksize int64, addrs []uint64, err error) { + sampleData := heapSampleRE.FindStringSubmatch(line) + if len(sampleData) != 6 { + return value, blocksize, addrs, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData)) + } + + // Use first two values by default; tcmalloc sampling generates the + // same value for both, only the older heap-profile collect separate + // stats for in-use and allocated objects. + valueIndex := 1 + if LegacyHeapAllocated { + valueIndex = 3 + } + + var v1, v2 int64 + if v1, err = strconv.ParseInt(sampleData[valueIndex], 10, 64); err != nil { + return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + if v2, err = strconv.ParseInt(sampleData[valueIndex+1], 10, 64); err != nil { + return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + if v1 == 0 { + if v2 != 0 { + return value, blocksize, addrs, fmt.Errorf("allocation count was 0 but allocation bytes was %d", v2) + } + } else { + blocksize = v2 / v1 + if sampling == "v2" { + v1, v2 = scaleHeapSample(v1, v2, rate) + } + } + + value = []int64{v1, v2} + addrs = parseHexAddresses(sampleData[5]) + + return value, blocksize, addrs, nil +} + +// extractHexAddresses extracts hex numbers from a string and returns +// them, together with their numeric value, in a slice. +func extractHexAddresses(s string) ([]string, []uint64) { + hexStrings := hexNumberRE.FindAllString(s, -1) + var ids []uint64 + for _, s := range hexStrings { + if id, err := strconv.ParseUint(s, 0, 64); err == nil { + ids = append(ids, id) + } else { + // Do not expect any parsing failures due to the regexp matching. + panic("failed to parse hex value:" + s) + } + } + return hexStrings, ids +} + +// parseHexAddresses parses hex numbers from a string and returns them +// in a slice. +func parseHexAddresses(s string) []uint64 { + _, ids := extractHexAddresses(s) + return ids +} + +// scaleHeapSample adjusts the data from a heapz Sample to +// account for its probability of appearing in the collected +// data. heapz profiles are a sampling of the memory allocations +// requests in a program. We estimate the unsampled value by dividing +// each collected sample by its probability of appearing in the +// profile. heapz v2 profiles rely on a poisson process to determine +// which samples to collect, based on the desired average collection +// rate R. The probability of a sample of size S to appear in that +// profile is 1-exp(-S/R). +func scaleHeapSample(count, size, rate int64) (int64, int64) { + if count == 0 || size == 0 { + return 0, 0 + } + + if rate <= 1 { + // if rate==1 all samples were collected so no adjustment is needed. + // if rate<1 treat as unknown and skip scaling. + return count, size + } + + avgSize := float64(size) / float64(count) + scale := 1 / (1 - math.Exp(-avgSize/float64(rate))) + + return int64(float64(count) * scale), int64(float64(size) * scale) +} + +// parseContention parses a mutex or contention profile. There are 2 cases: +// "--- contentionz " for legacy C++ profiles (and backwards compatibility) +// "--- mutex:" or "--- contention:" for profiles generated by the Go runtime. +// This code converts the text output from runtime into a *Profile. (In the future +// the runtime might write a serialized Profile directly making this unnecessary.) +func parseContention(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + l, err := r.ReadString('\n') + if err != nil { + return nil, errUnrecognized + } + if strings.HasPrefix(l, "--- contentionz ") { + return parseCppContention(r) + } else if strings.HasPrefix(l, "--- mutex:") { + return parseCppContention(r) + } else if strings.HasPrefix(l, "--- contention:") { + return parseCppContention(r) + } + return nil, errUnrecognized +} + +// parseCppContention parses the output from synchronization_profiling.cc +// for backward compatibility, and the compatible (non-debug) block profile +// output from the Go runtime. +func parseCppContention(r *bytes.Buffer) (*Profile, error) { + p := &Profile{ + PeriodType: &ValueType{Type: "contentions", Unit: "count"}, + Period: 1, + SampleType: []*ValueType{ + {Type: "contentions", Unit: "count"}, + {Type: "delay", Unit: "nanoseconds"}, + }, + } + + var cpuHz int64 + var l string + var err error + // Parse text of the form "attribute = value" before the samples. + const delimiter = "=" + for { + l, err = r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + + if l == "" { + break + } + } + + if l = strings.TrimSpace(l); l == "" { + continue + } + + if strings.HasPrefix(l, "---") { + break + } + + attr := strings.SplitN(l, delimiter, 2) + if len(attr) != 2 { + break + } + key, val := strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]) + var err error + switch key { + case "cycles/second": + if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil { + return nil, errUnrecognized + } + case "sampling period": + if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil { + return nil, errUnrecognized + } + case "ms since reset": + ms, err := strconv.ParseInt(val, 0, 64) + if err != nil { + return nil, errUnrecognized + } + p.DurationNanos = ms * 1000 * 1000 + case "format": + // CPP contentionz profiles don't have format. + return nil, errUnrecognized + case "resolution": + // CPP contentionz profiles don't have resolution. + return nil, errUnrecognized + case "discarded samples": + default: + return nil, errUnrecognized + } + } + + locs := make(map[uint64]*Location) + for { + if l = strings.TrimSpace(l); strings.HasPrefix(l, "---") { + break + } + value, addrs, err := parseContentionSample(l, p.Period, cpuHz) + if err != nil { + return nil, err + } + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + p.Sample = append(p.Sample, &Sample{ + Value: value, + Location: sloc, + }) + + if l, err = r.ReadString('\n'); err != nil { + if err != io.EOF { + return nil, err + } + if l == "" { + break + } + } + } + + if err = parseAdditionalSections(l, r, p); err != nil { + return nil, err + } + + return p, nil +} + +// parseContentionSample parses a single row from a contention profile +// into a new Sample. +func parseContentionSample(line string, period, cpuHz int64) (value []int64, addrs []uint64, err error) { + sampleData := contentionSampleRE.FindStringSubmatch(line) + if sampleData == nil { + return value, addrs, errUnrecognized + } + + v1, err := strconv.ParseInt(sampleData[1], 10, 64) + if err != nil { + return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + v2, err := strconv.ParseInt(sampleData[2], 10, 64) + if err != nil { + return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err) + } + + // Unsample values if period and cpuHz are available. + // - Delays are scaled to cycles and then to nanoseconds. + // - Contentions are scaled to cycles. + if period > 0 { + if cpuHz > 0 { + cpuGHz := float64(cpuHz) / 1e9 + v1 = int64(float64(v1) * float64(period) / cpuGHz) + } + v2 = v2 * period + } + + value = []int64{v2, v1} + addrs = parseHexAddresses(sampleData[3]) + + return value, addrs, nil +} + +// parseThread parses a Threadz profile and returns a new Profile. +func parseThread(b []byte) (*Profile, error) { + r := bytes.NewBuffer(b) + + var line string + var err error + for { + // Skip past comments and empty lines seeking a real header. + line, err = r.ReadString('\n') + if err != nil { + return nil, err + } + if !isSpaceOrComment(line) { + break + } + } + + if m := threadzStartRE.FindStringSubmatch(line); m != nil { + // Advance over initial comments until first stack trace. + for { + line, err = r.ReadString('\n') + if err != nil { + if err != io.EOF { + return nil, err + } + + if line == "" { + break + } + } + if sectionTrigger(line) != unrecognizedSection || line[0] == '-' { + break + } + } + } else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { + return nil, errUnrecognized + } + + p := &Profile{ + SampleType: []*ValueType{{Type: "thread", Unit: "count"}}, + PeriodType: &ValueType{Type: "thread", Unit: "count"}, + Period: 1, + } + + locs := make(map[uint64]*Location) + // Recognize each thread and populate profile samples. + for sectionTrigger(line) == unrecognizedSection { + if strings.HasPrefix(line, "---- no stack trace for") { + line = "" + break + } + if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 { + return nil, errUnrecognized + } + + var addrs []uint64 + line, addrs, err = parseThreadSample(r) + if err != nil { + return nil, errUnrecognized + } + if len(addrs) == 0 { + // We got a --same as previous threads--. Bump counters. + if len(p.Sample) > 0 { + s := p.Sample[len(p.Sample)-1] + s.Value[0]++ + } + continue + } + + var sloc []*Location + for _, addr := range addrs { + // Addresses from stack traces point to the next instruction after + // each call. Adjust by -1 to land somewhere on the actual call. + addr-- + loc := locs[addr] + if locs[addr] == nil { + loc = &Location{ + Address: addr, + } + p.Location = append(p.Location, loc) + locs[addr] = loc + } + sloc = append(sloc, loc) + } + + p.Sample = append(p.Sample, &Sample{ + Value: []int64{1}, + Location: sloc, + }) + } + + if err = parseAdditionalSections(line, r, p); err != nil { + return nil, err + } + + return p, nil +} + +// parseThreadSample parses a symbolized or unsymbolized stack trace. +// Returns the first line after the traceback, the sample (or nil if +// it hits a 'same-as-previous' marker) and an error. +func parseThreadSample(b *bytes.Buffer) (nextl string, addrs []uint64, err error) { + var l string + sameAsPrevious := false + for { + if l, err = b.ReadString('\n'); err != nil { + if err != io.EOF { + return "", nil, err + } + if l == "" { + break + } + } + if l = strings.TrimSpace(l); l == "" { + continue + } + + if strings.HasPrefix(l, "---") { + break + } + if strings.Contains(l, "same as previous thread") { + sameAsPrevious = true + continue + } + + addrs = append(addrs, parseHexAddresses(l)...) + } + + if sameAsPrevious { + return l, nil, nil + } + return l, addrs, nil +} + +// parseAdditionalSections parses any additional sections in the +// profile, ignoring any unrecognized sections. +func parseAdditionalSections(l string, b *bytes.Buffer, p *Profile) (err error) { + for { + if sectionTrigger(l) == memoryMapSection { + break + } + // Ignore any unrecognized sections. + if l, err := b.ReadString('\n'); err != nil { + if err != io.EOF { + return err + } + if l == "" { + break + } + } + } + return p.ParseMemoryMap(b) +} + +// ParseMemoryMap parses a memory map in the format of +// /proc/self/maps, and overrides the mappings in the current profile. +// It renumbers the samples and locations in the profile correspondingly. +func (p *Profile) ParseMemoryMap(rd io.Reader) error { + b := bufio.NewReader(rd) + + var attrs []string + var r *strings.Replacer + const delimiter = "=" + for { + l, err := b.ReadString('\n') + if err != nil { + if err != io.EOF { + return err + } + if l == "" { + break + } + } + if l = strings.TrimSpace(l); l == "" { + continue + } + + if r != nil { + l = r.Replace(l) + } + m, err := parseMappingEntry(l) + if err != nil { + if err == errUnrecognized { + // Recognize assignments of the form: attr=value, and replace + // $attr with value on subsequent mappings. + if attr := strings.SplitN(l, delimiter, 2); len(attr) == 2 { + attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1])) + r = strings.NewReplacer(attrs...) + } + // Ignore any unrecognized entries + continue + } + return err + } + if m == nil || (m.File == "" && len(p.Mapping) != 0) { + // In some cases the first entry may include the address range + // but not the name of the file. It should be followed by + // another entry with the name. + continue + } + if len(p.Mapping) == 1 && p.Mapping[0].File == "" { + // Update the name if this is the entry following that empty one. + p.Mapping[0].File = m.File + continue + } + p.Mapping = append(p.Mapping, m) + } + p.remapLocationIDs() + p.remapFunctionIDs() + p.remapMappingIDs() + return nil +} + +func parseMappingEntry(l string) (*Mapping, error) { + mapping := &Mapping{} + var err error + if me := procMapsRE.FindStringSubmatch(l); len(me) == 9 { + if !strings.Contains(me[3], "x") { + // Skip non-executable entries. + return nil, nil + } + if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil { + return nil, errUnrecognized + } + if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil { + return nil, errUnrecognized + } + if me[4] != "" { + if mapping.Offset, err = strconv.ParseUint(me[4], 16, 64); err != nil { + return nil, errUnrecognized + } + } + mapping.File = me[8] + return mapping, nil + } + + if me := briefMapsRE.FindStringSubmatch(l); len(me) == 6 { + if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil { + return nil, errUnrecognized + } + if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil { + return nil, errUnrecognized + } + mapping.File = me[3] + if me[5] != "" { + if mapping.Offset, err = strconv.ParseUint(me[5], 16, 64); err != nil { + return nil, errUnrecognized + } + } + return mapping, nil + } + + return nil, errUnrecognized +} + +type sectionType int + +const ( + unrecognizedSection sectionType = iota + memoryMapSection +) + +var memoryMapTriggers = []string{ + "--- Memory map: ---", + "MAPPED_LIBRARIES:", +} + +func sectionTrigger(line string) sectionType { + for _, trigger := range memoryMapTriggers { + if strings.Contains(line, trigger) { + return memoryMapSection + } + } + return unrecognizedSection +} + +func (p *Profile) addLegacyFrameInfo() { + switch { + case isProfileType(p, heapzSampleTypes) || + isProfileType(p, heapzInUseSampleTypes) || + isProfileType(p, heapzAllocSampleTypes): + p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr + case isProfileType(p, contentionzSampleTypes): + p.DropFrames, p.KeepFrames = lockRxStr, "" + default: + p.DropFrames, p.KeepFrames = cpuProfilerRxStr, "" + } +} + +var heapzSampleTypes = []string{"allocations", "size"} // early Go pprof profiles +var heapzInUseSampleTypes = []string{"inuse_objects", "inuse_space"} +var heapzAllocSampleTypes = []string{"alloc_objects", "alloc_space"} +var contentionzSampleTypes = []string{"contentions", "delay"} + +func isProfileType(p *Profile, t []string) bool { + st := p.SampleType + if len(st) != len(t) { + return false + } + + for i := range st { + if st[i].Type != t[i] { + return false + } + } + return true +} + +var allocRxStr = strings.Join([]string{ + // POSIX entry points. + `calloc`, + `cfree`, + `malloc`, + `free`, + `memalign`, + `do_memalign`, + `(__)?posix_memalign`, + `pvalloc`, + `valloc`, + `realloc`, + + // TC malloc. + `tcmalloc::.*`, + `tc_calloc`, + `tc_cfree`, + `tc_malloc`, + `tc_free`, + `tc_memalign`, + `tc_posix_memalign`, + `tc_pvalloc`, + `tc_valloc`, + `tc_realloc`, + `tc_new`, + `tc_delete`, + `tc_newarray`, + `tc_deletearray`, + `tc_new_nothrow`, + `tc_newarray_nothrow`, + + // Memory-allocation routines on OS X. + `malloc_zone_malloc`, + `malloc_zone_calloc`, + `malloc_zone_valloc`, + `malloc_zone_realloc`, + `malloc_zone_memalign`, + `malloc_zone_free`, + + // Go runtime + `runtime\..*`, + + // Other misc. memory allocation routines + `BaseArena::.*`, + `(::)?do_malloc_no_errno`, + `(::)?do_malloc_pages`, + `(::)?do_malloc`, + `DoSampledAllocation`, + `MallocedMemBlock::MallocedMemBlock`, + `_M_allocate`, + `__builtin_(vec_)?delete`, + `__builtin_(vec_)?new`, + `__gnu_cxx::new_allocator::allocate`, + `__libc_malloc`, + `__malloc_alloc_template::allocate`, + `allocate`, + `cpp_alloc`, + `operator new(\[\])?`, + `simple_alloc::allocate`, +}, `|`) + +var allocSkipRxStr = strings.Join([]string{ + // Preserve Go runtime frames that appear in the middle/bottom of + // the stack. + `runtime\.panic`, +}, `|`) + +var cpuProfilerRxStr = strings.Join([]string{ + `ProfileData::Add`, + `ProfileData::prof_handler`, + `CpuProfiler::prof_handler`, + `__pthread_sighandler`, + `__restore`, +}, `|`) + +var lockRxStr = strings.Join([]string{ + `RecordLockProfileData`, + `(base::)?RecordLockProfileData.*`, + `(base::)?SubmitMutexProfileData.*`, + `(base::)?SubmitSpinLockProfileData.*`, + `(Mutex::)?AwaitCommon.*`, + `(Mutex::)?Unlock.*`, + `(Mutex::)?UnlockSlow.*`, + `(Mutex::)?ReaderUnlock.*`, + `(MutexLock::)?~MutexLock.*`, + `(SpinLock::)?Unlock.*`, + `(SpinLock::)?SlowUnlock.*`, + `(SpinLockHolder::)?~SpinLockHolder.*`, +}, `|`) diff --git a/src/internal/pprof/profile/profile.go b/src/internal/pprof/profile/profile.go new file mode 100644 index 0000000000..28e713d7be --- /dev/null +++ b/src/internal/pprof/profile/profile.go @@ -0,0 +1,572 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package profile provides a representation of profile.proto and +// methods to encode/decode profiles in this format. +package profile + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "io/ioutil" + "regexp" + "strings" + "time" +) + +// Profile is an in-memory representation of profile.proto. +type Profile struct { + SampleType []*ValueType + Sample []*Sample + Mapping []*Mapping + Location []*Location + Function []*Function + + DropFrames string + KeepFrames string + + TimeNanos int64 + DurationNanos int64 + PeriodType *ValueType + Period int64 + + dropFramesX int64 + keepFramesX int64 + stringTable []string +} + +// ValueType corresponds to Profile.ValueType +type ValueType struct { + Type string // cpu, wall, inuse_space, etc + Unit string // seconds, nanoseconds, bytes, etc + + typeX int64 + unitX int64 +} + +// Sample corresponds to Profile.Sample +type Sample struct { + Location []*Location + Value []int64 + Label map[string][]string + NumLabel map[string][]int64 + + locationIDX []uint64 + labelX []Label +} + +// Label corresponds to Profile.Label +type Label struct { + keyX int64 + // Exactly one of the two following values must be set + strX int64 + numX int64 // Integer value for this label +} + +// Mapping corresponds to Profile.Mapping +type Mapping struct { + ID uint64 + Start uint64 + Limit uint64 + Offset uint64 + File string + BuildID string + HasFunctions bool + HasFilenames bool + HasLineNumbers bool + HasInlineFrames bool + + fileX int64 + buildIDX int64 +} + +// Location corresponds to Profile.Location +type Location struct { + ID uint64 + Mapping *Mapping + Address uint64 + Line []Line + + mappingIDX uint64 +} + +// Line corresponds to Profile.Line +type Line struct { + Function *Function + Line int64 + + functionIDX uint64 +} + +// Function corresponds to Profile.Function +type Function struct { + ID uint64 + Name string + SystemName string + Filename string + StartLine int64 + + nameX int64 + systemNameX int64 + filenameX int64 +} + +// Parse parses a profile and checks for its validity. The input +// may be a gzip-compressed encoded protobuf or one of many legacy +// profile formats which may be unsupported in the future. +func Parse(r io.Reader) (*Profile, error) { + orig, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + var p *Profile + if len(orig) >= 2 && orig[0] == 0x1f && orig[1] == 0x8b { + gz, err := gzip.NewReader(bytes.NewBuffer(orig)) + if err != nil { + return nil, fmt.Errorf("decompressing profile: %v", err) + } + data, err := ioutil.ReadAll(gz) + if err != nil { + return nil, fmt.Errorf("decompressing profile: %v", err) + } + orig = data + } + if p, err = parseUncompressed(orig); err != nil { + if p, err = parseLegacy(orig); err != nil { + return nil, fmt.Errorf("parsing profile: %v", err) + } + } + + if err := p.CheckValid(); err != nil { + return nil, fmt.Errorf("malformed profile: %v", err) + } + return p, nil +} + +var errUnrecognized = fmt.Errorf("unrecognized profile format") +var errMalformed = fmt.Errorf("malformed profile format") + +func parseLegacy(data []byte) (*Profile, error) { + parsers := []func([]byte) (*Profile, error){ + parseCPU, + parseHeap, + parseGoCount, // goroutine, threadcreate + parseThread, + parseContention, + } + + for _, parser := range parsers { + p, err := parser(data) + if err == nil { + p.setMain() + p.addLegacyFrameInfo() + return p, nil + } + if err != errUnrecognized { + return nil, err + } + } + return nil, errUnrecognized +} + +func parseUncompressed(data []byte) (*Profile, error) { + p := &Profile{} + if err := unmarshal(data, p); err != nil { + return nil, err + } + + if err := p.postDecode(); err != nil { + return nil, err + } + + return p, nil +} + +var libRx = regexp.MustCompile(`([.]so$|[.]so[._][0-9]+)`) + +// setMain scans Mapping entries and guesses which entry is main +// because legacy profiles don't obey the convention of putting main +// first. +func (p *Profile) setMain() { + for i := 0; i < len(p.Mapping); i++ { + file := strings.TrimSpace(strings.Replace(p.Mapping[i].File, "(deleted)", "", -1)) + if len(file) == 0 { + continue + } + if len(libRx.FindStringSubmatch(file)) > 0 { + continue + } + if strings.HasPrefix(file, "[") { + continue + } + // Swap what we guess is main to position 0. + tmp := p.Mapping[i] + p.Mapping[i] = p.Mapping[0] + p.Mapping[0] = tmp + break + } +} + +// Write writes the profile as a gzip-compressed marshaled protobuf. +func (p *Profile) Write(w io.Writer) error { + p.preEncode() + b := marshal(p) + zw := gzip.NewWriter(w) + defer zw.Close() + _, err := zw.Write(b) + return err +} + +// CheckValid tests whether the profile is valid. Checks include, but are +// not limited to: +// - len(Profile.Sample[n].value) == len(Profile.value_unit) +// - Sample.id has a corresponding Profile.Location +func (p *Profile) CheckValid() error { + // Check that sample values are consistent + sampleLen := len(p.SampleType) + if sampleLen == 0 && len(p.Sample) != 0 { + return fmt.Errorf("missing sample type information") + } + for _, s := range p.Sample { + if len(s.Value) != sampleLen { + return fmt.Errorf("mismatch: sample has: %d values vs. %d types", len(s.Value), len(p.SampleType)) + } + } + + // Check that all mappings/locations/functions are in the tables + // Check that there are no duplicate ids + mappings := make(map[uint64]*Mapping, len(p.Mapping)) + for _, m := range p.Mapping { + if m.ID == 0 { + return fmt.Errorf("found mapping with reserved ID=0") + } + if mappings[m.ID] != nil { + return fmt.Errorf("multiple mappings with same id: %d", m.ID) + } + mappings[m.ID] = m + } + functions := make(map[uint64]*Function, len(p.Function)) + for _, f := range p.Function { + if f.ID == 0 { + return fmt.Errorf("found function with reserved ID=0") + } + if functions[f.ID] != nil { + return fmt.Errorf("multiple functions with same id: %d", f.ID) + } + functions[f.ID] = f + } + locations := make(map[uint64]*Location, len(p.Location)) + for _, l := range p.Location { + if l.ID == 0 { + return fmt.Errorf("found location with reserved id=0") + } + if locations[l.ID] != nil { + return fmt.Errorf("multiple locations with same id: %d", l.ID) + } + locations[l.ID] = l + if m := l.Mapping; m != nil { + if m.ID == 0 || mappings[m.ID] != m { + return fmt.Errorf("inconsistent mapping %p: %d", m, m.ID) + } + } + for _, ln := range l.Line { + if f := ln.Function; f != nil { + if f.ID == 0 || functions[f.ID] != f { + return fmt.Errorf("inconsistent function %p: %d", f, f.ID) + } + } + } + } + return nil +} + +// Aggregate merges the locations in the profile into equivalence +// classes preserving the request attributes. It also updates the +// samples to point to the merged locations. +func (p *Profile) Aggregate(inlineFrame, function, filename, linenumber, address bool) error { + for _, m := range p.Mapping { + m.HasInlineFrames = m.HasInlineFrames && inlineFrame + m.HasFunctions = m.HasFunctions && function + m.HasFilenames = m.HasFilenames && filename + m.HasLineNumbers = m.HasLineNumbers && linenumber + } + + // Aggregate functions + if !function || !filename { + for _, f := range p.Function { + if !function { + f.Name = "" + f.SystemName = "" + } + if !filename { + f.Filename = "" + } + } + } + + // Aggregate locations + if !inlineFrame || !address || !linenumber { + for _, l := range p.Location { + if !inlineFrame && len(l.Line) > 1 { + l.Line = l.Line[len(l.Line)-1:] + } + if !linenumber { + for i := range l.Line { + l.Line[i].Line = 0 + } + } + if !address { + l.Address = 0 + } + } + } + + return p.CheckValid() +} + +// Print dumps a text representation of a profile. Intended mainly +// for debugging purposes. +func (p *Profile) String() string { + + ss := make([]string, 0, len(p.Sample)+len(p.Mapping)+len(p.Location)) + if pt := p.PeriodType; pt != nil { + ss = append(ss, fmt.Sprintf("PeriodType: %s %s", pt.Type, pt.Unit)) + } + ss = append(ss, fmt.Sprintf("Period: %d", p.Period)) + if p.TimeNanos != 0 { + ss = append(ss, fmt.Sprintf("Time: %v", time.Unix(0, p.TimeNanos))) + } + if p.DurationNanos != 0 { + ss = append(ss, fmt.Sprintf("Duration: %v", time.Duration(p.DurationNanos))) + } + + ss = append(ss, "Samples:") + var sh1 string + for _, s := range p.SampleType { + sh1 = sh1 + fmt.Sprintf("%s/%s ", s.Type, s.Unit) + } + ss = append(ss, strings.TrimSpace(sh1)) + for _, s := range p.Sample { + var sv string + for _, v := range s.Value { + sv = fmt.Sprintf("%s %10d", sv, v) + } + sv = sv + ": " + for _, l := range s.Location { + sv = sv + fmt.Sprintf("%d ", l.ID) + } + ss = append(ss, sv) + const labelHeader = " " + if len(s.Label) > 0 { + ls := labelHeader + for k, v := range s.Label { + ls = ls + fmt.Sprintf("%s:%v ", k, v) + } + ss = append(ss, ls) + } + if len(s.NumLabel) > 0 { + ls := labelHeader + for k, v := range s.NumLabel { + ls = ls + fmt.Sprintf("%s:%v ", k, v) + } + ss = append(ss, ls) + } + } + + ss = append(ss, "Locations") + for _, l := range p.Location { + locStr := fmt.Sprintf("%6d: %#x ", l.ID, l.Address) + if m := l.Mapping; m != nil { + locStr = locStr + fmt.Sprintf("M=%d ", m.ID) + } + if len(l.Line) == 0 { + ss = append(ss, locStr) + } + for li := range l.Line { + lnStr := "??" + if fn := l.Line[li].Function; fn != nil { + lnStr = fmt.Sprintf("%s %s:%d s=%d", + fn.Name, + fn.Filename, + l.Line[li].Line, + fn.StartLine) + if fn.Name != fn.SystemName { + lnStr = lnStr + "(" + fn.SystemName + ")" + } + } + ss = append(ss, locStr+lnStr) + // Do not print location details past the first line + locStr = " " + } + } + + ss = append(ss, "Mappings") + for _, m := range p.Mapping { + bits := "" + if m.HasFunctions { + bits = bits + "[FN]" + } + if m.HasFilenames { + bits = bits + "[FL]" + } + if m.HasLineNumbers { + bits = bits + "[LN]" + } + if m.HasInlineFrames { + bits = bits + "[IN]" + } + ss = append(ss, fmt.Sprintf("%d: %#x/%#x/%#x %s %s %s", + m.ID, + m.Start, m.Limit, m.Offset, + m.File, + m.BuildID, + bits)) + } + + return strings.Join(ss, "\n") + "\n" +} + +// Merge adds profile p adjusted by ratio r into profile p. Profiles +// must be compatible (same Type and SampleType). +// TODO(rsilvera): consider normalizing the profiles based on the +// total samples collected. +func (p *Profile) Merge(pb *Profile, r float64) error { + if err := p.Compatible(pb); err != nil { + return err + } + + pb = pb.Copy() + + // Keep the largest of the two periods. + if pb.Period > p.Period { + p.Period = pb.Period + } + + p.DurationNanos += pb.DurationNanos + + p.Mapping = append(p.Mapping, pb.Mapping...) + for i, m := range p.Mapping { + m.ID = uint64(i + 1) + } + p.Location = append(p.Location, pb.Location...) + for i, l := range p.Location { + l.ID = uint64(i + 1) + } + p.Function = append(p.Function, pb.Function...) + for i, f := range p.Function { + f.ID = uint64(i + 1) + } + + if r != 1.0 { + for _, s := range pb.Sample { + for i, v := range s.Value { + s.Value[i] = int64((float64(v) * r)) + } + } + } + p.Sample = append(p.Sample, pb.Sample...) + return p.CheckValid() +} + +// Compatible determines if two profiles can be compared/merged. +// returns nil if the profiles are compatible; otherwise an error with +// details on the incompatibility. +func (p *Profile) Compatible(pb *Profile) error { + if !compatibleValueTypes(p.PeriodType, pb.PeriodType) { + return fmt.Errorf("incompatible period types %v and %v", p.PeriodType, pb.PeriodType) + } + + if len(p.SampleType) != len(pb.SampleType) { + return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) + } + + for i := range p.SampleType { + if !compatibleValueTypes(p.SampleType[i], pb.SampleType[i]) { + return fmt.Errorf("incompatible sample types %v and %v", p.SampleType, pb.SampleType) + } + } + + return nil +} + +// HasFunctions determines if all locations in this profile have +// symbolized function information. +func (p *Profile) HasFunctions() bool { + for _, l := range p.Location { + if l.Mapping == nil || !l.Mapping.HasFunctions { + return false + } + } + return true +} + +// HasFileLines determines if all locations in this profile have +// symbolized file and line number information. +func (p *Profile) HasFileLines() bool { + for _, l := range p.Location { + if l.Mapping == nil || (!l.Mapping.HasFilenames || !l.Mapping.HasLineNumbers) { + return false + } + } + return true +} + +func compatibleValueTypes(v1, v2 *ValueType) bool { + if v1 == nil || v2 == nil { + return true // No grounds to disqualify. + } + return v1.Type == v2.Type && v1.Unit == v2.Unit +} + +// Copy makes a fully independent copy of a profile. +func (p *Profile) Copy() *Profile { + p.preEncode() + b := marshal(p) + + pp := &Profile{} + if err := unmarshal(b, pp); err != nil { + panic(err) + } + if err := pp.postDecode(); err != nil { + panic(err) + } + + return pp +} + +// Demangler maps symbol names to a human-readable form. This may +// include C++ demangling and additional simplification. Names that +// are not demangled may be missing from the resulting map. +type Demangler func(name []string) (map[string]string, error) + +// Demangle attempts to demangle and optionally simplify any function +// names referenced in the profile. It works on a best-effort basis: +// it will silently preserve the original names in case of any errors. +func (p *Profile) Demangle(d Demangler) error { + // Collect names to demangle. + var names []string + for _, fn := range p.Function { + names = append(names, fn.SystemName) + } + + // Update profile with demangled names. + demangled, err := d(names) + if err != nil { + return err + } + for _, fn := range p.Function { + if dd, ok := demangled[fn.SystemName]; ok { + fn.Name = dd + } + } + return nil +} + +// Empty returns true if the profile contains no samples. +func (p *Profile) Empty() bool { + return len(p.Sample) == 0 +} diff --git a/src/internal/pprof/profile/profile_test.go b/src/internal/pprof/profile/profile_test.go new file mode 100644 index 0000000000..09b11a456f --- /dev/null +++ b/src/internal/pprof/profile/profile_test.go @@ -0,0 +1,24 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package profile + +import ( + "bytes" + "testing" +) + +func TestEmptyProfile(t *testing.T) { + var buf bytes.Buffer + p, err := Parse(&buf) + if err != nil { + t.Error("Want no error, got", err) + } + if p == nil { + t.Fatal("Want a valid profile, got ") + } + if !p.Empty() { + t.Errorf("Profile should be empty, got %#v", p) + } +} diff --git a/src/internal/pprof/profile/proto.go b/src/internal/pprof/profile/proto.go new file mode 100644 index 0000000000..11d7f9ff9b --- /dev/null +++ b/src/internal/pprof/profile/proto.go @@ -0,0 +1,360 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file is a simple protocol buffer encoder and decoder. +// +// A protocol message must implement the message interface: +// decoder() []decoder +// encode(*buffer) +// +// The decode method returns a slice indexed by field number that gives the +// function to decode that field. +// The encode method encodes its receiver into the given buffer. +// +// The two methods are simple enough to be implemented by hand rather than +// by using a protocol compiler. +// +// See profile.go for examples of messages implementing this interface. +// +// There is no support for groups, message sets, or "has" bits. + +package profile + +import "errors" + +type buffer struct { + field int + typ int + u64 uint64 + data []byte + tmp [16]byte +} + +type decoder func(*buffer, message) error + +type message interface { + decoder() []decoder + encode(*buffer) +} + +func marshal(m message) []byte { + var b buffer + m.encode(&b) + return b.data +} + +func encodeVarint(b *buffer, x uint64) { + for x >= 128 { + b.data = append(b.data, byte(x)|0x80) + x >>= 7 + } + b.data = append(b.data, byte(x)) +} + +func encodeLength(b *buffer, tag int, len int) { + encodeVarint(b, uint64(tag)<<3|2) + encodeVarint(b, uint64(len)) +} + +func encodeUint64(b *buffer, tag int, x uint64) { + // append varint to b.data + encodeVarint(b, uint64(tag)<<3|0) + encodeVarint(b, x) +} + +func encodeUint64s(b *buffer, tag int, x []uint64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + encodeVarint(b, u) + } + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + encodeUint64(b, tag, u) + } +} + +func encodeUint64Opt(b *buffer, tag int, x uint64) { + if x == 0 { + return + } + encodeUint64(b, tag, x) +} + +func encodeInt64(b *buffer, tag int, x int64) { + u := uint64(x) + encodeUint64(b, tag, u) +} + +func encodeInt64Opt(b *buffer, tag int, x int64) { + if x == 0 { + return + } + encodeInt64(b, tag, x) +} + +func encodeInt64s(b *buffer, tag int, x []int64) { + if len(x) > 2 { + // Use packed encoding + n1 := len(b.data) + for _, u := range x { + encodeVarint(b, uint64(u)) + } + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) + return + } + for _, u := range x { + encodeInt64(b, tag, u) + } +} + +func encodeString(b *buffer, tag int, x string) { + encodeLength(b, tag, len(x)) + b.data = append(b.data, x...) +} + +func encodeStrings(b *buffer, tag int, x []string) { + for _, s := range x { + encodeString(b, tag, s) + } +} + +func encodeStringOpt(b *buffer, tag int, x string) { + if x == "" { + return + } + encodeString(b, tag, x) +} + +func encodeBool(b *buffer, tag int, x bool) { + if x { + encodeUint64(b, tag, 1) + } else { + encodeUint64(b, tag, 0) + } +} + +func encodeBoolOpt(b *buffer, tag int, x bool) { + if x == false { + return + } + encodeBool(b, tag, x) +} + +func encodeMessage(b *buffer, tag int, m message) { + n1 := len(b.data) + m.encode(b) + n2 := len(b.data) + encodeLength(b, tag, n2-n1) + n3 := len(b.data) + copy(b.tmp[:], b.data[n2:n3]) + copy(b.data[n1+(n3-n2):], b.data[n1:n2]) + copy(b.data[n1:], b.tmp[:n3-n2]) +} + +func unmarshal(data []byte, m message) (err error) { + b := buffer{data: data, typ: 2} + return decodeMessage(&b, m) +} + +func le64(p []byte) uint64 { + return uint64(p[0]) | uint64(p[1])<<8 | uint64(p[2])<<16 | uint64(p[3])<<24 | uint64(p[4])<<32 | uint64(p[5])<<40 | uint64(p[6])<<48 | uint64(p[7])<<56 +} + +func le32(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func decodeVarint(data []byte) (uint64, []byte, error) { + var i int + var u uint64 + for i = 0; ; i++ { + if i >= 10 || i >= len(data) { + return 0, nil, errors.New("bad varint") + } + u |= uint64(data[i]&0x7F) << uint(7*i) + if data[i]&0x80 == 0 { + return u, data[i+1:], nil + } + } +} + +func decodeField(b *buffer, data []byte) ([]byte, error) { + x, data, err := decodeVarint(data) + if err != nil { + return nil, err + } + b.field = int(x >> 3) + b.typ = int(x & 7) + b.data = nil + b.u64 = 0 + switch b.typ { + case 0: + b.u64, data, err = decodeVarint(data) + if err != nil { + return nil, err + } + case 1: + if len(data) < 8 { + return nil, errors.New("not enough data") + } + b.u64 = le64(data[:8]) + data = data[8:] + case 2: + var n uint64 + n, data, err = decodeVarint(data) + if err != nil { + return nil, err + } + if n > uint64(len(data)) { + return nil, errors.New("too much data") + } + b.data = data[:n] + data = data[n:] + case 5: + if len(data) < 4 { + return nil, errors.New("not enough data") + } + b.u64 = uint64(le32(data[:4])) + data = data[4:] + default: + return nil, errors.New("unknown type: " + string(b.typ)) + } + + return data, nil +} + +func checkType(b *buffer, typ int) error { + if b.typ != typ { + return errors.New("type mismatch") + } + return nil +} + +func decodeMessage(b *buffer, m message) error { + if err := checkType(b, 2); err != nil { + return err + } + dec := m.decoder() + data := b.data + for len(data) > 0 { + // pull varint field# + type + var err error + data, err = decodeField(b, data) + if err != nil { + return err + } + if b.field >= len(dec) || dec[b.field] == nil { + continue + } + if err := dec[b.field](b, m); err != nil { + return err + } + } + return nil +} + +func decodeInt64(b *buffer, x *int64) error { + if err := checkType(b, 0); err != nil { + return err + } + *x = int64(b.u64) + return nil +} + +func decodeInt64s(b *buffer, x *[]int64) error { + if b.typ == 2 { + // Packed encoding + data := b.data + for len(data) > 0 { + var u uint64 + var err error + + if u, data, err = decodeVarint(data); err != nil { + return err + } + *x = append(*x, int64(u)) + } + return nil + } + var i int64 + if err := decodeInt64(b, &i); err != nil { + return err + } + *x = append(*x, i) + return nil +} + +func decodeUint64(b *buffer, x *uint64) error { + if err := checkType(b, 0); err != nil { + return err + } + *x = b.u64 + return nil +} + +func decodeUint64s(b *buffer, x *[]uint64) error { + if b.typ == 2 { + data := b.data + // Packed encoding + for len(data) > 0 { + var u uint64 + var err error + + if u, data, err = decodeVarint(data); err != nil { + return err + } + *x = append(*x, u) + } + return nil + } + var u uint64 + if err := decodeUint64(b, &u); err != nil { + return err + } + *x = append(*x, u) + return nil +} + +func decodeString(b *buffer, x *string) error { + if err := checkType(b, 2); err != nil { + return err + } + *x = string(b.data) + return nil +} + +func decodeStrings(b *buffer, x *[]string) error { + var s string + if err := decodeString(b, &s); err != nil { + return err + } + *x = append(*x, s) + return nil +} + +func decodeBool(b *buffer, x *bool) error { + if err := checkType(b, 0); err != nil { + return err + } + if int64(b.u64) == 0 { + *x = false + } else { + *x = true + } + return nil +} diff --git a/src/internal/pprof/profile/proto_test.go b/src/internal/pprof/profile/proto_test.go new file mode 100644 index 0000000000..c2613fc375 --- /dev/null +++ b/src/internal/pprof/profile/proto_test.go @@ -0,0 +1,67 @@ +package profile + +import ( + "reflect" + "testing" +) + +func TestPackedEncoding(t *testing.T) { + + type testcase struct { + uint64s []uint64 + int64s []int64 + encoded []byte + } + for i, tc := range []testcase{ + { + []uint64{0, 1, 10, 100, 1000, 10000}, + []int64{1000, 0, 1000}, + []byte{10, 8, 0, 1, 10, 100, 232, 7, 144, 78, 18, 5, 232, 7, 0, 232, 7}, + }, + { + []uint64{10000}, + nil, + []byte{8, 144, 78}, + }, + { + nil, + []int64{-10000}, + []byte{16, 240, 177, 255, 255, 255, 255, 255, 255, 255, 1}, + }, + } { + source := &packedInts{tc.uint64s, tc.int64s} + if got, want := marshal(source), tc.encoded; !reflect.DeepEqual(got, want) { + t.Errorf("failed encode %d, got %v, want %v", i, got, want) + } + + dest := new(packedInts) + if err := unmarshal(tc.encoded, dest); err != nil { + t.Errorf("failed decode %d: %v", i, err) + continue + } + if got, want := dest.uint64s, tc.uint64s; !reflect.DeepEqual(got, want) { + t.Errorf("failed decode uint64s %d, got %v, want %v", i, got, want) + } + if got, want := dest.int64s, tc.int64s; !reflect.DeepEqual(got, want) { + t.Errorf("failed decode int64s %d, got %v, want %v", i, got, want) + } + } +} + +type packedInts struct { + uint64s []uint64 + int64s []int64 +} + +func (u *packedInts) decoder() []decoder { + return []decoder{ + nil, + func(b *buffer, m message) error { return decodeUint64s(b, &m.(*packedInts).uint64s) }, + func(b *buffer, m message) error { return decodeInt64s(b, &m.(*packedInts).int64s) }, + } +} + +func (u *packedInts) encode(b *buffer) { + encodeUint64s(b, 1, u.uint64s) + encodeInt64s(b, 2, u.int64s) +} diff --git a/src/internal/pprof/profile/prune.go b/src/internal/pprof/profile/prune.go new file mode 100644 index 0000000000..1924fada7a --- /dev/null +++ b/src/internal/pprof/profile/prune.go @@ -0,0 +1,97 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Implements methods to remove frames from profiles. + +package profile + +import ( + "fmt" + "regexp" +) + +// Prune removes all nodes beneath a node matching dropRx, and not +// matching keepRx. If the root node of a Sample matches, the sample +// will have an empty stack. +func (p *Profile) Prune(dropRx, keepRx *regexp.Regexp) { + prune := make(map[uint64]bool) + pruneBeneath := make(map[uint64]bool) + + for _, loc := range p.Location { + var i int + for i = len(loc.Line) - 1; i >= 0; i-- { + if fn := loc.Line[i].Function; fn != nil && fn.Name != "" { + funcName := fn.Name + // Account for leading '.' on the PPC ELF v1 ABI. + if funcName[0] == '.' { + funcName = funcName[1:] + } + if dropRx.MatchString(funcName) { + if keepRx == nil || !keepRx.MatchString(funcName) { + break + } + } + } + } + + if i >= 0 { + // Found matching entry to prune. + pruneBeneath[loc.ID] = true + + // Remove the matching location. + if i == len(loc.Line)-1 { + // Matched the top entry: prune the whole location. + prune[loc.ID] = true + } else { + loc.Line = loc.Line[i+1:] + } + } + } + + // Prune locs from each Sample + for _, sample := range p.Sample { + // Scan from the root to the leaves to find the prune location. + // Do not prune frames before the first user frame, to avoid + // pruning everything. + foundUser := false + for i := len(sample.Location) - 1; i >= 0; i-- { + id := sample.Location[i].ID + if !prune[id] && !pruneBeneath[id] { + foundUser = true + continue + } + if !foundUser { + continue + } + if prune[id] { + sample.Location = sample.Location[i+1:] + break + } + if pruneBeneath[id] { + sample.Location = sample.Location[i:] + break + } + } + } +} + +// RemoveUninteresting prunes and elides profiles using built-in +// tables of uninteresting function names. +func (p *Profile) RemoveUninteresting() error { + var keep, drop *regexp.Regexp + var err error + + if p.DropFrames != "" { + if drop, err = regexp.Compile("^(" + p.DropFrames + ")$"); err != nil { + return fmt.Errorf("failed to compile regexp %s: %v", p.DropFrames, err) + } + if p.KeepFrames != "" { + if keep, err = regexp.Compile("^(" + p.KeepFrames + ")$"); err != nil { + return fmt.Errorf("failed to compile regexp %s: %v", p.KeepFrames, err) + } + } + p.Prune(drop, keep) + } + return nil +} -- cgit v1.3