From cc700bdc269edc5fd29b14c1866c7f57f6f9b526 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 19 Jun 2020 17:59:16 -0400 Subject: cmd/trace: move viewer data structs into cmd/internal/traceviewer The ViewerEvent, ViewerData and ViewerFrame structs are moved into cmd/internal/traceviewer, and renamed Event, Data, and Frame. The structs are the same, except for the following: A definition for the JSON "bp" field that's defined in the trace format, but missing in the structs has been added. Also, the Tid and Pid fields on Event have been renamed TID and PID to better match Go style. Finally, the footer field on ViewerData, which hasn't been used for a while, has been removed. This CL is in preparation for the usage of these structs by cmd/go's tracing functionality. Updates #38714 Change-Id: I345f23617b96d4629b876ae717f89d56a67e05a3 Reviewed-on: https://go-review.googlesource.com/c/go/+/239098 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/internal/traceviewer/format.go | 38 ++++++++++ src/cmd/trace/trace.go | 125 +++++++++++++-------------------- src/cmd/trace/trace_test.go | 7 +- src/cmd/trace/trace_unix_test.go | 3 +- 4 files changed, 92 insertions(+), 81 deletions(-) create mode 100644 src/cmd/internal/traceviewer/format.go (limited to 'src/cmd') diff --git a/src/cmd/internal/traceviewer/format.go b/src/cmd/internal/traceviewer/format.go new file mode 100644 index 0000000000..871477447f --- /dev/null +++ b/src/cmd/internal/traceviewer/format.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package traceviewer provides definitions of the JSON data structures +// used by the Chrome trace viewer. +// +// The official description of the format is in this file: +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +package traceviewer + +type Data struct { + Events []*Event `json:"traceEvents"` + Frames map[string]Frame `json:"stackFrames"` + TimeUnit string `json:"displayTimeUnit"` +} + +type Event struct { + Name string `json:"name,omitempty"` + Phase string `json:"ph"` + Scope string `json:"s,omitempty"` + Time float64 `json:"ts"` + Dur float64 `json:"dur,omitempty"` + PID uint64 `json:"pid"` + TID uint64 `json:"tid"` + ID uint64 `json:"id,omitempty"` + BindPoint string `json:"bp,omitempty"` + Stack int `json:"sf,omitempty"` + EndStack int `json:"esf,omitempty"` + Arg interface{} `json:"args,omitempty"` + Cname string `json:"cname,omitempty"` + Category string `json:"cat,omitempty"` +} + +type Frame struct { + Name string `json:"name"` + Parent int `json:"parent,omitempty"` +} diff --git a/src/cmd/trace/trace.go b/src/cmd/trace/trace.go index b452376627..30c80f0e04 100644 --- a/src/cmd/trace/trace.go +++ b/src/cmd/trace/trace.go @@ -5,6 +5,7 @@ package main import ( + "cmd/internal/traceviewer" "encoding/json" "fmt" "internal/trace" @@ -325,7 +326,7 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { } var ( - data = ViewerData{Frames: make(map[string]ViewerFrame)} + data = traceviewer.Data{Frames: make(map[string]traceviewer.Frame)} sizes []eventSz cw countingWriter @@ -337,7 +338,7 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { consumeTimeUnit: func(unit string) { data.TimeUnit = unit }, - consumeViewerEvent: func(v *ViewerEvent, required bool) { + consumeViewerEvent: func(v *traceviewer.Event, required bool) { if required { // Store required events inside data // so flush can include them in the required @@ -350,7 +351,7 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { sizes = append(sizes, eventSz{v.Time, cw.size + 1}) // +1 for ",". cw.size = 0 }, - consumeViewerFrame: func(k string, v ViewerFrame) { + consumeViewerFrame: func(k string, v traceviewer.Frame) { data.Frames[k] = v }, flush: func() { @@ -478,36 +479,6 @@ type gInfo struct { markAssist *trace.Event // if non-nil, the mark assist currently running. } -type ViewerData struct { - Events []*ViewerEvent `json:"traceEvents"` - Frames map[string]ViewerFrame `json:"stackFrames"` - TimeUnit string `json:"displayTimeUnit"` - - // This is where mandatory part of the trace starts (e.g. thread names) - footer int -} - -type ViewerEvent struct { - Name string `json:"name,omitempty"` - Phase string `json:"ph"` - Scope string `json:"s,omitempty"` - Time float64 `json:"ts"` - Dur float64 `json:"dur,omitempty"` - Pid uint64 `json:"pid"` - Tid uint64 `json:"tid"` - ID uint64 `json:"id,omitempty"` - Stack int `json:"sf,omitempty"` - EndStack int `json:"esf,omitempty"` - Arg interface{} `json:"args,omitempty"` - Cname string `json:"cname,omitempty"` - Category string `json:"cat,omitempty"` -} - -type ViewerFrame struct { - Name string `json:"name"` - Parent int `json:"parent,omitempty"` -} - type NameArg struct { Name string `json:"name"` } @@ -528,8 +499,8 @@ type SortIndexArg struct { type traceConsumer struct { consumeTimeUnit func(unit string) - consumeViewerEvent func(v *ViewerEvent, required bool) - consumeViewerFrame func(key string, f ViewerFrame) + consumeViewerEvent func(v *traceviewer.Event, required bool) + consumeViewerFrame func(key string, f traceviewer.Frame) flush func() } @@ -775,23 +746,23 @@ func generateTrace(params *traceParams, consumer traceConsumer) error { ctx.emitSectionFooter(procsSection, "PROCS", 2) } - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.GCP, Arg: &NameArg{"GC"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.GCP, Arg: &SortIndexArg{-6}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.GCP, Arg: &NameArg{"GC"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.GCP, Arg: &SortIndexArg{-6}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.NetpollP, Arg: &NameArg{"Network"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.NetpollP, Arg: &SortIndexArg{-5}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.NetpollP, Arg: &NameArg{"Network"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.NetpollP, Arg: &SortIndexArg{-5}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.TimerP, Arg: &NameArg{"Timers"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.TimerP, Arg: &SortIndexArg{-4}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.TimerP, Arg: &NameArg{"Timers"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.TimerP, Arg: &SortIndexArg{-4}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.SyscallP, Arg: &NameArg{"Syscalls"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.SyscallP, Arg: &SortIndexArg{-3}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.SyscallP, Arg: &NameArg{"Syscalls"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.SyscallP, Arg: &SortIndexArg{-3}}) // Display rows for Ps if we are in the default trace view mode (not goroutine-oriented presentation) if ctx.mode&modeGoroutineOriented == 0 { for i := 0; i <= maxProc; i++ { - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: uint64(i), Arg: &NameArg{fmt.Sprintf("Proc %v", i)}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: uint64(i), Arg: &SortIndexArg{i}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: uint64(i), Arg: &NameArg{fmt.Sprintf("Proc %v", i)}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: uint64(i), Arg: &SortIndexArg{i}}) } } @@ -829,27 +800,27 @@ func generateTrace(params *traceParams, consumer traceConsumer) error { if !ctx.gs[k] { continue } - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: k, Arg: &NameArg{v.name}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: k, Arg: &NameArg{v.name}}) } // Row for the main goroutine (maing) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: ctx.maing, Arg: &SortIndexArg{-2}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: ctx.maing, Arg: &SortIndexArg{-2}}) // Row for GC or global state (specified with G=0) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: 0, Arg: &SortIndexArg{-1}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: 0, Arg: &SortIndexArg{-1}}) } return nil } -func (ctx *traceContext) emit(e *ViewerEvent) { +func (ctx *traceContext) emit(e *traceviewer.Event) { ctx.consumer.consumeViewerEvent(e, false) } -func (ctx *traceContext) emitFooter(e *ViewerEvent) { +func (ctx *traceContext) emitFooter(e *traceviewer.Event) { ctx.consumer.consumeViewerEvent(e, true) } func (ctx *traceContext) emitSectionFooter(sectionID uint64, name string, priority int) { - ctx.emitFooter(&ViewerEvent{Name: "process_name", Phase: "M", Pid: sectionID, Arg: &NameArg{name}}) - ctx.emitFooter(&ViewerEvent{Name: "process_sort_index", Phase: "M", Pid: sectionID, Arg: &SortIndexArg{priority}}) + ctx.emitFooter(&traceviewer.Event{Name: "process_name", Phase: "M", PID: sectionID, Arg: &NameArg{name}}) + ctx.emitFooter(&traceviewer.Event{Name: "process_sort_index", Phase: "M", PID: sectionID, Arg: &SortIndexArg{priority}}) } func (ctx *traceContext) time(ev *trace.Event) float64 { @@ -880,7 +851,7 @@ func (ctx *traceContext) emitSlice(ev *trace.Event, name string) { ctx.emit(ctx.makeSlice(ev, name)) } -func (ctx *traceContext) makeSlice(ev *trace.Event, name string) *ViewerEvent { +func (ctx *traceContext) makeSlice(ev *trace.Event, name string) *traceviewer.Event { // If ViewerEvent.Dur is not a positive value, // trace viewer handles it as a non-terminating time interval. // Avoid it by setting the field with a small value. @@ -888,12 +859,12 @@ func (ctx *traceContext) makeSlice(ev *trace.Event, name string) *ViewerEvent { if ev.Link.Ts-ev.Ts <= 0 { durationUsec = 0.0001 // 0.1 nanoseconds } - sl := &ViewerEvent{ + sl := &traceviewer.Event{ Name: name, Phase: "X", Time: ctx.time(ev), Dur: durationUsec, - Tid: ctx.proc(ev), + TID: ctx.proc(ev), Stack: ctx.stack(ev.Stk), EndStack: ctx.stack(ev.Link.Stk), } @@ -927,16 +898,16 @@ func (ctx *traceContext) emitTask(task *taskDesc, sortIndex int) { taskName := task.name durationUsec := float64(task.lastTimestamp()-task.firstTimestamp()) / 1e3 - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: tasksSection, Tid: taskRow, Arg: &NameArg{fmt.Sprintf("T%d %s", task.id, taskName)}}) - ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: tasksSection, Tid: taskRow, Arg: &SortIndexArg{sortIndex}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: tasksSection, TID: taskRow, Arg: &NameArg{fmt.Sprintf("T%d %s", task.id, taskName)}}) + ctx.emit(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: tasksSection, TID: taskRow, Arg: &SortIndexArg{sortIndex}}) ts := float64(task.firstTimestamp()) / 1e3 - sl := &ViewerEvent{ + sl := &traceviewer.Event{ Name: taskName, Phase: "X", Time: ts, Dur: durationUsec, - Pid: tasksSection, - Tid: taskRow, + PID: tasksSection, + TID: taskRow, Cname: pickTaskColor(task.id), } targ := TaskArg{ID: task.id} @@ -953,8 +924,8 @@ func (ctx *traceContext) emitTask(task *taskDesc, sortIndex int) { if task.create != nil && task.create.Type == trace.EvUserTaskCreate && task.create.Args[1] != 0 { ctx.arrowSeq++ - ctx.emit(&ViewerEvent{Name: "newTask", Phase: "s", Tid: task.create.Args[1], ID: ctx.arrowSeq, Time: ts, Pid: tasksSection}) - ctx.emit(&ViewerEvent{Name: "newTask", Phase: "t", Tid: taskRow, ID: ctx.arrowSeq, Time: ts, Pid: tasksSection}) + ctx.emit(&traceviewer.Event{Name: "newTask", Phase: "s", TID: task.create.Args[1], ID: ctx.arrowSeq, Time: ts, PID: tasksSection}) + ctx.emit(&traceviewer.Event{Name: "newTask", Phase: "t", TID: taskRow, ID: ctx.arrowSeq, Time: ts, PID: tasksSection}) } } @@ -975,12 +946,12 @@ func (ctx *traceContext) emitRegion(s regionDesc) { scopeID := fmt.Sprintf("%x", id) name := s.Name - sl0 := &ViewerEvent{ + sl0 := &traceviewer.Event{ Category: "Region", Name: name, Phase: "b", Time: float64(s.firstTimestamp()) / 1e3, - Tid: s.G, // only in goroutine-oriented view + TID: s.G, // only in goroutine-oriented view ID: uint64(regionID), Scope: scopeID, Cname: pickTaskColor(s.TaskID), @@ -990,12 +961,12 @@ func (ctx *traceContext) emitRegion(s regionDesc) { } ctx.emit(sl0) - sl1 := &ViewerEvent{ + sl1 := &traceviewer.Event{ Category: "Region", Name: name, Phase: "e", Time: float64(s.lastTimestamp()) / 1e3, - Tid: s.G, + TID: s.G, ID: uint64(regionID), Scope: scopeID, Cname: pickTaskColor(s.TaskID), @@ -1021,7 +992,7 @@ func (ctx *traceContext) emitHeapCounters(ev *trace.Event) { diff = ctx.heapStats.nextGC - ctx.heapStats.heapAlloc } if tsWithinRange(ev.Ts, ctx.startTime, ctx.endTime) { - ctx.emit(&ViewerEvent{Name: "Heap", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &heapCountersArg{ctx.heapStats.heapAlloc, diff}}) + ctx.emit(&traceviewer.Event{Name: "Heap", Phase: "C", Time: ctx.time(ev), PID: 1, Arg: &heapCountersArg{ctx.heapStats.heapAlloc, diff}}) } ctx.prevHeapStats = ctx.heapStats } @@ -1037,7 +1008,7 @@ func (ctx *traceContext) emitGoroutineCounters(ev *trace.Event) { return } if tsWithinRange(ev.Ts, ctx.startTime, ctx.endTime) { - ctx.emit(&ViewerEvent{Name: "Goroutines", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &goroutineCountersArg{uint64(ctx.gstates[gRunning]), uint64(ctx.gstates[gRunnable]), uint64(ctx.gstates[gWaitingGC])}}) + ctx.emit(&traceviewer.Event{Name: "Goroutines", Phase: "C", Time: ctx.time(ev), PID: 1, Arg: &goroutineCountersArg{uint64(ctx.gstates[gRunning]), uint64(ctx.gstates[gRunnable]), uint64(ctx.gstates[gWaitingGC])}}) } ctx.prevGstates = ctx.gstates } @@ -1052,7 +1023,7 @@ func (ctx *traceContext) emitThreadCounters(ev *trace.Event) { return } if tsWithinRange(ev.Ts, ctx.startTime, ctx.endTime) { - ctx.emit(&ViewerEvent{Name: "Threads", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &threadCountersArg{ + ctx.emit(&traceviewer.Event{Name: "Threads", Phase: "C", Time: ctx.time(ev), PID: 1, Arg: &threadCountersArg{ Running: ctx.threadStats.prunning, InSyscall: ctx.threadStats.insyscall}}) } @@ -1090,13 +1061,13 @@ func (ctx *traceContext) emitInstant(ev *trace.Event, name, category string) { } arg = &Arg{ev.Args[0]} } - ctx.emit(&ViewerEvent{ + ctx.emit(&traceviewer.Event{ Name: name, Category: category, Phase: "I", Scope: "t", Time: ctx.time(ev), - Tid: ctx.proc(ev), + TID: ctx.proc(ev), Stack: ctx.stack(ev.Stk), Cname: cname, Arg: arg}) @@ -1134,8 +1105,8 @@ func (ctx *traceContext) emitArrow(ev *trace.Event, name string) { } ctx.arrowSeq++ - ctx.emit(&ViewerEvent{Name: name, Phase: "s", Tid: ctx.proc(ev), ID: ctx.arrowSeq, Time: ctx.time(ev), Stack: ctx.stack(ev.Stk), Cname: color}) - ctx.emit(&ViewerEvent{Name: name, Phase: "t", Tid: ctx.proc(ev.Link), ID: ctx.arrowSeq, Time: ctx.time(ev.Link), Cname: color}) + ctx.emit(&traceviewer.Event{Name: name, Phase: "s", TID: ctx.proc(ev), ID: ctx.arrowSeq, Time: ctx.time(ev), Stack: ctx.stack(ev.Stk), Cname: color}) + ctx.emit(&traceviewer.Event{Name: name, Phase: "t", TID: ctx.proc(ev.Link), ID: ctx.arrowSeq, Time: ctx.time(ev.Link), Cname: color}) } func (ctx *traceContext) stack(stk []*trace.Frame) int { @@ -1157,7 +1128,7 @@ func (ctx *traceContext) buildBranch(parent frameNode, stk []*trace.Frame) int { node.id = ctx.frameSeq node.children = make(map[uint64]frameNode) parent.children[frame.PC] = node - ctx.consumer.consumeViewerFrame(strconv.Itoa(node.id), ViewerFrame{fmt.Sprintf("%v:%v", frame.Fn, frame.Line), parent.id}) + ctx.consumer.consumeViewerFrame(strconv.Itoa(node.id), traceviewer.Frame{Name: fmt.Sprintf("%v:%v", frame.Fn, frame.Line), Parent: parent.id}) } return ctx.buildBranch(node, stk) } @@ -1192,7 +1163,7 @@ type jsonWriter struct { } func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { - frames := make(map[string]ViewerFrame) + frames := make(map[string]traceviewer.Frame) enc := json.NewEncoder(w) written := 0 index := int64(-1) @@ -1204,7 +1175,7 @@ func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { enc.Encode(unit) io.WriteString(w, ",") }, - consumeViewerEvent: func(v *ViewerEvent, required bool) { + consumeViewerEvent: func(v *traceviewer.Event, required bool) { index++ if !required && (index < start || index > end) { // not in the range. Skip! @@ -1221,7 +1192,7 @@ func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { // Same should be applied to splittingTraceConsumer. written++ }, - consumeViewerFrame: func(k string, v ViewerFrame) { + consumeViewerFrame: func(k string, v traceviewer.Frame) { frames[k] = v }, flush: func() { diff --git a/src/cmd/trace/trace_test.go b/src/cmd/trace/trace_test.go index ef2d06c961..dd12e8cd20 100644 --- a/src/cmd/trace/trace_test.go +++ b/src/cmd/trace/trace_test.go @@ -7,6 +7,7 @@ package main import ( + "cmd/internal/traceviewer" "context" "internal/trace" "io/ioutil" @@ -78,7 +79,7 @@ func TestGoroutineCount(t *testing.T) { // Use the default viewerDataTraceConsumer but replace // consumeViewerEvent to intercept the ViewerEvents for testing. c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if ev.Name == "Goroutines" { cnt := ev.Arg.(*goroutineCountersArg) if cnt.Runnable+cnt.Running > 2 { @@ -165,7 +166,7 @@ func TestPreemptedMarkAssist(t *testing.T) { c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) marks := 0 - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if strings.Contains(ev.Name, "MARK ASSIST") { marks++ } @@ -216,7 +217,7 @@ func TestFoo(t *testing.T) { c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) var logBeforeTaskEnd, logAfterTaskEnd bool - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if ev.Name == "log before task ends" { logBeforeTaskEnd = true } diff --git a/src/cmd/trace/trace_unix_test.go b/src/cmd/trace/trace_unix_test.go index fec060e121..645978e0f8 100644 --- a/src/cmd/trace/trace_unix_test.go +++ b/src/cmd/trace/trace_unix_test.go @@ -8,6 +8,7 @@ package main import ( "bytes" + "cmd/internal/traceviewer" traceparser "internal/trace" "io/ioutil" "runtime" @@ -83,7 +84,7 @@ func TestGoroutineInSyscall(t *testing.T) { // Check only one thread for the pipe read goroutine is // considered in-syscall. c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if ev.Name == "Threads" { arg := ev.Arg.(*threadCountersArg) if arg.InSyscall > 1 { -- cgit v1.3 From 20254c02b38055c1bb6d5ac8aba70bb7b8d638ef Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 14 Apr 2020 17:29:27 -0400 Subject: cmd/go: add rudimentary tracing support. Updates #38714 Change-Id: I14da982d405074d65ccf5521d431df1bf1734f9a Reviewed-on: https://go-review.googlesource.com/c/go/+/230378 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/trace/trace.go | 139 +++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/cmd/go/internal/trace/trace.go (limited to 'src/cmd') diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go new file mode 100644 index 0000000000..7cb7636a34 --- /dev/null +++ b/src/cmd/go/internal/trace/trace.go @@ -0,0 +1,139 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package trace + +import ( + "cmd/internal/traceviewer" + "context" + "encoding/json" + "errors" + "os" + "strings" + "sync/atomic" + "time" +) + +var traceStarted int32 + +func getTraceContext(ctx context.Context) (traceContext, bool) { + if atomic.LoadInt32(&traceStarted) == 0 { + return traceContext{}, false + } + v := ctx.Value(traceKey{}) + if v == nil { + return traceContext{}, false + } + return v.(traceContext), true +} + +// StartSpan starts a trace event with the given name. The Span ends when its Done method is called. +func StartSpan(ctx context.Context, name string) (context.Context, *Span) { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx, nil + } + childSpan := &Span{t: tc.t, name: name, start: time.Now()} + tc.t.writeEvent(&traceviewer.Event{ + Name: childSpan.name, + Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), + Phase: "B", + }) + ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t}) + return ctx, childSpan +} + +type Span struct { + t *tracer + + name string + start time.Time + end time.Time +} + +func (s *Span) Done() { + if s == nil { + return + } + s.end = time.Now() + s.t.writeEvent(&traceviewer.Event{ + Name: s.name, + Time: float64(s.end.UnixNano()) / float64(time.Microsecond), + Phase: "E", + }) +} + +type tracer struct { + file chan traceFile // 1-buffered +} + +func (t *tracer) writeEvent(ev *traceviewer.Event) error { + f := <-t.file + defer func() { t.file <- f }() + var err error + if f.entries == 0 { + _, err = f.sb.WriteString("[\n") + } else { + _, err = f.sb.WriteString(",") + } + f.entries++ + if err != nil { + return nil + } + + if err := f.enc.Encode(ev); err != nil { + return err + } + + // Write event string to output file. + _, err = f.f.WriteString(f.sb.String()) + f.sb.Reset() + return err +} + +func (t *tracer) Close() error { + f := <-t.file + defer func() { t.file <- f }() + + _, firstErr := f.f.WriteString("]") + if err := f.f.Close(); firstErr == nil { + firstErr = err + } + return firstErr +} + +// traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in +// other packages. +type traceKey struct{} + +type traceContext struct { + t *tracer +} + +// Start starts a trace which writes to the given file. +func Start(ctx context.Context, file string) (context.Context, func() error, error) { + atomic.StoreInt32(&traceStarted, 1) + if file == "" { + return nil, nil, errors.New("no trace file supplied") + } + f, err := os.Create(file) + if err != nil { + return nil, nil, err + } + t := &tracer{file: make(chan traceFile, 1)} + sb := new(strings.Builder) + t.file <- traceFile{ + f: f, + sb: sb, + enc: json.NewEncoder(sb), + } + ctx = context.WithValue(ctx, traceKey{}, traceContext{t: t}) + return ctx, t.Close, nil +} + +type traceFile struct { + f *os.File + sb *strings.Builder + enc *json.Encoder + entries int64 +} -- cgit v1.3 From 52b0ea20ff10fdcfe570ef407bd462d23e13d782 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 14 Apr 2020 17:29:27 -0400 Subject: cmd/go: add a debug-trace flag to generate traces If cmd/go is provided with a -debug-trace= option, cmd/go will write an execution trace to that file. Updates #38714 Change-Id: I3e6521343902c08266a0292f4280298a3bf8b725 Reviewed-on: https://go-review.googlesource.com/c/go/+/237683 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/cfg/cfg.go | 1 + src/cmd/go/internal/work/build.go | 1 + src/cmd/go/main.go | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 7f8f8e92be..f9bbcd9180 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -51,6 +51,7 @@ var ( CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) + DebugTrace string // -debug-trace flag ) func defaultContext() build.Context { diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 7146c9ce00..fbd49b457b 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -270,6 +270,7 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") + cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") } // AddModCommonFlags adds the module-related flags common to build commands diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index fdf49b7380..3512866e4a 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "flag" "fmt" "log" @@ -34,6 +35,7 @@ import ( "cmd/go/internal/run" "cmd/go/internal/test" "cmd/go/internal/tool" + "cmd/go/internal/trace" "cmd/go/internal/version" "cmd/go/internal/vet" "cmd/go/internal/work" @@ -187,7 +189,11 @@ BigCmdLoop: cmd.Flag.Parse(args[1:]) args = cmd.Flag.Args() } + ctx := maybeStartTrace(context.Background()) + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + _ = ctx cmd.Run(cmd, args) + span.Done() base.Exit() return } @@ -209,3 +215,21 @@ func mainUsage() { help.PrintUsage(os.Stderr, base.Go) os.Exit(2) } + +func maybeStartTrace(pctx context.Context) context.Context { + if cfg.DebugTrace == "" { + return pctx + } + + ctx, close, err := trace.Start(pctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + base.AtExit(func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }) + + return ctx +} -- cgit v1.3 From 14715b24793176b30b3c41adf4ac4e676c2a56f0 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 12 Jun 2020 14:33:23 -0400 Subject: cmd/go: add Context parameter to base.command.Run One small step to start propagating the context in cmd/go for tracing purposes. Updates #38714 Change-Id: Ibb6debeb9233f84d55f0e81244487355cbe7b82c Reviewed-on: https://go-review.googlesource.com/c/go/+/237684 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/base/base.go | 3 ++- src/cmd/go/internal/bug/bug.go | 3 ++- src/cmd/go/internal/clean/clean.go | 3 ++- src/cmd/go/internal/doc/doc.go | 3 ++- src/cmd/go/internal/envcmd/env.go | 3 ++- src/cmd/go/internal/fix/fix.go | 3 ++- src/cmd/go/internal/fmtcmd/fmt.go | 3 ++- src/cmd/go/internal/generate/generate.go | 3 ++- src/cmd/go/internal/get/get.go | 3 ++- src/cmd/go/internal/list/list.go | 3 ++- src/cmd/go/internal/modcmd/download.go | 3 ++- src/cmd/go/internal/modcmd/edit.go | 3 ++- src/cmd/go/internal/modcmd/graph.go | 3 ++- src/cmd/go/internal/modcmd/init.go | 3 ++- src/cmd/go/internal/modcmd/tidy.go | 3 ++- src/cmd/go/internal/modcmd/vendor.go | 3 ++- src/cmd/go/internal/modcmd/verify.go | 3 ++- src/cmd/go/internal/modcmd/why.go | 3 ++- src/cmd/go/internal/modget/get.go | 3 ++- src/cmd/go/internal/run/run.go | 3 ++- src/cmd/go/internal/test/test.go | 3 ++- src/cmd/go/internal/tool/tool.go | 3 ++- src/cmd/go/internal/version/version.go | 3 ++- src/cmd/go/internal/vet/vet.go | 3 ++- src/cmd/go/internal/work/build.go | 5 +++-- src/cmd/go/main.go | 3 +-- 26 files changed, 52 insertions(+), 28 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/base/base.go b/src/cmd/go/internal/base/base.go index ab2f1bb4e2..db3ebef933 100644 --- a/src/cmd/go/internal/base/base.go +++ b/src/cmd/go/internal/base/base.go @@ -7,6 +7,7 @@ package base import ( + "context" "flag" "fmt" "log" @@ -24,7 +25,7 @@ import ( type Command struct { // Run runs the command. // The args are the arguments after the command name. - Run func(cmd *Command, args []string) + Run func(ctx context.Context, cmd *Command, args []string) // UsageLine is the one-line usage message. // The words between "go" and the first flag or argument in the line are taken to be the command name. diff --git a/src/cmd/go/internal/bug/bug.go b/src/cmd/go/internal/bug/bug.go index fe71281ef0..52bd40f2fb 100644 --- a/src/cmd/go/internal/bug/bug.go +++ b/src/cmd/go/internal/bug/bug.go @@ -7,6 +7,7 @@ package bug import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -37,7 +38,7 @@ func init() { CmdBug.Flag.BoolVar(&cfg.BuildV, "v", false, "") } -func runBug(cmd *base.Command, args []string) { +func runBug(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go bug: bug takes no arguments") } diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 99704cb2b1..8af3e3df9c 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -6,6 +6,7 @@ package clean import ( + "context" "fmt" "io/ioutil" "os" @@ -105,7 +106,7 @@ func init() { work.AddBuildFlags(CmdClean, work.DefaultBuildFlags) } -func runClean(cmd *base.Command, args []string) { +func runClean(ctx context.Context, cmd *base.Command, args []string) { // golang.org/issue/29925: only load packages before cleaning if // either the flags and arguments explicitly imply a package, // or no other target (such as a cache) was requested to be cleaned. diff --git a/src/cmd/go/internal/doc/doc.go b/src/cmd/go/internal/doc/doc.go index 4ff08bb928..67f76e2256 100644 --- a/src/cmd/go/internal/doc/doc.go +++ b/src/cmd/go/internal/doc/doc.go @@ -8,6 +8,7 @@ package doc import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "context" ) var CmdDoc = &base.Command{ @@ -129,6 +130,6 @@ Flags: `, } -func runDoc(cmd *base.Command, args []string) { +func runDoc(ctx context.Context, cmd *base.Command, args []string) { base.Run(cfg.BuildToolexec, base.Tool("doc"), args) } diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 252025dc25..403e0f4a7b 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -6,6 +6,7 @@ package envcmd import ( + "context" "encoding/json" "fmt" "go/build" @@ -186,7 +187,7 @@ func argKey(arg string) string { return arg[:i] } -func runEnv(cmd *base.Command, args []string) { +func runEnv(ctx context.Context, cmd *base.Command, args []string) { if *envJson && *envU { base.Fatalf("go env: cannot use -json with -u") } diff --git a/src/cmd/go/internal/fix/fix.go b/src/cmd/go/internal/fix/fix.go index 4d741df2b4..f16af05fc8 100644 --- a/src/cmd/go/internal/fix/fix.go +++ b/src/cmd/go/internal/fix/fix.go @@ -11,6 +11,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/str" + "context" "fmt" "os" ) @@ -31,7 +32,7 @@ See also: go fmt, go vet. `, } -func runFix(cmd *base.Command, args []string) { +func runFix(ctx context.Context, cmd *base.Command, args []string) { printed := false for _, pkg := range load.Packages(args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index d6894edc9f..9868efc7ef 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -6,6 +6,7 @@ package fmtcmd import ( + "context" "errors" "fmt" "os" @@ -48,7 +49,7 @@ See also: go fix, go vet. `, } -func runFmt(cmd *base.Command, args []string) { +func runFmt(ctx context.Context, cmd *base.Command, args []string) { printed := false gofmt := gofmtPath() procs := runtime.GOMAXPROCS(0) diff --git a/src/cmd/go/internal/generate/generate.go b/src/cmd/go/internal/generate/generate.go index 093b19817b..fb26f77f95 100644 --- a/src/cmd/go/internal/generate/generate.go +++ b/src/cmd/go/internal/generate/generate.go @@ -8,6 +8,7 @@ package generate import ( "bufio" "bytes" + "context" "fmt" "go/parser" "go/token" @@ -160,7 +161,7 @@ func init() { CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") } -func runGenerate(cmd *base.Command, args []string) { +func runGenerate(ctx context.Context, cmd *base.Command, args []string) { load.IgnoreImports = true if generateRunFlag != "" { diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index d38350c2a8..f7da5270b0 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -6,6 +6,7 @@ package get import ( + "context" "fmt" "os" "path/filepath" @@ -112,7 +113,7 @@ func init() { CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") } -func runGet(cmd *base.Command, args []string) { +func runGet(ctx context.Context, cmd *base.Command, args []string) { if cfg.ModulesEnabled { // Should not happen: main.go should install the separate module-enabled get code. base.Fatalf("go get: modules not implemented") diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 6ca1561121..ef0a5a2f2d 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -8,6 +8,7 @@ package list import ( "bufio" "bytes" + "context" "encoding/json" "io" "os" @@ -309,7 +310,7 @@ var ( var nl = []byte{'\n'} -func runList(cmd *base.Command, args []string) { +func runList(ctx context.Context, cmd *base.Command, args []string) { modload.LoadTests = *listTest work.BuildInit() out := newTrackingWriter(os.Stdout) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 584434935b..b43c32be5a 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -5,6 +5,7 @@ package modcmd import ( + "context" "encoding/json" "os" @@ -78,7 +79,7 @@ type moduleJSON struct { GoModSum string `json:",omitempty"` } -func runDownload(cmd *base.Command, args []string) { +func runDownload(ctx context.Context, cmd *base.Command, args []string) { // Check whether modules are enabled and whether we're in a module. if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") diff --git a/src/cmd/go/internal/modcmd/edit.go b/src/cmd/go/internal/modcmd/edit.go index dbbfb96e42..a81c25270f 100644 --- a/src/cmd/go/internal/modcmd/edit.go +++ b/src/cmd/go/internal/modcmd/edit.go @@ -8,6 +8,7 @@ package modcmd import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -141,7 +142,7 @@ func init() { base.AddBuildFlagsNX(&cmdEdit.Flag) } -func runEdit(cmd *base.Command, args []string) { +func runEdit(ctx context.Context, cmd *base.Command, args []string) { anyFlags := *editModule != "" || *editGo != "" || diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 27ae9354f3..fff5b02626 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -8,6 +8,7 @@ package modcmd import ( "bufio" + "context" "os" "sort" @@ -36,7 +37,7 @@ func init() { work.AddModCommonFlags(cmdGraph) } -func runGraph(cmd *base.Command, args []string) { +func runGraph(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod graph: graph takes no arguments") } diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 714ff2e205..ddb9aeebe9 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -10,6 +10,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" "cmd/go/internal/work" + "context" "os" "strings" ) @@ -32,7 +33,7 @@ func init() { work.AddModCommonFlags(cmdInit) } -func runInit(cmd *base.Command, args []string) { +func runInit(ctx context.Context, cmd *base.Command, args []string) { modload.CmdModInit = true if len(args) > 1 { base.Fatalf("go mod init: too many arguments") diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index af2b04c0c2..feb41a83b0 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -12,6 +12,7 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/modload" "cmd/go/internal/work" + "context" "golang.org/x/mod/module" ) @@ -37,7 +38,7 @@ func init() { work.AddModCommonFlags(cmdTidy) } -func runTidy(cmd *base.Command, args []string) { +func runTidy(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod tidy: no arguments allowed") } diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 8509ceb7a8..257d1cd0ef 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -6,6 +6,7 @@ package modcmd import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -43,7 +44,7 @@ func init() { work.AddModCommonFlags(cmdVendor) } -func runVendor(cmd *base.Command, args []string) { +func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index b7fd7fa8e0..570e571049 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -6,6 +6,7 @@ package modcmd import ( "bytes" + "context" "errors" "fmt" "io/ioutil" @@ -40,7 +41,7 @@ func init() { work.AddModCommonFlags(cmdVerify) } -func runVerify(cmd *base.Command, args []string) { +func runVerify(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { // NOTE(rsc): Could take a module pattern. base.Fatalf("go mod verify: verify takes no arguments") diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index 40d238519b..3f9cf0f120 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -5,6 +5,7 @@ package modcmd import ( + "context" "fmt" "strings" @@ -60,7 +61,7 @@ func init() { work.AddModCommonFlags(cmdWhy) } -func runWhy(cmd *base.Command, args []string) { +func runWhy(ctx context.Context, cmd *base.Command, args []string) { loadALL := modload.LoadALL if *whyVendor { loadALL = modload.LoadVendor diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 4c6982426f..9836a3e2cc 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -6,6 +6,7 @@ package modget import ( + "context" "errors" "fmt" "os" @@ -259,7 +260,7 @@ type query struct { m module.Version } -func runGet(cmd *base.Command, args []string) { +func runGet(ctx context.Context, cmd *base.Command, args []string) { switch getU { case "", "upgrade", "patch": // ok diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 2edae38cca..ca2c3db92c 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -6,6 +6,7 @@ package run import ( + "context" "fmt" "os" "path" @@ -57,7 +58,7 @@ func printStderr(args ...interface{}) (int, error) { return fmt.Fprint(os.Stderr, args...) } -func runRun(cmd *base.Command, args []string) { +func runRun(ctx context.Context, cmd *base.Command, args []string) { work.BuildInit() var b work.Builder b.Init() diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 873a76aa38..6648d4eab4 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -6,6 +6,7 @@ package test import ( "bytes" + "context" "crypto/sha256" "errors" "fmt" @@ -565,7 +566,7 @@ var defaultVetFlags = []string{ // "-unusedresult", } -func runTest(cmd *base.Command, args []string) { +func runTest(ctx context.Context, cmd *base.Command, args []string) { modload.LoadTests = true pkgArgs, testArgs = testFlags(args) diff --git a/src/cmd/go/internal/tool/tool.go b/src/cmd/go/internal/tool/tool.go index 930eecb63f..7f4dc86802 100644 --- a/src/cmd/go/internal/tool/tool.go +++ b/src/cmd/go/internal/tool/tool.go @@ -6,6 +6,7 @@ package tool import ( + "context" "fmt" "os" "os/exec" @@ -48,7 +49,7 @@ func init() { CmdTool.Flag.BoolVar(&toolN, "n", false, "") } -func runTool(cmd *base.Command, args []string) { +func runTool(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { listTools() return diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index ac2ae50155..056db7bf9e 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -7,6 +7,7 @@ package version import ( "bytes" + "context" "encoding/binary" "fmt" "os" @@ -51,7 +52,7 @@ var ( versionV = CmdVersion.Flag.Bool("v", false, "") ) -func runVersion(cmd *base.Command, args []string) { +func runVersion(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { if *versionM || *versionV { fmt.Fprintf(os.Stderr, "go version: flags can only be used with arguments\n") diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 4ec58de785..717ff2d0aa 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -10,6 +10,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" + "context" "path/filepath" ) @@ -48,7 +49,7 @@ See also: go fmt, go fix. `, } -func runVet(cmd *base.Command, args []string) { +func runVet(ctx context.Context, cmd *base.Command, args []string) { modload.LoadTests = true vetFlags, pkgArgs := vetFlags(args) diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index fbd49b457b..2bbee43ab4 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -5,6 +5,7 @@ package work import ( + "context" "errors" "fmt" "go/build" @@ -344,7 +345,7 @@ var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } var runtimeVersion = runtime.Version() -func runBuild(cmd *base.Command, args []string) { +func runBuild(ctx context.Context, cmd *base.Command, args []string) { BuildInit() var b Builder b.Init() @@ -515,7 +516,7 @@ func libname(args []string, pkgs []*load.Package) (string, error) { return "lib" + libname + ".so", nil } -func runInstall(cmd *base.Command, args []string) { +func runInstall(ctx context.Context, cmd *base.Command, args []string) { BuildInit() InstallPackages(args, load.PackagesForBuild(args)) } diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 3512866e4a..37bb7d6d27 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -191,8 +191,7 @@ BigCmdLoop: } ctx := maybeStartTrace(context.Background()) ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) - _ = ctx - cmd.Run(cmd, args) + cmd.Run(ctx, cmd, args) span.Done() base.Exit() return -- cgit v1.3 From 2bfa45cfa994512c47da2d98f3baca5bb474ec9b Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 15:50:14 -0400 Subject: cmd/go: propagate context into PackagesForBuild and Do for tracing This change propagates context into PackagesForErrors and Do for the purpose of tracing, and calls trace.StartSpan on PackagesForErrors and Do, so that the trace now shows the broad outline of where the "Loading" and "Execution" phases are in the build. Updates #38714 Change-Id: Ib9a7cf7030210f68f76663d1c8a7461e0a226611 Reviewed-on: https://go-review.googlesource.com/c/go/+/238541 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/clean/clean.go | 2 +- src/cmd/go/internal/fix/fix.go | 2 +- src/cmd/go/internal/fmtcmd/fmt.go | 2 +- src/cmd/go/internal/generate/generate.go | 2 +- src/cmd/go/internal/get/get.go | 4 ++-- src/cmd/go/internal/list/list.go | 6 +++--- src/cmd/go/internal/load/pkg.go | 15 ++++++++++----- src/cmd/go/internal/modget/get.go | 4 ++-- src/cmd/go/internal/run/run.go | 4 ++-- src/cmd/go/internal/test/test.go | 8 ++++---- src/cmd/go/internal/vet/vet.go | 4 ++-- src/cmd/go/internal/work/build.go | 20 ++++++++++++-------- src/cmd/go/internal/work/exec.go | 7 ++++++- 13 files changed, 47 insertions(+), 33 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 8af3e3df9c..6bfd7ae21e 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -117,7 +117,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { } if cleanPkg { - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { clean(pkg) } } diff --git a/src/cmd/go/internal/fix/fix.go b/src/cmd/go/internal/fix/fix.go index f16af05fc8..825624fcbb 100644 --- a/src/cmd/go/internal/fix/fix.go +++ b/src/cmd/go/internal/fix/fix.go @@ -34,7 +34,7 @@ See also: go fmt, go vet. func runFix(ctx context.Context, cmd *base.Command, args []string) { printed := false - for _, pkg := range load.Packages(args) { + for _, pkg := range load.Packages(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n") diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index 9868efc7ef..f96cff429c 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -64,7 +64,7 @@ func runFmt(ctx context.Context, cmd *base.Command, args []string) { } }() } - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n") diff --git a/src/cmd/go/internal/generate/generate.go b/src/cmd/go/internal/generate/generate.go index fb26f77f95..98c17bba8c 100644 --- a/src/cmd/go/internal/generate/generate.go +++ b/src/cmd/go/internal/generate/generate.go @@ -176,7 +176,7 @@ func runGenerate(ctx context.Context, cmd *base.Command, args []string) { // Even if the arguments are .go files, this loop suffices. printed := false - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n") diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index f7da5270b0..ef43602aca 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -172,7 +172,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // everything. load.ClearPackageCache() - pkgs := load.PackagesForBuild(args) + pkgs := load.PackagesForBuild(ctx, args) // Phase 3. Install. if *getD { @@ -182,7 +182,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { return } - work.InstallPackages(args, pkgs) + work.InstallPackages(ctx, args, pkgs) } // downloadPaths prepares the list of paths to pass to download. diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index ef0a5a2f2d..3ec243a759 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -449,9 +449,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { load.IgnoreImports = *listFind var pkgs []*load.Package if *listE { - pkgs = load.PackagesAndErrors(args) + pkgs = load.PackagesAndErrors(ctx, args) } else { - pkgs = load.Packages(args) + pkgs = load.Packages(ctx, args) base.ExitIfErrors() } @@ -539,7 +539,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { a.Deps = append(a.Deps, b.AutoAction(work.ModeInstall, work.ModeInstall, p)) } } - b.Do(a) + b.Do(ctx, a) } for _, p := range pkgs { diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 2b5fbb1c5b..32c2ba7912 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -7,6 +7,7 @@ package load import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -30,6 +31,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/trace" ) var ( @@ -2123,9 +2125,9 @@ func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, // to load dependencies of a named package, the named // package is still returned, with p.Incomplete = true // and details in p.DepsErrors. -func Packages(args []string) []*Package { +func Packages(ctx context.Context, args []string) []*Package { var pkgs []*Package - for _, pkg := range PackagesAndErrors(args) { + for _, pkg := range PackagesAndErrors(ctx, args) { if pkg.Error != nil { base.Errorf("%v", pkg.Error) continue @@ -2139,7 +2141,10 @@ func Packages(args []string) []*Package { // *Package for every argument, even the ones that // cannot be loaded at all. // The packages that fail to load will have p.Error != nil. -func PackagesAndErrors(patterns []string) []*Package { +func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { + ctx, span := trace.StartSpan(ctx, "load.PackagesAndErrors") + defer span.Done() + for _, p := range patterns { // Listing is only supported with all patterns referring to either: // - Files that are part of the same directory. @@ -2233,8 +2238,8 @@ func ImportPaths(args []string) []*search.Match { // PackagesForBuild is like Packages but exits // if any of the packages or their dependencies have errors // (cannot be built). -func PackagesForBuild(args []string) []*Package { - pkgs := PackagesAndErrors(args) +func PackagesForBuild(ctx context.Context, args []string) []*Package { + pkgs := PackagesAndErrors(ctx, args) printed := map[*PackageError]bool{} for _, pkg := range pkgs { if pkg.Error != nil { diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 9836a3e2cc..b217196931 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -715,8 +715,8 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { return } work.BuildInit() - pkgs := load.PackagesForBuild(pkgPatterns) - work.InstallPackages(pkgPatterns, pkgs) + pkgs := load.PackagesForBuild(ctx, pkgPatterns) + work.InstallPackages(ctx, pkgPatterns, pkgs) } // runQueries looks up modules at target versions in parallel. Results will be diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index ca2c3db92c..3630f68c54 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -79,7 +79,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { } p = load.GoFilesPackage(files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { - pkgs := load.PackagesAndErrors(args[:1]) + pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { base.Fatalf("go run: no packages loaded from %s", args[0]) } @@ -141,7 +141,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { } a1 := b.LinkAction(work.ModeBuild, work.ModeBuild, p) a := &work.Action{Mode: "go run", Func: buildRunProgram, Args: cmdArgs, Deps: []*work.Action{a1}} - b.Do(a) + b.Do(ctx, a) } // buildRunProgram is the action for running a binary that has already diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 6648d4eab4..d71d339828 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -577,7 +577,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { work.VetFlags = testVet.flags work.VetExplicit = testVet.explicit - pkgs = load.PackagesForBuild(pkgArgs) + pkgs = load.PackagesForBuild(ctx, pkgArgs) if len(pkgs) == 0 { base.Fatalf("no packages to test") } @@ -659,7 +659,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { sort.Strings(all) a := &work.Action{Mode: "go test -i"} - for _, p := range load.PackagesForBuild(all) { + for _, p := range load.PackagesForBuild(ctx, all) { if cfg.BuildToolchainName == "gccgo" && p.Standard { // gccgo's standard library packages // can not be reinstalled. @@ -667,7 +667,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } a.Deps = append(a.Deps, b.CompileAction(work.ModeInstall, work.ModeInstall, p)) } - b.Do(a) + b.Do(ctx, a) if !testC || a.Failed { return } @@ -787,7 +787,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } } - b.Do(root) + b.Do(ctx, root) } // ensures that package p imports the named package diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 717ff2d0aa..58f392eb96 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -67,7 +67,7 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { } } - pkgs := load.PackagesForBuild(pkgArgs) + pkgs := load.PackagesForBuild(ctx, pkgArgs) if len(pkgs) == 0 { base.Fatalf("no packages to vet") } @@ -93,5 +93,5 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { root.Deps = append(root.Deps, b.VetAction(work.ModeBuild, work.ModeBuild, pxtest)) } } - b.Do(root) + b.Do(ctx, root) } diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 2bbee43ab4..d020aa6e9f 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/search" + "cmd/go/internal/trace" ) var CmdBuild = &base.Command{ @@ -350,7 +351,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { var b Builder b.Init() - pkgs := load.PackagesForBuild(args) + pkgs := load.PackagesForBuild(ctx, args) explicitO := len(cfg.BuildO) > 0 @@ -379,7 +380,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { depMode = ModeInstall } - pkgs = omitTestOnly(pkgsFilter(load.Packages(args))) + pkgs = omitTestOnly(pkgsFilter(load.Packages(ctx, args))) // Special case -o /dev/null by not writing at all. if cfg.BuildO == os.DevNull { @@ -409,7 +410,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { if len(a.Deps) == 0 { base.Fatalf("go build: no main packages to build") } - b.Do(a) + b.Do(ctx, a) return } if len(pkgs) > 1 { @@ -422,7 +423,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { p.Stale = true // must build - not up to date p.StaleReason = "build -o flag in use" a := b.AutoAction(ModeInstall, depMode, p) - b.Do(a) + b.Do(ctx, a) return } @@ -433,7 +434,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { if cfg.BuildBuildmode == "shared" { a = b.buildmodeShared(ModeBuild, depMode, args, pkgs, a) } - b.Do(a) + b.Do(ctx, a) } var CmdInstall = &base.Command{ @@ -518,7 +519,7 @@ func libname(args []string, pkgs []*load.Package) (string, error) { func runInstall(ctx context.Context, cmd *base.Command, args []string) { BuildInit() - InstallPackages(args, load.PackagesForBuild(args)) + InstallPackages(ctx, args, load.PackagesForBuild(ctx, args)) } // omitTestOnly returns pkgs with test-only packages removed. @@ -538,7 +539,10 @@ func omitTestOnly(pkgs []*load.Package) []*load.Package { return list } -func InstallPackages(patterns []string, pkgs []*load.Package) { +func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Package) { + ctx, span := trace.StartSpan(ctx, "InstallPackages "+strings.Join(patterns, " ")) + defer span.Done() + if cfg.GOBIN != "" && !filepath.IsAbs(cfg.GOBIN) { base.Fatalf("cannot install, GOBIN must be an absolute path") } @@ -607,7 +611,7 @@ func InstallPackages(patterns []string, pkgs []*load.Package) { a = b.buildmodeShared(ModeInstall, ModeInstall, patterns, pkgs, a) } - b.Do(a) + b.Do(ctx, a) base.ExitIfErrors() // Success. If this command is 'go install' with no arguments diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 071c9d2db9..3ea3293ae1 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -8,6 +8,7 @@ package work import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -31,6 +32,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/str" + "cmd/go/internal/trace" ) // actionList returns the list of actions in the dag rooted at root @@ -54,7 +56,10 @@ func actionList(root *Action) []*Action { } // do runs the action graph rooted at root. -func (b *Builder) Do(root *Action) { +func (b *Builder) Do(ctx context.Context, root *Action) { + ctx, span := trace.StartSpan(ctx, "exec.Builder.Do ("+root.Mode+" "+root.Target+")") + defer span.Done() + if !b.IsCmdList { // If we're doing real work, take time at the end to trim the cache. c := cache.Default() -- cgit v1.3 From c2e73fb446bffd02c651e51c6641cc90fd065b70 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Tue, 23 Jun 2020 08:46:36 -0400 Subject: cmd/compile: remove AttrSeenGlobl (use AttrOnList instead) Minor cleanup: remove the symbol attribute AttrSeenGlobal, since it is redundant with the existing attribute AttrOnList (no need to have what amounts to a separate flag for checking the same property). Change-Id: Ia269b64de37c2bb4a2314bbecf3d2091c6d57424 Reviewed-on: https://go-review.googlesource.com/c/go/+/239477 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/obj.go | 2 +- src/cmd/internal/obj/link.go | 3 --- src/cmd/internal/obj/plist.go | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index 0826b04e33..af5037c5a8 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -352,7 +352,7 @@ func stringsym(pos src.XPos, s string) (data *obj.LSym) { symdata := Ctxt.Lookup(symdataname) - if !symdata.SeenGlobl() { + if !symdata.OnList() { // string data off := dsname(symdata, 0, s, pos, "string") ggloblsym(symdata, int32(off), obj.DUPOK|obj.RODATA|obj.LOCAL) diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index dc47e51be9..311e5ae2e8 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -480,7 +480,6 @@ const ( AttrWrapper AttrNeedCtxt AttrNoFrame - AttrSeenGlobl AttrOnList AttrStatic @@ -537,7 +536,6 @@ func (a Attribute) MakeTypelink() bool { return a&AttrMakeTypelink != 0 } func (a Attribute) CFunc() bool { return a&AttrCFunc != 0 } func (a Attribute) NoSplit() bool { return a&AttrNoSplit != 0 } func (a Attribute) Leaf() bool { return a&AttrLeaf != 0 } -func (a Attribute) SeenGlobl() bool { return a&AttrSeenGlobl != 0 } func (a Attribute) OnList() bool { return a&AttrOnList != 0 } func (a Attribute) ReflectMethod() bool { return a&AttrReflectMethod != 0 } func (a Attribute) Local() bool { return a&AttrLocal != 0 } @@ -574,7 +572,6 @@ var textAttrStrings = [...]struct { {bit: AttrCFunc, s: "CFUNC"}, {bit: AttrNoSplit, s: "NOSPLIT"}, {bit: AttrLeaf, s: "LEAF"}, - {bit: AttrSeenGlobl, s: ""}, {bit: AttrOnList, s: ""}, {bit: AttrReflectMethod, s: "REFLECTMETHOD"}, {bit: AttrLocal, s: "LOCAL"}, diff --git a/src/cmd/internal/obj/plist.go b/src/cmd/internal/obj/plist.go index afe0ee4ee0..6e33f29959 100644 --- a/src/cmd/internal/obj/plist.go +++ b/src/cmd/internal/obj/plist.go @@ -145,10 +145,6 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) { } func (ctxt *Link) Globl(s *LSym, size int64, flag int) { - if s.SeenGlobl() { - fmt.Printf("duplicate %v\n", s) - } - s.Set(AttrSeenGlobl, true) if s.OnList() { ctxt.Diag("symbol %s listed multiple times", s.Name) } -- cgit v1.3 From 7d7bd5abc7f7ac901830b79496f63ce86895e262 Mon Sep 17 00:00:00 2001 From: Lynn Boger Date: Tue, 11 Aug 2020 12:04:25 -0400 Subject: cmd/internal/obj/ppc64: don't remove NOP in assembler Previously, the assembler removed NOPs from the Prog list in obj9.go. NOPs shouldn't be removed if they were added as an inline mark, as described in the issue below. Fixes #40689 Once the NOPs were left in the Prog list, some instructions were flagged as invalid because they had an operand which was not represented in optab. In order to preserve the previous assembler behavior, entries were added to optab for those operand cases. They were not flagged as errors before because the NOP instructions were removed before the code to check the valid opcode/operand combinations. Change-Id: Iae5145f94459027cf458e914d7c5d6089807ccf8 Reviewed-on: https://go-review.googlesource.com/c/go/+/247842 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Paul Murphy Reviewed-by: Michael Munday Reviewed-by: Keith Randall --- src/cmd/internal/obj/ppc64/asm9.go | 3 +++ src/cmd/internal/obj/ppc64/obj9.go | 11 +++-------- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index 0fd0744a42..238ca8f0b7 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -613,6 +613,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_NONE, C_LCON, 0, 0, 0}, {obj.AFUNCDATA, C_SCON, C_NONE, C_NONE, C_ADDR, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // NOP operand variations added for #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // to preserve previous behavior + {obj.ANOP, C_FREG, C_NONE, C_NONE, C_NONE, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_LBRA, 11, 4, 0}, // same as ABR/ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_LBRA, 11, 4, 0}, // same as ABR/ABL {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // align code diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go index 16881c634b..749f7066de 100644 --- a/src/cmd/internal/obj/ppc64/obj9.go +++ b/src/cmd/internal/obj/ppc64/obj9.go @@ -429,7 +429,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs * expand RET * expand BECOME pseudo */ @@ -559,10 +558,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { q = p q1 = p.Pcond if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } + // NOPs are not removed due to #40689. if q1.Mark&LEAF == 0 { q1.Mark |= LABEL @@ -589,9 +585,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { continue case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark + // NOPs are not removed due to + // #40689 continue default: -- cgit v1.3 From cde5fd1c0f8c40804bfd942eec1e2d69bccf4e13 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Thu, 13 Aug 2020 12:39:04 -0700 Subject: cmd/compile: correct type of CvtBoolToUint8 values Fixes #40746 Change-Id: I539f07d1f958dacee87d846171a8889d03182d25 Reviewed-on: https://go-review.googlesource.com/c/go/+/248397 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/phiopt.go | 2 +- src/cmd/compile/internal/ssa/prove.go | 2 +- test/fixedbugs/issue40746.go | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 test/fixedbugs/issue40746.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/phiopt.go b/src/cmd/compile/internal/ssa/phiopt.go index 8643fa584c..db7b02275c 100644 --- a/src/cmd/compile/internal/ssa/phiopt.go +++ b/src/cmd/compile/internal/ssa/phiopt.go @@ -154,7 +154,7 @@ func phioptint(v *Value, b0 *Block, reverse int) { } v.AddArg(a) - cvt := v.Block.NewValue1(v.Pos, OpCvtBoolToUint8, a.Type, a) + cvt := v.Block.NewValue1(v.Pos, OpCvtBoolToUint8, v.Block.Func.Config.Types.UInt8, a) switch v.Type.Size() { case 1: v.reset(OpCopy) diff --git a/src/cmd/compile/internal/ssa/prove.go b/src/cmd/compile/internal/ssa/prove.go index 6c6be39d34..ce7d689f93 100644 --- a/src/cmd/compile/internal/ssa/prove.go +++ b/src/cmd/compile/internal/ssa/prove.go @@ -1334,7 +1334,7 @@ func removeBranch(b *Block, branch branch) { // isNonNegative reports whether v is known to be greater or equal to zero. func isNonNegative(v *Value) bool { if !v.Type.IsInteger() { - panic("isNonNegative bad type") + v.Fatalf("isNonNegative bad type: %v", v.Type) } // TODO: return true if !v.Type.IsSigned() // SSA isn't type-safe enough to do that now (issue 37753). diff --git a/test/fixedbugs/issue40746.go b/test/fixedbugs/issue40746.go new file mode 100644 index 0000000000..235282fd90 --- /dev/null +++ b/test/fixedbugs/issue40746.go @@ -0,0 +1,19 @@ +// compile + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f(x byte, b bool) byte { + var c byte + if b { + c = 1 + } + + if int8(c) < 0 { + x++ + } + return x +} -- cgit v1.3 From d0d6593d1d4e81acd073244f42b6893fa65c99d8 Mon Sep 17 00:00:00 2001 From: Michael Munday Date: Mon, 10 Aug 2020 08:01:21 -0700 Subject: cmd/internal/obj: fix inline marker issue on s390x The optimization that replaces inline markers with pre-existing instructions assumes that 'Prog' values produced by the compiler are still reachable after the assembler has run. This was not true on s390x where the assembler was removing NOP instructions from the linked list of 'Prog' values. This led to broken inlining data which in turn caused an infinite loop in the runtime traceback code. Fix this by stopping the s390x assembler backend removing NOP values. It does not make any difference to the output of the assembler because NOP instructions are 0 bytes long anyway. Fixes #40473. Change-Id: Ib4fabadd1de8adb80421f75950ee9aad2111147a Reviewed-on: https://go-review.googlesource.com/c/go/+/247697 Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/internal/obj/pcln.go | 15 +++++++++++++++ src/cmd/internal/obj/s390x/objz.go | 11 ----------- 2 files changed, 15 insertions(+), 11 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index 1f7ccf47ef..bffeda041d 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -278,6 +278,21 @@ func linkpcln(ctxt *Link, cursym *LSym) { funcpctab(ctxt, &pcln.Pcfile, cursym, "pctofile", pctofileline, pcln) funcpctab(ctxt, &pcln.Pcline, cursym, "pctoline", pctofileline, nil) + // Check that all the Progs used as inline markers are still reachable. + // See issue #40473. + inlMarkProgs := make(map[*Prog]struct{}, len(cursym.Func.InlMarks)) + for _, inlMark := range cursym.Func.InlMarks { + inlMarkProgs[inlMark.p] = struct{}{} + } + for p := cursym.Func.Text; p != nil; p = p.Link { + if _, ok := inlMarkProgs[p]; ok { + delete(inlMarkProgs, p) + } + } + if len(inlMarkProgs) > 0 { + ctxt.Diag("one or more instructions used as inline markers are no longer reachable") + } + pcinlineState := new(pcinlineState) funcpctab(ctxt, &pcln.Pcinline, cursym, "pctoinline", pcinlineState.pctoinline, nil) for _, inlMark := range cursym.Func.InlMarks { diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index b14dc810fa..ef6335d849 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -283,17 +283,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ACMPUBNE: q = p p.Mark |= BRANCH - if p.Pcond != nil { - q := p.Pcond - for q.As == obj.ANOP { - q = q.Link - p.Pcond = q - } - } - - case obj.ANOP: - q.Link = p.Link /* q is non-nop */ - p.Link.Mark |= p.Mark default: q = p -- cgit v1.3 From 5ae198087bd07e88009885ac96c864381f8d8272 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 4 Jun 2020 11:41:43 -0400 Subject: cmd/go: don't initialize Builder in envcmd.MkEnv The Builder isn't needed by MkEnv, and Builder.Init doesn't have side effects that change the environment. Builder.Init does currently call CheckGOOSARCHPair, but that's being moved out in CL 234658. Builder.Init creates the temporary work directory used by the builder. For the builder created in MkEnv, this directory is never used. Creating this directory can cause unnecessary errors for commands that don't use a builder like 'go clean' and 'go list'. Fixes #38395 Updates #24398 Change-Id: Ib93ae55afdf958000470657f4c4ff5bd92700e46 Reviewed-on: https://go-review.googlesource.com/c/go/+/236563 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/envcmd/env.go | 3 -- src/cmd/go/testdata/script/build_GOTMPDIR.txt | 49 +++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 10 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 403e0f4a7b..7bd75f7305 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -63,9 +63,6 @@ var ( ) func MkEnv() []cfg.EnvVar { - var b work.Builder - b.Init() - envFile, _ := cfg.EnvFile() env := []cfg.EnvVar{ {Name: "GO111MODULE", Value: cfg.Getenv("GO111MODULE")}, diff --git a/src/cmd/go/testdata/script/build_GOTMPDIR.txt b/src/cmd/go/testdata/script/build_GOTMPDIR.txt index c93ca932ca..1073517c29 100644 --- a/src/cmd/go/testdata/script/build_GOTMPDIR.txt +++ b/src/cmd/go/testdata/script/build_GOTMPDIR.txt @@ -1,15 +1,50 @@ -env GO111MODULE=off -[short] skip - # Set GOCACHE to a clean directory to ensure that 'go build' has work to report. -env GOCACHE=$WORK/gocache +[!windows] env GOCACHE=$WORK/gocache +[windows] env GOCACHE=$WORK\gocache -# Build should use GOTMPDIR if set. -env GOTMPDIR=$WORK/my-favorite-tmpdir +# 'go build' should use GOTMPDIR if set. +[!windows] env GOTMPDIR=$WORK/my-favorite-tmpdir +[windows] env GOTMPDIR=$WORK\my-favorite-tmpdir mkdir $GOTMPDIR -go build -work hello.go +go build -x hello.go stderr ^WORK=.*my-favorite-tmpdir +# Make GOTMPDIR a regular file. This prevents the creation of work directories, +# so we can check that certain commands don't create them. +# This simulates running on a full disk or a read-only volume. +rm $GOTMPDIR +cp hello.go $GOTMPDIR # any file will do + +# 'go build' should fail if GOTMPDIR is read-only. +! go build -x . +stderr '^go: creating work dir: \w+ '$GOTMPDIR + +# 'go list' should only fail if it needs to build something. +go list -x . +! stderr 'creating work dir' +stdout m +go list -m all +stdout m +! go list -x -export . +stderr '^go: creating work dir: \w+ '$GOTMPDIR + +# 'go clean -cache' and 'go clean -modcache' should not fail. +go clean -x -cache +! stderr 'creating work dir' +go clean -x -modcache +! stderr 'creating work dir' + +# 'go env' should not fail for specific variables. +# Without arguments, it needs to initialize a builder to load cgo flags, and +# that uses a temporary directory. +! go env +stderr '^go: creating work dir: \w+ '$GOTMPDIR +go env GOROOT + +-- go.mod -- +module m + +go 1.15 -- hello.go -- package main func main() { println("hello") } -- cgit v1.3 From 016e13df7475329c65524b2eabbc5207ceb4ee74 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 29 Jun 2020 16:28:18 -0400 Subject: cmd/go/internal/modfetch: stop migrating go.modverify to go.sum go.modverify was renamed to go.sum before vgo was merged into cmd/go. It's been long enough that we can safely drop support for it. For #25525 Change-Id: If8da66280a0fb6a4d4db0b170700775523c18571 Reviewed-on: https://go-review.googlesource.com/c/go/+/240458 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modfetch/fetch.go | 31 ++++--------------------------- src/cmd/go/testdata/script/mod_verify.txt | 8 +++----- 2 files changed, 7 insertions(+), 32 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index fd7a5cef83..8df2289097 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -374,12 +374,11 @@ type modSum struct { var goSum struct { mu sync.Mutex - m map[module.Version][]string // content of go.sum file (+ go.modverify if present) + m map[module.Version][]string // content of go.sum file checked map[modSum]bool // sums actually checked during execution dirty bool // whether we added any new sums to m overwrite bool // if true, overwrite go.sum without incorporating its contents enabled bool // whether to use go.sum at all - modverify string // path to go.modverify, to be deleted } // initGoSum initializes the go.sum data. @@ -403,19 +402,6 @@ func initGoSum() (bool, error) { goSum.enabled = true readGoSum(goSum.m, GoSumFile, data) - // Add old go.modverify file. - // We'll delete go.modverify in WriteGoSum. - alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify" - if data, err := renameio.ReadFile(alt); err == nil { - migrate := make(map[module.Version][]string) - readGoSum(migrate, alt, data) - for mod, sums := range migrate { - for _, sum := range sums { - addModSumLocked(mod, sum) - } - } - goSum.modverify = alt - } return true, nil } @@ -616,14 +602,9 @@ func WriteGoSum() { goSum.mu.Lock() defer goSum.mu.Unlock() - if !goSum.enabled { - // If we haven't read the go.sum file yet, don't bother writing it: at best, - // we could rename the go.modverify file if it isn't empty, but we haven't - // needed to touch it so far — how important could it be? - return - } - if !goSum.dirty { - // Don't bother opening the go.sum file if we don't have anything to add. + if !goSum.enabled || !goSum.dirty { + // If we haven't read go.sum yet or if we don't have anything to add, + // don't bother opening it. return } if cfg.BuildMod == "readonly" { @@ -674,10 +655,6 @@ func WriteGoSum() { goSum.checked = make(map[modSum]bool) goSum.dirty = false goSum.overwrite = false - - if goSum.modverify != "" { - os.Remove(goSum.modverify) // best effort - } } // TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. diff --git a/src/cmd/go/testdata/script/mod_verify.txt b/src/cmd/go/testdata/script/mod_verify.txt index 646bc62bb7..3918400435 100644 --- a/src/cmd/go/testdata/script/mod_verify.txt +++ b/src/cmd/go/testdata/script/mod_verify.txt @@ -12,20 +12,18 @@ go mod verify ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip # With bad go.sum, sync (which must download) fails. -# Even if the bad sum is in the old legacy go.modverify file. rm go.sum -cp go.sum.bad go.modverify +cp go.sum.bad go.sum ! go mod tidy stderr 'checksum mismatch' ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip -# With good go.sum, sync works (and moves go.modverify to go.sum). +# With good go.sum, sync works. rm go.sum -cp go.sum.good go.modverify +cp go.sum.good go.sum go mod tidy exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip exists $GOPATH/pkg/mod/rsc.io/quote@v1.1.0/quote.go -! exists go.modverify # go.sum should have the new checksum for go.mod grep '^rsc.io/quote v1.1.0/go.mod ' go.sum -- cgit v1.3 From 9a759593d7a71b4c061fd9bd053bd79584c632dc Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 8 Jun 2020 18:06:11 -0400 Subject: cmd/go: don't save sums for modules loaded for import resolution modfetch.WriteGoSum now accepts a map[module.Version]bool parameter. This is used to prevent some new sums from being saved to go.sum when they would be removed by the next 'go mod tidy'. Previusly, sums were saved for modules looked up during import resolution. A new function, modload.TrimGoSum, is also introduced, which marks sums for deletion. 'go mod tidy' now uses this. The new logic distinguishes between go.mod sums and content sums, which lets 'go mod tidy' delete sums for modules in the build graph but not the build list. Fixes #31580 Fixes #36260 Fixes #33008 Change-Id: I06c4125704a8bbc9969de05265967ec1d2e6d3e8 Reviewed-on: https://go-review.googlesource.com/c/go/+/237017 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modcmd/init.go | 1 + src/cmd/go/internal/modcmd/tidy.go | 38 +----------- src/cmd/go/internal/modfetch/fetch.go | 83 +++++++++++++++++++-------- src/cmd/go/internal/modload/init.go | 69 ++++++++++++++++++++-- src/cmd/go/testdata/script/mod_sum_lookup.txt | 33 +++++++++++ src/cmd/go/testdata/script/mod_tidy_old.txt | 46 +++++++++++++++ 6 files changed, 204 insertions(+), 66 deletions(-) create mode 100644 src/cmd/go/testdata/script/mod_sum_lookup.txt create mode 100644 src/cmd/go/testdata/script/mod_tidy_old.txt (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index ddb9aeebe9..95063e62f4 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -52,4 +52,5 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod init: module path must not contain '@'") } modload.InitMod() // does all the hard work + modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index feb41a83b0..769cd11fe8 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -9,12 +9,9 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" "cmd/go/internal/modload" "cmd/go/internal/work" "context" - - "golang.org/x/mod/module" ) var cmdTidy = &base.Command{ @@ -45,39 +42,6 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { modload.LoadALL() modload.TidyBuildList() - modTidyGoSum() // updates memory copy; WriteGoMod on next line flushes it out + modload.TrimGoSum() modload.WriteGoMod() } - -// modTidyGoSum resets the go.sum file content -// to be exactly what's needed for the current go.mod. -func modTidyGoSum() { - // Assuming go.sum already has at least enough from the successful load, - // we only have to tell modfetch what needs keeping. - reqs := modload.Reqs() - keep := make(map[module.Version]bool) - replaced := make(map[module.Version]bool) - var walk func(module.Version) - walk = func(m module.Version) { - // If we build using a replacement module, keep the sum for the replacement, - // since that's the code we'll actually use during a build. - // - // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the - // sums for both sets of transitive requirements. - r := modload.Replacement(m) - if r.Path == "" { - keep[m] = true - } else { - keep[r] = true - replaced[m] = true - } - list, _ := reqs.Required(m) - for _, r := range list { - if !keep[r] && !replaced[r] { - walk(r) - } - } - } - walk(modload.Target) - modfetch.TrimGoSum(keep) -} diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 8df2289097..e40158b535 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -375,12 +375,15 @@ type modSum struct { var goSum struct { mu sync.Mutex m map[module.Version][]string // content of go.sum file - checked map[modSum]bool // sums actually checked during execution - dirty bool // whether we added any new sums to m + status map[modSum]modSumStatus // state of sums in m overwrite bool // if true, overwrite go.sum without incorporating its contents enabled bool // whether to use go.sum at all } +type modSumStatus struct { + used, dirty bool +} + // initGoSum initializes the go.sum data. // The boolean it returns reports whether the // use of go.sum is now enabled. @@ -394,7 +397,7 @@ func initGoSum() (bool, error) { } goSum.m = make(map[module.Version][]string) - goSum.checked = make(map[modSum]bool) + goSum.status = make(map[modSum]modSumStatus) data, err := lockedfile.Read(GoSumFile) if err != nil && !os.IsNotExist(err) { return false, err @@ -504,6 +507,11 @@ func checkModSum(mod module.Version, h string) error { return err } done := inited && haveModSumLocked(mod, h) + if inited { + st := goSum.status[modSum{mod, h}] + st.used = true + goSum.status[modSum{mod, h}] = st + } goSum.mu.Unlock() if done { @@ -523,6 +531,9 @@ func checkModSum(mod module.Version, h string) error { if inited { goSum.mu.Lock() addModSumLocked(mod, h) + st := goSum.status[modSum{mod, h}] + st.dirty = true + goSum.status[modSum{mod, h}] = st goSum.mu.Unlock() } return nil @@ -532,7 +543,6 @@ func checkModSum(mod module.Version, h string) error { // If it finds a conflicting pair instead, it calls base.Fatalf. // goSum.mu must be locked. func haveModSumLocked(mod module.Version, h string) bool { - goSum.checked[modSum{mod, h}] = true for _, vh := range goSum.m[mod] { if h == vh { return true @@ -554,7 +564,6 @@ func addModSumLocked(mod module.Version, h string) { fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) } goSum.m[mod] = append(goSum.m[mod], h) - goSum.dirty = true } // checkSumDB checks the mod, h pair against the Go checksum database. @@ -598,13 +607,35 @@ func Sum(mod module.Version) string { } // WriteGoSum writes the go.sum file if it needs to be updated. -func WriteGoSum() { +// +// keep is used to check whether a newly added sum should be saved in go.sum. +// It should have entries for both module content sums and go.mod sums +// (version ends with "/go.mod"). Existing sums will be preserved unless they +// have been marked for deletion with TrimGoSum. +func WriteGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() - if !goSum.enabled || !goSum.dirty { - // If we haven't read go.sum yet or if we don't have anything to add, - // don't bother opening it. + // If we haven't read the go.sum file yet, don't bother writing it. + if !goSum.enabled { + return + } + + // Check whether we need to add sums for which keep[m] is true or remove + // unused sums marked with TrimGoSum. If there are no changes to make, + // just return without opening go.sum. + dirty := false +Outer: + for m, hs := range goSum.m { + for _, h := range hs { + st := goSum.status[modSum{m, h}] + if st.dirty && (!st.used || keep[m]) { + dirty = true + break Outer + } + } + } + if !dirty { return } if cfg.BuildMod == "readonly" { @@ -625,9 +656,10 @@ func WriteGoSum() { // them without good reason. goSum.m = make(map[module.Version][]string, len(goSum.m)) readGoSum(goSum.m, GoSumFile, data) - for ms := range goSum.checked { - addModSumLocked(ms.mod, ms.sum) - goSum.dirty = true + for ms, st := range goSum.status { + if st.used { + addModSumLocked(ms.mod, ms.sum) + } } } @@ -642,7 +674,10 @@ func WriteGoSum() { list := goSum.m[m] sort.Strings(list) for _, h := range list { - fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + st := goSum.status[modSum{m, h}] + if !st.dirty || (st.used && keep[m]) { + fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + } } } return buf.Bytes(), nil @@ -652,12 +687,16 @@ func WriteGoSum() { base.Fatalf("go: updating go.sum: %v", err) } - goSum.checked = make(map[modSum]bool) - goSum.dirty = false + goSum.status = make(map[modSum]modSumStatus) goSum.overwrite = false } -// TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. +// TrimGoSum trims go.sum to contain only the modules needed for reproducible +// builds. +// +// keep is used to check whether a sum should be retained in go.mod. It should +// have entries for both module content sums and go.mod sums (version ends +// with "/go.mod"). func TrimGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() @@ -669,13 +708,11 @@ func TrimGoSum(keep map[module.Version]bool) { return } - for m := range goSum.m { - // If we're keeping x@v we also keep x@v/go.mod. - // Map x@v/go.mod back to x@v for the keep lookup. - noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")} - if !keep[m] && !keep[noGoMod] { - delete(goSum.m, m) - goSum.dirty = true + for m, hs := range goSum.m { + if !keep[m] { + for _, h := range hs { + goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true} + } goSum.overwrite = true } } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 664a2a1594..95334211ef 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -331,7 +331,11 @@ func die() { } // InitMod sets Target and, if there is a main module, parses the initial build -// list from its go.mod file, creating and populating that file if needed. +// list from its go.mod file. If InitMod is called by 'go mod init', InitMod +// will populate go.mod in memory, possibly importing dependencies from a +// legacy configuration file. For other commands, InitMod may make other +// adjustments in memory, like adding a go directive. WriteGoMod should be +// called later to write changes out to disk. // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. @@ -352,7 +356,6 @@ func InitMod() { // Running go mod init: do legacy module conversion legacyModInit() modFileToBuildList() - WriteGoMod() return } @@ -391,9 +394,6 @@ func InitMod() { if cfg.BuildMod == "vendor" { readVendorList() checkVendorConsistency() - } else { - // TODO(golang.org/issue/33326): if cfg.BuildMod != "readonly"? - WriteGoMod() } } @@ -797,9 +797,10 @@ func WriteGoMod() { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") } } + // Always update go.sum, even if we didn't change go.mod: we may have // downloaded modules that we didn't have before. - modfetch.WriteGoSum() + modfetch.WriteGoSum(keepSums()) if !dirty && cfg.CmdName != "mod tidy" { // The go.mod file has the same semantic content that it had before @@ -849,3 +850,59 @@ func WriteGoMod() { base.Fatalf("go: updating go.mod: %v", err) } } + +// keepSums returns a set of module sums to preserve in go.sum. The set +// includes entries for all modules used to load packages (according to +// the last load function like ImportPaths, LoadALL, etc.). It also contains +// entries for go.mod files needed for MVS (the version of these entries +// ends with "/go.mod"). +func keepSums() map[module.Version]bool { + // Walk the module graph and keep sums needed by MVS. + modkey := func(m module.Version) module.Version { + return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} + } + keep := make(map[module.Version]bool) + replaced := make(map[module.Version]bool) + reqs := Reqs() + var walk func(module.Version) + walk = func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + // + // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the + // sums for both sets of transitive requirements. + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + replaced[m] = true + keep[modkey(r)] = true + } + list, _ := reqs.Required(m) + for _, r := range list { + if !keep[modkey(r)] && !replaced[r] { + walk(r) + } + } + } + walk(Target) + + // Add entries for modules that provided packages loaded with ImportPaths, + // LoadALL, or similar functions. + if loaded != nil { + for _, pkg := range loaded.pkgs { + m := pkg.mod + if r := Replacement(m); r.Path != "" { + keep[r] = true + } else { + keep[m] = true + } + } + } + + return keep +} + +func TrimGoSum() { + modfetch.TrimGoSum(keepSums()) +} diff --git a/src/cmd/go/testdata/script/mod_sum_lookup.txt b/src/cmd/go/testdata/script/mod_sum_lookup.txt new file mode 100644 index 0000000000..ed80a44984 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_sum_lookup.txt @@ -0,0 +1,33 @@ +# When we attempt to resolve an import that doesn't exist, we should not save +# hashes for downloaded modules. +# Verifies golang.org/issue/36260. +go list -e -tags=ignore ./noexist +! exists go.sum + +# When an import is resolved successfully, we should only save hashes for +# the module that provides the package, not for other modules looked up. +# Verifies golang.org/issue/31580. +go list ./exist +grep '^example.com/join v1.1.0 h1:' go.sum +! grep '^example.com/join/subpkg' go.sum +cp go.sum go.list.sum +go mod tidy +cmp go.sum go.list.sum + +-- go.mod -- +module m + +go 1.15 + +-- noexist/use.go -- +// ignore tags prevents errors in 'go mod tidy' +// +build ignore + +package use + +import _ "example.com/join/subpkg/noexist" + +-- exist/use.go -- +package use + +import _ "example.com/join/subpkg" diff --git a/src/cmd/go/testdata/script/mod_tidy_old.txt b/src/cmd/go/testdata/script/mod_tidy_old.txt new file mode 100644 index 0000000000..7428f0ce8a --- /dev/null +++ b/src/cmd/go/testdata/script/mod_tidy_old.txt @@ -0,0 +1,46 @@ +# 'go mod tidy' should remove content sums for module versions that aren't +# in the build list. It should preserve go.mod sums for module versions that +# are in the module graph though. +# Verifies golang.org/issue/33008. +go mod tidy +! grep '^rsc.io/quote v1.5.0 h1:' go.sum +grep '^rsc.io/quote v1.5.0/go.mod h1:' go.sum + +-- go.mod -- +module m + +go 1.15 + +require ( + rsc.io/quote v1.5.2 + example.com/r v0.0.0 +) + +replace example.com/r => ./r + +-- go.sum -- +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:pvCbr/wm8HzDD3fVywevekufpn6tCGPY3spdHeZJEsw= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +rsc.io/quote v1.5.0 h1:6fJa6E+wGadANKkUMlZ0DhXFpoKlslOQDCo259XtdIE= +rsc.io/quote v1.5.0/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= +rsc.io/quote v1.5.2 h1:3fEykkD9k7lYzXqCYrwGAf7iNhbk4yCjHmKBN9td4L0= +rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= +rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/testonly v1.0.0 h1:K/VWHdO+Jv7woUXG0GzVNx1czBXUt3Ib1deaMn+xk64= +rsc.io/testonly v1.0.0/go.mod h1:OqmGbIFOcF+XrFReLOGZ6BhMM7uMBiQwZsyNmh74SzY= + +-- r/go.mod -- +module example.com/r + +require rsc.io/quote v1.5.0 + +-- use.go -- +package use + +import _ "example.com/r" + +-- r/use.go -- +package use + +import _ "rsc.io/quote" -- cgit v1.3 From 8766f96dd72b5d124bf76bf5f88e260a88072683 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 25 Jun 2020 17:50:38 -0400 Subject: cmd/go: migrate to module.MatchPrefixPatterns In CL 239797, str.GlobsMatchPath was copied to golang.org/x/mod/module as MatchPrefixPatterns. This CL updates x/mod, switches calls to use the new function, and deletes the old function. For #38725 Change-Id: I7241032228b574aa539426a92d2f5aad9ee001e2 Reviewed-on: https://go-review.googlesource.com/c/go/+/240061 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Michael Matloob --- src/cmd/go.mod | 2 +- src/cmd/go.sum | 4 +- src/cmd/go/internal/modfetch/insecure.go | 5 ++- src/cmd/go/internal/modfetch/repo.go | 4 +- src/cmd/go/internal/modfetch/sumdb.go | 3 +- src/cmd/go/internal/str/path.go | 45 ----------------------- src/cmd/vendor/golang.org/x/mod/module/module.go | 47 ++++++++++++++++++++++++ src/cmd/vendor/modules.txt | 2 +- 8 files changed, 57 insertions(+), 55 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 6d57ceee79..21670b9996 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -7,7 +7,7 @@ require ( github.com/ianlancetaylor/demangle v0.0.0-20200414190113-039b1ae3a340 // indirect golang.org/x/arch v0.0.0-20200511175325-f7c78586839d golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/mod v0.3.0 + golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 golang.org/x/xerrors v0.0.0-20200806184451-1a77d5e9f316 // indirect diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 3fc693e3bf..1b5ef515c2 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -14,8 +14,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 h1:R11LxkoUvECaAHdM5/ZOevSR7n+016EgTw8nbE1l+XM= +golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/src/cmd/go/internal/modfetch/insecure.go b/src/cmd/go/internal/modfetch/insecure.go index 8420432d6c..b692669cba 100644 --- a/src/cmd/go/internal/modfetch/insecure.go +++ b/src/cmd/go/internal/modfetch/insecure.go @@ -7,10 +7,11 @@ package modfetch import ( "cmd/go/internal/cfg" "cmd/go/internal/get" - "cmd/go/internal/str" + + "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || str.GlobsMatchPath(cfg.GOINSECURE, path) + return get.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index f03bdd8d03..34f805d58a 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -16,9 +16,9 @@ import ( "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" - "cmd/go/internal/str" web "cmd/go/internal/web" + "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -217,7 +217,7 @@ func lookup(proxy, path string) (r Repo, err error) { return nil, errLookupDisabled } - if str.GlobsMatchPath(cfg.GONOPROXY, path) { + if module.MatchPrefixPatterns(cfg.GONOPROXY, path) { switch proxy { case "noproxy", "direct": return lookupDirect(path) diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 7973f47426..783c4a433b 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -24,7 +24,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/get" "cmd/go/internal/lockedfile" - "cmd/go/internal/str" "cmd/go/internal/web" "golang.org/x/mod/module" @@ -34,7 +33,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !get.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, diff --git a/src/cmd/go/internal/str/path.go b/src/cmd/go/internal/str/path.go index 95d91a3332..51ab2af82b 100644 --- a/src/cmd/go/internal/str/path.go +++ b/src/cmd/go/internal/str/path.go @@ -5,7 +5,6 @@ package str import ( - "path" "path/filepath" "strings" ) @@ -50,47 +49,3 @@ func HasFilePathPrefix(s, prefix string) bool { return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix } } - -// GlobsMatchPath reports whether any path prefix of target -// matches one of the glob patterns (as defined by path.Match) -// in the comma-separated globs list. -// It ignores any empty or malformed patterns in the list. -func GlobsMatchPath(globs, target string) bool { - for globs != "" { - // Extract next non-empty glob in comma-separated list. - var glob string - if i := strings.Index(globs, ","); i >= 0 { - glob, globs = globs[:i], globs[i+1:] - } else { - glob, globs = globs, "" - } - if glob == "" { - continue - } - - // A glob with N+1 path elements (N slashes) needs to be matched - // against the first N+1 path elements of target, - // which end just before the N+1'th slash. - n := strings.Count(glob, "/") - prefix := target - // Walk target, counting slashes, truncating at the N+1'th slash. - for i := 0; i < len(target); i++ { - if target[i] == '/' { - if n == 0 { - prefix = target[:i] - break - } - n-- - } - } - if n > 0 { - // Not enough prefix elements. - continue - } - matched, _ := path.Match(glob, prefix) - if matched { - return true - } - } - return false -} diff --git a/src/cmd/vendor/golang.org/x/mod/module/module.go b/src/cmd/vendor/golang.org/x/mod/module/module.go index 6cd37280a8..3a8b080c7b 100644 --- a/src/cmd/vendor/golang.org/x/mod/module/module.go +++ b/src/cmd/vendor/golang.org/x/mod/module/module.go @@ -97,6 +97,7 @@ package module import ( "fmt" + "path" "sort" "strings" "unicode" @@ -716,3 +717,49 @@ func unescapeString(escaped string) (string, bool) { } return string(buf), true } + +// MatchPrefixPatterns reports whether any path prefix of target matches one of +// the glob patterns (as defined by path.Match) in the comma-separated globs +// list. This implements the algorithm used when matching a module path to the +// GOPRIVATE environment variable, as described by 'go help module-private'. +// +// It ignores any empty or malformed patterns in the list. +func MatchPrefixPatterns(globs, target string) bool { + for globs != "" { + // Extract next non-empty glob in comma-separated list. + var glob string + if i := strings.Index(globs, ","); i >= 0 { + glob, globs = globs[:i], globs[i+1:] + } else { + glob, globs = globs, "" + } + if glob == "" { + continue + } + + // A glob with N+1 path elements (N slashes) needs to be matched + // against the first N+1 path elements of target, + // which end just before the N+1'th slash. + n := strings.Count(glob, "/") + prefix := target + // Walk target, counting slashes, truncating at the N+1'th slash. + for i := 0; i < len(target); i++ { + if target[i] == '/' { + if n == 0 { + prefix = target[:i] + break + } + n-- + } + } + if n > 0 { + // Not enough prefix elements. + continue + } + matched, _ := path.Match(glob, prefix) + if matched { + return true + } + } + return false +} diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index 21fc78c237..7272f04ff3 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -29,7 +29,7 @@ golang.org/x/arch/x86/x86asm golang.org/x/crypto/ed25519 golang.org/x/crypto/ed25519/internal/edwards25519 golang.org/x/crypto/ssh/terminal -# golang.org/x/mod v0.3.0 +# golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 ## explicit golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile -- cgit v1.3 From 02a7b4b4a70d0574f82776309feaf28f109f5399 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Tue, 30 Jun 2020 13:48:15 -0400 Subject: cmd/go/internal/modload: don't initialize build cache modload.Init initialized the build cache with the intent of providing a better error message in Go 1.12, when the build cache became mandatory (in module mode, packages aren't installed outside the build cache). Unfortunately, this didn't provide a more descriptive error (the cache calls base.Fatalf with its own message), and it caused errors for commands that don't use the cache (like 'go mod edit'). This CL removes the cache initialization from modload.Init. The builder will initialize it when it's needed. For #39882 Change-Id: Ibc01ae4e59358dcd08a07ffc97bf556514d0366f Reviewed-on: https://go-review.googlesource.com/c/go/+/240548 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Michael Matloob --- src/cmd/go/internal/modload/init.go | 7 ---- .../go/testdata/script/build_cache_disabled.txt | 46 ++++++++++++++++++++++ 2 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 src/cmd/go/testdata/script/build_cache_disabled.txt (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 95334211ef..fff060e665 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -20,7 +20,6 @@ import ( "strings" "cmd/go/internal/base" - "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/lockedfile" @@ -162,12 +161,6 @@ func Init() { // We're in module mode. Install the hooks to make it work. - if c := cache.Default(); c == nil { - // With modules, there are no install locations for packages - // other than the build cache. - base.Fatalf("go: cannot use modules with build cache disabled") - } - list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") diff --git a/src/cmd/go/testdata/script/build_cache_disabled.txt b/src/cmd/go/testdata/script/build_cache_disabled.txt new file mode 100644 index 0000000000..2e1327880b --- /dev/null +++ b/src/cmd/go/testdata/script/build_cache_disabled.txt @@ -0,0 +1,46 @@ +# The build cache is required to build anything. It also may be needed to +# initialize the build system, which is needed for commands like 'go env'. +# However, there are lots of commands the cache is not needed for, and we +# shouldn't require it when it won't be used. +# +# TODO(golang.org/issue/39882): commands below should work, too. +# * go clean -modcache +# * go env +# * go fix +# * go fmt +# * go generate +# * go get -d +# * go list (without -export or -compiled) + +env GOCACHE=off + +# Commands that don't completely load packages should work. +go doc fmt +stdout Printf + +go fmt . + +! go tool compile -h +stderr usage: + +go version +stdout '^go version' + + +# Module commands that don't load packages should work. +go mod init m +exists go.mod + +go mod edit -require rsc.io/quote@v1.5.2 + +go mod download rsc.io/quote + +go mod graph +stdout rsc.io/quote + +go mod verify + +-- main.go -- +package main + +func main() {} -- cgit v1.3 From 32a84c99e136ed5af0686dbedd31fd7dff40fb38 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 8 Aug 2020 07:58:04 -0700 Subject: cmd/compile: fix live variable computation for deferreturn Taking the live variable set from the last return point is problematic. See #40629 for details, but there may not be a return point, or it may be before the final defer. Additionally, keeping track of the last call as a *Value doesn't quite work. If it is dead-code eliminated, the storage for the Value is reused for some other random instruction. Its live variable information, if it is available at all, is wrong. Instead, just mark all the open-defer argument slots as live throughout the function. (They are already zero-initialized.) Fixes #40629 Change-Id: Ie456c7db3082d0de57eaa5234a0f32525a1cce13 Reviewed-on: https://go-review.googlesource.com/c/go/+/247522 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Dan Scales --- src/cmd/compile/internal/gc/plive.go | 118 ++++++++--------------------------- src/cmd/compile/internal/gc/ssa.go | 19 +----- src/cmd/compile/internal/ssa/func.go | 11 +--- test/fixedbugs/issue40629.go | 69 ++++++++++++++++++++ 4 files changed, 99 insertions(+), 118 deletions(-) create mode 100644 test/fixedbugs/issue40629.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index b366c8a4a0..0cb2661997 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -140,24 +140,14 @@ type Liveness struct { regMaps []liveRegMask cache progeffectscache - - // These are only populated if open-coded defers are being used. - // List of vars/stack slots storing defer args - openDeferVars []openDeferVarInfo - // Map from defer arg OpVarDef to the block where the OpVarDef occurs. - openDeferVardefToBlockMap map[*Node]*ssa.Block - // Map of blocks that cannot reach a return or exit (panic) - nonReturnBlocks map[*ssa.Block]bool -} - -type openDeferVarInfo struct { - n *Node // Var/stack slot storing a defer arg - varsIndex int // Index of variable in lv.vars } // LivenessMap maps from *ssa.Value to LivenessIndex. type LivenessMap struct { vals map[ssa.ID]LivenessIndex + // The set of live, pointer-containing variables at the deferreturn + // call (only set when open-coded defers are used). + deferreturn LivenessIndex } func (m *LivenessMap) reset() { @@ -168,6 +158,7 @@ func (m *LivenessMap) reset() { delete(m.vals, k) } } + m.deferreturn = LivenessInvalid } func (m *LivenessMap) set(v *ssa.Value, i LivenessIndex) { @@ -542,7 +533,7 @@ func newliveness(fn *Node, f *ssa.Func, vars []*Node, idx map[*Node]int32, stkpt if cap(lc.be) >= f.NumBlocks() { lv.be = lc.be[:f.NumBlocks()] } - lv.livenessMap = LivenessMap{lc.livenessMap.vals} + lv.livenessMap = LivenessMap{vals: lc.livenessMap.vals, deferreturn: LivenessInvalid} lc.livenessMap.vals = nil } if lv.be == nil { @@ -893,58 +884,12 @@ func (lv *Liveness) hasStackMap(v *ssa.Value) bool { func (lv *Liveness) prologue() { lv.initcache() - if lv.fn.Func.HasDefer() && !lv.fn.Func.OpenCodedDeferDisallowed() { - lv.openDeferVardefToBlockMap = make(map[*Node]*ssa.Block) - for i, n := range lv.vars { - if n.Name.OpenDeferSlot() { - lv.openDeferVars = append(lv.openDeferVars, openDeferVarInfo{n: n, varsIndex: i}) - } - } - - // Find any blocks that cannot reach a return or a BlockExit - // (panic) -- these must be because of an infinite loop. - reachesRet := make(map[ssa.ID]bool) - blockList := make([]*ssa.Block, 0, 256) - - for _, b := range lv.f.Blocks { - if b.Kind == ssa.BlockRet || b.Kind == ssa.BlockRetJmp || b.Kind == ssa.BlockExit { - blockList = append(blockList, b) - } - } - - for len(blockList) > 0 { - b := blockList[0] - blockList = blockList[1:] - if reachesRet[b.ID] { - continue - } - reachesRet[b.ID] = true - for _, e := range b.Preds { - blockList = append(blockList, e.Block()) - } - } - - lv.nonReturnBlocks = make(map[*ssa.Block]bool) - for _, b := range lv.f.Blocks { - if !reachesRet[b.ID] { - lv.nonReturnBlocks[b] = true - //fmt.Println("No reach ret", lv.f.Name, b.ID, b.Kind) - } - } - } - for _, b := range lv.f.Blocks { be := lv.blockEffects(b) // Walk the block instructions backward and update the block // effects with the each prog effects. for j := len(b.Values) - 1; j >= 0; j-- { - if b.Values[j].Op == ssa.OpVarDef { - n := b.Values[j].Aux.(*Node) - if n.Name.OpenDeferSlot() { - lv.openDeferVardefToBlockMap[n] = b - } - } pos, e := lv.valueEffects(b.Values[j]) regUevar, regKill := lv.regEffects(b.Values[j]) if e&varkill != 0 { @@ -961,20 +906,6 @@ func (lv *Liveness) prologue() { } } -// markDeferVarsLive marks each variable storing an open-coded defer arg as -// specially live in block b if the variable definition dominates block b. -func (lv *Liveness) markDeferVarsLive(b *ssa.Block, newliveout *varRegVec) { - // Only force computation of dominators if we have a block where we need - // to specially mark defer args live. - sdom := lv.f.Sdom() - for _, info := range lv.openDeferVars { - defB := lv.openDeferVardefToBlockMap[info.n] - if sdom.IsAncestorEq(defB, b) { - newliveout.vars.Set(int32(info.varsIndex)) - } - } -} - // Solve the liveness dataflow equations. func (lv *Liveness) solve() { // These temporary bitvectors exist to avoid successive allocations and @@ -1018,23 +949,6 @@ func (lv *Liveness) solve() { } } - if lv.fn.Func.HasDefer() && !lv.fn.Func.OpenCodedDeferDisallowed() && - (b.Kind == ssa.BlockExit || lv.nonReturnBlocks[b]) { - // Open-coded defer args slots must be live - // everywhere in a function, since a panic can - // occur (almost) anywhere. Force all appropriate - // defer arg slots to be live in BlockExit (panic) - // blocks and in blocks that do not reach a return - // (because of infinite loop). - // - // We are assuming that the defer exit code at - // BlockReturn/BlockReturnJmp accesses all of the - // defer args (with pointers), and so keeps them - // live. This analysis may have to be adjusted if - // that changes (because of optimizations). - lv.markDeferVarsLive(b, &newliveout) - } - if !be.liveout.Eq(newliveout) { change = true be.liveout.Copy(newliveout) @@ -1087,6 +1001,17 @@ func (lv *Liveness) epilogue() { n.Name.SetNeedzero(true) livedefer.Set(int32(i)) } + if n.Name.OpenDeferSlot() { + // Open-coded defer args slots must be live + // everywhere in a function, since a panic can + // occur (almost) anywhere. Because it is live + // everywhere, it must be zeroed on entry. + livedefer.Set(int32(i)) + // It was already marked as Needzero when created. + if !n.Name.Needzero() { + Fatalf("all pointer-containing defer arg slots should have Needzero set") + } + } } } @@ -1188,6 +1113,17 @@ func (lv *Liveness) epilogue() { lv.compact(b) } + // If we have an open-coded deferreturn call, make a liveness map for it. + if lv.fn.Func.OpenCodedDeferDisallowed() { + lv.livenessMap.deferreturn = LivenessInvalid + } else { + lv.livenessMap.deferreturn = LivenessIndex{ + stackMapIndex: lv.stackMapSet.add(livedefer), + regMapIndex: 0, // entry regMap, containing no live registers + isUnsafePoint: false, + } + } + // Done compacting. Throw out the stack map set. lv.stackMaps = lv.stackMapSet.extractUniqe() lv.stackMapSet = bvecSet{} diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index d4d23a2956..5d0098b4e6 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -4318,12 +4318,6 @@ func (s *state) openDeferExit() { } } - if i == len(s.openDefers)-1 { - // Record the call of the first defer. This will be used - // to set liveness info for the deferreturn (which is also - // used for any location that causes a runtime panic) - s.f.LastDeferExit = call - } s.endBlock() s.startBlock(bEnd) } @@ -5807,11 +5801,6 @@ type SSAGenState struct { // wasm: The number of values on the WebAssembly stack. This is only used as a safeguard. OnWasmStackSkipped int - - // Liveness index for the first function call in the final defer exit code - // path that we generated. All defer functions and args should be live at - // this point. This will be used to set the liveness for the deferreturn. - lastDeferLiveness LivenessIndex } // Prog appends a new Prog. @@ -6056,12 +6045,6 @@ func genssa(f *ssa.Func, pp *Progs) { // instruction. s.pp.nextLive = s.livenessMap.Get(v) - // Remember the liveness index of the first defer call of - // the last defer exit - if v.Block.Func.LastDeferExit != nil && v == v.Block.Func.LastDeferExit { - s.lastDeferLiveness = s.pp.nextLive - } - // Special case for first line in function; move it to the start. if firstPos != src.NoXPos { s.SetPos(firstPos) @@ -6122,7 +6105,7 @@ func genssa(f *ssa.Func, pp *Progs) { // When doing open-coded defers, generate a disconnected call to // deferreturn and a return. This will be used to during panic // recovery to unwind the stack and return back to the runtime. - s.pp.nextLive = s.lastDeferLiveness + s.pp.nextLive = s.livenessMap.deferreturn gencallret(pp, Deferreturn) } diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 7cf72a8e37..4b9189fb3e 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -33,15 +33,8 @@ type Func struct { Blocks []*Block // unordered set of all basic blocks (note: not indexable by ID) Entry *Block // the entry basic block - // If we are using open-coded defers, this is the first call to a deferred - // function in the final defer exit sequence that we generated. This call - // should be after all defer statements, and will have all args, etc. of - // all defer calls as live. The liveness info of this call will be used - // for the deferreturn/ret segment generated for functions with open-coded - // defers. - LastDeferExit *Value - bid idAlloc // block ID allocator - vid idAlloc // value ID allocator + bid idAlloc // block ID allocator + vid idAlloc // value ID allocator // Given an environment variable used for debug hash match, // what file (if any) receives the yes/no logging? diff --git a/test/fixedbugs/issue40629.go b/test/fixedbugs/issue40629.go new file mode 100644 index 0000000000..c6ef408f49 --- /dev/null +++ b/test/fixedbugs/issue40629.go @@ -0,0 +1,69 @@ +// run + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +const N = 40 + +func main() { + var x [N]int // stack-allocated memory + for i := range x { + x[i] = 0x999 + } + + // This defer checks to see if x is uncorrupted. + defer func(p *[N]int) { + recover() + for i := range p { + if p[i] != 0x999 { + for j := range p { + fmt.Printf("p[%d]=0x%x\n", j, p[j]) + } + panic("corrupted stack variable") + } + } + }(&x) + + // This defer starts a new goroutine, which will (hopefully) + // overwrite x on the garbage stack. + defer func() { + c := make(chan bool) + go func() { + useStack(1000) + c <- true + }() + <-c + + }() + + // This defer causes a stack copy. + // The old stack is now garbage. + defer func() { + useStack(1000) + }() + + // Trigger a segfault. + *g = 0 + + // Make the return statement unreachable. + // That makes the stack map at the deferreturn call empty. + // In particular, the argument to the first defer is not + // marked as a pointer, so it doesn't get adjusted + // during the stack copy. + for { + } +} + +var g *int64 + +func useStack(n int) { + if n == 0 { + return + } + useStack(n - 1) +} -- cgit v1.3 From 24ff2af65e27eed1e8c7f09c21a5ca68fc2e07ab Mon Sep 17 00:00:00 2001 From: lufia Date: Sun, 26 Jul 2020 19:04:37 +0900 Subject: cmd/dist: fix typo Change-Id: Ib5d7f3eadff03070043d52659af4312ee293c586 Reviewed-on: https://go-review.googlesource.com/c/go/+/244817 Reviewed-by: Alberto Donizetti Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot --- src/cmd/dist/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd') diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index a817e6fcd7..397b3bb88f 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -1209,7 +1209,7 @@ func timelog(op, name string) { } i := strings.Index(s, " start") if i < 0 { - log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBULDTIMELOGFILE")) + log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBUILDTIMELOGFILE")) } t, err := time.Parse(time.UnixDate, s[:i]) if err != nil { -- cgit v1.3 From 0031fa80a3c6685e44e84533edbae0dad0eb0395 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 7 May 2020 00:35:28 +0700 Subject: cmd/compile: another fix initializing blank fields in struct literal CL 230121 fixed the bug that struct literal blank fields type array/struct can not be initialized. But it still misses some cases when an expression causes "candiscard(value)" return false. When these happen, we recursively call fixedlit with "var_" set to "_", and hit the bug again. To fix it, just making splitnode return "nblank" whenever "var_" is "nblank". Fixes #38905 Change-Id: I281941b388acbd551a4d8ca1a235477f8d26fb6e Reviewed-on: https://go-review.googlesource.com/c/go/+/232617 Run-TryBot: Cuong Manh Le TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/sinit.go | 6 +++++- test/fixedbugs/issue38905.go | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue38905.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/sinit.go b/src/cmd/compile/internal/gc/sinit.go index 4a2edc7d21..71ed558461 100644 --- a/src/cmd/compile/internal/gc/sinit.go +++ b/src/cmd/compile/internal/gc/sinit.go @@ -506,6 +506,7 @@ const ( // fixedlit handles struct, array, and slice literals. // TODO: expand documentation. func fixedlit(ctxt initContext, kind initKind, n *Node, var_ *Node, init *Nodes) { + isBlank := var_ == nblank var splitnode func(*Node) (a *Node, value *Node) switch n.Op { case OARRAYLIT, OSLICELIT: @@ -520,6 +521,9 @@ func fixedlit(ctxt initContext, kind initKind, n *Node, var_ *Node, init *Nodes) } a := nod(OINDEX, var_, nodintconst(k)) k++ + if isBlank { + a = nblank + } return a, r } case OSTRUCTLIT: @@ -527,7 +531,7 @@ func fixedlit(ctxt initContext, kind initKind, n *Node, var_ *Node, init *Nodes) if r.Op != OSTRUCTKEY { Fatalf("fixedlit: rhs not OSTRUCTKEY: %v", r) } - if r.Sym.IsBlank() { + if r.Sym.IsBlank() || isBlank { return nblank, r.Left } setlineno(r) diff --git a/test/fixedbugs/issue38905.go b/test/fixedbugs/issue38905.go new file mode 100644 index 0000000000..6f411b8605 --- /dev/null +++ b/test/fixedbugs/issue38905.go @@ -0,0 +1,18 @@ +// compile + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure that literal value can be passed to struct +// blank field with expressions where candiscard(value) +// returns false, see #38905. + +package p + +type t struct{ _ u } +type u [10]struct{ f int } + +func f(x int) t { return t{u{{1 / x}, {1 % x}}} } +func g(p *int) t { return t{u{{*p}}} } +func h(s []int) t { return t{u{{s[0]}}} } -- cgit v1.3 From 82c45eb68187d7827bca392d528dbfa06607e3f0 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Jul 2020 12:08:56 +0700 Subject: cmd/compile: handle OCLOSURE/OCALLPART in mustHeapAlloc check Currently, generated struct wrapper for closure is not handled in mustHeapAlloc. That causes compiler crashes when the wrapper struct is too large for stack, and must be heap allocated instead. Fixes #39292 Change-Id: I14c1e591681d9d92317bb2396d6cf5207aa93e08 Reviewed-on: https://go-review.googlesource.com/c/go/+/244917 Run-TryBot: Cuong Manh Le TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/closure.go | 2 +- src/cmd/compile/internal/gc/esc.go | 7 +++++++ test/fixedbugs/issue39292.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue39292.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/closure.go b/src/cmd/compile/internal/gc/closure.go index 3bb7bb9834..04fb7d5495 100644 --- a/src/cmd/compile/internal/gc/closure.go +++ b/src/cmd/compile/internal/gc/closure.go @@ -526,7 +526,7 @@ func walkpartialcall(n *Node, init *Nodes) *Node { // Create closure in the form of a composite literal. // For x.M with receiver (x) type T, the generated code looks like: // - // clos = &struct{F uintptr; R T}{M.T·f, x} + // clos = &struct{F uintptr; R T}{T.M·f, x} // // Like walkclosure above. diff --git a/src/cmd/compile/internal/gc/esc.go b/src/cmd/compile/internal/gc/esc.go index f3e9ab78ef..628953741a 100644 --- a/src/cmd/compile/internal/gc/esc.go +++ b/src/cmd/compile/internal/gc/esc.go @@ -187,6 +187,13 @@ func mustHeapAlloc(n *Node) bool { return true } + if n.Op == OCLOSURE && closureType(n).Size() >= maxImplicitStackVarSize { + return true + } + if n.Op == OCALLPART && partialCallType(n).Size() >= maxImplicitStackVarSize { + return true + } + if n.Op == OMAKESLICE && !isSmallMakeSlice(n) { return true } diff --git a/test/fixedbugs/issue39292.go b/test/fixedbugs/issue39292.go new file mode 100644 index 0000000000..5d6595c234 --- /dev/null +++ b/test/fixedbugs/issue39292.go @@ -0,0 +1,29 @@ +// errorcheck -0 -m -l + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type t [10000]*int + +func (t) f() { +} + +func x() { + x := t{}.f // ERROR "t literal.f escapes to heap" + x() +} + +func y() { + var i int // ERROR "moved to heap: i" + y := (&t{&i}).f // ERROR "\(&t literal\).f escapes to heap" "&t literal escapes to heap" + y() +} + +func z() { + var i int // ERROR "moved to heap: i" + z := t{&i}.f // ERROR "t literal.f escapes to heap" + z() +} -- cgit v1.3 From ccc951637be806e6e7a3c2c922bf4746b60e7395 Mon Sep 17 00:00:00 2001 From: Alberto Donizetti Date: Thu, 13 Aug 2020 12:56:57 +0200 Subject: cmd/link: move comma outside quotes Change-Id: I2ecf8976a6289924ac7bfe7ace129a462537e11d Reviewed-on: https://go-review.googlesource.com/c/go/+/248339 Reviewed-by: Emmanuel Odeke Reviewed-by: Than McIntosh --- src/cmd/link/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd') diff --git a/src/cmd/link/doc.go b/src/cmd/link/doc.go index 219499be0a..604675caec 100644 --- a/src/cmd/link/doc.go +++ b/src/cmd/link/doc.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -Link, typically invoked as ``go tool link,'' reads the Go archive or object +Link, typically invoked as ``go tool link'', reads the Go archive or object for a package main, along with its dependencies, and combines them into an executable binary. -- cgit v1.3 From 01f99b4e9540f34b44e13b25f6dd04b82ac952d9 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 11 Aug 2020 13:07:35 -0700 Subject: cmd/compile: mark DUFFZERO/DUFFCOPY as async unsafe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These operations are async unsafe on architectures that use frame pointers. The reason is they rely on data being safe when stored below the stack pointer. They do: 45da69: 48 89 6c 24 f0 mov %rbp,-0x10(%rsp) 45da6e: 48 8d 6c 24 f0 lea -0x10(%rsp),%rbp 45da73: e8 7d d0 ff ff callq 45aaf5 45da78: 48 8b 6d 00 mov 0x0(%rbp),%rbp This dance ensures that inside duffzero, it looks like there is a proper frame pointer set up, so that stack walkbacks work correctly if the kernel samples during duffzero. However, this instruction sequence depends on data not being clobbered even though it is below the stack pointer. If there is an async interrupt at any of those last 3 instructions, and the interrupt decides to insert a call to asyncPreempt, then the saved frame pointer on the stack gets clobbered. The last instruction above then restores junk to the frame pointer. To prevent this, mark these instructions as async unsafe. (The body of duffzero is already async unsafe, as it is in package runtime.) Change-Id: I5562e82f9f5bd2fb543dcf2b6b9133d87ff83032 Reviewed-on: https://go-review.googlesource.com/c/go/+/248261 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Martin Möhrmann Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/ssa/gen/AMD64Ops.go | 2 ++ src/cmd/compile/internal/ssa/gen/ARM64Ops.go | 2 ++ src/cmd/compile/internal/ssa/opGen.go | 4 ++++ 3 files changed, 8 insertions(+) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go index a3b29049df..e6d66957dd 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go @@ -748,6 +748,7 @@ func init() { clobbers: buildReg("DI"), }, faultOnNilArg0: true, + unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts }, {name: "MOVOconst", reg: regInfo{nil, 0, []regMask{fp}}, typ: "Int128", aux: "Int128", rematerializeable: true}, @@ -786,6 +787,7 @@ func init() { clobberFlags: true, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts }, // arg0 = destination pointer diff --git a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go index b402e35ea6..2424e67e20 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go @@ -507,6 +507,7 @@ func init() { clobbers: buildReg("R20 R30"), }, faultOnNilArg0: true, + unsafePoint: true, // FP maintenance around DUFFZERO can be clobbered by interrupts }, // large zeroing @@ -547,6 +548,7 @@ func init() { }, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts }, // large move diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 9efa1bfcc4..408c855dbd 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -13119,6 +13119,7 @@ var opcodeTable = [...]opInfo{ auxType: auxInt64, argLen: 3, faultOnNilArg0: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 128}, // DI @@ -13196,6 +13197,7 @@ var opcodeTable = [...]opInfo{ clobberFlags: true, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 128}, // DI @@ -20734,6 +20736,7 @@ var opcodeTable = [...]opInfo{ auxType: auxInt64, argLen: 2, faultOnNilArg0: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 1048576}, // R20 @@ -20760,6 +20763,7 @@ var opcodeTable = [...]opInfo{ argLen: 3, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 2097152}, // R21 -- cgit v1.3 From 8c39bbf9c93f773ab351bbddb4c3dd93e4fddc76 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 11 Aug 2020 13:19:57 -0700 Subject: cmd/compile: stop race instrumentation from clobbering frame pointer There is an optimization rule that removes calls to racefuncenter and racefuncexit, if there are no other race calls in the function. The rule removes the call to racefuncenter, but it does *not* remove the store of its argument to the outargs section of the frame. If the outargs section is now size 0 (because the calls to racefuncenter/exit were the only calls), then that argument store clobbers the frame pointer instead. The fix is to remove the argument store when removing the call to racefuncenter. (Racefuncexit doesn't have an argument.) Change-Id: I183ec4d92bbb4920200e1be27b7b8f66b89a2a0a Reviewed-on: https://go-review.googlesource.com/c/go/+/248262 Reviewed-by: Robert Griesemer Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/racewalk.go | 2 +- src/cmd/compile/internal/ssa/rewrite.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/racewalk.go b/src/cmd/compile/internal/gc/racewalk.go index 6f251377c9..3552617401 100644 --- a/src/cmd/compile/internal/gc/racewalk.go +++ b/src/cmd/compile/internal/gc/racewalk.go @@ -42,7 +42,7 @@ var omit_pkgs = []string{ "internal/cpu", } -// Only insert racefuncenterfp/racefuncexit into the following packages. +// Don't insert racefuncenterfp/racefuncexit into the following packages. // Memory accesses in the packages are either uninteresting or will cause false positives. var norace_inst_pkgs = []string{"sync", "sync/atomic"} diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index 2152b1675a..e082bb1dfa 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -1379,6 +1379,15 @@ func needRaceCleanup(sym Sym, v *Value) bool { } } } + if symNamed(sym, "runtime.racefuncenter") { + // If we're removing racefuncenter, remove its argument as well. + if v.Args[0].Op != OpStore { + return false + } + mem := v.Args[0].Args[2] + v.Args[0].reset(OpCopy) + v.Args[0].AddArg(mem) + } return true } -- cgit v1.3 From 6f99b33c18266a8858af96163de97173bdf6f081 Mon Sep 17 00:00:00 2001 From: Polina Osadcha Date: Thu, 18 Jun 2020 16:17:13 +0300 Subject: all: replace Replace(..., -1) with ReplaceAll(...) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I8f7cff7a83a9c50bfa3331e8b40e4a6c2e1c0eee Reviewed-on: https://go-review.googlesource.com/c/go/+/245198 Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- doc/progs/run.go | 2 +- src/cmd/cover/cover_test.go | 2 +- src/cmd/go/internal/version/version.go | 2 +- src/runtime/mkpreempt.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'src/cmd') diff --git a/doc/progs/run.go b/doc/progs/run.go index baef3f79f9..8ac75cdcff 100644 --- a/doc/progs/run.go +++ b/doc/progs/run.go @@ -105,7 +105,7 @@ func test(tmpdir, file, want string) error { // Canonicalize output. out = bytes.TrimRight(out, "\n") - out = bytes.Replace(out, []byte{'\n'}, []byte{' '}, -1) + out = bytes.ReplaceAll(out, []byte{'\n'}, []byte{' '}) // Check the result. match, err := regexp.Match(want, out) diff --git a/src/cmd/cover/cover_test.go b/src/cmd/cover/cover_test.go index 8a56e39011..1c252e6e45 100644 --- a/src/cmd/cover/cover_test.go +++ b/src/cmd/cover/cover_test.go @@ -179,7 +179,7 @@ func TestCover(t *testing.T) { } lines := bytes.Split(file, []byte("\n")) for i, line := range lines { - lines[i] = bytes.Replace(line, []byte("LINE"), []byte(fmt.Sprint(i+1)), -1) + lines[i] = bytes.ReplaceAll(line, []byte("LINE"), []byte(fmt.Sprint(i+1))) } // Add a function that is not gofmt'ed. This used to cause a crash. diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index 056db7bf9e..c2de8d326d 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -138,7 +138,7 @@ func scanFile(file string, info os.FileInfo, mustPrint bool) { fmt.Printf("%s: %s\n", file, vers) if *versionM && mod != "" { - fmt.Printf("\t%s\n", strings.Replace(mod[:len(mod)-1], "\n", "\n\t", -1)) + fmt.Printf("\t%s\n", strings.ReplaceAll(mod[:len(mod)-1], "\n", "\n\t")) } } diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 1fe77663b9..44dea22ef3 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -131,7 +131,7 @@ func header(arch string) { func p(f string, args ...interface{}) { fmted := fmt.Sprintf(f, args...) - fmt.Fprintf(out, "\t%s\n", strings.Replace(fmted, "\n", "\n\t", -1)) + fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) } func label(l string) { -- cgit v1.3 From 9138a2a67f7f29948f6f608bf904b3605f1b45d0 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Tue, 26 May 2020 14:40:44 +1000 Subject: cmd/link: avoid duplicate DT_NEEDED entries When adding a new library entry, ensure we record it as seen to avoid adding duplicates of it. Fixes #39256 Change-Id: Id309adf80c533d78fd485517c18bc9ab5f1d29fb Reviewed-on: https://go-review.googlesource.com/c/go/+/235257 Run-TryBot: Joel Sing TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/elf.go | 1 + src/cmd/link/internal/ld/elf_test.go | 55 +++++++++++++++++++++++ src/cmd/link/internal/ld/testdata/issue39256/x.go | 20 +++++++++ src/cmd/link/internal/ld/testdata/issue39256/x.s | 10 +++++ 4 files changed, 86 insertions(+) create mode 100644 src/cmd/link/internal/ld/testdata/issue39256/x.go create mode 100644 src/cmd/link/internal/ld/testdata/issue39256/x.s (limited to 'src/cmd') diff --git a/src/cmd/link/internal/ld/elf.go b/src/cmd/link/internal/ld/elf.go index 957f5081f6..2862f65f9f 100644 --- a/src/cmd/link/internal/ld/elf.go +++ b/src/cmd/link/internal/ld/elf.go @@ -2378,6 +2378,7 @@ func elfadddynsym(ldr *loader.Loader, target *Target, syms *ArchSyms, s loader.S if target.Arch.Family == sys.AMD64 && !cgoeDynamic && dil != "" && !seenlib[dil] { du := ldr.MakeSymbolUpdater(syms.Dynamic) Elfwritedynent(target.Arch, du, DT_NEEDED, uint64(dstru.Addstring(dil))) + seenlib[dil] = true } } else { diff --git a/src/cmd/link/internal/ld/elf_test.go b/src/cmd/link/internal/ld/elf_test.go index 8e86beb1ec..37f0e77336 100644 --- a/src/cmd/link/internal/ld/elf_test.go +++ b/src/cmd/link/internal/ld/elf_test.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "testing" ) @@ -77,3 +78,57 @@ func main() { t.Fatalf("Unexpected sh info, want greater than 0, got: %d", section.Info) } } + +func TestNoDuplicateNeededEntries(t *testing.T) { + testenv.MustHaveGoBuild(t) + testenv.MustHaveCGO(t) + + // run this test on just a small set of platforms (no need to test it + // across the board given the nature of the test). + pair := runtime.GOOS + "-" + runtime.GOARCH + switch pair { + case "linux-amd64", "freebsd-amd64", "openbsd-amd64": + default: + t.Skip("no need for test on " + pair) + } + + t.Parallel() + + dir, err := ioutil.TempDir("", "no-dup-needed") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(dir) + + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + + path := filepath.Join(dir, "x") + argv := []string{"build", "-o", path, filepath.Join(wd, "testdata", "issue39256")} + out, err := exec.Command(testenv.GoToolPath(t), argv...).CombinedOutput() + if err != nil { + t.Fatalf("Build failure: %s\n%s\n", err, string(out)) + } + + f, err := elf.Open(path) + if err != nil { + t.Fatalf("Failed to open ELF file: %v", err) + } + libs, err := f.ImportedLibraries() + if err != nil { + t.Fatalf("Failed to read imported libraries: %v", err) + } + + var count int + for _, lib := range libs { + if lib == "libc.so" { + count++ + } + } + + if got, want := count, 1; got != want { + t.Errorf("Got %d entries for `libc.so`, want %d", got, want) + } +} diff --git a/src/cmd/link/internal/ld/testdata/issue39256/x.go b/src/cmd/link/internal/ld/testdata/issue39256/x.go new file mode 100644 index 0000000000..d8562ad172 --- /dev/null +++ b/src/cmd/link/internal/ld/testdata/issue39256/x.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + _ "unsafe" +) + +//go:cgo_import_dynamic libc_getpid getpid "libc.so" +//go:cgo_import_dynamic libc_kill kill "libc.so" +//go:cgo_import_dynamic libc_close close "libc.so" +//go:cgo_import_dynamic libc_open open "libc.so" + +func trampoline() + +func main() { + trampoline() +} diff --git a/src/cmd/link/internal/ld/testdata/issue39256/x.s b/src/cmd/link/internal/ld/testdata/issue39256/x.s new file mode 100644 index 0000000000..41a54b2e04 --- /dev/null +++ b/src/cmd/link/internal/ld/testdata/issue39256/x.s @@ -0,0 +1,10 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·trampoline(SB),0,$0 + CALL libc_getpid(SB) + CALL libc_kill(SB) + CALL libc_open(SB) + CALL libc_close(SB) + RET -- cgit v1.3 From dc12d5b0f5e9c1cfec2a8eb6dd7ff3473c36d45c Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Mon, 17 Aug 2020 11:28:26 +0200 Subject: all: add empty line between copyright header and package clause Makes sure the copyright notice is not interpreted as the package level godoc. Change-Id: I2afce7c9d620f19d51ec1438b1d0db1774b57146 Reviewed-on: https://go-review.googlesource.com/c/go/+/248760 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Dave Cheney --- src/cmd/compile/internal/ssa/debug.go | 1 + src/cmd/compile/internal/ssa/passbm_test.go | 1 + src/cmd/go/internal/trace/trace.go | 1 + src/cmd/link/internal/benchmark/bench_test.go | 1 + src/cmd/link/internal/ld/errors.go | 1 + src/runtime/closure_test.go | 1 + src/runtime/map_benchmark_test.go | 1 + src/runtime/slice_test.go | 1 + src/sync/cond_test.go | 1 + test/fixedbugs/issue15281.go | 1 + 10 files changed, 10 insertions(+) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/debug.go b/src/cmd/compile/internal/ssa/debug.go index 13fe67cbca..6353f72897 100644 --- a/src/cmd/compile/internal/ssa/debug.go +++ b/src/cmd/compile/internal/ssa/debug.go @@ -1,6 +1,7 @@ // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package ssa import ( diff --git a/src/cmd/compile/internal/ssa/passbm_test.go b/src/cmd/compile/internal/ssa/passbm_test.go index eefdbb8722..3fd3eb579b 100644 --- a/src/cmd/compile/internal/ssa/passbm_test.go +++ b/src/cmd/compile/internal/ssa/passbm_test.go @@ -1,6 +1,7 @@ // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package ssa import ( diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index 7cb7636a34..c8fac92c9f 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -1,6 +1,7 @@ // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package trace import ( diff --git a/src/cmd/link/internal/benchmark/bench_test.go b/src/cmd/link/internal/benchmark/bench_test.go index d8ec717c7c..419dc55724 100644 --- a/src/cmd/link/internal/benchmark/bench_test.go +++ b/src/cmd/link/internal/benchmark/bench_test.go @@ -1,6 +1,7 @@ // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package benchmark import ( diff --git a/src/cmd/link/internal/ld/errors.go b/src/cmd/link/internal/ld/errors.go index c5ce097fde..d6e8ff236d 100644 --- a/src/cmd/link/internal/ld/errors.go +++ b/src/cmd/link/internal/ld/errors.go @@ -1,6 +1,7 @@ // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package ld import ( diff --git a/src/runtime/closure_test.go b/src/runtime/closure_test.go index ea65fbd5f5..741c932eab 100644 --- a/src/runtime/closure_test.go +++ b/src/runtime/closure_test.go @@ -1,6 +1,7 @@ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package runtime_test import "testing" diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index 893cb6c5b6..d0becc9ddb 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -1,6 +1,7 @@ // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package runtime_test import ( diff --git a/src/runtime/slice_test.go b/src/runtime/slice_test.go index e963a43dd3..cd2bc26d1e 100644 --- a/src/runtime/slice_test.go +++ b/src/runtime/slice_test.go @@ -1,6 +1,7 @@ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package runtime_test import ( diff --git a/src/sync/cond_test.go b/src/sync/cond_test.go index 9d0d9adc74..859cae59bc 100644 --- a/src/sync/cond_test.go +++ b/src/sync/cond_test.go @@ -1,6 +1,7 @@ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package sync_test import ( diff --git a/test/fixedbugs/issue15281.go b/test/fixedbugs/issue15281.go index 187c96f218..390867c848 100644 --- a/test/fixedbugs/issue15281.go +++ b/test/fixedbugs/issue15281.go @@ -3,6 +3,7 @@ // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package main import "runtime" -- cgit v1.3 From a22ec6e650669f5101c7e0955d82e29d644eef4e Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 17 Aug 2020 13:17:26 +0000 Subject: Revert "cmd/internal/obj: fix inline marker issue on s390x" This reverts CL 247697. Reason for revert: This change broke the linux-arm builder. Change-Id: I8ca0d5b3b2ea0109ffbfadeab1406a1b60e7d18d Reviewed-on: https://go-review.googlesource.com/c/go/+/248718 Reviewed-by: Michael Munday Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot --- src/cmd/internal/obj/pcln.go | 15 --------------- src/cmd/internal/obj/s390x/objz.go | 11 +++++++++++ 2 files changed, 11 insertions(+), 15 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index bffeda041d..1f7ccf47ef 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -278,21 +278,6 @@ func linkpcln(ctxt *Link, cursym *LSym) { funcpctab(ctxt, &pcln.Pcfile, cursym, "pctofile", pctofileline, pcln) funcpctab(ctxt, &pcln.Pcline, cursym, "pctoline", pctofileline, nil) - // Check that all the Progs used as inline markers are still reachable. - // See issue #40473. - inlMarkProgs := make(map[*Prog]struct{}, len(cursym.Func.InlMarks)) - for _, inlMark := range cursym.Func.InlMarks { - inlMarkProgs[inlMark.p] = struct{}{} - } - for p := cursym.Func.Text; p != nil; p = p.Link { - if _, ok := inlMarkProgs[p]; ok { - delete(inlMarkProgs, p) - } - } - if len(inlMarkProgs) > 0 { - ctxt.Diag("one or more instructions used as inline markers are no longer reachable") - } - pcinlineState := new(pcinlineState) funcpctab(ctxt, &pcln.Pcinline, cursym, "pctoinline", pcinlineState.pctoinline, nil) for _, inlMark := range cursym.Func.InlMarks { diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index ef6335d849..b14dc810fa 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -283,6 +283,17 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ACMPUBNE: q = p p.Mark |= BRANCH + if p.Pcond != nil { + q := p.Pcond + for q.As == obj.ANOP { + q = q.Link + p.Pcond = q + } + } + + case obj.ANOP: + q.Link = p.Link /* q is non-nop */ + p.Link.Mark |= p.Mark default: q = p -- cgit v1.3 From 7ee26224436d80dca3f7e98c8fcf21185522d8e6 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 12 Aug 2020 20:27:57 -0400 Subject: cmd/link: link dynamic library automatically cgo_import_dynamic pragma indicates a symbol is imported from a dynamic library. Currently, the linker does not actually link against the dynamic library, so we have to "force" it by using //go:cgo_import_dynamic _ _ "dylib" syntax, which links in the library unconditionally. This CL changes it to link in the library automatically when a symbol is imported from the library, without using the "force" syntax. (The "force" syntax is still supported.) Remove the unconditional imports in the runtime. Now, Security.framework and CoreFoundation.framework are only linked when the x509 package is imported (or otherwise specified). Fixes #40727. Change-Id: Ied36b1f621cdcc5dc4a8f497cdf1c554a182d0e0 Reviewed-on: https://go-review.googlesource.com/c/go/+/248333 Run-TryBot: Cherry Zhang Reviewed-by: Filippo Valsorda Reviewed-by: Than McIntosh Reviewed-by: Keith Randall TryBot-Result: Gobot Gobot --- src/cmd/link/internal/ld/go.go | 3 +++ src/runtime/sys_darwin.go | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index bf5c9ca1ba..b3541c46c0 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -183,6 +183,9 @@ func setCgoAttr(ctxt *Link, lookup func(string, int) loader.Sym, file string, pk hostObjSyms[s] = struct{}{} } havedynamic = 1 + if lib != "" && ctxt.IsDarwin() { + machoadddynlib(lib, ctxt.LinkMode) + } } continue diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 06474434c9..e4f19bbf41 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -489,9 +489,3 @@ func setNonblock(fd int32) { //go:cgo_import_dynamic libc_pthread_cond_wait pthread_cond_wait "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_timedwait_relative_np pthread_cond_timedwait_relative_np "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" - -// Magic incantation to get libSystem and friends actually dynamically linked. -// TODO: Why does the code require this? See cmd/link/internal/ld/go.go -//go:cgo_import_dynamic _ _ "/usr/lib/libSystem.B.dylib" -//go:cgo_import_dynamic _ _ "/System/Library/Frameworks/Security.framework/Versions/A/Security" -//go:cgo_import_dynamic _ _ "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation" -- cgit v1.3 From a2a2237ae02016dd9ce16388963cfceece6744f3 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 13 Aug 2020 16:59:52 -0400 Subject: cmd/link: emit correct jump instruction on ARM for DYNIMPORT On ARM, for a JMP/CALL relocation, the instruction bytes is encoded in Reloc.Add (issue #19811). I really hate it, but before it is fixed we have to follow the rule and emit the right bits from r.Add. Fixes #40769. Change-Id: I862e105408d344c5cc58ca9140d2e552e4364453 Reviewed-on: https://go-review.googlesource.com/c/go/+/248399 Reviewed-by: Jeremy Faller Reviewed-by: Joel Sing Reviewed-by: Than McIntosh --- src/cmd/link/internal/arm/asm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd') diff --git a/src/cmd/link/internal/arm/asm.go b/src/cmd/link/internal/arm/asm.go index 22bcb518df..611c96ce35 100644 --- a/src/cmd/link/internal/arm/asm.go +++ b/src/cmd/link/internal/arm/asm.go @@ -220,7 +220,7 @@ func adddynrel(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, s loade addpltsym(target, ldr, syms, targ) su := ldr.MakeSymbolUpdater(s) su.SetRelocSym(rIdx, syms.PLT) - su.SetRelocAdd(rIdx, int64(ldr.SymPlt(targ))) + su.SetRelocAdd(rIdx, int64(braddoff(int32(r.Add()), ldr.SymPlt(targ)/4))) // TODO: don't use r.Add for instruction bytes (issue 19811) return true case objabi.R_ADDR: -- cgit v1.3 From 49003da6d437ef1a4e1e55cf86240480f17dc8ab Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 17:40:35 -0400 Subject: cmd/go/internal/trace: add function to distinguish goroutines trace.StartGoroutine will associate the trace information on the context with a new chrome profiler thread id. The chrome profiler doesn't expect multiple trace events to have the same thread id, so this will allow us to display concurrent events on the trace. Updates #38714 Change-Id: I888b0cce15a5a01db66366716fdd85bf86c832cd Reviewed-on: https://go-review.googlesource.com/c/go/+/248319 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/trace/trace.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index c8fac92c9f..32ce0408f5 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -34,20 +34,33 @@ func StartSpan(ctx context.Context, name string) (context.Context, *Span) { if !ok { return ctx, nil } - childSpan := &Span{t: tc.t, name: name, start: time.Now()} + childSpan := &Span{t: tc.t, name: name, tid: tc.tid, start: time.Now()} tc.t.writeEvent(&traceviewer.Event{ Name: childSpan.name, Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), + TID: childSpan.tid, Phase: "B", }) - ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t}) + ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.tid}) return ctx, childSpan } +// Goroutine associates the context with a new Thread ID. The Chrome trace viewer associates each +// trace event with a thread, and doesn't expect events with the same thread id to happen at the +// same time. +func Goroutine(ctx context.Context) context.Context { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx + } + return context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.t.getNextTID()}) +} + type Span struct { t *tracer name string + tid uint64 start time.Time end time.Time } @@ -60,12 +73,15 @@ func (s *Span) Done() { s.t.writeEvent(&traceviewer.Event{ Name: s.name, Time: float64(s.end.UnixNano()) / float64(time.Microsecond), + TID: s.tid, Phase: "E", }) } type tracer struct { file chan traceFile // 1-buffered + + nextTID uint64 } func (t *tracer) writeEvent(ev *traceviewer.Event) error { @@ -103,12 +119,17 @@ func (t *tracer) Close() error { return firstErr } +func (t *tracer) getNextTID() uint64 { + return atomic.AddUint64(&t.nextTID, 1) +} + // traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in // other packages. type traceKey struct{} type traceContext struct { - t *tracer + t *tracer + tid uint64 } // Start starts a trace which writes to the given file. -- cgit v1.3 From 023d4973851a25e2a47b1ebaf96833c9209efd7c Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 18:05:16 -0400 Subject: cmd/go: add trace events for each action This change adds a trace event for each action and also annotates each of the action execution goroutines with trace.Goroutine so that the actions eaxecuted by each goroutine appear on different threads in the chrome trace viewer. Updates #38714 Change-Id: I2e58dc5606b2e3f7f87076a61e1cc6a2014255c5 Reviewed-on: https://go-review.googlesource.com/c/go/+/248320 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/trace/trace.go | 4 ++-- src/cmd/go/internal/work/exec.go | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index 32ce0408f5..24130d9d72 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -45,10 +45,10 @@ func StartSpan(ctx context.Context, name string) (context.Context, *Span) { return ctx, childSpan } -// Goroutine associates the context with a new Thread ID. The Chrome trace viewer associates each +// StartGoroutine associates the context with a new Thread ID. The Chrome trace viewer associates each // trace event with a thread, and doesn't expect events with the same thread id to happen at the // same time. -func Goroutine(ctx context.Context) context.Context { +func StartGoroutine(ctx context.Context) context.Context { tc, ok := getTraceContext(ctx) if !ok { return ctx diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 3ea3293ae1..56a127f36f 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -115,13 +115,21 @@ func (b *Builder) Do(ctx context.Context, root *Action) { // Handle runs a single action and takes care of triggering // any actions that are runnable as a result. - handle := func(a *Action) { + handle := func(ctx context.Context, a *Action) { if a.json != nil { a.json.TimeStart = time.Now() } var err error if a.Func != nil && (!a.Failed || a.IgnoreFail) { + // TODO(matloob): Better action descriptions + desc := "Executing action " + if a.Package != nil { + desc += "(" + a.Mode + " " + a.Package.Desc() + ")" + } + ctx, span := trace.StartSpan(ctx, desc) + _ = ctx err = a.Func(b, a) + span.Done() } if a.json != nil { a.json.TimeDone = time.Now() @@ -169,6 +177,7 @@ func (b *Builder) Do(ctx context.Context, root *Action) { for i := 0; i < par; i++ { wg.Add(1) go func() { + ctx := trace.StartGoroutine(ctx) defer wg.Done() for { select { @@ -181,7 +190,7 @@ func (b *Builder) Do(ctx context.Context, root *Action) { b.exec.Lock() a := b.ready.pop() b.exec.Unlock() - handle(a) + handle(ctx, a) case <-base.Interrupted: base.SetExitStatus(1) return -- cgit v1.3 From a26d687ebb23fa14b777ef5bf69b56556124ff3b Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 18:18:23 -0400 Subject: cmd/go: propagate context into Action.Func calls Action.Func is now a func(*Builder, context.Context, *Action), so that contexts can be propagated into the action funcs. While context is traditionally the first parameter of a function, it's the second parameter of Action.Func's type to continue to allow for methods on Builder to be used as functions taking a *Builder as the first parameter. context.Context is instead the first parameter on those functions. Change-Id: I5f058d6a99a1e96fe2025f2e8ce30a033d12e935 Reviewed-on: https://go-review.googlesource.com/c/go/+/248321 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 10 +++++----- src/cmd/go/internal/work/action.go | 15 ++++++++------- src/cmd/go/internal/work/exec.go | 17 ++++++++--------- 4 files changed, 22 insertions(+), 22 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 3630f68c54..deec5106ff 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -146,7 +146,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { // buildRunProgram is the action for running a binary that has already // been compiled. We ignore exit status. -func buildRunProgram(b *work.Builder, a *work.Action) error { +func buildRunProgram(b *work.Builder, ctx context.Context, a *work.Action) error { cmdline := str.StringList(work.FindExecCmd(), a.Deps[0].Target, a.Args) if cfg.BuildN || cfg.BuildX { b.Showcmd("", "%s", strings.Join(cmdline, " ")) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index d71d339828..9788590938 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1069,7 +1069,7 @@ func (lockedStdout) Write(b []byte) (int, error) { } // builderRunTest is the action for running a test binary. -func (c *runCache) builderRunTest(b *work.Builder, a *work.Action) error { +func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work.Action) error { if a.Failed { // We were unable to build the binary. a.Failed = false @@ -1642,7 +1642,7 @@ func coveragePercentage(out []byte) string { } // builderCleanTest is the action for cleaning up after a test. -func builderCleanTest(b *work.Builder, a *work.Action) error { +func builderCleanTest(b *work.Builder, ctx context.Context, a *work.Action) error { if cfg.BuildWork { return nil } @@ -1654,7 +1654,7 @@ func builderCleanTest(b *work.Builder, a *work.Action) error { } // builderPrintTest is the action for printing a test result. -func builderPrintTest(b *work.Builder, a *work.Action) error { +func builderPrintTest(b *work.Builder, ctx context.Context, a *work.Action) error { clean := a.Deps[0] run := clean.Deps[0] if run.TestOutput != nil { @@ -1665,7 +1665,7 @@ func builderPrintTest(b *work.Builder, a *work.Action) error { } // builderNoTest is the action for testing a package with no test files. -func builderNoTest(b *work.Builder, a *work.Action) error { +func builderNoTest(b *work.Builder, ctx context.Context, a *work.Action) error { var stdout io.Writer = os.Stdout if testJSON { json := test2json.NewConverter(lockedStdout{}, a.Package.ImportPath, test2json.Timestamp) @@ -1677,7 +1677,7 @@ func builderNoTest(b *work.Builder, a *work.Action) error { } // printExitStatus is the action for printing the exit status -func printExitStatus(b *work.Builder, a *work.Action) error { +func printExitStatus(b *work.Builder, ctx context.Context, a *work.Action) error { if !testJSON && len(pkgArgs) != 0 { if base.GetExitStatus() != 0 { fmt.Println("FAIL") diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index 6b5f9e4807..a37a5e618d 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -10,6 +10,7 @@ import ( "bufio" "bytes" "container/heap" + "context" "debug/elf" "encoding/json" "fmt" @@ -63,13 +64,13 @@ type Builder struct { // An Action represents a single action in the action graph. type Action struct { - Mode string // description of action operation - Package *load.Package // the package this action works on - Deps []*Action // actions that must happen before this one - Func func(*Builder, *Action) error // the action itself (nil = no-op) - IgnoreFail bool // whether to run f even if dependencies fail - TestOutput *bytes.Buffer // test output buffer - Args []string // additional args for runProgram + Mode string // description of action operation + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Func func(*Builder, context.Context, *Action) error // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram triggers []*Action // inverse of deps diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 56a127f36f..3903502a67 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -127,8 +127,7 @@ func (b *Builder) Do(ctx context.Context, root *Action) { desc += "(" + a.Mode + " " + a.Package.Desc() + ")" } ctx, span := trace.StartSpan(ctx, desc) - _ = ctx - err = a.Func(b, a) + err = a.Func(b, ctx, a) span.Done() } if a.json != nil { @@ -400,7 +399,7 @@ const ( // build is the action for building a single package. // Note that any new influence on this logic must be reported in b.buildActionID above as well. -func (b *Builder) build(a *Action) (err error) { +func (b *Builder) build(ctx context.Context, a *Action) (err error) { p := a.Package bit := func(x uint32, b bool) uint32 { @@ -1005,7 +1004,7 @@ var VetFlags []string // VetExplicit records whether the vet flags were set explicitly on the command line. var VetExplicit bool -func (b *Builder) vet(a *Action) error { +func (b *Builder) vet(ctx context.Context, a *Action) error { // a.Deps[0] is the build of the package being vetted. // a.Deps[1] is the build of the "fmt" package. @@ -1196,7 +1195,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { // link is the action for linking a single command. // Note that any new influence on this logic must be reported in b.linkActionID above as well. -func (b *Builder) link(a *Action) (err error) { +func (b *Builder) link(ctx context.Context, a *Action) (err error) { if b.useCache(a, b.linkActionID(a), a.Package.Target) || b.IsCmdList { return nil } @@ -1388,7 +1387,7 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return } -func (b *Builder) installShlibname(a *Action) error { +func (b *Builder) installShlibname(ctx context.Context, a *Action) error { if err := allowInstall(a); err != nil { return err } @@ -1437,7 +1436,7 @@ func (b *Builder) linkSharedActionID(a *Action) cache.ActionID { return h.Sum() } -func (b *Builder) linkShared(a *Action) (err error) { +func (b *Builder) linkShared(ctx context.Context, a *Action) (err error) { if b.useCache(a, b.linkSharedActionID(a), a.Target) || b.IsCmdList { return nil } @@ -1463,7 +1462,7 @@ func (b *Builder) linkShared(a *Action) (err error) { } // BuildInstallFunc is the action for installing a single package or executable. -func BuildInstallFunc(b *Builder, a *Action) (err error) { +func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { defer func() { if err != nil && err != errPrintedOutput { // a.Package == nil is possible for the go install -buildmode=shared @@ -1716,7 +1715,7 @@ func (b *Builder) writeFile(file string, text []byte) error { } // Install the cgo export header file, if there is one. -func (b *Builder) installHeader(a *Action) error { +func (b *Builder) installHeader(ctx context.Context, a *Action) error { src := a.Objdir + "_cgo_install.h" if _, err := os.Stat(src); os.IsNotExist(err) { // If the file does not exist, there are no exported -- cgit v1.3 From 15b98e55d195bd876203506d5f513546dd4e3b36 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 18 Jun 2020 11:58:46 -0400 Subject: cmd/go: mark trace flows between actions This could help make it easier to identify blocking dependencies when examining traces. Flows can be turned off when viewing traces to remove potential distractions. Updates #38714 Change-Id: Ibfd3f1a1861e3cac31addb053a2fca7ee796c4d7 Reviewed-on: https://go-review.googlesource.com/c/go/+/248322 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/trace/trace.go | 51 +++++++++++++++++++++++++++++++++++--- src/cmd/go/internal/work/action.go | 10 +++++--- src/cmd/go/internal/work/exec.go | 4 +++ 3 files changed, 58 insertions(+), 7 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index 24130d9d72..f108a2b6ca 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -15,6 +15,18 @@ import ( "time" ) +// Constants used in event fields. +// See https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU +// for more details. +const ( + phaseDurationBegin = "B" + phaseDurationEnd = "E" + phaseFlowStart = "s" + phaseFlowEnd = "f" + + bindEnclosingSlice = "e" +) + var traceStarted int32 func getTraceContext(ctx context.Context) (traceContext, bool) { @@ -39,7 +51,7 @@ func StartSpan(ctx context.Context, name string) (context.Context, *Span) { Name: childSpan.name, Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), TID: childSpan.tid, - Phase: "B", + Phase: phaseDurationBegin, }) ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.tid}) return ctx, childSpan @@ -56,6 +68,34 @@ func StartGoroutine(ctx context.Context) context.Context { return context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.t.getNextTID()}) } +// Flow marks a flow indicating that the 'to' span depends on the 'from' span. +// Flow should be called while the 'to' span is in progress. +func Flow(ctx context.Context, from *Span, to *Span) { + tc, ok := getTraceContext(ctx) + if !ok || from == nil || to == nil { + return + } + + id := tc.t.getNextFlowID() + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", + ID: id, + Time: float64(from.end.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowStart, + TID: from.tid, + }) + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", // TODO(matloob): Add Category to Flow? + ID: id, + Time: float64(to.start.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowEnd, + TID: to.tid, + BindPoint: bindEnclosingSlice, + }) +} + type Span struct { t *tracer @@ -74,14 +114,15 @@ func (s *Span) Done() { Name: s.name, Time: float64(s.end.UnixNano()) / float64(time.Microsecond), TID: s.tid, - Phase: "E", + Phase: phaseDurationEnd, }) } type tracer struct { file chan traceFile // 1-buffered - nextTID uint64 + nextTID uint64 + nextFlowID uint64 } func (t *tracer) writeEvent(ev *traceviewer.Event) error { @@ -123,6 +164,10 @@ func (t *tracer) getNextTID() uint64 { return atomic.AddUint64(&t.nextTID, 1) } +func (t *tracer) getNextFlowID() uint64 { + return atomic.AddUint64(&t.nextFlowID, 1) +} + // traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in // other packages. type traceKey struct{} diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index a37a5e618d..825e763c03 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -26,6 +26,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/trace" "cmd/internal/buildid" ) @@ -92,10 +93,11 @@ type Action struct { output []byte // output redirect buffer (nil means use b.Print) // Execution state. - pending int // number of deps yet to complete - priority int // relative execution priority - Failed bool // whether the action failed - json *actionJSON // action graph information + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed + json *actionJSON // action graph information + traceSpan *trace.Span } // BuildActionID returns the action ID section of a's build ID. diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 3903502a67..681ecd7646 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -127,6 +127,10 @@ func (b *Builder) Do(ctx context.Context, root *Action) { desc += "(" + a.Mode + " " + a.Package.Desc() + ")" } ctx, span := trace.StartSpan(ctx, desc) + a.traceSpan = span + for _, d := range a.Deps { + trace.Flow(ctx, d.traceSpan, a.traceSpan) + } err = a.Func(b, ctx, a) span.Done() } -- cgit v1.3 From 38fea3a4ec97fbcfad1f2d329f3a12c53cc36301 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 22 Jun 2020 18:39:10 -0400 Subject: cmd/go: add tracing instrumentation to load.TestPackagesFor This change adds tracing instrumentation into load.TestPackagesFor, propagating context through its callers. Updates #38714 Change-Id: I80fefaf3116ccccffaa8bb7613a656bda867394c Reviewed-on: https://go-review.googlesource.com/c/go/+/248323 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/list/list.go | 4 ++-- src/cmd/go/internal/load/test.go | 14 ++++++++++---- src/cmd/go/internal/test/test.go | 6 +++--- src/cmd/go/internal/vet/vet.go | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 3ec243a759..7747e730ae 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -477,9 +477,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { var pmain, ptest, pxtest *load.Package var err error if *listE { - pmain, ptest, pxtest = load.TestPackagesAndErrors(p, nil) + pmain, ptest, pxtest = load.TestPackagesAndErrors(ctx, p, nil) } else { - pmain, ptest, pxtest, err = load.TestPackagesFor(p, nil) + pmain, ptest, pxtest, err = load.TestPackagesFor(ctx, p, nil) if err != nil { base.Errorf("can't load test package: %s", err) } diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 6d251e8358..6db8a00245 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -6,7 +6,7 @@ package load import ( "bytes" - "cmd/go/internal/str" + "context" "errors" "fmt" "go/ast" @@ -20,6 +20,9 @@ import ( "strings" "unicode" "unicode/utf8" + + "cmd/go/internal/str" + "cmd/go/internal/trace" ) var TestMainDeps = []string{ @@ -42,8 +45,8 @@ type TestCover struct { // TestPackagesFor is like TestPackagesAndErrors but it returns // an error if the test packages or their dependencies have errors. // Only test packages without errors are returned. -func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { - pmain, ptest, pxtest = TestPackagesAndErrors(p, cover) +func TestPackagesFor(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { + pmain, ptest, pxtest = TestPackagesAndErrors(ctx, p, cover) for _, p1 := range []*Package{ptest, pxtest, pmain} { if p1 == nil { // pxtest may be nil @@ -89,7 +92,10 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag // // The caller is expected to have checked that len(p.TestGoFiles)+len(p.XTestGoFiles) > 0, // or else there's no point in any of this. -func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { +func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { + ctx, span := trace.StartSpan(ctx, "load.TestPackagesAndErrors") + defer span.Done() + pre := newPreload() defer pre.flush() allImports := append([]string{}, p.TestImports...) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9788590938..cda51053fb 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -746,7 +746,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { ensureImport(p, "sync/atomic") } - buildTest, runTest, printTest, err := builderTest(&b, p) + buildTest, runTest, printTest, err := builderTest(&b, ctx, p) if err != nil { str := err.Error() str = strings.TrimPrefix(str, "\n") @@ -813,7 +813,7 @@ var windowsBadWords = []string{ "update", } -func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { +func builderTest(b *work.Builder, ctx context.Context, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { if len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { build := b.CompileAction(work.ModeBuild, work.ModeBuild, p) run := &work.Action{Mode: "test run", Package: p, Deps: []*work.Action{build}} @@ -836,7 +836,7 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin DeclVars: declareCoverVars, } } - pmain, ptest, pxtest, err := load.TestPackagesFor(p, cover) + pmain, ptest, pxtest, err := load.TestPackagesFor(ctx, p, cover) if err != nil { return nil, nil, nil, err } diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 58f392eb96..b306572281 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -77,7 +77,7 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { root := &work.Action{Mode: "go vet"} for _, p := range pkgs { - _, ptest, pxtest, err := load.TestPackagesFor(p, nil) + _, ptest, pxtest, err := load.TestPackagesFor(ctx, p, nil) if err != nil { base.Errorf("%v", err) continue -- cgit v1.3 From ebccba7954fe9507df993dda7ba78fa34e030390 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 22 Jun 2020 19:02:00 -0400 Subject: cmd/go: process -debug-trace flag for cmd/test and cmd/vet These commands are build-like commands that do their own flag processing, so the value of debug-trace isn't available until the command starts running. Start tracing in the cmd's run function. Updates #38714 Change-Id: I4d633e6ee907bf09feac52c2aff3daceb9b20e12 Reviewed-on: https://go-review.googlesource.com/c/go/+/248324 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/test/test.go | 18 ++++++++++++++++++ src/cmd/go/internal/vet/vet.go | 25 +++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index cda51053fb..9c120e08dc 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -31,6 +31,7 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/modload" "cmd/go/internal/str" + "cmd/go/internal/trace" "cmd/go/internal/work" "cmd/internal/test2json" ) @@ -571,6 +572,23 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { pkgArgs, testArgs = testFlags(args) + if cfg.DebugTrace != "" { + var close func() error + var err error + ctx, close, err = trace.Start(ctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + defer func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }() + } + + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + defer span.Done() + work.FindExecCmd() // initialize cached result work.BuildInit() diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index b306572281..cf2c8d59e8 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -6,12 +6,16 @@ package vet import ( + "context" + "fmt" + "path/filepath" + "cmd/go/internal/base" + "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" + "cmd/go/internal/trace" "cmd/go/internal/work" - "context" - "path/filepath" ) // Break init loop. @@ -54,6 +58,23 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { vetFlags, pkgArgs := vetFlags(args) + if cfg.DebugTrace != "" { + var close func() error + var err error + ctx, close, err = trace.Start(ctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + defer func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }() + } + + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + defer span.Done() + work.BuildInit() work.VetFlags = vetFlags if len(vetFlags) > 0 { -- cgit v1.3 From 2ac4bf3802f0786a0afb09488173507f40d5d885 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 24 Jun 2020 15:58:47 -0400 Subject: cmd/go: add span for modload.LoadBuildList This change adds context, and a span to modload.LoadBuildList and propagates context into modload.BuildList. It's the start of a run of CLs to add trace spans for module operations. Updates #38714 Change-Id: I0d58dd394051526338092dc9a5ec29a9e087e4e4 Reviewed-on: https://go-review.googlesource.com/c/go/+/248325 Run-TryBot: Michael Matloob Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/list/list.go | 4 ++-- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modcmd/graph.go | 2 +- src/cmd/go/internal/modcmd/verify.go | 2 +- src/cmd/go/internal/modcmd/why.go | 2 +- src/cmd/go/internal/modget/get.go | 8 ++++---- src/cmd/go/internal/modload/list.go | 9 +++++---- src/cmd/go/internal/modload/load.go | 23 ++++++++++++++--------- 8 files changed, 29 insertions(+), 23 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7747e730ae..7303e6c866 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -413,9 +413,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } - modload.LoadBuildList() + modload.LoadBuildList(ctx) - mods := modload.ListModules(args, *listU, *listVersions) + mods := modload.ListModules(ctx, args, *listU, *listVersions) if !*listE { for _, m := range mods { if m.Error != nil { diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index b43c32be5a..946e8ed3cf 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -106,7 +106,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { var work par.Work listU := false listVersions := false - for _, info := range modload.ListModules(args, listU, listVersions) { + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { if info.Replace != nil { info = info.Replace } diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index fff5b02626..4853503fd4 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -49,7 +49,7 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go: cannot find main module; see 'go help modules'") } } - modload.LoadBuildList() + modload.LoadBuildList(ctx) reqs := modload.MinReqs() format := func(m module.Version) string { diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index 570e571049..73ab714d10 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -60,7 +60,7 @@ func runVerify(ctx context.Context, cmd *base.Command, args []string) { sem := make(chan token, runtime.GOMAXPROCS(0)) // Use a slice of result channels, so that the output is deterministic. - mods := modload.LoadBuildList()[1:] + mods := modload.LoadBuildList(ctx)[1:] errsChans := make([]<-chan []error, len(mods)) for i, mod := range mods { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index 3f9cf0f120..f400339b25 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -74,7 +74,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod why: module query not allowed") } } - mods := modload.ListModules(args, listU, listVersions) + mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) for _, path := range loadALL() { m := modload.PackageModule(path) diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index b217196931..93a6bb54d5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -278,7 +278,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { } modload.LoadTests = *getT - buildList := modload.LoadBuildList() + buildList := modload.LoadBuildList(ctx) buildList = buildList[:len(buildList):len(buildList)] // copy on append versionByPath := make(map[string]string) for _, m := range buildList { @@ -444,7 +444,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // packages in unknown modules can't be expanded. This also avoids looking // up new modules while loading packages, only to downgrade later. queryCache := make(map[querySpec]*query) - byPath := runQueries(queryCache, queries, nil) + byPath := runQueries(ctx, queryCache, queries, nil) // Add missing modules to the build list. // We call SetBuildList here and elsewhere, since newUpgrader, @@ -586,7 +586,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Query target versions for modules providing packages matched by // command line arguments. - byPath = runQueries(queryCache, queries, modOnly) + byPath = runQueries(ctx, queryCache, queries, modOnly) // Handle upgrades. This is needed for arguments that didn't match // modules or matched different modules from a previous iteration. It @@ -724,7 +724,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // versions (including earlier queries in the modOnly map), an error will be // reported. A map from module paths to queries is returned, which includes // queries and modOnly. -func runQueries(cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { +func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { var lookup par.Work for _, q := range queries { if cached := cache[q.querySpec]; cached != nil { diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 9400793bcb..4768516e90 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -19,8 +20,8 @@ import ( "golang.org/x/mod/module" ) -func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePublic { - mods := listModules(args, listVersions) +func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions) if listU || listVersions { var work par.Work for _, m := range mods { @@ -42,8 +43,8 @@ func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePubli return mods } -func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { - LoadBuildList() +func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { + LoadBuildList(ctx) if len(args) == 0 { return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} } diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 30992e0cc2..8190009b23 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -6,14 +6,7 @@ package modload import ( "bytes" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/imports" - "cmd/go/internal/modfetch" - "cmd/go/internal/mvs" - "cmd/go/internal/par" - "cmd/go/internal/search" - "cmd/go/internal/str" + "context" "errors" "fmt" "go/build" @@ -24,6 +17,16 @@ import ( "sort" "strings" + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" + "cmd/go/internal/trace" + "golang.org/x/mod/module" ) @@ -385,7 +388,9 @@ func DirImportPath(dir string) string { // LoadBuildList need only be called if ImportPaths is not // (typically in commands that care about the module but // no particular package). -func LoadBuildList() []module.Version { +func LoadBuildList(ctx context.Context) []module.Version { + ctx, span := trace.StartSpan(ctx, "LoadBuildList") + defer span.Done() InitMod() ReloadBuildList() WriteGoMod() -- cgit v1.3 From f30044a03bc7cf107dbec03c02fb6d0072878252 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 25 Jun 2020 19:11:28 -0400 Subject: cmd/go/internal: remove some users of par.Work par.Work is used in a number of places as a parallel work queue. This change replaces it with goroutines and channels in a number of simpler places where it's used. Change-Id: I0620eda46ec7b2c0599a8b9361639af7bb73a05a Reviewed-on: https://go-review.googlesource.com/c/go/+/248326 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modcmd/download.go | 69 +++++++++++++++++++--------------- src/cmd/go/internal/modcmd/graph.go | 19 +++++----- src/cmd/go/internal/modconv/convert.go | 59 ++++++++++++++++------------- src/cmd/go/internal/modget/get.go | 41 +++++++++++++------- src/cmd/go/internal/modload/list.go | 37 +++++++++++------- 5 files changed, 133 insertions(+), 92 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 946e8ed3cf..857362a72e 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -5,15 +5,15 @@ package modcmd import ( + "cmd/go/internal/modfetch" "context" "encoding/json" "os" + "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -102,33 +102,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } } - var mods []*moduleJSON - var work par.Work - listU := false - listVersions := false - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - work.Add(m) - } - - work.Do(10, func(item interface{}) { - m := item.(*moduleJSON) + downloadModule := func(m *moduleJSON) { var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -157,7 +131,42 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { m.Error = err.Error() return } - }) + } + + var mods []*moduleJSON + listU := false + listVersions := false + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + sem <- token{} + go func() { + downloadModule(m) + <-sem + }() + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } if *downloadJSON { for _, m := range mods { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 4853503fd4..6da12b9cab 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,7 +15,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -59,23 +58,25 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { return m.Path + "@" + m.Version } - // Note: using par.Work only to manage work queue. - // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - var work par.Work - work.Add(modload.Target) - work.Do(1, func(item interface{}) { - m := item.(module.Version) + seen := map[module.Version]bool{modload.Target: true} + queue := []module.Version{modload.Target} + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] list, _ := reqs.Required(m) for _, r := range list { - work.Add(r) + if !seen[r] { + queue = append(queue, r) + seen[r] = true + } out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - }) + } sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/src/cmd/go/internal/modconv/convert.go b/src/cmd/go/internal/modconv/convert.go index f465a9f395..d5a0bc21e9 100644 --- a/src/cmd/go/internal/modconv/convert.go +++ b/src/cmd/go/internal/modconv/convert.go @@ -7,13 +7,12 @@ package modconv import ( "fmt" "os" + "runtime" "sort" "strings" - "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" - "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -42,46 +41,52 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var ( - work par.Work - mu sync.Mutex - need = make(map[string]string) - replace = make(map[string]*modfile.Replace) - ) + versions := make([]*module.Version, len(mf.Require)) + replace := make(map[string]*modfile.Replace) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - for _, r := range mf.Require { + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for i, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - work.Add(re.New) - continue + m = re.New } - work.Add(r.Mod) + sem <- token{} + go func(i int, m module.Version) { + repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) + return + } + + path := repo.ModulePath() + versions[i].Path = path + versions[i].Version = info.Version + + <-sem + }(i, m) + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} } - work.Do(10, func(item interface{}) { - r := item.(module.Version) - repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) - return - } - mu.Lock() - path := repo.ModulePath() + need := map[string]string{} + for _, v := range versions { // Don't use semver.Max here; need to preserve +incompatible suffix. - if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { - need[path] = info.Version + if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { + need[v.Path] = v.Version } - mu.Unlock() - }) - - var paths []string + } + paths := make([]string, 0, len(need)) for path := range need { paths = append(paths, path) } diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 93a6bb54d5..d02c9a8da5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -21,7 +22,6 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/mvs" - "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" @@ -725,18 +725,8 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // reported. A map from module paths to queries is returned, which includes // queries and modOnly. func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { - var lookup par.Work - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - cache[q.querySpec] = q - lookup.Add(q) - } - } - lookup.Do(10, func(item interface{}) { - q := item.(*query) + runQuery := func(q *query) { if q.vers == "none" { // Wait for downgrade step. q.m = module.Version{Path: q.path, Version: "none"} @@ -747,7 +737,32 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer base.Errorf("go get %s: %v", q.arg, err) } q.m = m - }) + } + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, q := range queries { + if cached := cache[q.querySpec]; cached != nil { + *q = *cached + } else { + sem <- token{} + go func(q *query) { + runQuery(q) + <-sem + }(q) + } + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + // Add to cache after concurrent section to avoid races... + for _, q := range queries { + cache[q.querySpec] = q + } + base.ExitIfErrors() byPath := make(map[string]*query) diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 4768516e90..8db4d64706 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "os" + "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" - "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" @@ -22,24 +22,35 @@ import ( func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { mods := listModules(ctx, args, listVersions) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) if listU || listVersions { - var work par.Work for _, m := range mods { - work.Add(m) + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(m) + } + if listVersions { + addVersions(m) + } + <-sem + }() + } + + add(m) if m.Replace != nil { - work.Add(m.Replace) + add(m.Replace) } } - work.Do(10, func(item interface{}) { - m := item.(*modinfo.ModulePublic) - if listU { - addUpdate(m) - } - if listVersions { - addVersions(m) - } - }) } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + return mods } -- cgit v1.3 From 1b86bdbdc3991c13c6ed156100a5f4918fdd9c6b Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 14 Aug 2020 17:44:22 -0400 Subject: cmd/test2json: do not emit a final Action if the result is not known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we are parsing a test output, and the test does not end in the usual PASS or FAIL line (say, because it panicked), then we need the exit status of the test binary in order to determine whether the test passed or failed. If we don't have that status available, we shouldn't guess arbitrarily — instead, we should omit the final "pass" or "fail" action entirely. (In practice, we nearly always DO have the final status, such as when running 'go test' or 'go tool test2json some.exe'.) Fixes #40132 Change-Id: Iae482577361a6033395fe4a05d746b980e18c3de Reviewed-on: https://go-review.googlesource.com/c/go/+/248624 Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/test/test.go | 8 +- src/cmd/go/testdata/script/test_json_exit.txt | 102 +++++++++++++++++++++ src/cmd/internal/test2json/test2json.go | 44 +++++---- .../internal/test2json/testdata/benchshort.json | 1 - src/cmd/internal/test2json/testdata/empty.json | 1 - src/cmd/test2json/main.go | 6 +- 6 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 src/cmd/go/testdata/script/test_json_exit.txt (limited to 'src/cmd') diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9c120e08dc..9cef8cf89c 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1098,9 +1098,13 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } var stdout io.Writer = os.Stdout + var err error if testJSON { json := test2json.NewConverter(lockedStdout{}, a.Package.ImportPath, test2json.Timestamp) - defer json.Close() + defer func() { + json.Exited(err) + json.Close() + }() stdout = json } @@ -1204,7 +1208,7 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } t0 := time.Now() - err := cmd.Start() + err = cmd.Start() // This is a last-ditch deadline to detect and // stop wedged test binaries, to keep the builders diff --git a/src/cmd/go/testdata/script/test_json_exit.txt b/src/cmd/go/testdata/script/test_json_exit.txt new file mode 100644 index 0000000000..dc7ffb06cf --- /dev/null +++ b/src/cmd/go/testdata/script/test_json_exit.txt @@ -0,0 +1,102 @@ +[short] skip + +go test -c -o mainpanic.exe ./mainpanic & +go test -c -o mainexit0.exe ./mainexit0 & +go test -c -o testpanic.exe ./testpanic & +go test -c -o testbgpanic.exe ./testbgpanic & +wait + +# Test binaries that panic in TestMain should be marked as failing. + +! go test -json ./mainpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./mainpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +# Test binaries that exit with status 0 should be marked as passing. + +go test -json ./mainexit0 +stdout '"Action":"pass"' +! stdout '"Action":"fail"' + +go tool test2json ./mainexit0.exe +stdout '"Action":"pass"' +! stdout '"Action":"fail"' + +# Test functions that panic should never be marked as passing +# (https://golang.org/issue/40132). + +! go test -json ./testpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testpanic.exe -test.v +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +# Tests that panic in a background goroutine should be marked as failing. + +! go test -json ./testbgpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testbgpanic.exe -test.v +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testbgpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +-- go.mod -- +module m +go 1.14 +-- mainpanic/mainpanic_test.go -- +package mainpanic_test + +import "testing" + +func TestMain(m *testing.M) { + panic("haha no") +} +-- mainexit0/mainexit0_test.go -- +package mainexit0_test + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + fmt.Println("nothing to do") + os.Exit(0) +} +-- testpanic/testpanic_test.go -- +package testpanic_test + +import "testing" + +func TestPanic(*testing.T) { + panic("haha no") +} +-- testbgpanic/testbgpanic_test.go -- +package testbgpanic_test + +import "testing" + +func TestPanicInBackground(*testing.T) { + c := make(chan struct{}) + go func() { + panic("haha no") + close(c) + }() + <-c +} diff --git a/src/cmd/internal/test2json/test2json.go b/src/cmd/internal/test2json/test2json.go index a01a8900e8..4eb6dd4838 100644 --- a/src/cmd/internal/test2json/test2json.go +++ b/src/cmd/internal/test2json/test2json.go @@ -45,10 +45,10 @@ type textBytes []byte func (b textBytes) MarshalText() ([]byte, error) { return b, nil } -// A converter holds the state of a test-to-JSON conversion. +// A Converter holds the state of a test-to-JSON conversion. // It implements io.WriteCloser; the caller writes test output in, // and the converter writes JSON output to w. -type converter struct { +type Converter struct { w io.Writer // JSON output stream pkg string // package to name in events mode Mode // mode bits @@ -100,9 +100,9 @@ var ( // // The pkg string, if present, specifies the import path to // report in the JSON stream. -func NewConverter(w io.Writer, pkg string, mode Mode) io.WriteCloser { - c := new(converter) - *c = converter{ +func NewConverter(w io.Writer, pkg string, mode Mode) *Converter { + c := new(Converter) + *c = Converter{ w: w, pkg: pkg, mode: mode, @@ -122,11 +122,20 @@ func NewConverter(w io.Writer, pkg string, mode Mode) io.WriteCloser { } // Write writes the test input to the converter. -func (c *converter) Write(b []byte) (int, error) { +func (c *Converter) Write(b []byte) (int, error) { c.input.write(b) return len(b), nil } +// Exited marks the test process as having exited with the given error. +func (c *Converter) Exited(err error) { + if err == nil { + c.result = "pass" + } else { + c.result = "fail" + } +} + var ( // printed by test on successful run. bigPass = []byte("PASS\n") @@ -160,7 +169,7 @@ var ( // handleInputLine handles a single whole test output line. // It must write the line to c.output but may choose to do so // before or after emitting other events. -func (c *converter) handleInputLine(line []byte) { +func (c *Converter) handleInputLine(line []byte) { // Final PASS or FAIL. if bytes.Equal(line, bigPass) || bytes.Equal(line, bigFail) || bytes.HasPrefix(line, bigFailErrorPrefix) { c.flushReport(0) @@ -286,7 +295,7 @@ func (c *converter) handleInputLine(line []byte) { } // flushReport flushes all pending PASS/FAIL reports at levels >= depth. -func (c *converter) flushReport(depth int) { +func (c *Converter) flushReport(depth int) { c.testName = "" for len(c.report) > depth { e := c.report[len(c.report)-1] @@ -298,23 +307,22 @@ func (c *converter) flushReport(depth int) { // Close marks the end of the go test output. // It flushes any pending input and then output (only partial lines at this point) // and then emits the final overall package-level pass/fail event. -func (c *converter) Close() error { +func (c *Converter) Close() error { c.input.flush() c.output.flush() - e := &event{Action: "pass"} if c.result != "" { - e.Action = c.result - } - if c.mode&Timestamp != 0 { - dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds() - e.Elapsed = &dt + e := &event{Action: c.result} + if c.mode&Timestamp != 0 { + dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds() + e.Elapsed = &dt + } + c.writeEvent(e) } - c.writeEvent(e) return nil } // writeOutputEvent writes a single output event with the given bytes. -func (c *converter) writeOutputEvent(out []byte) { +func (c *Converter) writeOutputEvent(out []byte) { c.writeEvent(&event{ Action: "output", Output: (*textBytes)(&out), @@ -323,7 +331,7 @@ func (c *converter) writeOutputEvent(out []byte) { // writeEvent writes a single event. // It adds the package, time (if requested), and test name (if needed). -func (c *converter) writeEvent(e *event) { +func (c *Converter) writeEvent(e *event) { e.Package = c.pkg if c.mode&Timestamp != 0 { t := time.Now() diff --git a/src/cmd/internal/test2json/testdata/benchshort.json b/src/cmd/internal/test2json/testdata/benchshort.json index 28e287c848..34b03b9362 100644 --- a/src/cmd/internal/test2json/testdata/benchshort.json +++ b/src/cmd/internal/test2json/testdata/benchshort.json @@ -4,4 +4,3 @@ {"Action":"output","Output":"# but to avoid questions of timing, we just use a file with no \\n at all.\n"} {"Action":"output","Output":"BenchmarkFoo \t"} {"Action":"output","Output":"10000 early EOF"} -{"Action":"pass"} diff --git a/src/cmd/internal/test2json/testdata/empty.json b/src/cmd/internal/test2json/testdata/empty.json index 80b5217501..e69de29bb2 100644 --- a/src/cmd/internal/test2json/testdata/empty.json +++ b/src/cmd/internal/test2json/testdata/empty.json @@ -1 +0,0 @@ -{"Action":"pass"} diff --git a/src/cmd/test2json/main.go b/src/cmd/test2json/main.go index 0385d8f246..57a874193e 100644 --- a/src/cmd/test2json/main.go +++ b/src/cmd/test2json/main.go @@ -118,12 +118,16 @@ func main() { w := &countWriter{0, c} cmd.Stdout = w cmd.Stderr = w - if err := cmd.Run(); err != nil { + err := cmd.Run() + if err != nil { if w.n > 0 { // Assume command printed why it failed. } else { fmt.Fprintf(c, "test2json: %v\n", err) } + } + c.Exited(err) + if err != nil { c.Close() os.Exit(1) } -- cgit v1.3 From 797124f5ff4bb80957007adbf3115287a4e90870 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 14 Aug 2020 15:47:49 -0400 Subject: cmd/go/internal/test: keep looking for go command flags after ambiguous test flag Fixes #40763 Change-Id: I275970d1f8561414571a5b93e368d68fa052c60f Reviewed-on: https://go-review.googlesource.com/c/go/+/248618 Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/test/testflag.go | 29 +++++++++++++++++++++---- src/cmd/go/testdata/script/test_flags.txt | 35 +++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 10 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 1ff34f7445..4f0a8924f1 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -214,9 +214,13 @@ func testFlags(args []string) (packageNames, passToTest []string) { explicitArgs := make([]string, 0, len(args)) inPkgList := false + afterFlagWithoutValue := false for len(args) > 0 { f, remainingArgs, err := cmdflag.ParseOne(&CmdTest.Flag, args) + wasAfterFlagWithoutValue := afterFlagWithoutValue + afterFlagWithoutValue = false // provisionally + if errors.Is(err, flag.ErrHelp) { exitWithUsage() } @@ -233,10 +237,24 @@ func testFlags(args []string) (packageNames, passToTest []string) { if nf := (cmdflag.NonFlagError{}); errors.As(err, &nf) { if !inPkgList && packageNames != nil { // We already saw the package list previously, and this argument is not - // a flag, so it — and everything after it — must be a literal argument - // to the test binary. - explicitArgs = append(explicitArgs, args...) - break + // a flag, so it — and everything after it — must be either a value for + // a preceding flag or a literal argument to the test binary. + if wasAfterFlagWithoutValue { + // This argument could syntactically be a flag value, so + // optimistically assume that it is and keep looking for go command + // flags after it. + // + // (If we're wrong, we'll at least be consistent with historical + // behavior; see https://golang.org/issue/40763.) + explicitArgs = append(explicitArgs, nf.RawArg) + args = remainingArgs + continue + } else { + // This argument syntactically cannot be a flag value, so it must be a + // positional argument, and so must everything after it. + explicitArgs = append(explicitArgs, args...) + break + } } inPkgList = true @@ -272,6 +290,9 @@ func testFlags(args []string) (packageNames, passToTest []string) { explicitArgs = append(explicitArgs, nd.RawArg) args = remainingArgs + if !nd.HasValue { + afterFlagWithoutValue = true + } continue } diff --git a/src/cmd/go/testdata/script/test_flags.txt b/src/cmd/go/testdata/script/test_flags.txt index d38e37f238..63385e6997 100644 --- a/src/cmd/go/testdata/script/test_flags.txt +++ b/src/cmd/go/testdata/script/test_flags.txt @@ -10,7 +10,7 @@ stdout '\Aok\s+example.com/x\s+[0-9.s]+\n\z' ! stderr . # For backward-compatibility with previous releases of the 'go' command, -# arguments that appear after unrecognized flags should not be treated +# arguments that appear after unrecognized flags should not be treated # as packages, even if they are unambiguously not arguments to flags. # Even though ./x looks like a package path, the real package should be # the implicit '.'. @@ -18,6 +18,22 @@ stdout '\Aok\s+example.com/x\s+[0-9.s]+\n\z' stderr '^no Go files in .+$' ! stderr '/x' +# However, *flags* that appear after unrecognized flags should still be +# interpreted as flags, under the (possibly-erroneous) assumption that +# unrecognized flags are non-boolean. + +go test -v -x ./x -timeout 24h -boolflag=true foo -timeout 25h +stdout 'args: foo -timeout 25h' +stdout 'timeout: 24h0m0s$' # -timeout is unambiguously not a flag, so the real flag wins. + +go test -v -x ./x -timeout 24h -boolflag foo -timeout 25h +stdout 'args: foo -test\.timeout=25h0m0s' # For legacy reasons, '-timeout ' is erroneously rewritten to -test.timeout; see https://golang.org/issue/40763. +stdout 'timeout: 24h0m0s$' # Actual flag wins. + +go test -v -x ./x -timeout 24h -stringflag foo -timeout 25h +stdout 'args: $' +stdout 'timeout: 25h0m0s$' # Later flag wins. + # An explicit '-outputdir=' argument should set test.outputdir # to the 'go' command's working directory, not zero it out # for the test binary. @@ -30,23 +46,23 @@ exists ./cover.out # with the 'test.' prefix in the GOFLAGS entry... env GOFLAGS='-test.timeout=24h0m0s -count=1' go test -v -x ./x -stdout '.*: 24h0m0s$' +stdout 'timeout: 24h0m0s$' stderr '-test.count=1' # ...or without. env GOFLAGS='-timeout=24h0m0s -count=1' go test -v -x ./x -stdout '.*: 24h0m0s$' +stdout 'timeout: 24h0m0s$' stderr '-test.count=1' # Arguments from the command line should override GOFLAGS... go test -v -x -timeout=25h0m0s ./x -stdout '.*: 25h0m0s$' +stdout 'timeout: 25h0m0s$' stderr '-test.count=1' # ...even if they use a different flag name. go test -v -x -test.timeout=26h0m0s ./x -stdout '.*: 26h0m0s$' +stdout 'timeout: 26h0m0s$' stderr '-test\.timeout=26h0m0s' ! stderr 'timeout=24h0m0s' stderr '-test.count=1' @@ -99,11 +115,18 @@ package x import ( "flag" + "strings" "testing" ) var _ = flag.String("usage_message", "", "dummy flag to check usage message") +var boolflag = flag.Bool("boolflag", false, "ignored boolean flag") +var stringflag = flag.String("stringflag", "", "ignored string flag") func TestLogTimeout(t *testing.T) { - t.Log(flag.Lookup("test.timeout").Value) + t.Logf("timeout: %v", flag.Lookup("test.timeout").Value) +} + +func TestLogArgs(t *testing.T) { + t.Logf("args: %s", strings.Join(flag.Args(), " ")) } -- cgit v1.3 From c0cf190d226cc3defb71d17c01d0b45bf49a8a85 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 30 Jun 2020 17:51:52 -0400 Subject: cmd/go: do context propagation for tracing downloads This change does context propagation (and only context propagation) necessary to add context to modfetch.Download and pkg.LoadImport. This was done by adding context to their callers, and then adding context to all call-sites, and then repeating adding context to callers of those enclosing functions and their callers until none were left. In some cases the call graph expansion was pruned by using context.TODOs. The next CL will add a span to Download. I kept it out of this change to avoid making it any larger (and harder to review) than it needs to be. Updates #38714 Change-Id: I5bf2d599aafef67334c384dfccd5e255198c85b4 Reviewed-on: https://go-review.googlesource.com/c/go/+/248327 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/get/get.go | 4 +-- src/cmd/go/internal/list/list.go | 5 ++-- src/cmd/go/internal/load/pkg.go | 40 ++++++++++++++--------------- src/cmd/go/internal/load/test.go | 6 ++--- src/cmd/go/internal/modcmd/download.go | 4 +-- src/cmd/go/internal/modcmd/init.go | 2 +- src/cmd/go/internal/modcmd/tidy.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modcmd/why.go | 6 ++--- src/cmd/go/internal/modconv/convert_test.go | 5 +++- src/cmd/go/internal/modfetch/fetch.go | 3 ++- src/cmd/go/internal/modget/get.go | 18 ++++++------- src/cmd/go/internal/modload/build.go | 17 ++++++------ src/cmd/go/internal/modload/import.go | 9 ++++--- src/cmd/go/internal/modload/import_test.go | 5 +++- src/cmd/go/internal/modload/init.go | 9 ++++--- src/cmd/go/internal/modload/list.go | 14 +++++----- src/cmd/go/internal/modload/load.go | 39 ++++++++++++++-------------- src/cmd/go/internal/modload/mvs.go | 5 ++-- src/cmd/go/internal/modload/query.go | 33 ++++++++++++------------ src/cmd/go/internal/modload/query_test.go | 5 +++- src/cmd/go/internal/modload/search.go | 5 ++-- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- 25 files changed, 131 insertions(+), 113 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index ef43602aca..e5bacadaa3 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7303e6c866..e68c39f392 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -20,6 +20,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -349,7 +350,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": modload.ModuleInfo, + "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -389,7 +390,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod() // Parses go.mod and sets cfg.BuildMod. + modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 32c2ba7912..71fd9b5538 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -42,10 +42,10 @@ var ( ModBinDir func() string // return effective bin directory ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(args []string) []*search.Match // expand import paths + ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files + ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files ModDirImportPath func(string) string // return effective import path for directory ) @@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -605,11 +605,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -657,7 +657,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(path, stk, importPos, bp, err) + p.load(ctx, path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1800,7 +1800,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(roots []*Package) []*Package { +func TestPackageList(ctx context.Context, roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2089,7 +2089,7 @@ func TestPackageList(roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2112,7 +2112,7 @@ func TestPackageList(roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(path, srcDir, parent, stk, importPos, mode) + p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } @@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(patterns)} + return []*Package{GoFilesPackage(ctx, patterns)} } } } - matches := ImportPaths(patterns) + matches := ImportPaths(ctx, patterns) var ( pkgs []*Package stk ImportStack @@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(args []string) []*search.Match { +func ImportPaths(ctx context.Context, args []string) []*search.Match { if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(args) + return ModImportPaths(ctx, args) } return search.ImportPaths(args) } @@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(gofiles []string) *Package { +func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ModInit() for _, f := range gofiles { @@ -2329,7 +2329,7 @@ func GoFilesPackage(gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(gofiles) + ModImportFromFiles(ctx, gofiles) } var err error @@ -2345,7 +2345,7 @@ func GoFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load("command-line-arguments", &stk, nil, bp, err) + pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 6db8a00245..a0e275095b 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 857362a72e..75785c2c48 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod() // to fill Target + modload.InitMod(ctx) // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" @@ -126,7 +126,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(mod) + m.Dir, err = modfetch.Download(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 95063e62f4..b6cffd332d 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if strings.Contains(modload.CmdModModule, "@") { base.Fatalf("go mod init: module path must not contain '@'") } - modload.InitMod() // does all the hard work + modload.InitMod(ctx) // does all the hard work modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index 769cd11fe8..c7c53d7c0c 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL() + modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 257d1cd0ef..e5353b5c7f 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor() + pkgs := modload.LoadVendor(ctx) vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index f400339b25..da33fff89e 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) - for _, path := range loadALL() { + for _, path := range loadALL(ctx) { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(args) // resolve to packages - loadALL() // rebuild graph, from main module (not from named packages) + matches := modload.ImportPaths(ctx, args) // resolve to packages + loadALL(ctx) // rebuild graph, from main module (not from named packages) sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index a04a13b14f..faa2b4c606 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -6,6 +6,7 @@ package modconv import ( "bytes" + "context" "fmt" "internal/testenv" "io/ioutil" @@ -146,6 +147,8 @@ func TestConvertLegacyConfig(t *testing.T) { }, } + ctx := context.Background() + for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -157,7 +160,7 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e40158b535..6606612658 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -7,6 +7,7 @@ package modfetch import ( "archive/zip" "bytes" + "context" "errors" "fmt" "io" @@ -34,7 +35,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(mod module.Version) (dir string, err error) { +func Download(ctx context.Context, mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index d02c9a8da5..ee9757912b 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -353,7 +353,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if !strings.Contains(path, "...") { m := search.NewMatch(path) if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(pkgPath) + m = modload.TargetPackages(ctx, pkgPath) } if len(m.Pkgs) == 0 { for _, err := range m.Errs { @@ -399,7 +399,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { default: // The argument is a package or module path. if modload.HasModRoot() { - if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { + if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { // The path is in the main module. Nothing to query. if vers != "upgrade" && vers != "patch" { base.Errorf("go get %s: can't request explicit version of path in main module", arg) @@ -491,7 +491,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if q.path == q.m.Path { wg.Add(1) go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { + if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil { base.Errorf("go get: %v", err) } else if !hasPkg { modOnlyMu.Lock() @@ -536,7 +536,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Don't load packages if pkgPatterns is empty. Both // modload.ImportPathsQuiet and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) + matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -732,7 +732,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer q.m = module.Version{Path: q.path, Version: "none"} return } - m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) + m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath) if err != nil { base.Errorf("go get %s: %v", q.arg, err) } @@ -790,7 +790,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer // to determine the underlying module version being requested. // If forceModulePath is set, getQuery must interpret path // as a module path. -func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { +func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { if (prevM.Version != "") != forceModulePath { // We resolve package patterns by calling QueryPattern, which does not // accept a previous version and therefore cannot take it into account for @@ -812,7 +812,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo } } - info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +838,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(path, vers, modload.Allowed) + results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -994,7 +994,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 5f8a2e7e05..a101681a1f 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/hex" "fmt" "internal/goroot" @@ -57,21 +58,21 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(m, true) + return moduleInfo(context.TODO(), m, true) } -func ModuleInfo(path string) *modinfo.ModulePublic { +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) + return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(m, true) + return moduleInfo(ctx, m, true) } } @@ -84,12 +85,12 @@ func ModuleInfo(path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(m *modinfo.ModulePublic) { +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -103,7 +104,7 @@ func addVersions(m *modinfo.ModulePublic) { m.Versions, _ = versions(m.Path) } -func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -132,7 +133,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(m.Path, m.Version, "", nil); err != nil { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 4d2bc805e2..5c51a79124 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "go/build" @@ -110,7 +111,7 @@ var _ load.ImportPathError = &AmbiguousImportError{} // Import returns an ImportMissingError as the error. // If Import can identify a module that could be added to supply the package, // the ImportMissingError records that module. -func Import(path string) (m module.Version, dir string, err error) { +func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -165,7 +166,7 @@ func Import(path string) (m module.Version, dir string, err error) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error. // Note that we don't know for sure this module is necessary, @@ -248,7 +249,7 @@ func Import(path string) (m module.Version, dir string, err error) { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error as above. return module.Version{}, "", err @@ -285,7 +286,7 @@ func Import(path string) (m module.Version, dir string, err error) { fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(path, "latest", Allowed) + candidates, err := QueryPackage(ctx, path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index accc60eecd..47ce89a084 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "regexp" "strings" @@ -49,10 +50,12 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() + ctx := context.Background() + for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(tt.path) + m, dir, err := Import(ctx, tt.path) if err == nil { t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index fff060e665..93027c44c4 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -332,7 +333,7 @@ func die() { // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod() { +func InitMod(ctx context.Context) { if len(buildList) > 0 { return } @@ -359,7 +360,7 @@ func InitMod() { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -397,7 +398,7 @@ func InitMod() { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(fixed *bool) modfile.VersionFixer { +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -429,7 +430,7 @@ func fixVersion(fixed *bool) modfile.VersionFixer { } } - info, err := Query(path, vers, "", nil) + info, err := Query(ctx, path, vers, "", nil) if err != nil { return "", err } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 8db4d64706..7bf4e86c8d 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -31,7 +31,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ sem <- token{} go func() { if listU { - addUpdate(m) + addUpdate(ctx, m) } if listVersions { addVersions(m) @@ -57,7 +57,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { LoadBuildList(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -83,7 +83,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(path, vers, current, nil) + info, err := Query(ctx, path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +92,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) continue } @@ -117,7 +117,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m, true)) + mods = append(mods, moduleInfo(ctx, m, true)) } } } @@ -127,9 +127,9 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", "", nil) + info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 8190009b23..686d491219 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -52,8 +52,8 @@ var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), // on the target platform. Modules may be added to the build list // to satisfy new imports. -func ImportPaths(patterns []string) []*search.Match { - matches := ImportPathsQuiet(patterns, imports.Tags()) +func ImportPaths(ctx context.Context, patterns []string) []*search.Match { + matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) search.WarnUnmatched(matches) return matches } @@ -62,7 +62,7 @@ func ImportPaths(patterns []string) []*search.Match { // no matches. It also lets the caller specify a set of build tags to match // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { +func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { updateMatches := func(matches []*search.Match, iterating bool) { for _, m := range matches { switch { @@ -103,7 +103,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, loaded.tags, includeStd, buildList) case m.Pattern() == "all": loaded.testAll = true @@ -111,7 +111,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { // Enumerate the packages in the main module. // We'll load the dependencies as we find them. m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -129,7 +129,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { } } - InitMod() + InitMod(ctx) var matches []*search.Match for _, pattern := range search.CleanPatterns(patterns) { @@ -338,8 +338,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(gofiles []string) { - InitMod() +func ImportFromFiles(ctx context.Context, gofiles []string) { + InitMod(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -391,7 +391,7 @@ func DirImportPath(dir string) string { func LoadBuildList(ctx context.Context) []module.Version { ctx, span := trace.StartSpan(ctx, "LoadBuildList") defer span.Done() - InitMod() + InitMod(ctx) ReloadBuildList() WriteGoMod() return buildList @@ -409,20 +409,20 @@ func ReloadBuildList() []module.Version { // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. -func LoadALL() []string { - return loadAll(true) +func LoadALL(ctx context.Context) []string { + return loadAll(ctx, true) } // LoadVendor is like LoadALL but only follows test dependencies // for tests in the main module. Tests in dependency modules are // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor() []string { - return loadAll(false) +func LoadVendor(ctx context.Context) []string { + return loadAll(ctx, false) } -func loadAll(testAll bool) []string { - InitMod() +func loadAll(ctx context.Context, testAll bool) []string { + InitMod(ctx) loaded = newLoader(imports.AnyTags()) loaded.isALL = true @@ -430,7 +430,7 @@ func loadAll(testAll bool) []string { if !testAll { loaded.testRoots = true } - all := TargetPackages("...") + all := TargetPackages(ctx, "...") loaded.load(func() []string { return all.Pkgs }) checkMultiplePaths() WriteGoMod() @@ -453,13 +453,13 @@ func loadAll(testAll bool) []string { // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(pattern string) *search.Match { +func TargetPackages(ctx context.Context, pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. ModRoot() m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } @@ -817,7 +817,8 @@ func (ld *loader) doPkg(item interface{}) { return } - pkg.mod, pkg.dir, pkg.err = Import(pkg.path) + // TODO(matloob): Handle TODO context. This needs to be threaded through Do. + pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) if pkg.dir == "" { return } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 5dd009d31d..67eb2c2e19 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -224,7 +225,7 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { // // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(mod module.Version) (dir string, isLocal bool, err error) { +func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -254,6 +255,6 @@ func fetch(mod module.Version) (dir string, isLocal bool, err error) { mod = r } - dir, err = modfetch.Download(mod) + dir, err = modfetch.Download(ctx, mod) return dir, false, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index acc886bf21..39a813447c 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -55,10 +56,10 @@ import ( // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(proxy, path, query, current, allowed) + info, err = queryProxy(ctx, proxy, path, query, current, allowed) return err }) return info, err @@ -75,7 +76,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -243,7 +244,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) if err != nil { return nil, err } - releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) if err != nil { return nil, err } @@ -327,7 +328,7 @@ func matchSemverPrefix(p, v string) bool { // 1. versions that do not satisfy the 'ok' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { if !ok(module.Version{Path: path, Version: v}) { @@ -343,7 +344,7 @@ func filterVersions(path string, versions []string, ok func(module.Version) bool // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) if err != nil { return nil, nil, err } @@ -380,12 +381,12 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) } - return QueryPattern(path, query, allowed) + return QueryPattern(ctx, path, query, allowed) } // QueryPattern looks up the module(s) containing at least one package matching @@ -401,7 +402,7 @@ func QueryPackage(path, query string, allowed func(module.Version) bool) ([]Quer // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { base := pattern firstError := func(m *search.Match) error { @@ -417,7 +418,7 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q base = pathpkg.Dir(pattern[:i+3]) match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -472,12 +473,12 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q queryModule := func(path string) (r QueryResult, err error) { current := findCurrentVersion(path) r.Mod.Path = path - r.Rev, err = queryProxy(proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(r.Mod) + root, isLocal, err := fetch(ctx, r.Mod) if err != nil { return r, err } @@ -698,8 +699,8 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(m module.Version) (bool, error) { - root, isLocal, err := fetch(m) +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + root, isLocal, err := fetch(ctx, m) if err != nil { return false, err } @@ -707,8 +708,8 @@ func ModuleHasRootPackage(m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(m module.Version) (bool, error) { - root, _, err := fetch(m) +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + root, _, err := fetch(ctx, m) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 247e4c40d2..77080e9b5b 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "io/ioutil" "log" @@ -179,6 +180,8 @@ func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") + ctx := context.Background() + for _, tt := range queryTests { allow := tt.allow if allow == "" { @@ -192,7 +195,7 @@ func TestQuery(t *testing.T) { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(tt.path, tt.query, tt.current, allowed) + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index c28e7c0c1e..a9bee0af4e 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -5,6 +5,7 @@ package modload import ( + "context" "fmt" "os" "path/filepath" @@ -27,7 +28,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -153,7 +154,7 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu isLocal = true } else { var err error - root, isLocal, err = fetch(mod) + root, isLocal, err = fetch(ctx, mod) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index deec5106ff..99578b244c 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -77,7 +77,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(files) + p = load.GoFilesPackage(ctx, files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9cef8cf89c..3aee6939d2 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -702,7 +702,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(pkgs) { + for _, p := range load.TestPackageList(ctx, pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 681ecd7646..d975c36306 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2900,7 +2900,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { } srcs := []string{src} - p := load.GoFilesPackage(srcs) + p := load.GoFilesPackage(context.TODO(), srcs) if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { return "32", nil -- cgit v1.3 From 1a3558341860357c2400e37773e5076bb3a51628 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 15 Jul 2020 17:38:36 -0400 Subject: cmd/go: add tracing for querying and downloading from the proxy This CL adds tracing spans for modload.queryPattern, modload.queryProxy, modload.QueryPattern, modload.QueryPattern.queryModule, modload.queryPrefixModules and modfetch.Download. Updates #38714 Change-Id: I537c7fa4f466c691c1b60ec73ef8a2277af49cd7 Reviewed-on: https://go-review.googlesource.com/c/go/+/242786 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modfetch/fetch.go | 21 +++++++++++++++------ .../internal/modfetch/zip_sum_test/zip_sum_test.go | 3 ++- src/cmd/go/internal/modload/query.go | 22 ++++++++++++++++++---- 4 files changed, 36 insertions(+), 12 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 75785c2c48..13e5cb066b 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -120,7 +120,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(mod) + m.Zip, err = modfetch.DownloadZip(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 6606612658..e29eb0a942 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -24,6 +24,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -48,7 +49,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(mod) + dir, err := download(ctx, mod) if err != nil { return cached{"", err} } @@ -58,7 +59,10 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(mod module.Version) (dir string, err error) { +func download(ctx context.Context, mod module.Version) (dir string, err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) + defer span.Done() + // If the directory exists, and no .partial file exists, the module has // already been completely extracted. .partial files may be created when a // module zip directory is extracted in place instead of being extracted to a @@ -73,7 +77,7 @@ func download(mod module.Version) (dir string, err error) { // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(mod) + zipfile, err := DownloadZip(ctx, mod) if err != nil { return "", err } @@ -143,6 +147,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) if unzipInPlace { if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { return "", err @@ -172,6 +177,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } } + defer span.Done() if !cfg.ModCacheRW { // Make dir read-only only *after* renaming it. @@ -196,7 +202,7 @@ var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(mod module.Version) (zipfile string, err error) { +func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -231,7 +237,7 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(mod, zipfile); err != nil { + if err := downloadZip(ctx, mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -239,7 +245,10 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { return c.zipfile, c.err } -func downloadZip(mod module.Version, zipfile string) (err error) { +func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) + defer span.Done() + // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. diff --git a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index eac9b32fa8..82398ebfed 100644 --- a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,6 +16,7 @@ package zip_sum_test import ( + "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -119,7 +120,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(test.m) + zipPath, err := modfetch.DownloadZip(context.Background(), test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 39a813447c..e82eb1506f 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -77,6 +78,9 @@ func (queryDisabledError) Error() string { } func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -403,6 +407,9 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // the main module and only the version "latest", without checking for other // possible modules. func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + base := pattern firstError := func(m *search.Match) error { @@ -470,7 +477,10 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(path string) (r QueryResult, err error) { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + current := findCurrentVersion(path) r.Mod.Path = path r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) @@ -499,7 +509,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } var err error - results, err = queryPrefixModules(candidateModules, queryModule) + results, err = queryPrefixModules(ctx, candidateModules, queryModule) return err }) @@ -543,7 +553,10 @@ type prefixResult struct { err error } -func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() + // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -556,8 +569,9 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(p) + r.QueryResult, r.err = queryModule(ctx, p) wg.Done() }(p, &results[i]) } -- cgit v1.3 From e30fbe3757d09a22988835835c41233df7c6cd00 Mon Sep 17 00:00:00 2001 From: Junchen Li Date: Fri, 10 Jul 2020 11:39:23 +0800 Subject: cmd/compile: optimize unsigned comparisons to 0 There are some architecture-independent rules in #21439, since an unsigned integer >= 0 is always true and < 0 is always false. This CL adds these optimizations to generic rules. Updates #21439 Change-Id: Iec7e3040b761ecb1e60908f764815fdd9bc62495 Reviewed-on: https://go-review.googlesource.com/c/go/+/246617 Reviewed-by: Keith Randall Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/ssa/gen/generic.rules | 4 ++ src/cmd/compile/internal/ssa/rewritegeneric.go | 80 ++++++++++++++++++++++++++ test/codegen/comparisons.go | 17 ++++++ 3 files changed, 101 insertions(+) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/generic.rules b/src/cmd/compile/internal/ssa/gen/generic.rules index ed5bfc81fd..2d39d27226 100644 --- a/src/cmd/compile/internal/ssa/gen/generic.rules +++ b/src/cmd/compile/internal/ssa/gen/generic.rules @@ -545,6 +545,10 @@ (Or(64|32|16|8) x (Or(64|32|16|8) x y)) => (Or(64|32|16|8) x y) (Xor(64|32|16|8) x (Xor(64|32|16|8) x y)) => y +// Unsigned comparisons to zero. +(Less(64U|32U|16U|8U) _ (Const(64|32|16|8) [0])) => (ConstBool [false]) +(Leq(64U|32U|16U|8U) (Const(64|32|16|8) [0]) _) => (ConstBool [true]) + // Ands clear bits. Ors set bits. // If a subsequent Or will set all the bits // that an And cleared, we can skip the And. diff --git a/src/cmd/compile/internal/ssa/rewritegeneric.go b/src/cmd/compile/internal/ssa/rewritegeneric.go index 9f4e1b95bd..68e49f46f3 100644 --- a/src/cmd/compile/internal/ssa/rewritegeneric.go +++ b/src/cmd/compile/internal/ssa/rewritegeneric.go @@ -9701,6 +9701,16 @@ func rewriteValuegeneric_OpLeq16U(v *Value) bool { v.AuxInt = boolToAuxInt(uint16(c) <= uint16(d)) return true } + // match: (Leq16U (Const16 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst16 || auxIntToInt16(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLeq32(v *Value) bool { @@ -9805,6 +9815,16 @@ func rewriteValuegeneric_OpLeq32U(v *Value) bool { v.AuxInt = boolToAuxInt(uint32(c) <= uint32(d)) return true } + // match: (Leq32U (Const32 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst32 || auxIntToInt32(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLeq64(v *Value) bool { @@ -9909,6 +9929,16 @@ func rewriteValuegeneric_OpLeq64U(v *Value) bool { v.AuxInt = boolToAuxInt(uint64(c) <= uint64(d)) return true } + // match: (Leq64U (Const64 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst64 || auxIntToInt64(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLeq8(v *Value) bool { @@ -9993,6 +10023,16 @@ func rewriteValuegeneric_OpLeq8U(v *Value) bool { v.AuxInt = boolToAuxInt(uint8(c) <= uint8(d)) return true } + // match: (Leq8U (Const8 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst8 || auxIntToInt8(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLess16(v *Value) bool { @@ -10033,6 +10073,16 @@ func rewriteValuegeneric_OpLess16U(v *Value) bool { v.AuxInt = boolToAuxInt(uint16(c) < uint16(d)) return true } + // match: (Less16U _ (Const16 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst16 || auxIntToInt16(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLess32(v *Value) bool { @@ -10093,6 +10143,16 @@ func rewriteValuegeneric_OpLess32U(v *Value) bool { v.AuxInt = boolToAuxInt(uint32(c) < uint32(d)) return true } + // match: (Less32U _ (Const32 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst32 || auxIntToInt32(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLess64(v *Value) bool { @@ -10153,6 +10213,16 @@ func rewriteValuegeneric_OpLess64U(v *Value) bool { v.AuxInt = boolToAuxInt(uint64(c) < uint64(d)) return true } + // match: (Less64U _ (Const64 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst64 || auxIntToInt64(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLess8(v *Value) bool { @@ -10193,6 +10263,16 @@ func rewriteValuegeneric_OpLess8U(v *Value) bool { v.AuxInt = boolToAuxInt(uint8(c) < uint8(d)) return true } + // match: (Less8U _ (Const8 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst8 || auxIntToInt8(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLoad(v *Value) bool { diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index 90808573c2..f3c15538a8 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -407,3 +407,20 @@ func CmpToZero_ex5(e, f int32, u uint32) int { } return 0 } +func UintLtZero(a uint8, b uint16, c uint32, d uint64) int { + // amd64: -`(TESTB|TESTW|TESTL|TESTQ|JCC|JCS)` + // arm64: -`(CMPW|CMP|BHS|BLO)` + if a < 0 || b < 0 || c < 0 || d < 0 { + return 1 + } + return 0 +} + +func UintGeqZero(a uint8, b uint16, c uint32, d uint64) int { + // amd64: -`(TESTB|TESTW|TESTL|TESTQ|JCS|JCC)` + // arm64: -`(CMPW|CMP|BLO|BHS)` + if a >= 0 || b >= 0 || c >= 0 || d >= 0 { + return 1 + } + return 0 +} -- cgit v1.3 From 17553c6e7154deab8501595f4fa387c0b718337a Mon Sep 17 00:00:00 2001 From: surechen Date: Mon, 27 Jul 2020 12:00:19 +0800 Subject: cmd/compile: move dumpFileSeq I noticed that there is a Todo comment here. This variable is only used for filename when dump a function's ssa passes result in details. It is no problem to print a function alone, but may be edited by not only one goroutine if dump multiple functions at the same time. Although it looks only dump one function's ssa passes now. As far as I am concerned this variable can be a member variable of the struct Func. I'm not sure if this change is necessary. Looking forward to your advices, thank you very much. Change-Id: I35dd7247889e0cc7f19c0b400b597206592dee75 Reviewed-on: https://go-review.googlesource.com/c/go/+/244918 Reviewed-by: Keith Randall Run-TryBot: Keith Randall --- src/cmd/compile/internal/ssa/compile.go | 7 ++----- src/cmd/compile/internal/ssa/func.go | 7 ++++--- 2 files changed, 6 insertions(+), 8 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/compile.go b/src/cmd/compile/internal/ssa/compile.go index dbdd027716..444475d67a 100644 --- a/src/cmd/compile/internal/ssa/compile.go +++ b/src/cmd/compile/internal/ssa/compile.go @@ -160,15 +160,12 @@ func Compile(f *Func) { phaseName = "" } -// TODO: should be a config field -var dumpFileSeq int - // dumpFile creates a file from the phase name and function name // Dumping is done to files to avoid buffering huge strings before // output. func (f *Func) dumpFile(phaseName string) { - dumpFileSeq++ - fname := fmt.Sprintf("%s_%02d__%s.dump", f.Name, dumpFileSeq, phaseName) + f.dumpFileSeq++ + fname := fmt.Sprintf("%s_%02d__%s.dump", f.Name, int(f.dumpFileSeq), phaseName) fname = strings.Replace(fname, " ", "_", -1) fname = strings.Replace(fname, "/", "_", -1) fname = strings.Replace(fname, ":", "_", -1) diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 4b9189fb3e..9e40b6214c 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -44,9 +44,10 @@ type Func struct { PrintOrHtmlSSA bool // true if GOSSAFUNC matches, true even if fe.Log() (spew phase results to stdout) is false. ruleMatches map[string]int // number of times countRule was called during compilation for any given string - scheduled bool // Values in Blocks are in final order - laidout bool // Blocks are ordered - NoSplit bool // true if function is marked as nosplit. Used by schedule check pass. + scheduled bool // Values in Blocks are in final order + laidout bool // Blocks are ordered + NoSplit bool // true if function is marked as nosplit. Used by schedule check pass. + dumpFileSeq uint8 // the sequence numbers of dump file. (%s_%02d__%s.dump", funcname, dumpFileSeq, phaseName) // when register allocation is done, maps value ids to locations RegAlloc []Location -- cgit v1.3 From ef9c8a38ad177fa7f48dfaad5d0e27f39a03529d Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 7 Jul 2020 09:32:12 -0700 Subject: cmd/compile: don't rewrite (CMP (AND x y) 0) to TEST if AND has other uses If the AND has other uses, we end up saving an argument to the AND in another register, so we can use it for the TEST. No point in doing that. Change-Id: I73444a6aeddd6f55e2328ce04d77c3e6cf4a83e0 Reviewed-on: https://go-review.googlesource.com/c/go/+/241280 Run-TryBot: Keith Randall Reviewed-by: Josh Bleecher Snyder TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/ssa/gen/AMD64.rules | 16 ++-- src/cmd/compile/internal/ssa/rewriteAMD64.go | 128 ++++++++++++++++++++------- test/codegen/logic.go | 24 +++++ 3 files changed, 128 insertions(+), 40 deletions(-) create mode 100644 test/codegen/logic.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules index 9967c7b030..5111ef79d3 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules @@ -1463,14 +1463,14 @@ (MULQconst [c] (NEGQ x)) && c != -(1<<31) -> (MULQconst [-c] x) // checking AND against 0. -(CMPQconst (ANDQ x y) [0]) -> (TESTQ x y) -(CMPLconst (ANDL x y) [0]) -> (TESTL x y) -(CMPWconst (ANDL x y) [0]) -> (TESTW x y) -(CMPBconst (ANDL x y) [0]) -> (TESTB x y) -(CMPQconst (ANDQconst [c] x) [0]) -> (TESTQconst [c] x) -(CMPLconst (ANDLconst [c] x) [0]) -> (TESTLconst [c] x) -(CMPWconst (ANDLconst [c] x) [0]) -> (TESTWconst [int64(int16(c))] x) -(CMPBconst (ANDLconst [c] x) [0]) -> (TESTBconst [int64(int8(c))] x) +(CMPQconst a:(ANDQ x y) [0]) && a.Uses == 1 -> (TESTQ x y) +(CMPLconst a:(ANDL x y) [0]) && a.Uses == 1 -> (TESTL x y) +(CMPWconst a:(ANDL x y) [0]) && a.Uses == 1 -> (TESTW x y) +(CMPBconst a:(ANDL x y) [0]) && a.Uses == 1 -> (TESTB x y) +(CMPQconst a:(ANDQconst [c] x) [0]) && a.Uses == 1 -> (TESTQconst [c] x) +(CMPLconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 -> (TESTLconst [c] x) +(CMPWconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 -> (TESTWconst [int64(int16(c))] x) +(CMPBconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 -> (TESTBconst [int64(int8(c))] x) // Convert TESTx to TESTxconst if possible. (TESTQ (MOVQconst [c]) x) && is32Bit(c) -> (TESTQconst [c] x) diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index 20eab05e9c..cda9df56f4 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -6924,26 +6924,42 @@ func rewriteValueAMD64_OpAMD64CMPBconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPBconst (ANDL x y) [0]) + // match: (CMPBconst a:(ANDL x y) [0]) + // cond: a.Uses == 1 // result: (TESTB x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDL { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDL { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTB) v.AddArg2(x, y) return true } - // match: (CMPBconst (ANDLconst [c] x) [0]) + // match: (CMPBconst a:(ANDLconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTBconst [int64(int8(c))] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDLconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDLconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTBconst) v.AuxInt = int64(int8(c)) v.AddArg(x) @@ -7309,26 +7325,42 @@ func rewriteValueAMD64_OpAMD64CMPLconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPLconst (ANDL x y) [0]) + // match: (CMPLconst a:(ANDL x y) [0]) + // cond: a.Uses == 1 // result: (TESTL x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDL { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDL { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTL) v.AddArg2(x, y) return true } - // match: (CMPLconst (ANDLconst [c] x) [0]) + // match: (CMPLconst a:(ANDLconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTLconst [c] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDLconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDLconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTLconst) v.AuxInt = c v.AddArg(x) @@ -7874,26 +7906,42 @@ func rewriteValueAMD64_OpAMD64CMPQconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPQconst (ANDQ x y) [0]) + // match: (CMPQconst a:(ANDQ x y) [0]) + // cond: a.Uses == 1 // result: (TESTQ x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDQ { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDQ { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTQ) v.AddArg2(x, y) return true } - // match: (CMPQconst (ANDQconst [c] x) [0]) + // match: (CMPQconst a:(ANDQconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTQconst [c] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDQconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDQconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTQconst) v.AuxInt = c v.AddArg(x) @@ -8244,26 +8292,42 @@ func rewriteValueAMD64_OpAMD64CMPWconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPWconst (ANDL x y) [0]) + // match: (CMPWconst a:(ANDL x y) [0]) + // cond: a.Uses == 1 // result: (TESTW x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDL { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDL { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTW) v.AddArg2(x, y) return true } - // match: (CMPWconst (ANDLconst [c] x) [0]) + // match: (CMPWconst a:(ANDLconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTWconst [int64(int16(c))] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDLconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDLconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTWconst) v.AuxInt = int64(int16(c)) v.AddArg(x) diff --git a/test/codegen/logic.go b/test/codegen/logic.go new file mode 100644 index 0000000000..9afdfd760f --- /dev/null +++ b/test/codegen/logic.go @@ -0,0 +1,24 @@ +// asmcheck + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +var gx, gy int + +// Test to make sure that (CMPQ (ANDQ x y) [0]) does not get rewritten to +// (TESTQ x y) if the ANDQ has other uses. If that rewrite happens, then one +// of the args of the ANDQ needs to be saved so it can be used as the arg to TESTQ. +func andWithUse(x, y int) int { + // Load x,y into registers, so those MOVQ will not appear at the z := x&y line. + gx, gy = x, y + // amd64:-"MOVQ" + z := x & y + if z == 0 { + return 77 + } + // use z by returning it + return z +} -- cgit v1.3 From 06337823ef4d9c57b1b01f9b97a348a47277a9df Mon Sep 17 00:00:00 2001 From: Junchen Li Date: Fri, 10 Jul 2020 11:39:23 +0800 Subject: cmd/compile: optimize unsigned comparisons to 0/1 on arm64 For an unsigned integer, it's useful to convert its order test with 0/1 to its equality test with 0. We can save a comparison instruction that followed by a conditional branch on arm64 since it supports compare-with-zero-and-branch instructions. For example, if x > 0 { ... } else { ... } the original version: CMP $0, R0 BLS 9 the optimized version: CBZ R0, 8 Updates #21439 Change-Id: Id1de6f865f6aa72c5d45b29f7894818857288425 Reviewed-on: https://go-review.googlesource.com/c/go/+/246857 Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/gen/ARM64.rules | 10 ++ src/cmd/compile/internal/ssa/rewriteARM64.go | 204 +++++++++++++++++++++++++++ test/codegen/comparisons.go | 32 +++++ 3 files changed, 246 insertions(+) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/ARM64.rules b/src/cmd/compile/internal/ssa/gen/ARM64.rules index 442d769fdd..27959d01fc 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/gen/ARM64.rules @@ -279,6 +279,16 @@ (Less32F x y) => (LessThanF (FCMPS x y)) (Less64F x y) => (LessThanF (FCMPD x y)) +// For an unsigned integer x, the following rules are useful when combining branch +// 0 < x => x != 0 +// x <= 0 => x == 0 +// x < 1 => x == 0 +// 1 <= x => x != 0 +(Less(8U|16U|32U|64U) zero:(MOVDconst [0]) x) => (Neq(8|16|32|64) zero x) +(Leq(8U|16U|32U|64U) x zero:(MOVDconst [0])) => (Eq(8|16|32|64) x zero) +(Less(8U|16U|32U|64U) x (MOVDconst [1])) => (Eq(8|16|32|64) x (MOVDconst [0])) +(Leq(8U|16U|32U|64U) (MOVDconst [1]) x) => (Neq(8|16|32|64) (MOVDconst [0]) x) + (Less8U x y) => (LessThanU (CMPW (ZeroExt8to32 x) (ZeroExt8to32 y))) (Less16U x y) => (LessThanU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y))) (Less32U x y) => (LessThanU (CMPW x y)) diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go index 8e48b33628..023d9908c2 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM64.go +++ b/src/cmd/compile/internal/ssa/rewriteARM64.go @@ -21976,6 +21976,31 @@ func rewriteValueARM64_OpLeq16U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Leq16U x zero:(MOVDconst [0])) + // result: (Eq16 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq16) + v.AddArg2(x, zero) + return true + } + // match: (Leq16U (MOVDconst [1]) x) + // result: (Neq16 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq16) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq16U x y) // result: (LessEqualU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y))) for { @@ -22028,6 +22053,32 @@ func rewriteValueARM64_OpLeq32U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Leq32U x zero:(MOVDconst [0])) + // result: (Eq32 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq32) + v.AddArg2(x, zero) + return true + } + // match: (Leq32U (MOVDconst [1]) x) + // result: (Neq32 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq32) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq32U x y) // result: (LessEqualU (CMPW x y)) for { @@ -22076,6 +22127,32 @@ func rewriteValueARM64_OpLeq64U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Leq64U x zero:(MOVDconst [0])) + // result: (Eq64 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq64) + v.AddArg2(x, zero) + return true + } + // match: (Leq64U (MOVDconst [1]) x) + // result: (Neq64 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq64) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq64U x y) // result: (LessEqualU (CMP x y)) for { @@ -22114,6 +22191,31 @@ func rewriteValueARM64_OpLeq8U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Leq8U x zero:(MOVDconst [0])) + // result: (Eq8 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq8) + v.AddArg2(x, zero) + return true + } + // match: (Leq8U (MOVDconst [1]) x) + // result: (Neq8 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq8) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq8U x y) // result: (LessEqualU (CMPW (ZeroExt8to32 x) (ZeroExt8to32 y))) for { @@ -22156,6 +22258,31 @@ func rewriteValueARM64_OpLess16U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Less16U zero:(MOVDconst [0]) x) + // result: (Neq16 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq16) + v.AddArg2(zero, x) + return true + } + // match: (Less16U x (MOVDconst [1])) + // result: (Eq16 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq16) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less16U x y) // result: (LessThanU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y))) for { @@ -22208,6 +22335,32 @@ func rewriteValueARM64_OpLess32U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Less32U zero:(MOVDconst [0]) x) + // result: (Neq32 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq32) + v.AddArg2(zero, x) + return true + } + // match: (Less32U x (MOVDconst [1])) + // result: (Eq32 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq32) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less32U x y) // result: (LessThanU (CMPW x y)) for { @@ -22256,6 +22409,32 @@ func rewriteValueARM64_OpLess64U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Less64U zero:(MOVDconst [0]) x) + // result: (Neq64 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq64) + v.AddArg2(zero, x) + return true + } + // match: (Less64U x (MOVDconst [1])) + // result: (Eq64 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq64) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less64U x y) // result: (LessThanU (CMP x y)) for { @@ -22294,6 +22473,31 @@ func rewriteValueARM64_OpLess8U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Less8U zero:(MOVDconst [0]) x) + // result: (Neq8 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq8) + v.AddArg2(zero, x) + return true + } + // match: (Less8U x (MOVDconst [1])) + // result: (Eq8 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq8) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less8U x y) // result: (LessThanU (CMPW (ZeroExt8to32 x) (ZeroExt8to32 y))) for { diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index f3c15538a8..3c2dcb7eba 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -424,3 +424,35 @@ func UintGeqZero(a uint8, b uint16, c uint32, d uint64) int { } return 0 } + +func UintGtZero(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBZW`, `CBNZW`, `CBNZ`, -`(CMPW|CMP|BLS|BHI)` + if a > 0 || b > 0 || c > 0 || d > 0 { + return 1 + } + return 0 +} + +func UintLeqZero(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBNZW`, `CBZW`, `CBZ`, -`(CMPW|CMP|BHI|BLS)` + if a <= 0 || b <= 0 || c <= 0 || d <= 0 { + return 1 + } + return 0 +} + +func UintLtOne(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBNZW`, `CBZW`, `CBZW`, `CBZ`, -`(CMPW|CMP|BHS|BLO)` + if a < 1 || b < 1 || c < 1 || d < 1 { + return 1 + } + return 0 +} + +func UintGeqOne(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBZW`, `CBNZW`, `CBNZ`, -`(CMPW|CMP|BLO|BHS)` + if a >= 1 || b >= 1 || c >= 1 || d >= 1 { + return 1 + } + return 0 +} -- cgit v1.3 From 6dad1b4c9bb63cad432bcddb7aec0d36f03f8341 Mon Sep 17 00:00:00 2001 From: alex-semenyuk Date: Sun, 17 May 2020 16:05:05 +0000 Subject: cmd/cgo: close file Change-Id: Ia70edc8ba22e31e498fe07946db41882804bd39f GitHub-Last-Rev: 280232e879965b34fae3aa02e6215eff5bd7be46 GitHub-Pull-Request: golang/go#39120 Reviewed-on: https://go-review.googlesource.com/c/go/+/234317 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/cgo/out.go | 1 + 1 file changed, 1 insertion(+) (limited to 'src/cmd') diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 4064f0ae41..50d2811f1b 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -243,6 +243,7 @@ func (p *Package) writeDefs() { if err != nil { fatalf("%s", err) } + defer fgcch.Close() _, err = io.Copy(fexp, fgcch) if err != nil { fatalf("%s", err) -- cgit v1.3 From a745171e6b30394b661a040d04e8807b4bd0c7da Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 17 Aug 2020 21:59:07 -0700 Subject: cmd/compile: fix SSA type comparison A typo in the conversion code caused comparisons of SSA types to report CMPeq when they were not in fact equal. Fixes #40837 Change-Id: I0627eee51d524a585908b34a4590bc533c8415fc Reviewed-on: https://go-review.googlesource.com/c/go/+/248781 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Cuong Manh Le Reviewed-by: Emmanuel Odeke --- src/cmd/compile/internal/types/type.go | 3 ++- src/cmd/compile/internal/types/type_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 src/cmd/compile/internal/types/type_test.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index 3b7b31c5d6..91b54b43d4 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -131,6 +131,7 @@ type Type struct { // TPTR: Ptr // TARRAY: *Array // TSLICE: Slice + // TSSA: string Extra interface{} // Width is the width of this Type in bytes. @@ -1026,7 +1027,7 @@ func (t *Type) cmp(x *Type) Cmp { case TSSA: tname := t.Extra.(string) - xname := t.Extra.(string) + xname := x.Extra.(string) // desire fast sorting, not pretty sorting. if len(tname) == len(xname) { if tname == xname { diff --git a/src/cmd/compile/internal/types/type_test.go b/src/cmd/compile/internal/types/type_test.go new file mode 100644 index 0000000000..fe3f380b21 --- /dev/null +++ b/src/cmd/compile/internal/types/type_test.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "cmd/compile/internal/types" + "testing" +) + +func TestSSACompare(t *testing.T) { + a := []*types.Type{ + types.TypeInvalid, + types.TypeMem, + types.TypeFlags, + types.TypeVoid, + types.TypeInt128, + } + for _, x := range a { + for _, y := range a { + c := x.Compare(y) + if x == y && c != types.CMPeq || x != y && c == types.CMPeq { + t.Errorf("%s compare %s == %d\n", x.Extra, y.Extra, c) + } + } + } +} -- cgit v1.3 From 6e876f19857a8fbd259571080f7f91bc03276559 Mon Sep 17 00:00:00 2001 From: Michael Munday Date: Thu, 4 Jun 2020 10:55:01 -0700 Subject: cmd/compile: clean up and optimize s390x multiplication rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of the existing optimizations aren't triggered because they are handled by the generic rules so this CL removes them. Also some constraints were copied without much thought from the amd64 rules and they don't make sense on s390x, so we remove those constraints. Finally, add a 'multiply by the sum of two powers of two' optimization. This makes sense on s390x as shifts are low latency and can also sometimes be optimized further (especially if we add support for RISBG instructions). name old time/op new time/op delta IntMulByConst/3-8 1.70ns ±11% 1.10ns ± 5% -35.26% (p=0.000 n=10+10) IntMulByConst/5-8 1.64ns ± 7% 1.10ns ± 4% -32.94% (p=0.000 n=10+9) IntMulByConst/12-8 1.65ns ± 6% 1.20ns ± 4% -27.16% (p=0.000 n=10+9) IntMulByConst/120-8 1.66ns ± 4% 1.22ns ±13% -26.43% (p=0.000 n=10+10) IntMulByConst/-120-8 1.65ns ± 7% 1.19ns ± 4% -28.06% (p=0.000 n=9+10) IntMulByConst/65537-8 0.86ns ± 9% 1.12ns ±12% +30.41% (p=0.000 n=10+10) IntMulByConst/65538-8 1.65ns ± 5% 1.23ns ± 5% -25.11% (p=0.000 n=10+10) Change-Id: Ib196e6bff1e97febfd266134d0a2b2a62897989f Reviewed-on: https://go-review.googlesource.com/c/go/+/248937 Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/gen/S390X.rules | 51 +++-- src/cmd/compile/internal/ssa/rewriteS390X.go | 268 ++++++++++++++----------- src/cmd/compile/internal/test/mulconst_test.go | 242 ++++++++++++++++++++++ test/codegen/arithmetic.go | 6 + 4 files changed, 441 insertions(+), 126 deletions(-) create mode 100644 src/cmd/compile/internal/test/mulconst_test.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/S390X.rules b/src/cmd/compile/internal/ssa/gen/S390X.rules index d3234c1a00..5e4c436ca1 100644 --- a/src/cmd/compile/internal/ssa/gen/S390X.rules +++ b/src/cmd/compile/internal/ssa/gen/S390X.rules @@ -716,20 +716,40 @@ (ANDWconst [0xFF] x) => (MOVBZreg x) (ANDWconst [0xFFFF] x) => (MOVHZreg x) -// strength reduction -(MULLDconst [-1] x) => (NEG x) -(MULLDconst [0] _) => (MOVDconst [0]) -(MULLDconst [1] x) => x -(MULLDconst [c] x) && isPowerOfTwo(c) -> (SLDconst [log2(c)] x) -(MULLDconst [c] x) && isPowerOfTwo(c+1) && c >= 15 -> (SUB (SLDconst [log2(c+1)] x) x) -(MULLDconst [c] x) && isPowerOfTwo(c-1) && c >= 17 -> (ADD (SLDconst [log2(c-1)] x) x) - -(MULLWconst [-1] x) => (NEGW x) -(MULLWconst [0] _) => (MOVDconst [0]) -(MULLWconst [1] x) => x -(MULLWconst [c] x) && isPowerOfTwo(c) -> (SLWconst [log2(c)] x) -(MULLWconst [c] x) && isPowerOfTwo(c+1) && c >= 15 -> (SUBW (SLWconst [log2(c+1)] x) x) -(MULLWconst [c] x) && isPowerOfTwo(c-1) && c >= 17 -> (ADDW (SLWconst [log2(c-1)] x) x) +// Strength reduce multiplication to the sum (or difference) of two powers of two. +// +// Examples: +// 5x -> 4x + 1x +// 10x -> 8x + 2x +// 120x -> 128x - 8x +// -120x -> 8x - 128x +// +// We know that the rightmost bit of any positive value, once isolated, must either +// be a power of 2 (because it is a single bit) or 0 (if the original value is 0). +// In all of these rules we use a rightmost bit calculation to determine one operand +// for the addition or subtraction. We then just need to calculate if the other +// operand is a valid power of 2 before we can match the rule. +// +// Notes: +// - the generic rules have already matched single powers of two so we ignore them here +// - isPowerOfTwo32 asserts that its argument is greater than 0 +// - c&(c-1) = clear rightmost bit +// - c&^(c-1) = isolate rightmost bit + +// c = 2ˣ + 2ʸ => c - 2ˣ = 2ʸ +(MULL(D|W)const x [c]) && isPowerOfTwo32(c&(c-1)) + => ((ADD|ADDW) (SL(D|W)const x [int8(log32(c&(c-1)))]) + (SL(D|W)const x [int8(log32(c&^(c-1)))])) + +// c = 2ʸ - 2ˣ => c + 2ˣ = 2ʸ +(MULL(D|W)const x [c]) && isPowerOfTwo32(c+(c&^(c-1))) + => ((SUB|SUBW) (SL(D|W)const x [int8(log32(c+(c&^(c-1))))]) + (SL(D|W)const x [int8(log32(c&^(c-1)))])) + +// c = 2ˣ - 2ʸ => -c + 2ˣ = 2ʸ +(MULL(D|W)const x [c]) && isPowerOfTwo32(-c+(-c&^(-c-1))) + => ((SUB|SUBW) (SL(D|W)const x [int8(log32(-c&^(-c-1)))]) + (SL(D|W)const x [int8(log32(-c+(-c&^(-c-1))))])) // Fold ADD into MOVDaddr. Odd offsets from SB shouldn't be folded (LARL can't handle them). (ADDconst [c] (MOVDaddr [d] {s} x:(SB))) && ((c+d)&1 == 0) && is32Bit(c+d) -> (MOVDaddr [c+d] {s} x) @@ -1133,6 +1153,9 @@ (XORconst [0] x) => x (XORWconst [c] x) && int32(c)==0 => x +// Shifts by zero (may be inserted during multiplication strength reduction). +((SLD|SLW|SRD|SRW|SRAD|SRAW)const x [0]) => x + // Convert constant subtracts to constant adds. (SUBconst [c] x) && c != -(1<<31) => (ADDconst [-c] x) (SUBWconst [c] x) -> (ADDWconst [int64(int32(-c))] x) diff --git a/src/cmd/compile/internal/ssa/rewriteS390X.go b/src/cmd/compile/internal/ssa/rewriteS390X.go index dc9b143562..536f8db320 100644 --- a/src/cmd/compile/internal/ssa/rewriteS390X.go +++ b/src/cmd/compile/internal/ssa/rewriteS390X.go @@ -732,8 +732,12 @@ func rewriteValueS390X(v *Value) bool { return rewriteValueS390X_OpS390XRLLG(v) case OpS390XSLD: return rewriteValueS390X_OpS390XSLD(v) + case OpS390XSLDconst: + return rewriteValueS390X_OpS390XSLDconst(v) case OpS390XSLW: return rewriteValueS390X_OpS390XSLW(v) + case OpS390XSLWconst: + return rewriteValueS390X_OpS390XSLWconst(v) case OpS390XSRAD: return rewriteValueS390X_OpS390XSRAD(v) case OpS390XSRADconst: @@ -748,6 +752,8 @@ func rewriteValueS390X(v *Value) bool { return rewriteValueS390X_OpS390XSRDconst(v) case OpS390XSRW: return rewriteValueS390X_OpS390XSRW(v) + case OpS390XSRWconst: + return rewriteValueS390X_OpS390XSRWconst(v) case OpS390XSTM2: return rewriteValueS390X_OpS390XSTM2(v) case OpS390XSTMG2: @@ -13853,81 +13859,64 @@ func rewriteValueS390X_OpS390XMULLD(v *Value) bool { func rewriteValueS390X_OpS390XMULLDconst(v *Value) bool { v_0 := v.Args[0] b := v.Block - // match: (MULLDconst [-1] x) - // result: (NEG x) + // match: (MULLDconst x [c]) + // cond: isPowerOfTwo32(c&(c-1)) + // result: (ADD (SLDconst x [int8(log32(c&(c-1)))]) (SLDconst x [int8(log32(c&^(c-1)))])) for { - if auxIntToInt32(v.AuxInt) != -1 { - break - } - x := v_0 - v.reset(OpS390XNEG) - v.AddArg(x) - return true - } - // match: (MULLDconst [0] _) - // result: (MOVDconst [0]) - for { - if auxIntToInt32(v.AuxInt) != 0 { - break - } - v.reset(OpS390XMOVDconst) - v.AuxInt = int64ToAuxInt(0) - return true - } - // match: (MULLDconst [1] x) - // result: x - for { - if auxIntToInt32(v.AuxInt) != 1 { - break - } - x := v_0 - v.copyOf(x) - return true - } - // match: (MULLDconst [c] x) - // cond: isPowerOfTwo(c) - // result: (SLDconst [log2(c)] x) - for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c)) { + if !(isPowerOfTwo32(c & (c - 1))) { break } - v.reset(OpS390XSLDconst) - v.AuxInt = log2(c) - v.AddArg(x) + v.reset(OpS390XADD) + v0 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c & (c - 1)))) + v0.AddArg(x) + v1 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLDconst [c] x) - // cond: isPowerOfTwo(c+1) && c >= 15 - // result: (SUB (SLDconst [log2(c+1)] x) x) + // match: (MULLDconst x [c]) + // cond: isPowerOfTwo32(c+(c&^(c-1))) + // result: (SUB (SLDconst x [int8(log32(c+(c&^(c-1))))]) (SLDconst x [int8(log32(c&^(c-1)))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c+1) && c >= 15) { + if !(isPowerOfTwo32(c + (c &^ (c - 1)))) { break } v.reset(OpS390XSUB) - v0 := b.NewValue0(v.Pos, OpS390XSLDconst, v.Type) - v0.AuxInt = log2(c + 1) + v0 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c + (c &^ (c - 1))))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLDconst [c] x) - // cond: isPowerOfTwo(c-1) && c >= 17 - // result: (ADD (SLDconst [log2(c-1)] x) x) + // match: (MULLDconst x [c]) + // cond: isPowerOfTwo32(-c+(-c&^(-c-1))) + // result: (SUB (SLDconst x [int8(log32(-c&^(-c-1)))]) (SLDconst x [int8(log32(-c+(-c&^(-c-1))))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c-1) && c >= 17) { + if !(isPowerOfTwo32(-c + (-c &^ (-c - 1)))) { break } - v.reset(OpS390XADD) - v0 := b.NewValue0(v.Pos, OpS390XSLDconst, v.Type) - v0.AuxInt = log2(c - 1) + v.reset(OpS390XSUB) + v0 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(-c &^ (-c - 1)))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(-c + (-c &^ (-c - 1))))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } // match: (MULLDconst [c] (MOVDconst [d])) @@ -14097,81 +14086,64 @@ func rewriteValueS390X_OpS390XMULLW(v *Value) bool { func rewriteValueS390X_OpS390XMULLWconst(v *Value) bool { v_0 := v.Args[0] b := v.Block - // match: (MULLWconst [-1] x) - // result: (NEGW x) - for { - if auxIntToInt32(v.AuxInt) != -1 { - break - } - x := v_0 - v.reset(OpS390XNEGW) - v.AddArg(x) - return true - } - // match: (MULLWconst [0] _) - // result: (MOVDconst [0]) - for { - if auxIntToInt32(v.AuxInt) != 0 { - break - } - v.reset(OpS390XMOVDconst) - v.AuxInt = int64ToAuxInt(0) - return true - } - // match: (MULLWconst [1] x) - // result: x + // match: (MULLWconst x [c]) + // cond: isPowerOfTwo32(c&(c-1)) + // result: (ADDW (SLWconst x [int8(log32(c&(c-1)))]) (SLWconst x [int8(log32(c&^(c-1)))])) for { - if auxIntToInt32(v.AuxInt) != 1 { - break - } - x := v_0 - v.copyOf(x) - return true - } - // match: (MULLWconst [c] x) - // cond: isPowerOfTwo(c) - // result: (SLWconst [log2(c)] x) - for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c)) { + if !(isPowerOfTwo32(c & (c - 1))) { break } - v.reset(OpS390XSLWconst) - v.AuxInt = log2(c) - v.AddArg(x) + v.reset(OpS390XADDW) + v0 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c & (c - 1)))) + v0.AddArg(x) + v1 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLWconst [c] x) - // cond: isPowerOfTwo(c+1) && c >= 15 - // result: (SUBW (SLWconst [log2(c+1)] x) x) + // match: (MULLWconst x [c]) + // cond: isPowerOfTwo32(c+(c&^(c-1))) + // result: (SUBW (SLWconst x [int8(log32(c+(c&^(c-1))))]) (SLWconst x [int8(log32(c&^(c-1)))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c+1) && c >= 15) { + if !(isPowerOfTwo32(c + (c &^ (c - 1)))) { break } v.reset(OpS390XSUBW) - v0 := b.NewValue0(v.Pos, OpS390XSLWconst, v.Type) - v0.AuxInt = log2(c + 1) + v0 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c + (c &^ (c - 1))))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLWconst [c] x) - // cond: isPowerOfTwo(c-1) && c >= 17 - // result: (ADDW (SLWconst [log2(c-1)] x) x) + // match: (MULLWconst x [c]) + // cond: isPowerOfTwo32(-c+(-c&^(-c-1))) + // result: (SUBW (SLWconst x [int8(log32(-c&^(-c-1)))]) (SLWconst x [int8(log32(-c+(-c&^(-c-1))))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c-1) && c >= 17) { + if !(isPowerOfTwo32(-c + (-c &^ (-c - 1)))) { break } - v.reset(OpS390XADDW) - v0 := b.NewValue0(v.Pos, OpS390XSLWconst, v.Type) - v0.AuxInt = log2(c - 1) + v.reset(OpS390XSUBW) + v0 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(-c &^ (-c - 1)))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(-c + (-c &^ (-c - 1))))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } // match: (MULLWconst [c] (MOVDconst [d])) @@ -16826,6 +16798,20 @@ func rewriteValueS390X_OpS390XSLD(v *Value) bool { } return false } +func rewriteValueS390X_OpS390XSLDconst(v *Value) bool { + v_0 := v.Args[0] + // match: (SLDconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } + return false +} func rewriteValueS390X_OpS390XSLW(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -16960,6 +16946,20 @@ func rewriteValueS390X_OpS390XSLW(v *Value) bool { } return false } +func rewriteValueS390X_OpS390XSLWconst(v *Value) bool { + v_0 := v.Args[0] + // match: (SLWconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } + return false +} func rewriteValueS390X_OpS390XSRAD(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -17096,6 +17096,16 @@ func rewriteValueS390X_OpS390XSRAD(v *Value) bool { } func rewriteValueS390X_OpS390XSRADconst(v *Value) bool { v_0 := v.Args[0] + // match: (SRADconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } // match: (SRADconst [c] (MOVDconst [d])) // result: (MOVDconst [d>>uint64(c)]) for { @@ -17246,6 +17256,16 @@ func rewriteValueS390X_OpS390XSRAW(v *Value) bool { } func rewriteValueS390X_OpS390XSRAWconst(v *Value) bool { v_0 := v.Args[0] + // match: (SRAWconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } // match: (SRAWconst [c] (MOVDconst [d])) // result: (MOVDconst [int64(int32(d))>>uint64(c)]) for { @@ -17416,6 +17436,16 @@ func rewriteValueS390X_OpS390XSRDconst(v *Value) bool { v.AddArg(v0) return true } + // match: (SRDconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } return false } func rewriteValueS390X_OpS390XSRW(v *Value) bool { @@ -17552,6 +17582,20 @@ func rewriteValueS390X_OpS390XSRW(v *Value) bool { } return false } +func rewriteValueS390X_OpS390XSRWconst(v *Value) bool { + v_0 := v.Args[0] + // match: (SRWconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } + return false +} func rewriteValueS390X_OpS390XSTM2(v *Value) bool { v_3 := v.Args[3] v_2 := v.Args[2] diff --git a/src/cmd/compile/internal/test/mulconst_test.go b/src/cmd/compile/internal/test/mulconst_test.go new file mode 100644 index 0000000000..314cab32de --- /dev/null +++ b/src/cmd/compile/internal/test/mulconst_test.go @@ -0,0 +1,242 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package test + +import "testing" + +// Benchmark multiplication of an integer by various constants. +// +// The comment above each sub-benchmark provides an example of how the +// target multiplication operation might be implemented using shift +// (multiplication by a power of 2), addition and subtraction +// operations. It is platform-dependent whether these transformations +// are actually applied. + +var ( + mulSinkI32 int32 + mulSinkI64 int64 + mulSinkU32 uint32 + mulSinkU64 uint64 +) + +func BenchmarkMulconstI32(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkI32 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkI32 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkI32 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkI32 = x + }) + // -120x = 8x - 120x + b.Run("-120", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= -120 + } + mulSinkI32 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkI32 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkI32 = x + }) +} + +func BenchmarkMulconstI64(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkI64 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkI64 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkI64 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkI64 = x + }) + // -120x = 8x - 120x + b.Run("-120", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= -120 + } + mulSinkI64 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkI64 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkI64 = x + }) +} + +func BenchmarkMulconstU32(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkU32 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkU32 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkU32 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkU32 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkU32 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkU32 = x + }) +} + +func BenchmarkMulconstU64(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkU64 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkU64 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkU64 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkU64 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkU64 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkU64 = x + }) +} diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index 8f25974376..9f30ec8ce4 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -71,9 +71,15 @@ func Mul_96(n int) int { // 386:`SHLL\t[$]5`,`LEAL\t\(.*\)\(.*\*2\),`,-`IMULL` // arm64:`LSL\t[$]5`,`ADD\sR[0-9]+<<1,\sR[0-9]+`,-`MUL` // arm:`SLL\t[$]5`,`ADD\sR[0-9]+<<1,\sR[0-9]+`,-`MUL` + // s390x:`SLD\t[$]5`,`SLD\t[$]6`,-`MULLD` return n * 96 } +func Mul_n120(n int) int { + // s390x:`SLD\t[$]3`,`SLD\t[$]7`,-`MULLD` + return n * -120 +} + func MulMemSrc(a []uint32, b []float32) { // 386:`IMULL\s4\([A-Z]+\),\s[A-Z]+` a[0] *= a[1] -- cgit v1.3 From c12d9ed87596688aeeeb61111e408f6a176aa287 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 18 Aug 2020 10:54:11 -0400 Subject: cmd/go: revert 3 CLs affecting par.Work, context propagation, tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts the following changes: • cmd/go: add tracing for querying and downloading from the proxy CL 242786, commit 1a3558341860357c2400e37773e5076bb3a51628 • cmd/go: do context propagation for tracing downloads CL 248327, commit c0cf190d226cc3defb71d17c01d0b45bf49a8a85 • cmd/go/internal: remove some users of par.Work CL 248326, commit f30044a03bc7cf107dbec03c02fb6d0072878252 Reason for revert: broke linux 386 and amd64 longtest builders. The problem started with CL 248326, but CL 248327 and CL 242786 are reverted as well due to conflicts. Updates #38714. Fixes #40861. Change-Id: I68496b4e5a27e47a42183553c3a645b288edac83 Reviewed-on: https://go-review.googlesource.com/c/go/+/249017 Run-TryBot: Dmitri Shuralyov Reviewed-by: Bryan C. Mills Reviewed-by: Carlos Amedee TryBot-Result: Gobot Gobot --- src/cmd/go/internal/get/get.go | 4 +- src/cmd/go/internal/list/list.go | 5 +- src/cmd/go/internal/load/pkg.go | 40 ++++++------ src/cmd/go/internal/load/test.go | 6 +- src/cmd/go/internal/modcmd/download.go | 75 ++++++++++------------ src/cmd/go/internal/modcmd/graph.go | 19 +++--- src/cmd/go/internal/modcmd/init.go | 2 +- src/cmd/go/internal/modcmd/tidy.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modcmd/why.go | 6 +- src/cmd/go/internal/modconv/convert.go | 59 ++++++++--------- src/cmd/go/internal/modconv/convert_test.go | 5 +- src/cmd/go/internal/modfetch/fetch.go | 24 ++----- .../internal/modfetch/zip_sum_test/zip_sum_test.go | 3 +- src/cmd/go/internal/modget/get.go | 59 +++++++---------- src/cmd/go/internal/modload/build.go | 17 +++-- src/cmd/go/internal/modload/import.go | 9 ++- src/cmd/go/internal/modload/import_test.go | 5 +- src/cmd/go/internal/modload/init.go | 9 ++- src/cmd/go/internal/modload/list.go | 49 ++++++-------- src/cmd/go/internal/modload/load.go | 39 ++++++----- src/cmd/go/internal/modload/mvs.go | 5 +- src/cmd/go/internal/modload/query.go | 55 ++++++---------- src/cmd/go/internal/modload/query_test.go | 5 +- src/cmd/go/internal/modload/search.go | 5 +- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- 28 files changed, 216 insertions(+), 299 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index e5bacadaa3..ef43602aca 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index e68c39f392..7303e6c866 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -20,7 +20,6 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" - "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -350,7 +349,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, + "module": modload.ModuleInfo, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -390,7 +389,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. + modload.InitMod() // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 71fd9b5538..32c2ba7912 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -42,10 +42,10 @@ var ( ModBinDir func() string // return effective bin directory ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths + ModImportPaths func(args []string) []*search.Match // expand import paths ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files + ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files ModDirImportPath func(string) string // return effective import path for directory ) @@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -605,11 +605,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -657,7 +657,7 @@ func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent * // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(ctx, path, stk, importPos, bp, err) + p.load(path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1800,7 +1800,7 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor if path == "C" { continue } - p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(ctx context.Context, roots []*Package) []*Package { +func TestPackageList(roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2089,7 +2089,7 @@ func TestPackageList(ctx context.Context, roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2112,7 +2112,7 @@ func TestPackageList(ctx context.Context, roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) + p := LoadImport(path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } @@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(ctx, patterns)} + return []*Package{GoFilesPackage(patterns)} } } } - matches := ImportPaths(ctx, patterns) + matches := ImportPaths(patterns) var ( pkgs []*Package stk ImportStack @@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(ctx context.Context, args []string) []*search.Match { +func ImportPaths(args []string) []*search.Match { if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(ctx, args) + return ModImportPaths(args) } return search.ImportPaths(args) } @@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(ctx context.Context, gofiles []string) *Package { +func GoFilesPackage(gofiles []string) *Package { ModInit() for _, f := range gofiles { @@ -2329,7 +2329,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(ctx, gofiles) + ModImportFromFiles(gofiles) } var err error @@ -2345,7 +2345,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) + pkg.load("command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index a0e275095b..6db8a00245 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 13e5cb066b..946e8ed3cf 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -5,15 +5,15 @@ package modcmd import ( - "cmd/go/internal/modfetch" "context" "encoding/json" "os" - "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/modfetch" "cmd/go/internal/modload" + "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod(ctx) // to fill Target + modload.InitMod() // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" @@ -102,7 +102,33 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } } - downloadModule := func(m *moduleJSON) { + var mods []*moduleJSON + var work par.Work + listU := false + listVersions := false + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + work.Add(m) + } + + work.Do(10, func(item interface{}) { + m := item.(*moduleJSON) var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -120,53 +146,18 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(ctx, mod) + m.Zip, err = modfetch.DownloadZip(mod) if err != nil { m.Error = err.Error() return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(ctx, mod) + m.Dir, err = modfetch.Download(mod) if err != nil { m.Error = err.Error() return } - } - - var mods []*moduleJSON - listU := false - listVersions := false - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - sem <- token{} - go func() { - downloadModule(m) - <-sem - }() - } - - // Fill semaphore channel to wait for goroutines to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} - } + }) if *downloadJSON { for _, m := range mods { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 6da12b9cab..4853503fd4 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,6 +15,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" + "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -58,25 +59,23 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { return m.Path + "@" + m.Version } + // Note: using par.Work only to manage work queue. + // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - seen := map[module.Version]bool{modload.Target: true} - queue := []module.Version{modload.Target} - for len(queue) > 0 { - var m module.Version - m, queue = queue[0], queue[1:] + var work par.Work + work.Add(modload.Target) + work.Do(1, func(item interface{}) { + m := item.(module.Version) list, _ := reqs.Required(m) for _, r := range list { - if !seen[r] { - queue = append(queue, r) - seen[r] = true - } + work.Add(r) out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - } + }) sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index b6cffd332d..95063e62f4 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if strings.Contains(modload.CmdModModule, "@") { base.Fatalf("go mod init: module path must not contain '@'") } - modload.InitMod(ctx) // does all the hard work + modload.InitMod() // does all the hard work modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index c7c53d7c0c..769cd11fe8 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL(ctx) + modload.LoadALL() modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index e5353b5c7f..257d1cd0ef 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor(ctx) + pkgs := modload.LoadVendor() vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index da33fff89e..f400339b25 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) - for _, path := range loadALL(ctx) { + for _, path := range loadALL() { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(ctx, args) // resolve to packages - loadALL(ctx) // rebuild graph, from main module (not from named packages) + matches := modload.ImportPaths(args) // resolve to packages + loadALL() // rebuild graph, from main module (not from named packages) sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modconv/convert.go b/src/cmd/go/internal/modconv/convert.go index d5a0bc21e9..f465a9f395 100644 --- a/src/cmd/go/internal/modconv/convert.go +++ b/src/cmd/go/internal/modconv/convert.go @@ -7,12 +7,13 @@ package modconv import ( "fmt" "os" - "runtime" "sort" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" + "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -41,52 +42,46 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - versions := make([]*module.Version, len(mf.Require)) - replace := make(map[string]*modfile.Replace) + var ( + work par.Work + mu sync.Mutex + need = make(map[string]string) + replace = make(map[string]*modfile.Replace) + ) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) - for i, r := range mf.Require { + for _, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - m = re.New + work.Add(re.New) + continue } - sem <- token{} - go func(i int, m module.Version) { - repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) - return - } - - path := repo.ModulePath() - versions[i].Path = path - versions[i].Version = info.Version - - <-sem - }(i, m) - } - // Fill semaphore channel to wait for all tasks to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} + work.Add(r.Mod) } - need := map[string]string{} - for _, v := range versions { + work.Do(10, func(item interface{}) { + r := item.(module.Version) + repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) + return + } + mu.Lock() + path := repo.ModulePath() // Don't use semver.Max here; need to preserve +incompatible suffix. - if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { - need[v.Path] = v.Version + if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { + need[path] = info.Version } - } - paths := make([]string, 0, len(need)) + mu.Unlock() + }) + + var paths []string for path := range need { paths = append(paths, path) } diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index faa2b4c606..a04a13b14f 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -6,7 +6,6 @@ package modconv import ( "bytes" - "context" "fmt" "internal/testenv" "io/ioutil" @@ -147,8 +146,6 @@ func TestConvertLegacyConfig(t *testing.T) { }, } - ctx := context.Background() - for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -160,7 +157,7 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e29eb0a942..e40158b535 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -7,7 +7,6 @@ package modfetch import ( "archive/zip" "bytes" - "context" "errors" "fmt" "io" @@ -24,7 +23,6 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" - "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -36,7 +34,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(ctx context.Context, mod module.Version) (dir string, err error) { +func Download(mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. @@ -49,7 +47,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(ctx, mod) + dir, err := download(mod) if err != nil { return cached{"", err} } @@ -59,10 +57,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(ctx context.Context, mod module.Version) (dir string, err error) { - ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) - defer span.Done() - +func download(mod module.Version) (dir string, err error) { // If the directory exists, and no .partial file exists, the module has // already been completely extracted. .partial files may be created when a // module zip directory is extracted in place instead of being extracted to a @@ -77,7 +72,7 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(ctx, mod) + zipfile, err := DownloadZip(mod) if err != nil { return "", err } @@ -147,7 +142,6 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { return "", err } - ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) if unzipInPlace { if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { return "", err @@ -177,7 +171,6 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { return "", err } } - defer span.Done() if !cfg.ModCacheRW { // Make dir read-only only *after* renaming it. @@ -202,7 +195,7 @@ var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { +func DownloadZip(mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -237,7 +230,7 @@ func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err e if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(ctx, mod, zipfile); err != nil { + if err := downloadZip(mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -245,10 +238,7 @@ func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err e return c.zipfile, c.err } -func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { - ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) - defer span.Done() - +func downloadZip(mod module.Version, zipfile string) (err error) { // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. diff --git a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index 82398ebfed..eac9b32fa8 100644 --- a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,7 +16,6 @@ package zip_sum_test import ( - "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -120,7 +119,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(context.Background(), test.m) + zipPath, err := modfetch.DownloadZip(test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index ee9757912b..93a6bb54d5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "sort" "strings" "sync" @@ -22,6 +21,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/mvs" + "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" @@ -353,7 +353,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if !strings.Contains(path, "...") { m := search.NewMatch(path) if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(ctx, pkgPath) + m = modload.TargetPackages(pkgPath) } if len(m.Pkgs) == 0 { for _, err := range m.Errs { @@ -399,7 +399,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { default: // The argument is a package or module path. if modload.HasModRoot() { - if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { + if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { // The path is in the main module. Nothing to query. if vers != "upgrade" && vers != "patch" { base.Errorf("go get %s: can't request explicit version of path in main module", arg) @@ -491,7 +491,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if q.path == q.m.Path { wg.Add(1) go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil { + if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { base.Errorf("go get: %v", err) } else if !hasPkg { modOnlyMu.Lock() @@ -536,7 +536,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Don't load packages if pkgPatterns is empty. Both // modload.ImportPathsQuiet and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) + matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -725,44 +725,29 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // reported. A map from module paths to queries is returned, which includes // queries and modOnly. func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { + var lookup par.Work + for _, q := range queries { + if cached := cache[q.querySpec]; cached != nil { + *q = *cached + } else { + cache[q.querySpec] = q + lookup.Add(q) + } + } - runQuery := func(q *query) { + lookup.Do(10, func(item interface{}) { + q := item.(*query) if q.vers == "none" { // Wait for downgrade step. q.m = module.Version{Path: q.path, Version: "none"} return } - m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath) + m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) if err != nil { base.Errorf("go get %s: %v", q.arg, err) } q.m = m - } - - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - sem <- token{} - go func(q *query) { - runQuery(q) - <-sem - }(q) - } - } - - // Fill semaphore channel to wait for goroutines to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} - } - - // Add to cache after concurrent section to avoid races... - for _, q := range queries { - cache[q.querySpec] = q - } - + }) base.ExitIfErrors() byPath := make(map[string]*query) @@ -790,7 +775,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer // to determine the underlying module version being requested. // If forceModulePath is set, getQuery must interpret path // as a module path. -func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { +func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { if (prevM.Version != "") != forceModulePath { // We resolve package patterns by calling QueryPattern, which does not // accept a previous version and therefore cannot take it into account for @@ -812,7 +797,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc } } - info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +823,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) + results, err := modload.QueryPattern(path, vers, modload.Allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -994,7 +979,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) + info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index a101681a1f..5f8a2e7e05 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -6,7 +6,6 @@ package modload import ( "bytes" - "context" "encoding/hex" "fmt" "internal/goroot" @@ -58,21 +57,21 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(context.TODO(), m, true) + return moduleInfo(m, true) } -func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { +func ModuleInfo(path string) *modinfo.ModulePublic { if !Enabled() { return nil } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) + return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(ctx, m, true) + return moduleInfo(m, true) } } @@ -85,12 +84,12 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { +func addUpdate(m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -104,7 +103,7 @@ func addVersions(m *modinfo.ModulePublic) { m.Versions, _ = versions(m.Path) } -func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -133,7 +132,7 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { + if q, err := Query(m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 5c51a79124..4d2bc805e2 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -5,7 +5,6 @@ package modload import ( - "context" "errors" "fmt" "go/build" @@ -111,7 +110,7 @@ var _ load.ImportPathError = &AmbiguousImportError{} // Import returns an ImportMissingError as the error. // If Import can identify a module that could be added to supply the package, // the ImportMissingError records that module. -func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { +func Import(path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -166,7 +165,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(ctx, m) + root, isLocal, err := fetch(m) if err != nil { // Report fetch error. // Note that we don't know for sure this module is necessary, @@ -249,7 +248,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(ctx, m) + root, isLocal, err := fetch(m) if err != nil { // Report fetch error as above. return module.Version{}, "", err @@ -286,7 +285,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(ctx, path, "latest", Allowed) + candidates, err := QueryPackage(path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index 47ce89a084..accc60eecd 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -5,7 +5,6 @@ package modload import ( - "context" "internal/testenv" "regexp" "strings" @@ -50,12 +49,10 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() - ctx := context.Background() - for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(ctx, tt.path) + m, dir, err := Import(tt.path) if err == nil { t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 93027c44c4..fff060e665 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -6,7 +6,6 @@ package modload import ( "bytes" - "context" "encoding/json" "errors" "fmt" @@ -333,7 +332,7 @@ func die() { // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod(ctx context.Context) { +func InitMod() { if len(buildList) > 0 { return } @@ -360,7 +359,7 @@ func InitMod(ctx context.Context) { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -398,7 +397,7 @@ func InitMod(ctx context.Context) { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { +func fixVersion(fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -430,7 +429,7 @@ func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { } } - info, err := Query(ctx, path, vers, "", nil) + info, err := Query(path, vers, "", nil) if err != nil { return "", err } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 7bf4e86c8d..4768516e90 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "os" - "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" + "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" @@ -22,42 +22,31 @@ import ( func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { mods := listModules(ctx, args, listVersions) - - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) if listU || listVersions { + var work par.Work for _, m := range mods { - add := func(m *modinfo.ModulePublic) { - sem <- token{} - go func() { - if listU { - addUpdate(ctx, m) - } - if listVersions { - addVersions(m) - } - <-sem - }() - } - - add(m) + work.Add(m) if m.Replace != nil { - add(m.Replace) + work.Add(m.Replace) } } + work.Do(10, func(item interface{}) { + m := item.(*modinfo.ModulePublic) + if listU { + addUpdate(m) + } + if listVersions { + addVersions(m) + } + }) } - // Fill semaphore channel to wait for all tasks to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} - } - return mods } func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { LoadBuildList(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -83,7 +72,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(ctx, path, vers, current, nil) + info, err := Query(path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +81,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) + mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) continue } @@ -117,7 +106,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(ctx, m, true)) + mods = append(mods, moduleInfo(m, true)) } } } @@ -127,9 +116,9 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(ctx, arg, "latest", "", nil) + info, err := Query(arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) + mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 686d491219..8190009b23 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -52,8 +52,8 @@ var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), // on the target platform. Modules may be added to the build list // to satisfy new imports. -func ImportPaths(ctx context.Context, patterns []string) []*search.Match { - matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) +func ImportPaths(patterns []string) []*search.Match { + matches := ImportPathsQuiet(patterns, imports.Tags()) search.WarnUnmatched(matches) return matches } @@ -62,7 +62,7 @@ func ImportPaths(ctx context.Context, patterns []string) []*search.Match { // no matches. It also lets the caller specify a set of build tags to match // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { +func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { updateMatches := func(matches []*search.Match, iterating bool) { for _, m := range matches { switch { @@ -103,7 +103,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, includeStd, buildList) + matchPackages(m, loaded.tags, includeStd, buildList) case m.Pattern() == "all": loaded.testAll = true @@ -111,7 +111,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo // Enumerate the packages in the main module. // We'll load the dependencies as we find them. m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -129,7 +129,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo } } - InitMod(ctx) + InitMod() var matches []*search.Match for _, pattern := range search.CleanPatterns(patterns) { @@ -338,8 +338,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(ctx context.Context, gofiles []string) { - InitMod(ctx) +func ImportFromFiles(gofiles []string) { + InitMod() tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -391,7 +391,7 @@ func DirImportPath(dir string) string { func LoadBuildList(ctx context.Context) []module.Version { ctx, span := trace.StartSpan(ctx, "LoadBuildList") defer span.Done() - InitMod(ctx) + InitMod() ReloadBuildList() WriteGoMod() return buildList @@ -409,20 +409,20 @@ func ReloadBuildList() []module.Version { // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. -func LoadALL(ctx context.Context) []string { - return loadAll(ctx, true) +func LoadALL() []string { + return loadAll(true) } // LoadVendor is like LoadALL but only follows test dependencies // for tests in the main module. Tests in dependency modules are // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor(ctx context.Context) []string { - return loadAll(ctx, false) +func LoadVendor() []string { + return loadAll(false) } -func loadAll(ctx context.Context, testAll bool) []string { - InitMod(ctx) +func loadAll(testAll bool) []string { + InitMod() loaded = newLoader(imports.AnyTags()) loaded.isALL = true @@ -430,7 +430,7 @@ func loadAll(ctx context.Context, testAll bool) []string { if !testAll { loaded.testRoots = true } - all := TargetPackages(ctx, "...") + all := TargetPackages("...") loaded.load(func() []string { return all.Pkgs }) checkMultiplePaths() WriteGoMod() @@ -453,13 +453,13 @@ func loadAll(ctx context.Context, testAll bool) []string { // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(ctx context.Context, pattern string) *search.Match { +func TargetPackages(pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. ModRoot() m := search.NewMatch(pattern) - matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } @@ -817,8 +817,7 @@ func (ld *loader) doPkg(item interface{}) { return } - // TODO(matloob): Handle TODO context. This needs to be threaded through Do. - pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) + pkg.mod, pkg.dir, pkg.err = Import(pkg.path) if pkg.dir == "" { return } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 67eb2c2e19..5dd009d31d 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -5,7 +5,6 @@ package modload import ( - "context" "errors" "fmt" "os" @@ -225,7 +224,7 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { // // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { +func fetch(mod module.Version) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -255,6 +254,6 @@ func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, e mod = r } - dir, err = modfetch.Download(ctx, mod) + dir, err = modfetch.Download(mod) return dir, false, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index e82eb1506f..acc886bf21 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -5,7 +5,6 @@ package modload import ( - "context" "errors" "fmt" "os" @@ -19,7 +18,6 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/search" "cmd/go/internal/str" - "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -57,10 +55,10 @@ import ( // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(ctx, proxy, path, query, current, allowed) + info, err = queryProxy(proxy, path, query, current, allowed) return err }) return info, err @@ -77,10 +75,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { - ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) - defer span.Done() - +func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -248,7 +243,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if err != nil { return nil, err } - releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) + releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) if err != nil { return nil, err } @@ -332,7 +327,7 @@ func matchSemverPrefix(p, v string) bool { // 1. versions that do not satisfy the 'ok' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { if !ok(module.Version{Path: path, Version: v}) { @@ -348,7 +343,7 @@ func filterVersions(ctx context.Context, path string, versions []string, ok func // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) if err != nil { return nil, nil, err } @@ -385,12 +380,12 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) } - return QueryPattern(ctx, path, query, allowed) + return QueryPattern(path, query, allowed) } // QueryPattern looks up the module(s) containing at least one package matching @@ -406,10 +401,7 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { - ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) - defer span.Done() - +func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { base := pattern firstError := func(m *search.Match) error { @@ -425,7 +417,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul base = pathpkg.Dir(pattern[:i+3]) match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -477,18 +469,15 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { - ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) - defer span.Done() - + queryModule := func(path string) (r QueryResult, err error) { current := findCurrentVersion(path) r.Mod.Path = path - r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) + r.Rev, err = queryProxy(proxy, path, query, current, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(ctx, r.Mod) + root, isLocal, err := fetch(r.Mod) if err != nil { return r, err } @@ -509,7 +498,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } var err error - results, err = queryPrefixModules(ctx, candidateModules, queryModule) + results, err = queryPrefixModules(candidateModules, queryModule) return err }) @@ -553,10 +542,7 @@ type prefixResult struct { err error } -func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { - ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") - defer span.Done() - +func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -569,9 +555,8 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { - ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(ctx, p) + r.QueryResult, r.err = queryModule(p) wg.Done() }(p, &results[i]) } @@ -713,8 +698,8 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { - root, isLocal, err := fetch(ctx, m) +func ModuleHasRootPackage(m module.Version) (bool, error) { + root, isLocal, err := fetch(m) if err != nil { return false, err } @@ -722,8 +707,8 @@ func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { - root, _, err := fetch(ctx, m) +func versionHasGoMod(m module.Version) (bool, error) { + root, _, err := fetch(m) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 77080e9b5b..247e4c40d2 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -5,7 +5,6 @@ package modload import ( - "context" "internal/testenv" "io/ioutil" "log" @@ -180,8 +179,6 @@ func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") - ctx := context.Background() - for _, tt := range queryTests { allow := tt.allow if allow == "" { @@ -195,7 +192,7 @@ func TestQuery(t *testing.T) { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) + info, err := Query(tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index a9bee0af4e..c28e7c0c1e 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -5,7 +5,6 @@ package modload import ( - "context" "fmt" "os" "path/filepath" @@ -28,7 +27,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -154,7 +153,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f isLocal = true } else { var err error - root, isLocal, err = fetch(ctx, mod) + root, isLocal, err = fetch(mod) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 99578b244c..deec5106ff 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -77,7 +77,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(ctx, files) + p = load.GoFilesPackage(files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 3aee6939d2..9cef8cf89c 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -702,7 +702,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(ctx, pkgs) { + for _, p := range load.TestPackageList(pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index d975c36306..681ecd7646 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2900,7 +2900,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { } srcs := []string{src} - p := load.GoFilesPackage(context.TODO(), srcs) + p := load.GoFilesPackage(srcs) if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { return "32", nil -- cgit v1.3 From b58d29741650c7bf10b17f455666e2727e1cdd2e Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 17 Aug 2020 19:06:19 -0400 Subject: cmd/compile, runtime: mark R12 clobbered for write barrier call on PPC64 When external linking, for large binaries, the external linker may insert a trampoline for the write barrier call, which looks 0000000005a98cc8 <__long_branch_runtime.gcWriteBarrier>: 5a98cc8: 86 01 82 3d addis r12,r2,390 5a98ccc: d8 bd 8c e9 ld r12,-16936(r12) 5a98cd0: a6 03 89 7d mtctr r12 5a98cd4: 20 04 80 4e bctr It clobbers R12 (and CTR, which is never live across a call). As at compile time we don't know whether the binary is big and what link mode will be used, I think we need to mark R12 as clobbered for write barrier call. For extra safety (future-proof) we mark caller-saved register that cannot be used for function arguments, which includes R11, as potentially clobbered as well. Fixes #40851. Change-Id: Iedd901c5072f1127cc59b0a48cfeb4aaec81b519 Reviewed-on: https://go-review.googlesource.com/c/go/+/248917 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements --- src/cmd/compile/internal/ssa/gen/PPC64Ops.go | 4 +-- src/cmd/compile/internal/ssa/opGen.go | 2 +- src/runtime/asm_ppc64x.s | 41 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 23 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go index f8bc6cb20b..0261dc283b 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go @@ -645,9 +645,9 @@ func init() { {name: "LoweredAtomicOr8", argLength: 3, reg: gpstore, asm: "OR", faultOnNilArg0: true, hasSideEffects: true}, // LoweredWB invokes runtime.gcWriteBarrier. arg0=destptr, arg1=srcptr, arg2=mem, aux=runtime.gcWriteBarrier - // It preserves R0 through R15, g, and its arguments R20 and R21, + // It preserves R0 through R17 (except special registers R1, R2, R11, R12, R13), g, and its arguments R20 and R21, // but may clobber anything else, including R31 (REGTMP). - {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("R20"), buildReg("R21")}, clobbers: (callerSave &^ buildReg("R0 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R20 R21 g")) | buildReg("R31")}, clobberFlags: true, aux: "Sym", symEffect: "None"}, + {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("R20"), buildReg("R21")}, clobbers: (callerSave &^ buildReg("R0 R3 R4 R5 R6 R7 R8 R9 R10 R14 R15 R16 R17 R20 R21 g")) | buildReg("R31")}, clobberFlags: true, aux: "Sym", symEffect: "None"}, // There are three of these functions so that they can have three different register inputs. // When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 408c855dbd..df2a27368b 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -26889,7 +26889,7 @@ var opcodeTable = [...]opInfo{ {0, 1048576}, // R20 {1, 2097152}, // R21 }, - clobbers: 576460746931503104, // R16 R17 R18 R19 R22 R23 R24 R25 R26 R27 R28 R29 R31 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 + clobbers: 576460746931312640, // R11 R12 R18 R19 R22 R23 R24 R25 R26 R27 R28 R29 R31 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 }, }, { diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 11d2f2f51a..23387a2165 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -916,23 +916,23 @@ TEXT ·checkASM(SB),NOSPLIT,$0-1 // - R20 is the destination of the write // - R21 is the value being written at R20. // It clobbers condition codes. -// It does not clobber R0 through R15, +// It does not clobber R0 through R17 (except special registers), // but may clobber any other register, *including* R31. TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112 // The standard prologue clobbers R31. - // We use R16 and R17 as scratch registers. - MOVD g_m(g), R16 - MOVD m_p(R16), R16 - MOVD (p_wbBuf+wbBuf_next)(R16), R17 + // We use R18 and R19 as scratch registers. + MOVD g_m(g), R18 + MOVD m_p(R18), R18 + MOVD (p_wbBuf+wbBuf_next)(R18), R19 // Increment wbBuf.next position. - ADD $16, R17 - MOVD R17, (p_wbBuf+wbBuf_next)(R16) - MOVD (p_wbBuf+wbBuf_end)(R16), R16 - CMP R16, R17 + ADD $16, R19 + MOVD R19, (p_wbBuf+wbBuf_next)(R18) + MOVD (p_wbBuf+wbBuf_end)(R18), R18 + CMP R18, R19 // Record the write. - MOVD R21, -16(R17) // Record value - MOVD (R20), R16 // TODO: This turns bad writes into bad reads. - MOVD R16, -8(R17) // Record *slot + MOVD R21, -16(R19) // Record value + MOVD (R20), R18 // TODO: This turns bad writes into bad reads. + MOVD R18, -8(R19) // Record *slot // Is the buffer full? (flags set in CMP above) BEQ flush ret: @@ -956,11 +956,12 @@ flush: MOVD R8, (FIXED_FRAME+56)(R1) MOVD R9, (FIXED_FRAME+64)(R1) MOVD R10, (FIXED_FRAME+72)(R1) - MOVD R11, (FIXED_FRAME+80)(R1) - MOVD R12, (FIXED_FRAME+88)(R1) + // R11, R12 may be clobbered by external-linker-inserted trampoline // R13 is REGTLS - MOVD R14, (FIXED_FRAME+96)(R1) - MOVD R15, (FIXED_FRAME+104)(R1) + MOVD R14, (FIXED_FRAME+80)(R1) + MOVD R15, (FIXED_FRAME+88)(R1) + MOVD R16, (FIXED_FRAME+96)(R1) + MOVD R17, (FIXED_FRAME+104)(R1) // This takes arguments R20 and R21. CALL runtime·wbBufFlush(SB) @@ -975,10 +976,10 @@ flush: MOVD (FIXED_FRAME+56)(R1), R8 MOVD (FIXED_FRAME+64)(R1), R9 MOVD (FIXED_FRAME+72)(R1), R10 - MOVD (FIXED_FRAME+80)(R1), R11 - MOVD (FIXED_FRAME+88)(R1), R12 - MOVD (FIXED_FRAME+96)(R1), R14 - MOVD (FIXED_FRAME+104)(R1), R15 + MOVD (FIXED_FRAME+80)(R1), R14 + MOVD (FIXED_FRAME+88)(R1), R15 + MOVD (FIXED_FRAME+96)(R1), R16 + MOVD (FIXED_FRAME+104)(R1), R17 JMP ret // Note: these functions use a special calling convention to save generated code space. -- cgit v1.3 From 20b4987d3e658a1cd93472185d7998f745bd062e Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Wed, 13 May 2020 13:57:36 -0400 Subject: cmd/dist: use GO_TEST_SHORT value more consistently There were two places where the -short flag was added in order to speed up tests when run in short mode, in CL 178399 and CL 177417. It appears viable to re-use the GO_TEST_SHORT value so that -short flag is not used when the tests are executed on a longtest builder, where it is not a goal to skip slow tests for improved performance. Do so, in order to make the testing configurations simpler and more predictable. Factor out the flag name out of the string returned by short, so that it can be used in context of 'go test' which can accept a -short flag, and a test binary which requires the use of a longer -test.short flag. For #39054. For #29252. Change-Id: I52dfbef73cc8307735c52e2ebaa609305fb05933 Reviewed-on: https://go-review.googlesource.com/c/go/+/233898 Run-TryBot: Dmitri Shuralyov TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/dist/test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 2dc9459215..a83ae35293 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -241,13 +241,15 @@ func (t *tester) shouldRunTest(name string) bool { return false } -// short returns a -short flag to pass to 'go test'. -// It returns "-short", unless the environment variable +// short returns a -short flag value to use with 'go test' +// or a test binary for tests intended to run in short mode. +// It returns "true", unless the environment variable // GO_TEST_SHORT is set to a non-empty, false-ish string. // // This environment variable is meant to be an internal -// detail between the Go build system and cmd/dist -// and is not intended for use by users. +// detail between the Go build system and cmd/dist for +// the purpose of longtest builders, and is not intended +// for use by users. See golang.org/issue/12508. func short() string { if v := os.Getenv("GO_TEST_SHORT"); v != "" { short, err := strconv.ParseBool(v) @@ -255,10 +257,10 @@ func short() string { fatalf("invalid GO_TEST_SHORT %q: %v", v, err) } if !short { - return "-short=false" + return "false" } } - return "-short" + return "true" } // goTest returns the beginning of the go test command line. @@ -266,7 +268,7 @@ func short() string { // defaults as later arguments in the command line. func (t *tester) goTest() []string { return []string{ - "go", "test", short(), "-count=1", t.tags(), t.runFlag(""), + "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""), } } @@ -335,7 +337,7 @@ func (t *tester) registerStdTest(pkg string) { } args := []string{ "test", - short(), + "-short=" + short(), t.tags(), t.timeout(timeoutSec), "-gcflags=all=" + gogcflags, @@ -373,7 +375,7 @@ func (t *tester) registerRaceBenchTest(pkg string) { ranGoBench = true args := []string{ "test", - short(), + "-short=" + short(), "-race", t.timeout(1200), // longer timeout for race with benchmarks "-run=^$", // nothing. only benchmarks. @@ -1069,7 +1071,7 @@ func (t *tester) runHostTest(dir, pkg string) error { if err := cmd.Run(); err != nil { return err } - return t.dirCmd(dir, f.Name(), "-test.short").Run() + return t.dirCmd(dir, f.Name(), "-test.short="+short()).Run() } func (t *tester) cgoTest(dt *distTest) error { @@ -1570,7 +1572,7 @@ func (t *tester) prebuiltGoPackageTestBinary() string { func (t *tester) runPrecompiledStdTest(timeout time.Duration) error { bin := t.prebuiltGoPackageTestBinary() fmt.Fprintf(os.Stderr, "# %s: using pre-built %s...\n", stdMatches[0], bin) - cmd := exec.Command(bin, "-test.short", "-test.timeout="+timeout.String()) + cmd := exec.Command(bin, "-test.short="+short(), "-test.timeout="+timeout.String()) cmd.Dir = filepath.Dir(bin) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr -- cgit v1.3 From 93eeb819cab491d4e429b7aa85a864a045979a18 Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Mon, 8 Jun 2020 13:58:53 +0800 Subject: cmd/asm: Add SHA512 hardware instructions for ARM64 ARMv8.2-SHA add SHA512 intructions: 1. SHA512H Vm.D2, Vn, Vd 2. SHA512H2 Vm.D2, Vn, Vd 3. SHA512SU0 Vn.D2, Vd.D2 4. SHA512SU1 Vm.D2, Vn.D2, Vd.D2 ARMv8 Architecture Reference Manual C7.2.234-C7.2.234 Change-Id: Ie970fef1bba5312ad466f246035da4c40a1bbb39 Reviewed-on: https://go-review.googlesource.com/c/go/+/180057 Reviewed-by: Cherry Zhang Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot --- src/cmd/asm/internal/asm/testdata/arm64.s | 4 ++++ src/cmd/internal/obj/arm64/a.out.go | 4 ++++ src/cmd/internal/obj/arm64/anames.go | 4 ++++ src/cmd/internal/obj/arm64/asm7.go | 16 ++++++++++++++++ 4 files changed, 28 insertions(+) (limited to 'src/cmd') diff --git a/src/cmd/asm/internal/asm/testdata/arm64.s b/src/cmd/asm/internal/asm/testdata/arm64.s index 69267bfa63..5a6db05074 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64.s +++ b/src/cmd/asm/internal/asm/testdata/arm64.s @@ -77,6 +77,10 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 SHA1H V5, V4 // a408285e SHA1M V8.S4, V7, V6 // e620085e SHA1P V11.S4, V10, V9 // 49110b5e + SHA512H V2.D2, V1, V0 // 208062ce + SHA512H2 V4.D2, V3, V2 // 628464ce + SHA512SU0 V9.D2, V8.D2 // 2881c0ce + SHA512SU1 V7.D2, V6.D2, V5.D2 // c58867ce VADDV V0.S4, V0 // 00b8b14e VMOVI $82, V0.B16 // 40e6024f VUADDLV V6.B16, V6 // c638306e diff --git a/src/cmd/internal/obj/arm64/a.out.go b/src/cmd/internal/obj/arm64/a.out.go index 152c493a65..03e0278a33 100644 --- a/src/cmd/internal/obj/arm64/a.out.go +++ b/src/cmd/internal/obj/arm64/a.out.go @@ -946,6 +946,10 @@ const ( ASHA256H2 ASHA256SU0 ASHA256SU1 + ASHA512H + ASHA512H2 + ASHA512SU0 + ASHA512SU1 AVADD AVADDP AVAND diff --git a/src/cmd/internal/obj/arm64/anames.go b/src/cmd/internal/obj/arm64/anames.go index 565f70aaf9..65ecd007ea 100644 --- a/src/cmd/internal/obj/arm64/anames.go +++ b/src/cmd/internal/obj/arm64/anames.go @@ -453,6 +453,10 @@ var Anames = []string{ "SHA256H2", "SHA256SU0", "SHA256SU1", + "SHA512H", + "SHA512H2", + "SHA512SU0", + "SHA512SU1", "VADD", "VADDP", "VAND", diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index df17729a76..8f8981479b 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -2747,6 +2747,7 @@ func buildop(ctxt *obj.Link) { oprangeset(AAESIMC, t) oprangeset(ASHA1SU1, t) oprangeset(ASHA256SU0, t) + oprangeset(ASHA512SU0, t) case ASHA1C: oprangeset(ASHA1P, t) @@ -2754,9 +2755,12 @@ func buildop(ctxt *obj.Link) { case ASHA256H: oprangeset(ASHA256H2, t) + oprangeset(ASHA512H, t) + oprangeset(ASHA512H2, t) case ASHA1SU0: oprangeset(ASHA256SU1, t) + oprangeset(ASHA512SU1, t) case AVADDV: oprangeset(AVUADDLV, t) @@ -5391,6 +5395,18 @@ func (c *ctxt7) oprrr(p *obj.Prog, a obj.As) uint32 { case ASHA256SU0: return 0x5E<<24 | 2<<20 | 8<<16 | 2<<12 | 2<<10 + case ASHA512H: + return 0xCE<<24 | 3<<21 | 8<<12 + + case ASHA512H2: + return 0xCE<<24 | 3<<21 | 8<<12 | 4<<8 + + case ASHA512SU1: + return 0xCE<<24 | 3<<21 | 8<<12 | 8<<8 + + case ASHA512SU0: + return 0xCE<<24 | 3<<22 | 8<<12 + case AFCVTZSD: return FPCVTI(1, 0, 1, 3, 0) -- cgit v1.3 From cdc77d34d7770ed02d84b9193380f9646017dce6 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 17 Aug 2020 17:31:21 -0400 Subject: testing: treat PAUSE lines as changing the active test name We could instead fix cmd/test2json to treat PAUSE lines as *not* changing the active test name, but that seems like it would be more confusing to humans, and also wouldn't fix tools that parse output using existing builds of cmd/test2json. Fixes #40657 Change-Id: I937611778f5b1e7dd1d6e9f44424d7e725a589ed Reviewed-on: https://go-review.googlesource.com/c/go/+/248727 Run-TryBot: Bryan C. Mills Reviewed-by: Jean de Klerk --- .../go/testdata/script/test_json_interleaved.txt | 27 ++++++++++++++++++++++ src/testing/testing.go | 25 ++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 src/cmd/go/testdata/script/test_json_interleaved.txt (limited to 'src/cmd') diff --git a/src/cmd/go/testdata/script/test_json_interleaved.txt b/src/cmd/go/testdata/script/test_json_interleaved.txt new file mode 100644 index 0000000000..e2d349e3fb --- /dev/null +++ b/src/cmd/go/testdata/script/test_json_interleaved.txt @@ -0,0 +1,27 @@ +# Regression test for https://golang.org/issue/40657: output from the main test +# function should be attributed correctly even if interleaved with the PAUSE +# line for a new parallel subtest. + +[short] skip + +go test -json +stdout '"Test":"TestWeirdTiming","Output":"[^"]* logging to outer again\\n"' + +-- go.mod -- +module example.com +go 1.15 +-- main_test.go -- +package main + +import ( + "testing" +) + +func TestWeirdTiming(outer *testing.T) { + outer.Run("pauser", func(pauser *testing.T) { + outer.Logf("logging to outer") + pauser.Parallel() + }) + + outer.Logf("logging to outer again") +} diff --git a/src/testing/testing.go b/src/testing/testing.go index 061142b9ab..6fc8c4fa9f 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -357,10 +357,19 @@ func (p *testPrinter) Fprint(w io.Writer, testName, out string) { defer p.lastNameMu.Unlock() if !p.chatty || - strings.HasPrefix(out, "--- PASS") || - strings.HasPrefix(out, "--- FAIL") || - strings.HasPrefix(out, "=== CONT") || - strings.HasPrefix(out, "=== RUN") { + strings.HasPrefix(out, "--- PASS: ") || + strings.HasPrefix(out, "--- FAIL: ") || + strings.HasPrefix(out, "--- SKIP: ") || + strings.HasPrefix(out, "=== RUN ") || + strings.HasPrefix(out, "=== CONT ") || + strings.HasPrefix(out, "=== PAUSE ") { + // If we're buffering test output (!p.chatty), we don't really care which + // test is emitting which line so long as they are serialized. + // + // If the message already implies an association with a specific new test, + // we don't need to check what the old test name was or log an extra CONT + // line for it. (We're updating it anyway, and the current message already + // includes the test name.) p.lastName = testName fmt.Fprint(w, out) return @@ -976,7 +985,13 @@ func (t *T) Parallel() { for ; root.parent != nil; root = root.parent { } root.mu.Lock() - fmt.Fprintf(root.w, "=== PAUSE %s\n", t.name) + // Unfortunately, even though PAUSE indicates that the named test is *no + // longer* running, cmd/test2json interprets it as changing the active test + // for the purpose of log parsing. We could fix cmd/test2json, but that + // won't fix existing deployments of third-party tools that already shell + // out to older builds of cmd/test2json — so merely fixing cmd/test2json + // isn't enough for now. + printer.Fprint(root.w, t.name, fmt.Sprintf("=== PAUSE %s\n", t.name)) root.mu.Unlock() } -- cgit v1.3 From 78a1064d5dd05fc669342df3a6a5e11d49749d85 Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 14:05:05 -0700 Subject: runtime: remove scase.releasetime field selectgo will report at most one block event, so there's no need to keep a releasetime for every select case. It suffices to simply track the releasetime of the case responsible for the wakeup. Updates #40410. Change-Id: I72679cd43dde80d7e6dbab21a78952a4372d1e79 Reviewed-on: https://go-review.googlesource.com/c/go/+/245122 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/select.go | 1 - src/runtime/select.go | 23 ++++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index 49cc23cd3d..eb5ff8469b 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -386,7 +386,6 @@ func scasetype() *types.Type { namedfield("elem", types.Types[TUNSAFEPTR]), namedfield("kind", types.Types[TUINT16]), namedfield("pc", types.Types[TUINTPTR]), - namedfield("releasetime", types.Types[TINT64]), }) scase.SetNoalg(true) } diff --git a/src/runtime/select.go b/src/runtime/select.go index 081db7bad4..2f8b139155 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -26,11 +26,10 @@ const ( // Known to compiler. // Changes here must also be made in src/cmd/internal/gc/select.go's scasetype. type scase struct { - c *hchan // chan - elem unsafe.Pointer // data element - kind uint16 - pc uintptr // race pc (for race detector / msan) - releasetime int64 + c *hchan // chan + elem unsafe.Pointer // data element + kind uint16 + pc uintptr // race pc (for race detector / msan) } var ( @@ -142,9 +141,6 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { var t0 int64 if blockprofilerate > 0 { t0 = cputicks() - for i := 0; i < ncases; i++ { - scases[i].releasetime = -1 - } } // The compiler rewrites selects that statically have @@ -227,6 +223,7 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { var casi int var cas *scase var caseSuccess bool + var caseReleaseTime int64 = -1 var recvOK bool for i := 0; i < ncases; i++ { casi = int(pollorder[i]) @@ -346,14 +343,14 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { if k.kind == caseNil { continue } - if sglist.releasetime > 0 { - k.releasetime = sglist.releasetime - } if sg == sglist { // sg has already been dequeued by the G that woke us up. casi = int(casei) cas = k caseSuccess = sglist.success + if sglist.releasetime > 0 { + caseReleaseTime = sglist.releasetime + } } else { c = k.c if k.kind == caseSend { @@ -483,8 +480,8 @@ send: goto retc retc: - if cas.releasetime > 0 { - blockevent(cas.releasetime-t0, 1) + if caseReleaseTime > 0 { + blockevent(caseReleaseTime-t0, 1) } return casi, recvOK -- cgit v1.3 From d36bc7d78ad226b20056c08fb8bca041e25b3d1d Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 15:20:18 -0700 Subject: runtime: split PCs out of scase Per-case PCs are only needed for race detector builds, so this allows skipping allocating stack space for them for non-race builds. It's possible to arrange the PCs and order arrays consecutively in memory so that we could just reuse the order0 pointer to identify both. However, there's more risk of that silently going wrong, so this commit passes them as separate arguments for now. We can revisit this in the future. Updates #40410. Change-Id: I8468bc25749e559891cb0cb007d1cc4a40fdd0f8 Reviewed-on: https://go-review.googlesource.com/c/go/+/245124 Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/builtin.go | 199 +++++++++++++------------ src/cmd/compile/internal/gc/builtin/runtime.go | 4 +- src/cmd/compile/internal/gc/select.go | 18 ++- src/runtime/select.go | 50 +++++-- 4 files changed, 153 insertions(+), 118 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index 2cf2f4687e..eafdb0ebe7 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -126,74 +126,74 @@ var runtimeDecls = [...]struct { {"selectnbsend", funcTag, 94}, {"selectnbrecv", funcTag, 95}, {"selectnbrecv2", funcTag, 97}, - {"selectsetpc", funcTag, 62}, - {"selectgo", funcTag, 98}, + {"selectsetpc", funcTag, 98}, + {"selectgo", funcTag, 99}, {"block", funcTag, 9}, - {"makeslice", funcTag, 99}, - {"makeslice64", funcTag, 100}, - {"makeslicecopy", funcTag, 101}, - {"growslice", funcTag, 103}, - {"memmove", funcTag, 104}, - {"memclrNoHeapPointers", funcTag, 105}, - {"memclrHasPointers", funcTag, 105}, - {"memequal", funcTag, 106}, - {"memequal0", funcTag, 107}, - {"memequal8", funcTag, 107}, - {"memequal16", funcTag, 107}, - {"memequal32", funcTag, 107}, - {"memequal64", funcTag, 107}, - {"memequal128", funcTag, 107}, - {"f32equal", funcTag, 108}, - {"f64equal", funcTag, 108}, - {"c64equal", funcTag, 108}, - {"c128equal", funcTag, 108}, - {"strequal", funcTag, 108}, - {"interequal", funcTag, 108}, - {"nilinterequal", funcTag, 108}, - {"memhash", funcTag, 109}, - {"memhash0", funcTag, 110}, - {"memhash8", funcTag, 110}, - {"memhash16", funcTag, 110}, - {"memhash32", funcTag, 110}, - {"memhash64", funcTag, 110}, - {"memhash128", funcTag, 110}, - {"f32hash", funcTag, 110}, - {"f64hash", funcTag, 110}, - {"c64hash", funcTag, 110}, - {"c128hash", funcTag, 110}, - {"strhash", funcTag, 110}, - {"interhash", funcTag, 110}, - {"nilinterhash", funcTag, 110}, - {"int64div", funcTag, 111}, - {"uint64div", funcTag, 112}, - {"int64mod", funcTag, 111}, - {"uint64mod", funcTag, 112}, - {"float64toint64", funcTag, 113}, - {"float64touint64", funcTag, 114}, - {"float64touint32", funcTag, 115}, - {"int64tofloat64", funcTag, 116}, - {"uint64tofloat64", funcTag, 117}, - {"uint32tofloat64", funcTag, 118}, - {"complex128div", funcTag, 119}, - {"racefuncenter", funcTag, 120}, + {"makeslice", funcTag, 100}, + {"makeslice64", funcTag, 101}, + {"makeslicecopy", funcTag, 102}, + {"growslice", funcTag, 104}, + {"memmove", funcTag, 105}, + {"memclrNoHeapPointers", funcTag, 106}, + {"memclrHasPointers", funcTag, 106}, + {"memequal", funcTag, 107}, + {"memequal0", funcTag, 108}, + {"memequal8", funcTag, 108}, + {"memequal16", funcTag, 108}, + {"memequal32", funcTag, 108}, + {"memequal64", funcTag, 108}, + {"memequal128", funcTag, 108}, + {"f32equal", funcTag, 109}, + {"f64equal", funcTag, 109}, + {"c64equal", funcTag, 109}, + {"c128equal", funcTag, 109}, + {"strequal", funcTag, 109}, + {"interequal", funcTag, 109}, + {"nilinterequal", funcTag, 109}, + {"memhash", funcTag, 110}, + {"memhash0", funcTag, 111}, + {"memhash8", funcTag, 111}, + {"memhash16", funcTag, 111}, + {"memhash32", funcTag, 111}, + {"memhash64", funcTag, 111}, + {"memhash128", funcTag, 111}, + {"f32hash", funcTag, 111}, + {"f64hash", funcTag, 111}, + {"c64hash", funcTag, 111}, + {"c128hash", funcTag, 111}, + {"strhash", funcTag, 111}, + {"interhash", funcTag, 111}, + {"nilinterhash", funcTag, 111}, + {"int64div", funcTag, 112}, + {"uint64div", funcTag, 113}, + {"int64mod", funcTag, 112}, + {"uint64mod", funcTag, 113}, + {"float64toint64", funcTag, 114}, + {"float64touint64", funcTag, 115}, + {"float64touint32", funcTag, 116}, + {"int64tofloat64", funcTag, 117}, + {"uint64tofloat64", funcTag, 118}, + {"uint32tofloat64", funcTag, 119}, + {"complex128div", funcTag, 120}, + {"racefuncenter", funcTag, 121}, {"racefuncenterfp", funcTag, 9}, {"racefuncexit", funcTag, 9}, - {"raceread", funcTag, 120}, - {"racewrite", funcTag, 120}, - {"racereadrange", funcTag, 121}, - {"racewriterange", funcTag, 121}, - {"msanread", funcTag, 121}, - {"msanwrite", funcTag, 121}, - {"checkptrAlignment", funcTag, 122}, - {"checkptrArithmetic", funcTag, 124}, - {"libfuzzerTraceCmp1", funcTag, 126}, - {"libfuzzerTraceCmp2", funcTag, 128}, - {"libfuzzerTraceCmp4", funcTag, 129}, - {"libfuzzerTraceCmp8", funcTag, 130}, - {"libfuzzerTraceConstCmp1", funcTag, 126}, - {"libfuzzerTraceConstCmp2", funcTag, 128}, - {"libfuzzerTraceConstCmp4", funcTag, 129}, - {"libfuzzerTraceConstCmp8", funcTag, 130}, + {"raceread", funcTag, 121}, + {"racewrite", funcTag, 121}, + {"racereadrange", funcTag, 122}, + {"racewriterange", funcTag, 122}, + {"msanread", funcTag, 122}, + {"msanwrite", funcTag, 122}, + {"checkptrAlignment", funcTag, 123}, + {"checkptrArithmetic", funcTag, 125}, + {"libfuzzerTraceCmp1", funcTag, 127}, + {"libfuzzerTraceCmp2", funcTag, 129}, + {"libfuzzerTraceCmp4", funcTag, 130}, + {"libfuzzerTraceCmp8", funcTag, 131}, + {"libfuzzerTraceConstCmp1", funcTag, 127}, + {"libfuzzerTraceConstCmp2", funcTag, 129}, + {"libfuzzerTraceConstCmp4", funcTag, 130}, + {"libfuzzerTraceConstCmp8", funcTag, 131}, {"x86HasPOPCNT", varTag, 6}, {"x86HasSSE41", varTag, 6}, {"x86HasFMA", varTag, 6}, @@ -202,7 +202,7 @@ var runtimeDecls = [...]struct { } func runtimeTypes() []*types.Type { - var typs [131]*types.Type + var typs [132]*types.Type typs[0] = types.Bytetype typs[1] = types.NewPtr(typs[0]) typs[2] = types.Types[TANY] @@ -301,38 +301,39 @@ func runtimeTypes() []*types.Type { typs[95] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[84])}, []*Node{anonfield(typs[6])}) typs[96] = types.NewPtr(typs[6]) typs[97] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[96]), anonfield(typs[84])}, []*Node{anonfield(typs[6])}) - typs[98] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) - typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])}) - typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])}) - typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])}) - typs[102] = types.NewSlice(typs[2]) - typs[103] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[102]), anonfield(typs[15])}, []*Node{anonfield(typs[102])}) - typs[104] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, nil) - typs[105] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, nil) - typs[106] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, []*Node{anonfield(typs[6])}) - typs[107] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[6])}) - typs[108] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[7])}, []*Node{anonfield(typs[6])}) - typs[109] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) - typs[110] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) - typs[111] = functype(nil, []*Node{anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[22])}) - typs[112] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, []*Node{anonfield(typs[24])}) - typs[113] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[22])}) - typs[114] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[24])}) - typs[115] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[65])}) - typs[116] = functype(nil, []*Node{anonfield(typs[22])}, []*Node{anonfield(typs[20])}) - typs[117] = functype(nil, []*Node{anonfield(typs[24])}, []*Node{anonfield(typs[20])}) - typs[118] = functype(nil, []*Node{anonfield(typs[65])}, []*Node{anonfield(typs[20])}) - typs[119] = functype(nil, []*Node{anonfield(typs[26]), anonfield(typs[26])}, []*Node{anonfield(typs[26])}) - typs[120] = functype(nil, []*Node{anonfield(typs[5])}, nil) - typs[121] = functype(nil, []*Node{anonfield(typs[5]), anonfield(typs[5])}, nil) - typs[122] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[1]), anonfield(typs[5])}, nil) - typs[123] = types.NewSlice(typs[7]) - typs[124] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[123])}, nil) - typs[125] = types.Types[TUINT8] - typs[126] = functype(nil, []*Node{anonfield(typs[125]), anonfield(typs[125])}, nil) - typs[127] = types.Types[TUINT16] - typs[128] = functype(nil, []*Node{anonfield(typs[127]), anonfield(typs[127])}, nil) - typs[129] = functype(nil, []*Node{anonfield(typs[65]), anonfield(typs[65])}, nil) - typs[130] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, nil) + typs[98] = functype(nil, []*Node{anonfield(typs[63])}, nil) + typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) + typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])}) + typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])}) + typs[102] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])}) + typs[103] = types.NewSlice(typs[2]) + typs[104] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[103]), anonfield(typs[15])}, []*Node{anonfield(typs[103])}) + typs[105] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, nil) + typs[106] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, nil) + typs[107] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, []*Node{anonfield(typs[6])}) + typs[108] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[6])}) + typs[109] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[7])}, []*Node{anonfield(typs[6])}) + typs[110] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) + typs[111] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) + typs[112] = functype(nil, []*Node{anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[22])}) + typs[113] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, []*Node{anonfield(typs[24])}) + typs[114] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[22])}) + typs[115] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[24])}) + typs[116] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[65])}) + typs[117] = functype(nil, []*Node{anonfield(typs[22])}, []*Node{anonfield(typs[20])}) + typs[118] = functype(nil, []*Node{anonfield(typs[24])}, []*Node{anonfield(typs[20])}) + typs[119] = functype(nil, []*Node{anonfield(typs[65])}, []*Node{anonfield(typs[20])}) + typs[120] = functype(nil, []*Node{anonfield(typs[26]), anonfield(typs[26])}, []*Node{anonfield(typs[26])}) + typs[121] = functype(nil, []*Node{anonfield(typs[5])}, nil) + typs[122] = functype(nil, []*Node{anonfield(typs[5]), anonfield(typs[5])}, nil) + typs[123] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[1]), anonfield(typs[5])}, nil) + typs[124] = types.NewSlice(typs[7]) + typs[125] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[124])}, nil) + typs[126] = types.Types[TUINT8] + typs[127] = functype(nil, []*Node{anonfield(typs[126]), anonfield(typs[126])}, nil) + typs[128] = types.Types[TUINT16] + typs[129] = functype(nil, []*Node{anonfield(typs[128]), anonfield(typs[128])}, nil) + typs[130] = functype(nil, []*Node{anonfield(typs[65]), anonfield(typs[65])}, nil) + typs[131] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, nil) return typs[:] } diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index 00448272c5..25f86efdd6 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -169,8 +169,8 @@ func selectnbsend(hchan chan<- any, elem *any) bool func selectnbrecv(elem *any, hchan <-chan any) bool func selectnbrecv2(elem *any, received *bool, hchan <-chan any) bool -func selectsetpc(cas *byte) -func selectgo(cas0 *byte, order0 *byte, ncases int) (int, bool) +func selectsetpc(pc *uintptr) +func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, ncases int) (int, bool) func block() func makeslice(typ *byte, len int, cap int) unsafe.Pointer diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index eb5ff8469b..8eb31eb5c1 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -271,6 +271,14 @@ func walkselectcases(cases *Nodes) []*Node { r = typecheck(r, ctxStmt) init = append(init, r) + var pc0, pcs *Node + if flag_race { + pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(n))) + pc0 = typecheck(nod(OADDR, nod(OINDEX, pcs, nodintconst(0)), nil), ctxExpr) + } else { + pc0 = nodnil() + } + // register cases for i, cas := range cases.Slice() { setlineno(cas) @@ -324,8 +332,8 @@ func walkselectcases(cases *Nodes) []*Node { // TODO(mdempsky): There should be a cleaner way to // handle this. - if instrumenting { - r = mkcall("selectsetpc", nil, nil, bytePtrToIndex(selv, int64(i))) + if flag_race { + r = mkcall("selectsetpc", nil, nil, nod(OADDR, nod(OINDEX, pcs, nodintconst(int64(i))), nil)) init = append(init, r) } } @@ -337,13 +345,16 @@ func walkselectcases(cases *Nodes) []*Node { r = nod(OAS2, nil, nil) r.List.Set2(chosen, recvOK) fn := syslook("selectgo") - r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), nodintconst(int64(n)))) + r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(n)))) r = typecheck(r, ctxStmt) init = append(init, r) // selv and order are no longer alive after selectgo. init = append(init, nod(OVARKILL, selv, nil)) init = append(init, nod(OVARKILL, order, nil)) + if flag_race { + init = append(init, nod(OVARKILL, pcs, nil)) + } // dispatch cases for i, cas := range cases.Slice() { @@ -385,7 +396,6 @@ func scasetype() *types.Type { namedfield("c", types.Types[TUNSAFEPTR]), namedfield("elem", types.Types[TUNSAFEPTR]), namedfield("kind", types.Types[TUINT16]), - namedfield("pc", types.Types[TUINTPTR]), }) scase.SetNoalg(true) } diff --git a/src/runtime/select.go b/src/runtime/select.go index d540dd2e69..d7c7d9f26f 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -29,7 +29,6 @@ type scase struct { c *hchan // chan elem unsafe.Pointer // data element kind uint16 - pc uintptr // race pc (for race detector / msan) } var ( @@ -37,8 +36,8 @@ var ( chanrecvpc = funcPC(chanrecv) ) -func selectsetpc(cas *scase) { - cas.pc = getcallerpc() +func selectsetpc(pc *uintptr) { + *pc = getcallerpc() } func sellock(scases []scase, lockorder []uint16) { @@ -108,11 +107,15 @@ func block() { // Both reside on the goroutine's stack (regardless of any escaping in // selectgo). // +// For race detector builds, pc0 points to an array of type +// [ncases]uintptr (also on the stack); for other builds, it's set to +// nil. +// // selectgo returns the index of the chosen scase, which matches the // ordinal position of its respective select{recv,send,default} call. // Also, if the chosen scase was a receive operation, it reports whether // a value was received. -func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { +func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) { if debugSelect { print("select: cas0=", cas0, "\n") } @@ -126,6 +129,21 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { pollorder := order1[:ncases:ncases] lockorder := order1[ncases:][:ncases:ncases] + // Even when raceenabled is true, there might be select + // statements in packages compiled without -race (e.g., + // ensureSigM in runtime/signal_unix.go). + var pcs []uintptr + if raceenabled && pc0 != nil { + pc1 := (*[1 << 16]uintptr)(unsafe.Pointer(pc0)) + pcs = pc1[:ncases:ncases] + } + casePC := func(casi int) uintptr { + if pcs == nil { + return 0 + } + return pcs[casi] + } + var t0 int64 if blockprofilerate > 0 { t0 = cputicks() @@ -247,7 +265,7 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { case caseSend: if raceenabled { - racereadpc(c.raceaddr(), cas.pc, chansendpc) + racereadpc(c.raceaddr(), casePC(casi), chansendpc) } if c.closed != 0 { goto sclose @@ -371,9 +389,9 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { if raceenabled { if cas.kind == caseRecv && cas.elem != nil { - raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) } else if cas.kind == caseSend { - raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) } } if msanenabled { @@ -391,7 +409,7 @@ bufrecv: // can receive from buffer if raceenabled { if cas.elem != nil { - raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) } raceacquire(chanbuf(c, c.recvx)) racerelease(chanbuf(c, c.recvx)) @@ -418,7 +436,7 @@ bufsend: if raceenabled { raceacquire(chanbuf(c, c.sendx)) racerelease(chanbuf(c, c.sendx)) - raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) } if msanenabled { msanread(cas.elem, c.elemtype.size) @@ -456,7 +474,7 @@ rclose: send: // can send to a sleeping receiver (sg) if raceenabled { - raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) } if msanenabled { msanread(cas.elem, c.elemtype.size) @@ -519,12 +537,18 @@ func reflect_rselect(cases []runtimeSelect) (int, bool) { case selectRecv: sel[i] = scase{kind: caseRecv, c: rc.ch, elem: rc.val} } - if raceenabled || msanenabled { - selectsetpc(&sel[i]) + } + + var pc0 *uintptr + if raceenabled { + pcs := make([]uintptr, len(cases)) + for i := range pcs { + selectsetpc(&pcs[i]) } + pc0 = &pcs[0] } - return selectgo(&sel[0], &order[0], len(cases)) + return selectgo(&sel[0], &order[0], pc0, len(cases)) } func (q *waitq) dequeueSudoG(sgp *sudog) { -- cgit v1.3 From fe23ba4a145ce8465d16ea2a92b9a7e96e15c28e Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 16:19:15 -0700 Subject: runtime: eliminate scase.kind field Currently, we include a "kind" field on scase to distinguish the three kinds of cases in a select statement: sends, receives, and defaults. This commit removes by kind field by instead arranging for the compiler to always place sends before receives, and to provide their counts separately. It also passes an explicit "block bool" parameter to avoid needing to include a default case in the array. It's safe to shuffle cases like this because the runtime will randomize the order they're polled in anyway. Fixes #40410. Change-Id: Iaeaed4cf7bddd576d78f2c863bd91a03a5c82df2 Reviewed-on: https://go-review.googlesource.com/c/go/+/245125 Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/builtin.go | 2 +- src/cmd/compile/internal/gc/builtin/runtime.go | 2 +- src/cmd/compile/internal/gc/select.go | 108 +++++++++++++------------ src/reflect/all_test.go | 8 ++ src/runtime/select.go | 104 +++++++++++++----------- 5 files changed, 123 insertions(+), 101 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index eafdb0ebe7..861ffaaa5b 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -302,7 +302,7 @@ func runtimeTypes() []*types.Type { typs[96] = types.NewPtr(typs[6]) typs[97] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[96]), anonfield(typs[84])}, []*Node{anonfield(typs[6])}) typs[98] = functype(nil, []*Node{anonfield(typs[63])}, nil) - typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) + typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[6])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])}) typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])}) typs[102] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])}) diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index 25f86efdd6..635da80f7c 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -170,7 +170,7 @@ func selectnbrecv(elem *any, hchan <-chan any) bool func selectnbrecv2(elem *any, received *bool, hchan <-chan any) bool func selectsetpc(pc *uintptr) -func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, ncases int) (int, bool) +func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, nsends int, nrecvs int, block bool) (int, bool) func block() func makeslice(typ *byte, len int, cap int) unsafe.Pointer diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index 8eb31eb5c1..bae7ed30e2 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -106,18 +106,16 @@ func walkselect(sel *Node) { } func walkselectcases(cases *Nodes) []*Node { - n := cases.Len() + ncas := cases.Len() sellineno := lineno // optimization: zero-case select - if n == 0 { + if ncas == 0 { return []*Node{mkcall("block", nil, nil)} } // optimization: one-case select: single op. - // TODO(rsc): Reenable optimization once order.go can handle it. - // golang.org/issue/7672. - if n == 1 { + if ncas == 1 { cas := cases.First() setlineno(cas) l := cas.Ninit.Slice() @@ -178,10 +176,12 @@ func walkselectcases(cases *Nodes) []*Node { // convert case value arguments to addresses. // this rewrite is used by both the general code and the next optimization. + var dflt *Node for _, cas := range cases.Slice() { setlineno(cas) n := cas.Left if n == nil { + dflt = cas continue } switch n.Op { @@ -202,15 +202,10 @@ func walkselectcases(cases *Nodes) []*Node { } // optimization: two-case select but one is default: single non-blocking op. - if n == 2 && (cases.First().Left == nil || cases.Second().Left == nil) { - var cas *Node - var dflt *Node - if cases.First().Left == nil { + if ncas == 2 && dflt != nil { + cas := cases.First() + if cas == dflt { cas = cases.Second() - dflt = cases.First() - } else { - dflt = cases.Second() - cas = cases.First() } n := cas.Left @@ -257,74 +252,73 @@ func walkselectcases(cases *Nodes) []*Node { return []*Node{r, nod(OBREAK, nil, nil)} } + if dflt != nil { + ncas-- + } + casorder := make([]*Node, ncas) + nsends, nrecvs := 0, 0 + var init []*Node // generate sel-struct lineno = sellineno - selv := temp(types.NewArray(scasetype(), int64(n))) + selv := temp(types.NewArray(scasetype(), int64(ncas))) r := nod(OAS, selv, nil) r = typecheck(r, ctxStmt) init = append(init, r) - order := temp(types.NewArray(types.Types[TUINT16], 2*int64(n))) + order := temp(types.NewArray(types.Types[TUINT16], 2*int64(ncas))) r = nod(OAS, order, nil) r = typecheck(r, ctxStmt) init = append(init, r) var pc0, pcs *Node if flag_race { - pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(n))) + pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(ncas))) pc0 = typecheck(nod(OADDR, nod(OINDEX, pcs, nodintconst(0)), nil), ctxExpr) } else { pc0 = nodnil() } // register cases - for i, cas := range cases.Slice() { + for _, cas := range cases.Slice() { setlineno(cas) init = append(init, cas.Ninit.Slice()...) cas.Ninit.Set(nil) - // Keep in sync with runtime/select.go. - const ( - caseNil = iota - caseRecv - caseSend - caseDefault - ) + n := cas.Left + if n == nil { // default: + continue + } + var i int var c, elem *Node - var kind int64 = caseDefault - - if n := cas.Left; n != nil { - init = append(init, n.Ninit.Slice()...) - - switch n.Op { - default: - Fatalf("select %v", n.Op) - case OSEND: - kind = caseSend - c = n.Left - elem = n.Right - case OSELRECV, OSELRECV2: - kind = caseRecv - c = n.Right.Left - elem = n.Left - } + switch n.Op { + default: + Fatalf("select %v", n.Op) + case OSEND: + i = nsends + nsends++ + c = n.Left + elem = n.Right + case OSELRECV, OSELRECV2: + nrecvs++ + i = ncas - nrecvs + c = n.Right.Left + elem = n.Left } + casorder[i] = cas + setField := func(f string, val *Node) { r := nod(OAS, nodSym(ODOT, nod(OINDEX, selv, nodintconst(int64(i))), lookup(f)), val) r = typecheck(r, ctxStmt) init = append(init, r) } - setField("kind", nodintconst(kind)) - if c != nil { - c = convnop(c, types.Types[TUNSAFEPTR]) - setField("c", c) - } + c = convnop(c, types.Types[TUNSAFEPTR]) + setField("c", c) if elem != nil { elem = convnop(elem, types.Types[TUNSAFEPTR]) setField("elem", elem) @@ -337,6 +331,9 @@ func walkselectcases(cases *Nodes) []*Node { init = append(init, r) } } + if nsends+nrecvs != ncas { + Fatalf("walkselectcases: miscount: %v + %v != %v", nsends, nrecvs, ncas) + } // run the select lineno = sellineno @@ -345,7 +342,7 @@ func walkselectcases(cases *Nodes) []*Node { r = nod(OAS2, nil, nil) r.List.Set2(chosen, recvOK) fn := syslook("selectgo") - r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(n)))) + r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(nsends)), nodintconst(int64(nrecvs)), nodbool(dflt == nil))) r = typecheck(r, ctxStmt) init = append(init, r) @@ -357,14 +354,11 @@ func walkselectcases(cases *Nodes) []*Node { } // dispatch cases - for i, cas := range cases.Slice() { - setlineno(cas) - - cond := nod(OEQ, chosen, nodintconst(int64(i))) + dispatch := func(cond, cas *Node) { cond = typecheck(cond, ctxExpr) cond = defaultlit(cond, nil) - r = nod(OIF, cond, nil) + r := nod(OIF, cond, nil) if n := cas.Left; n != nil && n.Op == OSELRECV2 { x := nod(OAS, n.List.First(), recvOK) @@ -377,6 +371,15 @@ func walkselectcases(cases *Nodes) []*Node { init = append(init, r) } + if dflt != nil { + setlineno(dflt) + dispatch(nod(OLT, chosen, nodintconst(0)), dflt) + } + for i, cas := range casorder { + setlineno(cas) + dispatch(nod(OEQ, chosen, nodintconst(int64(i))), cas) + } + return init } @@ -395,7 +398,6 @@ func scasetype() *types.Type { scase = tostruct([]*Node{ namedfield("c", types.Types[TUNSAFEPTR]), namedfield("elem", types.Types[TUNSAFEPTR]), - namedfield("kind", types.Types[TUINT16]), }) scase.SetNoalg(true) } diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index ed2f225077..5a12699472 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -1725,6 +1725,14 @@ func TestSelectMaxCases(t *testing.T) { _, _, _ = Select(sCases) } +func TestSelectNop(t *testing.T) { + // "select { default: }" should always return the default case. + chosen, _, _ := Select([]SelectCase{{Dir: SelectDefault}}) + if chosen != 0 { + t.Fatalf("expected Select to return 0, but got %#v", chosen) + } +} + func BenchmarkSelect(b *testing.B) { channel := make(chan int) close(channel) diff --git a/src/runtime/select.go b/src/runtime/select.go index d7c7d9f26f..80768b285b 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -12,23 +12,12 @@ import ( const debugSelect = false -// scase.kind values. -// Known to compiler. -// Changes here must also be made in src/cmd/compile/internal/gc/select.go's walkselectcases. -const ( - caseNil = iota - caseRecv - caseSend - caseDefault -) - // Select case descriptor. // Known to compiler. // Changes here must also be made in src/cmd/internal/gc/select.go's scasetype. type scase struct { c *hchan // chan elem unsafe.Pointer // data element - kind uint16 } var ( @@ -115,7 +104,7 @@ func block() { // ordinal position of its respective select{recv,send,default} call. // Also, if the chosen scase was a receive operation, it reports whether // a value was received. -func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) { +func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, block bool) (int, bool) { if debugSelect { print("select: cas0=", cas0, "\n") } @@ -125,6 +114,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) cas1 := (*[1 << 16]scase)(unsafe.Pointer(cas0)) order1 := (*[1 << 17]uint16)(unsafe.Pointer(order0)) + ncases := nsends + nrecvs scases := cas1[:ncases:ncases] pollorder := order1[:ncases:ncases] lockorder := order1[ncases:][:ncases:ncases] @@ -158,16 +148,12 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) // optimizing (and needing to test). // generate permuted order - dfli := -1 norder := 0 for i := range scases { cas := &scases[i] // Omit cases without channels from the poll and lock orders. if cas.c == nil { - if cas.kind == caseDefault { - dfli = i - } cas.elem = nil // allow GC continue } @@ -250,8 +236,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) cas = &scases[casi] c = cas.c - switch cas.kind { - case caseRecv: + if casi >= nsends { sg = c.sendq.dequeue() if sg != nil { goto recv @@ -262,8 +247,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) if c.closed != 0 { goto rclose } - - case caseSend: + } else { if raceenabled { racereadpc(c.raceaddr(), casePC(casi), chansendpc) } @@ -280,9 +264,9 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) } } - if dfli >= 0 { + if !block { selunlock(scases, lockorder) - casi = dfli + casi = -1 goto retc } @@ -311,12 +295,10 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) *nextp = sg nextp = &sg.waitlink - switch cas.kind { - case caseRecv: - c.recvq.enqueue(sg) - - case caseSend: + if casi < nsends { c.sendq.enqueue(sg) + } else { + c.recvq.enqueue(sg) } } @@ -359,7 +341,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) } } else { c = k.c - if k.kind == caseSend { + if int(casei) < nsends { c.sendq.dequeueSudoG(sglist) } else { c.recvq.dequeueSudoG(sglist) @@ -378,27 +360,29 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) c = cas.c if debugSelect { - print("wait-return: cas0=", cas0, " c=", c, " cas=", cas, " kind=", cas.kind, "\n") + print("wait-return: cas0=", cas0, " c=", c, " cas=", cas, " send=", casi < nsends, "\n") } - if cas.kind == caseRecv { + if casi < nsends { + if !caseSuccess { + goto sclose + } + } else { recvOK = caseSuccess - } else if cas.kind == caseSend && !caseSuccess { - goto sclose } if raceenabled { - if cas.kind == caseRecv && cas.elem != nil { - raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) - } else if cas.kind == caseSend { + if casi < nsends { raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) + } else if cas.elem != nil { + raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) } } if msanenabled { - if cas.kind == caseRecv && cas.elem != nil { - msanwrite(cas.elem, c.elemtype.size) - } else if cas.kind == caseSend { + if casi < nsends { msanread(cas.elem, c.elemtype.size) + } else if cas.elem != nil { + msanwrite(cas.elem, c.elemtype.size) } } @@ -526,29 +510,57 @@ func reflect_rselect(cases []runtimeSelect) (int, bool) { block() } sel := make([]scase, len(cases)) - order := make([]uint16, 2*len(cases)) - for i := range cases { - rc := &cases[i] + orig := make([]int, len(cases)) + nsends, nrecvs := 0, 0 + dflt := -1 + for i, rc := range cases { + var j int switch rc.dir { case selectDefault: - sel[i] = scase{kind: caseDefault} + dflt = i + continue case selectSend: - sel[i] = scase{kind: caseSend, c: rc.ch, elem: rc.val} + j = nsends + nsends++ case selectRecv: - sel[i] = scase{kind: caseRecv, c: rc.ch, elem: rc.val} + nrecvs++ + j = len(cases) - nrecvs } + + sel[j] = scase{c: rc.ch, elem: rc.val} + orig[j] = i } + // Only a default case. + if nsends+nrecvs == 0 { + return dflt, false + } + + // Compact sel and orig if necessary. + if nsends+nrecvs < len(cases) { + copy(sel[nsends:], sel[len(cases)-nrecvs:]) + copy(orig[nsends:], orig[len(cases)-nrecvs:]) + } + + order := make([]uint16, 2*(nsends+nrecvs)) var pc0 *uintptr if raceenabled { - pcs := make([]uintptr, len(cases)) + pcs := make([]uintptr, nsends+nrecvs) for i := range pcs { selectsetpc(&pcs[i]) } pc0 = &pcs[0] } - return selectgo(&sel[0], &order[0], pc0, len(cases)) + chosen, recvOK := selectgo(&sel[0], &order[0], pc0, nsends, nrecvs, dflt == -1) + + // Translate chosen back to caller's ordering. + if chosen < 0 { + chosen = dflt + } else { + chosen = orig[chosen] + } + return chosen, recvOK } func (q *waitq) dequeueSudoG(sgp *sudog) { -- cgit v1.3 From a40171857595db60e95a04b64aad8ba262cf64a7 Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 16:45:48 -0700 Subject: cmd/compile/internal/gc: cleanup walkselectcases slightly Remove some unnecessary code. Most significantly, we can skip testing "if ch == nil { block() }", because this is already the semantics implied by normal send/receive operations. Updates #40410. Change-Id: I4acd33383cc876719fc3b998d85244d4ac1ff9d9 Reviewed-on: https://go-review.googlesource.com/c/go/+/245126 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Cuong Manh Le --- src/cmd/compile/internal/gc/select.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index bae7ed30e2..3812a0e1fa 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -123,17 +123,14 @@ func walkselectcases(cases *Nodes) []*Node { n := cas.Left l = append(l, n.Ninit.Slice()...) n.Ninit.Set(nil) - var ch *Node switch n.Op { default: Fatalf("select %v", n.Op) - // ok already case OSEND: - ch = n.Left + // already ok case OSELRECV, OSELRECV2: - ch = n.Right.Left if n.Op == OSELRECV || n.List.Len() == 0 { if n.Left == nil { n = n.Right @@ -157,16 +154,7 @@ func walkselectcases(cases *Nodes) []*Node { n = typecheck(n, ctxStmt) } - // if ch == nil { block() }; n; - a := nod(OIF, nil, nil) - - a.Left = nod(OEQ, ch, nodnil()) - var ln Nodes - ln.Set(l) - a.Nbody.Set1(mkcall("block", nil, &ln)) - l = ln.Slice() - a = typecheck(a, ctxStmt) - l = append(l, a, n) + l = append(l, n) } l = append(l, cas.Nbody.Slice()...) @@ -223,8 +211,6 @@ func walkselectcases(cases *Nodes) []*Node { case OSELRECV: // if selectnbrecv(&v, c) { body } else { default body } - r = nod(OIF, nil, nil) - r.Ninit.Set(cas.Ninit.Slice()) ch := n.Right.Left elem := n.Left if elem == nil { @@ -234,8 +220,6 @@ func walkselectcases(cases *Nodes) []*Node { case OSELRECV2: // if selectnbrecv2(&v, &received, c) { body } else { default body } - r = nod(OIF, nil, nil) - r.Ninit.Set(cas.Ninit.Slice()) ch := n.Right.Left elem := n.Left if elem == nil { -- cgit v1.3 From e7c7ce646f37b260fe5a5635bc52243d28125dd8 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Mon, 17 Aug 2020 16:14:48 -0500 Subject: cmd/compile: combine multiply/add into maddld on ppc64le/power9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new lowering rule to match and replace such instances with the MADDLD instruction available on power9 where possible. Likewise, this plumbs in a new ppc64 ssa opcode to house the newly generated MADDLD instructions. When testing ed25519, this reduced binary size by 936B. Similarly, MADDLD combination occcurs in a few other less obvious cases such as division by constant. Testing of golang.org/x/crypto/ed25519 shows non-trivial speedup during keygeneration: name old time/op new time/op delta KeyGeneration 65.2µs ± 0% 63.1µs ± 0% -3.19% Signing 64.3µs ± 0% 64.4µs ± 0% +0.16% Verification 147µs ± 0% 147µs ± 0% +0.11% Similarly, this test binary has shrunk by 66488B. Change-Id: I077aeda7943119b41f07e4e62e44a648f16e4ad0 Reviewed-on: https://go-review.googlesource.com/c/go/+/248723 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Lynn Boger --- src/cmd/compile/internal/ppc64/ssa.go | 14 ++++++++++++++ src/cmd/compile/internal/ssa/gen/PPC64.rules | 3 +++ src/cmd/compile/internal/ssa/gen/PPC64Ops.go | 2 ++ src/cmd/compile/internal/ssa/opGen.go | 16 ++++++++++++++++ src/cmd/compile/internal/ssa/rewritePPC64.go | 21 +++++++++++++++++++++ test/codegen/arithmetic.go | 12 ++++++++---- 6 files changed, 64 insertions(+), 4 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go index 0efdd710fb..4d2ad48135 100644 --- a/src/cmd/compile/internal/ppc64/ssa.go +++ b/src/cmd/compile/internal/ppc64/ssa.go @@ -601,6 +601,20 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() + case ssa.OpPPC64MADDLD: + r := v.Reg() + r1 := v.Args[0].Reg() + r2 := v.Args[1].Reg() + r3 := v.Args[2].Reg() + // r = r1*r2 ± r3 + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = r1 + p.Reg = r2 + p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: r3}) + p.To.Type = obj.TYPE_REG + p.To.Reg = r + case ssa.OpPPC64FMADD, ssa.OpPPC64FMADDS, ssa.OpPPC64FMSUB, ssa.OpPPC64FMSUBS: r := v.Reg() r1 := v.Args[0].Reg() diff --git a/src/cmd/compile/internal/ssa/gen/PPC64.rules b/src/cmd/compile/internal/ssa/gen/PPC64.rules index fd28e10098..14942d50f9 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64.rules +++ b/src/cmd/compile/internal/ssa/gen/PPC64.rules @@ -11,6 +11,9 @@ (Sub32F ...) => (FSUBS ...) (Sub64F ...) => (FSUB ...) +// Combine 64 bit integer multiply and adds +(ADD l:(MULLD x y) z) && objabi.GOPPC64 >= 9 && l.Uses == 1 && clobber(l) => (MADDLD x y z) + (Mod16 x y) => (Mod32 (SignExt16to32 x) (SignExt16to32 y)) (Mod16u x y) => (Mod32u (ZeroExt16to32 x) (ZeroExt16to32 y)) (Mod8 x y) => (Mod32 (SignExt8to32 x) (SignExt8to32 y)) diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go index 0261dc283b..825d0faf34 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go @@ -137,6 +137,7 @@ func init() { gp01 = regInfo{inputs: nil, outputs: []regMask{gp}} gp11 = regInfo{inputs: []regMask{gp | sp | sb}, outputs: []regMask{gp}} gp21 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp}} + gp31 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp}} gp22 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp, gp}} gp32 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp, gp}} gp1cr = regInfo{inputs: []regMask{gp | sp | sb}} @@ -179,6 +180,7 @@ func init() { {name: "MULLD", argLength: 2, reg: gp21, asm: "MULLD", typ: "Int64", commutative: true}, // arg0*arg1 (signed 64-bit) {name: "MULLW", argLength: 2, reg: gp21, asm: "MULLW", typ: "Int32", commutative: true}, // arg0*arg1 (signed 32-bit) + {name: "MADDLD", argLength: 3, reg: gp31, asm: "MADDLD", typ: "Int64"}, // (arg0*arg1)+arg2 (signed 64-bit) {name: "MULHD", argLength: 2, reg: gp21, asm: "MULHD", commutative: true}, // (arg0 * arg1) >> 64, signed {name: "MULHW", argLength: 2, reg: gp21, asm: "MULHW", commutative: true}, // (arg0 * arg1) >> 32, signed diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index df2a27368b..4cd72799e8 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -1832,6 +1832,7 @@ const ( OpPPC64FSUBS OpPPC64MULLD OpPPC64MULLW + OpPPC64MADDLD OpPPC64MULHD OpPPC64MULHW OpPPC64MULHDU @@ -24374,6 +24375,21 @@ var opcodeTable = [...]opInfo{ }, }, }, + { + name: "MADDLD", + argLen: 3, + asm: ppc64.AMADDLD, + reg: regInfo{ + inputs: []inputInfo{ + {0, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + {1, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + {2, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + }, + outputs: []outputInfo{ + {0, 1073733624}, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + }, + }, + }, { name: "MULHD", argLen: 2, diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go index 37b75cc58a..7704b80dc6 100644 --- a/src/cmd/compile/internal/ssa/rewritePPC64.go +++ b/src/cmd/compile/internal/ssa/rewritePPC64.go @@ -3852,6 +3852,27 @@ func rewriteValuePPC64_OpPPC64ADD(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (ADD l:(MULLD x y) z) + // cond: objabi.GOPPC64 >= 9 && l.Uses == 1 && clobber(l) + // result: (MADDLD x y z) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + l := v_0 + if l.Op != OpPPC64MULLD { + continue + } + y := l.Args[1] + x := l.Args[0] + z := v_1 + if !(objabi.GOPPC64 >= 9 && l.Uses == 1 && clobber(l)) { + continue + } + v.reset(OpPPC64MADDLD) + v.AddArg3(x, y, z) + return true + } + break + } // match: (ADD (SLDconst x [c]) (SRDconst x [d])) // cond: d == 64-c // result: (ROTLconst [c] x) diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index 9f30ec8ce4..45fdb68903 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -253,16 +253,20 @@ func Divisible(n1 uint, n2 int) (bool, bool, bool, bool) { // 386:"IMUL3L\t[$]-1431655765","ADDL\t[$]715827882","ROLL\t[$]31",-"DIVQ" // arm64:"MUL","ADD\t[$]3074457345618258602","ROR",-"DIV" // arm:"MUL","ADD\t[$]715827882",-".*udiv" - // ppc64:"MULLD","ADD","ROTL\t[$]63" - // ppc64le:"MULLD","ADD","ROTL\t[$]63" + // ppc64/power8:"MULLD","ADD","ROTL\t[$]63" + // ppc64le/power8:"MULLD","ADD","ROTL\t[$]63" + // ppc64/power9:"MADDLD","ROTL\t[$]63" + // ppc64le/power9:"MADDLD","ROTL\t[$]63" evenS := n2%6 == 0 // amd64:"IMULQ","ADD",-"ROLQ",-"DIVQ" // 386:"IMUL3L\t[$]678152731","ADDL\t[$]113025455",-"ROLL",-"DIVQ" // arm64:"MUL","ADD\t[$]485440633518672410",-"ROR",-"DIV" // arm:"MUL","ADD\t[$]113025455",-".*udiv" - // ppc64:"MULLD","ADD",-"ROTL" - // ppc64le:"MULLD","ADD",-"ROTL" + // ppc64/power8:"MULLD","ADD",-"ROTL" + // ppc64/power9:"MADDLD",-"ROTL" + // ppc64le/power8:"MULLD","ADD",-"ROTL" + // ppc64le/power9:"MADDLD",-"ROTL" oddS := n2%19 == 0 return evenU, oddU, evenS, oddS -- cgit v1.3 From 84a62453e5c01df3f7d0c48d9aca32832c2052c1 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 8 May 2020 17:12:45 -0400 Subject: cmd/compile: remove unnecessary error condition on reading fingerprint io.ReadFull guarantees n == len(buf) if and only if err == nil, so the length check is redundant. Change-Id: I15bff97868e27a65648acd791883cac8dab77630 Reviewed-on: https://go-review.googlesource.com/c/go/+/232988 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Dmitri Shuralyov --- src/cmd/compile/internal/gc/iimport.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/iimport.go b/src/cmd/compile/internal/gc/iimport.go index 0eeb047c06..4169222c14 100644 --- a/src/cmd/compile/internal/gc/iimport.go +++ b/src/cmd/compile/internal/gc/iimport.go @@ -191,9 +191,9 @@ func iimport(pkg *types.Pkg, in *bio.Reader) (fingerprint goobj.FingerprintType) } } - // Fingerprint - n, err := io.ReadFull(in, fingerprint[:]) - if err != nil || n != len(fingerprint) { + // Fingerprint. + _, err = io.ReadFull(in, fingerprint[:]) + if err != nil { yyerror("import %s: error reading fingerprint", pkg.Path) errorexit() } -- cgit v1.3 From ac875bc923db2b7350f244f06a06557e6fd97e05 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Sat, 15 Aug 2020 00:44:58 +0700 Subject: cmd/compile: don't bother to declare closure inside redeclared func Fixes #17758 Change-Id: I75f5dc5be85fd8a6791ac89dfc0681be759cca36 Reviewed-on: https://go-review.googlesource.com/c/go/+/248517 Run-TryBot: Cuong Manh Le Reviewed-by: Matthew Dempsky TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/closure.go | 12 +++++++++++- test/fixedbugs/issue17758.go | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue17758.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/closure.go b/src/cmd/compile/internal/gc/closure.go index 04fb7d5495..23e48939b4 100644 --- a/src/cmd/compile/internal/gc/closure.go +++ b/src/cmd/compile/internal/gc/closure.go @@ -108,7 +108,17 @@ func typecheckclosure(clo *Node, top int) { xfunc.Func.Nname.Sym = closurename(Curfn) disableExport(xfunc.Func.Nname.Sym) - declare(xfunc.Func.Nname, PFUNC) + if xfunc.Func.Nname.Sym.Def != nil { + // The only case we can reach here is when the outer function was redeclared. + // In that case, don't bother to redeclare the closure. Otherwise, we will get + // a spurious error message, see #17758. While we are here, double check that + // we already reported other error. + if nsavederrors+nerrors == 0 { + Fatalf("unexpected symbol collision %v", xfunc.Func.Nname.Sym) + } + } else { + declare(xfunc.Func.Nname, PFUNC) + } xfunc = typecheck(xfunc, ctxStmt) // Type check the body now, but only if we're inside a function. diff --git a/test/fixedbugs/issue17758.go b/test/fixedbugs/issue17758.go new file mode 100644 index 0000000000..e7f2f3af91 --- /dev/null +++ b/test/fixedbugs/issue17758.go @@ -0,0 +1,17 @@ +// errorcheck + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func foo() { + _ = func() {} +} + +func foo() { // ERROR "foo redeclared in this block" + _ = func() {} +} + +func main() {} -- cgit v1.3 From 31da1d993a498acefcf3dd5fdfbefa8eec132791 Mon Sep 17 00:00:00 2001 From: Tao Qingyun Date: Wed, 19 Aug 2020 01:38:43 +0000 Subject: cmd/internal/objfile: cache computation of goobj.Arch Change-Id: I23774cf185e5fa6b89398001cd0655fb0c5bdb46 GitHub-Last-Rev: ca8cae2469b5fad84bd636a3305a484dfdcb0db2 GitHub-Pull-Request: golang/go#40877 Reviewed-on: https://go-review.googlesource.com/c/go/+/249180 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/internal/archive/archive.go | 6 ++++++ src/cmd/internal/objfile/goobj.go | 32 +++++++++++++------------------- 2 files changed, 19 insertions(+), 19 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/internal/archive/archive.go b/src/cmd/internal/archive/archive.go index db67ce424b..c1661d7711 100644 --- a/src/cmd/internal/archive/archive.go +++ b/src/cmd/internal/archive/archive.go @@ -17,6 +17,7 @@ import ( "log" "os" "strconv" + "strings" "time" "unicode/utf8" ) @@ -83,6 +84,7 @@ func (e *Entry) String() string { type GoObj struct { TextHeader []byte + Arch string Data } @@ -404,6 +406,10 @@ func (r *objReader) parseObject(o *GoObj, size int64) error { } } o.TextHeader = h + hs := strings.Fields(string(h)) + if len(hs) >= 4 { + o.Arch = hs[3] + } o.Offset = r.offset o.Size = size - int64(len(h)) diff --git a/src/cmd/internal/objfile/goobj.go b/src/cmd/internal/objfile/goobj.go index e838f58aed..af9ada3324 100644 --- a/src/cmd/internal/objfile/goobj.go +++ b/src/cmd/internal/objfile/goobj.go @@ -17,13 +17,13 @@ import ( "fmt" "io" "os" - "strings" ) type goobjFile struct { goobj *archive.GoObj r *goobj.Reader f *os.File + arch *sys.Arch } func openGoFile(f *os.File) (*File, error) { @@ -45,9 +45,16 @@ L: return nil, err } r := goobj.NewReaderFromBytes(b, false) + var arch *sys.Arch + for _, a := range sys.Archs { + if a.Name == e.Obj.Arch { + arch = a + break + } + } entries = append(entries, &Entry{ name: e.Name, - raw: &goobjFile{e.Obj, r, f}, + raw: &goobjFile{e.Obj, r, f, arch}, }) continue case archive.EntryNativeObj: @@ -223,17 +230,8 @@ func (f *goobjFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) // Returns "",0,nil if unknown. // This function implements the Liner interface in preference to pcln() above. func (f *goobjFile) PCToLine(pc uint64) (string, int, *gosym.Func) { - // TODO: this is really inefficient. Binary search? Memoize last result? r := f.r - var arch *sys.Arch - archname := f.goarch() - for _, a := range sys.Archs { - if a.Name == archname { - arch = a - break - } - } - if arch == nil { + if f.arch == nil { return "", 0, nil } pcdataBase := r.PcdataBase() @@ -264,10 +262,10 @@ func (f *goobjFile) PCToLine(pc uint64) (string, int, *gosym.Func) { lengths := info.ReadFuncInfoLengths(b) off, end := info.ReadPcline(b) pcline := r.BytesAt(pcdataBase+off, int(end-off)) - line := int(pcValue(pcline, pc-addr, arch)) + line := int(pcValue(pcline, pc-addr, f.arch)) off, end = info.ReadPcfile(b) pcfile := r.BytesAt(pcdataBase+off, int(end-off)) - fileID := pcValue(pcfile, pc-addr, arch) + fileID := pcValue(pcfile, pc-addr, f.arch) globalFileID := info.ReadFile(b, lengths.FileOff, uint32(fileID)) fileName := r.File(int(globalFileID)) // Note: we provide only the name in the Func structure. @@ -332,11 +330,7 @@ func (f *goobjFile) text() (textStart uint64, text []byte, err error) { } func (f *goobjFile) goarch() string { - hs := strings.Fields(string(f.goobj.TextHeader)) - if len(hs) >= 4 { - return hs[3] - } - return "" + return f.goobj.Arch } func (f *goobjFile) loadAddress() (uint64, error) { -- cgit v1.3 From 8d91d736e2eb518e0a95eb47ae463c0dd80827c4 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 18 Aug 2020 12:01:41 -0400 Subject: cmd/dist: ignore _test.s files in bootstrap Ignore all _test.s files in bootstrap, not only the ARM64 one. They are for testing only. Fixes #40855. Change-Id: I00e6b4ab5349e317c9ad3a503997de85aed49373 Reviewed-on: https://go-review.googlesource.com/c/go/+/249018 Reviewed-by: Ian Lance Taylor --- src/cmd/dist/buildtool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'src/cmd') diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index 40d28b535b..0b50f66c32 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -113,14 +113,15 @@ var ignorePrefixes = []string{ // File suffixes that use build tags introduced since Go 1.4. // These must not be copied into the bootstrap build directory. +// Also igonore test files. var ignoreSuffixes = []string{ "_arm64.s", - "_arm64_test.s", "_arm64.go", "_riscv64.s", "_riscv64.go", "_wasm.s", "_wasm.go", + "_test.s", } func bootstrapBuildTools() { -- cgit v1.3 From 01aad9ea939fed313d5c51778485349435302ead Mon Sep 17 00:00:00 2001 From: diaxu01 Date: Fri, 5 Jun 2020 03:53:53 +0000 Subject: cmd/compile: Optimize ARM64's code with EON This patch fuses pattern '(MVN (XOR x y))' into '(EON x y)'. Change-Id: I269c98ce198d51a4945ce8bd0e1024acbd1b7609 Reviewed-on: https://go-review.googlesource.com/c/go/+/239638 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/ssa/gen/ARM64.rules | 1 + src/cmd/compile/internal/ssa/rewriteARM64.go | 12 ++++++++++++ test/codegen/bits.go | 13 +++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/ssa/gen/ARM64.rules b/src/cmd/compile/internal/ssa/gen/ARM64.rules index 27959d01fc..80e8c7137b 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/gen/ARM64.rules @@ -1323,6 +1323,7 @@ (AND x (MVN y)) -> (BIC x y) (XOR x (MVN y)) -> (EON x y) (OR x (MVN y)) -> (ORN x y) +(MVN (XOR x y)) -> (EON x y) (CSEL {cc} x (MOVDconst [0]) flag) -> (CSEL0 {cc} x flag) (CSEL {cc} (MOVDconst [0]) y flag) -> (CSEL0 {arm64Negate(cc.(Op))} y flag) (SUB x (SUB y z)) -> (SUB (ADD x z) y) diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go index 023d9908c2..842eddbf4a 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM64.go +++ b/src/cmd/compile/internal/ssa/rewriteARM64.go @@ -14593,6 +14593,18 @@ func rewriteValueARM64_OpARM64MULW(v *Value) bool { } func rewriteValueARM64_OpARM64MVN(v *Value) bool { v_0 := v.Args[0] + // match: (MVN (XOR x y)) + // result: (EON x y) + for { + if v_0.Op != OpARM64XOR { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + v.reset(OpARM64EON) + v.AddArg2(x, y) + return true + } // match: (MVN (MOVDconst [c])) // result: (MOVDconst [^c]) for { diff --git a/test/codegen/bits.go b/test/codegen/bits.go index 0a5428b55a..398dd84e9e 100644 --- a/test/codegen/bits.go +++ b/test/codegen/bits.go @@ -310,9 +310,18 @@ func op_bic(x, y uint32) uint32 { return x &^ y } -func op_eon(x, y uint32) uint32 { +func op_eon(x, y, z uint32, a []uint32, n, m uint64) uint64 { + // arm64:`EON\t`,-`EOR`,-`MVN` + a[0] = x ^ (y ^ 0xffffffff) + + // arm64:`EON\t`,-`EOR`,-`MVN` + a[1] = ^(y ^ z) + // arm64:`EON\t`,-`XOR` - return x ^ ^y + a[2] = x ^ ^z + + // arm64:`EON\t`,-`EOR`,-`MVN` + return n ^ (m ^ 0xffffffffffffffff) } func op_orn(x, y uint32) uint32 { -- cgit v1.3 From 64350f1eabeb688e997c6cd0c103e21c02ab2a46 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Tue, 18 Aug 2020 16:26:42 -0500 Subject: cmd/asm,cmd/internal/obj/ppc64: add {l,st}xvx power9 instructions These are the indexed vsx load operations with the same endian and alignment benefits of {l,st}vx. Likewise, cleanup redundant comments in op{load,store}x and fix ISA 3.0 typos nearby. Change-Id: Ie1ace17c6150cf9168a834e435114028ff6eb07c Reviewed-on: https://go-review.googlesource.com/c/go/+/249025 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Lynn Boger --- src/cmd/asm/internal/asm/testdata/ppc64.s | 2 ++ src/cmd/asm/internal/asm/testdata/ppc64enc.s | 2 ++ src/cmd/internal/obj/ppc64/asm9.go | 30 ++++++++++++++++------------ 3 files changed, 21 insertions(+), 13 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/asm/internal/asm/testdata/ppc64.s b/src/cmd/asm/internal/asm/testdata/ppc64.s index b3736bf6a4..ba64d84a35 100644 --- a/src/cmd/asm/internal/asm/testdata/ppc64.s +++ b/src/cmd/asm/internal/asm/testdata/ppc64.s @@ -1037,6 +1037,7 @@ label1: // VSX load with length X-form (also left-justified) LXVL R3,R4, VS0 LXVLL R3,R4, VS0 + LXVX R3,R4, VS0 // VSX load, DQ-form // DQ(RA), XS produces // XS, DQ(RA) @@ -1060,6 +1061,7 @@ label1: // VSX store with length, X-form (also left-justified) STXVL VS0, R3,R4 STXVLL VS0, R3,R4 + STXVX VS0, R3,R4 // VSX move from VSR, XX1-form // XS,RA produces diff --git a/src/cmd/asm/internal/asm/testdata/ppc64enc.s b/src/cmd/asm/internal/asm/testdata/ppc64enc.s index 07a8a540cd..10a05ec402 100644 --- a/src/cmd/asm/internal/asm/testdata/ppc64enc.s +++ b/src/cmd/asm/internal/asm/testdata/ppc64enc.s @@ -595,11 +595,13 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0 LXV 16(R3), VS1 // f4230011 LXVL R3, R4, VS1 // 7c23221a LXVLL R3, R4, VS1 // 7c23225a + LXVX R3, R4, VS1 // 7c232218 LXSDX (R3)(R4), VS1 // 7c241c98 STXVD2X VS1, (R3)(R4) // 7c241f98 STXV VS1,16(R3) // f4230015 STXVL VS1, R3, R4 // 7c23231a STXVLL VS1, R3, R4 // 7c23235a + STXVX VS1, R3, R4 // 7c232318 STXSDX VS1, (R3)(R4) // 7c241d98 LXSIWAX (R3)(R4), VS1 // 7c241898 STXSIWX VS1, (R3)(R4) // 7c241918 diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index 238ca8f0b7..3c82477fc4 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -1584,8 +1584,9 @@ func buildop(ctxt *obj.Link) { case ALXV: /* lxv */ opset(ALXV, r0) - case ALXVL: /* lxvl */ + case ALXVL: /* lxvl, lxvll, lxvx */ opset(ALXVLL, r0) + opset(ALXVX, r0) case ASTXVD2X: /* stxvd2x, stxvdsx, stxvw4x, stxvh8x, stxvb16x */ opset(ASTXVW4X, r0) @@ -1595,8 +1596,9 @@ func buildop(ctxt *obj.Link) { case ASTXV: /* stxv */ opset(ASTXV, r0) - case ASTXVL: /* stxvl, stxvll */ + case ASTXVL: /* stxvl, stxvll, stvx */ opset(ASTXVLL, r0) + opset(ASTXVX, r0) case ALXSDX: /* lxsdx */ opset(ALXSDX, r0) @@ -5020,11 +5022,13 @@ func (c *ctxt9) opload(a obj.As) uint32 { case AMOVW: return OPVCC(58, 0, 0, 0) | 1<<1 /* lwa */ case ALXV: - return OPDQ(61, 1, 0) /* lxv - ISA v3.00 */ + return OPDQ(61, 1, 0) /* lxv - ISA v3.0 */ case ALXVL: - return OPVXX1(31, 269, 0) /* lxvl - ISA v3.00 */ + return OPVXX1(31, 269, 0) /* lxvl - ISA v3.0 */ case ALXVLL: - return OPVXX1(31, 301, 0) /* lxvll - ISA v3.00 */ + return OPVXX1(31, 301, 0) /* lxvll - ISA v3.0 */ + case ALXVX: + return OPVXX1(31, 268, 0) /* lxvx - ISA v3.0 */ /* no AMOVWU */ case AMOVB, AMOVBZ: @@ -5122,8 +5126,6 @@ func (c *ctxt9) oploadx(a obj.As) uint32 { return OPVCC(31, 309, 0, 0) /* ldmx */ /* Vector (VMX/Altivec) instructions */ - /* ISA 2.03 enables these for PPC970. For POWERx processors, these */ - /* are enabled starting at POWER6 (ISA 2.05). */ case ALVEBX: return OPVCC(31, 7, 0, 0) /* lvebx - v2.03 */ case ALVEHX: @@ -5141,7 +5143,8 @@ func (c *ctxt9) oploadx(a obj.As) uint32 { /* End of vector instructions */ /* Vector scalar (VSX) instructions */ - /* ISA 2.06 enables these for POWER7. */ + case ALXVX: + return OPVXX1(31, 268, 0) /* lxvx - ISA v3.0 */ case ALXVD2X: return OPVXX1(31, 844, 0) /* lxvd2x - v2.06 */ case ALXVW4X: @@ -5208,6 +5211,8 @@ func (c *ctxt9) opstore(a obj.As) uint32 { return OPVXX1(31, 397, 0) /* stxvl ISA 3.0 */ case ASTXVLL: return OPVXX1(31, 429, 0) /* stxvll ISA 3.0 */ + case ASTXVX: + return OPVXX1(31, 396, 0) /* stxvx - ISA v3.0 */ } @@ -5271,8 +5276,6 @@ func (c *ctxt9) opstorex(a obj.As) uint32 { return OPVCC(31, 181, 0, 0) /* stdux */ /* Vector (VMX/Altivec) instructions */ - /* ISA 2.03 enables these for PPC970. For POWERx processors, these */ - /* are enabled starting at POWER6 (ISA 2.05). */ case ASTVEBX: return OPVCC(31, 135, 0, 0) /* stvebx - v2.03 */ case ASTVEHX: @@ -5286,15 +5289,16 @@ func (c *ctxt9) opstorex(a obj.As) uint32 { /* End of vector instructions */ /* Vector scalar (VSX) instructions */ - /* ISA 2.06 enables these for POWER7. */ + case ASTXVX: + return OPVXX1(31, 396, 0) /* stxvx - v3.0 */ case ASTXVD2X: return OPVXX1(31, 972, 0) /* stxvd2x - v2.06 */ case ASTXVW4X: return OPVXX1(31, 908, 0) /* stxvw4x - v2.06 */ case ASTXVH8X: - return OPVXX1(31, 940, 0) /* stxvh8x - v3.00 */ + return OPVXX1(31, 940, 0) /* stxvh8x - v3.0 */ case ASTXVB16X: - return OPVXX1(31, 1004, 0) /* stxvb16x - v3.00 */ + return OPVXX1(31, 1004, 0) /* stxvb16x - v3.0 */ case ASTXSDX: return OPVXX1(31, 716, 0) /* stxsdx - v2.06 */ -- cgit v1.3 From 18239be10a0b5caa1b3222b228ff590b1c036382 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 25 Jun 2020 19:11:28 -0400 Subject: cmd/go/internal: remove some users of par.Work par.Work is used in a number of places as a parallel work queue. This change replaces it with goroutines and channels in a number of simpler places where it's used. This is the same CL as golang.org/cl/240062 and golang.org/cl/248326 except for the following changes in convert.go (all line numbers from this CL), as well as fixing up imports in download.go: - On line 44, the "*" before modules.Versions is removed (we were trying to assign to a nil value on lines 72 and 73). - Line 64 is new, and ensures that we receive on the semaphore channel once the goroutine function exits. (The previous versions of this CL only received at the end of the function, ignoring the return point in the branch in the middle of the function.) - The semaphore channel receive right before line 74 is gone, replaced with the deferred receive above. - The if block at line 83 is new, accounting for cases where modfetch.ImportRepoRev returned an error in the goroutine, so that versions[i] is ignored. Change-Id: I0e33670bb2eb0a1e4d7a5fa693a471e61ffbc8b7 Reviewed-on: https://go-review.googlesource.com/c/go/+/249020 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 69 +++++++++++++++++++--------------- src/cmd/go/internal/modcmd/graph.go | 19 +++++----- src/cmd/go/internal/modconv/convert.go | 59 ++++++++++++++++------------- src/cmd/go/internal/modget/get.go | 41 +++++++++++++------- src/cmd/go/internal/modload/list.go | 37 +++++++++++------- 5 files changed, 134 insertions(+), 91 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 946e8ed3cf..2f42d4306b 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -8,12 +8,12 @@ import ( "context" "encoding/json" "os" + "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/par" + "cmd/go/internal/modfetch" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -102,33 +102,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } } - var mods []*moduleJSON - var work par.Work - listU := false - listVersions := false - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - work.Add(m) - } - - work.Do(10, func(item interface{}) { - m := item.(*moduleJSON) + downloadModule := func(m *moduleJSON) { var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -157,7 +131,42 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { m.Error = err.Error() return } - }) + } + + var mods []*moduleJSON + listU := false + listVersions := false + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + sem <- token{} + go func() { + downloadModule(m) + <-sem + }() + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } if *downloadJSON { for _, m := range mods { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 4853503fd4..6da12b9cab 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,7 +15,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -59,23 +58,25 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { return m.Path + "@" + m.Version } - // Note: using par.Work only to manage work queue. - // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - var work par.Work - work.Add(modload.Target) - work.Do(1, func(item interface{}) { - m := item.(module.Version) + seen := map[module.Version]bool{modload.Target: true} + queue := []module.Version{modload.Target} + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] list, _ := reqs.Required(m) for _, r := range list { - work.Add(r) + if !seen[r] { + queue = append(queue, r) + seen[r] = true + } out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - }) + } sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/src/cmd/go/internal/modconv/convert.go b/src/cmd/go/internal/modconv/convert.go index f465a9f395..5d4165c944 100644 --- a/src/cmd/go/internal/modconv/convert.go +++ b/src/cmd/go/internal/modconv/convert.go @@ -7,13 +7,12 @@ package modconv import ( "fmt" "os" + "runtime" "sort" "strings" - "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" - "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -42,46 +41,54 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var ( - work par.Work - mu sync.Mutex - need = make(map[string]string) - replace = make(map[string]*modfile.Replace) - ) + versions := make([]module.Version, len(mf.Require)) + replace := make(map[string]*modfile.Replace) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - for _, r := range mf.Require { + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for i, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - work.Add(re.New) - continue + m = re.New } - work.Add(r.Mod) + sem <- token{} + go func(i int, m module.Version) { + defer func() { <-sem }() + repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) + return + } + + path := repo.ModulePath() + versions[i].Path = path + versions[i].Version = info.Version + }(i, m) + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} } - work.Do(10, func(item interface{}) { - r := item.(module.Version) - repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) - return + need := map[string]string{} + for _, v := range versions { + if v.Path == "" { + continue } - mu.Lock() - path := repo.ModulePath() // Don't use semver.Max here; need to preserve +incompatible suffix. - if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { - need[path] = info.Version + if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { + need[v.Path] = v.Version } - mu.Unlock() - }) - - var paths []string + } + paths := make([]string, 0, len(need)) for path := range need { paths = append(paths, path) } diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 93a6bb54d5..d02c9a8da5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -21,7 +22,6 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/mvs" - "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" @@ -725,18 +725,8 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // reported. A map from module paths to queries is returned, which includes // queries and modOnly. func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { - var lookup par.Work - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - cache[q.querySpec] = q - lookup.Add(q) - } - } - lookup.Do(10, func(item interface{}) { - q := item.(*query) + runQuery := func(q *query) { if q.vers == "none" { // Wait for downgrade step. q.m = module.Version{Path: q.path, Version: "none"} @@ -747,7 +737,32 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer base.Errorf("go get %s: %v", q.arg, err) } q.m = m - }) + } + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, q := range queries { + if cached := cache[q.querySpec]; cached != nil { + *q = *cached + } else { + sem <- token{} + go func(q *query) { + runQuery(q) + <-sem + }(q) + } + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + // Add to cache after concurrent section to avoid races... + for _, q := range queries { + cache[q.querySpec] = q + } + base.ExitIfErrors() byPath := make(map[string]*query) diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 4768516e90..8db4d64706 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "os" + "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" - "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" @@ -22,24 +22,35 @@ import ( func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { mods := listModules(ctx, args, listVersions) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) if listU || listVersions { - var work par.Work for _, m := range mods { - work.Add(m) + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(m) + } + if listVersions { + addVersions(m) + } + <-sem + }() + } + + add(m) if m.Replace != nil { - work.Add(m.Replace) + add(m.Replace) } } - work.Do(10, func(item interface{}) { - m := item.(*modinfo.ModulePublic) - if listU { - addUpdate(m) - } - if listVersions { - addVersions(m) - } - }) } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + return mods } -- cgit v1.3 From 2d34f3d023586fc001d64b372bd709efaab54a18 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 19 Aug 2020 14:42:11 -0700 Subject: cmd/dist: fix tipo in comment Change-Id: I98fbf0dc94cf182adec5d414a56b9cc5126c38f4 Reviewed-on: https://go-review.googlesource.com/c/go/+/249437 Run-TryBot: Ian Lance Taylor Reviewed-by: Cherry Zhang TryBot-Result: Gobot Gobot --- src/cmd/dist/buildtool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/cmd') diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index 0b50f66c32..79eab24d29 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -113,7 +113,7 @@ var ignorePrefixes = []string{ // File suffixes that use build tags introduced since Go 1.4. // These must not be copied into the bootstrap build directory. -// Also igonore test files. +// Also ignore test files. var ignoreSuffixes = []string{ "_arm64.s", "_arm64.go", -- cgit v1.3 From 46ca7b5ee2a8582736f1ddac27d8660e1104c345 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 15 Aug 2020 08:08:37 -0700 Subject: cmd/internal/obj: stop removing NOPs from instruction stream This has already been done for s390x, ppc64. This CL is for all the other architectures. Fixes #40796 Change-Id: Idd1816e057df63022d47e99fa06617811d8c8489 Reviewed-on: https://go-review.googlesource.com/c/go/+/248684 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/internal/obj/arm/asm5.go | 3 ++ src/cmd/internal/obj/arm/obj5.go | 46 ------------------------------- src/cmd/internal/obj/arm64/asm7.go | 3 ++ src/cmd/internal/obj/arm64/obj7.go | 56 ++------------------------------------ src/cmd/internal/obj/mips/asm0.go | 3 ++ src/cmd/internal/obj/mips/obj0.go | 26 ++---------------- 6 files changed, 14 insertions(+), 123 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/internal/obj/arm/asm5.go b/src/cmd/internal/obj/arm/asm5.go index f66f8aaf84..7b7e42ee2e 100644 --- a/src/cmd/internal/obj/arm/asm5.go +++ b/src/cmd/internal/obj/arm/asm5.go @@ -327,6 +327,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_LCON, 0, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_LCON, C_NONE, C_ADDR, 0, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, + {obj.ANOP, C_FREG, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0, 0}, // same as ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0, 0}, // same as ABL {obj.AXXX, C_NONE, C_NONE, C_NONE, 0, 4, 0, 0, 0, 0}, diff --git a/src/cmd/internal/obj/arm/obj5.go b/src/cmd/internal/obj/arm/obj5.go index 008118c47b..86831f2b44 100644 --- a/src/cmd/internal/obj/arm/obj5.go +++ b/src/cmd/internal/obj/arm/obj5.go @@ -276,67 +276,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs - * expand RET - * expand BECOME pseudo */ - var q1 *obj.Prog - var q *obj.Prog for p := cursym.Func.Text; p != nil; p = p.Link { switch p.As { case obj.ATEXT: p.Mark |= LEAF - case obj.ARET: - break - case ADIV, ADIVU, AMOD, AMODU: - q = p cursym.Func.Text.Mark &^= LEAF - continue - - case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - if q1 != nil { - q1.Mark |= p.Mark - } - continue case ABL, ABX, obj.ADUFFZERO, obj.ADUFFCOPY: cursym.Func.Text.Mark &^= LEAF - fallthrough - - case AB, - ABEQ, - ABNE, - ABCS, - ABHS, - ABCC, - ABLO, - ABMI, - ABPL, - ABVS, - ABVC, - ABHI, - ABLS, - ABGE, - ABLT, - ABGT, - ABLE: - q1 = p.Pcond - if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } - } } - - q = p } var q2 *obj.Prog diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index 8f8981479b..7a5a8ff38c 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -837,6 +837,9 @@ var optab = []Optab{ {obj.APCDATA, C_VCON, C_NONE, C_NONE, C_VCON, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_VCON, C_NONE, C_NONE, C_ADDR, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_VREG, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0}, // same as AB/ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0}, // same as AB/ABL {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // align code diff --git a/src/cmd/internal/obj/arm64/obj7.go b/src/cmd/internal/obj/arm64/obj7.go index b046685ada..0d74430053 100644 --- a/src/cmd/internal/obj/arm64/obj7.go +++ b/src/cmd/internal/obj/arm64/obj7.go @@ -468,73 +468,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs - * expand RET */ - q := (*obj.Prog)(nil) - var q1 *obj.Prog for p := c.cursym.Func.Text; p != nil; p = p.Link { switch p.As { case obj.ATEXT: p.Mark |= LEAF - case obj.ARET: - break - - case obj.ANOP: - if p.Link != nil { - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark - } - continue - case ABL, obj.ADUFFZERO, obj.ADUFFCOPY: c.cursym.Func.Text.Mark &^= LEAF - fallthrough - - case ACBNZ, - ACBZ, - ACBNZW, - ACBZW, - ATBZ, - ATBNZ, - AB, - ABEQ, - ABNE, - ABCS, - ABHS, - ABCC, - ABLO, - ABMI, - ABPL, - ABVS, - ABVC, - ABHI, - ABLS, - ABGE, - ABLT, - ABGT, - ABLE, - AADR, /* strange */ - AADRP: - q1 = p.Pcond - - if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } - } - - break } - - q = p } + var q *obj.Prog + var q1 *obj.Prog var retjmp *obj.LSym for p := c.cursym.Func.Text; p != nil; p = p.Link { o := p.As diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index faa12bf133..faa827da9f 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -391,6 +391,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_LCON, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_SCON, C_NONE, C_ADDR, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_FREG, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_LBRA, 11, 4, 0, 0, 0}, // same as AJMP {obj.ADUFFCOPY, C_NONE, C_NONE, C_LBRA, 11, 4, 0, 0, 0}, // same as AJMP diff --git a/src/cmd/internal/obj/mips/obj0.go b/src/cmd/internal/obj/mips/obj0.go index 3106143844..77cad979a6 100644 --- a/src/cmd/internal/obj/mips/obj0.go +++ b/src/cmd/internal/obj/mips/obj0.go @@ -158,19 +158,14 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs * expand RET * expand BECOME pseudo */ - var q *obj.Prog - var q1 *obj.Prog for p := c.cursym.Func.Text; p != nil; p = p.Link { switch p.As { /* too hard, just leave alone */ case obj.ATEXT: - q = p - p.Mark |= LABEL | LEAF | SYNC if p.Link != nil { p.Link.Mark |= LABEL @@ -179,7 +174,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* too hard, just leave alone */ case AMOVW, AMOVV: - q = p if p.To.Type == obj.TYPE_REG && p.To.Reg >= REG_SPECIAL { p.Mark |= LABEL | SYNC break @@ -195,11 +189,9 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ATLBWI, ATLBP, ATLBR: - q = p p.Mark |= LABEL | SYNC case ANOR: - q = p if p.To.Type == obj.TYPE_REG { if p.To.Reg == REGZERO { p.Mark |= LABEL | SYNC @@ -235,8 +227,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { } else { p.Mark |= BRANCH } - q = p - q1 = p.Pcond + q1 := p.Pcond if q1 != nil { for q1.As == obj.ANOP { q1 = q1.Link @@ -254,24 +245,11 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { if q1 != nil { q1.Mark |= LABEL } - continue case ARET: - q = p if p.Link != nil { p.Link.Mark |= LABEL } - continue - - case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark - continue - - default: - q = p - continue } } @@ -284,6 +262,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { mov = AMOVW } + var q *obj.Prog + var q1 *obj.Prog autosize := int32(0) var p1 *obj.Prog var p2 *obj.Prog -- cgit v1.3 From c57c0212ebf44b7896477177c8e35be79ff3a586 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 19 Aug 2020 23:01:58 -0700 Subject: cmd/compile: define starting block before opening html writer Opening the html writer can fail, and the failure printer wants to use the entry block's line number. So make sure we set up the entry block first. Fixes #40919 Change-Id: I4ffa2839b45a721bbaf04ff84418e8108fa1cc37 Reviewed-on: https://go-review.googlesource.com/c/go/+/249497 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: David Chase --- src/cmd/compile/internal/gc/ssa.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 5d0098b4e6..4124655b79 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -338,6 +338,10 @@ func buildssa(fn *Node, worker int) *ssa.Func { s.panics = map[funcLine]*ssa.Block{} s.softFloat = s.config.SoftFloat + // Allocate starting block + s.f.Entry = s.f.NewBlock(ssa.BlockPlain) + s.f.Entry.Pos = fn.Pos + if printssa { s.f.HTMLWriter = ssa.NewHTMLWriter(ssaDumpFile, s.f, ssaDumpCFG) // TODO: generate and print a mapping from nodes to values and blocks @@ -345,9 +349,6 @@ func buildssa(fn *Node, worker int) *ssa.Func { s.f.HTMLWriter.WriteAST("AST", astBuf) } - // Allocate starting block - s.f.Entry = s.f.NewBlock(ssa.BlockPlain) - // Allocate starting values s.labels = map[string]*ssaLabel{} s.labeledNodes = map[*Node]*ssaLabel{} -- cgit v1.3 From 3f568625980f6e23a603cfd1cb4fcd2bf5c895d7 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 30 Jun 2020 17:51:52 -0400 Subject: cmd/go: do context propagation for tracing downloads This change does context propagation (and only context propagation) necessary to add context to modfetch.Download and pkg.LoadImport. This was done by adding context to their callers, and then adding context to all call-sites, and then repeating adding context to callers of those enclosing functions and their callers until none were left. In some cases the call graph expansion was pruned by using context.TODOs. The next CL will add a span to Download. I kept it out of this change to avoid making it any larger (and harder to review) than it needs to be. Updates #38714 Change-Id: I7a03416e04a14ca71636d96f2c1bda2c4c30d348 Reviewed-on: https://go-review.googlesource.com/c/go/+/249021 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/get/get.go | 4 +-- src/cmd/go/internal/list/list.go | 5 ++-- src/cmd/go/internal/load/pkg.go | 40 ++++++++++++++--------------- src/cmd/go/internal/load/test.go | 6 ++--- src/cmd/go/internal/modcmd/download.go | 4 +-- src/cmd/go/internal/modcmd/init.go | 2 +- src/cmd/go/internal/modcmd/tidy.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modcmd/why.go | 6 ++--- src/cmd/go/internal/modconv/convert_test.go | 5 +++- src/cmd/go/internal/modfetch/fetch.go | 3 ++- src/cmd/go/internal/modget/get.go | 18 ++++++------- src/cmd/go/internal/modload/build.go | 17 ++++++------ src/cmd/go/internal/modload/import.go | 9 ++++--- src/cmd/go/internal/modload/import_test.go | 5 +++- src/cmd/go/internal/modload/init.go | 9 ++++--- src/cmd/go/internal/modload/list.go | 14 +++++----- src/cmd/go/internal/modload/load.go | 39 ++++++++++++++-------------- src/cmd/go/internal/modload/mvs.go | 5 ++-- src/cmd/go/internal/modload/query.go | 33 ++++++++++++------------ src/cmd/go/internal/modload/query_test.go | 5 +++- src/cmd/go/internal/modload/search.go | 5 ++-- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- 25 files changed, 131 insertions(+), 113 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index ef43602aca..e5bacadaa3 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7303e6c866..e68c39f392 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -20,6 +20,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -349,7 +350,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": modload.ModuleInfo, + "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -389,7 +390,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod() // Parses go.mod and sets cfg.BuildMod. + modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 32c2ba7912..71fd9b5538 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -42,10 +42,10 @@ var ( ModBinDir func() string // return effective bin directory ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(args []string) []*search.Match // expand import paths + ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files + ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files ModDirImportPath func(string) string // return effective import path for directory ) @@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -605,11 +605,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -657,7 +657,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(path, stk, importPos, bp, err) + p.load(ctx, path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1800,7 +1800,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(roots []*Package) []*Package { +func TestPackageList(ctx context.Context, roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2089,7 +2089,7 @@ func TestPackageList(roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2112,7 +2112,7 @@ func TestPackageList(roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(path, srcDir, parent, stk, importPos, mode) + p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } @@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(patterns)} + return []*Package{GoFilesPackage(ctx, patterns)} } } } - matches := ImportPaths(patterns) + matches := ImportPaths(ctx, patterns) var ( pkgs []*Package stk ImportStack @@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(args []string) []*search.Match { +func ImportPaths(ctx context.Context, args []string) []*search.Match { if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(args) + return ModImportPaths(ctx, args) } return search.ImportPaths(args) } @@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(gofiles []string) *Package { +func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ModInit() for _, f := range gofiles { @@ -2329,7 +2329,7 @@ func GoFilesPackage(gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(gofiles) + ModImportFromFiles(ctx, gofiles) } var err error @@ -2345,7 +2345,7 @@ func GoFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load("command-line-arguments", &stk, nil, bp, err) + pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 6db8a00245..a0e275095b 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 2f42d4306b..6c4ec572fc 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod() // to fill Target + modload.InitMod(ctx) // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" @@ -126,7 +126,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(mod) + m.Dir, err = modfetch.Download(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 95063e62f4..b6cffd332d 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if strings.Contains(modload.CmdModModule, "@") { base.Fatalf("go mod init: module path must not contain '@'") } - modload.InitMod() // does all the hard work + modload.InitMod(ctx) // does all the hard work modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index 769cd11fe8..c7c53d7c0c 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL() + modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 257d1cd0ef..e5353b5c7f 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor() + pkgs := modload.LoadVendor(ctx) vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index f400339b25..da33fff89e 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) - for _, path := range loadALL() { + for _, path := range loadALL(ctx) { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(args) // resolve to packages - loadALL() // rebuild graph, from main module (not from named packages) + matches := modload.ImportPaths(ctx, args) // resolve to packages + loadALL(ctx) // rebuild graph, from main module (not from named packages) sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index a04a13b14f..faa2b4c606 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -6,6 +6,7 @@ package modconv import ( "bytes" + "context" "fmt" "internal/testenv" "io/ioutil" @@ -146,6 +147,8 @@ func TestConvertLegacyConfig(t *testing.T) { }, } + ctx := context.Background() + for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -157,7 +160,7 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e40158b535..6606612658 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -7,6 +7,7 @@ package modfetch import ( "archive/zip" "bytes" + "context" "errors" "fmt" "io" @@ -34,7 +35,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(mod module.Version) (dir string, err error) { +func Download(ctx context.Context, mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index d02c9a8da5..ee9757912b 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -353,7 +353,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if !strings.Contains(path, "...") { m := search.NewMatch(path) if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(pkgPath) + m = modload.TargetPackages(ctx, pkgPath) } if len(m.Pkgs) == 0 { for _, err := range m.Errs { @@ -399,7 +399,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { default: // The argument is a package or module path. if modload.HasModRoot() { - if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { + if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { // The path is in the main module. Nothing to query. if vers != "upgrade" && vers != "patch" { base.Errorf("go get %s: can't request explicit version of path in main module", arg) @@ -491,7 +491,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if q.path == q.m.Path { wg.Add(1) go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { + if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil { base.Errorf("go get: %v", err) } else if !hasPkg { modOnlyMu.Lock() @@ -536,7 +536,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Don't load packages if pkgPatterns is empty. Both // modload.ImportPathsQuiet and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) + matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -732,7 +732,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer q.m = module.Version{Path: q.path, Version: "none"} return } - m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) + m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath) if err != nil { base.Errorf("go get %s: %v", q.arg, err) } @@ -790,7 +790,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer // to determine the underlying module version being requested. // If forceModulePath is set, getQuery must interpret path // as a module path. -func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { +func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { if (prevM.Version != "") != forceModulePath { // We resolve package patterns by calling QueryPattern, which does not // accept a previous version and therefore cannot take it into account for @@ -812,7 +812,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo } } - info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +838,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(path, vers, modload.Allowed) + results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -994,7 +994,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 5f8a2e7e05..a101681a1f 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/hex" "fmt" "internal/goroot" @@ -57,21 +58,21 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(m, true) + return moduleInfo(context.TODO(), m, true) } -func ModuleInfo(path string) *modinfo.ModulePublic { +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) + return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(m, true) + return moduleInfo(ctx, m, true) } } @@ -84,12 +85,12 @@ func ModuleInfo(path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(m *modinfo.ModulePublic) { +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -103,7 +104,7 @@ func addVersions(m *modinfo.ModulePublic) { m.Versions, _ = versions(m.Path) } -func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -132,7 +133,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(m.Path, m.Version, "", nil); err != nil { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 4d2bc805e2..5c51a79124 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "go/build" @@ -110,7 +111,7 @@ var _ load.ImportPathError = &AmbiguousImportError{} // Import returns an ImportMissingError as the error. // If Import can identify a module that could be added to supply the package, // the ImportMissingError records that module. -func Import(path string) (m module.Version, dir string, err error) { +func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -165,7 +166,7 @@ func Import(path string) (m module.Version, dir string, err error) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error. // Note that we don't know for sure this module is necessary, @@ -248,7 +249,7 @@ func Import(path string) (m module.Version, dir string, err error) { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error as above. return module.Version{}, "", err @@ -285,7 +286,7 @@ func Import(path string) (m module.Version, dir string, err error) { fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(path, "latest", Allowed) + candidates, err := QueryPackage(ctx, path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index accc60eecd..47ce89a084 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "regexp" "strings" @@ -49,10 +50,12 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() + ctx := context.Background() + for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(tt.path) + m, dir, err := Import(ctx, tt.path) if err == nil { t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index fff060e665..93027c44c4 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -332,7 +333,7 @@ func die() { // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod() { +func InitMod(ctx context.Context) { if len(buildList) > 0 { return } @@ -359,7 +360,7 @@ func InitMod() { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -397,7 +398,7 @@ func InitMod() { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(fixed *bool) modfile.VersionFixer { +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -429,7 +430,7 @@ func fixVersion(fixed *bool) modfile.VersionFixer { } } - info, err := Query(path, vers, "", nil) + info, err := Query(ctx, path, vers, "", nil) if err != nil { return "", err } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 8db4d64706..7bf4e86c8d 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -31,7 +31,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ sem <- token{} go func() { if listU { - addUpdate(m) + addUpdate(ctx, m) } if listVersions { addVersions(m) @@ -57,7 +57,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { LoadBuildList(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -83,7 +83,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(path, vers, current, nil) + info, err := Query(ctx, path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +92,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) continue } @@ -117,7 +117,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m, true)) + mods = append(mods, moduleInfo(ctx, m, true)) } } } @@ -127,9 +127,9 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", "", nil) + info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 8190009b23..686d491219 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -52,8 +52,8 @@ var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), // on the target platform. Modules may be added to the build list // to satisfy new imports. -func ImportPaths(patterns []string) []*search.Match { - matches := ImportPathsQuiet(patterns, imports.Tags()) +func ImportPaths(ctx context.Context, patterns []string) []*search.Match { + matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) search.WarnUnmatched(matches) return matches } @@ -62,7 +62,7 @@ func ImportPaths(patterns []string) []*search.Match { // no matches. It also lets the caller specify a set of build tags to match // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { +func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { updateMatches := func(matches []*search.Match, iterating bool) { for _, m := range matches { switch { @@ -103,7 +103,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, loaded.tags, includeStd, buildList) case m.Pattern() == "all": loaded.testAll = true @@ -111,7 +111,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { // Enumerate the packages in the main module. // We'll load the dependencies as we find them. m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -129,7 +129,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { } } - InitMod() + InitMod(ctx) var matches []*search.Match for _, pattern := range search.CleanPatterns(patterns) { @@ -338,8 +338,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(gofiles []string) { - InitMod() +func ImportFromFiles(ctx context.Context, gofiles []string) { + InitMod(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -391,7 +391,7 @@ func DirImportPath(dir string) string { func LoadBuildList(ctx context.Context) []module.Version { ctx, span := trace.StartSpan(ctx, "LoadBuildList") defer span.Done() - InitMod() + InitMod(ctx) ReloadBuildList() WriteGoMod() return buildList @@ -409,20 +409,20 @@ func ReloadBuildList() []module.Version { // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. -func LoadALL() []string { - return loadAll(true) +func LoadALL(ctx context.Context) []string { + return loadAll(ctx, true) } // LoadVendor is like LoadALL but only follows test dependencies // for tests in the main module. Tests in dependency modules are // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor() []string { - return loadAll(false) +func LoadVendor(ctx context.Context) []string { + return loadAll(ctx, false) } -func loadAll(testAll bool) []string { - InitMod() +func loadAll(ctx context.Context, testAll bool) []string { + InitMod(ctx) loaded = newLoader(imports.AnyTags()) loaded.isALL = true @@ -430,7 +430,7 @@ func loadAll(testAll bool) []string { if !testAll { loaded.testRoots = true } - all := TargetPackages("...") + all := TargetPackages(ctx, "...") loaded.load(func() []string { return all.Pkgs }) checkMultiplePaths() WriteGoMod() @@ -453,13 +453,13 @@ func loadAll(testAll bool) []string { // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(pattern string) *search.Match { +func TargetPackages(ctx context.Context, pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. ModRoot() m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } @@ -817,7 +817,8 @@ func (ld *loader) doPkg(item interface{}) { return } - pkg.mod, pkg.dir, pkg.err = Import(pkg.path) + // TODO(matloob): Handle TODO context. This needs to be threaded through Do. + pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) if pkg.dir == "" { return } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 5dd009d31d..67eb2c2e19 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -224,7 +225,7 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { // // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(mod module.Version) (dir string, isLocal bool, err error) { +func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -254,6 +255,6 @@ func fetch(mod module.Version) (dir string, isLocal bool, err error) { mod = r } - dir, err = modfetch.Download(mod) + dir, err = modfetch.Download(ctx, mod) return dir, false, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index acc886bf21..39a813447c 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -55,10 +56,10 @@ import ( // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(proxy, path, query, current, allowed) + info, err = queryProxy(ctx, proxy, path, query, current, allowed) return err }) return info, err @@ -75,7 +76,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -243,7 +244,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) if err != nil { return nil, err } - releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) if err != nil { return nil, err } @@ -327,7 +328,7 @@ func matchSemverPrefix(p, v string) bool { // 1. versions that do not satisfy the 'ok' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { if !ok(module.Version{Path: path, Version: v}) { @@ -343,7 +344,7 @@ func filterVersions(path string, versions []string, ok func(module.Version) bool // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) if err != nil { return nil, nil, err } @@ -380,12 +381,12 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) } - return QueryPattern(path, query, allowed) + return QueryPattern(ctx, path, query, allowed) } // QueryPattern looks up the module(s) containing at least one package matching @@ -401,7 +402,7 @@ func QueryPackage(path, query string, allowed func(module.Version) bool) ([]Quer // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { base := pattern firstError := func(m *search.Match) error { @@ -417,7 +418,7 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q base = pathpkg.Dir(pattern[:i+3]) match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -472,12 +473,12 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q queryModule := func(path string) (r QueryResult, err error) { current := findCurrentVersion(path) r.Mod.Path = path - r.Rev, err = queryProxy(proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(r.Mod) + root, isLocal, err := fetch(ctx, r.Mod) if err != nil { return r, err } @@ -698,8 +699,8 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(m module.Version) (bool, error) { - root, isLocal, err := fetch(m) +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + root, isLocal, err := fetch(ctx, m) if err != nil { return false, err } @@ -707,8 +708,8 @@ func ModuleHasRootPackage(m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(m module.Version) (bool, error) { - root, _, err := fetch(m) +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + root, _, err := fetch(ctx, m) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 247e4c40d2..77080e9b5b 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "io/ioutil" "log" @@ -179,6 +180,8 @@ func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") + ctx := context.Background() + for _, tt := range queryTests { allow := tt.allow if allow == "" { @@ -192,7 +195,7 @@ func TestQuery(t *testing.T) { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(tt.path, tt.query, tt.current, allowed) + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index c28e7c0c1e..a9bee0af4e 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -5,6 +5,7 @@ package modload import ( + "context" "fmt" "os" "path/filepath" @@ -27,7 +28,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -153,7 +154,7 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu isLocal = true } else { var err error - root, isLocal, err = fetch(mod) + root, isLocal, err = fetch(ctx, mod) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index deec5106ff..99578b244c 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -77,7 +77,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(files) + p = load.GoFilesPackage(ctx, files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9cef8cf89c..3aee6939d2 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -702,7 +702,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(pkgs) { + for _, p := range load.TestPackageList(ctx, pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 681ecd7646..d975c36306 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2900,7 +2900,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { } srcs := []string{src} - p := load.GoFilesPackage(srcs) + p := load.GoFilesPackage(context.TODO(), srcs) if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { return "32", nil -- cgit v1.3 From 822dca4b383611335385ef05c2882d23f6f89d88 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 15 Jul 2020 17:38:36 -0400 Subject: cmd/go: add tracing for querying and downloading from the proxy This CL adds tracing spans for modload.queryPattern, modload.queryProxy, modload.QueryPattern, modload.QueryPattern.queryModule, modload.queryPrefixModules and modfetch.Download. Updates #38714 Change-Id: I91af3f022a6e18ab8d9c1d9b0ccf4928b6c236bd Reviewed-on: https://go-review.googlesource.com/c/go/+/249022 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modfetch/fetch.go | 21 +++++++++++++++------ .../internal/modfetch/zip_sum_test/zip_sum_test.go | 3 ++- src/cmd/go/internal/modload/query.go | 22 ++++++++++++++++++---- 4 files changed, 36 insertions(+), 12 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 6c4ec572fc..d4c161fca1 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -120,7 +120,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(mod) + m.Zip, err = modfetch.DownloadZip(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 6606612658..e29eb0a942 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -24,6 +24,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -48,7 +49,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(mod) + dir, err := download(ctx, mod) if err != nil { return cached{"", err} } @@ -58,7 +59,10 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(mod module.Version) (dir string, err error) { +func download(ctx context.Context, mod module.Version) (dir string, err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) + defer span.Done() + // If the directory exists, and no .partial file exists, the module has // already been completely extracted. .partial files may be created when a // module zip directory is extracted in place instead of being extracted to a @@ -73,7 +77,7 @@ func download(mod module.Version) (dir string, err error) { // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(mod) + zipfile, err := DownloadZip(ctx, mod) if err != nil { return "", err } @@ -143,6 +147,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) if unzipInPlace { if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { return "", err @@ -172,6 +177,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } } + defer span.Done() if !cfg.ModCacheRW { // Make dir read-only only *after* renaming it. @@ -196,7 +202,7 @@ var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(mod module.Version) (zipfile string, err error) { +func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -231,7 +237,7 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(mod, zipfile); err != nil { + if err := downloadZip(ctx, mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -239,7 +245,10 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { return c.zipfile, c.err } -func downloadZip(mod module.Version, zipfile string) (err error) { +func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) + defer span.Done() + // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. diff --git a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index eac9b32fa8..82398ebfed 100644 --- a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,6 +16,7 @@ package zip_sum_test import ( + "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -119,7 +120,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(test.m) + zipPath, err := modfetch.DownloadZip(context.Background(), test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 39a813447c..e82eb1506f 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -77,6 +78,9 @@ func (queryDisabledError) Error() string { } func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -403,6 +407,9 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // the main module and only the version "latest", without checking for other // possible modules. func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + base := pattern firstError := func(m *search.Match) error { @@ -470,7 +477,10 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(path string) (r QueryResult, err error) { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + current := findCurrentVersion(path) r.Mod.Path = path r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) @@ -499,7 +509,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } var err error - results, err = queryPrefixModules(candidateModules, queryModule) + results, err = queryPrefixModules(ctx, candidateModules, queryModule) return err }) @@ -543,7 +553,10 @@ type prefixResult struct { err error } -func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() + // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -556,8 +569,9 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(p) + r.QueryResult, r.err = queryModule(ctx, p) wg.Done() }(p, &results[i]) } -- cgit v1.3 From 268dd2e5a7a8919bd26f0a59c847f8268a2437d1 Mon Sep 17 00:00:00 2001 From: Michael Munday Date: Mon, 10 Aug 2020 08:01:21 -0700 Subject: cmd/internal/obj: fix inline marker issue on s390x The optimization that replaces inline markers with pre-existing instructions assumes that 'Prog' values produced by the compiler are still reachable after the assembler has run. This was not true on s390x where the assembler was removing NOP instructions from the linked list of 'Prog' values. This led to broken inlining data which in turn caused an infinite loop in the runtime traceback code. Fix this by stopping the s390x assembler backend removing NOP values. It does not make any difference to the output of the assembler because NOP instructions are 0 bytes long anyway. Fixes #40473. Change-Id: I9b97c494afaae2d5ed6bca4cd428b4132b5f8133 Reviewed-on: https://go-review.googlesource.com/c/go/+/249448 Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/internal/obj/pcln.go | 15 +++++++++++++++ src/cmd/internal/obj/s390x/objz.go | 11 ----------- 2 files changed, 15 insertions(+), 11 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index 1f7ccf47ef..bffeda041d 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -278,6 +278,21 @@ func linkpcln(ctxt *Link, cursym *LSym) { funcpctab(ctxt, &pcln.Pcfile, cursym, "pctofile", pctofileline, pcln) funcpctab(ctxt, &pcln.Pcline, cursym, "pctoline", pctofileline, nil) + // Check that all the Progs used as inline markers are still reachable. + // See issue #40473. + inlMarkProgs := make(map[*Prog]struct{}, len(cursym.Func.InlMarks)) + for _, inlMark := range cursym.Func.InlMarks { + inlMarkProgs[inlMark.p] = struct{}{} + } + for p := cursym.Func.Text; p != nil; p = p.Link { + if _, ok := inlMarkProgs[p]; ok { + delete(inlMarkProgs, p) + } + } + if len(inlMarkProgs) > 0 { + ctxt.Diag("one or more instructions used as inline markers are no longer reachable") + } + pcinlineState := new(pcinlineState) funcpctab(ctxt, &pcln.Pcinline, cursym, "pctoinline", pcinlineState.pctoinline, nil) for _, inlMark := range cursym.Func.InlMarks { diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index b14dc810fa..ef6335d849 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -283,17 +283,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ACMPUBNE: q = p p.Mark |= BRANCH - if p.Pcond != nil { - q := p.Pcond - for q.As == obj.ANOP { - q = q.Link - p.Pcond = q - } - } - - case obj.ANOP: - q.Link = p.Link /* q is non-nop */ - p.Link.Mark |= p.Mark default: q = p -- cgit v1.3 From e94544cf012535da6b3c9e735bc4026e2db1c99c Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Wed, 19 Aug 2020 21:39:12 -0700 Subject: cmd/compile: fix checkptr handling of &^ checkptr has code to recognize &^ expressions, but it didn't take into account that "p &^ x" gets rewritten to "p & ^x" during walk, which resulted in false positive diagnostics. This CL changes walkexpr to mark OANDNOT expressions with Implicit when they're rewritten to OAND, so that walkCheckPtrArithmetic can still recognize them later. It would be slightly more idiomatic to instead mark the OBITNOT expression as Implicit (as it's a compiler-generated Node), but the OBITNOT expression might get constant folded. It's not worth the extra complexity/subtlety of relying on n.Right.Orig, so we set Implicit on the OAND node instead. To atone for this transgression, I add documentation for nodeImplicit. Fixes #40917. Change-Id: I386304171ad299c530e151e5924f179e9a5fd5b8 Reviewed-on: https://go-review.googlesource.com/c/go/+/249477 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Cuong Manh Le --- src/cmd/compile/internal/gc/syntax.go | 4 ++-- src/cmd/compile/internal/gc/walk.go | 7 ++++++- src/runtime/checkptr_test.go | 1 + src/runtime/testdata/testprog/checkptr.go | 8 ++++++++ test/fixedbugs/issue40917.go | 23 +++++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 test/fixedbugs/issue40917.go (limited to 'src/cmd') diff --git a/src/cmd/compile/internal/gc/syntax.go b/src/cmd/compile/internal/gc/syntax.go index b658410c53..47e5e59156 100644 --- a/src/cmd/compile/internal/gc/syntax.go +++ b/src/cmd/compile/internal/gc/syntax.go @@ -141,8 +141,8 @@ const ( nodeInitorder, _ // tracks state during init1; two bits _, _ // second nodeInitorder bit _, nodeHasBreak - _, nodeNoInline // used internally by inliner to indicate that a function call should not be inlined; set for OCALLFUNC and OCALLMETH only - _, nodeImplicit + _, nodeNoInline // used internally by inliner to indicate that a function call should not be inlined; set for OCALLFUNC and OCALLMETH only + _, nodeImplicit // implicit OADDR or ODEREF; ++/-- statement represented as OASOP; or ANDNOT lowered to OAND _, nodeIsDDD // is the argument variadic _, nodeDiag // already printed error about this _, nodeColas // OAS resulting from := diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 8ae3d9a5c7..74ed0411bd 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -973,6 +973,7 @@ opswitch: case OANDNOT: n.Left = walkexpr(n.Left, init) n.Op = OAND + n.SetImplicit(true) // for walkCheckPtrArithmetic n.Right = nod(OBITNOT, n.Right, nil) n.Right = typecheck(n.Right, ctxExpr) n.Right = walkexpr(n.Right, init) @@ -4003,8 +4004,12 @@ func walkCheckPtrArithmetic(n *Node, init *Nodes) *Node { case OADD: walk(n.Left) walk(n.Right) - case OSUB, OANDNOT: + case OSUB: walk(n.Left) + case OAND: + if n.Implicit() { // was OANDNOT + walk(n.Left) + } case OCONVNOP: if n.Left.Type.Etype == TUNSAFEPTR { n.Left = cheapexpr(n.Left, init) diff --git a/src/runtime/checkptr_test.go b/src/runtime/checkptr_test.go index 8ab8a4937c..194cc1243a 100644 --- a/src/runtime/checkptr_test.go +++ b/src/runtime/checkptr_test.go @@ -27,6 +27,7 @@ func TestCheckPtr(t *testing.T) { {"CheckPtrAlignmentPtr", "fatal error: checkptr: misaligned pointer conversion\n"}, {"CheckPtrAlignmentNoPtr", ""}, {"CheckPtrArithmetic", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, + {"CheckPtrArithmetic2", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, {"CheckPtrSize", "fatal error: checkptr: converted pointer straddles multiple allocations\n"}, {"CheckPtrSmall", "fatal error: checkptr: pointer arithmetic computed bad pointer value\n"}, } diff --git a/src/runtime/testdata/testprog/checkptr.go b/src/runtime/testdata/testprog/checkptr.go index 45e6fb1aa5..e0a2794f4c 100644 --- a/src/runtime/testdata/testprog/checkptr.go +++ b/src/runtime/testdata/testprog/checkptr.go @@ -10,6 +10,7 @@ func init() { register("CheckPtrAlignmentNoPtr", CheckPtrAlignmentNoPtr) register("CheckPtrAlignmentPtr", CheckPtrAlignmentPtr) register("CheckPtrArithmetic", CheckPtrArithmetic) + register("CheckPtrArithmetic2", CheckPtrArithmetic2) register("CheckPtrSize", CheckPtrSize) register("CheckPtrSmall", CheckPtrSmall) } @@ -32,6 +33,13 @@ func CheckPtrArithmetic() { sink2 = (*int)(unsafe.Pointer(i)) } +func CheckPtrArithmetic2() { + var x [2]int64 + p := unsafe.Pointer(&x[1]) + var one uintptr = 1 + sink2 = unsafe.Pointer(uintptr(p) & ^one) +} + func CheckPtrSize() { p := new(int64) sink2 = p diff --git a/test/fixedbugs/issue40917.go b/test/fixedbugs/issue40917.go new file mode 100644 index 0000000000..2128be5eca --- /dev/null +++ b/test/fixedbugs/issue40917.go @@ -0,0 +1,23 @@ +// run -gcflags=-d=checkptr + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "unsafe" + +func main() { + var x [2]uint64 + a := unsafe.Pointer(&x[1]) + + b := a + b = unsafe.Pointer(uintptr(b) + 2) + b = unsafe.Pointer(uintptr(b) - 1) + b = unsafe.Pointer(uintptr(b) &^ 1) + + if a != b { + panic("pointer arithmetic failed") + } +} -- cgit v1.3 From 8535008765b4fcd5c7dc3fb2b73a856af4d51f9b Mon Sep 17 00:00:00 2001 From: Michał Łowicki Date: Mon, 11 May 2020 22:57:26 +0100 Subject: cmd/go: clarify error for invalid proxy responses Add information that error comes from parsing module proxy responses. Fixes #38680 Change-Id: Ic318b9cdbca789c1b0d983e083e692a914892fbd Reviewed-on: https://go-review.googlesource.com/c/go/+/233437 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modfetch/proxy.go | 13 +++++++------ src/cmd/go/proxy_test.go | 6 ++++++ src/cmd/go/testdata/script/mod_proxy_invalid.txt | 8 ++++++++ src/cmd/go/testdata/script/mod_query_empty.txt | 2 +- 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 src/cmd/go/testdata/script/mod_proxy_invalid.txt (limited to 'src/cmd') diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index 1c35d0b99b..4ac26650a9 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -242,8 +242,9 @@ func TryProxies(f func(proxy string) error) error { } type proxyRepo struct { - url *url.URL - path string + url *url.URL + path string + redactedURL string } func newProxyRepo(baseURL, path string) (Repo, error) { @@ -268,10 +269,10 @@ func newProxyRepo(baseURL, path string) (Repo, error) { if err != nil { return nil, err } - + redactedURL := base.Redacted() base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc) - return &proxyRepo{base, path}, nil + return &proxyRepo{base, path, redactedURL}, nil } func (p *proxyRepo) ModulePath() string { @@ -413,7 +414,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError(rev, err) + return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil { // If we request a correct, appropriate version for the module path, the @@ -434,7 +435,7 @@ func (p *proxyRepo) Latest() (*RevInfo, error) { } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError("", err) + return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } return info, nil } diff --git a/src/cmd/go/proxy_test.go b/src/cmd/go/proxy_test.go index 2a4d2935b3..7f58fb8ce4 100644 --- a/src/cmd/go/proxy_test.go +++ b/src/cmd/go/proxy_test.go @@ -131,6 +131,12 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { } path := r.URL.Path[len("/mod/"):] + // /mod/invalid returns faulty responses. + if strings.HasPrefix(path, "invalid/") { + w.Write([]byte("invalid")) + return + } + // /mod/quiet/ does not print errors. quiet := false if strings.HasPrefix(path, "quiet/") { diff --git a/src/cmd/go/testdata/script/mod_proxy_invalid.txt b/src/cmd/go/testdata/script/mod_proxy_invalid.txt new file mode 100644 index 0000000000..6427cc1527 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_proxy_invalid.txt @@ -0,0 +1,8 @@ +env GO111MODULE=on +env GOPROXY=$GOPROXY/invalid + +! go list -m rsc.io/quote@latest +stderr '^go list -m: module rsc.io/quote: invalid response from proxy "'$GOPROXY'": invalid character ''i'' looking for beginning of value$' + +! go list -m rsc.io/quote@1.5.2 +stderr '^go list -m: rsc.io/quote@1.5.2: invalid version: invalid response from proxy "'$GOPROXY'": invalid character ''i'' looking for beginning of value$' diff --git a/src/cmd/go/testdata/script/mod_query_empty.txt b/src/cmd/go/testdata/script/mod_query_empty.txt index 4d8259b40f..b3ea3e3de0 100644 --- a/src/cmd/go/testdata/script/mod_query_empty.txt +++ b/src/cmd/go/testdata/script/mod_query_empty.txt @@ -40,7 +40,7 @@ env GOPROXY=file:///$WORK/gatekeeper chmod 0000 $WORK/gatekeeper/example.com/join/subpkg/@latest cp go.mod.orig go.mod ! go get -d example.com/join/subpkg -stderr 'go get example.com/join/subpkg: module example.com/join/subpkg: (invalid character .+|reading file://.*/gatekeeper/example.com/join/subpkg/@latest: .+)' +stderr 'go get example.com/join/subpkg: module example.com/join/subpkg: (invalid response from proxy ".+": invalid character .+|reading file://.*/gatekeeper/example.com/join/subpkg/@latest: .+)' -- go.mod.orig -- module example.com/othermodule -- cgit v1.3 From 9679b307334bce77cc6e50751956a4c717e9458c Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 23 Apr 2020 18:06:21 -0400 Subject: cmd/go/testdata/script: make list_case_collision's behavior more clear Implementing the suggestion made by bcmills on a comment on golang.org/cl/228783. Change-Id: I314a24a002c65b582ea51610dcc1a54a69afbb8c Reviewed-on: https://go-review.googlesource.com/c/go/+/229705 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/script_test.go | 38 ++++++++++++++++++++++ src/cmd/go/testdata/script/README | 1 + src/cmd/go/testdata/script/list_case_collision.txt | 25 +++++--------- 3 files changed, 48 insertions(+), 16 deletions(-) (limited to 'src/cmd') diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 2e8f18a897..986646252a 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -22,6 +22,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -296,6 +297,8 @@ Script: ok = os.Geteuid() == 0 case "symlink": ok = testenv.HasSymlink() + case "case-sensitive": + ok = isCaseSensitive(ts.t) default: if strings.HasPrefix(cond.tag, "exec:") { prog := cond.tag[len("exec:"):] @@ -364,6 +367,41 @@ Script: } } +var ( + onceCaseSensitive sync.Once + caseSensitive bool +) + +func isCaseSensitive(t *testing.T) bool { + onceCaseSensitive.Do(func() { + tmpdir, err := ioutil.TempDir("", "case-sensitive") + if err != nil { + t.Fatal("failed to create directory to determine case-sensitivity:", err) + } + defer os.RemoveAll(tmpdir) + + fcap := filepath.Join(tmpdir, "FILE") + if err := ioutil.WriteFile(fcap, []byte{}, 0644); err != nil { + t.Fatal("error writing file to determine case-sensitivity:", err) + } + + flow := filepath.Join(tmpdir, "file") + _, err = ioutil.ReadFile(flow) + switch { + case err == nil: + caseSensitive = false + return + case os.IsNotExist(err): + caseSensitive = true + return + default: + t.Fatal("unexpected error reading file when determining case-sensitivity:", err) + } + }) + + return caseSensitive +} + // scriptCmds are the script command implementations. // Keep list and the implementations below sorted by name. // diff --git a/src/cmd/go/testdata/script/README b/src/cmd/go/testdata/script/README index 76d6651718..d658cebfce 100644 --- a/src/cmd/go/testdata/script/README +++ b/src/cmd/go/testdata/script/README @@ -85,6 +85,7 @@ should only run when the condition is satisfied. The available conditions are: - [link] for testenv.HasLink() - [root] for os.Geteuid() == 0 - [symlink] for testenv.HasSymlink() + - [case-sensitive] for whether the file system is case-sensitive - [exec:prog] for whether prog is available for execution (found by exec.LookPath) - [GODEBUG:value] for whether value is one of the comma-separated entries in the GODEBUG variable - [buildmode:value] for whether -buildmode=value is supported diff --git a/src/cmd/go/testdata/script/list_case_collision.txt b/src/cmd/go/testdata/script/list_case_collision.txt index 1b5f305587..73f44b63a0 100644 --- a/src/cmd/go/testdata/script/list_case_collision.txt +++ b/src/cmd/go/testdata/script/list_case_collision.txt @@ -6,23 +6,20 @@ stdout 'case-insensitive import collision' ! go build example/a stderr 'case-insensitive import collision' -# If we're not guaranteed to have a case-sensitive file system, list files explicitly on command line. -# Otherwise, let directory read find both files. -[darwin] ! go list example/b/file.go example/b/FILE.go -[windows] ! go list example/b/file.go example/b/FILE.go -[!darwin] [!windows] ! go list example/b +# List files explicitly on command line, to encounter case-checking +# logic even on case-insensitive filesystems. +cp example/b/file.go example/b/FILE.go # no-op on case-insensitive filesystems +! go list example/b/file.go example/b/FILE.go stderr 'case-insensitive file name collision' +mkdir example/a/Pkg # no-op on case-insensitive filesystems +cp example/a/pkg/pkg.go example/a/Pkg/pkg.go # no-op on case-insensitive filesystems ! go list example/a/pkg example/a/Pkg -stderr 'case-insensitive import collision' -go list -json -e example/a/pkg example/a/Pkg -stdout 'case-insensitive import collision' -! go build example/a/pkg example/a/Pkg -stderr 'case-insensitive import collision' # Test that the path reported with an indirect import is correct. -[!darwin] [!windows] ! go build example/c -[!darwin] [!windows] stderr '^package example/c\n\timports example/b: case-insensitive file name collision: "FILE.go" and "file.go"$' +cp example/b/file.go example/b/FILE.go +[case-sensitive] ! go build example/c +[case-sensitive] stderr '^package example/c\n\timports example/b: case-insensitive file name collision: "FILE.go" and "file.go"$' -- example/a/a.go -- package p @@ -32,12 +29,8 @@ import ( ) -- example/a/pkg/pkg.go -- package pkg --- example/a/Pkg/pkg.go -- -package pkg -- example/b/file.go -- package b --- example/b/FILE.go -- -package b -- example/c/c.go -- package c -- cgit v1.3