From cc700bdc269edc5fd29b14c1866c7f57f6f9b526 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 19 Jun 2020 17:59:16 -0400 Subject: cmd/trace: move viewer data structs into cmd/internal/traceviewer The ViewerEvent, ViewerData and ViewerFrame structs are moved into cmd/internal/traceviewer, and renamed Event, Data, and Frame. The structs are the same, except for the following: A definition for the JSON "bp" field that's defined in the trace format, but missing in the structs has been added. Also, the Tid and Pid fields on Event have been renamed TID and PID to better match Go style. Finally, the footer field on ViewerData, which hasn't been used for a while, has been removed. This CL is in preparation for the usage of these structs by cmd/go's tracing functionality. Updates #38714 Change-Id: I345f23617b96d4629b876ae717f89d56a67e05a3 Reviewed-on: https://go-review.googlesource.com/c/go/+/239098 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/internal/traceviewer/format.go | 38 ++++++++++ src/cmd/trace/trace.go | 125 +++++++++++++-------------------- src/cmd/trace/trace_test.go | 7 +- src/cmd/trace/trace_unix_test.go | 3 +- 4 files changed, 92 insertions(+), 81 deletions(-) create mode 100644 src/cmd/internal/traceviewer/format.go diff --git a/src/cmd/internal/traceviewer/format.go b/src/cmd/internal/traceviewer/format.go new file mode 100644 index 0000000000..871477447f --- /dev/null +++ b/src/cmd/internal/traceviewer/format.go @@ -0,0 +1,38 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package traceviewer provides definitions of the JSON data structures +// used by the Chrome trace viewer. +// +// The official description of the format is in this file: +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +package traceviewer + +type Data struct { + Events []*Event `json:"traceEvents"` + Frames map[string]Frame `json:"stackFrames"` + TimeUnit string `json:"displayTimeUnit"` +} + +type Event struct { + Name string `json:"name,omitempty"` + Phase string `json:"ph"` + Scope string `json:"s,omitempty"` + Time float64 `json:"ts"` + Dur float64 `json:"dur,omitempty"` + PID uint64 `json:"pid"` + TID uint64 `json:"tid"` + ID uint64 `json:"id,omitempty"` + BindPoint string `json:"bp,omitempty"` + Stack int `json:"sf,omitempty"` + EndStack int `json:"esf,omitempty"` + Arg interface{} `json:"args,omitempty"` + Cname string `json:"cname,omitempty"` + Category string `json:"cat,omitempty"` +} + +type Frame struct { + Name string `json:"name"` + Parent int `json:"parent,omitempty"` +} diff --git a/src/cmd/trace/trace.go b/src/cmd/trace/trace.go index b452376627..30c80f0e04 100644 --- a/src/cmd/trace/trace.go +++ b/src/cmd/trace/trace.go @@ -5,6 +5,7 @@ package main import ( + "cmd/internal/traceviewer" "encoding/json" "fmt" "internal/trace" @@ -325,7 +326,7 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { } var ( - data = ViewerData{Frames: make(map[string]ViewerFrame)} + data = traceviewer.Data{Frames: make(map[string]traceviewer.Frame)} sizes []eventSz cw countingWriter @@ -337,7 +338,7 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { consumeTimeUnit: func(unit string) { data.TimeUnit = unit }, - consumeViewerEvent: func(v *ViewerEvent, required bool) { + consumeViewerEvent: func(v *traceviewer.Event, required bool) { if required { // Store required events inside data // so flush can include them in the required @@ -350,7 +351,7 @@ func splittingTraceConsumer(max int) (*splitter, traceConsumer) { sizes = append(sizes, eventSz{v.Time, cw.size + 1}) // +1 for ",". cw.size = 0 }, - consumeViewerFrame: func(k string, v ViewerFrame) { + consumeViewerFrame: func(k string, v traceviewer.Frame) { data.Frames[k] = v }, flush: func() { @@ -478,36 +479,6 @@ type gInfo struct { markAssist *trace.Event // if non-nil, the mark assist currently running. } -type ViewerData struct { - Events []*ViewerEvent `json:"traceEvents"` - Frames map[string]ViewerFrame `json:"stackFrames"` - TimeUnit string `json:"displayTimeUnit"` - - // This is where mandatory part of the trace starts (e.g. thread names) - footer int -} - -type ViewerEvent struct { - Name string `json:"name,omitempty"` - Phase string `json:"ph"` - Scope string `json:"s,omitempty"` - Time float64 `json:"ts"` - Dur float64 `json:"dur,omitempty"` - Pid uint64 `json:"pid"` - Tid uint64 `json:"tid"` - ID uint64 `json:"id,omitempty"` - Stack int `json:"sf,omitempty"` - EndStack int `json:"esf,omitempty"` - Arg interface{} `json:"args,omitempty"` - Cname string `json:"cname,omitempty"` - Category string `json:"cat,omitempty"` -} - -type ViewerFrame struct { - Name string `json:"name"` - Parent int `json:"parent,omitempty"` -} - type NameArg struct { Name string `json:"name"` } @@ -528,8 +499,8 @@ type SortIndexArg struct { type traceConsumer struct { consumeTimeUnit func(unit string) - consumeViewerEvent func(v *ViewerEvent, required bool) - consumeViewerFrame func(key string, f ViewerFrame) + consumeViewerEvent func(v *traceviewer.Event, required bool) + consumeViewerFrame func(key string, f traceviewer.Frame) flush func() } @@ -775,23 +746,23 @@ func generateTrace(params *traceParams, consumer traceConsumer) error { ctx.emitSectionFooter(procsSection, "PROCS", 2) } - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.GCP, Arg: &NameArg{"GC"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.GCP, Arg: &SortIndexArg{-6}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.GCP, Arg: &NameArg{"GC"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.GCP, Arg: &SortIndexArg{-6}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.NetpollP, Arg: &NameArg{"Network"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.NetpollP, Arg: &SortIndexArg{-5}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.NetpollP, Arg: &NameArg{"Network"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.NetpollP, Arg: &SortIndexArg{-5}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.TimerP, Arg: &NameArg{"Timers"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.TimerP, Arg: &SortIndexArg{-4}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.TimerP, Arg: &NameArg{"Timers"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.TimerP, Arg: &SortIndexArg{-4}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: trace.SyscallP, Arg: &NameArg{"Syscalls"}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: trace.SyscallP, Arg: &SortIndexArg{-3}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: trace.SyscallP, Arg: &NameArg{"Syscalls"}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: trace.SyscallP, Arg: &SortIndexArg{-3}}) // Display rows for Ps if we are in the default trace view mode (not goroutine-oriented presentation) if ctx.mode&modeGoroutineOriented == 0 { for i := 0; i <= maxProc; i++ { - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: uint64(i), Arg: &NameArg{fmt.Sprintf("Proc %v", i)}}) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: uint64(i), Arg: &SortIndexArg{i}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: uint64(i), Arg: &NameArg{fmt.Sprintf("Proc %v", i)}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: uint64(i), Arg: &SortIndexArg{i}}) } } @@ -829,27 +800,27 @@ func generateTrace(params *traceParams, consumer traceConsumer) error { if !ctx.gs[k] { continue } - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: procsSection, Tid: k, Arg: &NameArg{v.name}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: procsSection, TID: k, Arg: &NameArg{v.name}}) } // Row for the main goroutine (maing) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: ctx.maing, Arg: &SortIndexArg{-2}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: ctx.maing, Arg: &SortIndexArg{-2}}) // Row for GC or global state (specified with G=0) - ctx.emitFooter(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: procsSection, Tid: 0, Arg: &SortIndexArg{-1}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: procsSection, TID: 0, Arg: &SortIndexArg{-1}}) } return nil } -func (ctx *traceContext) emit(e *ViewerEvent) { +func (ctx *traceContext) emit(e *traceviewer.Event) { ctx.consumer.consumeViewerEvent(e, false) } -func (ctx *traceContext) emitFooter(e *ViewerEvent) { +func (ctx *traceContext) emitFooter(e *traceviewer.Event) { ctx.consumer.consumeViewerEvent(e, true) } func (ctx *traceContext) emitSectionFooter(sectionID uint64, name string, priority int) { - ctx.emitFooter(&ViewerEvent{Name: "process_name", Phase: "M", Pid: sectionID, Arg: &NameArg{name}}) - ctx.emitFooter(&ViewerEvent{Name: "process_sort_index", Phase: "M", Pid: sectionID, Arg: &SortIndexArg{priority}}) + ctx.emitFooter(&traceviewer.Event{Name: "process_name", Phase: "M", PID: sectionID, Arg: &NameArg{name}}) + ctx.emitFooter(&traceviewer.Event{Name: "process_sort_index", Phase: "M", PID: sectionID, Arg: &SortIndexArg{priority}}) } func (ctx *traceContext) time(ev *trace.Event) float64 { @@ -880,7 +851,7 @@ func (ctx *traceContext) emitSlice(ev *trace.Event, name string) { ctx.emit(ctx.makeSlice(ev, name)) } -func (ctx *traceContext) makeSlice(ev *trace.Event, name string) *ViewerEvent { +func (ctx *traceContext) makeSlice(ev *trace.Event, name string) *traceviewer.Event { // If ViewerEvent.Dur is not a positive value, // trace viewer handles it as a non-terminating time interval. // Avoid it by setting the field with a small value. @@ -888,12 +859,12 @@ func (ctx *traceContext) makeSlice(ev *trace.Event, name string) *ViewerEvent { if ev.Link.Ts-ev.Ts <= 0 { durationUsec = 0.0001 // 0.1 nanoseconds } - sl := &ViewerEvent{ + sl := &traceviewer.Event{ Name: name, Phase: "X", Time: ctx.time(ev), Dur: durationUsec, - Tid: ctx.proc(ev), + TID: ctx.proc(ev), Stack: ctx.stack(ev.Stk), EndStack: ctx.stack(ev.Link.Stk), } @@ -927,16 +898,16 @@ func (ctx *traceContext) emitTask(task *taskDesc, sortIndex int) { taskName := task.name durationUsec := float64(task.lastTimestamp()-task.firstTimestamp()) / 1e3 - ctx.emitFooter(&ViewerEvent{Name: "thread_name", Phase: "M", Pid: tasksSection, Tid: taskRow, Arg: &NameArg{fmt.Sprintf("T%d %s", task.id, taskName)}}) - ctx.emit(&ViewerEvent{Name: "thread_sort_index", Phase: "M", Pid: tasksSection, Tid: taskRow, Arg: &SortIndexArg{sortIndex}}) + ctx.emitFooter(&traceviewer.Event{Name: "thread_name", Phase: "M", PID: tasksSection, TID: taskRow, Arg: &NameArg{fmt.Sprintf("T%d %s", task.id, taskName)}}) + ctx.emit(&traceviewer.Event{Name: "thread_sort_index", Phase: "M", PID: tasksSection, TID: taskRow, Arg: &SortIndexArg{sortIndex}}) ts := float64(task.firstTimestamp()) / 1e3 - sl := &ViewerEvent{ + sl := &traceviewer.Event{ Name: taskName, Phase: "X", Time: ts, Dur: durationUsec, - Pid: tasksSection, - Tid: taskRow, + PID: tasksSection, + TID: taskRow, Cname: pickTaskColor(task.id), } targ := TaskArg{ID: task.id} @@ -953,8 +924,8 @@ func (ctx *traceContext) emitTask(task *taskDesc, sortIndex int) { if task.create != nil && task.create.Type == trace.EvUserTaskCreate && task.create.Args[1] != 0 { ctx.arrowSeq++ - ctx.emit(&ViewerEvent{Name: "newTask", Phase: "s", Tid: task.create.Args[1], ID: ctx.arrowSeq, Time: ts, Pid: tasksSection}) - ctx.emit(&ViewerEvent{Name: "newTask", Phase: "t", Tid: taskRow, ID: ctx.arrowSeq, Time: ts, Pid: tasksSection}) + ctx.emit(&traceviewer.Event{Name: "newTask", Phase: "s", TID: task.create.Args[1], ID: ctx.arrowSeq, Time: ts, PID: tasksSection}) + ctx.emit(&traceviewer.Event{Name: "newTask", Phase: "t", TID: taskRow, ID: ctx.arrowSeq, Time: ts, PID: tasksSection}) } } @@ -975,12 +946,12 @@ func (ctx *traceContext) emitRegion(s regionDesc) { scopeID := fmt.Sprintf("%x", id) name := s.Name - sl0 := &ViewerEvent{ + sl0 := &traceviewer.Event{ Category: "Region", Name: name, Phase: "b", Time: float64(s.firstTimestamp()) / 1e3, - Tid: s.G, // only in goroutine-oriented view + TID: s.G, // only in goroutine-oriented view ID: uint64(regionID), Scope: scopeID, Cname: pickTaskColor(s.TaskID), @@ -990,12 +961,12 @@ func (ctx *traceContext) emitRegion(s regionDesc) { } ctx.emit(sl0) - sl1 := &ViewerEvent{ + sl1 := &traceviewer.Event{ Category: "Region", Name: name, Phase: "e", Time: float64(s.lastTimestamp()) / 1e3, - Tid: s.G, + TID: s.G, ID: uint64(regionID), Scope: scopeID, Cname: pickTaskColor(s.TaskID), @@ -1021,7 +992,7 @@ func (ctx *traceContext) emitHeapCounters(ev *trace.Event) { diff = ctx.heapStats.nextGC - ctx.heapStats.heapAlloc } if tsWithinRange(ev.Ts, ctx.startTime, ctx.endTime) { - ctx.emit(&ViewerEvent{Name: "Heap", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &heapCountersArg{ctx.heapStats.heapAlloc, diff}}) + ctx.emit(&traceviewer.Event{Name: "Heap", Phase: "C", Time: ctx.time(ev), PID: 1, Arg: &heapCountersArg{ctx.heapStats.heapAlloc, diff}}) } ctx.prevHeapStats = ctx.heapStats } @@ -1037,7 +1008,7 @@ func (ctx *traceContext) emitGoroutineCounters(ev *trace.Event) { return } if tsWithinRange(ev.Ts, ctx.startTime, ctx.endTime) { - ctx.emit(&ViewerEvent{Name: "Goroutines", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &goroutineCountersArg{uint64(ctx.gstates[gRunning]), uint64(ctx.gstates[gRunnable]), uint64(ctx.gstates[gWaitingGC])}}) + ctx.emit(&traceviewer.Event{Name: "Goroutines", Phase: "C", Time: ctx.time(ev), PID: 1, Arg: &goroutineCountersArg{uint64(ctx.gstates[gRunning]), uint64(ctx.gstates[gRunnable]), uint64(ctx.gstates[gWaitingGC])}}) } ctx.prevGstates = ctx.gstates } @@ -1052,7 +1023,7 @@ func (ctx *traceContext) emitThreadCounters(ev *trace.Event) { return } if tsWithinRange(ev.Ts, ctx.startTime, ctx.endTime) { - ctx.emit(&ViewerEvent{Name: "Threads", Phase: "C", Time: ctx.time(ev), Pid: 1, Arg: &threadCountersArg{ + ctx.emit(&traceviewer.Event{Name: "Threads", Phase: "C", Time: ctx.time(ev), PID: 1, Arg: &threadCountersArg{ Running: ctx.threadStats.prunning, InSyscall: ctx.threadStats.insyscall}}) } @@ -1090,13 +1061,13 @@ func (ctx *traceContext) emitInstant(ev *trace.Event, name, category string) { } arg = &Arg{ev.Args[0]} } - ctx.emit(&ViewerEvent{ + ctx.emit(&traceviewer.Event{ Name: name, Category: category, Phase: "I", Scope: "t", Time: ctx.time(ev), - Tid: ctx.proc(ev), + TID: ctx.proc(ev), Stack: ctx.stack(ev.Stk), Cname: cname, Arg: arg}) @@ -1134,8 +1105,8 @@ func (ctx *traceContext) emitArrow(ev *trace.Event, name string) { } ctx.arrowSeq++ - ctx.emit(&ViewerEvent{Name: name, Phase: "s", Tid: ctx.proc(ev), ID: ctx.arrowSeq, Time: ctx.time(ev), Stack: ctx.stack(ev.Stk), Cname: color}) - ctx.emit(&ViewerEvent{Name: name, Phase: "t", Tid: ctx.proc(ev.Link), ID: ctx.arrowSeq, Time: ctx.time(ev.Link), Cname: color}) + ctx.emit(&traceviewer.Event{Name: name, Phase: "s", TID: ctx.proc(ev), ID: ctx.arrowSeq, Time: ctx.time(ev), Stack: ctx.stack(ev.Stk), Cname: color}) + ctx.emit(&traceviewer.Event{Name: name, Phase: "t", TID: ctx.proc(ev.Link), ID: ctx.arrowSeq, Time: ctx.time(ev.Link), Cname: color}) } func (ctx *traceContext) stack(stk []*trace.Frame) int { @@ -1157,7 +1128,7 @@ func (ctx *traceContext) buildBranch(parent frameNode, stk []*trace.Frame) int { node.id = ctx.frameSeq node.children = make(map[uint64]frameNode) parent.children[frame.PC] = node - ctx.consumer.consumeViewerFrame(strconv.Itoa(node.id), ViewerFrame{fmt.Sprintf("%v:%v", frame.Fn, frame.Line), parent.id}) + ctx.consumer.consumeViewerFrame(strconv.Itoa(node.id), traceviewer.Frame{Name: fmt.Sprintf("%v:%v", frame.Fn, frame.Line), Parent: parent.id}) } return ctx.buildBranch(node, stk) } @@ -1192,7 +1163,7 @@ type jsonWriter struct { } func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { - frames := make(map[string]ViewerFrame) + frames := make(map[string]traceviewer.Frame) enc := json.NewEncoder(w) written := 0 index := int64(-1) @@ -1204,7 +1175,7 @@ func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { enc.Encode(unit) io.WriteString(w, ",") }, - consumeViewerEvent: func(v *ViewerEvent, required bool) { + consumeViewerEvent: func(v *traceviewer.Event, required bool) { index++ if !required && (index < start || index > end) { // not in the range. Skip! @@ -1221,7 +1192,7 @@ func viewerDataTraceConsumer(w io.Writer, start, end int64) traceConsumer { // Same should be applied to splittingTraceConsumer. written++ }, - consumeViewerFrame: func(k string, v ViewerFrame) { + consumeViewerFrame: func(k string, v traceviewer.Frame) { frames[k] = v }, flush: func() { diff --git a/src/cmd/trace/trace_test.go b/src/cmd/trace/trace_test.go index ef2d06c961..dd12e8cd20 100644 --- a/src/cmd/trace/trace_test.go +++ b/src/cmd/trace/trace_test.go @@ -7,6 +7,7 @@ package main import ( + "cmd/internal/traceviewer" "context" "internal/trace" "io/ioutil" @@ -78,7 +79,7 @@ func TestGoroutineCount(t *testing.T) { // Use the default viewerDataTraceConsumer but replace // consumeViewerEvent to intercept the ViewerEvents for testing. c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if ev.Name == "Goroutines" { cnt := ev.Arg.(*goroutineCountersArg) if cnt.Runnable+cnt.Running > 2 { @@ -165,7 +166,7 @@ func TestPreemptedMarkAssist(t *testing.T) { c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) marks := 0 - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if strings.Contains(ev.Name, "MARK ASSIST") { marks++ } @@ -216,7 +217,7 @@ func TestFoo(t *testing.T) { c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) var logBeforeTaskEnd, logAfterTaskEnd bool - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if ev.Name == "log before task ends" { logBeforeTaskEnd = true } diff --git a/src/cmd/trace/trace_unix_test.go b/src/cmd/trace/trace_unix_test.go index fec060e121..645978e0f8 100644 --- a/src/cmd/trace/trace_unix_test.go +++ b/src/cmd/trace/trace_unix_test.go @@ -8,6 +8,7 @@ package main import ( "bytes" + "cmd/internal/traceviewer" traceparser "internal/trace" "io/ioutil" "runtime" @@ -83,7 +84,7 @@ func TestGoroutineInSyscall(t *testing.T) { // Check only one thread for the pipe read goroutine is // considered in-syscall. c := viewerDataTraceConsumer(ioutil.Discard, 0, 1<<63-1) - c.consumeViewerEvent = func(ev *ViewerEvent, _ bool) { + c.consumeViewerEvent = func(ev *traceviewer.Event, _ bool) { if ev.Name == "Threads" { arg := ev.Arg.(*threadCountersArg) if arg.InSyscall > 1 { -- cgit v1.3-5-g9baa From 20254c02b38055c1bb6d5ac8aba70bb7b8d638ef Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 14 Apr 2020 17:29:27 -0400 Subject: cmd/go: add rudimentary tracing support. Updates #38714 Change-Id: I14da982d405074d65ccf5521d431df1bf1734f9a Reviewed-on: https://go-review.googlesource.com/c/go/+/230378 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/trace/trace.go | 139 +++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 src/cmd/go/internal/trace/trace.go diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go new file mode 100644 index 0000000000..7cb7636a34 --- /dev/null +++ b/src/cmd/go/internal/trace/trace.go @@ -0,0 +1,139 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +package trace + +import ( + "cmd/internal/traceviewer" + "context" + "encoding/json" + "errors" + "os" + "strings" + "sync/atomic" + "time" +) + +var traceStarted int32 + +func getTraceContext(ctx context.Context) (traceContext, bool) { + if atomic.LoadInt32(&traceStarted) == 0 { + return traceContext{}, false + } + v := ctx.Value(traceKey{}) + if v == nil { + return traceContext{}, false + } + return v.(traceContext), true +} + +// StartSpan starts a trace event with the given name. The Span ends when its Done method is called. +func StartSpan(ctx context.Context, name string) (context.Context, *Span) { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx, nil + } + childSpan := &Span{t: tc.t, name: name, start: time.Now()} + tc.t.writeEvent(&traceviewer.Event{ + Name: childSpan.name, + Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), + Phase: "B", + }) + ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t}) + return ctx, childSpan +} + +type Span struct { + t *tracer + + name string + start time.Time + end time.Time +} + +func (s *Span) Done() { + if s == nil { + return + } + s.end = time.Now() + s.t.writeEvent(&traceviewer.Event{ + Name: s.name, + Time: float64(s.end.UnixNano()) / float64(time.Microsecond), + Phase: "E", + }) +} + +type tracer struct { + file chan traceFile // 1-buffered +} + +func (t *tracer) writeEvent(ev *traceviewer.Event) error { + f := <-t.file + defer func() { t.file <- f }() + var err error + if f.entries == 0 { + _, err = f.sb.WriteString("[\n") + } else { + _, err = f.sb.WriteString(",") + } + f.entries++ + if err != nil { + return nil + } + + if err := f.enc.Encode(ev); err != nil { + return err + } + + // Write event string to output file. + _, err = f.f.WriteString(f.sb.String()) + f.sb.Reset() + return err +} + +func (t *tracer) Close() error { + f := <-t.file + defer func() { t.file <- f }() + + _, firstErr := f.f.WriteString("]") + if err := f.f.Close(); firstErr == nil { + firstErr = err + } + return firstErr +} + +// traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in +// other packages. +type traceKey struct{} + +type traceContext struct { + t *tracer +} + +// Start starts a trace which writes to the given file. +func Start(ctx context.Context, file string) (context.Context, func() error, error) { + atomic.StoreInt32(&traceStarted, 1) + if file == "" { + return nil, nil, errors.New("no trace file supplied") + } + f, err := os.Create(file) + if err != nil { + return nil, nil, err + } + t := &tracer{file: make(chan traceFile, 1)} + sb := new(strings.Builder) + t.file <- traceFile{ + f: f, + sb: sb, + enc: json.NewEncoder(sb), + } + ctx = context.WithValue(ctx, traceKey{}, traceContext{t: t}) + return ctx, t.Close, nil +} + +type traceFile struct { + f *os.File + sb *strings.Builder + enc *json.Encoder + entries int64 +} -- cgit v1.3-5-g9baa From 52b0ea20ff10fdcfe570ef407bd462d23e13d782 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 14 Apr 2020 17:29:27 -0400 Subject: cmd/go: add a debug-trace flag to generate traces If cmd/go is provided with a -debug-trace= option, cmd/go will write an execution trace to that file. Updates #38714 Change-Id: I3e6521343902c08266a0292f4280298a3bf8b725 Reviewed-on: https://go-review.googlesource.com/c/go/+/237683 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/cfg/cfg.go | 1 + src/cmd/go/internal/work/build.go | 1 + src/cmd/go/main.go | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/src/cmd/go/internal/cfg/cfg.go b/src/cmd/go/internal/cfg/cfg.go index 7f8f8e92be..f9bbcd9180 100644 --- a/src/cmd/go/internal/cfg/cfg.go +++ b/src/cmd/go/internal/cfg/cfg.go @@ -51,6 +51,7 @@ var ( CmdName string // "build", "install", "list", "mod tidy", etc. DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable) + DebugTrace string // -debug-trace flag ) func defaultContext() build.Context { diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 7146c9ce00..fbd49b457b 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -270,6 +270,7 @@ func AddBuildFlags(cmd *base.Command, mask BuildFlagMask) { // Undocumented, unstable debugging flags. cmd.Flag.StringVar(&cfg.DebugActiongraph, "debug-actiongraph", "", "") + cmd.Flag.StringVar(&cfg.DebugTrace, "debug-trace", "", "") } // AddModCommonFlags adds the module-related flags common to build commands diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index fdf49b7380..3512866e4a 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "flag" "fmt" "log" @@ -34,6 +35,7 @@ import ( "cmd/go/internal/run" "cmd/go/internal/test" "cmd/go/internal/tool" + "cmd/go/internal/trace" "cmd/go/internal/version" "cmd/go/internal/vet" "cmd/go/internal/work" @@ -187,7 +189,11 @@ BigCmdLoop: cmd.Flag.Parse(args[1:]) args = cmd.Flag.Args() } + ctx := maybeStartTrace(context.Background()) + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + _ = ctx cmd.Run(cmd, args) + span.Done() base.Exit() return } @@ -209,3 +215,21 @@ func mainUsage() { help.PrintUsage(os.Stderr, base.Go) os.Exit(2) } + +func maybeStartTrace(pctx context.Context) context.Context { + if cfg.DebugTrace == "" { + return pctx + } + + ctx, close, err := trace.Start(pctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + base.AtExit(func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }) + + return ctx +} -- cgit v1.3-5-g9baa From 14715b24793176b30b3c41adf4ac4e676c2a56f0 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Fri, 12 Jun 2020 14:33:23 -0400 Subject: cmd/go: add Context parameter to base.command.Run One small step to start propagating the context in cmd/go for tracing purposes. Updates #38714 Change-Id: Ibb6debeb9233f84d55f0e81244487355cbe7b82c Reviewed-on: https://go-review.googlesource.com/c/go/+/237684 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/base/base.go | 3 ++- src/cmd/go/internal/bug/bug.go | 3 ++- src/cmd/go/internal/clean/clean.go | 3 ++- src/cmd/go/internal/doc/doc.go | 3 ++- src/cmd/go/internal/envcmd/env.go | 3 ++- src/cmd/go/internal/fix/fix.go | 3 ++- src/cmd/go/internal/fmtcmd/fmt.go | 3 ++- src/cmd/go/internal/generate/generate.go | 3 ++- src/cmd/go/internal/get/get.go | 3 ++- src/cmd/go/internal/list/list.go | 3 ++- src/cmd/go/internal/modcmd/download.go | 3 ++- src/cmd/go/internal/modcmd/edit.go | 3 ++- src/cmd/go/internal/modcmd/graph.go | 3 ++- src/cmd/go/internal/modcmd/init.go | 3 ++- src/cmd/go/internal/modcmd/tidy.go | 3 ++- src/cmd/go/internal/modcmd/vendor.go | 3 ++- src/cmd/go/internal/modcmd/verify.go | 3 ++- src/cmd/go/internal/modcmd/why.go | 3 ++- src/cmd/go/internal/modget/get.go | 3 ++- src/cmd/go/internal/run/run.go | 3 ++- src/cmd/go/internal/test/test.go | 3 ++- src/cmd/go/internal/tool/tool.go | 3 ++- src/cmd/go/internal/version/version.go | 3 ++- src/cmd/go/internal/vet/vet.go | 3 ++- src/cmd/go/internal/work/build.go | 5 +++-- src/cmd/go/main.go | 3 +-- 26 files changed, 52 insertions(+), 28 deletions(-) diff --git a/src/cmd/go/internal/base/base.go b/src/cmd/go/internal/base/base.go index ab2f1bb4e2..db3ebef933 100644 --- a/src/cmd/go/internal/base/base.go +++ b/src/cmd/go/internal/base/base.go @@ -7,6 +7,7 @@ package base import ( + "context" "flag" "fmt" "log" @@ -24,7 +25,7 @@ import ( type Command struct { // Run runs the command. // The args are the arguments after the command name. - Run func(cmd *Command, args []string) + Run func(ctx context.Context, cmd *Command, args []string) // UsageLine is the one-line usage message. // The words between "go" and the first flag or argument in the line are taken to be the command name. diff --git a/src/cmd/go/internal/bug/bug.go b/src/cmd/go/internal/bug/bug.go index fe71281ef0..52bd40f2fb 100644 --- a/src/cmd/go/internal/bug/bug.go +++ b/src/cmd/go/internal/bug/bug.go @@ -7,6 +7,7 @@ package bug import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -37,7 +38,7 @@ func init() { CmdBug.Flag.BoolVar(&cfg.BuildV, "v", false, "") } -func runBug(cmd *base.Command, args []string) { +func runBug(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go bug: bug takes no arguments") } diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 99704cb2b1..8af3e3df9c 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -6,6 +6,7 @@ package clean import ( + "context" "fmt" "io/ioutil" "os" @@ -105,7 +106,7 @@ func init() { work.AddBuildFlags(CmdClean, work.DefaultBuildFlags) } -func runClean(cmd *base.Command, args []string) { +func runClean(ctx context.Context, cmd *base.Command, args []string) { // golang.org/issue/29925: only load packages before cleaning if // either the flags and arguments explicitly imply a package, // or no other target (such as a cache) was requested to be cleaned. diff --git a/src/cmd/go/internal/doc/doc.go b/src/cmd/go/internal/doc/doc.go index 4ff08bb928..67f76e2256 100644 --- a/src/cmd/go/internal/doc/doc.go +++ b/src/cmd/go/internal/doc/doc.go @@ -8,6 +8,7 @@ package doc import ( "cmd/go/internal/base" "cmd/go/internal/cfg" + "context" ) var CmdDoc = &base.Command{ @@ -129,6 +130,6 @@ Flags: `, } -func runDoc(cmd *base.Command, args []string) { +func runDoc(ctx context.Context, cmd *base.Command, args []string) { base.Run(cfg.BuildToolexec, base.Tool("doc"), args) } diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 252025dc25..403e0f4a7b 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -6,6 +6,7 @@ package envcmd import ( + "context" "encoding/json" "fmt" "go/build" @@ -186,7 +187,7 @@ func argKey(arg string) string { return arg[:i] } -func runEnv(cmd *base.Command, args []string) { +func runEnv(ctx context.Context, cmd *base.Command, args []string) { if *envJson && *envU { base.Fatalf("go env: cannot use -json with -u") } diff --git a/src/cmd/go/internal/fix/fix.go b/src/cmd/go/internal/fix/fix.go index 4d741df2b4..f16af05fc8 100644 --- a/src/cmd/go/internal/fix/fix.go +++ b/src/cmd/go/internal/fix/fix.go @@ -11,6 +11,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/str" + "context" "fmt" "os" ) @@ -31,7 +32,7 @@ See also: go fmt, go vet. `, } -func runFix(cmd *base.Command, args []string) { +func runFix(ctx context.Context, cmd *base.Command, args []string) { printed := false for _, pkg := range load.Packages(args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index d6894edc9f..9868efc7ef 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -6,6 +6,7 @@ package fmtcmd import ( + "context" "errors" "fmt" "os" @@ -48,7 +49,7 @@ See also: go fix, go vet. `, } -func runFmt(cmd *base.Command, args []string) { +func runFmt(ctx context.Context, cmd *base.Command, args []string) { printed := false gofmt := gofmtPath() procs := runtime.GOMAXPROCS(0) diff --git a/src/cmd/go/internal/generate/generate.go b/src/cmd/go/internal/generate/generate.go index 093b19817b..fb26f77f95 100644 --- a/src/cmd/go/internal/generate/generate.go +++ b/src/cmd/go/internal/generate/generate.go @@ -8,6 +8,7 @@ package generate import ( "bufio" "bytes" + "context" "fmt" "go/parser" "go/token" @@ -160,7 +161,7 @@ func init() { CmdGenerate.Flag.StringVar(&generateRunFlag, "run", "", "") } -func runGenerate(cmd *base.Command, args []string) { +func runGenerate(ctx context.Context, cmd *base.Command, args []string) { load.IgnoreImports = true if generateRunFlag != "" { diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index d38350c2a8..f7da5270b0 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -6,6 +6,7 @@ package get import ( + "context" "fmt" "os" "path/filepath" @@ -112,7 +113,7 @@ func init() { CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "") } -func runGet(cmd *base.Command, args []string) { +func runGet(ctx context.Context, cmd *base.Command, args []string) { if cfg.ModulesEnabled { // Should not happen: main.go should install the separate module-enabled get code. base.Fatalf("go get: modules not implemented") diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 6ca1561121..ef0a5a2f2d 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -8,6 +8,7 @@ package list import ( "bufio" "bytes" + "context" "encoding/json" "io" "os" @@ -309,7 +310,7 @@ var ( var nl = []byte{'\n'} -func runList(cmd *base.Command, args []string) { +func runList(ctx context.Context, cmd *base.Command, args []string) { modload.LoadTests = *listTest work.BuildInit() out := newTrackingWriter(os.Stdout) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 584434935b..b43c32be5a 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -5,6 +5,7 @@ package modcmd import ( + "context" "encoding/json" "os" @@ -78,7 +79,7 @@ type moduleJSON struct { GoModSum string `json:",omitempty"` } -func runDownload(cmd *base.Command, args []string) { +func runDownload(ctx context.Context, cmd *base.Command, args []string) { // Check whether modules are enabled and whether we're in a module. if cfg.Getenv("GO111MODULE") == "off" { base.Fatalf("go: modules disabled by GO111MODULE=off; see 'go help modules'") diff --git a/src/cmd/go/internal/modcmd/edit.go b/src/cmd/go/internal/modcmd/edit.go index dbbfb96e42..a81c25270f 100644 --- a/src/cmd/go/internal/modcmd/edit.go +++ b/src/cmd/go/internal/modcmd/edit.go @@ -8,6 +8,7 @@ package modcmd import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -141,7 +142,7 @@ func init() { base.AddBuildFlagsNX(&cmdEdit.Flag) } -func runEdit(cmd *base.Command, args []string) { +func runEdit(ctx context.Context, cmd *base.Command, args []string) { anyFlags := *editModule != "" || *editGo != "" || diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 27ae9354f3..fff5b02626 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -8,6 +8,7 @@ package modcmd import ( "bufio" + "context" "os" "sort" @@ -36,7 +37,7 @@ func init() { work.AddModCommonFlags(cmdGraph) } -func runGraph(cmd *base.Command, args []string) { +func runGraph(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod graph: graph takes no arguments") } diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 714ff2e205..ddb9aeebe9 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -10,6 +10,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/modload" "cmd/go/internal/work" + "context" "os" "strings" ) @@ -32,7 +33,7 @@ func init() { work.AddModCommonFlags(cmdInit) } -func runInit(cmd *base.Command, args []string) { +func runInit(ctx context.Context, cmd *base.Command, args []string) { modload.CmdModInit = true if len(args) > 1 { base.Fatalf("go mod init: too many arguments") diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index af2b04c0c2..feb41a83b0 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -12,6 +12,7 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/modload" "cmd/go/internal/work" + "context" "golang.org/x/mod/module" ) @@ -37,7 +38,7 @@ func init() { work.AddModCommonFlags(cmdTidy) } -func runTidy(cmd *base.Command, args []string) { +func runTidy(ctx context.Context, cmd *base.Command, args []string) { if len(args) > 0 { base.Fatalf("go mod tidy: no arguments allowed") } diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 8509ceb7a8..257d1cd0ef 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -6,6 +6,7 @@ package modcmd import ( "bytes" + "context" "fmt" "io" "io/ioutil" @@ -43,7 +44,7 @@ func init() { work.AddModCommonFlags(cmdVendor) } -func runVendor(cmd *base.Command, args []string) { +func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index b7fd7fa8e0..570e571049 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -6,6 +6,7 @@ package modcmd import ( "bytes" + "context" "errors" "fmt" "io/ioutil" @@ -40,7 +41,7 @@ func init() { work.AddModCommonFlags(cmdVerify) } -func runVerify(cmd *base.Command, args []string) { +func runVerify(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { // NOTE(rsc): Could take a module pattern. base.Fatalf("go mod verify: verify takes no arguments") diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index 40d238519b..3f9cf0f120 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -5,6 +5,7 @@ package modcmd import ( + "context" "fmt" "strings" @@ -60,7 +61,7 @@ func init() { work.AddModCommonFlags(cmdWhy) } -func runWhy(cmd *base.Command, args []string) { +func runWhy(ctx context.Context, cmd *base.Command, args []string) { loadALL := modload.LoadALL if *whyVendor { loadALL = modload.LoadVendor diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 4c6982426f..9836a3e2cc 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -6,6 +6,7 @@ package modget import ( + "context" "errors" "fmt" "os" @@ -259,7 +260,7 @@ type query struct { m module.Version } -func runGet(cmd *base.Command, args []string) { +func runGet(ctx context.Context, cmd *base.Command, args []string) { switch getU { case "", "upgrade", "patch": // ok diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 2edae38cca..ca2c3db92c 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -6,6 +6,7 @@ package run import ( + "context" "fmt" "os" "path" @@ -57,7 +58,7 @@ func printStderr(args ...interface{}) (int, error) { return fmt.Fprint(os.Stderr, args...) } -func runRun(cmd *base.Command, args []string) { +func runRun(ctx context.Context, cmd *base.Command, args []string) { work.BuildInit() var b work.Builder b.Init() diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 873a76aa38..6648d4eab4 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -6,6 +6,7 @@ package test import ( "bytes" + "context" "crypto/sha256" "errors" "fmt" @@ -565,7 +566,7 @@ var defaultVetFlags = []string{ // "-unusedresult", } -func runTest(cmd *base.Command, args []string) { +func runTest(ctx context.Context, cmd *base.Command, args []string) { modload.LoadTests = true pkgArgs, testArgs = testFlags(args) diff --git a/src/cmd/go/internal/tool/tool.go b/src/cmd/go/internal/tool/tool.go index 930eecb63f..7f4dc86802 100644 --- a/src/cmd/go/internal/tool/tool.go +++ b/src/cmd/go/internal/tool/tool.go @@ -6,6 +6,7 @@ package tool import ( + "context" "fmt" "os" "os/exec" @@ -48,7 +49,7 @@ func init() { CmdTool.Flag.BoolVar(&toolN, "n", false, "") } -func runTool(cmd *base.Command, args []string) { +func runTool(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { listTools() return diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index ac2ae50155..056db7bf9e 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -7,6 +7,7 @@ package version import ( "bytes" + "context" "encoding/binary" "fmt" "os" @@ -51,7 +52,7 @@ var ( versionV = CmdVersion.Flag.Bool("v", false, "") ) -func runVersion(cmd *base.Command, args []string) { +func runVersion(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { if *versionM || *versionV { fmt.Fprintf(os.Stderr, "go version: flags can only be used with arguments\n") diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 4ec58de785..717ff2d0aa 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -10,6 +10,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/work" + "context" "path/filepath" ) @@ -48,7 +49,7 @@ See also: go fmt, go fix. `, } -func runVet(cmd *base.Command, args []string) { +func runVet(ctx context.Context, cmd *base.Command, args []string) { modload.LoadTests = true vetFlags, pkgArgs := vetFlags(args) diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index fbd49b457b..2bbee43ab4 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -5,6 +5,7 @@ package work import ( + "context" "errors" "fmt" "go/build" @@ -344,7 +345,7 @@ var pkgsFilter = func(pkgs []*load.Package) []*load.Package { return pkgs } var runtimeVersion = runtime.Version() -func runBuild(cmd *base.Command, args []string) { +func runBuild(ctx context.Context, cmd *base.Command, args []string) { BuildInit() var b Builder b.Init() @@ -515,7 +516,7 @@ func libname(args []string, pkgs []*load.Package) (string, error) { return "lib" + libname + ".so", nil } -func runInstall(cmd *base.Command, args []string) { +func runInstall(ctx context.Context, cmd *base.Command, args []string) { BuildInit() InstallPackages(args, load.PackagesForBuild(args)) } diff --git a/src/cmd/go/main.go b/src/cmd/go/main.go index 3512866e4a..37bb7d6d27 100644 --- a/src/cmd/go/main.go +++ b/src/cmd/go/main.go @@ -191,8 +191,7 @@ BigCmdLoop: } ctx := maybeStartTrace(context.Background()) ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) - _ = ctx - cmd.Run(cmd, args) + cmd.Run(ctx, cmd, args) span.Done() base.Exit() return -- cgit v1.3-5-g9baa From 2bfa45cfa994512c47da2d98f3baca5bb474ec9b Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 15:50:14 -0400 Subject: cmd/go: propagate context into PackagesForBuild and Do for tracing This change propagates context into PackagesForErrors and Do for the purpose of tracing, and calls trace.StartSpan on PackagesForErrors and Do, so that the trace now shows the broad outline of where the "Loading" and "Execution" phases are in the build. Updates #38714 Change-Id: Ib9a7cf7030210f68f76663d1c8a7461e0a226611 Reviewed-on: https://go-review.googlesource.com/c/go/+/238541 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/clean/clean.go | 2 +- src/cmd/go/internal/fix/fix.go | 2 +- src/cmd/go/internal/fmtcmd/fmt.go | 2 +- src/cmd/go/internal/generate/generate.go | 2 +- src/cmd/go/internal/get/get.go | 4 ++-- src/cmd/go/internal/list/list.go | 6 +++--- src/cmd/go/internal/load/pkg.go | 15 ++++++++++----- src/cmd/go/internal/modget/get.go | 4 ++-- src/cmd/go/internal/run/run.go | 4 ++-- src/cmd/go/internal/test/test.go | 8 ++++---- src/cmd/go/internal/vet/vet.go | 4 ++-- src/cmd/go/internal/work/build.go | 20 ++++++++++++-------- src/cmd/go/internal/work/exec.go | 7 ++++++- 13 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/cmd/go/internal/clean/clean.go b/src/cmd/go/internal/clean/clean.go index 8af3e3df9c..6bfd7ae21e 100644 --- a/src/cmd/go/internal/clean/clean.go +++ b/src/cmd/go/internal/clean/clean.go @@ -117,7 +117,7 @@ func runClean(ctx context.Context, cmd *base.Command, args []string) { } if cleanPkg { - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { clean(pkg) } } diff --git a/src/cmd/go/internal/fix/fix.go b/src/cmd/go/internal/fix/fix.go index f16af05fc8..825624fcbb 100644 --- a/src/cmd/go/internal/fix/fix.go +++ b/src/cmd/go/internal/fix/fix.go @@ -34,7 +34,7 @@ See also: go fmt, go vet. func runFix(ctx context.Context, cmd *base.Command, args []string) { printed := false - for _, pkg := range load.Packages(args) { + for _, pkg := range load.Packages(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not fixing packages in dependency modules\n") diff --git a/src/cmd/go/internal/fmtcmd/fmt.go b/src/cmd/go/internal/fmtcmd/fmt.go index 9868efc7ef..f96cff429c 100644 --- a/src/cmd/go/internal/fmtcmd/fmt.go +++ b/src/cmd/go/internal/fmtcmd/fmt.go @@ -64,7 +64,7 @@ func runFmt(ctx context.Context, cmd *base.Command, args []string) { } }() } - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not formatting packages in dependency modules\n") diff --git a/src/cmd/go/internal/generate/generate.go b/src/cmd/go/internal/generate/generate.go index fb26f77f95..98c17bba8c 100644 --- a/src/cmd/go/internal/generate/generate.go +++ b/src/cmd/go/internal/generate/generate.go @@ -176,7 +176,7 @@ func runGenerate(ctx context.Context, cmd *base.Command, args []string) { // Even if the arguments are .go files, this loop suffices. printed := false - for _, pkg := range load.PackagesAndErrors(args) { + for _, pkg := range load.PackagesAndErrors(ctx, args) { if modload.Enabled() && pkg.Module != nil && !pkg.Module.Main { if !printed { fmt.Fprintf(os.Stderr, "go: not generating in packages in dependency modules\n") diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index f7da5270b0..ef43602aca 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -172,7 +172,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // everything. load.ClearPackageCache() - pkgs := load.PackagesForBuild(args) + pkgs := load.PackagesForBuild(ctx, args) // Phase 3. Install. if *getD { @@ -182,7 +182,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { return } - work.InstallPackages(args, pkgs) + work.InstallPackages(ctx, args, pkgs) } // downloadPaths prepares the list of paths to pass to download. diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index ef0a5a2f2d..3ec243a759 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -449,9 +449,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { load.IgnoreImports = *listFind var pkgs []*load.Package if *listE { - pkgs = load.PackagesAndErrors(args) + pkgs = load.PackagesAndErrors(ctx, args) } else { - pkgs = load.Packages(args) + pkgs = load.Packages(ctx, args) base.ExitIfErrors() } @@ -539,7 +539,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { a.Deps = append(a.Deps, b.AutoAction(work.ModeInstall, work.ModeInstall, p)) } } - b.Do(a) + b.Do(ctx, a) } for _, p := range pkgs { diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 2b5fbb1c5b..32c2ba7912 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -7,6 +7,7 @@ package load import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -30,6 +31,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/trace" ) var ( @@ -2123,9 +2125,9 @@ func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, // to load dependencies of a named package, the named // package is still returned, with p.Incomplete = true // and details in p.DepsErrors. -func Packages(args []string) []*Package { +func Packages(ctx context.Context, args []string) []*Package { var pkgs []*Package - for _, pkg := range PackagesAndErrors(args) { + for _, pkg := range PackagesAndErrors(ctx, args) { if pkg.Error != nil { base.Errorf("%v", pkg.Error) continue @@ -2139,7 +2141,10 @@ func Packages(args []string) []*Package { // *Package for every argument, even the ones that // cannot be loaded at all. // The packages that fail to load will have p.Error != nil. -func PackagesAndErrors(patterns []string) []*Package { +func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { + ctx, span := trace.StartSpan(ctx, "load.PackagesAndErrors") + defer span.Done() + for _, p := range patterns { // Listing is only supported with all patterns referring to either: // - Files that are part of the same directory. @@ -2233,8 +2238,8 @@ func ImportPaths(args []string) []*search.Match { // PackagesForBuild is like Packages but exits // if any of the packages or their dependencies have errors // (cannot be built). -func PackagesForBuild(args []string) []*Package { - pkgs := PackagesAndErrors(args) +func PackagesForBuild(ctx context.Context, args []string) []*Package { + pkgs := PackagesAndErrors(ctx, args) printed := map[*PackageError]bool{} for _, pkg := range pkgs { if pkg.Error != nil { diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 9836a3e2cc..b217196931 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -715,8 +715,8 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { return } work.BuildInit() - pkgs := load.PackagesForBuild(pkgPatterns) - work.InstallPackages(pkgPatterns, pkgs) + pkgs := load.PackagesForBuild(ctx, pkgPatterns) + work.InstallPackages(ctx, pkgPatterns, pkgs) } // runQueries looks up modules at target versions in parallel. Results will be diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index ca2c3db92c..3630f68c54 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -79,7 +79,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { } p = load.GoFilesPackage(files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { - pkgs := load.PackagesAndErrors(args[:1]) + pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { base.Fatalf("go run: no packages loaded from %s", args[0]) } @@ -141,7 +141,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { } a1 := b.LinkAction(work.ModeBuild, work.ModeBuild, p) a := &work.Action{Mode: "go run", Func: buildRunProgram, Args: cmdArgs, Deps: []*work.Action{a1}} - b.Do(a) + b.Do(ctx, a) } // buildRunProgram is the action for running a binary that has already diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 6648d4eab4..d71d339828 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -577,7 +577,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { work.VetFlags = testVet.flags work.VetExplicit = testVet.explicit - pkgs = load.PackagesForBuild(pkgArgs) + pkgs = load.PackagesForBuild(ctx, pkgArgs) if len(pkgs) == 0 { base.Fatalf("no packages to test") } @@ -659,7 +659,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { sort.Strings(all) a := &work.Action{Mode: "go test -i"} - for _, p := range load.PackagesForBuild(all) { + for _, p := range load.PackagesForBuild(ctx, all) { if cfg.BuildToolchainName == "gccgo" && p.Standard { // gccgo's standard library packages // can not be reinstalled. @@ -667,7 +667,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } a.Deps = append(a.Deps, b.CompileAction(work.ModeInstall, work.ModeInstall, p)) } - b.Do(a) + b.Do(ctx, a) if !testC || a.Failed { return } @@ -787,7 +787,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } } - b.Do(root) + b.Do(ctx, root) } // ensures that package p imports the named package diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 717ff2d0aa..58f392eb96 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -67,7 +67,7 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { } } - pkgs := load.PackagesForBuild(pkgArgs) + pkgs := load.PackagesForBuild(ctx, pkgArgs) if len(pkgs) == 0 { base.Fatalf("no packages to vet") } @@ -93,5 +93,5 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { root.Deps = append(root.Deps, b.VetAction(work.ModeBuild, work.ModeBuild, pxtest)) } } - b.Do(root) + b.Do(ctx, root) } diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index 2bbee43ab4..d020aa6e9f 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/search" + "cmd/go/internal/trace" ) var CmdBuild = &base.Command{ @@ -350,7 +351,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { var b Builder b.Init() - pkgs := load.PackagesForBuild(args) + pkgs := load.PackagesForBuild(ctx, args) explicitO := len(cfg.BuildO) > 0 @@ -379,7 +380,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { depMode = ModeInstall } - pkgs = omitTestOnly(pkgsFilter(load.Packages(args))) + pkgs = omitTestOnly(pkgsFilter(load.Packages(ctx, args))) // Special case -o /dev/null by not writing at all. if cfg.BuildO == os.DevNull { @@ -409,7 +410,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { if len(a.Deps) == 0 { base.Fatalf("go build: no main packages to build") } - b.Do(a) + b.Do(ctx, a) return } if len(pkgs) > 1 { @@ -422,7 +423,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { p.Stale = true // must build - not up to date p.StaleReason = "build -o flag in use" a := b.AutoAction(ModeInstall, depMode, p) - b.Do(a) + b.Do(ctx, a) return } @@ -433,7 +434,7 @@ func runBuild(ctx context.Context, cmd *base.Command, args []string) { if cfg.BuildBuildmode == "shared" { a = b.buildmodeShared(ModeBuild, depMode, args, pkgs, a) } - b.Do(a) + b.Do(ctx, a) } var CmdInstall = &base.Command{ @@ -518,7 +519,7 @@ func libname(args []string, pkgs []*load.Package) (string, error) { func runInstall(ctx context.Context, cmd *base.Command, args []string) { BuildInit() - InstallPackages(args, load.PackagesForBuild(args)) + InstallPackages(ctx, args, load.PackagesForBuild(ctx, args)) } // omitTestOnly returns pkgs with test-only packages removed. @@ -538,7 +539,10 @@ func omitTestOnly(pkgs []*load.Package) []*load.Package { return list } -func InstallPackages(patterns []string, pkgs []*load.Package) { +func InstallPackages(ctx context.Context, patterns []string, pkgs []*load.Package) { + ctx, span := trace.StartSpan(ctx, "InstallPackages "+strings.Join(patterns, " ")) + defer span.Done() + if cfg.GOBIN != "" && !filepath.IsAbs(cfg.GOBIN) { base.Fatalf("cannot install, GOBIN must be an absolute path") } @@ -607,7 +611,7 @@ func InstallPackages(patterns []string, pkgs []*load.Package) { a = b.buildmodeShared(ModeInstall, ModeInstall, patterns, pkgs, a) } - b.Do(a) + b.Do(ctx, a) base.ExitIfErrors() // Success. If this command is 'go install' with no arguments diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 071c9d2db9..3ea3293ae1 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -8,6 +8,7 @@ package work import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -31,6 +32,7 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/str" + "cmd/go/internal/trace" ) // actionList returns the list of actions in the dag rooted at root @@ -54,7 +56,10 @@ func actionList(root *Action) []*Action { } // do runs the action graph rooted at root. -func (b *Builder) Do(root *Action) { +func (b *Builder) Do(ctx context.Context, root *Action) { + ctx, span := trace.StartSpan(ctx, "exec.Builder.Do ("+root.Mode+" "+root.Target+")") + defer span.Done() + if !b.IsCmdList { // If we're doing real work, take time at the end to trim the cache. c := cache.Default() -- cgit v1.3-5-g9baa From 50f63a7ae4b7f951fa894b96633b1716adca55fa Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 12 Aug 2020 13:30:43 -0400 Subject: doc/go1.15: clarify external linking can still be used for building PIE In Go 1.15 we switched the default linking mode for PIE on Linux/AMD64 and Linux/ARM64 to internal linking. Clarify that the previous behavior (external linking) can still be used with a flag. Fixes #40719. Change-Id: Ib7042622bc91e1b1aa31f520990d03b5eb6c56bb Reviewed-on: https://go-review.googlesource.com/c/go/+/248199 Reviewed-by: Ian Lance Taylor --- doc/go1.15.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/go1.15.html b/doc/go1.15.html index 8872d71138..3a12ecfa11 100644 --- a/doc/go1.15.html +++ b/doc/go1.15.html @@ -357,7 +357,10 @@ Do not send CLs removing the interior tags from such phrases. The linker now defaults to internal linking mode for -buildmode=pie on linux/amd64 and linux/arm64, so these - configurations no longer require a C linker. + configurations no longer require a C linker. External linking + mode (which was the default in Go 1.14 for + -buildmode=pie) can still be requested with + -ldflags=-linkmode=external flag.

Objdump

-- cgit v1.3-5-g9baa From b2353174db1349f15113416b4df2a833db6058a6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 12 Aug 2020 17:45:48 -0400 Subject: doc/go1.15: include behavior updates to the context package Fixes #40737 Change-Id: I8e2c1e1653d427af1ded6d61df1aa450e3c4d35c Reviewed-on: https://go-review.googlesource.com/c/go/+/248329 Run-TryBot: Andrew Bonventre TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- doc/go1.15.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/go1.15.html b/doc/go1.15.html index 3a12ecfa11..c691bf3bd5 100644 --- a/doc/go1.15.html +++ b/doc/go1.15.html @@ -436,6 +436,19 @@ Do not send CLs removing the interior tags from such phrases. +
context
+
+

+ Creating a derived Context using a nil parent is now explicitly + disallowed. Any attempt to do so with the + WithValue, + WithDeadline, or + WithCancel functions + will cause a panic. +

+
+
+
crypto

-- cgit v1.3-5-g9baa From 92bda33d2771a9b12868d9025f113538fa7a84de Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 31 Jul 2020 15:58:00 -0400 Subject: runtime: revert signal stack mlocking Go 1.14 included a (rather awful) workaround for a Linux kernel bug that corrupted vector registers on x86 CPUs during signal delivery (https://bugzilla.kernel.org/show_bug.cgi?id=205663). This bug was introduced in Linux 5.2 and fixed in 5.3.15, 5.4.2 and all 5.5 and later kernels. The fix was also back-ported by major distros. This workaround was necessary, but had unfortunate downsides, including causing Go programs to exceed the mlock ulimit in many configurations (#37436). We're reasonably confident that by the Go 1.16 release, the number of systems running affected kernels will be vanishingly small. Hence, this CL removes this workaround. This effectively reverts CLs 209597 (version parser), 209899 (mlock top of signal stack), 210299 (better failure message), 223121 (soft mlock failure handling), and 244059 (special-case patched Ubuntu kernels). The one thing we keep is the osArchInit function. It's empty everywhere now, but is a reasonable hook to have. Updates #35326, #35777 (the original register corruption bugs). Updates #40184 (request to revert in 1.15). Fixes #35979. Change-Id: Ie213270837095576f1f3ef46bf3de187dc486c50 Reviewed-on: https://go-review.googlesource.com/c/go/+/246200 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/defs_linux_386.go | 11 ---- src/runtime/defs_linux_amd64.go | 11 ---- src/runtime/export_test.go | 2 - src/runtime/os_linux.go | 9 --- src/runtime/os_linux_x86.go | 118 +--------------------------------------- src/runtime/panic.go | 10 ---- src/runtime/string.go | 34 ------------ src/runtime/string_test.go | 31 ----------- src/runtime/sys_linux_386.s | 19 ------- src/runtime/sys_linux_amd64.s | 19 ------- 10 files changed, 1 insertion(+), 263 deletions(-) diff --git a/src/runtime/defs_linux_386.go b/src/runtime/defs_linux_386.go index f4db8cf927..64a0fbcaaa 100644 --- a/src/runtime/defs_linux_386.go +++ b/src/runtime/defs_linux_386.go @@ -226,14 +226,3 @@ type sockaddr_un struct { family uint16 path [108]byte } - -const __NEW_UTS_LEN = 64 - -type new_utsname struct { - sysname [__NEW_UTS_LEN + 1]byte - nodename [__NEW_UTS_LEN + 1]byte - release [__NEW_UTS_LEN + 1]byte - version [__NEW_UTS_LEN + 1]byte - machine [__NEW_UTS_LEN + 1]byte - domainname [__NEW_UTS_LEN + 1]byte -} diff --git a/src/runtime/defs_linux_amd64.go b/src/runtime/defs_linux_amd64.go index 8480d85219..1ae18a309b 100644 --- a/src/runtime/defs_linux_amd64.go +++ b/src/runtime/defs_linux_amd64.go @@ -262,14 +262,3 @@ type sockaddr_un struct { family uint16 path [108]byte } - -const __NEW_UTS_LEN = 64 - -type new_utsname struct { - sysname [__NEW_UTS_LEN + 1]byte - nodename [__NEW_UTS_LEN + 1]byte - release [__NEW_UTS_LEN + 1]byte - version [__NEW_UTS_LEN + 1]byte - machine [__NEW_UTS_LEN + 1]byte - domainname [__NEW_UTS_LEN + 1]byte -} diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go index 5ab03f3f99..d591fdc4e9 100644 --- a/src/runtime/export_test.go +++ b/src/runtime/export_test.go @@ -43,8 +43,6 @@ var PhysHugePageSize = physHugePageSize var NetpollGenericInit = netpollGenericInit -var ParseRelease = parseRelease - var Memmove = memmove var MemclrNoHeapPointers = memclrNoHeapPointers diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 7b95ff2428..22931b4d5c 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -328,20 +328,11 @@ func libpreinit() { initsig(true) } -// gsignalInitQuirk, if non-nil, is called for every allocated gsignal G. -// -// TODO(austin): Remove this after Go 1.15 when we remove the -// mlockGsignal workaround. -var gsignalInitQuirk func(gsignal *g) - // Called to initialize a new m (including the bootstrap m). // Called on the parent thread (main thread in case of bootstrap), can allocate memory. func mpreinit(mp *m) { mp.gsignal = malg(32 * 1024) // Linux wants >= 2K mp.gsignal.m = mp - if gsignalInitQuirk != nil { - gsignalInitQuirk(mp.gsignal) - } } func gettid() uint32 diff --git a/src/runtime/os_linux_x86.go b/src/runtime/os_linux_x86.go index 97f870707d..d91fa1a0d1 100644 --- a/src/runtime/os_linux_x86.go +++ b/src/runtime/os_linux_x86.go @@ -7,120 +7,4 @@ package runtime -import ( - "runtime/internal/atomic" - "unsafe" -) - -//go:noescape -func uname(utsname *new_utsname) int - -func mlock(addr, len uintptr) int - -func osArchInit() { - // Linux 5.2 introduced a bug that can corrupt vector - // registers on return from a signal if the signal stack isn't - // faulted in: - // https://bugzilla.kernel.org/show_bug.cgi?id=205663 - // - // It was fixed in 5.3.15, 5.4.2, and all 5.5 and later - // kernels. - // - // If we're on an affected kernel, work around this issue by - // mlocking the top page of every signal stack. This doesn't - // help for signal stacks created in C, but there's not much - // we can do about that. - // - // TODO(austin): Remove this in Go 1.15, at which point it - // will be unlikely to encounter any of the affected kernels - // in the wild. - - var uts new_utsname - if uname(&uts) < 0 { - throw("uname failed") - } - // Check for null terminator to ensure gostringnocopy doesn't - // walk off the end of the release string. - found := false - for _, b := range uts.release { - if b == 0 { - found = true - break - } - } - if !found { - return - } - rel := gostringnocopy(&uts.release[0]) - - major, minor, patch, ok := parseRelease(rel) - if !ok { - return - } - - if major == 5 && minor == 4 && patch < 2 { - // All 5.4 versions of Ubuntu are patched. - procVersion := []byte("/proc/version\000") - f := open(&procVersion[0], _O_RDONLY, 0) - if f >= 0 { - var buf [512]byte - p := noescape(unsafe.Pointer(&buf[0])) - n := read(f, p, int32(len(buf))) - closefd(f) - - needle := []byte("Ubuntu") - contains: - for i, c := range buf[:n] { - if c != needle[0] { - continue - } - if int(n)-i < len(needle) { - break - } - for j, c2 := range needle { - if c2 != buf[i+j] { - continue contains - } - } - // This is an Ubuntu system. - return - } - } - } - - if major == 5 && (minor == 2 || minor == 3 && patch < 15 || minor == 4 && patch < 2) { - gsignalInitQuirk = mlockGsignal - if m0.gsignal != nil { - throw("gsignal quirk too late") - } - throwReportQuirk = throwBadKernel - } -} - -func mlockGsignal(gsignal *g) { - if atomic.Load(&touchStackBeforeSignal) != 0 { - // mlock has already failed, don't try again. - return - } - - // This mlock call may fail, but we don't report the failure. - // Instead, if something goes badly wrong, we rely on prepareSignalM - // and throwBadKernel to do further mitigation and to report a problem - // to the user if mitigation fails. This is because many - // systems have a limit on the total mlock size, and many kernels - // that appear to have bad versions are actually patched to avoid the - // bug described above. We want Go 1.14 to run on those systems. - // See #37436. - if errno := mlock(gsignal.stack.hi-physPageSize, physPageSize); errno < 0 { - atomic.Store(&touchStackBeforeSignal, uint32(-errno)) - } -} - -// throwBadKernel is called, via throwReportQuirk, by throw. -func throwBadKernel() { - if errno := atomic.Load(&touchStackBeforeSignal); errno != 0 { - println("runtime: note: your Linux kernel may be buggy") - println("runtime: note: see https://golang.org/wiki/LinuxKernelSignalVectorBug") - println("runtime: note: mlock workaround for kernel bug failed with errno", errno) - } -} +func osArchInit() {} diff --git a/src/runtime/panic.go b/src/runtime/panic.go index 615249f33c..127843b081 100644 --- a/src/runtime/panic.go +++ b/src/runtime/panic.go @@ -1283,12 +1283,6 @@ func startpanic_m() bool { } } -// throwReportQuirk, if non-nil, is called by throw after dumping the stacks. -// -// TODO(austin): Remove this after Go 1.15 when we remove the -// mlockGsignal workaround. -var throwReportQuirk func() - var didothers bool var deadlock mutex @@ -1335,10 +1329,6 @@ func dopanic_m(gp *g, pc, sp uintptr) bool { printDebugLog() - if throwReportQuirk != nil { - throwReportQuirk() - } - return docrash } diff --git a/src/runtime/string.go b/src/runtime/string.go index 0515b56573..251044231e 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -499,37 +499,3 @@ func gostringw(strw *uint16) string { b[n2] = 0 // for luck return s[:n2] } - -// parseRelease parses a dot-separated version number. It follows the -// semver syntax, but allows the minor and patch versions to be -// elided. -func parseRelease(rel string) (major, minor, patch int, ok bool) { - // Strip anything after a dash or plus. - for i := 0; i < len(rel); i++ { - if rel[i] == '-' || rel[i] == '+' { - rel = rel[:i] - break - } - } - - next := func() (int, bool) { - for i := 0; i < len(rel); i++ { - if rel[i] == '.' { - ver, ok := atoi(rel[:i]) - rel = rel[i+1:] - return ver, ok - } - } - ver, ok := atoi(rel) - rel = "" - return ver, ok - } - if major, ok = next(); !ok || rel == "" { - return - } - if minor, ok = next(); !ok || rel == "" { - return - } - patch, ok = next() - return -} diff --git a/src/runtime/string_test.go b/src/runtime/string_test.go index b9ac667533..4eda12c35d 100644 --- a/src/runtime/string_test.go +++ b/src/runtime/string_test.go @@ -454,34 +454,3 @@ func TestAtoi32(t *testing.T) { } } } - -type parseReleaseTest struct { - in string - major, minor, patch int -} - -var parseReleaseTests = []parseReleaseTest{ - {"", -1, -1, -1}, - {"x", -1, -1, -1}, - {"5", 5, 0, 0}, - {"5.12", 5, 12, 0}, - {"5.12-x", 5, 12, 0}, - {"5.12.1", 5, 12, 1}, - {"5.12.1-x", 5, 12, 1}, - {"5.12.1.0", 5, 12, 1}, - {"5.20496382327982653440", -1, -1, -1}, -} - -func TestParseRelease(t *testing.T) { - for _, test := range parseReleaseTests { - major, minor, patch, ok := runtime.ParseRelease(test.in) - if !ok { - major, minor, patch = -1, -1, -1 - } - if test.major != major || test.minor != minor || test.patch != patch { - t.Errorf("parseRelease(%q) = (%v, %v, %v) want (%v, %v, %v)", - test.in, major, minor, patch, - test.major, test.minor, test.patch) - } - } -} diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 5b9b638ad7..1e3a834812 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -39,8 +39,6 @@ #define SYS_socketcall 102 #define SYS_setittimer 104 #define SYS_clone 120 -#define SYS_uname 122 -#define SYS_mlock 150 #define SYS_sched_yield 158 #define SYS_nanosleep 162 #define SYS_rt_sigreturn 173 @@ -808,20 +806,3 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-4 INVOKE_SYSCALL MOVL AX, ret+0(FP) RET - -// func uname(utsname *new_utsname) int -TEXT ·uname(SB),NOSPLIT,$0-8 - MOVL $SYS_uname, AX - MOVL utsname+0(FP), BX - INVOKE_SYSCALL - MOVL AX, ret+4(FP) - RET - -// func mlock(addr, len uintptr) int -TEXT ·mlock(SB),NOSPLIT,$0-12 - MOVL $SYS_mlock, AX - MOVL addr+0(FP), BX - MOVL len+4(FP), CX - INVOKE_SYSCALL - MOVL AX, ret+8(FP) - RET diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index fe9c6bce85..b60057ce83 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -33,10 +33,8 @@ #define SYS_clone 56 #define SYS_exit 60 #define SYS_kill 62 -#define SYS_uname 63 #define SYS_fcntl 72 #define SYS_sigaltstack 131 -#define SYS_mlock 149 #define SYS_arch_prctl 158 #define SYS_gettid 186 #define SYS_futex 202 @@ -789,20 +787,3 @@ TEXT runtime·sbrk0(SB),NOSPLIT,$0-8 SYSCALL MOVQ AX, ret+0(FP) RET - -// func uname(utsname *new_utsname) int -TEXT ·uname(SB),NOSPLIT,$0-16 - MOVQ utsname+0(FP), DI - MOVL $SYS_uname, AX - SYSCALL - MOVQ AX, ret+8(FP) - RET - -// func mlock(addr, len uintptr) int -TEXT ·mlock(SB),NOSPLIT,$0-24 - MOVQ addr+0(FP), DI - MOVQ len+8(FP), SI - MOVL $SYS_mlock, AX - SYSCALL - MOVQ AX, ret+16(FP) - RET -- cgit v1.3-5-g9baa From 933ca0cfdc5b16a28661707b95485ce6d739bb97 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Wed, 12 Aug 2020 12:59:55 -0400 Subject: doc: add a release notes blurb on 1.16 linker improvements Add a draft version of a blurb on improvements to the linker. This will need to be finalized later in the release since there are still some additional changes to be made to the linker in 1.16. Updates #40703. Change-Id: Id85c7e129071cc2faacb09c53a2968bd52b0a7b4 Reviewed-on: https://go-review.googlesource.com/c/go/+/248238 Reviewed-by: Cherry Zhang Reviewed-by: Austin Clements --- doc/go1.16.html | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/go1.16.html b/doc/go1.16.html index f0e26a1033..1fea359436 100644 --- a/doc/go1.16.html +++ b/doc/go1.16.html @@ -62,7 +62,26 @@ Do not send CLs removing the interior tags from such phrases.

Linker

- TODO + This release includes additional improvements to the Go linker, + reducing linker resource usage (both time and memory) and improving + code robustness/maintainability. These changes form the second half + of a two-release project to + modernize the Go + linker. +

+ +

+ The linker changes in 1.16 extend the 1.15 improvements to all + supported architecture/OS combinations (the 1.15 performance improvements + were primarily focused on ELF-based OSes and + amd64 architectures). For a representative set of + large Go programs, linking is 20-35% faster than 1.15 and requires + 5-15% less memory on average for linux/amd64, with larger + improvements for other architectures and OSes. +

+ +

+ TODO: update with final numbers later in the release.

Core library

-- cgit v1.3-5-g9baa From c2e73fb446bffd02c651e51c6641cc90fd065b70 Mon Sep 17 00:00:00 2001 From: Than McIntosh Date: Tue, 23 Jun 2020 08:46:36 -0400 Subject: cmd/compile: remove AttrSeenGlobl (use AttrOnList instead) Minor cleanup: remove the symbol attribute AttrSeenGlobal, since it is redundant with the existing attribute AttrOnList (no need to have what amounts to a separate flag for checking the same property). Change-Id: Ia269b64de37c2bb4a2314bbecf3d2091c6d57424 Reviewed-on: https://go-review.googlesource.com/c/go/+/239477 Run-TryBot: Than McIntosh TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/obj.go | 2 +- src/cmd/internal/obj/link.go | 3 --- src/cmd/internal/obj/plist.go | 4 ---- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index 0826b04e33..af5037c5a8 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -352,7 +352,7 @@ func stringsym(pos src.XPos, s string) (data *obj.LSym) { symdata := Ctxt.Lookup(symdataname) - if !symdata.SeenGlobl() { + if !symdata.OnList() { // string data off := dsname(symdata, 0, s, pos, "string") ggloblsym(symdata, int32(off), obj.DUPOK|obj.RODATA|obj.LOCAL) diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index dc47e51be9..311e5ae2e8 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -480,7 +480,6 @@ const ( AttrWrapper AttrNeedCtxt AttrNoFrame - AttrSeenGlobl AttrOnList AttrStatic @@ -537,7 +536,6 @@ func (a Attribute) MakeTypelink() bool { return a&AttrMakeTypelink != 0 } func (a Attribute) CFunc() bool { return a&AttrCFunc != 0 } func (a Attribute) NoSplit() bool { return a&AttrNoSplit != 0 } func (a Attribute) Leaf() bool { return a&AttrLeaf != 0 } -func (a Attribute) SeenGlobl() bool { return a&AttrSeenGlobl != 0 } func (a Attribute) OnList() bool { return a&AttrOnList != 0 } func (a Attribute) ReflectMethod() bool { return a&AttrReflectMethod != 0 } func (a Attribute) Local() bool { return a&AttrLocal != 0 } @@ -574,7 +572,6 @@ var textAttrStrings = [...]struct { {bit: AttrCFunc, s: "CFUNC"}, {bit: AttrNoSplit, s: "NOSPLIT"}, {bit: AttrLeaf, s: "LEAF"}, - {bit: AttrSeenGlobl, s: ""}, {bit: AttrOnList, s: ""}, {bit: AttrReflectMethod, s: "REFLECTMETHOD"}, {bit: AttrLocal, s: "LOCAL"}, diff --git a/src/cmd/internal/obj/plist.go b/src/cmd/internal/obj/plist.go index afe0ee4ee0..6e33f29959 100644 --- a/src/cmd/internal/obj/plist.go +++ b/src/cmd/internal/obj/plist.go @@ -145,10 +145,6 @@ func (ctxt *Link) InitTextSym(s *LSym, flag int) { } func (ctxt *Link) Globl(s *LSym, size int64, flag int) { - if s.SeenGlobl() { - fmt.Printf("duplicate %v\n", s) - } - s.Set(AttrSeenGlobl, true) if s.OnList() { ctxt.Diag("symbol %s listed multiple times", s.Name) } -- cgit v1.3-5-g9baa From 7d7bd5abc7f7ac901830b79496f63ce86895e262 Mon Sep 17 00:00:00 2001 From: Lynn Boger Date: Tue, 11 Aug 2020 12:04:25 -0400 Subject: cmd/internal/obj/ppc64: don't remove NOP in assembler Previously, the assembler removed NOPs from the Prog list in obj9.go. NOPs shouldn't be removed if they were added as an inline mark, as described in the issue below. Fixes #40689 Once the NOPs were left in the Prog list, some instructions were flagged as invalid because they had an operand which was not represented in optab. In order to preserve the previous assembler behavior, entries were added to optab for those operand cases. They were not flagged as errors before because the NOP instructions were removed before the code to check the valid opcode/operand combinations. Change-Id: Iae5145f94459027cf458e914d7c5d6089807ccf8 Reviewed-on: https://go-review.googlesource.com/c/go/+/247842 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Paul Murphy Reviewed-by: Michael Munday Reviewed-by: Keith Randall --- src/cmd/internal/obj/ppc64/asm9.go | 3 +++ src/cmd/internal/obj/ppc64/obj9.go | 11 +++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index 0fd0744a42..238ca8f0b7 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -613,6 +613,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_NONE, C_LCON, 0, 0, 0}, {obj.AFUNCDATA, C_SCON, C_NONE, C_NONE, C_ADDR, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // NOP operand variations added for #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // to preserve previous behavior + {obj.ANOP, C_FREG, C_NONE, C_NONE, C_NONE, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_LBRA, 11, 4, 0}, // same as ABR/ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_LBRA, 11, 4, 0}, // same as ABR/ABL {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0}, // align code diff --git a/src/cmd/internal/obj/ppc64/obj9.go b/src/cmd/internal/obj/ppc64/obj9.go index 16881c634b..749f7066de 100644 --- a/src/cmd/internal/obj/ppc64/obj9.go +++ b/src/cmd/internal/obj/ppc64/obj9.go @@ -429,7 +429,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs * expand RET * expand BECOME pseudo */ @@ -559,10 +558,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { q = p q1 = p.Pcond if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } + // NOPs are not removed due to #40689. if q1.Mark&LEAF == 0 { q1.Mark |= LABEL @@ -589,9 +585,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { continue case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark + // NOPs are not removed due to + // #40689 continue default: -- cgit v1.3-5-g9baa From cde5fd1c0f8c40804bfd942eec1e2d69bccf4e13 Mon Sep 17 00:00:00 2001 From: Josh Bleecher Snyder Date: Thu, 13 Aug 2020 12:39:04 -0700 Subject: cmd/compile: correct type of CvtBoolToUint8 values Fixes #40746 Change-Id: I539f07d1f958dacee87d846171a8889d03182d25 Reviewed-on: https://go-review.googlesource.com/c/go/+/248397 Run-TryBot: Josh Bleecher Snyder TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/phiopt.go | 2 +- src/cmd/compile/internal/ssa/prove.go | 2 +- test/fixedbugs/issue40746.go | 19 +++++++++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 test/fixedbugs/issue40746.go diff --git a/src/cmd/compile/internal/ssa/phiopt.go b/src/cmd/compile/internal/ssa/phiopt.go index 8643fa584c..db7b02275c 100644 --- a/src/cmd/compile/internal/ssa/phiopt.go +++ b/src/cmd/compile/internal/ssa/phiopt.go @@ -154,7 +154,7 @@ func phioptint(v *Value, b0 *Block, reverse int) { } v.AddArg(a) - cvt := v.Block.NewValue1(v.Pos, OpCvtBoolToUint8, a.Type, a) + cvt := v.Block.NewValue1(v.Pos, OpCvtBoolToUint8, v.Block.Func.Config.Types.UInt8, a) switch v.Type.Size() { case 1: v.reset(OpCopy) diff --git a/src/cmd/compile/internal/ssa/prove.go b/src/cmd/compile/internal/ssa/prove.go index 6c6be39d34..ce7d689f93 100644 --- a/src/cmd/compile/internal/ssa/prove.go +++ b/src/cmd/compile/internal/ssa/prove.go @@ -1334,7 +1334,7 @@ func removeBranch(b *Block, branch branch) { // isNonNegative reports whether v is known to be greater or equal to zero. func isNonNegative(v *Value) bool { if !v.Type.IsInteger() { - panic("isNonNegative bad type") + v.Fatalf("isNonNegative bad type: %v", v.Type) } // TODO: return true if !v.Type.IsSigned() // SSA isn't type-safe enough to do that now (issue 37753). diff --git a/test/fixedbugs/issue40746.go b/test/fixedbugs/issue40746.go new file mode 100644 index 0000000000..235282fd90 --- /dev/null +++ b/test/fixedbugs/issue40746.go @@ -0,0 +1,19 @@ +// compile + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +func f(x byte, b bool) byte { + var c byte + if b { + c = 1 + } + + if int8(c) < 0 { + x++ + } + return x +} -- cgit v1.3-5-g9baa From d0d6593d1d4e81acd073244f42b6893fa65c99d8 Mon Sep 17 00:00:00 2001 From: Michael Munday Date: Mon, 10 Aug 2020 08:01:21 -0700 Subject: cmd/internal/obj: fix inline marker issue on s390x The optimization that replaces inline markers with pre-existing instructions assumes that 'Prog' values produced by the compiler are still reachable after the assembler has run. This was not true on s390x where the assembler was removing NOP instructions from the linked list of 'Prog' values. This led to broken inlining data which in turn caused an infinite loop in the runtime traceback code. Fix this by stopping the s390x assembler backend removing NOP values. It does not make any difference to the output of the assembler because NOP instructions are 0 bytes long anyway. Fixes #40473. Change-Id: Ib4fabadd1de8adb80421f75950ee9aad2111147a Reviewed-on: https://go-review.googlesource.com/c/go/+/247697 Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/internal/obj/pcln.go | 15 +++++++++++++++ src/cmd/internal/obj/s390x/objz.go | 11 ----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index 1f7ccf47ef..bffeda041d 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -278,6 +278,21 @@ func linkpcln(ctxt *Link, cursym *LSym) { funcpctab(ctxt, &pcln.Pcfile, cursym, "pctofile", pctofileline, pcln) funcpctab(ctxt, &pcln.Pcline, cursym, "pctoline", pctofileline, nil) + // Check that all the Progs used as inline markers are still reachable. + // See issue #40473. + inlMarkProgs := make(map[*Prog]struct{}, len(cursym.Func.InlMarks)) + for _, inlMark := range cursym.Func.InlMarks { + inlMarkProgs[inlMark.p] = struct{}{} + } + for p := cursym.Func.Text; p != nil; p = p.Link { + if _, ok := inlMarkProgs[p]; ok { + delete(inlMarkProgs, p) + } + } + if len(inlMarkProgs) > 0 { + ctxt.Diag("one or more instructions used as inline markers are no longer reachable") + } + pcinlineState := new(pcinlineState) funcpctab(ctxt, &pcln.Pcinline, cursym, "pctoinline", pcinlineState.pctoinline, nil) for _, inlMark := range cursym.Func.InlMarks { diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index b14dc810fa..ef6335d849 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -283,17 +283,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ACMPUBNE: q = p p.Mark |= BRANCH - if p.Pcond != nil { - q := p.Pcond - for q.As == obj.ANOP { - q = q.Link - p.Pcond = q - } - } - - case obj.ANOP: - q.Link = p.Link /* q is non-nop */ - p.Link.Mark |= p.Mark default: q = p -- cgit v1.3-5-g9baa From a20cb4ca5c14ff27bdf16989d450c83b22f156d8 Mon Sep 17 00:00:00 2001 From: Tim Möhlmann Date: Thu, 13 Aug 2020 11:56:31 +0300 Subject: database/sql: make Rows.Scan properly wrap underlying errors The prior implementation used the format verb %v which unfortunately improperly wrapped any underlying scanner errors, and we couldn't use errors.Is nor errors.As. This change fixes that by using the %w verb. Added a unit to ensure that both error sub string matching works, but also that errors.Is works as expected. Fixes #38099 Change-Id: Iea667041dd8081d961246f77f2542330417292dc Reviewed-on: https://go-review.googlesource.com/c/go/+/248337 Reviewed-by: Emmanuel Odeke Reviewed-by: Daniel Theophanes Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot --- src/database/sql/sql.go | 5 ++++- src/database/sql/sql_test.go | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/database/sql/sql.go b/src/database/sql/sql.go index b3d0653f5c..0b85db66b9 100644 --- a/src/database/sql/sql.go +++ b/src/database/sql/sql.go @@ -3110,6 +3110,9 @@ func rowsColumnInfoSetupConnLocked(rowsi driver.Rows) []*ColumnType { // "select cursor(select * from my_table) from dual", into a // *Rows value that can itself be scanned from. The parent // select query will close any cursor *Rows if the parent *Rows is closed. +// +// If any of the first arguments implementing Scanner returns an error, +// that error will be wrapped in the returned error func (rs *Rows) Scan(dest ...interface{}) error { rs.closemu.RLock() @@ -3133,7 +3136,7 @@ func (rs *Rows) Scan(dest ...interface{}) error { for i, sv := range rs.lastcols { err := convertAssignRows(dest[i], sv, rs) if err != nil { - return fmt.Errorf(`sql: Scan error on column index %d, name %q: %v`, i, rs.rowsi.Columns()[i], err) + return fmt.Errorf(`sql: Scan error on column index %d, name %q: %w`, i, rs.rowsi.Columns()[i], err) } } return nil diff --git a/src/database/sql/sql_test.go b/src/database/sql/sql_test.go index 5727f0d8aa..762d42f54b 100644 --- a/src/database/sql/sql_test.go +++ b/src/database/sql/sql_test.go @@ -4149,6 +4149,41 @@ func TestQueryExecContextOnly(t *testing.T) { } } +type alwaysErrScanner struct{} + +var errTestScanWrap = errors.New("errTestScanWrap") + +func (alwaysErrScanner) Scan(interface{}) error { + return errTestScanWrap +} + +// Issue 38099: Ensure that Rows.Scan properly wraps underlying errors. +func TestRowsScanProperlyWrapsErrors(t *testing.T) { + db := newTestDB(t, "people") + defer closeDB(t, db) + + rows, err := db.Query("SELECT|people|age|") + if err != nil { + t.Fatalf("Query: %v", err) + } + + var res alwaysErrScanner + + for rows.Next() { + err = rows.Scan(&res) + if err == nil { + t.Fatal("expecting back an error") + } + if !errors.Is(err, errTestScanWrap) { + t.Fatalf("errors.Is mismatch\n%v\nWant: %v", err, errTestScanWrap) + } + // Ensure that error substring matching still correctly works. + if !strings.Contains(err.Error(), errTestScanWrap.Error()) { + t.Fatalf("Error %v does not contain %v", err, errTestScanWrap) + } + } +} + // badConn implements a bad driver.Conn, for TestBadDriver. // The Exec method panics. type badConn struct{} -- cgit v1.3-5-g9baa From 5ae198087bd07e88009885ac96c864381f8d8272 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 4 Jun 2020 11:41:43 -0400 Subject: cmd/go: don't initialize Builder in envcmd.MkEnv The Builder isn't needed by MkEnv, and Builder.Init doesn't have side effects that change the environment. Builder.Init does currently call CheckGOOSARCHPair, but that's being moved out in CL 234658. Builder.Init creates the temporary work directory used by the builder. For the builder created in MkEnv, this directory is never used. Creating this directory can cause unnecessary errors for commands that don't use a builder like 'go clean' and 'go list'. Fixes #38395 Updates #24398 Change-Id: Ib93ae55afdf958000470657f4c4ff5bd92700e46 Reviewed-on: https://go-review.googlesource.com/c/go/+/236563 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/envcmd/env.go | 3 -- src/cmd/go/testdata/script/build_GOTMPDIR.txt | 49 +++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index 403e0f4a7b..7bd75f7305 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -63,9 +63,6 @@ var ( ) func MkEnv() []cfg.EnvVar { - var b work.Builder - b.Init() - envFile, _ := cfg.EnvFile() env := []cfg.EnvVar{ {Name: "GO111MODULE", Value: cfg.Getenv("GO111MODULE")}, diff --git a/src/cmd/go/testdata/script/build_GOTMPDIR.txt b/src/cmd/go/testdata/script/build_GOTMPDIR.txt index c93ca932ca..1073517c29 100644 --- a/src/cmd/go/testdata/script/build_GOTMPDIR.txt +++ b/src/cmd/go/testdata/script/build_GOTMPDIR.txt @@ -1,15 +1,50 @@ -env GO111MODULE=off -[short] skip - # Set GOCACHE to a clean directory to ensure that 'go build' has work to report. -env GOCACHE=$WORK/gocache +[!windows] env GOCACHE=$WORK/gocache +[windows] env GOCACHE=$WORK\gocache -# Build should use GOTMPDIR if set. -env GOTMPDIR=$WORK/my-favorite-tmpdir +# 'go build' should use GOTMPDIR if set. +[!windows] env GOTMPDIR=$WORK/my-favorite-tmpdir +[windows] env GOTMPDIR=$WORK\my-favorite-tmpdir mkdir $GOTMPDIR -go build -work hello.go +go build -x hello.go stderr ^WORK=.*my-favorite-tmpdir +# Make GOTMPDIR a regular file. This prevents the creation of work directories, +# so we can check that certain commands don't create them. +# This simulates running on a full disk or a read-only volume. +rm $GOTMPDIR +cp hello.go $GOTMPDIR # any file will do + +# 'go build' should fail if GOTMPDIR is read-only. +! go build -x . +stderr '^go: creating work dir: \w+ '$GOTMPDIR + +# 'go list' should only fail if it needs to build something. +go list -x . +! stderr 'creating work dir' +stdout m +go list -m all +stdout m +! go list -x -export . +stderr '^go: creating work dir: \w+ '$GOTMPDIR + +# 'go clean -cache' and 'go clean -modcache' should not fail. +go clean -x -cache +! stderr 'creating work dir' +go clean -x -modcache +! stderr 'creating work dir' + +# 'go env' should not fail for specific variables. +# Without arguments, it needs to initialize a builder to load cgo flags, and +# that uses a temporary directory. +! go env +stderr '^go: creating work dir: \w+ '$GOTMPDIR +go env GOROOT + +-- go.mod -- +module m + +go 1.15 -- hello.go -- package main func main() { println("hello") } -- cgit v1.3-5-g9baa From 016e13df7475329c65524b2eabbc5207ceb4ee74 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 29 Jun 2020 16:28:18 -0400 Subject: cmd/go/internal/modfetch: stop migrating go.modverify to go.sum go.modverify was renamed to go.sum before vgo was merged into cmd/go. It's been long enough that we can safely drop support for it. For #25525 Change-Id: If8da66280a0fb6a4d4db0b170700775523c18571 Reviewed-on: https://go-review.googlesource.com/c/go/+/240458 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modfetch/fetch.go | 31 ++++--------------------------- src/cmd/go/testdata/script/mod_verify.txt | 8 +++----- 2 files changed, 7 insertions(+), 32 deletions(-) diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index fd7a5cef83..8df2289097 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -374,12 +374,11 @@ type modSum struct { var goSum struct { mu sync.Mutex - m map[module.Version][]string // content of go.sum file (+ go.modverify if present) + m map[module.Version][]string // content of go.sum file checked map[modSum]bool // sums actually checked during execution dirty bool // whether we added any new sums to m overwrite bool // if true, overwrite go.sum without incorporating its contents enabled bool // whether to use go.sum at all - modverify string // path to go.modverify, to be deleted } // initGoSum initializes the go.sum data. @@ -403,19 +402,6 @@ func initGoSum() (bool, error) { goSum.enabled = true readGoSum(goSum.m, GoSumFile, data) - // Add old go.modverify file. - // We'll delete go.modverify in WriteGoSum. - alt := strings.TrimSuffix(GoSumFile, ".sum") + ".modverify" - if data, err := renameio.ReadFile(alt); err == nil { - migrate := make(map[module.Version][]string) - readGoSum(migrate, alt, data) - for mod, sums := range migrate { - for _, sum := range sums { - addModSumLocked(mod, sum) - } - } - goSum.modverify = alt - } return true, nil } @@ -616,14 +602,9 @@ func WriteGoSum() { goSum.mu.Lock() defer goSum.mu.Unlock() - if !goSum.enabled { - // If we haven't read the go.sum file yet, don't bother writing it: at best, - // we could rename the go.modverify file if it isn't empty, but we haven't - // needed to touch it so far — how important could it be? - return - } - if !goSum.dirty { - // Don't bother opening the go.sum file if we don't have anything to add. + if !goSum.enabled || !goSum.dirty { + // If we haven't read go.sum yet or if we don't have anything to add, + // don't bother opening it. return } if cfg.BuildMod == "readonly" { @@ -674,10 +655,6 @@ func WriteGoSum() { goSum.checked = make(map[modSum]bool) goSum.dirty = false goSum.overwrite = false - - if goSum.modverify != "" { - os.Remove(goSum.modverify) // best effort - } } // TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. diff --git a/src/cmd/go/testdata/script/mod_verify.txt b/src/cmd/go/testdata/script/mod_verify.txt index 646bc62bb7..3918400435 100644 --- a/src/cmd/go/testdata/script/mod_verify.txt +++ b/src/cmd/go/testdata/script/mod_verify.txt @@ -12,20 +12,18 @@ go mod verify ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip # With bad go.sum, sync (which must download) fails. -# Even if the bad sum is in the old legacy go.modverify file. rm go.sum -cp go.sum.bad go.modverify +cp go.sum.bad go.sum ! go mod tidy stderr 'checksum mismatch' ! exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip -# With good go.sum, sync works (and moves go.modverify to go.sum). +# With good go.sum, sync works. rm go.sum -cp go.sum.good go.modverify +cp go.sum.good go.sum go mod tidy exists $GOPATH/pkg/mod/cache/download/rsc.io/quote/@v/v1.1.0.zip exists $GOPATH/pkg/mod/rsc.io/quote@v1.1.0/quote.go -! exists go.modverify # go.sum should have the new checksum for go.mod grep '^rsc.io/quote v1.1.0/go.mod ' go.sum -- cgit v1.3-5-g9baa From 9a759593d7a71b4c061fd9bd053bd79584c632dc Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Mon, 8 Jun 2020 18:06:11 -0400 Subject: cmd/go: don't save sums for modules loaded for import resolution modfetch.WriteGoSum now accepts a map[module.Version]bool parameter. This is used to prevent some new sums from being saved to go.sum when they would be removed by the next 'go mod tidy'. Previusly, sums were saved for modules looked up during import resolution. A new function, modload.TrimGoSum, is also introduced, which marks sums for deletion. 'go mod tidy' now uses this. The new logic distinguishes between go.mod sums and content sums, which lets 'go mod tidy' delete sums for modules in the build graph but not the build list. Fixes #31580 Fixes #36260 Fixes #33008 Change-Id: I06c4125704a8bbc9969de05265967ec1d2e6d3e8 Reviewed-on: https://go-review.googlesource.com/c/go/+/237017 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Michael Matloob Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modcmd/init.go | 1 + src/cmd/go/internal/modcmd/tidy.go | 38 +----------- src/cmd/go/internal/modfetch/fetch.go | 83 +++++++++++++++++++-------- src/cmd/go/internal/modload/init.go | 69 ++++++++++++++++++++-- src/cmd/go/testdata/script/mod_sum_lookup.txt | 33 +++++++++++ src/cmd/go/testdata/script/mod_tidy_old.txt | 46 +++++++++++++++ 6 files changed, 204 insertions(+), 66 deletions(-) create mode 100644 src/cmd/go/testdata/script/mod_sum_lookup.txt create mode 100644 src/cmd/go/testdata/script/mod_tidy_old.txt diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index ddb9aeebe9..95063e62f4 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -52,4 +52,5 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod init: module path must not contain '@'") } modload.InitMod() // does all the hard work + modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index feb41a83b0..769cd11fe8 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -9,12 +9,9 @@ package modcmd import ( "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" "cmd/go/internal/modload" "cmd/go/internal/work" "context" - - "golang.org/x/mod/module" ) var cmdTidy = &base.Command{ @@ -45,39 +42,6 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { modload.LoadALL() modload.TidyBuildList() - modTidyGoSum() // updates memory copy; WriteGoMod on next line flushes it out + modload.TrimGoSum() modload.WriteGoMod() } - -// modTidyGoSum resets the go.sum file content -// to be exactly what's needed for the current go.mod. -func modTidyGoSum() { - // Assuming go.sum already has at least enough from the successful load, - // we only have to tell modfetch what needs keeping. - reqs := modload.Reqs() - keep := make(map[module.Version]bool) - replaced := make(map[module.Version]bool) - var walk func(module.Version) - walk = func(m module.Version) { - // If we build using a replacement module, keep the sum for the replacement, - // since that's the code we'll actually use during a build. - // - // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the - // sums for both sets of transitive requirements. - r := modload.Replacement(m) - if r.Path == "" { - keep[m] = true - } else { - keep[r] = true - replaced[m] = true - } - list, _ := reqs.Required(m) - for _, r := range list { - if !keep[r] && !replaced[r] { - walk(r) - } - } - } - walk(modload.Target) - modfetch.TrimGoSum(keep) -} diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 8df2289097..e40158b535 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -375,12 +375,15 @@ type modSum struct { var goSum struct { mu sync.Mutex m map[module.Version][]string // content of go.sum file - checked map[modSum]bool // sums actually checked during execution - dirty bool // whether we added any new sums to m + status map[modSum]modSumStatus // state of sums in m overwrite bool // if true, overwrite go.sum without incorporating its contents enabled bool // whether to use go.sum at all } +type modSumStatus struct { + used, dirty bool +} + // initGoSum initializes the go.sum data. // The boolean it returns reports whether the // use of go.sum is now enabled. @@ -394,7 +397,7 @@ func initGoSum() (bool, error) { } goSum.m = make(map[module.Version][]string) - goSum.checked = make(map[modSum]bool) + goSum.status = make(map[modSum]modSumStatus) data, err := lockedfile.Read(GoSumFile) if err != nil && !os.IsNotExist(err) { return false, err @@ -504,6 +507,11 @@ func checkModSum(mod module.Version, h string) error { return err } done := inited && haveModSumLocked(mod, h) + if inited { + st := goSum.status[modSum{mod, h}] + st.used = true + goSum.status[modSum{mod, h}] = st + } goSum.mu.Unlock() if done { @@ -523,6 +531,9 @@ func checkModSum(mod module.Version, h string) error { if inited { goSum.mu.Lock() addModSumLocked(mod, h) + st := goSum.status[modSum{mod, h}] + st.dirty = true + goSum.status[modSum{mod, h}] = st goSum.mu.Unlock() } return nil @@ -532,7 +543,6 @@ func checkModSum(mod module.Version, h string) error { // If it finds a conflicting pair instead, it calls base.Fatalf. // goSum.mu must be locked. func haveModSumLocked(mod module.Version, h string) bool { - goSum.checked[modSum{mod, h}] = true for _, vh := range goSum.m[mod] { if h == vh { return true @@ -554,7 +564,6 @@ func addModSumLocked(mod module.Version, h string) { fmt.Fprintf(os.Stderr, "warning: verifying %s@%s: unknown hashes in go.sum: %v; adding %v"+hashVersionMismatch, mod.Path, mod.Version, strings.Join(goSum.m[mod], ", "), h) } goSum.m[mod] = append(goSum.m[mod], h) - goSum.dirty = true } // checkSumDB checks the mod, h pair against the Go checksum database. @@ -598,13 +607,35 @@ func Sum(mod module.Version) string { } // WriteGoSum writes the go.sum file if it needs to be updated. -func WriteGoSum() { +// +// keep is used to check whether a newly added sum should be saved in go.sum. +// It should have entries for both module content sums and go.mod sums +// (version ends with "/go.mod"). Existing sums will be preserved unless they +// have been marked for deletion with TrimGoSum. +func WriteGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() - if !goSum.enabled || !goSum.dirty { - // If we haven't read go.sum yet or if we don't have anything to add, - // don't bother opening it. + // If we haven't read the go.sum file yet, don't bother writing it. + if !goSum.enabled { + return + } + + // Check whether we need to add sums for which keep[m] is true or remove + // unused sums marked with TrimGoSum. If there are no changes to make, + // just return without opening go.sum. + dirty := false +Outer: + for m, hs := range goSum.m { + for _, h := range hs { + st := goSum.status[modSum{m, h}] + if st.dirty && (!st.used || keep[m]) { + dirty = true + break Outer + } + } + } + if !dirty { return } if cfg.BuildMod == "readonly" { @@ -625,9 +656,10 @@ func WriteGoSum() { // them without good reason. goSum.m = make(map[module.Version][]string, len(goSum.m)) readGoSum(goSum.m, GoSumFile, data) - for ms := range goSum.checked { - addModSumLocked(ms.mod, ms.sum) - goSum.dirty = true + for ms, st := range goSum.status { + if st.used { + addModSumLocked(ms.mod, ms.sum) + } } } @@ -642,7 +674,10 @@ func WriteGoSum() { list := goSum.m[m] sort.Strings(list) for _, h := range list { - fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + st := goSum.status[modSum{m, h}] + if !st.dirty || (st.used && keep[m]) { + fmt.Fprintf(&buf, "%s %s %s\n", m.Path, m.Version, h) + } } } return buf.Bytes(), nil @@ -652,12 +687,16 @@ func WriteGoSum() { base.Fatalf("go: updating go.sum: %v", err) } - goSum.checked = make(map[modSum]bool) - goSum.dirty = false + goSum.status = make(map[modSum]modSumStatus) goSum.overwrite = false } -// TrimGoSum trims go.sum to contain only the modules for which keep[m] is true. +// TrimGoSum trims go.sum to contain only the modules needed for reproducible +// builds. +// +// keep is used to check whether a sum should be retained in go.mod. It should +// have entries for both module content sums and go.mod sums (version ends +// with "/go.mod"). func TrimGoSum(keep map[module.Version]bool) { goSum.mu.Lock() defer goSum.mu.Unlock() @@ -669,13 +708,11 @@ func TrimGoSum(keep map[module.Version]bool) { return } - for m := range goSum.m { - // If we're keeping x@v we also keep x@v/go.mod. - // Map x@v/go.mod back to x@v for the keep lookup. - noGoMod := module.Version{Path: m.Path, Version: strings.TrimSuffix(m.Version, "/go.mod")} - if !keep[m] && !keep[noGoMod] { - delete(goSum.m, m) - goSum.dirty = true + for m, hs := range goSum.m { + if !keep[m] { + for _, h := range hs { + goSum.status[modSum{m, h}] = modSumStatus{used: false, dirty: true} + } goSum.overwrite = true } } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 664a2a1594..95334211ef 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -331,7 +331,11 @@ func die() { } // InitMod sets Target and, if there is a main module, parses the initial build -// list from its go.mod file, creating and populating that file if needed. +// list from its go.mod file. If InitMod is called by 'go mod init', InitMod +// will populate go.mod in memory, possibly importing dependencies from a +// legacy configuration file. For other commands, InitMod may make other +// adjustments in memory, like adding a go directive. WriteGoMod should be +// called later to write changes out to disk. // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. @@ -352,7 +356,6 @@ func InitMod() { // Running go mod init: do legacy module conversion legacyModInit() modFileToBuildList() - WriteGoMod() return } @@ -391,9 +394,6 @@ func InitMod() { if cfg.BuildMod == "vendor" { readVendorList() checkVendorConsistency() - } else { - // TODO(golang.org/issue/33326): if cfg.BuildMod != "readonly"? - WriteGoMod() } } @@ -797,9 +797,10 @@ func WriteGoMod() { base.Fatalf("go: updates to go.mod needed, disabled by -mod=readonly") } } + // Always update go.sum, even if we didn't change go.mod: we may have // downloaded modules that we didn't have before. - modfetch.WriteGoSum() + modfetch.WriteGoSum(keepSums()) if !dirty && cfg.CmdName != "mod tidy" { // The go.mod file has the same semantic content that it had before @@ -849,3 +850,59 @@ func WriteGoMod() { base.Fatalf("go: updating go.mod: %v", err) } } + +// keepSums returns a set of module sums to preserve in go.sum. The set +// includes entries for all modules used to load packages (according to +// the last load function like ImportPaths, LoadALL, etc.). It also contains +// entries for go.mod files needed for MVS (the version of these entries +// ends with "/go.mod"). +func keepSums() map[module.Version]bool { + // Walk the module graph and keep sums needed by MVS. + modkey := func(m module.Version) module.Version { + return module.Version{Path: m.Path, Version: m.Version + "/go.mod"} + } + keep := make(map[module.Version]bool) + replaced := make(map[module.Version]bool) + reqs := Reqs() + var walk func(module.Version) + walk = func(m module.Version) { + // If we build using a replacement module, keep the sum for the replacement, + // since that's the code we'll actually use during a build. + // + // TODO(golang.org/issue/29182): Perhaps we should keep both sums, and the + // sums for both sets of transitive requirements. + r := Replacement(m) + if r.Path == "" { + keep[modkey(m)] = true + } else { + replaced[m] = true + keep[modkey(r)] = true + } + list, _ := reqs.Required(m) + for _, r := range list { + if !keep[modkey(r)] && !replaced[r] { + walk(r) + } + } + } + walk(Target) + + // Add entries for modules that provided packages loaded with ImportPaths, + // LoadALL, or similar functions. + if loaded != nil { + for _, pkg := range loaded.pkgs { + m := pkg.mod + if r := Replacement(m); r.Path != "" { + keep[r] = true + } else { + keep[m] = true + } + } + } + + return keep +} + +func TrimGoSum() { + modfetch.TrimGoSum(keepSums()) +} diff --git a/src/cmd/go/testdata/script/mod_sum_lookup.txt b/src/cmd/go/testdata/script/mod_sum_lookup.txt new file mode 100644 index 0000000000..ed80a44984 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_sum_lookup.txt @@ -0,0 +1,33 @@ +# When we attempt to resolve an import that doesn't exist, we should not save +# hashes for downloaded modules. +# Verifies golang.org/issue/36260. +go list -e -tags=ignore ./noexist +! exists go.sum + +# When an import is resolved successfully, we should only save hashes for +# the module that provides the package, not for other modules looked up. +# Verifies golang.org/issue/31580. +go list ./exist +grep '^example.com/join v1.1.0 h1:' go.sum +! grep '^example.com/join/subpkg' go.sum +cp go.sum go.list.sum +go mod tidy +cmp go.sum go.list.sum + +-- go.mod -- +module m + +go 1.15 + +-- noexist/use.go -- +// ignore tags prevents errors in 'go mod tidy' +// +build ignore + +package use + +import _ "example.com/join/subpkg/noexist" + +-- exist/use.go -- +package use + +import _ "example.com/join/subpkg" diff --git a/src/cmd/go/testdata/script/mod_tidy_old.txt b/src/cmd/go/testdata/script/mod_tidy_old.txt new file mode 100644 index 0000000000..7428f0ce8a --- /dev/null +++ b/src/cmd/go/testdata/script/mod_tidy_old.txt @@ -0,0 +1,46 @@ +# 'go mod tidy' should remove content sums for module versions that aren't +# in the build list. It should preserve go.mod sums for module versions that +# are in the module graph though. +# Verifies golang.org/issue/33008. +go mod tidy +! grep '^rsc.io/quote v1.5.0 h1:' go.sum +grep '^rsc.io/quote v1.5.0/go.mod h1:' go.sum + +-- go.mod -- +module m + +go 1.15 + +require ( + rsc.io/quote v1.5.2 + example.com/r v0.0.0 +) + +replace example.com/r => ./r + +-- go.sum -- +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:pvCbr/wm8HzDD3fVywevekufpn6tCGPY3spdHeZJEsw= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +rsc.io/quote v1.5.0 h1:6fJa6E+wGadANKkUMlZ0DhXFpoKlslOQDCo259XtdIE= +rsc.io/quote v1.5.0/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= +rsc.io/quote v1.5.2 h1:3fEykkD9k7lYzXqCYrwGAf7iNhbk4yCjHmKBN9td4L0= +rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0= +rsc.io/sampler v1.3.0 h1:HLGR/BgEtI3r0uymSP/nl2uPLsUnNJX8toRyhfpBTII= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/testonly v1.0.0 h1:K/VWHdO+Jv7woUXG0GzVNx1czBXUt3Ib1deaMn+xk64= +rsc.io/testonly v1.0.0/go.mod h1:OqmGbIFOcF+XrFReLOGZ6BhMM7uMBiQwZsyNmh74SzY= + +-- r/go.mod -- +module example.com/r + +require rsc.io/quote v1.5.0 + +-- use.go -- +package use + +import _ "example.com/r" + +-- r/use.go -- +package use + +import _ "rsc.io/quote" -- cgit v1.3-5-g9baa From 8766f96dd72b5d124bf76bf5f88e260a88072683 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Thu, 25 Jun 2020 17:50:38 -0400 Subject: cmd/go: migrate to module.MatchPrefixPatterns In CL 239797, str.GlobsMatchPath was copied to golang.org/x/mod/module as MatchPrefixPatterns. This CL updates x/mod, switches calls to use the new function, and deletes the old function. For #38725 Change-Id: I7241032228b574aa539426a92d2f5aad9ee001e2 Reviewed-on: https://go-review.googlesource.com/c/go/+/240061 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Michael Matloob --- src/cmd/go.mod | 2 +- src/cmd/go.sum | 4 +- src/cmd/go/internal/modfetch/insecure.go | 5 ++- src/cmd/go/internal/modfetch/repo.go | 4 +- src/cmd/go/internal/modfetch/sumdb.go | 3 +- src/cmd/go/internal/str/path.go | 45 ----------------------- src/cmd/vendor/golang.org/x/mod/module/module.go | 47 ++++++++++++++++++++++++ src/cmd/vendor/modules.txt | 2 +- 8 files changed, 57 insertions(+), 55 deletions(-) diff --git a/src/cmd/go.mod b/src/cmd/go.mod index 6d57ceee79..21670b9996 100644 --- a/src/cmd/go.mod +++ b/src/cmd/go.mod @@ -7,7 +7,7 @@ require ( github.com/ianlancetaylor/demangle v0.0.0-20200414190113-039b1ae3a340 // indirect golang.org/x/arch v0.0.0-20200511175325-f7c78586839d golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/mod v0.3.0 + golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3 // indirect golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 golang.org/x/xerrors v0.0.0-20200806184451-1a77d5e9f316 // indirect diff --git a/src/cmd/go.sum b/src/cmd/go.sum index 3fc693e3bf..1b5ef515c2 100644 --- a/src/cmd/go.sum +++ b/src/cmd/go.sum @@ -14,8 +14,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 h1:R11LxkoUvECaAHdM5/ZOevSR7n+016EgTw8nbE1l+XM= +golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/src/cmd/go/internal/modfetch/insecure.go b/src/cmd/go/internal/modfetch/insecure.go index 8420432d6c..b692669cba 100644 --- a/src/cmd/go/internal/modfetch/insecure.go +++ b/src/cmd/go/internal/modfetch/insecure.go @@ -7,10 +7,11 @@ package modfetch import ( "cmd/go/internal/cfg" "cmd/go/internal/get" - "cmd/go/internal/str" + + "golang.org/x/mod/module" ) // allowInsecure reports whether we are allowed to fetch this path in an insecure manner. func allowInsecure(path string) bool { - return get.Insecure || str.GlobsMatchPath(cfg.GOINSECURE, path) + return get.Insecure || module.MatchPrefixPatterns(cfg.GOINSECURE, path) } diff --git a/src/cmd/go/internal/modfetch/repo.go b/src/cmd/go/internal/modfetch/repo.go index f03bdd8d03..34f805d58a 100644 --- a/src/cmd/go/internal/modfetch/repo.go +++ b/src/cmd/go/internal/modfetch/repo.go @@ -16,9 +16,9 @@ import ( "cmd/go/internal/get" "cmd/go/internal/modfetch/codehost" "cmd/go/internal/par" - "cmd/go/internal/str" web "cmd/go/internal/web" + "golang.org/x/mod/module" "golang.org/x/mod/semver" ) @@ -217,7 +217,7 @@ func lookup(proxy, path string) (r Repo, err error) { return nil, errLookupDisabled } - if str.GlobsMatchPath(cfg.GONOPROXY, path) { + if module.MatchPrefixPatterns(cfg.GONOPROXY, path) { switch proxy { case "noproxy", "direct": return lookupDirect(path) diff --git a/src/cmd/go/internal/modfetch/sumdb.go b/src/cmd/go/internal/modfetch/sumdb.go index 7973f47426..783c4a433b 100644 --- a/src/cmd/go/internal/modfetch/sumdb.go +++ b/src/cmd/go/internal/modfetch/sumdb.go @@ -24,7 +24,6 @@ import ( "cmd/go/internal/cfg" "cmd/go/internal/get" "cmd/go/internal/lockedfile" - "cmd/go/internal/str" "cmd/go/internal/web" "golang.org/x/mod/module" @@ -34,7 +33,7 @@ import ( // useSumDB reports whether to use the Go checksum database for the given module. func useSumDB(mod module.Version) bool { - return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path) + return cfg.GOSUMDB != "off" && !get.Insecure && !module.MatchPrefixPatterns(cfg.GONOSUMDB, mod.Path) } // lookupSumDB returns the Go checksum database's go.sum lines for the given module, diff --git a/src/cmd/go/internal/str/path.go b/src/cmd/go/internal/str/path.go index 95d91a3332..51ab2af82b 100644 --- a/src/cmd/go/internal/str/path.go +++ b/src/cmd/go/internal/str/path.go @@ -5,7 +5,6 @@ package str import ( - "path" "path/filepath" "strings" ) @@ -50,47 +49,3 @@ func HasFilePathPrefix(s, prefix string) bool { return s[len(prefix)] == filepath.Separator && s[:len(prefix)] == prefix } } - -// GlobsMatchPath reports whether any path prefix of target -// matches one of the glob patterns (as defined by path.Match) -// in the comma-separated globs list. -// It ignores any empty or malformed patterns in the list. -func GlobsMatchPath(globs, target string) bool { - for globs != "" { - // Extract next non-empty glob in comma-separated list. - var glob string - if i := strings.Index(globs, ","); i >= 0 { - glob, globs = globs[:i], globs[i+1:] - } else { - glob, globs = globs, "" - } - if glob == "" { - continue - } - - // A glob with N+1 path elements (N slashes) needs to be matched - // against the first N+1 path elements of target, - // which end just before the N+1'th slash. - n := strings.Count(glob, "/") - prefix := target - // Walk target, counting slashes, truncating at the N+1'th slash. - for i := 0; i < len(target); i++ { - if target[i] == '/' { - if n == 0 { - prefix = target[:i] - break - } - n-- - } - } - if n > 0 { - // Not enough prefix elements. - continue - } - matched, _ := path.Match(glob, prefix) - if matched { - return true - } - } - return false -} diff --git a/src/cmd/vendor/golang.org/x/mod/module/module.go b/src/cmd/vendor/golang.org/x/mod/module/module.go index 6cd37280a8..3a8b080c7b 100644 --- a/src/cmd/vendor/golang.org/x/mod/module/module.go +++ b/src/cmd/vendor/golang.org/x/mod/module/module.go @@ -97,6 +97,7 @@ package module import ( "fmt" + "path" "sort" "strings" "unicode" @@ -716,3 +717,49 @@ func unescapeString(escaped string) (string, bool) { } return string(buf), true } + +// MatchPrefixPatterns reports whether any path prefix of target matches one of +// the glob patterns (as defined by path.Match) in the comma-separated globs +// list. This implements the algorithm used when matching a module path to the +// GOPRIVATE environment variable, as described by 'go help module-private'. +// +// It ignores any empty or malformed patterns in the list. +func MatchPrefixPatterns(globs, target string) bool { + for globs != "" { + // Extract next non-empty glob in comma-separated list. + var glob string + if i := strings.Index(globs, ","); i >= 0 { + glob, globs = globs[:i], globs[i+1:] + } else { + glob, globs = globs, "" + } + if glob == "" { + continue + } + + // A glob with N+1 path elements (N slashes) needs to be matched + // against the first N+1 path elements of target, + // which end just before the N+1'th slash. + n := strings.Count(glob, "/") + prefix := target + // Walk target, counting slashes, truncating at the N+1'th slash. + for i := 0; i < len(target); i++ { + if target[i] == '/' { + if n == 0 { + prefix = target[:i] + break + } + n-- + } + } + if n > 0 { + // Not enough prefix elements. + continue + } + matched, _ := path.Match(glob, prefix) + if matched { + return true + } + } + return false +} diff --git a/src/cmd/vendor/modules.txt b/src/cmd/vendor/modules.txt index 21fc78c237..7272f04ff3 100644 --- a/src/cmd/vendor/modules.txt +++ b/src/cmd/vendor/modules.txt @@ -29,7 +29,7 @@ golang.org/x/arch/x86/x86asm golang.org/x/crypto/ed25519 golang.org/x/crypto/ed25519/internal/edwards25519 golang.org/x/crypto/ssh/terminal -# golang.org/x/mod v0.3.0 +# golang.org/x/mod v0.3.1-0.20200625141748-0b26df4a2231 ## explicit golang.org/x/mod/internal/lazyregexp golang.org/x/mod/modfile -- cgit v1.3-5-g9baa From 02a7b4b4a70d0574f82776309feaf28f109f5399 Mon Sep 17 00:00:00 2001 From: Jay Conrod Date: Tue, 30 Jun 2020 13:48:15 -0400 Subject: cmd/go/internal/modload: don't initialize build cache modload.Init initialized the build cache with the intent of providing a better error message in Go 1.12, when the build cache became mandatory (in module mode, packages aren't installed outside the build cache). Unfortunately, this didn't provide a more descriptive error (the cache calls base.Fatalf with its own message), and it caused errors for commands that don't use the cache (like 'go mod edit'). This CL removes the cache initialization from modload.Init. The builder will initialize it when it's needed. For #39882 Change-Id: Ibc01ae4e59358dcd08a07ffc97bf556514d0366f Reviewed-on: https://go-review.googlesource.com/c/go/+/240548 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Michael Matloob --- src/cmd/go/internal/modload/init.go | 7 ---- .../go/testdata/script/build_cache_disabled.txt | 46 ++++++++++++++++++++++ 2 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 src/cmd/go/testdata/script/build_cache_disabled.txt diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 95334211ef..fff060e665 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -20,7 +20,6 @@ import ( "strings" "cmd/go/internal/base" - "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/lockedfile" @@ -162,12 +161,6 @@ func Init() { // We're in module mode. Install the hooks to make it work. - if c := cache.Default(); c == nil { - // With modules, there are no install locations for packages - // other than the build cache. - base.Fatalf("go: cannot use modules with build cache disabled") - } - list := filepath.SplitList(cfg.BuildContext.GOPATH) if len(list) == 0 || list[0] == "" { base.Fatalf("missing $GOPATH") diff --git a/src/cmd/go/testdata/script/build_cache_disabled.txt b/src/cmd/go/testdata/script/build_cache_disabled.txt new file mode 100644 index 0000000000..2e1327880b --- /dev/null +++ b/src/cmd/go/testdata/script/build_cache_disabled.txt @@ -0,0 +1,46 @@ +# The build cache is required to build anything. It also may be needed to +# initialize the build system, which is needed for commands like 'go env'. +# However, there are lots of commands the cache is not needed for, and we +# shouldn't require it when it won't be used. +# +# TODO(golang.org/issue/39882): commands below should work, too. +# * go clean -modcache +# * go env +# * go fix +# * go fmt +# * go generate +# * go get -d +# * go list (without -export or -compiled) + +env GOCACHE=off + +# Commands that don't completely load packages should work. +go doc fmt +stdout Printf + +go fmt . + +! go tool compile -h +stderr usage: + +go version +stdout '^go version' + + +# Module commands that don't load packages should work. +go mod init m +exists go.mod + +go mod edit -require rsc.io/quote@v1.5.2 + +go mod download rsc.io/quote + +go mod graph +stdout rsc.io/quote + +go mod verify + +-- main.go -- +package main + +func main() {} -- cgit v1.3-5-g9baa From 32a84c99e136ed5af0686dbedd31fd7dff40fb38 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 8 Aug 2020 07:58:04 -0700 Subject: cmd/compile: fix live variable computation for deferreturn Taking the live variable set from the last return point is problematic. See #40629 for details, but there may not be a return point, or it may be before the final defer. Additionally, keeping track of the last call as a *Value doesn't quite work. If it is dead-code eliminated, the storage for the Value is reused for some other random instruction. Its live variable information, if it is available at all, is wrong. Instead, just mark all the open-defer argument slots as live throughout the function. (They are already zero-initialized.) Fixes #40629 Change-Id: Ie456c7db3082d0de57eaa5234a0f32525a1cce13 Reviewed-on: https://go-review.googlesource.com/c/go/+/247522 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Dan Scales --- src/cmd/compile/internal/gc/plive.go | 118 ++++++++--------------------------- src/cmd/compile/internal/gc/ssa.go | 19 +----- src/cmd/compile/internal/ssa/func.go | 11 +--- test/fixedbugs/issue40629.go | 69 ++++++++++++++++++++ 4 files changed, 99 insertions(+), 118 deletions(-) create mode 100644 test/fixedbugs/issue40629.go diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index b366c8a4a0..0cb2661997 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -140,24 +140,14 @@ type Liveness struct { regMaps []liveRegMask cache progeffectscache - - // These are only populated if open-coded defers are being used. - // List of vars/stack slots storing defer args - openDeferVars []openDeferVarInfo - // Map from defer arg OpVarDef to the block where the OpVarDef occurs. - openDeferVardefToBlockMap map[*Node]*ssa.Block - // Map of blocks that cannot reach a return or exit (panic) - nonReturnBlocks map[*ssa.Block]bool -} - -type openDeferVarInfo struct { - n *Node // Var/stack slot storing a defer arg - varsIndex int // Index of variable in lv.vars } // LivenessMap maps from *ssa.Value to LivenessIndex. type LivenessMap struct { vals map[ssa.ID]LivenessIndex + // The set of live, pointer-containing variables at the deferreturn + // call (only set when open-coded defers are used). + deferreturn LivenessIndex } func (m *LivenessMap) reset() { @@ -168,6 +158,7 @@ func (m *LivenessMap) reset() { delete(m.vals, k) } } + m.deferreturn = LivenessInvalid } func (m *LivenessMap) set(v *ssa.Value, i LivenessIndex) { @@ -542,7 +533,7 @@ func newliveness(fn *Node, f *ssa.Func, vars []*Node, idx map[*Node]int32, stkpt if cap(lc.be) >= f.NumBlocks() { lv.be = lc.be[:f.NumBlocks()] } - lv.livenessMap = LivenessMap{lc.livenessMap.vals} + lv.livenessMap = LivenessMap{vals: lc.livenessMap.vals, deferreturn: LivenessInvalid} lc.livenessMap.vals = nil } if lv.be == nil { @@ -893,58 +884,12 @@ func (lv *Liveness) hasStackMap(v *ssa.Value) bool { func (lv *Liveness) prologue() { lv.initcache() - if lv.fn.Func.HasDefer() && !lv.fn.Func.OpenCodedDeferDisallowed() { - lv.openDeferVardefToBlockMap = make(map[*Node]*ssa.Block) - for i, n := range lv.vars { - if n.Name.OpenDeferSlot() { - lv.openDeferVars = append(lv.openDeferVars, openDeferVarInfo{n: n, varsIndex: i}) - } - } - - // Find any blocks that cannot reach a return or a BlockExit - // (panic) -- these must be because of an infinite loop. - reachesRet := make(map[ssa.ID]bool) - blockList := make([]*ssa.Block, 0, 256) - - for _, b := range lv.f.Blocks { - if b.Kind == ssa.BlockRet || b.Kind == ssa.BlockRetJmp || b.Kind == ssa.BlockExit { - blockList = append(blockList, b) - } - } - - for len(blockList) > 0 { - b := blockList[0] - blockList = blockList[1:] - if reachesRet[b.ID] { - continue - } - reachesRet[b.ID] = true - for _, e := range b.Preds { - blockList = append(blockList, e.Block()) - } - } - - lv.nonReturnBlocks = make(map[*ssa.Block]bool) - for _, b := range lv.f.Blocks { - if !reachesRet[b.ID] { - lv.nonReturnBlocks[b] = true - //fmt.Println("No reach ret", lv.f.Name, b.ID, b.Kind) - } - } - } - for _, b := range lv.f.Blocks { be := lv.blockEffects(b) // Walk the block instructions backward and update the block // effects with the each prog effects. for j := len(b.Values) - 1; j >= 0; j-- { - if b.Values[j].Op == ssa.OpVarDef { - n := b.Values[j].Aux.(*Node) - if n.Name.OpenDeferSlot() { - lv.openDeferVardefToBlockMap[n] = b - } - } pos, e := lv.valueEffects(b.Values[j]) regUevar, regKill := lv.regEffects(b.Values[j]) if e&varkill != 0 { @@ -961,20 +906,6 @@ func (lv *Liveness) prologue() { } } -// markDeferVarsLive marks each variable storing an open-coded defer arg as -// specially live in block b if the variable definition dominates block b. -func (lv *Liveness) markDeferVarsLive(b *ssa.Block, newliveout *varRegVec) { - // Only force computation of dominators if we have a block where we need - // to specially mark defer args live. - sdom := lv.f.Sdom() - for _, info := range lv.openDeferVars { - defB := lv.openDeferVardefToBlockMap[info.n] - if sdom.IsAncestorEq(defB, b) { - newliveout.vars.Set(int32(info.varsIndex)) - } - } -} - // Solve the liveness dataflow equations. func (lv *Liveness) solve() { // These temporary bitvectors exist to avoid successive allocations and @@ -1018,23 +949,6 @@ func (lv *Liveness) solve() { } } - if lv.fn.Func.HasDefer() && !lv.fn.Func.OpenCodedDeferDisallowed() && - (b.Kind == ssa.BlockExit || lv.nonReturnBlocks[b]) { - // Open-coded defer args slots must be live - // everywhere in a function, since a panic can - // occur (almost) anywhere. Force all appropriate - // defer arg slots to be live in BlockExit (panic) - // blocks and in blocks that do not reach a return - // (because of infinite loop). - // - // We are assuming that the defer exit code at - // BlockReturn/BlockReturnJmp accesses all of the - // defer args (with pointers), and so keeps them - // live. This analysis may have to be adjusted if - // that changes (because of optimizations). - lv.markDeferVarsLive(b, &newliveout) - } - if !be.liveout.Eq(newliveout) { change = true be.liveout.Copy(newliveout) @@ -1087,6 +1001,17 @@ func (lv *Liveness) epilogue() { n.Name.SetNeedzero(true) livedefer.Set(int32(i)) } + if n.Name.OpenDeferSlot() { + // Open-coded defer args slots must be live + // everywhere in a function, since a panic can + // occur (almost) anywhere. Because it is live + // everywhere, it must be zeroed on entry. + livedefer.Set(int32(i)) + // It was already marked as Needzero when created. + if !n.Name.Needzero() { + Fatalf("all pointer-containing defer arg slots should have Needzero set") + } + } } } @@ -1188,6 +1113,17 @@ func (lv *Liveness) epilogue() { lv.compact(b) } + // If we have an open-coded deferreturn call, make a liveness map for it. + if lv.fn.Func.OpenCodedDeferDisallowed() { + lv.livenessMap.deferreturn = LivenessInvalid + } else { + lv.livenessMap.deferreturn = LivenessIndex{ + stackMapIndex: lv.stackMapSet.add(livedefer), + regMapIndex: 0, // entry regMap, containing no live registers + isUnsafePoint: false, + } + } + // Done compacting. Throw out the stack map set. lv.stackMaps = lv.stackMapSet.extractUniqe() lv.stackMapSet = bvecSet{} diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index d4d23a2956..5d0098b4e6 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -4318,12 +4318,6 @@ func (s *state) openDeferExit() { } } - if i == len(s.openDefers)-1 { - // Record the call of the first defer. This will be used - // to set liveness info for the deferreturn (which is also - // used for any location that causes a runtime panic) - s.f.LastDeferExit = call - } s.endBlock() s.startBlock(bEnd) } @@ -5807,11 +5801,6 @@ type SSAGenState struct { // wasm: The number of values on the WebAssembly stack. This is only used as a safeguard. OnWasmStackSkipped int - - // Liveness index for the first function call in the final defer exit code - // path that we generated. All defer functions and args should be live at - // this point. This will be used to set the liveness for the deferreturn. - lastDeferLiveness LivenessIndex } // Prog appends a new Prog. @@ -6056,12 +6045,6 @@ func genssa(f *ssa.Func, pp *Progs) { // instruction. s.pp.nextLive = s.livenessMap.Get(v) - // Remember the liveness index of the first defer call of - // the last defer exit - if v.Block.Func.LastDeferExit != nil && v == v.Block.Func.LastDeferExit { - s.lastDeferLiveness = s.pp.nextLive - } - // Special case for first line in function; move it to the start. if firstPos != src.NoXPos { s.SetPos(firstPos) @@ -6122,7 +6105,7 @@ func genssa(f *ssa.Func, pp *Progs) { // When doing open-coded defers, generate a disconnected call to // deferreturn and a return. This will be used to during panic // recovery to unwind the stack and return back to the runtime. - s.pp.nextLive = s.lastDeferLiveness + s.pp.nextLive = s.livenessMap.deferreturn gencallret(pp, Deferreturn) } diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 7cf72a8e37..4b9189fb3e 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -33,15 +33,8 @@ type Func struct { Blocks []*Block // unordered set of all basic blocks (note: not indexable by ID) Entry *Block // the entry basic block - // If we are using open-coded defers, this is the first call to a deferred - // function in the final defer exit sequence that we generated. This call - // should be after all defer statements, and will have all args, etc. of - // all defer calls as live. The liveness info of this call will be used - // for the deferreturn/ret segment generated for functions with open-coded - // defers. - LastDeferExit *Value - bid idAlloc // block ID allocator - vid idAlloc // value ID allocator + bid idAlloc // block ID allocator + vid idAlloc // value ID allocator // Given an environment variable used for debug hash match, // what file (if any) receives the yes/no logging? diff --git a/test/fixedbugs/issue40629.go b/test/fixedbugs/issue40629.go new file mode 100644 index 0000000000..c6ef408f49 --- /dev/null +++ b/test/fixedbugs/issue40629.go @@ -0,0 +1,69 @@ +// run + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "fmt" + +const N = 40 + +func main() { + var x [N]int // stack-allocated memory + for i := range x { + x[i] = 0x999 + } + + // This defer checks to see if x is uncorrupted. + defer func(p *[N]int) { + recover() + for i := range p { + if p[i] != 0x999 { + for j := range p { + fmt.Printf("p[%d]=0x%x\n", j, p[j]) + } + panic("corrupted stack variable") + } + } + }(&x) + + // This defer starts a new goroutine, which will (hopefully) + // overwrite x on the garbage stack. + defer func() { + c := make(chan bool) + go func() { + useStack(1000) + c <- true + }() + <-c + + }() + + // This defer causes a stack copy. + // The old stack is now garbage. + defer func() { + useStack(1000) + }() + + // Trigger a segfault. + *g = 0 + + // Make the return statement unreachable. + // That makes the stack map at the deferreturn call empty. + // In particular, the argument to the first defer is not + // marked as a pointer, so it doesn't get adjusted + // during the stack copy. + for { + } +} + +var g *int64 + +func useStack(n int) { + if n == 0 { + return + } + useStack(n - 1) +} -- cgit v1.3-5-g9baa From 407bf0ca67200463cdd451937623078e0240335e Mon Sep 17 00:00:00 2001 From: Alexander Klauer Date: Mon, 29 Jun 2020 16:02:07 +0000 Subject: reflect: add parentheses to properly bind <- in ChanOf’s string MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds parentheses so as to properly bind <- to the right most channel. This meant that previously given: ChanOf(<-chan T) it would mistakenly try to look up the type as chan <-chan T instead of chan (<-chan T) Fixes #39897 Change-Id: I8564916055f5fadde3382e41fe8820a1071e5f13 GitHub-Last-Rev: f8f2abe8d4c9e3d1414c89cadca8a16ce5cdeab9 GitHub-Pull-Request: golang/go#39898 Reviewed-on: https://go-review.googlesource.com/c/go/+/240280 Reviewed-by: Emmanuel Odeke Reviewed-by: Keith Randall Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot --- src/reflect/all_test.go | 16 ++++++++++++++++ src/reflect/type.go | 12 ++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index 63f6a92157..6b31568bb9 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -74,6 +74,10 @@ var typeTests = []pair{ {struct{ x ([]int8) }{}, "[]int8"}, {struct{ x (map[string]int32) }{}, "map[string]int32"}, {struct{ x (chan<- string) }{}, "chan<- string"}, + {struct{ x (chan<- chan string) }{}, "chan<- chan string"}, + {struct{ x (chan<- <-chan string) }{}, "chan<- <-chan string"}, + {struct{ x (<-chan <-chan string) }{}, "<-chan <-chan string"}, + {struct{ x (chan (<-chan string)) }{}, "chan (<-chan string)"}, {struct { x struct { c chan *int32 @@ -5491,6 +5495,18 @@ func TestChanOf(t *testing.T) { // check that type already in binary is found type T1 int checkSameType(t, ChanOf(BothDir, TypeOf(T1(1))), (chan T1)(nil)) + + // Check arrow token association in undefined chan types. + var left chan<- chan T + var right chan (<-chan T) + tLeft := ChanOf(SendDir, ChanOf(BothDir, TypeOf(T("")))) + tRight := ChanOf(BothDir, ChanOf(RecvDir, TypeOf(T("")))) + if tLeft != TypeOf(left) { + t.Errorf("chan<-chan: have %s, want %T", tLeft, left) + } + if tRight != TypeOf(right) { + t.Errorf("chan<-chan: have %s, want %T", tRight, right) + } } func TestChanOfDir(t *testing.T) { diff --git a/src/reflect/type.go b/src/reflect/type.go index 38b1283d42..44c96fea82 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -1789,7 +1789,6 @@ func ChanOf(dir ChanDir, t Type) Type { } // Look in known types. - // TODO: Precedence when constructing string. var s string switch dir { default: @@ -1799,7 +1798,16 @@ func ChanOf(dir ChanDir, t Type) Type { case RecvDir: s = "<-chan " + typ.String() case BothDir: - s = "chan " + typ.String() + typeStr := typ.String() + if typeStr[0] == '<' { + // typ is recv chan, need parentheses as "<-" associates with leftmost + // chan possible, see: + // * https://golang.org/ref/spec#Channel_types + // * https://github.com/golang/go/issues/39897 + s = "chan (" + typeStr + ")" + } else { + s = "chan " + typeStr + } } for _, tt := range typesByString(s) { ch := (*chanType)(unsafe.Pointer(tt)) -- cgit v1.3-5-g9baa From 24ff2af65e27eed1e8c7f09c21a5ca68fc2e07ab Mon Sep 17 00:00:00 2001 From: lufia Date: Sun, 26 Jul 2020 19:04:37 +0900 Subject: cmd/dist: fix typo Change-Id: Ib5d7f3eadff03070043d52659af4312ee293c586 Reviewed-on: https://go-review.googlesource.com/c/go/+/244817 Reviewed-by: Alberto Donizetti Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot --- src/cmd/dist/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/dist/build.go b/src/cmd/dist/build.go index a817e6fcd7..397b3bb88f 100644 --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -1209,7 +1209,7 @@ func timelog(op, name string) { } i := strings.Index(s, " start") if i < 0 { - log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBULDTIMELOGFILE")) + log.Fatalf("time log %s does not begin with start line", os.Getenv("GOBUILDTIMELOGFILE")) } t, err := time.Parse(time.UnixDate, s[:i]) if err != nil { -- cgit v1.3-5-g9baa From 5a18e0b58ca2d08f3988018a8759207cb64e651a Mon Sep 17 00:00:00 2001 From: Gaurav Singh Date: Thu, 23 Jul 2020 23:27:05 +0000 Subject: sync: fix goroutine leak for when TestMutexFairness times out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the timeout triggers before writing to the done channel, the goroutine will be blocked waiting for a corresponding read that’s no longer existent, thus a goroutine leak. This change fixes that by using a buffered channel instead. Change-Id: I9cf4067a58bc5a729ab31e4426edd78bd359e8e0 GitHub-Last-Rev: a7d811a7be6d875175a894e53d474aa0034e7d2c GitHub-Pull-Request: golang/go#40236 Reviewed-on: https://go-review.googlesource.com/c/go/+/242902 Reviewed-by: Emmanuel Odeke Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot --- src/sync/mutex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync/mutex_test.go b/src/sync/mutex_test.go index e61a853642..98c1bf2a5f 100644 --- a/src/sync/mutex_test.go +++ b/src/sync/mutex_test.go @@ -194,7 +194,7 @@ func TestMutexFairness(t *testing.T) { } } }() - done := make(chan bool) + done := make(chan bool, 1) go func() { for i := 0; i < 10; i++ { time.Sleep(100 * time.Microsecond) -- cgit v1.3-5-g9baa From f71444955a4c0962abb334a8f39438466c57a4db Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 14 Aug 2020 14:39:19 -0700 Subject: test: add test case that caused gccgo undefined symbol reference For #40252 Change-Id: Ie23d2789ca9b4b9081adb39ab64c80c412ad58ce Reviewed-on: https://go-review.googlesource.com/c/go/+/248637 Run-TryBot: Ian Lance Taylor Reviewed-by: Cherry Zhang TryBot-Result: Gobot Gobot --- test/fixedbugs/issue40252.dir/a.go | 14 ++++++++++++++ test/fixedbugs/issue40252.dir/main.go | 16 ++++++++++++++++ test/fixedbugs/issue40252.go | 8 ++++++++ 3 files changed, 38 insertions(+) create mode 100644 test/fixedbugs/issue40252.dir/a.go create mode 100644 test/fixedbugs/issue40252.dir/main.go create mode 100644 test/fixedbugs/issue40252.go diff --git a/test/fixedbugs/issue40252.dir/a.go b/test/fixedbugs/issue40252.dir/a.go new file mode 100644 index 0000000000..5519e9331a --- /dev/null +++ b/test/fixedbugs/issue40252.dir/a.go @@ -0,0 +1,14 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package a + +type I interface { + Func() +} + +func Call() { + f := I.Func + f(nil) +} diff --git a/test/fixedbugs/issue40252.dir/main.go b/test/fixedbugs/issue40252.dir/main.go new file mode 100644 index 0000000000..93f5b70624 --- /dev/null +++ b/test/fixedbugs/issue40252.dir/main.go @@ -0,0 +1,16 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "./a" + +func main() { + defer func() { + if recover() == nil { + panic("expected nil pointer dereference") + } + }() + a.Call() +} diff --git a/test/fixedbugs/issue40252.go b/test/fixedbugs/issue40252.go new file mode 100644 index 0000000000..9be4e665d2 --- /dev/null +++ b/test/fixedbugs/issue40252.go @@ -0,0 +1,8 @@ +// rundir + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// gccgo got an undefined symbol reference when inlining a method expression. +package ignored -- cgit v1.3-5-g9baa From 6072d6ee3e173bb46370dabc158e2cb25a6f4877 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 10 Jul 2020 09:56:00 -0700 Subject: test: add a test case that gccgo fails to compile Change-Id: If36394e059cdae49834d26ad4ffdd3092a72a0b8 Reviewed-on: https://go-review.googlesource.com/c/go/+/241997 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- test/fixedbugs/bug509.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/fixedbugs/bug509.go diff --git a/test/fixedbugs/bug509.go b/test/fixedbugs/bug509.go new file mode 100644 index 0000000000..df6ed61f89 --- /dev/null +++ b/test/fixedbugs/bug509.go @@ -0,0 +1,30 @@ +// compile + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Gccgo mishandles a couple of alias cases. + +package p + +type S struct{} + +func (*S) M() {} + +type I interface { + M() +} + +type A = *S + +var V1 I +var _ = V1.(*S) +var _ = V1.(A) + +func F() { + var v I + v = (*S)(nil) + v = A(nil) + _ = v +} -- cgit v1.3-5-g9baa From 12d40adac46b5c771247a789205f7893bfd808b2 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 10 Jul 2020 12:21:41 -0700 Subject: test: add test for conversion of untyped bool to interface gccgo miscompiled this case. Updates #40152 Change-Id: I8448c155e802e39d8fc7cda4930ce62cb6363ce5 Reviewed-on: https://go-review.googlesource.com/c/go/+/242000 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Emmanuel Odeke Reviewed-by: Cherry Zhang --- test/fixedbugs/issue40152.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/fixedbugs/issue40152.go diff --git a/test/fixedbugs/issue40152.go b/test/fixedbugs/issue40152.go new file mode 100644 index 0000000000..1cb68e9914 --- /dev/null +++ b/test/fixedbugs/issue40152.go @@ -0,0 +1,21 @@ +// run + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Gccgo mishandles converting an untyped boolean to an interface type. + +package main + +func t(args ...interface{}) bool { + x := true + return x == args[0] +} + +func main() { + r := t("x" == "x" && "y" == "y") + if !r { + panic(r) + } +} -- cgit v1.3-5-g9baa From 441b52f5660ccde7848f034ba345d2f0088ea383 Mon Sep 17 00:00:00 2001 From: kakulisen Date: Thu, 14 May 2020 16:20:58 +0800 Subject: math: simplify the code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Simplifying some code without compromising performance. My CPU is Intel Xeon Gold 6161, 2.20GHz, 64-bit operating system. The memory is 8GB. This is my test environment, I hope to help you judge. Benchmark: name old time/op new time/op delta Log1p-4 21.8ns ± 5% 21.8ns ± 4% ~ (p=0.973 n=20+20) Change-Id: Icd8f96f1325b00007602d114300b92d4c57de409 Reviewed-on: https://go-review.googlesource.com/c/go/+/233940 Reviewed-by: Robert Griesemer --- src/math/log1p.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/math/log1p.go b/src/math/log1p.go index c4ec61b225..e34e1ff4f2 100644 --- a/src/math/log1p.go +++ b/src/math/log1p.go @@ -122,10 +122,7 @@ func log1p(x float64) float64 { return Inf(1) } - absx := x - if absx < 0 { - absx = -absx - } + absx := Abs(x) var f float64 var iu uint64 -- cgit v1.3-5-g9baa From 0031fa80a3c6685e44e84533edbae0dad0eb0395 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Thu, 7 May 2020 00:35:28 +0700 Subject: cmd/compile: another fix initializing blank fields in struct literal CL 230121 fixed the bug that struct literal blank fields type array/struct can not be initialized. But it still misses some cases when an expression causes "candiscard(value)" return false. When these happen, we recursively call fixedlit with "var_" set to "_", and hit the bug again. To fix it, just making splitnode return "nblank" whenever "var_" is "nblank". Fixes #38905 Change-Id: I281941b388acbd551a4d8ca1a235477f8d26fb6e Reviewed-on: https://go-review.googlesource.com/c/go/+/232617 Run-TryBot: Cuong Manh Le TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/sinit.go | 6 +++++- test/fixedbugs/issue38905.go | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue38905.go diff --git a/src/cmd/compile/internal/gc/sinit.go b/src/cmd/compile/internal/gc/sinit.go index 4a2edc7d21..71ed558461 100644 --- a/src/cmd/compile/internal/gc/sinit.go +++ b/src/cmd/compile/internal/gc/sinit.go @@ -506,6 +506,7 @@ const ( // fixedlit handles struct, array, and slice literals. // TODO: expand documentation. func fixedlit(ctxt initContext, kind initKind, n *Node, var_ *Node, init *Nodes) { + isBlank := var_ == nblank var splitnode func(*Node) (a *Node, value *Node) switch n.Op { case OARRAYLIT, OSLICELIT: @@ -520,6 +521,9 @@ func fixedlit(ctxt initContext, kind initKind, n *Node, var_ *Node, init *Nodes) } a := nod(OINDEX, var_, nodintconst(k)) k++ + if isBlank { + a = nblank + } return a, r } case OSTRUCTLIT: @@ -527,7 +531,7 @@ func fixedlit(ctxt initContext, kind initKind, n *Node, var_ *Node, init *Nodes) if r.Op != OSTRUCTKEY { Fatalf("fixedlit: rhs not OSTRUCTKEY: %v", r) } - if r.Sym.IsBlank() { + if r.Sym.IsBlank() || isBlank { return nblank, r.Left } setlineno(r) diff --git a/test/fixedbugs/issue38905.go b/test/fixedbugs/issue38905.go new file mode 100644 index 0000000000..6f411b8605 --- /dev/null +++ b/test/fixedbugs/issue38905.go @@ -0,0 +1,18 @@ +// compile + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure that literal value can be passed to struct +// blank field with expressions where candiscard(value) +// returns false, see #38905. + +package p + +type t struct{ _ u } +type u [10]struct{ f int } + +func f(x int) t { return t{u{{1 / x}, {1 % x}}} } +func g(p *int) t { return t{u{{*p}}} } +func h(s []int) t { return t{u{{s[0]}}} } -- cgit v1.3-5-g9baa From 82c45eb68187d7827bca392d528dbfa06607e3f0 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 27 Jul 2020 12:08:56 +0700 Subject: cmd/compile: handle OCLOSURE/OCALLPART in mustHeapAlloc check Currently, generated struct wrapper for closure is not handled in mustHeapAlloc. That causes compiler crashes when the wrapper struct is too large for stack, and must be heap allocated instead. Fixes #39292 Change-Id: I14c1e591681d9d92317bb2396d6cf5207aa93e08 Reviewed-on: https://go-review.googlesource.com/c/go/+/244917 Run-TryBot: Cuong Manh Le TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- src/cmd/compile/internal/gc/closure.go | 2 +- src/cmd/compile/internal/gc/esc.go | 7 +++++++ test/fixedbugs/issue39292.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue39292.go diff --git a/src/cmd/compile/internal/gc/closure.go b/src/cmd/compile/internal/gc/closure.go index 3bb7bb9834..04fb7d5495 100644 --- a/src/cmd/compile/internal/gc/closure.go +++ b/src/cmd/compile/internal/gc/closure.go @@ -526,7 +526,7 @@ func walkpartialcall(n *Node, init *Nodes) *Node { // Create closure in the form of a composite literal. // For x.M with receiver (x) type T, the generated code looks like: // - // clos = &struct{F uintptr; R T}{M.T·f, x} + // clos = &struct{F uintptr; R T}{T.M·f, x} // // Like walkclosure above. diff --git a/src/cmd/compile/internal/gc/esc.go b/src/cmd/compile/internal/gc/esc.go index f3e9ab78ef..628953741a 100644 --- a/src/cmd/compile/internal/gc/esc.go +++ b/src/cmd/compile/internal/gc/esc.go @@ -187,6 +187,13 @@ func mustHeapAlloc(n *Node) bool { return true } + if n.Op == OCLOSURE && closureType(n).Size() >= maxImplicitStackVarSize { + return true + } + if n.Op == OCALLPART && partialCallType(n).Size() >= maxImplicitStackVarSize { + return true + } + if n.Op == OMAKESLICE && !isSmallMakeSlice(n) { return true } diff --git a/test/fixedbugs/issue39292.go b/test/fixedbugs/issue39292.go new file mode 100644 index 0000000000..5d6595c234 --- /dev/null +++ b/test/fixedbugs/issue39292.go @@ -0,0 +1,29 @@ +// errorcheck -0 -m -l + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +type t [10000]*int + +func (t) f() { +} + +func x() { + x := t{}.f // ERROR "t literal.f escapes to heap" + x() +} + +func y() { + var i int // ERROR "moved to heap: i" + y := (&t{&i}).f // ERROR "\(&t literal\).f escapes to heap" "&t literal escapes to heap" + y() +} + +func z() { + var i int // ERROR "moved to heap: i" + z := t{&i}.f // ERROR "t literal.f escapes to heap" + z() +} -- cgit v1.3-5-g9baa From 948d324f7d8641a042da46c25417ebabd84e5e78 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 12 Aug 2020 23:12:57 +0700 Subject: cmd/compile: add failing test case for #24305 Updates #24305 Change-Id: Ib0b093e33004a978467cdd1e77186798392d4eca Reviewed-on: https://go-review.googlesource.com/c/go/+/248217 Run-TryBot: Cuong Manh Le TryBot-Result: Gobot Gobot Reviewed-by: Matthew Dempsky --- test/escape5.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/escape5.go b/test/escape5.go index 061e57a069..2ed2023cd2 100644 --- a/test/escape5.go +++ b/test/escape5.go @@ -179,6 +179,13 @@ func _() { u.N() } +func fbad24305() { + // BAD u should not be heap allocated + var u U // ERROR "moved to heap: u" + (*U).M(&u) + (*U).N(&u) +} + // Issue 24730: taking address in a loop causes unnecessary escape type T24730 struct { x [64]byte -- cgit v1.3-5-g9baa From 69d34e2c6965f70fe1ead3e7e8ab45ada3267ebc Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Fri, 14 Aug 2020 21:30:04 -0700 Subject: test: bump array size in fixedbugs/issue39292.go The previous array length was large enough to exceed maxImplicitStackSize on 64-bit architectures, but not on 32-bit architectures. Fixes #40808. Change-Id: I69e9abb447454b2e7875ba503a0cb772e965ae31 Reviewed-on: https://go-review.googlesource.com/c/go/+/248680 Run-TryBot: Matthew Dempsky Reviewed-by: Cuong Manh Le TryBot-Result: Gobot Gobot --- test/fixedbugs/issue39292.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fixedbugs/issue39292.go b/test/fixedbugs/issue39292.go index 5d6595c234..1be88653e9 100644 --- a/test/fixedbugs/issue39292.go +++ b/test/fixedbugs/issue39292.go @@ -6,7 +6,7 @@ package p -type t [10000]*int +type t [20000]*int func (t) f() { } -- cgit v1.3-5-g9baa From c0dded04f7ded5048b44200078a1f723f5e1bcc1 Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Tue, 14 Jul 2020 01:41:03 -0600 Subject: runtime: do not explicitly exit on ctrl handler The default ctrl+c handler should process exits in situations where it makes sense, like console apps, but not in situations where it doesn't, like libraries or services. Therefore, we should remove the exit(2) so that the default handler is used for this. This also uses the more proper windows exit code of STATUS_CONTROL_C_EXIT, with the base case handler installed by KernelBase.dll. In particular, this helps in the case of services, which previously would terminate when receiving shutdown signals, instead of passing them onward to the service program. In this CL, contrary to CL 244959, we do not need to special case services with expensive detection algorithms, or rely on hard-coded library/archive flags. Fixes #40167. Fixes #40074. Change-Id: I9bf6ed6f65cefeff754d270aa33fa4df8d0b451f Reviewed-on: https://go-review.googlesource.com/c/go/+/243597 Run-TryBot: Jason A. Donenfeld TryBot-Result: Gobot Gobot Reviewed-by: Alex Brainman Reviewed-by: Jason A. Donenfeld --- src/runtime/os_windows.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index a584ada702..a62e941229 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -1010,11 +1010,6 @@ func ctrlhandler1(_type uint32) uint32 { if sigsend(s) { return 1 } - if !islibrary && !isarchive { - // Only exit the program if we don't have a DLL. - // See https://golang.org/issues/35965. - exit(2) // SIGINT, SIGTERM, etc - } return 0 } -- cgit v1.3-5-g9baa From ccc951637be806e6e7a3c2c922bf4746b60e7395 Mon Sep 17 00:00:00 2001 From: Alberto Donizetti Date: Thu, 13 Aug 2020 12:56:57 +0200 Subject: cmd/link: move comma outside quotes Change-Id: I2ecf8976a6289924ac7bfe7ace129a462537e11d Reviewed-on: https://go-review.googlesource.com/c/go/+/248339 Reviewed-by: Emmanuel Odeke Reviewed-by: Than McIntosh --- src/cmd/link/doc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/doc.go b/src/cmd/link/doc.go index 219499be0a..604675caec 100644 --- a/src/cmd/link/doc.go +++ b/src/cmd/link/doc.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. /* -Link, typically invoked as ``go tool link,'' reads the Go archive or object +Link, typically invoked as ``go tool link'', reads the Go archive or object for a package main, along with its dependencies, and combines them into an executable binary. -- cgit v1.3-5-g9baa From b6ad2880323191713a5525bae5eb27d62c1d1c35 Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Sat, 13 Jun 2020 10:37:22 +0530 Subject: net/http: avoid setting body when NoBody is set for js/wasm When http.NoBody is set, it is equivalent to Body being zero bytes. We therefore set the body only if it is of length greater than 0. Manually verified with wasmbrowsertest. Fixes #36339 Change-Id: I9c108c38f99409f72ea101819af572429505a8ad Reviewed-on: https://go-review.googlesource.com/c/go/+/237758 Run-TryBot: Agniva De Sarker TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick Reviewed-by: Johan Brandhorst Reviewed-by: Emmanuel Odeke --- src/net/http/roundtrip_js.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/net/http/roundtrip_js.go b/src/net/http/roundtrip_js.go index 509d229aad..b09923c386 100644 --- a/src/net/http/roundtrip_js.go +++ b/src/net/http/roundtrip_js.go @@ -98,9 +98,11 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { return nil, err } req.Body.Close() - buf := uint8Array.New(len(body)) - js.CopyBytesToJS(buf, body) - opt.Set("body", buf) + if len(body) != 0 { + buf := uint8Array.New(len(body)) + js.CopyBytesToJS(buf, body) + opt.Set("body", buf) + } } fetchPromise := js.Global().Call("fetch", req.URL.String(), opt) -- cgit v1.3-5-g9baa From cf9b4f63a57b4360be700831781885fc6cf5a0b1 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Tue, 5 May 2020 03:15:58 +1000 Subject: runtime: use riscv64 RDTIME instruction Use the actual RDTIME instruction, rather than a WORD. Generated code is the same. Change-Id: I6f6f5a1836eae2d05af34d4a22db2ede4fdcb458 Reviewed-on: https://go-review.googlesource.com/c/go/+/231997 Reviewed-by: Cherry Zhang Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot --- src/runtime/asm_riscv64.s | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/asm_riscv64.s b/src/runtime/asm_riscv64.s index d7c45a183d..8f6c8773eb 100644 --- a/src/runtime/asm_riscv64.s +++ b/src/runtime/asm_riscv64.s @@ -79,7 +79,7 @@ TEXT setg_gcc<>(SB),NOSPLIT,$0-0 // func cputicks() int64 TEXT runtime·cputicks(SB),NOSPLIT,$0-8 - WORD $0xc0102573 // rdtime a0 + RDTIME A0 MOV A0, ret+0(FP) RET -- cgit v1.3-5-g9baa From d30363062283dcdca4392ef61f13f9b332ca8bc3 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Mon, 25 May 2020 03:23:30 +1000 Subject: syscall: support rawVforkSyscall on linux/riscv64 Updates #31936 Change-Id: I7dcb8987d4c306ccc97704b9c1b12313ba8bf242 Reviewed-on: https://go-review.googlesource.com/c/go/+/234960 Reviewed-by: Cherry Zhang Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot --- src/syscall/asm_linux_riscv64.s | 22 ++++++++++++++++++++++ src/syscall/exec_linux.go | 6 +++++- src/syscall/syscall_linux_riscv64.go | 4 +--- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/syscall/asm_linux_riscv64.s b/src/syscall/asm_linux_riscv64.s index ad0b6b17d9..f172dd3d9b 100644 --- a/src/syscall/asm_linux_riscv64.s +++ b/src/syscall/asm_linux_riscv64.s @@ -104,6 +104,28 @@ err: MOV A0, err+72(FP) // errno RET +// func rawVforkSyscall(trap, a1 uintptr) (r1, err uintptr) +TEXT ·rawVforkSyscall(SB),NOSPLIT|NOFRAME,$0-32 + MOV a1+8(FP), A0 + MOV ZERO, A1 + MOV ZERO, A2 + MOV ZERO, A3 + MOV ZERO, A4 + MOV ZERO, A5 + MOV trap+0(FP), A7 // syscall entry + ECALL + MOV $-4096, T0 + BLTU T0, A0, err + MOV A0, r1+16(FP) // r1 + MOV ZERO, err+24(FP) // errno + RET +err: + MOV $-1, T0 + MOV T0, r1+16(FP) // r1 + SUB A0, ZERO, A0 + MOV A0, err+24(FP) // errno + RET + TEXT ·rawSyscallNoError(SB),NOSPLIT,$0-48 MOV a1+8(FP), A0 MOV a2+16(FP), A1 diff --git a/src/syscall/exec_linux.go b/src/syscall/exec_linux.go index b7351cda82..b6acad96ea 100644 --- a/src/syscall/exec_linux.go +++ b/src/syscall/exec_linux.go @@ -207,7 +207,11 @@ func forkAndExecInChild1(argv0 *byte, argv, envv []*byte, chroot, dir *byte, att } } - hasRawVforkSyscall := runtime.GOARCH == "amd64" || runtime.GOARCH == "ppc64" || runtime.GOARCH == "s390x" || runtime.GOARCH == "arm64" + var hasRawVforkSyscall bool + switch runtime.GOARCH { + case "amd64", "arm64", "ppc64", "riscv64", "s390x": + hasRawVforkSyscall = true + } // About to call fork. // No more allocation or calls of non-assembly functions. diff --git a/src/syscall/syscall_linux_riscv64.go b/src/syscall/syscall_linux_riscv64.go index d54bd38510..088e23439f 100644 --- a/src/syscall/syscall_linux_riscv64.go +++ b/src/syscall/syscall_linux_riscv64.go @@ -199,6 +199,4 @@ func Pause() error { return err } -func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) { - panic("not implemented") -} +func rawVforkSyscall(trap, a1 uintptr) (r1 uintptr, err Errno) -- cgit v1.3-5-g9baa From c6a11f0dd279f374602794af60c7cde4585a1e6f Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 11 Aug 2020 13:04:48 -0700 Subject: crypto,internal/bytealg: fix assembly that clobbers BP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BP should be callee-save. It will be saved automatically if there is a nonzero frame size. Otherwise, we need to avoid this register. Change-Id: If3f551efa42d830c8793d9f0183cb8daad7a2ab5 Reviewed-on: https://go-review.googlesource.com/c/go/+/248260 Run-TryBot: Keith Randall Reviewed-by: Michael Knyszek Reviewed-by: Martin Möhrmann TryBot-Result: Gobot Gobot --- src/crypto/elliptic/p256_asm_amd64.s | 5 ++-- src/crypto/md5/md5block_amd64.s | 2 +- src/internal/bytealg/index_amd64.s | 52 ++++++++++++++++++------------------ src/runtime/sys_linux_amd64.s | 8 +++--- 4 files changed, 33 insertions(+), 34 deletions(-) diff --git a/src/crypto/elliptic/p256_asm_amd64.s b/src/crypto/elliptic/p256_asm_amd64.s index 7afa54a58c..c77b11bcf2 100644 --- a/src/crypto/elliptic/p256_asm_amd64.s +++ b/src/crypto/elliptic/p256_asm_amd64.s @@ -1336,7 +1336,7 @@ TEXT p256SubInternal(SB),NOSPLIT,$0 RET /* ---------------------------------------*/ -TEXT p256MulInternal(SB),NOSPLIT,$0 +TEXT p256MulInternal(SB),NOSPLIT,$8 MOVQ acc4, mul0 MULQ t0 MOVQ mul0, acc0 @@ -1519,7 +1519,7 @@ TEXT p256MulInternal(SB),NOSPLIT,$0 RET /* ---------------------------------------*/ -TEXT p256SqrInternal(SB),NOSPLIT,$0 +TEXT p256SqrInternal(SB),NOSPLIT,$8 MOVQ acc4, mul0 MULQ acc5 @@ -2345,4 +2345,3 @@ TEXT ·p256PointDoubleAsm(SB),NOSPLIT,$256-48 RET /* ---------------------------------------*/ - diff --git a/src/crypto/md5/md5block_amd64.s b/src/crypto/md5/md5block_amd64.s index 90d932b146..7c7d92d7e8 100644 --- a/src/crypto/md5/md5block_amd64.s +++ b/src/crypto/md5/md5block_amd64.s @@ -13,7 +13,7 @@ // Licence: I hereby disclaim the copyright on this code and place it // in the public domain. -TEXT ·block(SB),NOSPLIT,$0-32 +TEXT ·block(SB),NOSPLIT,$8-32 MOVQ dig+0(FP), BP MOVQ p+8(FP), SI MOVQ p_len+16(FP), DX diff --git a/src/internal/bytealg/index_amd64.s b/src/internal/bytealg/index_amd64.s index 4459820801..6193b57239 100644 --- a/src/internal/bytealg/index_amd64.s +++ b/src/internal/bytealg/index_amd64.s @@ -8,7 +8,7 @@ TEXT ·Index(SB),NOSPLIT,$0-56 MOVQ a_base+0(FP), DI MOVQ a_len+8(FP), DX - MOVQ b_base+24(FP), BP + MOVQ b_base+24(FP), R8 MOVQ b_len+32(FP), AX MOVQ DI, R10 LEAQ ret+48(FP), R11 @@ -17,7 +17,7 @@ TEXT ·Index(SB),NOSPLIT,$0-56 TEXT ·IndexString(SB),NOSPLIT,$0-40 MOVQ a_base+0(FP), DI MOVQ a_len+8(FP), DX - MOVQ b_base+16(FP), BP + MOVQ b_base+16(FP), R8 MOVQ b_len+24(FP), AX MOVQ DI, R10 LEAQ ret+32(FP), R11 @@ -26,7 +26,7 @@ TEXT ·IndexString(SB),NOSPLIT,$0-40 // AX: length of string, that we are searching for // DX: length of string, in which we are searching // DI: pointer to string, in which we are searching -// BP: pointer to string, that we are searching for +// R8: pointer to string, that we are searching for // R11: address, where to put return value // Note: We want len in DX and AX, because PCMPESTRI implicitly consumes them TEXT indexbody<>(SB),NOSPLIT,$0 @@ -37,11 +37,11 @@ TEXT indexbody<>(SB),NOSPLIT,$0 no_sse42: CMPQ AX, $2 JA _3_or_more - MOVW (BP), BP + MOVW (R8), R8 LEAQ -1(DI)(DX*1), DX loop2: MOVW (DI), SI - CMPW SI,BP + CMPW SI,R8 JZ success ADDQ $1,DI CMPQ DI,DX @@ -50,12 +50,12 @@ loop2: _3_or_more: CMPQ AX, $3 JA _4_or_more - MOVW 1(BP), BX - MOVW (BP), BP + MOVW 1(R8), BX + MOVW (R8), R8 LEAQ -2(DI)(DX*1), DX loop3: MOVW (DI), SI - CMPW SI,BP + CMPW SI,R8 JZ partial_success3 ADDQ $1,DI CMPQ DI,DX @@ -72,11 +72,11 @@ partial_success3: _4_or_more: CMPQ AX, $4 JA _5_or_more - MOVL (BP), BP + MOVL (R8), R8 LEAQ -3(DI)(DX*1), DX loop4: MOVL (DI), SI - CMPL SI,BP + CMPL SI,R8 JZ success ADDQ $1,DI CMPQ DI,DX @@ -87,11 +87,11 @@ _5_or_more: JA _8_or_more LEAQ 1(DI)(DX*1), DX SUBQ AX, DX - MOVL -4(BP)(AX*1), BX - MOVL (BP), BP + MOVL -4(R8)(AX*1), BX + MOVL (R8), R8 loop5to7: MOVL (DI), SI - CMPL SI,BP + CMPL SI,R8 JZ partial_success5to7 ADDQ $1,DI CMPQ DI,DX @@ -108,11 +108,11 @@ partial_success5to7: _8_or_more: CMPQ AX, $8 JA _9_or_more - MOVQ (BP), BP + MOVQ (R8), R8 LEAQ -7(DI)(DX*1), DX loop8: MOVQ (DI), SI - CMPQ SI,BP + CMPQ SI,R8 JZ success ADDQ $1,DI CMPQ DI,DX @@ -123,11 +123,11 @@ _9_or_more: JA _16_or_more LEAQ 1(DI)(DX*1), DX SUBQ AX, DX - MOVQ -8(BP)(AX*1), BX - MOVQ (BP), BP + MOVQ -8(R8)(AX*1), BX + MOVQ (R8), R8 loop9to15: MOVQ (DI), SI - CMPQ SI,BP + CMPQ SI,R8 JZ partial_success9to15 ADDQ $1,DI CMPQ DI,DX @@ -144,7 +144,7 @@ partial_success9to15: _16_or_more: CMPQ AX, $16 JA _17_or_more - MOVOU (BP), X1 + MOVOU (R8), X1 LEAQ -15(DI)(DX*1), DX loop16: MOVOU (DI), X2 @@ -161,8 +161,8 @@ _17_or_more: JA _32_or_more LEAQ 1(DI)(DX*1), DX SUBQ AX, DX - MOVOU -16(BP)(AX*1), X0 - MOVOU (BP), X1 + MOVOU -16(R8)(AX*1), X0 + MOVOU (R8), X1 loop17to31: MOVOU (DI), X2 PCMPEQB X1,X2 @@ -188,7 +188,7 @@ partial_success17to31: _32_or_more: CMPQ AX, $32 JA _33_to_63 - VMOVDQU (BP), Y1 + VMOVDQU (R8), Y1 LEAQ -31(DI)(DX*1), DX loop32: VMOVDQU (DI), Y2 @@ -203,8 +203,8 @@ loop32: _33_to_63: LEAQ 1(DI)(DX*1), DX SUBQ AX, DX - VMOVDQU -32(BP)(AX*1), Y0 - VMOVDQU (BP), Y1 + VMOVDQU -32(R8)(AX*1), Y0 + VMOVDQU (R8), Y1 loop33to63: VMOVDQU (DI), Y2 VPCMPEQB Y1, Y2, Y3 @@ -241,10 +241,10 @@ sse42: // This value was determined experimentally and is the ~same // on Nehalem (first with SSE42) and Haswell. JAE _9_or_more - LEAQ 16(BP), SI + LEAQ 16(R8), SI TESTW $0xff0, SI JEQ no_sse42 - MOVOU (BP), X1 + MOVOU (R8), X1 LEAQ -15(DI)(DX*1), SI MOVQ $16, R9 SUBQ AX, R9 // We advance by 16-len(sep) each iteration, so precalculate it into R9 diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index b60057ce83..621c01b365 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -212,7 +212,7 @@ TEXT runtime·walltime1(SB),NOSPLIT,$16-12 // due to stack probes inserted to avoid stack/heap collisions. // See issue #20427. - MOVQ SP, BP // Save old SP; BP unchanged by C code. + MOVQ SP, R12 // Save old SP; R12 unchanged by C code. get_tls(CX) MOVQ g(CX), AX @@ -250,7 +250,7 @@ noswitch: MOVQ 0(SP), AX // sec MOVQ 8(SP), DX // nsec ret: - MOVQ BP, SP // Restore real SP + MOVQ R12, SP // Restore real SP // Restore vdsoPC, vdsoSP // We don't worry about being signaled between the two stores. // If we are not in a signal handler, we'll restore vdsoSP to 0, @@ -277,7 +277,7 @@ fallback: TEXT runtime·nanotime1(SB),NOSPLIT,$16-8 // Switch to g0 stack. See comment above in runtime·walltime. - MOVQ SP, BP // Save old SP; BP unchanged by C code. + MOVQ SP, R12 // Save old SP; R12 unchanged by C code. get_tls(CX) MOVQ g(CX), AX @@ -315,7 +315,7 @@ noswitch: MOVQ 0(SP), AX // sec MOVQ 8(SP), DX // nsec ret: - MOVQ BP, SP // Restore real SP + MOVQ R12, SP // Restore real SP // Restore vdsoPC, vdsoSP // We don't worry about being signaled between the two stores. // If we are not in a signal handler, we'll restore vdsoSP to 0, -- cgit v1.3-5-g9baa From 01f99b4e9540f34b44e13b25f6dd04b82ac952d9 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 11 Aug 2020 13:07:35 -0700 Subject: cmd/compile: mark DUFFZERO/DUFFCOPY as async unsafe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These operations are async unsafe on architectures that use frame pointers. The reason is they rely on data being safe when stored below the stack pointer. They do: 45da69: 48 89 6c 24 f0 mov %rbp,-0x10(%rsp) 45da6e: 48 8d 6c 24 f0 lea -0x10(%rsp),%rbp 45da73: e8 7d d0 ff ff callq 45aaf5 45da78: 48 8b 6d 00 mov 0x0(%rbp),%rbp This dance ensures that inside duffzero, it looks like there is a proper frame pointer set up, so that stack walkbacks work correctly if the kernel samples during duffzero. However, this instruction sequence depends on data not being clobbered even though it is below the stack pointer. If there is an async interrupt at any of those last 3 instructions, and the interrupt decides to insert a call to asyncPreempt, then the saved frame pointer on the stack gets clobbered. The last instruction above then restores junk to the frame pointer. To prevent this, mark these instructions as async unsafe. (The body of duffzero is already async unsafe, as it is in package runtime.) Change-Id: I5562e82f9f5bd2fb543dcf2b6b9133d87ff83032 Reviewed-on: https://go-review.googlesource.com/c/go/+/248261 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Martin Möhrmann Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/ssa/gen/AMD64Ops.go | 2 ++ src/cmd/compile/internal/ssa/gen/ARM64Ops.go | 2 ++ src/cmd/compile/internal/ssa/opGen.go | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go index a3b29049df..e6d66957dd 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/AMD64Ops.go @@ -748,6 +748,7 @@ func init() { clobbers: buildReg("DI"), }, faultOnNilArg0: true, + unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts }, {name: "MOVOconst", reg: regInfo{nil, 0, []regMask{fp}}, typ: "Int128", aux: "Int128", rematerializeable: true}, @@ -786,6 +787,7 @@ func init() { clobberFlags: true, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts }, // arg0 = destination pointer diff --git a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go index b402e35ea6..2424e67e20 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/ARM64Ops.go @@ -507,6 +507,7 @@ func init() { clobbers: buildReg("R20 R30"), }, faultOnNilArg0: true, + unsafePoint: true, // FP maintenance around DUFFZERO can be clobbered by interrupts }, // large zeroing @@ -547,6 +548,7 @@ func init() { }, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, // FP maintenance around DUFFCOPY can be clobbered by interrupts }, // large move diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 9efa1bfcc4..408c855dbd 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -13119,6 +13119,7 @@ var opcodeTable = [...]opInfo{ auxType: auxInt64, argLen: 3, faultOnNilArg0: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 128}, // DI @@ -13196,6 +13197,7 @@ var opcodeTable = [...]opInfo{ clobberFlags: true, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 128}, // DI @@ -20734,6 +20736,7 @@ var opcodeTable = [...]opInfo{ auxType: auxInt64, argLen: 2, faultOnNilArg0: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 1048576}, // R20 @@ -20760,6 +20763,7 @@ var opcodeTable = [...]opInfo{ argLen: 3, faultOnNilArg0: true, faultOnNilArg1: true, + unsafePoint: true, reg: regInfo{ inputs: []inputInfo{ {0, 2097152}, // R21 -- cgit v1.3-5-g9baa From 8c39bbf9c93f773ab351bbddb4c3dd93e4fddc76 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 11 Aug 2020 13:19:57 -0700 Subject: cmd/compile: stop race instrumentation from clobbering frame pointer There is an optimization rule that removes calls to racefuncenter and racefuncexit, if there are no other race calls in the function. The rule removes the call to racefuncenter, but it does *not* remove the store of its argument to the outargs section of the frame. If the outargs section is now size 0 (because the calls to racefuncenter/exit were the only calls), then that argument store clobbers the frame pointer instead. The fix is to remove the argument store when removing the call to racefuncenter. (Racefuncexit doesn't have an argument.) Change-Id: I183ec4d92bbb4920200e1be27b7b8f66b89a2a0a Reviewed-on: https://go-review.googlesource.com/c/go/+/248262 Reviewed-by: Robert Griesemer Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/racewalk.go | 2 +- src/cmd/compile/internal/ssa/rewrite.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/gc/racewalk.go b/src/cmd/compile/internal/gc/racewalk.go index 6f251377c9..3552617401 100644 --- a/src/cmd/compile/internal/gc/racewalk.go +++ b/src/cmd/compile/internal/gc/racewalk.go @@ -42,7 +42,7 @@ var omit_pkgs = []string{ "internal/cpu", } -// Only insert racefuncenterfp/racefuncexit into the following packages. +// Don't insert racefuncenterfp/racefuncexit into the following packages. // Memory accesses in the packages are either uninteresting or will cause false positives. var norace_inst_pkgs = []string{"sync", "sync/atomic"} diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index 2152b1675a..e082bb1dfa 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -1379,6 +1379,15 @@ func needRaceCleanup(sym Sym, v *Value) bool { } } } + if symNamed(sym, "runtime.racefuncenter") { + // If we're removing racefuncenter, remove its argument as well. + if v.Args[0].Op != OpStore { + return false + } + mem := v.Args[0].Args[2] + v.Args[0].reset(OpCopy) + v.Args[0].AddArg(mem) + } return true } -- cgit v1.3-5-g9baa From c810c6db101b53154b06f9ef1ff7455aaff16c36 Mon Sep 17 00:00:00 2001 From: Chirag Sukhala Date: Sun, 16 Aug 2020 21:49:53 +0000 Subject: doc/articles/wiki: add missing log import to net/http tutorial The log package is used with the net/http but was not in the import clause. Change-Id: Ic45b987633adf0ee15defd4d136b5d37027e22b0 GitHub-Last-Rev: e74aff53370569864b7ec8c18617a5d992d34bf2 GitHub-Pull-Request: golang/go#36674 Reviewed-on: https://go-review.googlesource.com/c/go/+/215618 Reviewed-by: Emmanuel Odeke --- doc/articles/wiki/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/articles/wiki/index.html b/doc/articles/wiki/index.html index 4e3a5deab5..a74a58e317 100644 --- a/doc/articles/wiki/index.html +++ b/doc/articles/wiki/index.html @@ -257,6 +257,7 @@ To use the net/http package, it must be imported: import ( "fmt" "io/ioutil" + "log" "net/http" ) -- cgit v1.3-5-g9baa From f7fc25ed5a16ee7678680ffd0bcc3078cc249e0a Mon Sep 17 00:00:00 2001 From: Nigel Tao Date: Fri, 14 Aug 2020 00:04:05 +1000 Subject: image/gif: add more writer benchmarks The two existing benchmarks encode randomized pixels, which isn't very representative. The two new benchmarks encode a PNG photo as a GIF. Also rename the benchmarks for consistency. Also fix the bytes-per-op measure for paletted images, which are 1 (not 4) bytes per pixel. Also simplify BenchmarkEncodeRandomPaletted (formerly just called BenchmarkEncode). It doesn't need to generate a random palette (and the GIF encoder largely doesn't care about the palette's RGBA values). Use palette.Plan9 instead, a pre-existing 256-element color palette. Change-Id: I10a6ea4e9590bb0d9f76e8cc0f4a88d43b1d650d Reviewed-on: https://go-review.googlesource.com/c/go/+/248218 Reviewed-by: David Symonds --- src/image/gif/writer_test.go | 53 +++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/src/image/gif/writer_test.go b/src/image/gif/writer_test.go index 9b15c8d99d..5d1b2c439e 100644 --- a/src/image/gif/writer_test.go +++ b/src/image/gif/writer_test.go @@ -9,6 +9,7 @@ import ( "image" "image/color" "image/color/palette" + "image/draw" _ "image/png" "io/ioutil" "math/rand" @@ -656,25 +657,14 @@ func TestEncodeWrappedImage(t *testing.T) { } } -func BenchmarkEncode(b *testing.B) { +func BenchmarkEncodeRandomPaletted(b *testing.B) { + img := image.NewPaletted(image.Rect(0, 0, 640, 480), palette.Plan9) rnd := rand.New(rand.NewSource(123)) - - // Restrict to a 256-color paletted image to avoid quantization path. - palette := make(color.Palette, 256) - for i := range palette { - palette[i] = color.RGBA{ - uint8(rnd.Intn(256)), - uint8(rnd.Intn(256)), - uint8(rnd.Intn(256)), - 255, - } - } - img := image.NewPaletted(image.Rect(0, 0, 640, 480), palette) for i := range img.Pix { img.Pix[i] = uint8(rnd.Intn(256)) } - b.SetBytes(640 * 480 * 4) + b.SetBytes(640 * 480 * 1) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -682,7 +672,7 @@ func BenchmarkEncode(b *testing.B) { } } -func BenchmarkQuantizedEncode(b *testing.B) { +func BenchmarkEncodeRandomRGBA(b *testing.B) { img := image.NewRGBA(image.Rect(0, 0, 640, 480)) bo := img.Bounds() rnd := rand.New(rand.NewSource(123)) @@ -696,6 +686,7 @@ func BenchmarkQuantizedEncode(b *testing.B) { }) } } + b.SetBytes(640 * 480 * 4) b.ReportAllocs() b.ResetTimer() @@ -703,3 +694,35 @@ func BenchmarkQuantizedEncode(b *testing.B) { Encode(ioutil.Discard, img, nil) } } + +func BenchmarkEncodeRealisticPaletted(b *testing.B) { + rgba, err := readImg("../testdata/video-001.png") + if err != nil { + b.Fatalf("readImg: %v", err) + } + bo := rgba.Bounds() + img := image.NewPaletted(bo, palette.Plan9) + draw.Draw(img, bo, rgba, bo.Min, draw.Src) + + b.SetBytes(int64(bo.Dx() * bo.Dy() * 1)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Encode(ioutil.Discard, img, nil) + } +} + +func BenchmarkEncodeRealisticRGBA(b *testing.B) { + img, err := readImg("../testdata/video-001.png") + if err != nil { + b.Fatalf("readImg: %v", err) + } + bo := img.Bounds() + + b.SetBytes(int64(bo.Dx() * bo.Dy() * 4)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + Encode(ioutil.Discard, img, nil) + } +} -- cgit v1.3-5-g9baa From 6f99b33c18266a8858af96163de97173bdf6f081 Mon Sep 17 00:00:00 2001 From: Polina Osadcha Date: Thu, 18 Jun 2020 16:17:13 +0300 Subject: all: replace Replace(..., -1) with ReplaceAll(...) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change-Id: I8f7cff7a83a9c50bfa3331e8b40e4a6c2e1c0eee Reviewed-on: https://go-review.googlesource.com/c/go/+/245198 Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- doc/progs/run.go | 2 +- src/cmd/cover/cover_test.go | 2 +- src/cmd/go/internal/version/version.go | 2 +- src/runtime/mkpreempt.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/progs/run.go b/doc/progs/run.go index baef3f79f9..8ac75cdcff 100644 --- a/doc/progs/run.go +++ b/doc/progs/run.go @@ -105,7 +105,7 @@ func test(tmpdir, file, want string) error { // Canonicalize output. out = bytes.TrimRight(out, "\n") - out = bytes.Replace(out, []byte{'\n'}, []byte{' '}, -1) + out = bytes.ReplaceAll(out, []byte{'\n'}, []byte{' '}) // Check the result. match, err := regexp.Match(want, out) diff --git a/src/cmd/cover/cover_test.go b/src/cmd/cover/cover_test.go index 8a56e39011..1c252e6e45 100644 --- a/src/cmd/cover/cover_test.go +++ b/src/cmd/cover/cover_test.go @@ -179,7 +179,7 @@ func TestCover(t *testing.T) { } lines := bytes.Split(file, []byte("\n")) for i, line := range lines { - lines[i] = bytes.Replace(line, []byte("LINE"), []byte(fmt.Sprint(i+1)), -1) + lines[i] = bytes.ReplaceAll(line, []byte("LINE"), []byte(fmt.Sprint(i+1))) } // Add a function that is not gofmt'ed. This used to cause a crash. diff --git a/src/cmd/go/internal/version/version.go b/src/cmd/go/internal/version/version.go index 056db7bf9e..c2de8d326d 100644 --- a/src/cmd/go/internal/version/version.go +++ b/src/cmd/go/internal/version/version.go @@ -138,7 +138,7 @@ func scanFile(file string, info os.FileInfo, mustPrint bool) { fmt.Printf("%s: %s\n", file, vers) if *versionM && mod != "" { - fmt.Printf("\t%s\n", strings.Replace(mod[:len(mod)-1], "\n", "\n\t", -1)) + fmt.Printf("\t%s\n", strings.ReplaceAll(mod[:len(mod)-1], "\n", "\n\t")) } } diff --git a/src/runtime/mkpreempt.go b/src/runtime/mkpreempt.go index 1fe77663b9..44dea22ef3 100644 --- a/src/runtime/mkpreempt.go +++ b/src/runtime/mkpreempt.go @@ -131,7 +131,7 @@ func header(arch string) { func p(f string, args ...interface{}) { fmted := fmt.Sprintf(f, args...) - fmt.Fprintf(out, "\t%s\n", strings.Replace(fmted, "\n", "\n\t", -1)) + fmt.Fprintf(out, "\t%s\n", strings.ReplaceAll(fmted, "\n", "\n\t")) } func label(l string) { -- cgit v1.3-5-g9baa From 51ac0f0f4cb432204dee3d434335fd1e61ca8446 Mon Sep 17 00:00:00 2001 From: Polina Osadcha Date: Thu, 18 Jun 2020 16:17:13 +0300 Subject: strings: optimize Replace by using a strings.Builder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit name old time/op new time/op delta ReplaceAll 162ns ±26% 134ns ±26% -17.44% (p=0.014 n=10+10) name old alloc/op new alloc/op delta ReplaceAll 32.0B ± 0% 16.0B ± 0% -50.00% (p=0.000 n=10+10) name old allocs/op new allocs/op delta ReplaceAll 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=10+10) Change-Id: Ia8377141d3adb84c7bd94e511ac8f739915aeb40 Reviewed-on: https://go-review.googlesource.com/c/go/+/245197 Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/strings/strings.go | 12 ++++++------ src/strings/strings_test.go | 9 +++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/strings/strings.go b/src/strings/strings.go index d6f5cea6e6..b429735fea 100644 --- a/src/strings/strings.go +++ b/src/strings/strings.go @@ -934,8 +934,8 @@ func Replace(s, old, new string, n int) string { } // Apply replacements to buffer. - t := make([]byte, len(s)+n*(len(new)-len(old))) - w := 0 + var b Builder + b.Grow(len(s) + n*(len(new)-len(old))) start := 0 for i := 0; i < n; i++ { j := start @@ -947,12 +947,12 @@ func Replace(s, old, new string, n int) string { } else { j += Index(s[start:], old) } - w += copy(t[w:], s[start:j]) - w += copy(t[w:], new) + b.WriteString(s[start:j]) + b.WriteString(new) start = j + len(old) } - w += copy(t[w:], s[start:]) - return string(t[0:w]) + b.WriteString(s[start:]) + return b.String() } // ReplaceAll returns a copy of the string s with all diff --git a/src/strings/strings_test.go b/src/strings/strings_test.go index c01c4dabc5..09e5b27cc3 100644 --- a/src/strings/strings_test.go +++ b/src/strings/strings_test.go @@ -1900,3 +1900,12 @@ func BenchmarkTrimSpace(b *testing.B) { }) } } + +var stringSink string + +func BenchmarkReplaceAll(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + stringSink = ReplaceAll("banana", "a", "<>") + } +} -- cgit v1.3-5-g9baa From 99f179f55a66f35dc7861fa411b42ed61bd0df31 Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Mon, 20 Jul 2020 07:57:06 +0200 Subject: fmt: avoid badverb formatting for %q when used with integers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of returning a bad verb error format for runes above utf8.Maxrune return a quoted utf8.RuneError rune (\ufffd). This makes the behaviour consistent with the "c" verb and aligns behaviour to not return bad verb error format when a verb is applied to the correct argument type. Fixes #14569 Change-Id: I679485f6bb90ebe408423ab68af16cce38816cd0 Reviewed-on: https://go-review.googlesource.com/c/go/+/248759 Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot Reviewed-by: Rob Pike --- src/fmt/fmt_test.go | 8 ++++---- src/fmt/print.go | 6 +----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/fmt/fmt_test.go b/src/fmt/fmt_test.go index 6004061020..87fb323809 100644 --- a/src/fmt/fmt_test.go +++ b/src/fmt/fmt_test.go @@ -290,11 +290,11 @@ var fmtTests = []struct { {"%q", '\U00000e00', `'\u0e00'`}, {"%q", '\U0010ffff', `'\U0010ffff'`}, // Runes that are not valid. - {"%q", int32(-1), "%!q(int32=-1)"}, + {"%q", int32(-1), `'�'`}, {"%q", 0xDC80, `'�'`}, - {"%q", rune(0x110000), "%!q(int32=1114112)"}, - {"%q", int64(0xFFFFFFFFF), "%!q(int64=68719476735)"}, - {"%q", uint64(0xFFFFFFFFF), "%!q(uint64=68719476735)"}, + {"%q", rune(0x110000), `'�'`}, + {"%q", int64(0xFFFFFFFFF), `'�'`}, + {"%q", uint64(0xFFFFFFFFF), `'�'`}, // width {"%5s", "abc", " abc"}, diff --git a/src/fmt/print.go b/src/fmt/print.go index 595869140a..778b5b0938 100644 --- a/src/fmt/print.go +++ b/src/fmt/print.go @@ -388,11 +388,7 @@ func (p *pp) fmtInteger(v uint64, isSigned bool, verb rune) { case 'c': p.fmt.fmtC(v) case 'q': - if v <= utf8.MaxRune { - p.fmt.fmtQc(v) - } else { - p.badVerb(verb) - } + p.fmt.fmtQc(v) case 'U': p.fmt.fmtUnicode(v) default: -- cgit v1.3-5-g9baa From f979d072d339a24e4938d46588c153587d61af19 Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Sun, 3 May 2020 16:26:05 +0200 Subject: runtime: avoid memclr call for keys in mapdelete_fast MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace memclrHasPointers calls for keys in mapdelete_fast* functions with direct writes since the key sizes are known at compile time. name old time/op new time/op delta MapDelete/Pointer/100 33.7ns ± 1% 23.7ns ± 2% -29.68% (p=0.000 n=7+9) MapDelete/Pointer/1000 41.6ns ± 5% 34.9ns ± 4% -16.01% (p=0.000 n=9+10) MapDelete/Pointer/10000 45.6ns ± 1% 38.2ns ± 2% -16.34% (p=0.000 n=8+10) Change-Id: Icaac43b520b93c2cf9fd192b822fae7203a7bbf7 Reviewed-on: https://go-review.googlesource.com/c/go/+/231737 Run-TryBot: Martin Möhrmann TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/map_fast32.go | 8 ++++++-- src/runtime/map_fast64.go | 8 +++++++- src/runtime/map_test.go | 22 ++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/src/runtime/map_fast32.go b/src/runtime/map_fast32.go index 534454f3ad..d035ed0386 100644 --- a/src/runtime/map_fast32.go +++ b/src/runtime/map_fast32.go @@ -299,8 +299,12 @@ search: continue } // Only clear key if there are pointers in it. - if t.key.ptrdata != 0 { - memclrHasPointers(k, t.key.size) + // This can only happen if pointers are 32 bit + // wide as 64 bit pointers do not fit into a 32 bit key. + if sys.PtrSize == 4 && t.key.ptrdata != 0 { + // The key must be a pointer as we checked pointers are + // 32 bits wide and the key is 32 bits wide also. + *(*unsafe.Pointer)(k) = nil } e := add(unsafe.Pointer(b), dataOffset+bucketCnt*4+i*uintptr(t.elemsize)) if t.elem.ptrdata != 0 { diff --git a/src/runtime/map_fast64.go b/src/runtime/map_fast64.go index 1669c7cfe9..f1f3927598 100644 --- a/src/runtime/map_fast64.go +++ b/src/runtime/map_fast64.go @@ -300,7 +300,13 @@ search: } // Only clear key if there are pointers in it. if t.key.ptrdata != 0 { - memclrHasPointers(k, t.key.size) + if sys.PtrSize == 8 { + *(*unsafe.Pointer)(k) = nil + } else { + // There are three ways to squeeze at one ore more 32 bit pointers into 64 bits. + // Just call memclrHasPointers instead of trying to handle all cases here. + memclrHasPointers(k, 8) + } } e := add(unsafe.Pointer(b), dataOffset+bucketCnt*8+i*uintptr(t.elemsize)) if t.elem.ptrdata != 0 { diff --git a/src/runtime/map_test.go b/src/runtime/map_test.go index 1b7ccad6ed..302b3c23c1 100644 --- a/src/runtime/map_test.go +++ b/src/runtime/map_test.go @@ -993,6 +993,27 @@ func benchmarkMapDeleteStr(b *testing.B, n int) { } } +func benchmarkMapDeletePointer(b *testing.B, n int) { + i2p := make([]*int, n) + for i := 0; i < n; i++ { + i2p[i] = new(int) + } + a := make(map[*int]int, n) + b.ResetTimer() + k := 0 + for i := 0; i < b.N; i++ { + if len(a) == 0 { + b.StopTimer() + for j := 0; j < n; j++ { + a[i2p[j]] = j + } + k = i + b.StartTimer() + } + delete(a, i2p[i-k]) + } +} + func runWith(f func(*testing.B, int), v ...int) func(*testing.B) { return func(b *testing.B) { for _, n := range v { @@ -1023,6 +1044,7 @@ func BenchmarkMapDelete(b *testing.B) { b.Run("Int32", runWith(benchmarkMapDeleteInt32, 100, 1000, 10000)) b.Run("Int64", runWith(benchmarkMapDeleteInt64, 100, 1000, 10000)) b.Run("Str", runWith(benchmarkMapDeleteStr, 100, 1000, 10000)) + b.Run("Pointer", runWith(benchmarkMapDeletePointer, 100, 1000, 10000)) } func TestDeferDeleteSlow(t *testing.T) { -- cgit v1.3-5-g9baa From 681559e1f10f83a053b4ebab101de3d77ede8353 Mon Sep 17 00:00:00 2001 From: "zero.xu" Date: Mon, 17 Aug 2020 07:06:32 +0000 Subject: runtime: update comment: modTimer is called by Timer.Reset Change-Id: I97d0d1343d41b603a68388e496411fb040dc6d66 GitHub-Last-Rev: d11177ad249bd844dd9e7e355eea28596d0b1fa8 GitHub-Pull-Request: golang/go#38625 Reviewed-on: https://go-review.googlesource.com/c/go/+/229767 Reviewed-by: Emmanuel Odeke --- src/runtime/time.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/time.go b/src/runtime/time.go index fdb5066b24..f895bf8443 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -403,7 +403,7 @@ func dodeltimer0(pp *p) { } // modtimer modifies an existing timer. -// This is called by the netpoll code or time.Ticker.Reset. +// This is called by the netpoll code or time.Ticker.Reset or time.Timer.Reset. // Reports whether the timer was modified before it was run. func modtimer(t *timer, when, period int64, f func(interface{}, uintptr), arg interface{}, seq uintptr) bool { if when < 0 { -- cgit v1.3-5-g9baa From 9138a2a67f7f29948f6f608bf904b3605f1b45d0 Mon Sep 17 00:00:00 2001 From: Joel Sing Date: Tue, 26 May 2020 14:40:44 +1000 Subject: cmd/link: avoid duplicate DT_NEEDED entries When adding a new library entry, ensure we record it as seen to avoid adding duplicates of it. Fixes #39256 Change-Id: Id309adf80c533d78fd485517c18bc9ab5f1d29fb Reviewed-on: https://go-review.googlesource.com/c/go/+/235257 Run-TryBot: Joel Sing TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/link/internal/ld/elf.go | 1 + src/cmd/link/internal/ld/elf_test.go | 55 +++++++++++++++++++++++ src/cmd/link/internal/ld/testdata/issue39256/x.go | 20 +++++++++ src/cmd/link/internal/ld/testdata/issue39256/x.s | 10 +++++ 4 files changed, 86 insertions(+) create mode 100644 src/cmd/link/internal/ld/testdata/issue39256/x.go create mode 100644 src/cmd/link/internal/ld/testdata/issue39256/x.s diff --git a/src/cmd/link/internal/ld/elf.go b/src/cmd/link/internal/ld/elf.go index 957f5081f6..2862f65f9f 100644 --- a/src/cmd/link/internal/ld/elf.go +++ b/src/cmd/link/internal/ld/elf.go @@ -2378,6 +2378,7 @@ func elfadddynsym(ldr *loader.Loader, target *Target, syms *ArchSyms, s loader.S if target.Arch.Family == sys.AMD64 && !cgoeDynamic && dil != "" && !seenlib[dil] { du := ldr.MakeSymbolUpdater(syms.Dynamic) Elfwritedynent(target.Arch, du, DT_NEEDED, uint64(dstru.Addstring(dil))) + seenlib[dil] = true } } else { diff --git a/src/cmd/link/internal/ld/elf_test.go b/src/cmd/link/internal/ld/elf_test.go index 8e86beb1ec..37f0e77336 100644 --- a/src/cmd/link/internal/ld/elf_test.go +++ b/src/cmd/link/internal/ld/elf_test.go @@ -13,6 +13,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "testing" ) @@ -77,3 +78,57 @@ func main() { t.Fatalf("Unexpected sh info, want greater than 0, got: %d", section.Info) } } + +func TestNoDuplicateNeededEntries(t *testing.T) { + testenv.MustHaveGoBuild(t) + testenv.MustHaveCGO(t) + + // run this test on just a small set of platforms (no need to test it + // across the board given the nature of the test). + pair := runtime.GOOS + "-" + runtime.GOARCH + switch pair { + case "linux-amd64", "freebsd-amd64", "openbsd-amd64": + default: + t.Skip("no need for test on " + pair) + } + + t.Parallel() + + dir, err := ioutil.TempDir("", "no-dup-needed") + if err != nil { + t.Fatalf("Failed to create temp dir: %v", err) + } + defer os.RemoveAll(dir) + + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + + path := filepath.Join(dir, "x") + argv := []string{"build", "-o", path, filepath.Join(wd, "testdata", "issue39256")} + out, err := exec.Command(testenv.GoToolPath(t), argv...).CombinedOutput() + if err != nil { + t.Fatalf("Build failure: %s\n%s\n", err, string(out)) + } + + f, err := elf.Open(path) + if err != nil { + t.Fatalf("Failed to open ELF file: %v", err) + } + libs, err := f.ImportedLibraries() + if err != nil { + t.Fatalf("Failed to read imported libraries: %v", err) + } + + var count int + for _, lib := range libs { + if lib == "libc.so" { + count++ + } + } + + if got, want := count, 1; got != want { + t.Errorf("Got %d entries for `libc.so`, want %d", got, want) + } +} diff --git a/src/cmd/link/internal/ld/testdata/issue39256/x.go b/src/cmd/link/internal/ld/testdata/issue39256/x.go new file mode 100644 index 0000000000..d8562ad172 --- /dev/null +++ b/src/cmd/link/internal/ld/testdata/issue39256/x.go @@ -0,0 +1,20 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + _ "unsafe" +) + +//go:cgo_import_dynamic libc_getpid getpid "libc.so" +//go:cgo_import_dynamic libc_kill kill "libc.so" +//go:cgo_import_dynamic libc_close close "libc.so" +//go:cgo_import_dynamic libc_open open "libc.so" + +func trampoline() + +func main() { + trampoline() +} diff --git a/src/cmd/link/internal/ld/testdata/issue39256/x.s b/src/cmd/link/internal/ld/testdata/issue39256/x.s new file mode 100644 index 0000000000..41a54b2e04 --- /dev/null +++ b/src/cmd/link/internal/ld/testdata/issue39256/x.s @@ -0,0 +1,10 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +TEXT ·trampoline(SB),0,$0 + CALL libc_getpid(SB) + CALL libc_kill(SB) + CALL libc_open(SB) + CALL libc_close(SB) + RET -- cgit v1.3-5-g9baa From d79350bac73670c04a91b6761d334b810201f6ee Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Mon, 4 May 2020 18:36:31 +0200 Subject: runtime: use hw.ncpuonline sysctl in getncpu on netbsd Since NetBSD 7, hw.ncpuonline reports the number of CPUs online, while hw.cpu reports the number of CPUs configured. Try hw.cpuonline first and fall back to hw.ncpu in case it fails (which is the case on NetBSD before 7.0). This follows the behavior on OpenBSD (see CL 161757). Also, Go in pkgsrc is patched to use hw.cpuonline, so this CL would allow said patch to be dropped. Updates #30824 Change-Id: Id1c19dff2c1e4401e6074179fae7c708ba0e3098 Reviewed-on: https://go-review.googlesource.com/c/go/+/231957 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor Reviewed-by: Benny Siegert --- src/runtime/os_netbsd.go | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/src/runtime/os_netbsd.go b/src/runtime/os_netbsd.go index 97106c7b9d..f7f90cedc1 100644 --- a/src/runtime/os_netbsd.go +++ b/src/runtime/os_netbsd.go @@ -95,18 +95,28 @@ var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0) // From NetBSD's const ( - _CTL_HW = 6 - _HW_NCPU = 3 - _HW_PAGESIZE = 7 + _CTL_HW = 6 + _HW_NCPU = 3 + _HW_PAGESIZE = 7 + _HW_NCPUONLINE = 16 ) -func getncpu() int32 { - mib := [2]uint32{_CTL_HW, _HW_NCPU} - out := uint32(0) +func sysctlInt(mib []uint32) (int32, bool) { + var out int32 nout := unsafe.Sizeof(out) - ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) - if ret >= 0 { - return int32(out) + ret := sysctl(&mib[0], uint32(len(mib)), (*byte)(unsafe.Pointer(&out)), &nout, nil, 0) + if ret < 0 { + return 0, false + } + return out, true +} + +func getncpu() int32 { + if n, ok := sysctlInt([]uint32{_CTL_HW, _HW_NCPUONLINE}); ok { + return int32(n) + } + if n, ok := sysctlInt([]uint32{_CTL_HW, _HW_NCPU}); ok { + return int32(n) } return 1 } -- cgit v1.3-5-g9baa From bf512685fee6282f1a50069ef444412bdf59611b Mon Sep 17 00:00:00 2001 From: Cholerae Hu Date: Wed, 5 Aug 2020 13:52:32 +0800 Subject: syscall: cap RLIMIT_NOFILE soft limit in TestRlimit on darwin On some machines, kern.maxfilesperproc is 4096. If Rlimit.Cur is larger than that, Setrlimit will get an errEINVAL. Fixes #40564. Change-Id: Ib94303c790a489ff0559c88d41a021e514d18f8d Reviewed-on: https://go-review.googlesource.com/c/go/+/246658 Reviewed-by: Tobias Klauser Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot --- src/syscall/syscall_unix_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/syscall/syscall_unix_test.go b/src/syscall/syscall_unix_test.go index 13b79ca8d8..3c7982eefe 100644 --- a/src/syscall/syscall_unix_test.go +++ b/src/syscall/syscall_unix_test.go @@ -336,11 +336,11 @@ func TestRlimit(t *testing.T) { } set := rlimit set.Cur = set.Max - 1 - if runtime.GOOS == "darwin" && set.Cur > 10240 { - // The max file limit is 10240, even though - // the max returned by Getrlimit is 1<<63-1. - // This is OPEN_MAX in sys/syslimits.h. - set.Cur = 10240 + if runtime.GOOS == "darwin" && set.Cur > 4096 { + // rlim_min for RLIMIT_NOFILE should be equal to + // or lower than kern.maxfilesperproc, which on + // some machines are 4096. See #40564. + set.Cur = 4096 } err = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &set) if err != nil { @@ -353,8 +353,8 @@ func TestRlimit(t *testing.T) { } set = rlimit set.Cur = set.Max - 1 - if runtime.GOOS == "darwin" && set.Cur > 10240 { - set.Cur = 10240 + if runtime.GOOS == "darwin" && set.Cur > 4096 { + set.Cur = 4096 } if set != get { t.Fatalf("Rlimit: change failed: wanted %#v got %#v", set, get) -- cgit v1.3-5-g9baa From dc12d5b0f5e9c1cfec2a8eb6dd7ff3473c36d45c Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Mon, 17 Aug 2020 11:28:26 +0200 Subject: all: add empty line between copyright header and package clause Makes sure the copyright notice is not interpreted as the package level godoc. Change-Id: I2afce7c9d620f19d51ec1438b1d0db1774b57146 Reviewed-on: https://go-review.googlesource.com/c/go/+/248760 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Dave Cheney --- src/cmd/compile/internal/ssa/debug.go | 1 + src/cmd/compile/internal/ssa/passbm_test.go | 1 + src/cmd/go/internal/trace/trace.go | 1 + src/cmd/link/internal/benchmark/bench_test.go | 1 + src/cmd/link/internal/ld/errors.go | 1 + src/runtime/closure_test.go | 1 + src/runtime/map_benchmark_test.go | 1 + src/runtime/slice_test.go | 1 + src/sync/cond_test.go | 1 + test/fixedbugs/issue15281.go | 1 + 10 files changed, 10 insertions(+) diff --git a/src/cmd/compile/internal/ssa/debug.go b/src/cmd/compile/internal/ssa/debug.go index 13fe67cbca..6353f72897 100644 --- a/src/cmd/compile/internal/ssa/debug.go +++ b/src/cmd/compile/internal/ssa/debug.go @@ -1,6 +1,7 @@ // Copyright 2017 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package ssa import ( diff --git a/src/cmd/compile/internal/ssa/passbm_test.go b/src/cmd/compile/internal/ssa/passbm_test.go index eefdbb8722..3fd3eb579b 100644 --- a/src/cmd/compile/internal/ssa/passbm_test.go +++ b/src/cmd/compile/internal/ssa/passbm_test.go @@ -1,6 +1,7 @@ // Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package ssa import ( diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index 7cb7636a34..c8fac92c9f 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -1,6 +1,7 @@ // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package trace import ( diff --git a/src/cmd/link/internal/benchmark/bench_test.go b/src/cmd/link/internal/benchmark/bench_test.go index d8ec717c7c..419dc55724 100644 --- a/src/cmd/link/internal/benchmark/bench_test.go +++ b/src/cmd/link/internal/benchmark/bench_test.go @@ -1,6 +1,7 @@ // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package benchmark import ( diff --git a/src/cmd/link/internal/ld/errors.go b/src/cmd/link/internal/ld/errors.go index c5ce097fde..d6e8ff236d 100644 --- a/src/cmd/link/internal/ld/errors.go +++ b/src/cmd/link/internal/ld/errors.go @@ -1,6 +1,7 @@ // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package ld import ( diff --git a/src/runtime/closure_test.go b/src/runtime/closure_test.go index ea65fbd5f5..741c932eab 100644 --- a/src/runtime/closure_test.go +++ b/src/runtime/closure_test.go @@ -1,6 +1,7 @@ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package runtime_test import "testing" diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go index 893cb6c5b6..d0becc9ddb 100644 --- a/src/runtime/map_benchmark_test.go +++ b/src/runtime/map_benchmark_test.go @@ -1,6 +1,7 @@ // Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package runtime_test import ( diff --git a/src/runtime/slice_test.go b/src/runtime/slice_test.go index e963a43dd3..cd2bc26d1e 100644 --- a/src/runtime/slice_test.go +++ b/src/runtime/slice_test.go @@ -1,6 +1,7 @@ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package runtime_test import ( diff --git a/src/sync/cond_test.go b/src/sync/cond_test.go index 9d0d9adc74..859cae59bc 100644 --- a/src/sync/cond_test.go +++ b/src/sync/cond_test.go @@ -1,6 +1,7 @@ // Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package sync_test import ( diff --git a/test/fixedbugs/issue15281.go b/test/fixedbugs/issue15281.go index 187c96f218..390867c848 100644 --- a/test/fixedbugs/issue15281.go +++ b/test/fixedbugs/issue15281.go @@ -3,6 +3,7 @@ // Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + package main import "runtime" -- cgit v1.3-5-g9baa From 7bbd5ca5a6a94f58d33de6b1244248a32dc8cd9c Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 22 Jul 2020 11:21:36 -0400 Subject: runtime: replace index and contains with bytealg calls The runtime has its own implementation of string indexing. To reduce code duplication and cognitive load, replace this with calls to the internal/bytealg package. We can't do this on Plan 9 because it needs string indexing in a note handler (which isn't allowed to use the optimized bytealg version because it uses SSE), so we can't just eliminate the index function, but this CL does down-scope it so make it clear it's only for note handlers on Plan 9. Change-Id: Ie1a142678262048515c481e8c26313b80c5875df Reviewed-on: https://go-review.googlesource.com/c/go/+/244537 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Michael Knyszek Reviewed-by: Michael Pratt --- src/runtime/os_plan9.go | 18 ++++++++++++++++-- src/runtime/proc.go | 3 ++- src/runtime/runtime1.go | 5 +++-- src/runtime/string.go | 16 ---------------- src/runtime/traceback.go | 3 ++- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index 9e187d2220..128c30adeb 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -82,10 +82,10 @@ func sigpanic() { note := gostringnocopy((*byte)(unsafe.Pointer(g.m.notesig))) switch g.sig { case _SIGRFAULT, _SIGWFAULT: - i := index(note, "addr=") + i := indexNoFloat(note, "addr=") if i >= 0 { i += 5 - } else if i = index(note, "va="); i >= 0 { + } else if i = indexNoFloat(note, "va="); i >= 0 { i += 3 } else { panicmem() @@ -111,6 +111,20 @@ func sigpanic() { } } +// indexNoFloat is bytealg.IndexString but safe to use in a note +// handler. +func indexNoFloat(s, t string) int { + if len(t) == 0 { + return 0 + } + for i := 0; i < len(s); i++ { + if s[i] == t[0] && hasPrefix(s[i:], t) { + return i + } + } + return -1 +} + func atolwhex(p string) int64 { for hasPrefix(p, " ") || hasPrefix(p, "\t") { p = p[1:] diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 035822216d..ed7e2128ae 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -5,6 +5,7 @@ package runtime import ( + "internal/bytealg" "internal/cpu" "runtime/internal/atomic" "runtime/internal/sys" @@ -5460,7 +5461,7 @@ func haveexperiment(name string) bool { x := sys.Goexperiment for x != "" { xname := "" - i := index(x, ",") + i := bytealg.IndexByteString(x, ',') if i < 0 { xname, x = x, "" } else { diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index c65a534ef6..7c893aa25c 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -5,6 +5,7 @@ package runtime import ( + "internal/bytealg" "runtime/internal/atomic" "runtime/internal/sys" "unsafe" @@ -347,13 +348,13 @@ func parsedebugvars() { for p := gogetenv("GODEBUG"); p != ""; { field := "" - i := index(p, ",") + i := bytealg.IndexByteString(p, ',') if i < 0 { field, p = p, "" } else { field, p = p[:i], p[i+1:] } - i = index(field, "=") + i = bytealg.IndexByteString(field, '=') if i < 0 { continue } diff --git a/src/runtime/string.go b/src/runtime/string.go index 251044231e..9a601f0094 100644 --- a/src/runtime/string.go +++ b/src/runtime/string.go @@ -335,22 +335,6 @@ func gostringn(p *byte, l int) string { return s } -func index(s, t string) int { - if len(t) == 0 { - return 0 - } - for i := 0; i < len(s); i++ { - if s[i] == t[0] && hasPrefix(s[i:], t) { - return i - } - } - return -1 -} - -func contains(s, t string) bool { - return index(s, t) >= 0 -} - func hasPrefix(s, prefix string) bool { return len(s) >= len(prefix) && s[:len(prefix)] == prefix } diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 944c8473d2..96e552524e 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -5,6 +5,7 @@ package runtime import ( + "internal/bytealg" "runtime/internal/atomic" "runtime/internal/sys" "unsafe" @@ -848,7 +849,7 @@ func showfuncinfo(f funcInfo, firstFrame bool, funcID, childID funcID) bool { return true } - return contains(name, ".") && (!hasPrefix(name, "runtime.") || isExportedRuntime(name)) + return bytealg.IndexByteString(name, '.') >= 0 && (!hasPrefix(name, "runtime.") || isExportedRuntime(name)) } // isExportedRuntime reports whether name is an exported runtime function. -- cgit v1.3-5-g9baa From 7148abc1b900555199998aac25af11783a9eb41c Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 5 Jun 2020 16:44:29 -0400 Subject: runtime: simplify heapBitsSetType doubleCheck The heapBitsSetType function has a slow doubleCheck debugging mode that checks the bitmap written out by the rest of the function using far more obvious logic. But even this has some surprisingly complex logic in it. Simplify it a bit. This also happens to fix the logic on 32-bit. Fixes #40335. Change-Id: I5cee482ad8adbd01cf5b98e35a270fe941ba4940 Reviewed-on: https://go-review.googlesource.com/c/go/+/244538 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Michael Knyszek --- src/runtime/mbitmap.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 35332c91c4..cad6f56404 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -1403,17 +1403,20 @@ Phase4: // Double check the whole bitmap. if doubleCheck { // x+size may not point to the heap, so back up one - // word and then call next(). - end := heapBitsForAddr(x + size - sys.PtrSize).next() - endAI := arenaIdx(end.arena) - if !outOfPlace && (end.bitp == nil || (end.shift == 0 && end.bitp == &mheap_.arenas[endAI.l1()][endAI.l2()].bitmap[0])) { - // The unrolling code above walks hbitp just - // past the bitmap without moving to the next - // arena. Synthesize this for end.bitp. - end.arena-- - endAI = arenaIdx(end.arena) - end.bitp = addb(&mheap_.arenas[endAI.l1()][endAI.l2()].bitmap[0], heapArenaBitmapBytes) - end.last = nil + // word and then advance it the way we do above. + end := heapBitsForAddr(x + size - sys.PtrSize) + if outOfPlace { + // In out-of-place copying, we just advance + // using next. + end = end.next() + } else { + // Don't use next because that may advance to + // the next arena and the in-place logic + // doesn't do that. + end.shift += heapBitsShift + if end.shift == 4*heapBitsShift { + end.bitp, end.shift = add1(end.bitp), 0 + } } if typ.kind&kindGCProg == 0 && (hbitp != end.bitp || (w == nw+2) != (end.shift == 2)) { println("ended at wrong bitmap byte for", typ.string(), "x", dataSize/typ.size) @@ -1437,8 +1440,9 @@ Phase4: var have, want uint8 have = (*h.bitp >> h.shift) & (bitPointer | bitScan) if i >= totalptr { - want = 0 // deadmarker if typ.kind&kindGCProg != 0 && i < (totalptr+3)/4*4 { + // heapBitsSetTypeGCProg always fills + // in full nibbles of bitScan. want = bitScan } } else { -- cgit v1.3-5-g9baa From d19fedd180fceb6a60961e19387893ddb047e4e6 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 5 Jun 2020 16:48:03 -0400 Subject: runtime: move checkmarks to a separate bitmap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently, the GC stores the object marks for checkmarks mode in the heap bitmap using a rather complex encoding: for one word objects, the checkmark is stored in the pointer/scalar bit since one word objects must be pointers; for larger objects, the checkmark is stored in what would be the scan/dead bit for the second word of the object. This encoding made more sense when the runtime used the first scan/dead bit as the regular mark bit, but we moved away from that long ago. This encoding and overloading of the heap bitmap bits causes a great deal of complexity in many parts of the allocator and garbage collector and leads to some subtle bugs like #15903. This CL moves the checkmarks mark bits into their own per-arena bitmap and reclaims the second scan/dead bit as a regular scan/dead bit. I tested this by enabling doubleCheck mode in heapBitsSetType and running in both regular and GODEBUG=gccheckmark=1 mode. Fixes #15903. No performance degradation. (Very slight improvement on a few benchmarks, but it's probably just noise.) name old time/op new time/op delta BiogoIgor 16.6s ± 1% 16.4s ± 1% -0.94% (p=0.000 n=25+24) BiogoKrishna 19.2s ± 3% 19.2s ± 3% ~ (p=0.638 n=23+25) BleveIndexBatch100 6.12s ± 5% 6.17s ± 4% ~ (p=0.170 n=25+25) CompileTemplate 206ms ± 1% 205ms ± 1% -0.43% (p=0.005 n=24+24) CompileUnicode 82.2ms ± 2% 81.5ms ± 2% -0.95% (p=0.001 n=22+22) CompileGoTypes 755ms ± 3% 754ms ± 4% ~ (p=0.715 n=25+25) CompileCompiler 3.73s ± 1% 3.73s ± 1% ~ (p=0.445 n=25+24) CompileSSA 8.67s ± 1% 8.66s ± 1% ~ (p=0.836 n=24+22) CompileFlate 134ms ± 2% 133ms ± 1% -0.66% (p=0.001 n=24+23) CompileGoParser 164ms ± 1% 163ms ± 1% -0.85% (p=0.000 n=24+24) CompileReflect 466ms ± 5% 466ms ± 3% ~ (p=0.863 n=25+25) CompileTar 182ms ± 1% 182ms ± 1% -0.31% (p=0.048 n=24+24) CompileXML 249ms ± 1% 248ms ± 1% -0.32% (p=0.031 n=21+25) CompileStdCmd 10.3s ± 1% 10.3s ± 1% ~ (p=0.459 n=23+23) FoglemanFauxGLRenderRotateBoat 8.66s ± 1% 8.62s ± 1% -0.47% (p=0.000 n=23+24) FoglemanPathTraceRenderGopherIter1 20.3s ± 3% 20.2s ± 2% ~ (p=0.893 n=25+25) GopherLuaKNucleotide 29.7s ± 1% 29.8s ± 2% ~ (p=0.421 n=24+25) MarkdownRenderXHTML 246ms ± 1% 247ms ± 1% ~ (p=0.558 n=25+24) Tile38WithinCircle100kmRequest 779µs ± 4% 779µs ± 3% ~ (p=0.954 n=25+25) Tile38IntersectsCircle100kmRequest 1.02ms ± 3% 1.01ms ± 4% ~ (p=0.658 n=25+25) Tile38KNearestLimit100Request 984µs ± 4% 986µs ± 4% ~ (p=0.627 n=24+25) [Geo mean] 552ms 551ms -0.19% https://perf.golang.org/search?q=upload:20200723.6 Change-Id: Ic703f26a83fb034941dc6f4788fc997d56890dec Reviewed-on: https://go-review.googlesource.com/c/go/+/244539 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Michael Knyszek Reviewed-by: Martin Möhrmann --- src/reflect/all_test.go | 5 +- src/runtime/cgocall.go | 2 +- src/runtime/gcinfo_test.go | 19 +---- src/runtime/heapdump.go | 2 +- src/runtime/mbitmap.go | 173 +++++++++------------------------------------ src/runtime/mcheckmark.go | 100 ++++++++++++++++++++++++++ src/runtime/mgc.go | 4 +- src/runtime/mgcmark.go | 70 +----------------- src/runtime/mheap.go | 4 ++ 9 files changed, 148 insertions(+), 231 deletions(-) create mode 100644 src/runtime/mcheckmark.go diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index 6b31568bb9..ed2f225077 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -6467,12 +6467,9 @@ func verifyGCBitsSlice(t *testing.T, typ Type, cap int, bits []byte) { // Repeat the bitmap for the slice size, trimming scalars in // the last element. bits = rep(cap, bits) - for len(bits) > 2 && bits[len(bits)-1] == 0 { + for len(bits) > 0 && bits[len(bits)-1] == 0 { bits = bits[:len(bits)-1] } - if len(bits) == 2 && bits[0] == 0 && bits[1] == 0 { - bits = bits[:0] - } if !bytes.Equal(heapBits, bits) { t.Errorf("heapBits incorrect for make(%v, 0, %v)\nhave %v\nwant %v", typ, cap, heapBits, bits) } diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go index a4e64b00cc..099aa540e0 100644 --- a/src/runtime/cgocall.go +++ b/src/runtime/cgocall.go @@ -605,7 +605,7 @@ func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) { hbits := heapBitsForAddr(base) n := span.elemsize for i = uintptr(0); i < n; i += sys.PtrSize { - if i != 1*sys.PtrSize && !hbits.morePointers() { + if !hbits.morePointers() { // No more possible pointers. break } diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index ec1ba90c2e..0808b416f0 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -77,7 +77,7 @@ func TestGCInfo(t *testing.T) { } for i := 0; i < 10; i++ { - verifyGCInfo(t, "heap Ptr", escape(new(Ptr)), trimDead(padDead(infoPtr))) + verifyGCInfo(t, "heap Ptr", escape(new(Ptr)), trimDead(infoPtr)) verifyGCInfo(t, "heap PtrSlice", escape(&make([]*byte, 10)[0]), trimDead(infoPtr10)) verifyGCInfo(t, "heap ScalarPtr", escape(new(ScalarPtr)), trimDead(infoScalarPtr)) verifyGCInfo(t, "heap ScalarPtrSlice", escape(&make([]ScalarPtr, 4)[0]), trimDead(infoScalarPtr4)) @@ -97,25 +97,10 @@ func verifyGCInfo(t *testing.T, name string, p interface{}, mask0 []byte) { } } -func padDead(mask []byte) []byte { - // Because the dead bit isn't encoded in the second word, - // and because on 32-bit systems a one-word allocation - // uses a two-word block, the pointer info for a one-word - // object needs to be expanded to include an extra scalar - // on 32-bit systems to match the heap bitmap. - if runtime.PtrSize == 4 && len(mask) == 1 { - return []byte{mask[0], 0} - } - return mask -} - func trimDead(mask []byte) []byte { - for len(mask) > 2 && mask[len(mask)-1] == typeScalar { + for len(mask) > 0 && mask[len(mask)-1] == typeScalar { mask = mask[:len(mask)-1] } - if len(mask) == 2 && mask[0] == typeScalar && mask[1] == typeScalar { - mask = mask[:0] - } return mask } diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index cfd5c251b4..4c35309211 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -713,7 +713,7 @@ func makeheapobjbv(p uintptr, size uintptr) bitvector { i := uintptr(0) hbits := heapBitsForAddr(p) for ; i < nptr; i++ { - if i != 1 && !hbits.morePointers() { + if !hbits.morePointers() { break // end of object } if hbits.isPointer() { diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index cad6f56404..8de44c14b9 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -6,10 +6,11 @@ // // Stack, data, and bss bitmaps // -// Stack frames and global variables in the data and bss sections are described -// by 1-bit bitmaps in which 0 means uninteresting and 1 means live pointer -// to be visited during GC. The bits in each byte are consumed starting with -// the low bit: 1<<0, 1<<1, and so on. +// Stack frames and global variables in the data and bss sections are +// described by bitmaps with 1 bit per pointer-sized word. A "1" bit +// means the word is a live pointer to be visited by the GC (referred to +// as "pointer"). A "0" bit means the word should be ignored by GC +// (referred to as "scalar", though it could be a dead pointer value). // // Heap bitmap // @@ -20,18 +21,13 @@ // through start+3*ptrSize, ha.bitmap[1] holds the entries for // start+4*ptrSize through start+7*ptrSize, and so on. // -// In each 2-bit entry, the lower bit holds the same information as in the 1-bit -// bitmaps: 0 means uninteresting and 1 means live pointer to be visited during GC. -// The meaning of the high bit depends on the position of the word being described -// in its allocated object. In all words *except* the second word, the -// high bit indicates that the object is still being described. In -// these words, if a bit pair with a high bit 0 is encountered, the -// low bit can also be assumed to be 0, and the object description is -// over. This 00 is called the ``dead'' encoding: it signals that the -// rest of the words in the object are uninteresting to the garbage -// collector. -// -// In the second word, the high bit is the GC ``checkmarked'' bit (see below). +// In each 2-bit entry, the lower bit is a pointer/scalar bit, just +// like in the stack/data bitmaps described above. The upper bit +// indicates scan/dead: a "1" value ("scan") indicates that there may +// be pointers in later words of the allocation, and a "0" value +// ("dead") indicates there are no more pointers in the allocation. If +// the upper bit is 0, the lower bit must also be 0, and this +// indicates scanning can ignore the rest of the allocation. // // The 2-bit entries are split when written into the byte, so that the top half // of the byte contains 4 high bits and the bottom half contains 4 low (pointer) @@ -39,38 +35,14 @@ // This form allows a copy from the 1-bit to the 4-bit form to keep the // pointer bits contiguous, instead of having to space them out. // -// The code makes use of the fact that the zero value for a heap bitmap -// has no live pointer bit set and is (depending on position), not used, -// not checkmarked, and is the dead encoding. -// These properties must be preserved when modifying the encoding. +// The code makes use of the fact that the zero value for a heap +// bitmap means scalar/dead. This property must be preserved when +// modifying the encoding. // // The bitmap for noscan spans is not maintained. Code must ensure // that an object is scannable before consulting its bitmap by // checking either the noscan bit in the span or by consulting its // type's information. -// -// Checkmarks -// -// In a concurrent garbage collector, one worries about failing to mark -// a live object due to mutations without write barriers or bugs in the -// collector implementation. As a sanity check, the GC has a 'checkmark' -// mode that retraverses the object graph with the world stopped, to make -// sure that everything that should be marked is marked. -// In checkmark mode, in the heap bitmap, the high bit of the 2-bit entry -// for the second word of the object holds the checkmark bit. -// When not in checkmark mode, this bit is set to 1. -// -// The smallest possible allocation is 8 bytes. On a 32-bit machine, that -// means every allocated object has two words, so there is room for the -// checkmark bit. On a 64-bit machine, however, the 8-byte allocation is -// just one word, so the second bit pair is not available for encoding the -// checkmark. However, because non-pointer allocations are combined -// into larger 16-byte (maxTinySize) allocations, a plain 8-byte allocation -// must be a pointer, so the type bit in the first word is not actually needed. -// It is still used in general, except in checkmark the type bit is repurposed -// as the checkmark bit and then reinitialized (to 1) as the type bit when -// finished. -// package runtime @@ -551,33 +523,6 @@ func (h heapBits) isPointer() bool { return h.bits()&bitPointer != 0 } -// isCheckmarked reports whether the heap bits have the checkmarked bit set. -// It must be told how large the object at h is, because the encoding of the -// checkmark bit varies by size. -// h must describe the initial word of the object. -func (h heapBits) isCheckmarked(size uintptr) bool { - if size == sys.PtrSize { - return (*h.bitp>>h.shift)&bitPointer != 0 - } - // All multiword objects are 2-word aligned, - // so we know that the initial word's 2-bit pair - // and the second word's 2-bit pair are in the - // same heap bitmap byte, *h.bitp. - return (*h.bitp>>(heapBitsShift+h.shift))&bitScan != 0 -} - -// setCheckmarked sets the checkmarked bit. -// It must be told how large the object at h is, because the encoding of the -// checkmark bit varies by size. -// h must describe the initial word of the object. -func (h heapBits) setCheckmarked(size uintptr) { - if size == sys.PtrSize { - atomic.Or8(h.bitp, bitPointer<= nw { goto Phase3 } @@ -1203,14 +1103,13 @@ func heapBitsSetType(x, size, dataSize uintptr, typ *_type) { // We took care of 1-word and 2-word objects above, // so this is at least a 6-word object. hb = (b & (bitPointer | bitPointer< 1 { + hb |= bitScan << (3 * heapBitsShift) + } b >>= 2 nb -= 2 - // Note: no bitScan for second word because that's - // the checkmark. - *hbitp &^= uint8((bitPointer | bitScan | (bitPointer << heapBitsShift)) << (2 * heapBitsShift)) + *hbitp &^= uint8((bitPointer | bitScan | ((bitPointer | bitScan) << heapBitsShift)) << (2 * heapBitsShift)) *hbitp |= uint8(hb) hbitp = add1(hbitp) if w += 2; w >= nw { @@ -1449,11 +1348,7 @@ Phase4: if j < nptr && (*addb(ptrmask, j/8)>>(j%8))&1 != 0 { want |= bitPointer } - if i != 1 { - want |= bitScan - } else { - have &^= bitScan - } + want |= bitScan } if have != want { println("mismatch writing bits for", typ.string(), "x", dataSize/typ.size) @@ -2013,7 +1908,7 @@ func getgcmask(ep interface{}) (mask []byte) { if hbits.isPointer() { mask[i/sys.PtrSize] = 1 } - if i != 1*sys.PtrSize && !hbits.morePointers() { + if !hbits.morePointers() { mask = mask[:i/sys.PtrSize] break } diff --git a/src/runtime/mcheckmark.go b/src/runtime/mcheckmark.go new file mode 100644 index 0000000000..1fd8e4e78f --- /dev/null +++ b/src/runtime/mcheckmark.go @@ -0,0 +1,100 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// GC checkmarks +// +// In a concurrent garbage collector, one worries about failing to mark +// a live object due to mutations without write barriers or bugs in the +// collector implementation. As a sanity check, the GC has a 'checkmark' +// mode that retraverses the object graph with the world stopped, to make +// sure that everything that should be marked is marked. + +package runtime + +import ( + "runtime/internal/atomic" + "runtime/internal/sys" + "unsafe" +) + +// A checkmarksMap stores the GC marks in "checkmarks" mode. It is a +// per-arena bitmap with a bit for every word in the arena. The mark +// is stored on the bit corresponding to the first word of the marked +// allocation. +// +//go:notinheap +type checkmarksMap [heapArenaBytes / sys.PtrSize / 8]uint8 + +// If useCheckmark is true, marking of an object uses the checkmark +// bits instead of the standard mark bits. +var useCheckmark = false + +// startCheckmarks prepares for the checkmarks phase. +// +// The world must be stopped. +func startCheckmarks() { + // Clear all checkmarks. + for _, ai := range mheap_.allArenas { + arena := mheap_.arenas[ai.l1()][ai.l2()] + bitmap := arena.checkmarks + + if bitmap == nil { + // Allocate bitmap on first use. + bitmap = (*checkmarksMap)(persistentalloc(unsafe.Sizeof(*bitmap), 0, &memstats.gc_sys)) + if bitmap == nil { + throw("out of memory allocating checkmarks bitmap") + } + arena.checkmarks = bitmap + } else { + // Otherwise clear the existing bitmap. + for i := range bitmap { + bitmap[i] = 0 + } + } + } + // Enable checkmarking. + useCheckmark = true +} + +// endCheckmarks ends the checkmarks phase. +func endCheckmarks() { + if gcMarkWorkAvailable(nil) { + throw("GC work not flushed") + } + useCheckmark = false +} + +// setCheckmark throws if marking object is a checkmarks violation, +// and otherwise sets obj's checkmark. It returns true if obj was +// already checkmarked. +func setCheckmark(obj, base, off uintptr, mbits markBits) bool { + if !mbits.isMarked() { + printlock() + print("runtime: checkmarks found unexpected unmarked object obj=", hex(obj), "\n") + print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n") + + // Dump the source (base) object + gcDumpObject("base", base, off) + + // Dump the object + gcDumpObject("obj", obj, ^uintptr(0)) + + getg().m.traceback = 2 + throw("checkmark found unmarked object") + } + + ai := arenaIndex(obj) + arena := mheap_.arenas[ai.l1()][ai.l2()] + arenaWord := (obj / heapArenaBytes / 8) % uintptr(len(arena.checkmarks)) + mask := byte(1 << ((obj / heapArenaBytes) % 8)) + bytep := &arena.checkmarks[arenaWord] + + if atomic.Load8(bytep)&mask != 0 { + // Already checkmarked. + return true + } + + atomic.Or8(bytep, mask) + return false +} diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index b3499516f6..c8c4a4c758 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1670,13 +1670,13 @@ func gcMarkTermination(nextTriggerRatio float64) { // mark using checkmark bits, to check that we // didn't forget to mark anything during the // concurrent mark process. + startCheckmarks() gcResetMarkState() - initCheckmarks() gcw := &getg().m.p.ptr().gcw gcDrain(gcw, 0) wbBufFlush1(getg().m.p.ptr()) gcw.dispose() - clearCheckmarks() + endCheckmarks() } // marking is complete so we can turn the write barrier off diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index fe988c46d9..96910ff729 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1354,11 +1354,7 @@ func scanobject(b uintptr, gcw *gcWork) { } // Load bits once. See CL 22712 and issue 16973 for discussion. bits := hbits.bits() - // During checkmarking, 1-word objects store the checkmark - // in the type bit for the one word. The only one-word objects - // are pointers, or else they'd be merged with other non-pointer - // data into larger allocations. - if i != 1*sys.PtrSize && bits&bitScan == 0 { + if bits&bitScan == 0 { break // no more pointers in this object } if bits&bitPointer == 0 { @@ -1511,28 +1507,10 @@ func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintp mbits := span.markBitsForIndex(objIndex) if useCheckmark { - if !mbits.isMarked() { - printlock() - print("runtime:greyobject: checkmarks finds unexpected unmarked object obj=", hex(obj), "\n") - print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n") - - // Dump the source (base) object - gcDumpObject("base", base, off) - - // Dump the object - gcDumpObject("obj", obj, ^uintptr(0)) - - getg().m.traceback = 2 - throw("checkmark found unmarked object") - } - hbits := heapBitsForAddr(obj) - if hbits.isCheckmarked(span.elemsize) { + if setCheckmark(obj, base, off, mbits) { + // Already marked. return } - hbits.setCheckmarked(span.elemsize) - if !hbits.isCheckmarked(span.elemsize) { - throw("setCheckmarked and isCheckmarked disagree") - } } else { if debug.gccheckmark > 0 && span.isFree(objIndex) { print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n") @@ -1661,45 +1639,3 @@ func gcMarkTinyAllocs() { greyobject(c.tiny, 0, 0, span, gcw, objIndex) } } - -// Checkmarking - -// To help debug the concurrent GC we remark with the world -// stopped ensuring that any object encountered has their normal -// mark bit set. To do this we use an orthogonal bit -// pattern to indicate the object is marked. The following pattern -// uses the upper two bits in the object's boundary nibble. -// 01: scalar not marked -// 10: pointer not marked -// 11: pointer marked -// 00: scalar marked -// Xoring with 01 will flip the pattern from marked to unmarked and vica versa. -// The higher bit is 1 for pointers and 0 for scalars, whether the object -// is marked or not. -// The first nibble no longer holds the typeDead pattern indicating that the -// there are no more pointers in the object. This information is held -// in the second nibble. - -// If useCheckmark is true, marking of an object uses the -// checkmark bits (encoding above) instead of the standard -// mark bits. -var useCheckmark = false - -//go:nowritebarrier -func initCheckmarks() { - useCheckmark = true - for _, s := range mheap_.allspans { - if s.state.get() == mSpanInUse { - heapBitsForAddr(s.base()).initCheckmarkSpan(s.layout()) - } - } -} - -func clearCheckmarks() { - useCheckmark = false - for _, s := range mheap_.allspans { - if s.state.get() == mSpanInUse { - heapBitsForAddr(s.base()).clearCheckmarkSpan(s.layout()) - } - } -} diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 2c7bfd8a59..6341375160 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -300,6 +300,10 @@ type heapArena struct { // during marking. pageSpecials [pagesPerArena / 8]uint8 + // checkmarks stores the debug.gccheckmark state. It is only + // used if debug.gccheckmark > 0. + checkmarks *checkmarksMap + // zeroedBase marks the first byte of the first page in this // arena which hasn't been used yet and is therefore already // zero. zeroedBase is relative to the arena base. -- cgit v1.3-5-g9baa From a22ec6e650669f5101c7e0955d82e29d644eef4e Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 17 Aug 2020 13:17:26 +0000 Subject: Revert "cmd/internal/obj: fix inline marker issue on s390x" This reverts CL 247697. Reason for revert: This change broke the linux-arm builder. Change-Id: I8ca0d5b3b2ea0109ffbfadeab1406a1b60e7d18d Reviewed-on: https://go-review.googlesource.com/c/go/+/248718 Reviewed-by: Michael Munday Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot --- src/cmd/internal/obj/pcln.go | 15 --------------- src/cmd/internal/obj/s390x/objz.go | 11 +++++++++++ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index bffeda041d..1f7ccf47ef 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -278,21 +278,6 @@ func linkpcln(ctxt *Link, cursym *LSym) { funcpctab(ctxt, &pcln.Pcfile, cursym, "pctofile", pctofileline, pcln) funcpctab(ctxt, &pcln.Pcline, cursym, "pctoline", pctofileline, nil) - // Check that all the Progs used as inline markers are still reachable. - // See issue #40473. - inlMarkProgs := make(map[*Prog]struct{}, len(cursym.Func.InlMarks)) - for _, inlMark := range cursym.Func.InlMarks { - inlMarkProgs[inlMark.p] = struct{}{} - } - for p := cursym.Func.Text; p != nil; p = p.Link { - if _, ok := inlMarkProgs[p]; ok { - delete(inlMarkProgs, p) - } - } - if len(inlMarkProgs) > 0 { - ctxt.Diag("one or more instructions used as inline markers are no longer reachable") - } - pcinlineState := new(pcinlineState) funcpctab(ctxt, &pcln.Pcinline, cursym, "pctoinline", pcinlineState.pctoinline, nil) for _, inlMark := range cursym.Func.InlMarks { diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index ef6335d849..b14dc810fa 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -283,6 +283,17 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ACMPUBNE: q = p p.Mark |= BRANCH + if p.Pcond != nil { + q := p.Pcond + for q.As == obj.ANOP { + q = q.Link + p.Pcond = q + } + } + + case obj.ANOP: + q.Link = p.Link /* q is non-nop */ + p.Link.Mark |= p.Mark default: q = p -- cgit v1.3-5-g9baa From 7ee26224436d80dca3f7e98c8fcf21185522d8e6 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Wed, 12 Aug 2020 20:27:57 -0400 Subject: cmd/link: link dynamic library automatically cgo_import_dynamic pragma indicates a symbol is imported from a dynamic library. Currently, the linker does not actually link against the dynamic library, so we have to "force" it by using //go:cgo_import_dynamic _ _ "dylib" syntax, which links in the library unconditionally. This CL changes it to link in the library automatically when a symbol is imported from the library, without using the "force" syntax. (The "force" syntax is still supported.) Remove the unconditional imports in the runtime. Now, Security.framework and CoreFoundation.framework are only linked when the x509 package is imported (or otherwise specified). Fixes #40727. Change-Id: Ied36b1f621cdcc5dc4a8f497cdf1c554a182d0e0 Reviewed-on: https://go-review.googlesource.com/c/go/+/248333 Run-TryBot: Cherry Zhang Reviewed-by: Filippo Valsorda Reviewed-by: Than McIntosh Reviewed-by: Keith Randall TryBot-Result: Gobot Gobot --- src/cmd/link/internal/ld/go.go | 3 +++ src/runtime/sys_darwin.go | 6 ------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/cmd/link/internal/ld/go.go b/src/cmd/link/internal/ld/go.go index bf5c9ca1ba..b3541c46c0 100644 --- a/src/cmd/link/internal/ld/go.go +++ b/src/cmd/link/internal/ld/go.go @@ -183,6 +183,9 @@ func setCgoAttr(ctxt *Link, lookup func(string, int) loader.Sym, file string, pk hostObjSyms[s] = struct{}{} } havedynamic = 1 + if lib != "" && ctxt.IsDarwin() { + machoadddynlib(lib, ctxt.LinkMode) + } } continue diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 06474434c9..e4f19bbf41 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -489,9 +489,3 @@ func setNonblock(fd int32) { //go:cgo_import_dynamic libc_pthread_cond_wait pthread_cond_wait "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_timedwait_relative_np pthread_cond_timedwait_relative_np "/usr/lib/libSystem.B.dylib" //go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" - -// Magic incantation to get libSystem and friends actually dynamically linked. -// TODO: Why does the code require this? See cmd/link/internal/ld/go.go -//go:cgo_import_dynamic _ _ "/usr/lib/libSystem.B.dylib" -//go:cgo_import_dynamic _ _ "/System/Library/Frameworks/Security.framework/Versions/A/Security" -//go:cgo_import_dynamic _ _ "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation" -- cgit v1.3-5-g9baa From a2a2237ae02016dd9ce16388963cfceece6744f3 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Thu, 13 Aug 2020 16:59:52 -0400 Subject: cmd/link: emit correct jump instruction on ARM for DYNIMPORT On ARM, for a JMP/CALL relocation, the instruction bytes is encoded in Reloc.Add (issue #19811). I really hate it, but before it is fixed we have to follow the rule and emit the right bits from r.Add. Fixes #40769. Change-Id: I862e105408d344c5cc58ca9140d2e552e4364453 Reviewed-on: https://go-review.googlesource.com/c/go/+/248399 Reviewed-by: Jeremy Faller Reviewed-by: Joel Sing Reviewed-by: Than McIntosh --- src/cmd/link/internal/arm/asm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/link/internal/arm/asm.go b/src/cmd/link/internal/arm/asm.go index 22bcb518df..611c96ce35 100644 --- a/src/cmd/link/internal/arm/asm.go +++ b/src/cmd/link/internal/arm/asm.go @@ -220,7 +220,7 @@ func adddynrel(target *ld.Target, ldr *loader.Loader, syms *ld.ArchSyms, s loade addpltsym(target, ldr, syms, targ) su := ldr.MakeSymbolUpdater(s) su.SetRelocSym(rIdx, syms.PLT) - su.SetRelocAdd(rIdx, int64(ldr.SymPlt(targ))) + su.SetRelocAdd(rIdx, int64(braddoff(int32(r.Add()), ldr.SymPlt(targ)/4))) // TODO: don't use r.Add for instruction bytes (issue 19811) return true case objabi.R_ADDR: -- cgit v1.3-5-g9baa From abfeec5eb0356d1ac91a097d2124a6b7c8cfccd4 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 16 Aug 2020 21:58:40 +0000 Subject: testing/iotest: add ErrReader Adds an io.Reader that always returns 0 and a non-nil error. Fixes #38781 Change-Id: I56bd124de07bc8809e77c6cfaab33a1e32cfe2ee GitHub-Last-Rev: 4e232b17e9120405d4ea4743350ee361a3505043 GitHub-Pull-Request: golang/go#34741 Reviewed-on: https://go-review.googlesource.com/c/go/+/199501 Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot Reviewed-by: Emmanuel Odeke --- src/testing/iotest/logger_test.go | 12 ++---------- src/testing/iotest/reader.go | 15 +++++++++++++++ src/testing/iotest/reader_test.go | 10 ++++++++++ 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/testing/iotest/logger_test.go b/src/testing/iotest/logger_test.go index c121bf48f7..575f37e05c 100644 --- a/src/testing/iotest/logger_test.go +++ b/src/testing/iotest/logger_test.go @@ -81,14 +81,6 @@ func TestWriteLogger_errorOnWrite(t *testing.T) { } } -type errReader struct { - err error -} - -func (r errReader) Read([]byte) (int, error) { - return 0, r.err -} - func TestReadLogger(t *testing.T) { olw := log.Writer() olf := log.Flags() @@ -146,14 +138,14 @@ func TestReadLogger_errorOnRead(t *testing.T) { data := []byte("Hello, World!") p := make([]byte, len(data)) - lr := errReader{err: errors.New("Read Error!")} + lr := ErrReader() rl := NewReadLogger("read", lr) n, err := rl.Read(p) if err == nil { t.Fatalf("Unexpectedly succeeded to read: %v", err) } - wantLogWithHex := fmt.Sprintf("lr: read %x: %v\n", p[:n], "Read Error!") + wantLogWithHex := fmt.Sprintf("lr: read %x: %v\n", p[:n], "io") if g, w := lOut.String(), wantLogWithHex; g != w { t.Errorf("ReadLogger mismatch\n\tgot: %q\n\twant: %q", g, w) } diff --git a/src/testing/iotest/reader.go b/src/testing/iotest/reader.go index 8d82018fd6..b18e912f27 100644 --- a/src/testing/iotest/reader.go +++ b/src/testing/iotest/reader.go @@ -68,6 +68,7 @@ func (r *dataErrReader) Read(p []byte) (n int, err error) { return } +// ErrTimeout is a fake timeout error. var ErrTimeout = errors.New("timeout") // TimeoutReader returns ErrTimeout on the second read @@ -86,3 +87,17 @@ func (r *timeoutReader) Read(p []byte) (int, error) { } return r.r.Read(p) } + +// ErrIO is a fake IO error. +var ErrIO = errors.New("io") + +// ErrReader returns a fake error every time it is read from. +func ErrReader() io.Reader { + return errReader(0) +} + +type errReader int + +func (r errReader) Read(p []byte) (int, error) { + return 0, ErrIO +} diff --git a/src/testing/iotest/reader_test.go b/src/testing/iotest/reader_test.go index 9397837e08..ccba22ee29 100644 --- a/src/testing/iotest/reader_test.go +++ b/src/testing/iotest/reader_test.go @@ -224,3 +224,13 @@ func TestDataErrReader_emptyReader(t *testing.T) { t.Errorf("Unexpectedly read %d bytes, wanted %d", g, w) } } + +func TestErrReader(t *testing.T) { + n, err := ErrReader().Read([]byte{}) + if err != ErrIO { + t.Errorf("ErrReader.Read(any) should have returned ErrIO, returned %v", err) + } + if n != 0 { + t.Errorf("ErrReader.Read(any) should have read 0 bytes, read %v", n) + } +} -- cgit v1.3-5-g9baa From 49003da6d437ef1a4e1e55cf86240480f17dc8ab Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 17:40:35 -0400 Subject: cmd/go/internal/trace: add function to distinguish goroutines trace.StartGoroutine will associate the trace information on the context with a new chrome profiler thread id. The chrome profiler doesn't expect multiple trace events to have the same thread id, so this will allow us to display concurrent events on the trace. Updates #38714 Change-Id: I888b0cce15a5a01db66366716fdd85bf86c832cd Reviewed-on: https://go-review.googlesource.com/c/go/+/248319 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/trace/trace.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index c8fac92c9f..32ce0408f5 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -34,20 +34,33 @@ func StartSpan(ctx context.Context, name string) (context.Context, *Span) { if !ok { return ctx, nil } - childSpan := &Span{t: tc.t, name: name, start: time.Now()} + childSpan := &Span{t: tc.t, name: name, tid: tc.tid, start: time.Now()} tc.t.writeEvent(&traceviewer.Event{ Name: childSpan.name, Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), + TID: childSpan.tid, Phase: "B", }) - ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t}) + ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.tid}) return ctx, childSpan } +// Goroutine associates the context with a new Thread ID. The Chrome trace viewer associates each +// trace event with a thread, and doesn't expect events with the same thread id to happen at the +// same time. +func Goroutine(ctx context.Context) context.Context { + tc, ok := getTraceContext(ctx) + if !ok { + return ctx + } + return context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.t.getNextTID()}) +} + type Span struct { t *tracer name string + tid uint64 start time.Time end time.Time } @@ -60,12 +73,15 @@ func (s *Span) Done() { s.t.writeEvent(&traceviewer.Event{ Name: s.name, Time: float64(s.end.UnixNano()) / float64(time.Microsecond), + TID: s.tid, Phase: "E", }) } type tracer struct { file chan traceFile // 1-buffered + + nextTID uint64 } func (t *tracer) writeEvent(ev *traceviewer.Event) error { @@ -103,12 +119,17 @@ func (t *tracer) Close() error { return firstErr } +func (t *tracer) getNextTID() uint64 { + return atomic.AddUint64(&t.nextTID, 1) +} + // traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in // other packages. type traceKey struct{} type traceContext struct { - t *tracer + t *tracer + tid uint64 } // Start starts a trace which writes to the given file. -- cgit v1.3-5-g9baa From 023d4973851a25e2a47b1ebaf96833c9209efd7c Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 18:05:16 -0400 Subject: cmd/go: add trace events for each action This change adds a trace event for each action and also annotates each of the action execution goroutines with trace.Goroutine so that the actions eaxecuted by each goroutine appear on different threads in the chrome trace viewer. Updates #38714 Change-Id: I2e58dc5606b2e3f7f87076a61e1cc6a2014255c5 Reviewed-on: https://go-review.googlesource.com/c/go/+/248320 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/trace/trace.go | 4 ++-- src/cmd/go/internal/work/exec.go | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index 32ce0408f5..24130d9d72 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -45,10 +45,10 @@ func StartSpan(ctx context.Context, name string) (context.Context, *Span) { return ctx, childSpan } -// Goroutine associates the context with a new Thread ID. The Chrome trace viewer associates each +// StartGoroutine associates the context with a new Thread ID. The Chrome trace viewer associates each // trace event with a thread, and doesn't expect events with the same thread id to happen at the // same time. -func Goroutine(ctx context.Context) context.Context { +func StartGoroutine(ctx context.Context) context.Context { tc, ok := getTraceContext(ctx) if !ok { return ctx diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 3ea3293ae1..56a127f36f 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -115,13 +115,21 @@ func (b *Builder) Do(ctx context.Context, root *Action) { // Handle runs a single action and takes care of triggering // any actions that are runnable as a result. - handle := func(a *Action) { + handle := func(ctx context.Context, a *Action) { if a.json != nil { a.json.TimeStart = time.Now() } var err error if a.Func != nil && (!a.Failed || a.IgnoreFail) { + // TODO(matloob): Better action descriptions + desc := "Executing action " + if a.Package != nil { + desc += "(" + a.Mode + " " + a.Package.Desc() + ")" + } + ctx, span := trace.StartSpan(ctx, desc) + _ = ctx err = a.Func(b, a) + span.Done() } if a.json != nil { a.json.TimeDone = time.Now() @@ -169,6 +177,7 @@ func (b *Builder) Do(ctx context.Context, root *Action) { for i := 0; i < par; i++ { wg.Add(1) go func() { + ctx := trace.StartGoroutine(ctx) defer wg.Done() for { select { @@ -181,7 +190,7 @@ func (b *Builder) Do(ctx context.Context, root *Action) { b.exec.Lock() a := b.ready.pop() b.exec.Unlock() - handle(a) + handle(ctx, a) case <-base.Interrupted: base.SetExitStatus(1) return -- cgit v1.3-5-g9baa From a26d687ebb23fa14b777ef5bf69b56556124ff3b Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 17 Jun 2020 18:18:23 -0400 Subject: cmd/go: propagate context into Action.Func calls Action.Func is now a func(*Builder, context.Context, *Action), so that contexts can be propagated into the action funcs. While context is traditionally the first parameter of a function, it's the second parameter of Action.Func's type to continue to allow for methods on Builder to be used as functions taking a *Builder as the first parameter. context.Context is instead the first parameter on those functions. Change-Id: I5f058d6a99a1e96fe2025f2e8ce30a033d12e935 Reviewed-on: https://go-review.googlesource.com/c/go/+/248321 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 10 +++++----- src/cmd/go/internal/work/action.go | 15 ++++++++------- src/cmd/go/internal/work/exec.go | 17 ++++++++--------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 3630f68c54..deec5106ff 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -146,7 +146,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { // buildRunProgram is the action for running a binary that has already // been compiled. We ignore exit status. -func buildRunProgram(b *work.Builder, a *work.Action) error { +func buildRunProgram(b *work.Builder, ctx context.Context, a *work.Action) error { cmdline := str.StringList(work.FindExecCmd(), a.Deps[0].Target, a.Args) if cfg.BuildN || cfg.BuildX { b.Showcmd("", "%s", strings.Join(cmdline, " ")) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index d71d339828..9788590938 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1069,7 +1069,7 @@ func (lockedStdout) Write(b []byte) (int, error) { } // builderRunTest is the action for running a test binary. -func (c *runCache) builderRunTest(b *work.Builder, a *work.Action) error { +func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work.Action) error { if a.Failed { // We were unable to build the binary. a.Failed = false @@ -1642,7 +1642,7 @@ func coveragePercentage(out []byte) string { } // builderCleanTest is the action for cleaning up after a test. -func builderCleanTest(b *work.Builder, a *work.Action) error { +func builderCleanTest(b *work.Builder, ctx context.Context, a *work.Action) error { if cfg.BuildWork { return nil } @@ -1654,7 +1654,7 @@ func builderCleanTest(b *work.Builder, a *work.Action) error { } // builderPrintTest is the action for printing a test result. -func builderPrintTest(b *work.Builder, a *work.Action) error { +func builderPrintTest(b *work.Builder, ctx context.Context, a *work.Action) error { clean := a.Deps[0] run := clean.Deps[0] if run.TestOutput != nil { @@ -1665,7 +1665,7 @@ func builderPrintTest(b *work.Builder, a *work.Action) error { } // builderNoTest is the action for testing a package with no test files. -func builderNoTest(b *work.Builder, a *work.Action) error { +func builderNoTest(b *work.Builder, ctx context.Context, a *work.Action) error { var stdout io.Writer = os.Stdout if testJSON { json := test2json.NewConverter(lockedStdout{}, a.Package.ImportPath, test2json.Timestamp) @@ -1677,7 +1677,7 @@ func builderNoTest(b *work.Builder, a *work.Action) error { } // printExitStatus is the action for printing the exit status -func printExitStatus(b *work.Builder, a *work.Action) error { +func printExitStatus(b *work.Builder, ctx context.Context, a *work.Action) error { if !testJSON && len(pkgArgs) != 0 { if base.GetExitStatus() != 0 { fmt.Println("FAIL") diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index 6b5f9e4807..a37a5e618d 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -10,6 +10,7 @@ import ( "bufio" "bytes" "container/heap" + "context" "debug/elf" "encoding/json" "fmt" @@ -63,13 +64,13 @@ type Builder struct { // An Action represents a single action in the action graph. type Action struct { - Mode string // description of action operation - Package *load.Package // the package this action works on - Deps []*Action // actions that must happen before this one - Func func(*Builder, *Action) error // the action itself (nil = no-op) - IgnoreFail bool // whether to run f even if dependencies fail - TestOutput *bytes.Buffer // test output buffer - Args []string // additional args for runProgram + Mode string // description of action operation + Package *load.Package // the package this action works on + Deps []*Action // actions that must happen before this one + Func func(*Builder, context.Context, *Action) error // the action itself (nil = no-op) + IgnoreFail bool // whether to run f even if dependencies fail + TestOutput *bytes.Buffer // test output buffer + Args []string // additional args for runProgram triggers []*Action // inverse of deps diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 56a127f36f..3903502a67 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -127,8 +127,7 @@ func (b *Builder) Do(ctx context.Context, root *Action) { desc += "(" + a.Mode + " " + a.Package.Desc() + ")" } ctx, span := trace.StartSpan(ctx, desc) - _ = ctx - err = a.Func(b, a) + err = a.Func(b, ctx, a) span.Done() } if a.json != nil { @@ -400,7 +399,7 @@ const ( // build is the action for building a single package. // Note that any new influence on this logic must be reported in b.buildActionID above as well. -func (b *Builder) build(a *Action) (err error) { +func (b *Builder) build(ctx context.Context, a *Action) (err error) { p := a.Package bit := func(x uint32, b bool) uint32 { @@ -1005,7 +1004,7 @@ var VetFlags []string // VetExplicit records whether the vet flags were set explicitly on the command line. var VetExplicit bool -func (b *Builder) vet(a *Action) error { +func (b *Builder) vet(ctx context.Context, a *Action) error { // a.Deps[0] is the build of the package being vetted. // a.Deps[1] is the build of the "fmt" package. @@ -1196,7 +1195,7 @@ func (b *Builder) printLinkerConfig(h io.Writer, p *load.Package) { // link is the action for linking a single command. // Note that any new influence on this logic must be reported in b.linkActionID above as well. -func (b *Builder) link(a *Action) (err error) { +func (b *Builder) link(ctx context.Context, a *Action) (err error) { if b.useCache(a, b.linkActionID(a), a.Package.Target) || b.IsCmdList { return nil } @@ -1388,7 +1387,7 @@ func (b *Builder) getPkgConfigFlags(p *load.Package) (cflags, ldflags []string, return } -func (b *Builder) installShlibname(a *Action) error { +func (b *Builder) installShlibname(ctx context.Context, a *Action) error { if err := allowInstall(a); err != nil { return err } @@ -1437,7 +1436,7 @@ func (b *Builder) linkSharedActionID(a *Action) cache.ActionID { return h.Sum() } -func (b *Builder) linkShared(a *Action) (err error) { +func (b *Builder) linkShared(ctx context.Context, a *Action) (err error) { if b.useCache(a, b.linkSharedActionID(a), a.Target) || b.IsCmdList { return nil } @@ -1463,7 +1462,7 @@ func (b *Builder) linkShared(a *Action) (err error) { } // BuildInstallFunc is the action for installing a single package or executable. -func BuildInstallFunc(b *Builder, a *Action) (err error) { +func BuildInstallFunc(b *Builder, ctx context.Context, a *Action) (err error) { defer func() { if err != nil && err != errPrintedOutput { // a.Package == nil is possible for the go install -buildmode=shared @@ -1716,7 +1715,7 @@ func (b *Builder) writeFile(file string, text []byte) error { } // Install the cgo export header file, if there is one. -func (b *Builder) installHeader(a *Action) error { +func (b *Builder) installHeader(ctx context.Context, a *Action) error { src := a.Objdir + "_cgo_install.h" if _, err := os.Stat(src); os.IsNotExist(err) { // If the file does not exist, there are no exported -- cgit v1.3-5-g9baa From 15b98e55d195bd876203506d5f513546dd4e3b36 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 18 Jun 2020 11:58:46 -0400 Subject: cmd/go: mark trace flows between actions This could help make it easier to identify blocking dependencies when examining traces. Flows can be turned off when viewing traces to remove potential distractions. Updates #38714 Change-Id: Ibfd3f1a1861e3cac31addb053a2fca7ee796c4d7 Reviewed-on: https://go-review.googlesource.com/c/go/+/248322 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/trace/trace.go | 51 +++++++++++++++++++++++++++++++++++--- src/cmd/go/internal/work/action.go | 10 +++++--- src/cmd/go/internal/work/exec.go | 4 +++ 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/cmd/go/internal/trace/trace.go b/src/cmd/go/internal/trace/trace.go index 24130d9d72..f108a2b6ca 100644 --- a/src/cmd/go/internal/trace/trace.go +++ b/src/cmd/go/internal/trace/trace.go @@ -15,6 +15,18 @@ import ( "time" ) +// Constants used in event fields. +// See https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU +// for more details. +const ( + phaseDurationBegin = "B" + phaseDurationEnd = "E" + phaseFlowStart = "s" + phaseFlowEnd = "f" + + bindEnclosingSlice = "e" +) + var traceStarted int32 func getTraceContext(ctx context.Context) (traceContext, bool) { @@ -39,7 +51,7 @@ func StartSpan(ctx context.Context, name string) (context.Context, *Span) { Name: childSpan.name, Time: float64(childSpan.start.UnixNano()) / float64(time.Microsecond), TID: childSpan.tid, - Phase: "B", + Phase: phaseDurationBegin, }) ctx = context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.tid}) return ctx, childSpan @@ -56,6 +68,34 @@ func StartGoroutine(ctx context.Context) context.Context { return context.WithValue(ctx, traceKey{}, traceContext{tc.t, tc.t.getNextTID()}) } +// Flow marks a flow indicating that the 'to' span depends on the 'from' span. +// Flow should be called while the 'to' span is in progress. +func Flow(ctx context.Context, from *Span, to *Span) { + tc, ok := getTraceContext(ctx) + if !ok || from == nil || to == nil { + return + } + + id := tc.t.getNextFlowID() + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", + ID: id, + Time: float64(from.end.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowStart, + TID: from.tid, + }) + tc.t.writeEvent(&traceviewer.Event{ + Name: from.name + " -> " + to.name, + Category: "flow", // TODO(matloob): Add Category to Flow? + ID: id, + Time: float64(to.start.UnixNano()) / float64(time.Microsecond), + Phase: phaseFlowEnd, + TID: to.tid, + BindPoint: bindEnclosingSlice, + }) +} + type Span struct { t *tracer @@ -74,14 +114,15 @@ func (s *Span) Done() { Name: s.name, Time: float64(s.end.UnixNano()) / float64(time.Microsecond), TID: s.tid, - Phase: "E", + Phase: phaseDurationEnd, }) } type tracer struct { file chan traceFile // 1-buffered - nextTID uint64 + nextTID uint64 + nextFlowID uint64 } func (t *tracer) writeEvent(ev *traceviewer.Event) error { @@ -123,6 +164,10 @@ func (t *tracer) getNextTID() uint64 { return atomic.AddUint64(&t.nextTID, 1) } +func (t *tracer) getNextFlowID() uint64 { + return atomic.AddUint64(&t.nextFlowID, 1) +} + // traceKey is the context key for tracing information. It is unexported to prevent collisions with context keys defined in // other packages. type traceKey struct{} diff --git a/src/cmd/go/internal/work/action.go b/src/cmd/go/internal/work/action.go index a37a5e618d..825e763c03 100644 --- a/src/cmd/go/internal/work/action.go +++ b/src/cmd/go/internal/work/action.go @@ -26,6 +26,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/trace" "cmd/internal/buildid" ) @@ -92,10 +93,11 @@ type Action struct { output []byte // output redirect buffer (nil means use b.Print) // Execution state. - pending int // number of deps yet to complete - priority int // relative execution priority - Failed bool // whether the action failed - json *actionJSON // action graph information + pending int // number of deps yet to complete + priority int // relative execution priority + Failed bool // whether the action failed + json *actionJSON // action graph information + traceSpan *trace.Span } // BuildActionID returns the action ID section of a's build ID. diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 3903502a67..681ecd7646 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -127,6 +127,10 @@ func (b *Builder) Do(ctx context.Context, root *Action) { desc += "(" + a.Mode + " " + a.Package.Desc() + ")" } ctx, span := trace.StartSpan(ctx, desc) + a.traceSpan = span + for _, d := range a.Deps { + trace.Flow(ctx, d.traceSpan, a.traceSpan) + } err = a.Func(b, ctx, a) span.Done() } -- cgit v1.3-5-g9baa From 38fea3a4ec97fbcfad1f2d329f3a12c53cc36301 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 22 Jun 2020 18:39:10 -0400 Subject: cmd/go: add tracing instrumentation to load.TestPackagesFor This change adds tracing instrumentation into load.TestPackagesFor, propagating context through its callers. Updates #38714 Change-Id: I80fefaf3116ccccffaa8bb7613a656bda867394c Reviewed-on: https://go-review.googlesource.com/c/go/+/248323 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/list/list.go | 4 ++-- src/cmd/go/internal/load/test.go | 14 ++++++++++---- src/cmd/go/internal/test/test.go | 6 +++--- src/cmd/go/internal/vet/vet.go | 2 +- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 3ec243a759..7747e730ae 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -477,9 +477,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { var pmain, ptest, pxtest *load.Package var err error if *listE { - pmain, ptest, pxtest = load.TestPackagesAndErrors(p, nil) + pmain, ptest, pxtest = load.TestPackagesAndErrors(ctx, p, nil) } else { - pmain, ptest, pxtest, err = load.TestPackagesFor(p, nil) + pmain, ptest, pxtest, err = load.TestPackagesFor(ctx, p, nil) if err != nil { base.Errorf("can't load test package: %s", err) } diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 6d251e8358..6db8a00245 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -6,7 +6,7 @@ package load import ( "bytes" - "cmd/go/internal/str" + "context" "errors" "fmt" "go/ast" @@ -20,6 +20,9 @@ import ( "strings" "unicode" "unicode/utf8" + + "cmd/go/internal/str" + "cmd/go/internal/trace" ) var TestMainDeps = []string{ @@ -42,8 +45,8 @@ type TestCover struct { // TestPackagesFor is like TestPackagesAndErrors but it returns // an error if the test packages or their dependencies have errors. // Only test packages without errors are returned. -func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { - pmain, ptest, pxtest = TestPackagesAndErrors(p, cover) +func TestPackagesFor(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package, err error) { + pmain, ptest, pxtest = TestPackagesAndErrors(ctx, p, cover) for _, p1 := range []*Package{ptest, pxtest, pmain} { if p1 == nil { // pxtest may be nil @@ -89,7 +92,10 @@ func TestPackagesFor(p *Package, cover *TestCover) (pmain, ptest, pxtest *Packag // // The caller is expected to have checked that len(p.TestGoFiles)+len(p.XTestGoFiles) > 0, // or else there's no point in any of this. -func TestPackagesAndErrors(p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { +func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (pmain, ptest, pxtest *Package) { + ctx, span := trace.StartSpan(ctx, "load.TestPackagesAndErrors") + defer span.Done() + pre := newPreload() defer pre.flush() allImports := append([]string{}, p.TestImports...) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9788590938..cda51053fb 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -746,7 +746,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { ensureImport(p, "sync/atomic") } - buildTest, runTest, printTest, err := builderTest(&b, p) + buildTest, runTest, printTest, err := builderTest(&b, ctx, p) if err != nil { str := err.Error() str = strings.TrimPrefix(str, "\n") @@ -813,7 +813,7 @@ var windowsBadWords = []string{ "update", } -func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { +func builderTest(b *work.Builder, ctx context.Context, p *load.Package) (buildAction, runAction, printAction *work.Action, err error) { if len(p.TestGoFiles)+len(p.XTestGoFiles) == 0 { build := b.CompileAction(work.ModeBuild, work.ModeBuild, p) run := &work.Action{Mode: "test run", Package: p, Deps: []*work.Action{build}} @@ -836,7 +836,7 @@ func builderTest(b *work.Builder, p *load.Package) (buildAction, runAction, prin DeclVars: declareCoverVars, } } - pmain, ptest, pxtest, err := load.TestPackagesFor(p, cover) + pmain, ptest, pxtest, err := load.TestPackagesFor(ctx, p, cover) if err != nil { return nil, nil, nil, err } diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index 58f392eb96..b306572281 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -77,7 +77,7 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { root := &work.Action{Mode: "go vet"} for _, p := range pkgs { - _, ptest, pxtest, err := load.TestPackagesFor(p, nil) + _, ptest, pxtest, err := load.TestPackagesFor(ctx, p, nil) if err != nil { base.Errorf("%v", err) continue -- cgit v1.3-5-g9baa From ebccba7954fe9507df993dda7ba78fa34e030390 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Mon, 22 Jun 2020 19:02:00 -0400 Subject: cmd/go: process -debug-trace flag for cmd/test and cmd/vet These commands are build-like commands that do their own flag processing, so the value of debug-trace isn't available until the command starts running. Start tracing in the cmd's run function. Updates #38714 Change-Id: I4d633e6ee907bf09feac52c2aff3daceb9b20e12 Reviewed-on: https://go-review.googlesource.com/c/go/+/248324 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/test/test.go | 18 ++++++++++++++++++ src/cmd/go/internal/vet/vet.go | 25 +++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index cda51053fb..9c120e08dc 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -31,6 +31,7 @@ import ( "cmd/go/internal/lockedfile" "cmd/go/internal/modload" "cmd/go/internal/str" + "cmd/go/internal/trace" "cmd/go/internal/work" "cmd/internal/test2json" ) @@ -571,6 +572,23 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { pkgArgs, testArgs = testFlags(args) + if cfg.DebugTrace != "" { + var close func() error + var err error + ctx, close, err = trace.Start(ctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + defer func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }() + } + + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + defer span.Done() + work.FindExecCmd() // initialize cached result work.BuildInit() diff --git a/src/cmd/go/internal/vet/vet.go b/src/cmd/go/internal/vet/vet.go index b306572281..cf2c8d59e8 100644 --- a/src/cmd/go/internal/vet/vet.go +++ b/src/cmd/go/internal/vet/vet.go @@ -6,12 +6,16 @@ package vet import ( + "context" + "fmt" + "path/filepath" + "cmd/go/internal/base" + "cmd/go/internal/cfg" "cmd/go/internal/load" "cmd/go/internal/modload" + "cmd/go/internal/trace" "cmd/go/internal/work" - "context" - "path/filepath" ) // Break init loop. @@ -54,6 +58,23 @@ func runVet(ctx context.Context, cmd *base.Command, args []string) { vetFlags, pkgArgs := vetFlags(args) + if cfg.DebugTrace != "" { + var close func() error + var err error + ctx, close, err = trace.Start(ctx, cfg.DebugTrace) + if err != nil { + base.Fatalf("failed to start trace: %v", err) + } + defer func() { + if err := close(); err != nil { + base.Fatalf("failed to stop trace: %v", err) + } + }() + } + + ctx, span := trace.StartSpan(ctx, fmt.Sprint("Running ", cmd.Name(), " command")) + defer span.Done() + work.BuildInit() work.VetFlags = vetFlags if len(vetFlags) > 0 { -- cgit v1.3-5-g9baa From 2ac4bf3802f0786a0afb09488173507f40d5d885 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 24 Jun 2020 15:58:47 -0400 Subject: cmd/go: add span for modload.LoadBuildList This change adds context, and a span to modload.LoadBuildList and propagates context into modload.BuildList. It's the start of a run of CLs to add trace spans for module operations. Updates #38714 Change-Id: I0d58dd394051526338092dc9a5ec29a9e087e4e4 Reviewed-on: https://go-review.googlesource.com/c/go/+/248325 Run-TryBot: Michael Matloob Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/list/list.go | 4 ++-- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modcmd/graph.go | 2 +- src/cmd/go/internal/modcmd/verify.go | 2 +- src/cmd/go/internal/modcmd/why.go | 2 +- src/cmd/go/internal/modget/get.go | 8 ++++---- src/cmd/go/internal/modload/list.go | 9 +++++---- src/cmd/go/internal/modload/load.go | 23 ++++++++++++++--------- 8 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7747e730ae..7303e6c866 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -413,9 +413,9 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { } } - modload.LoadBuildList() + modload.LoadBuildList(ctx) - mods := modload.ListModules(args, *listU, *listVersions) + mods := modload.ListModules(ctx, args, *listU, *listVersions) if !*listE { for _, m := range mods { if m.Error != nil { diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index b43c32be5a..946e8ed3cf 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -106,7 +106,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { var work par.Work listU := false listVersions := false - for _, info := range modload.ListModules(args, listU, listVersions) { + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { if info.Replace != nil { info = info.Replace } diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index fff5b02626..4853503fd4 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -49,7 +49,7 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go: cannot find main module; see 'go help modules'") } } - modload.LoadBuildList() + modload.LoadBuildList(ctx) reqs := modload.MinReqs() format := func(m module.Version) string { diff --git a/src/cmd/go/internal/modcmd/verify.go b/src/cmd/go/internal/modcmd/verify.go index 570e571049..73ab714d10 100644 --- a/src/cmd/go/internal/modcmd/verify.go +++ b/src/cmd/go/internal/modcmd/verify.go @@ -60,7 +60,7 @@ func runVerify(ctx context.Context, cmd *base.Command, args []string) { sem := make(chan token, runtime.GOMAXPROCS(0)) // Use a slice of result channels, so that the output is deterministic. - mods := modload.LoadBuildList()[1:] + mods := modload.LoadBuildList(ctx)[1:] errsChans := make([]<-chan []error, len(mods)) for i, mod := range mods { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index 3f9cf0f120..f400339b25 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -74,7 +74,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod why: module query not allowed") } } - mods := modload.ListModules(args, listU, listVersions) + mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) for _, path := range loadALL() { m := modload.PackageModule(path) diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index b217196931..93a6bb54d5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -278,7 +278,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { } modload.LoadTests = *getT - buildList := modload.LoadBuildList() + buildList := modload.LoadBuildList(ctx) buildList = buildList[:len(buildList):len(buildList)] // copy on append versionByPath := make(map[string]string) for _, m := range buildList { @@ -444,7 +444,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // packages in unknown modules can't be expanded. This also avoids looking // up new modules while loading packages, only to downgrade later. queryCache := make(map[querySpec]*query) - byPath := runQueries(queryCache, queries, nil) + byPath := runQueries(ctx, queryCache, queries, nil) // Add missing modules to the build list. // We call SetBuildList here and elsewhere, since newUpgrader, @@ -586,7 +586,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Query target versions for modules providing packages matched by // command line arguments. - byPath = runQueries(queryCache, queries, modOnly) + byPath = runQueries(ctx, queryCache, queries, modOnly) // Handle upgrades. This is needed for arguments that didn't match // modules or matched different modules from a previous iteration. It @@ -724,7 +724,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // versions (including earlier queries in the modOnly map), an error will be // reported. A map from module paths to queries is returned, which includes // queries and modOnly. -func runQueries(cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { +func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { var lookup par.Work for _, q := range queries { if cached := cache[q.querySpec]; cached != nil { diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 9400793bcb..4768516e90 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -19,8 +20,8 @@ import ( "golang.org/x/mod/module" ) -func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePublic { - mods := listModules(args, listVersions) +func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { + mods := listModules(ctx, args, listVersions) if listU || listVersions { var work par.Work for _, m := range mods { @@ -42,8 +43,8 @@ func ListModules(args []string, listU, listVersions bool) []*modinfo.ModulePubli return mods } -func listModules(args []string, listVersions bool) []*modinfo.ModulePublic { - LoadBuildList() +func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { + LoadBuildList(ctx) if len(args) == 0 { return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} } diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 30992e0cc2..8190009b23 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -6,14 +6,7 @@ package modload import ( "bytes" - "cmd/go/internal/base" - "cmd/go/internal/cfg" - "cmd/go/internal/imports" - "cmd/go/internal/modfetch" - "cmd/go/internal/mvs" - "cmd/go/internal/par" - "cmd/go/internal/search" - "cmd/go/internal/str" + "context" "errors" "fmt" "go/build" @@ -24,6 +17,16 @@ import ( "sort" "strings" + "cmd/go/internal/base" + "cmd/go/internal/cfg" + "cmd/go/internal/imports" + "cmd/go/internal/modfetch" + "cmd/go/internal/mvs" + "cmd/go/internal/par" + "cmd/go/internal/search" + "cmd/go/internal/str" + "cmd/go/internal/trace" + "golang.org/x/mod/module" ) @@ -385,7 +388,9 @@ func DirImportPath(dir string) string { // LoadBuildList need only be called if ImportPaths is not // (typically in commands that care about the module but // no particular package). -func LoadBuildList() []module.Version { +func LoadBuildList(ctx context.Context) []module.Version { + ctx, span := trace.StartSpan(ctx, "LoadBuildList") + defer span.Done() InitMod() ReloadBuildList() WriteGoMod() -- cgit v1.3-5-g9baa From f30044a03bc7cf107dbec03c02fb6d0072878252 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 25 Jun 2020 19:11:28 -0400 Subject: cmd/go/internal: remove some users of par.Work par.Work is used in a number of places as a parallel work queue. This change replaces it with goroutines and channels in a number of simpler places where it's used. Change-Id: I0620eda46ec7b2c0599a8b9361639af7bb73a05a Reviewed-on: https://go-review.googlesource.com/c/go/+/248326 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/modcmd/download.go | 69 +++++++++++++++++++--------------- src/cmd/go/internal/modcmd/graph.go | 19 +++++----- src/cmd/go/internal/modconv/convert.go | 59 ++++++++++++++++------------- src/cmd/go/internal/modget/get.go | 41 +++++++++++++------- src/cmd/go/internal/modload/list.go | 37 +++++++++++------- 5 files changed, 133 insertions(+), 92 deletions(-) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 946e8ed3cf..857362a72e 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -5,15 +5,15 @@ package modcmd import ( + "cmd/go/internal/modfetch" "context" "encoding/json" "os" + "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -102,33 +102,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } } - var mods []*moduleJSON - var work par.Work - listU := false - listVersions := false - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - work.Add(m) - } - - work.Do(10, func(item interface{}) { - m := item.(*moduleJSON) + downloadModule := func(m *moduleJSON) { var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -157,7 +131,42 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { m.Error = err.Error() return } - }) + } + + var mods []*moduleJSON + listU := false + listVersions := false + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + sem <- token{} + go func() { + downloadModule(m) + <-sem + }() + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } if *downloadJSON { for _, m := range mods { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 4853503fd4..6da12b9cab 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,7 +15,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -59,23 +58,25 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { return m.Path + "@" + m.Version } - // Note: using par.Work only to manage work queue. - // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - var work par.Work - work.Add(modload.Target) - work.Do(1, func(item interface{}) { - m := item.(module.Version) + seen := map[module.Version]bool{modload.Target: true} + queue := []module.Version{modload.Target} + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] list, _ := reqs.Required(m) for _, r := range list { - work.Add(r) + if !seen[r] { + queue = append(queue, r) + seen[r] = true + } out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - }) + } sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/src/cmd/go/internal/modconv/convert.go b/src/cmd/go/internal/modconv/convert.go index f465a9f395..d5a0bc21e9 100644 --- a/src/cmd/go/internal/modconv/convert.go +++ b/src/cmd/go/internal/modconv/convert.go @@ -7,13 +7,12 @@ package modconv import ( "fmt" "os" + "runtime" "sort" "strings" - "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" - "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -42,46 +41,52 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var ( - work par.Work - mu sync.Mutex - need = make(map[string]string) - replace = make(map[string]*modfile.Replace) - ) + versions := make([]*module.Version, len(mf.Require)) + replace := make(map[string]*modfile.Replace) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - for _, r := range mf.Require { + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for i, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - work.Add(re.New) - continue + m = re.New } - work.Add(r.Mod) + sem <- token{} + go func(i int, m module.Version) { + repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) + return + } + + path := repo.ModulePath() + versions[i].Path = path + versions[i].Version = info.Version + + <-sem + }(i, m) + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} } - work.Do(10, func(item interface{}) { - r := item.(module.Version) - repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) - return - } - mu.Lock() - path := repo.ModulePath() + need := map[string]string{} + for _, v := range versions { // Don't use semver.Max here; need to preserve +incompatible suffix. - if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { - need[path] = info.Version + if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { + need[v.Path] = v.Version } - mu.Unlock() - }) - - var paths []string + } + paths := make([]string, 0, len(need)) for path := range need { paths = append(paths, path) } diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 93a6bb54d5..d02c9a8da5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -21,7 +22,6 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/mvs" - "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" @@ -725,18 +725,8 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // reported. A map from module paths to queries is returned, which includes // queries and modOnly. func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { - var lookup par.Work - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - cache[q.querySpec] = q - lookup.Add(q) - } - } - lookup.Do(10, func(item interface{}) { - q := item.(*query) + runQuery := func(q *query) { if q.vers == "none" { // Wait for downgrade step. q.m = module.Version{Path: q.path, Version: "none"} @@ -747,7 +737,32 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer base.Errorf("go get %s: %v", q.arg, err) } q.m = m - }) + } + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, q := range queries { + if cached := cache[q.querySpec]; cached != nil { + *q = *cached + } else { + sem <- token{} + go func(q *query) { + runQuery(q) + <-sem + }(q) + } + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + // Add to cache after concurrent section to avoid races... + for _, q := range queries { + cache[q.querySpec] = q + } + base.ExitIfErrors() byPath := make(map[string]*query) diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 4768516e90..8db4d64706 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "os" + "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" - "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" @@ -22,24 +22,35 @@ import ( func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { mods := listModules(ctx, args, listVersions) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) if listU || listVersions { - var work par.Work for _, m := range mods { - work.Add(m) + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(m) + } + if listVersions { + addVersions(m) + } + <-sem + }() + } + + add(m) if m.Replace != nil { - work.Add(m.Replace) + add(m.Replace) } } - work.Do(10, func(item interface{}) { - m := item.(*modinfo.ModulePublic) - if listU { - addUpdate(m) - } - if listVersions { - addVersions(m) - } - }) } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + return mods } -- cgit v1.3-5-g9baa From 1b86bdbdc3991c13c6ed156100a5f4918fdd9c6b Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 14 Aug 2020 17:44:22 -0400 Subject: cmd/test2json: do not emit a final Action if the result is not known MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we are parsing a test output, and the test does not end in the usual PASS or FAIL line (say, because it panicked), then we need the exit status of the test binary in order to determine whether the test passed or failed. If we don't have that status available, we shouldn't guess arbitrarily — instead, we should omit the final "pass" or "fail" action entirely. (In practice, we nearly always DO have the final status, such as when running 'go test' or 'go tool test2json some.exe'.) Fixes #40132 Change-Id: Iae482577361a6033395fe4a05d746b980e18c3de Reviewed-on: https://go-review.googlesource.com/c/go/+/248624 Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/test/test.go | 8 +- src/cmd/go/testdata/script/test_json_exit.txt | 102 +++++++++++++++++++++ src/cmd/internal/test2json/test2json.go | 44 +++++---- .../internal/test2json/testdata/benchshort.json | 1 - src/cmd/internal/test2json/testdata/empty.json | 1 - src/cmd/test2json/main.go | 6 +- 6 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 src/cmd/go/testdata/script/test_json_exit.txt diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9c120e08dc..9cef8cf89c 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -1098,9 +1098,13 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } var stdout io.Writer = os.Stdout + var err error if testJSON { json := test2json.NewConverter(lockedStdout{}, a.Package.ImportPath, test2json.Timestamp) - defer json.Close() + defer func() { + json.Exited(err) + json.Close() + }() stdout = json } @@ -1204,7 +1208,7 @@ func (c *runCache) builderRunTest(b *work.Builder, ctx context.Context, a *work. } t0 := time.Now() - err := cmd.Start() + err = cmd.Start() // This is a last-ditch deadline to detect and // stop wedged test binaries, to keep the builders diff --git a/src/cmd/go/testdata/script/test_json_exit.txt b/src/cmd/go/testdata/script/test_json_exit.txt new file mode 100644 index 0000000000..dc7ffb06cf --- /dev/null +++ b/src/cmd/go/testdata/script/test_json_exit.txt @@ -0,0 +1,102 @@ +[short] skip + +go test -c -o mainpanic.exe ./mainpanic & +go test -c -o mainexit0.exe ./mainexit0 & +go test -c -o testpanic.exe ./testpanic & +go test -c -o testbgpanic.exe ./testbgpanic & +wait + +# Test binaries that panic in TestMain should be marked as failing. + +! go test -json ./mainpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./mainpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +# Test binaries that exit with status 0 should be marked as passing. + +go test -json ./mainexit0 +stdout '"Action":"pass"' +! stdout '"Action":"fail"' + +go tool test2json ./mainexit0.exe +stdout '"Action":"pass"' +! stdout '"Action":"fail"' + +# Test functions that panic should never be marked as passing +# (https://golang.org/issue/40132). + +! go test -json ./testpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testpanic.exe -test.v +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +# Tests that panic in a background goroutine should be marked as failing. + +! go test -json ./testbgpanic +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testbgpanic.exe -test.v +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +! go tool test2json ./testbgpanic.exe +stdout '"Action":"fail"' +! stdout '"Action":"pass"' + +-- go.mod -- +module m +go 1.14 +-- mainpanic/mainpanic_test.go -- +package mainpanic_test + +import "testing" + +func TestMain(m *testing.M) { + panic("haha no") +} +-- mainexit0/mainexit0_test.go -- +package mainexit0_test + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + fmt.Println("nothing to do") + os.Exit(0) +} +-- testpanic/testpanic_test.go -- +package testpanic_test + +import "testing" + +func TestPanic(*testing.T) { + panic("haha no") +} +-- testbgpanic/testbgpanic_test.go -- +package testbgpanic_test + +import "testing" + +func TestPanicInBackground(*testing.T) { + c := make(chan struct{}) + go func() { + panic("haha no") + close(c) + }() + <-c +} diff --git a/src/cmd/internal/test2json/test2json.go b/src/cmd/internal/test2json/test2json.go index a01a8900e8..4eb6dd4838 100644 --- a/src/cmd/internal/test2json/test2json.go +++ b/src/cmd/internal/test2json/test2json.go @@ -45,10 +45,10 @@ type textBytes []byte func (b textBytes) MarshalText() ([]byte, error) { return b, nil } -// A converter holds the state of a test-to-JSON conversion. +// A Converter holds the state of a test-to-JSON conversion. // It implements io.WriteCloser; the caller writes test output in, // and the converter writes JSON output to w. -type converter struct { +type Converter struct { w io.Writer // JSON output stream pkg string // package to name in events mode Mode // mode bits @@ -100,9 +100,9 @@ var ( // // The pkg string, if present, specifies the import path to // report in the JSON stream. -func NewConverter(w io.Writer, pkg string, mode Mode) io.WriteCloser { - c := new(converter) - *c = converter{ +func NewConverter(w io.Writer, pkg string, mode Mode) *Converter { + c := new(Converter) + *c = Converter{ w: w, pkg: pkg, mode: mode, @@ -122,11 +122,20 @@ func NewConverter(w io.Writer, pkg string, mode Mode) io.WriteCloser { } // Write writes the test input to the converter. -func (c *converter) Write(b []byte) (int, error) { +func (c *Converter) Write(b []byte) (int, error) { c.input.write(b) return len(b), nil } +// Exited marks the test process as having exited with the given error. +func (c *Converter) Exited(err error) { + if err == nil { + c.result = "pass" + } else { + c.result = "fail" + } +} + var ( // printed by test on successful run. bigPass = []byte("PASS\n") @@ -160,7 +169,7 @@ var ( // handleInputLine handles a single whole test output line. // It must write the line to c.output but may choose to do so // before or after emitting other events. -func (c *converter) handleInputLine(line []byte) { +func (c *Converter) handleInputLine(line []byte) { // Final PASS or FAIL. if bytes.Equal(line, bigPass) || bytes.Equal(line, bigFail) || bytes.HasPrefix(line, bigFailErrorPrefix) { c.flushReport(0) @@ -286,7 +295,7 @@ func (c *converter) handleInputLine(line []byte) { } // flushReport flushes all pending PASS/FAIL reports at levels >= depth. -func (c *converter) flushReport(depth int) { +func (c *Converter) flushReport(depth int) { c.testName = "" for len(c.report) > depth { e := c.report[len(c.report)-1] @@ -298,23 +307,22 @@ func (c *converter) flushReport(depth int) { // Close marks the end of the go test output. // It flushes any pending input and then output (only partial lines at this point) // and then emits the final overall package-level pass/fail event. -func (c *converter) Close() error { +func (c *Converter) Close() error { c.input.flush() c.output.flush() - e := &event{Action: "pass"} if c.result != "" { - e.Action = c.result - } - if c.mode&Timestamp != 0 { - dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds() - e.Elapsed = &dt + e := &event{Action: c.result} + if c.mode&Timestamp != 0 { + dt := time.Since(c.start).Round(1 * time.Millisecond).Seconds() + e.Elapsed = &dt + } + c.writeEvent(e) } - c.writeEvent(e) return nil } // writeOutputEvent writes a single output event with the given bytes. -func (c *converter) writeOutputEvent(out []byte) { +func (c *Converter) writeOutputEvent(out []byte) { c.writeEvent(&event{ Action: "output", Output: (*textBytes)(&out), @@ -323,7 +331,7 @@ func (c *converter) writeOutputEvent(out []byte) { // writeEvent writes a single event. // It adds the package, time (if requested), and test name (if needed). -func (c *converter) writeEvent(e *event) { +func (c *Converter) writeEvent(e *event) { e.Package = c.pkg if c.mode&Timestamp != 0 { t := time.Now() diff --git a/src/cmd/internal/test2json/testdata/benchshort.json b/src/cmd/internal/test2json/testdata/benchshort.json index 28e287c848..34b03b9362 100644 --- a/src/cmd/internal/test2json/testdata/benchshort.json +++ b/src/cmd/internal/test2json/testdata/benchshort.json @@ -4,4 +4,3 @@ {"Action":"output","Output":"# but to avoid questions of timing, we just use a file with no \\n at all.\n"} {"Action":"output","Output":"BenchmarkFoo \t"} {"Action":"output","Output":"10000 early EOF"} -{"Action":"pass"} diff --git a/src/cmd/internal/test2json/testdata/empty.json b/src/cmd/internal/test2json/testdata/empty.json index 80b5217501..e69de29bb2 100644 --- a/src/cmd/internal/test2json/testdata/empty.json +++ b/src/cmd/internal/test2json/testdata/empty.json @@ -1 +0,0 @@ -{"Action":"pass"} diff --git a/src/cmd/test2json/main.go b/src/cmd/test2json/main.go index 0385d8f246..57a874193e 100644 --- a/src/cmd/test2json/main.go +++ b/src/cmd/test2json/main.go @@ -118,12 +118,16 @@ func main() { w := &countWriter{0, c} cmd.Stdout = w cmd.Stderr = w - if err := cmd.Run(); err != nil { + err := cmd.Run() + if err != nil { if w.n > 0 { // Assume command printed why it failed. } else { fmt.Fprintf(c, "test2json: %v\n", err) } + } + c.Exited(err) + if err != nil { c.Close() os.Exit(1) } -- cgit v1.3-5-g9baa From 797124f5ff4bb80957007adbf3115287a4e90870 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 14 Aug 2020 15:47:49 -0400 Subject: cmd/go/internal/test: keep looking for go command flags after ambiguous test flag Fixes #40763 Change-Id: I275970d1f8561414571a5b93e368d68fa052c60f Reviewed-on: https://go-review.googlesource.com/c/go/+/248618 Run-TryBot: Bryan C. Mills TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/test/testflag.go | 29 +++++++++++++++++++++---- src/cmd/go/testdata/script/test_flags.txt | 35 +++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/cmd/go/internal/test/testflag.go b/src/cmd/go/internal/test/testflag.go index 1ff34f7445..4f0a8924f1 100644 --- a/src/cmd/go/internal/test/testflag.go +++ b/src/cmd/go/internal/test/testflag.go @@ -214,9 +214,13 @@ func testFlags(args []string) (packageNames, passToTest []string) { explicitArgs := make([]string, 0, len(args)) inPkgList := false + afterFlagWithoutValue := false for len(args) > 0 { f, remainingArgs, err := cmdflag.ParseOne(&CmdTest.Flag, args) + wasAfterFlagWithoutValue := afterFlagWithoutValue + afterFlagWithoutValue = false // provisionally + if errors.Is(err, flag.ErrHelp) { exitWithUsage() } @@ -233,10 +237,24 @@ func testFlags(args []string) (packageNames, passToTest []string) { if nf := (cmdflag.NonFlagError{}); errors.As(err, &nf) { if !inPkgList && packageNames != nil { // We already saw the package list previously, and this argument is not - // a flag, so it — and everything after it — must be a literal argument - // to the test binary. - explicitArgs = append(explicitArgs, args...) - break + // a flag, so it — and everything after it — must be either a value for + // a preceding flag or a literal argument to the test binary. + if wasAfterFlagWithoutValue { + // This argument could syntactically be a flag value, so + // optimistically assume that it is and keep looking for go command + // flags after it. + // + // (If we're wrong, we'll at least be consistent with historical + // behavior; see https://golang.org/issue/40763.) + explicitArgs = append(explicitArgs, nf.RawArg) + args = remainingArgs + continue + } else { + // This argument syntactically cannot be a flag value, so it must be a + // positional argument, and so must everything after it. + explicitArgs = append(explicitArgs, args...) + break + } } inPkgList = true @@ -272,6 +290,9 @@ func testFlags(args []string) (packageNames, passToTest []string) { explicitArgs = append(explicitArgs, nd.RawArg) args = remainingArgs + if !nd.HasValue { + afterFlagWithoutValue = true + } continue } diff --git a/src/cmd/go/testdata/script/test_flags.txt b/src/cmd/go/testdata/script/test_flags.txt index d38e37f238..63385e6997 100644 --- a/src/cmd/go/testdata/script/test_flags.txt +++ b/src/cmd/go/testdata/script/test_flags.txt @@ -10,7 +10,7 @@ stdout '\Aok\s+example.com/x\s+[0-9.s]+\n\z' ! stderr . # For backward-compatibility with previous releases of the 'go' command, -# arguments that appear after unrecognized flags should not be treated +# arguments that appear after unrecognized flags should not be treated # as packages, even if they are unambiguously not arguments to flags. # Even though ./x looks like a package path, the real package should be # the implicit '.'. @@ -18,6 +18,22 @@ stdout '\Aok\s+example.com/x\s+[0-9.s]+\n\z' stderr '^no Go files in .+$' ! stderr '/x' +# However, *flags* that appear after unrecognized flags should still be +# interpreted as flags, under the (possibly-erroneous) assumption that +# unrecognized flags are non-boolean. + +go test -v -x ./x -timeout 24h -boolflag=true foo -timeout 25h +stdout 'args: foo -timeout 25h' +stdout 'timeout: 24h0m0s$' # -timeout is unambiguously not a flag, so the real flag wins. + +go test -v -x ./x -timeout 24h -boolflag foo -timeout 25h +stdout 'args: foo -test\.timeout=25h0m0s' # For legacy reasons, '-timeout ' is erroneously rewritten to -test.timeout; see https://golang.org/issue/40763. +stdout 'timeout: 24h0m0s$' # Actual flag wins. + +go test -v -x ./x -timeout 24h -stringflag foo -timeout 25h +stdout 'args: $' +stdout 'timeout: 25h0m0s$' # Later flag wins. + # An explicit '-outputdir=' argument should set test.outputdir # to the 'go' command's working directory, not zero it out # for the test binary. @@ -30,23 +46,23 @@ exists ./cover.out # with the 'test.' prefix in the GOFLAGS entry... env GOFLAGS='-test.timeout=24h0m0s -count=1' go test -v -x ./x -stdout '.*: 24h0m0s$' +stdout 'timeout: 24h0m0s$' stderr '-test.count=1' # ...or without. env GOFLAGS='-timeout=24h0m0s -count=1' go test -v -x ./x -stdout '.*: 24h0m0s$' +stdout 'timeout: 24h0m0s$' stderr '-test.count=1' # Arguments from the command line should override GOFLAGS... go test -v -x -timeout=25h0m0s ./x -stdout '.*: 25h0m0s$' +stdout 'timeout: 25h0m0s$' stderr '-test.count=1' # ...even if they use a different flag name. go test -v -x -test.timeout=26h0m0s ./x -stdout '.*: 26h0m0s$' +stdout 'timeout: 26h0m0s$' stderr '-test\.timeout=26h0m0s' ! stderr 'timeout=24h0m0s' stderr '-test.count=1' @@ -99,11 +115,18 @@ package x import ( "flag" + "strings" "testing" ) var _ = flag.String("usage_message", "", "dummy flag to check usage message") +var boolflag = flag.Bool("boolflag", false, "ignored boolean flag") +var stringflag = flag.String("stringflag", "", "ignored string flag") func TestLogTimeout(t *testing.T) { - t.Log(flag.Lookup("test.timeout").Value) + t.Logf("timeout: %v", flag.Lookup("test.timeout").Value) +} + +func TestLogArgs(t *testing.T) { + t.Logf("args: %s", strings.Join(flag.Args(), " ")) } -- cgit v1.3-5-g9baa From c0cf190d226cc3defb71d17c01d0b45bf49a8a85 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 30 Jun 2020 17:51:52 -0400 Subject: cmd/go: do context propagation for tracing downloads This change does context propagation (and only context propagation) necessary to add context to modfetch.Download and pkg.LoadImport. This was done by adding context to their callers, and then adding context to all call-sites, and then repeating adding context to callers of those enclosing functions and their callers until none were left. In some cases the call graph expansion was pruned by using context.TODOs. The next CL will add a span to Download. I kept it out of this change to avoid making it any larger (and harder to review) than it needs to be. Updates #38714 Change-Id: I5bf2d599aafef67334c384dfccd5e255198c85b4 Reviewed-on: https://go-review.googlesource.com/c/go/+/248327 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/internal/get/get.go | 4 +-- src/cmd/go/internal/list/list.go | 5 ++-- src/cmd/go/internal/load/pkg.go | 40 ++++++++++++++--------------- src/cmd/go/internal/load/test.go | 6 ++--- src/cmd/go/internal/modcmd/download.go | 4 +-- src/cmd/go/internal/modcmd/init.go | 2 +- src/cmd/go/internal/modcmd/tidy.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modcmd/why.go | 6 ++--- src/cmd/go/internal/modconv/convert_test.go | 5 +++- src/cmd/go/internal/modfetch/fetch.go | 3 ++- src/cmd/go/internal/modget/get.go | 18 ++++++------- src/cmd/go/internal/modload/build.go | 17 ++++++------ src/cmd/go/internal/modload/import.go | 9 ++++--- src/cmd/go/internal/modload/import_test.go | 5 +++- src/cmd/go/internal/modload/init.go | 9 ++++--- src/cmd/go/internal/modload/list.go | 14 +++++----- src/cmd/go/internal/modload/load.go | 39 ++++++++++++++-------------- src/cmd/go/internal/modload/mvs.go | 5 ++-- src/cmd/go/internal/modload/query.go | 33 ++++++++++++------------ src/cmd/go/internal/modload/query_test.go | 5 +++- src/cmd/go/internal/modload/search.go | 5 ++-- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- 25 files changed, 131 insertions(+), 113 deletions(-) diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index ef43602aca..e5bacadaa3 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7303e6c866..e68c39f392 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -20,6 +20,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -349,7 +350,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": modload.ModuleInfo, + "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -389,7 +390,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod() // Parses go.mod and sets cfg.BuildMod. + modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 32c2ba7912..71fd9b5538 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -42,10 +42,10 @@ var ( ModBinDir func() string // return effective bin directory ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(args []string) []*search.Match // expand import paths + ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files + ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files ModDirImportPath func(string) string // return effective import path for directory ) @@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -605,11 +605,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -657,7 +657,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(path, stk, importPos, bp, err) + p.load(ctx, path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1800,7 +1800,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(roots []*Package) []*Package { +func TestPackageList(ctx context.Context, roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2089,7 +2089,7 @@ func TestPackageList(roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2112,7 +2112,7 @@ func TestPackageList(roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(path, srcDir, parent, stk, importPos, mode) + p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } @@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(patterns)} + return []*Package{GoFilesPackage(ctx, patterns)} } } } - matches := ImportPaths(patterns) + matches := ImportPaths(ctx, patterns) var ( pkgs []*Package stk ImportStack @@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(args []string) []*search.Match { +func ImportPaths(ctx context.Context, args []string) []*search.Match { if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(args) + return ModImportPaths(ctx, args) } return search.ImportPaths(args) } @@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(gofiles []string) *Package { +func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ModInit() for _, f := range gofiles { @@ -2329,7 +2329,7 @@ func GoFilesPackage(gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(gofiles) + ModImportFromFiles(ctx, gofiles) } var err error @@ -2345,7 +2345,7 @@ func GoFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load("command-line-arguments", &stk, nil, bp, err) + pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 6db8a00245..a0e275095b 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 857362a72e..75785c2c48 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod() // to fill Target + modload.InitMod(ctx) // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" @@ -126,7 +126,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(mod) + m.Dir, err = modfetch.Download(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 95063e62f4..b6cffd332d 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if strings.Contains(modload.CmdModModule, "@") { base.Fatalf("go mod init: module path must not contain '@'") } - modload.InitMod() // does all the hard work + modload.InitMod(ctx) // does all the hard work modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index 769cd11fe8..c7c53d7c0c 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL() + modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 257d1cd0ef..e5353b5c7f 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor() + pkgs := modload.LoadVendor(ctx) vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index f400339b25..da33fff89e 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) - for _, path := range loadALL() { + for _, path := range loadALL(ctx) { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(args) // resolve to packages - loadALL() // rebuild graph, from main module (not from named packages) + matches := modload.ImportPaths(ctx, args) // resolve to packages + loadALL(ctx) // rebuild graph, from main module (not from named packages) sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index a04a13b14f..faa2b4c606 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -6,6 +6,7 @@ package modconv import ( "bytes" + "context" "fmt" "internal/testenv" "io/ioutil" @@ -146,6 +147,8 @@ func TestConvertLegacyConfig(t *testing.T) { }, } + ctx := context.Background() + for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -157,7 +160,7 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e40158b535..6606612658 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -7,6 +7,7 @@ package modfetch import ( "archive/zip" "bytes" + "context" "errors" "fmt" "io" @@ -34,7 +35,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(mod module.Version) (dir string, err error) { +func Download(ctx context.Context, mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index d02c9a8da5..ee9757912b 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -353,7 +353,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if !strings.Contains(path, "...") { m := search.NewMatch(path) if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(pkgPath) + m = modload.TargetPackages(ctx, pkgPath) } if len(m.Pkgs) == 0 { for _, err := range m.Errs { @@ -399,7 +399,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { default: // The argument is a package or module path. if modload.HasModRoot() { - if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { + if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { // The path is in the main module. Nothing to query. if vers != "upgrade" && vers != "patch" { base.Errorf("go get %s: can't request explicit version of path in main module", arg) @@ -491,7 +491,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if q.path == q.m.Path { wg.Add(1) go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { + if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil { base.Errorf("go get: %v", err) } else if !hasPkg { modOnlyMu.Lock() @@ -536,7 +536,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Don't load packages if pkgPatterns is empty. Both // modload.ImportPathsQuiet and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) + matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -732,7 +732,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer q.m = module.Version{Path: q.path, Version: "none"} return } - m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) + m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath) if err != nil { base.Errorf("go get %s: %v", q.arg, err) } @@ -790,7 +790,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer // to determine the underlying module version being requested. // If forceModulePath is set, getQuery must interpret path // as a module path. -func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { +func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { if (prevM.Version != "") != forceModulePath { // We resolve package patterns by calling QueryPattern, which does not // accept a previous version and therefore cannot take it into account for @@ -812,7 +812,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo } } - info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +838,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(path, vers, modload.Allowed) + results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -994,7 +994,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 5f8a2e7e05..a101681a1f 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/hex" "fmt" "internal/goroot" @@ -57,21 +58,21 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(m, true) + return moduleInfo(context.TODO(), m, true) } -func ModuleInfo(path string) *modinfo.ModulePublic { +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) + return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(m, true) + return moduleInfo(ctx, m, true) } } @@ -84,12 +85,12 @@ func ModuleInfo(path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(m *modinfo.ModulePublic) { +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -103,7 +104,7 @@ func addVersions(m *modinfo.ModulePublic) { m.Versions, _ = versions(m.Path) } -func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -132,7 +133,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(m.Path, m.Version, "", nil); err != nil { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 4d2bc805e2..5c51a79124 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "go/build" @@ -110,7 +111,7 @@ var _ load.ImportPathError = &AmbiguousImportError{} // Import returns an ImportMissingError as the error. // If Import can identify a module that could be added to supply the package, // the ImportMissingError records that module. -func Import(path string) (m module.Version, dir string, err error) { +func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -165,7 +166,7 @@ func Import(path string) (m module.Version, dir string, err error) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error. // Note that we don't know for sure this module is necessary, @@ -248,7 +249,7 @@ func Import(path string) (m module.Version, dir string, err error) { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error as above. return module.Version{}, "", err @@ -285,7 +286,7 @@ func Import(path string) (m module.Version, dir string, err error) { fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(path, "latest", Allowed) + candidates, err := QueryPackage(ctx, path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index accc60eecd..47ce89a084 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "regexp" "strings" @@ -49,10 +50,12 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() + ctx := context.Background() + for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(tt.path) + m, dir, err := Import(ctx, tt.path) if err == nil { t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index fff060e665..93027c44c4 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -332,7 +333,7 @@ func die() { // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod() { +func InitMod(ctx context.Context) { if len(buildList) > 0 { return } @@ -359,7 +360,7 @@ func InitMod() { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -397,7 +398,7 @@ func InitMod() { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(fixed *bool) modfile.VersionFixer { +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -429,7 +430,7 @@ func fixVersion(fixed *bool) modfile.VersionFixer { } } - info, err := Query(path, vers, "", nil) + info, err := Query(ctx, path, vers, "", nil) if err != nil { return "", err } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 8db4d64706..7bf4e86c8d 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -31,7 +31,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ sem <- token{} go func() { if listU { - addUpdate(m) + addUpdate(ctx, m) } if listVersions { addVersions(m) @@ -57,7 +57,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { LoadBuildList(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -83,7 +83,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(path, vers, current, nil) + info, err := Query(ctx, path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +92,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) continue } @@ -117,7 +117,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m, true)) + mods = append(mods, moduleInfo(ctx, m, true)) } } } @@ -127,9 +127,9 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", "", nil) + info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 8190009b23..686d491219 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -52,8 +52,8 @@ var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), // on the target platform. Modules may be added to the build list // to satisfy new imports. -func ImportPaths(patterns []string) []*search.Match { - matches := ImportPathsQuiet(patterns, imports.Tags()) +func ImportPaths(ctx context.Context, patterns []string) []*search.Match { + matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) search.WarnUnmatched(matches) return matches } @@ -62,7 +62,7 @@ func ImportPaths(patterns []string) []*search.Match { // no matches. It also lets the caller specify a set of build tags to match // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { +func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { updateMatches := func(matches []*search.Match, iterating bool) { for _, m := range matches { switch { @@ -103,7 +103,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, loaded.tags, includeStd, buildList) case m.Pattern() == "all": loaded.testAll = true @@ -111,7 +111,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { // Enumerate the packages in the main module. // We'll load the dependencies as we find them. m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -129,7 +129,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { } } - InitMod() + InitMod(ctx) var matches []*search.Match for _, pattern := range search.CleanPatterns(patterns) { @@ -338,8 +338,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(gofiles []string) { - InitMod() +func ImportFromFiles(ctx context.Context, gofiles []string) { + InitMod(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -391,7 +391,7 @@ func DirImportPath(dir string) string { func LoadBuildList(ctx context.Context) []module.Version { ctx, span := trace.StartSpan(ctx, "LoadBuildList") defer span.Done() - InitMod() + InitMod(ctx) ReloadBuildList() WriteGoMod() return buildList @@ -409,20 +409,20 @@ func ReloadBuildList() []module.Version { // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. -func LoadALL() []string { - return loadAll(true) +func LoadALL(ctx context.Context) []string { + return loadAll(ctx, true) } // LoadVendor is like LoadALL but only follows test dependencies // for tests in the main module. Tests in dependency modules are // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor() []string { - return loadAll(false) +func LoadVendor(ctx context.Context) []string { + return loadAll(ctx, false) } -func loadAll(testAll bool) []string { - InitMod() +func loadAll(ctx context.Context, testAll bool) []string { + InitMod(ctx) loaded = newLoader(imports.AnyTags()) loaded.isALL = true @@ -430,7 +430,7 @@ func loadAll(testAll bool) []string { if !testAll { loaded.testRoots = true } - all := TargetPackages("...") + all := TargetPackages(ctx, "...") loaded.load(func() []string { return all.Pkgs }) checkMultiplePaths() WriteGoMod() @@ -453,13 +453,13 @@ func loadAll(testAll bool) []string { // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(pattern string) *search.Match { +func TargetPackages(ctx context.Context, pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. ModRoot() m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } @@ -817,7 +817,8 @@ func (ld *loader) doPkg(item interface{}) { return } - pkg.mod, pkg.dir, pkg.err = Import(pkg.path) + // TODO(matloob): Handle TODO context. This needs to be threaded through Do. + pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) if pkg.dir == "" { return } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 5dd009d31d..67eb2c2e19 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -224,7 +225,7 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { // // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(mod module.Version) (dir string, isLocal bool, err error) { +func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -254,6 +255,6 @@ func fetch(mod module.Version) (dir string, isLocal bool, err error) { mod = r } - dir, err = modfetch.Download(mod) + dir, err = modfetch.Download(ctx, mod) return dir, false, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index acc886bf21..39a813447c 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -55,10 +56,10 @@ import ( // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(proxy, path, query, current, allowed) + info, err = queryProxy(ctx, proxy, path, query, current, allowed) return err }) return info, err @@ -75,7 +76,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -243,7 +244,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) if err != nil { return nil, err } - releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) if err != nil { return nil, err } @@ -327,7 +328,7 @@ func matchSemverPrefix(p, v string) bool { // 1. versions that do not satisfy the 'ok' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { if !ok(module.Version{Path: path, Version: v}) { @@ -343,7 +344,7 @@ func filterVersions(path string, versions []string, ok func(module.Version) bool // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) if err != nil { return nil, nil, err } @@ -380,12 +381,12 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) } - return QueryPattern(path, query, allowed) + return QueryPattern(ctx, path, query, allowed) } // QueryPattern looks up the module(s) containing at least one package matching @@ -401,7 +402,7 @@ func QueryPackage(path, query string, allowed func(module.Version) bool) ([]Quer // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { base := pattern firstError := func(m *search.Match) error { @@ -417,7 +418,7 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q base = pathpkg.Dir(pattern[:i+3]) match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -472,12 +473,12 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q queryModule := func(path string) (r QueryResult, err error) { current := findCurrentVersion(path) r.Mod.Path = path - r.Rev, err = queryProxy(proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(r.Mod) + root, isLocal, err := fetch(ctx, r.Mod) if err != nil { return r, err } @@ -698,8 +699,8 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(m module.Version) (bool, error) { - root, isLocal, err := fetch(m) +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + root, isLocal, err := fetch(ctx, m) if err != nil { return false, err } @@ -707,8 +708,8 @@ func ModuleHasRootPackage(m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(m module.Version) (bool, error) { - root, _, err := fetch(m) +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + root, _, err := fetch(ctx, m) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 247e4c40d2..77080e9b5b 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "io/ioutil" "log" @@ -179,6 +180,8 @@ func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") + ctx := context.Background() + for _, tt := range queryTests { allow := tt.allow if allow == "" { @@ -192,7 +195,7 @@ func TestQuery(t *testing.T) { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(tt.path, tt.query, tt.current, allowed) + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index c28e7c0c1e..a9bee0af4e 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -5,6 +5,7 @@ package modload import ( + "context" "fmt" "os" "path/filepath" @@ -27,7 +28,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -153,7 +154,7 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu isLocal = true } else { var err error - root, isLocal, err = fetch(mod) + root, isLocal, err = fetch(ctx, mod) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index deec5106ff..99578b244c 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -77,7 +77,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(files) + p = load.GoFilesPackage(ctx, files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9cef8cf89c..3aee6939d2 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -702,7 +702,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(pkgs) { + for _, p := range load.TestPackageList(ctx, pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 681ecd7646..d975c36306 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2900,7 +2900,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { } srcs := []string{src} - p := load.GoFilesPackage(srcs) + p := load.GoFilesPackage(context.TODO(), srcs) if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { return "32", nil -- cgit v1.3-5-g9baa From 1a3558341860357c2400e37773e5076bb3a51628 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 15 Jul 2020 17:38:36 -0400 Subject: cmd/go: add tracing for querying and downloading from the proxy This CL adds tracing spans for modload.queryPattern, modload.queryProxy, modload.QueryPattern, modload.QueryPattern.queryModule, modload.queryPrefixModules and modfetch.Download. Updates #38714 Change-Id: I537c7fa4f466c691c1b60ec73ef8a2277af49cd7 Reviewed-on: https://go-review.googlesource.com/c/go/+/242786 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modfetch/fetch.go | 21 +++++++++++++++------ .../internal/modfetch/zip_sum_test/zip_sum_test.go | 3 ++- src/cmd/go/internal/modload/query.go | 22 ++++++++++++++++++---- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 75785c2c48..13e5cb066b 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -120,7 +120,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(mod) + m.Zip, err = modfetch.DownloadZip(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 6606612658..e29eb0a942 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -24,6 +24,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -48,7 +49,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(mod) + dir, err := download(ctx, mod) if err != nil { return cached{"", err} } @@ -58,7 +59,10 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(mod module.Version) (dir string, err error) { +func download(ctx context.Context, mod module.Version) (dir string, err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) + defer span.Done() + // If the directory exists, and no .partial file exists, the module has // already been completely extracted. .partial files may be created when a // module zip directory is extracted in place instead of being extracted to a @@ -73,7 +77,7 @@ func download(mod module.Version) (dir string, err error) { // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(mod) + zipfile, err := DownloadZip(ctx, mod) if err != nil { return "", err } @@ -143,6 +147,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) if unzipInPlace { if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { return "", err @@ -172,6 +177,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } } + defer span.Done() if !cfg.ModCacheRW { // Make dir read-only only *after* renaming it. @@ -196,7 +202,7 @@ var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(mod module.Version) (zipfile string, err error) { +func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -231,7 +237,7 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(mod, zipfile); err != nil { + if err := downloadZip(ctx, mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -239,7 +245,10 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { return c.zipfile, c.err } -func downloadZip(mod module.Version, zipfile string) (err error) { +func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) + defer span.Done() + // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. diff --git a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index eac9b32fa8..82398ebfed 100644 --- a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,6 +16,7 @@ package zip_sum_test import ( + "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -119,7 +120,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(test.m) + zipPath, err := modfetch.DownloadZip(context.Background(), test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 39a813447c..e82eb1506f 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -77,6 +78,9 @@ func (queryDisabledError) Error() string { } func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -403,6 +407,9 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // the main module and only the version "latest", without checking for other // possible modules. func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + base := pattern firstError := func(m *search.Match) error { @@ -470,7 +477,10 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(path string) (r QueryResult, err error) { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + current := findCurrentVersion(path) r.Mod.Path = path r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) @@ -499,7 +509,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } var err error - results, err = queryPrefixModules(candidateModules, queryModule) + results, err = queryPrefixModules(ctx, candidateModules, queryModule) return err }) @@ -543,7 +553,10 @@ type prefixResult struct { err error } -func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() + // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -556,8 +569,9 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(p) + r.QueryResult, r.err = queryModule(ctx, p) wg.Done() }(p, &results[i]) } -- cgit v1.3-5-g9baa From e30fbe3757d09a22988835835c41233df7c6cd00 Mon Sep 17 00:00:00 2001 From: Junchen Li Date: Fri, 10 Jul 2020 11:39:23 +0800 Subject: cmd/compile: optimize unsigned comparisons to 0 There are some architecture-independent rules in #21439, since an unsigned integer >= 0 is always true and < 0 is always false. This CL adds these optimizations to generic rules. Updates #21439 Change-Id: Iec7e3040b761ecb1e60908f764815fdd9bc62495 Reviewed-on: https://go-review.googlesource.com/c/go/+/246617 Reviewed-by: Keith Randall Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/ssa/gen/generic.rules | 4 ++ src/cmd/compile/internal/ssa/rewritegeneric.go | 80 ++++++++++++++++++++++++++ test/codegen/comparisons.go | 17 ++++++ 3 files changed, 101 insertions(+) diff --git a/src/cmd/compile/internal/ssa/gen/generic.rules b/src/cmd/compile/internal/ssa/gen/generic.rules index ed5bfc81fd..2d39d27226 100644 --- a/src/cmd/compile/internal/ssa/gen/generic.rules +++ b/src/cmd/compile/internal/ssa/gen/generic.rules @@ -545,6 +545,10 @@ (Or(64|32|16|8) x (Or(64|32|16|8) x y)) => (Or(64|32|16|8) x y) (Xor(64|32|16|8) x (Xor(64|32|16|8) x y)) => y +// Unsigned comparisons to zero. +(Less(64U|32U|16U|8U) _ (Const(64|32|16|8) [0])) => (ConstBool [false]) +(Leq(64U|32U|16U|8U) (Const(64|32|16|8) [0]) _) => (ConstBool [true]) + // Ands clear bits. Ors set bits. // If a subsequent Or will set all the bits // that an And cleared, we can skip the And. diff --git a/src/cmd/compile/internal/ssa/rewritegeneric.go b/src/cmd/compile/internal/ssa/rewritegeneric.go index 9f4e1b95bd..68e49f46f3 100644 --- a/src/cmd/compile/internal/ssa/rewritegeneric.go +++ b/src/cmd/compile/internal/ssa/rewritegeneric.go @@ -9701,6 +9701,16 @@ func rewriteValuegeneric_OpLeq16U(v *Value) bool { v.AuxInt = boolToAuxInt(uint16(c) <= uint16(d)) return true } + // match: (Leq16U (Const16 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst16 || auxIntToInt16(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLeq32(v *Value) bool { @@ -9805,6 +9815,16 @@ func rewriteValuegeneric_OpLeq32U(v *Value) bool { v.AuxInt = boolToAuxInt(uint32(c) <= uint32(d)) return true } + // match: (Leq32U (Const32 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst32 || auxIntToInt32(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLeq64(v *Value) bool { @@ -9909,6 +9929,16 @@ func rewriteValuegeneric_OpLeq64U(v *Value) bool { v.AuxInt = boolToAuxInt(uint64(c) <= uint64(d)) return true } + // match: (Leq64U (Const64 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst64 || auxIntToInt64(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLeq8(v *Value) bool { @@ -9993,6 +10023,16 @@ func rewriteValuegeneric_OpLeq8U(v *Value) bool { v.AuxInt = boolToAuxInt(uint8(c) <= uint8(d)) return true } + // match: (Leq8U (Const8 [0]) _) + // result: (ConstBool [true]) + for { + if v_0.Op != OpConst8 || auxIntToInt8(v_0.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(true) + return true + } return false } func rewriteValuegeneric_OpLess16(v *Value) bool { @@ -10033,6 +10073,16 @@ func rewriteValuegeneric_OpLess16U(v *Value) bool { v.AuxInt = boolToAuxInt(uint16(c) < uint16(d)) return true } + // match: (Less16U _ (Const16 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst16 || auxIntToInt16(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLess32(v *Value) bool { @@ -10093,6 +10143,16 @@ func rewriteValuegeneric_OpLess32U(v *Value) bool { v.AuxInt = boolToAuxInt(uint32(c) < uint32(d)) return true } + // match: (Less32U _ (Const32 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst32 || auxIntToInt32(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLess64(v *Value) bool { @@ -10153,6 +10213,16 @@ func rewriteValuegeneric_OpLess64U(v *Value) bool { v.AuxInt = boolToAuxInt(uint64(c) < uint64(d)) return true } + // match: (Less64U _ (Const64 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst64 || auxIntToInt64(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLess8(v *Value) bool { @@ -10193,6 +10263,16 @@ func rewriteValuegeneric_OpLess8U(v *Value) bool { v.AuxInt = boolToAuxInt(uint8(c) < uint8(d)) return true } + // match: (Less8U _ (Const8 [0])) + // result: (ConstBool [false]) + for { + if v_1.Op != OpConst8 || auxIntToInt8(v_1.AuxInt) != 0 { + break + } + v.reset(OpConstBool) + v.AuxInt = boolToAuxInt(false) + return true + } return false } func rewriteValuegeneric_OpLoad(v *Value) bool { diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index 90808573c2..f3c15538a8 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -407,3 +407,20 @@ func CmpToZero_ex5(e, f int32, u uint32) int { } return 0 } +func UintLtZero(a uint8, b uint16, c uint32, d uint64) int { + // amd64: -`(TESTB|TESTW|TESTL|TESTQ|JCC|JCS)` + // arm64: -`(CMPW|CMP|BHS|BLO)` + if a < 0 || b < 0 || c < 0 || d < 0 { + return 1 + } + return 0 +} + +func UintGeqZero(a uint8, b uint16, c uint32, d uint64) int { + // amd64: -`(TESTB|TESTW|TESTL|TESTQ|JCS|JCC)` + // arm64: -`(CMPW|CMP|BLO|BHS)` + if a >= 0 || b >= 0 || c >= 0 || d >= 0 { + return 1 + } + return 0 +} -- cgit v1.3-5-g9baa From 260dff3ca3b06385dc298523791a2079162f546e Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 19 Feb 2020 19:45:57 +0000 Subject: runtime: clean up old markrootSpans This change removes the old markrootSpans implementation and deletes the feature flag. Updates #37487. Change-Id: Idb5a2559abcc3be5a7da6f2ccce1a86e1d7634e3 Reviewed-on: https://go-review.googlesource.com/c/go/+/221183 Run-TryBot: Michael Knyszek TryBot-Result: Gobot Gobot Reviewed-by: Michael Pratt Reviewed-by: Austin Clements --- src/runtime/mgcmark.go | 120 ++++----------------------------------------- src/runtime/mgcsweep.go | 2 +- src/runtime/mgcsweepbuf.go | 38 -------------- src/runtime/mheap.go | 8 ++- 4 files changed, 14 insertions(+), 154 deletions(-) diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 96910ff729..2b84945471 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -47,10 +47,6 @@ const ( // Must be a multiple of the pageInUse bitmap element size and // must also evenly divide pagesPerArena. pagesPerSpanRoot = 512 - - // go115NewMarkrootSpans is a feature flag that indicates whether - // to use the new bitmap-based markrootSpans implementation. - go115NewMarkrootSpans = true ) // gcMarkRootPrepare queues root scanning jobs (stacks, globals, and @@ -87,24 +83,16 @@ func gcMarkRootPrepare() { // // We depend on addfinalizer to mark objects that get // finalizers after root marking. - if go115NewMarkrootSpans { - // We're going to scan the whole heap (that was available at the time the - // mark phase started, i.e. markArenas) for in-use spans which have specials. - // - // Break up the work into arenas, and further into chunks. - // - // Snapshot allArenas as markArenas. This snapshot is safe because allArenas - // is append-only. - mheap_.markArenas = mheap_.allArenas[:len(mheap_.allArenas):len(mheap_.allArenas)] - work.nSpanRoots = len(mheap_.markArenas) * (pagesPerArena / pagesPerSpanRoot) - } else { - // We're only interested in scanning the in-use spans, - // which will all be swept at this point. More spans - // may be added to this list during concurrent GC, but - // we only care about spans that were allocated before - // this mark phase. - work.nSpanRoots = mheap_.sweepSpans[mheap_.sweepgen/2%2].numBlocks() - } + // + // We're going to scan the whole heap (that was available at the time the + // mark phase started, i.e. markArenas) for in-use spans which have specials. + // + // Break up the work into arenas, and further into chunks. + // + // Snapshot allArenas as markArenas. This snapshot is safe because allArenas + // is append-only. + mheap_.markArenas = mheap_.allArenas[:len(mheap_.allArenas):len(mheap_.allArenas)] + work.nSpanRoots = len(mheap_.markArenas) * (pagesPerArena / pagesPerSpanRoot) // Scan stacks. // @@ -316,10 +304,6 @@ func markrootFreeGStacks() { // //go:nowritebarrier func markrootSpans(gcw *gcWork, shard int) { - if !go115NewMarkrootSpans { - oldMarkrootSpans(gcw, shard) - return - } // Objects with finalizers have two GC-related invariants: // // 1) Everything reachable from the object must be marked. @@ -396,90 +380,6 @@ func markrootSpans(gcw *gcWork, shard int) { } } -// oldMarkrootSpans marks roots for one shard of work.spans. -// -// For go115NewMarkrootSpans = false. -// -//go:nowritebarrier -func oldMarkrootSpans(gcw *gcWork, shard int) { - // Objects with finalizers have two GC-related invariants: - // - // 1) Everything reachable from the object must be marked. - // This ensures that when we pass the object to its finalizer, - // everything the finalizer can reach will be retained. - // - // 2) Finalizer specials (which are not in the garbage - // collected heap) are roots. In practice, this means the fn - // field must be scanned. - // - // TODO(austin): There are several ideas for making this more - // efficient in issue #11485. - - sg := mheap_.sweepgen - spans := mheap_.sweepSpans[mheap_.sweepgen/2%2].block(shard) - // Note that work.spans may not include spans that were - // allocated between entering the scan phase and now. We may - // also race with spans being added into sweepSpans when they're - // just created, and as a result we may see nil pointers in the - // spans slice. This is okay because any objects with finalizers - // in those spans must have been allocated and given finalizers - // after we entered the scan phase, so addfinalizer will have - // ensured the above invariants for them. - for i := 0; i < len(spans); i++ { - // sweepBuf.block requires that we read pointers from the block atomically. - // It also requires that we ignore nil pointers. - s := (*mspan)(atomic.Loadp(unsafe.Pointer(&spans[i]))) - - // This is racing with spans being initialized, so - // check the state carefully. - if s == nil || s.state.get() != mSpanInUse { - continue - } - // Check that this span was swept (it may be cached or uncached). - if !useCheckmark && !(s.sweepgen == sg || s.sweepgen == sg+3) { - // sweepgen was updated (+2) during non-checkmark GC pass - print("sweep ", s.sweepgen, " ", sg, "\n") - throw("gc: unswept span") - } - - // Speculatively check if there are any specials - // without acquiring the span lock. This may race with - // adding the first special to a span, but in that - // case addfinalizer will observe that the GC is - // active (which is globally synchronized) and ensure - // the above invariants. We may also ensure the - // invariants, but it's okay to scan an object twice. - if s.specials == nil { - continue - } - - // Lock the specials to prevent a special from being - // removed from the list while we're traversing it. - lock(&s.speciallock) - - for sp := s.specials; sp != nil; sp = sp.next { - if sp.kind != _KindSpecialFinalizer { - continue - } - // don't mark finalized object, but scan it so we - // retain everything it points to. - spf := (*specialfinalizer)(unsafe.Pointer(sp)) - // A finalizer can be set for an inner byte of an object, find object beginning. - p := s.base() + uintptr(spf.special.offset)/s.elemsize*s.elemsize - - // Mark everything that can be reached from - // the object (but *not* the object itself or - // we'll never collect it). - scanobject(p, gcw) - - // The special itself is a root. - scanblock(uintptr(unsafe.Pointer(&spf.fn)), sys.PtrSize, &oneptrmask[0], gcw, nil) - } - - unlock(&s.speciallock) - } -} - // gcAssistAlloc performs GC work to make gp's assist debt positive. // gp must be the calling user gorountine. // diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index 3aa3afc028..9244174403 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -662,7 +662,7 @@ func (s *mspan) oldSweep(preserve bool) bool { special = *specialp } } - if go115NewMarkrootSpans && hadSpecials && s.specials == nil { + if hadSpecials && s.specials == nil { spanHasNoSpecials(s) } diff --git a/src/runtime/mgcsweepbuf.go b/src/runtime/mgcsweepbuf.go index 1f722c3d58..5e5ca3dd2f 100644 --- a/src/runtime/mgcsweepbuf.go +++ b/src/runtime/mgcsweepbuf.go @@ -136,41 +136,3 @@ func (b *gcSweepBuf) pop() *mspan { block.spans[bottom] = nil return s } - -// numBlocks returns the number of blocks in buffer b. numBlocks is -// safe to call concurrently with any other operation. Spans that have -// been pushed prior to the call to numBlocks are guaranteed to appear -// in some block in the range [0, numBlocks()), assuming there are no -// intervening pops. Spans that are pushed after the call may also -// appear in these blocks. -func (b *gcSweepBuf) numBlocks() int { - return int(divRoundUp(uintptr(atomic.Load(&b.index)), gcSweepBlockEntries)) -} - -// block returns the spans in the i'th block of buffer b. block is -// safe to call concurrently with push. The block may contain nil -// pointers that must be ignored, and each entry in the block must be -// loaded atomically. -func (b *gcSweepBuf) block(i int) []*mspan { - // Perform bounds check before loading spine address since - // push ensures the allocated length is at least spineLen. - if i < 0 || uintptr(i) >= atomic.Loaduintptr(&b.spineLen) { - throw("block index out of range") - } - - // Get block i. - spine := atomic.Loadp(unsafe.Pointer(&b.spine)) - blockp := add(spine, sys.PtrSize*uintptr(i)) - block := (*gcSweepBlock)(atomic.Loadp(blockp)) - - // Slice the block if necessary. - cursor := uintptr(atomic.Load(&b.index)) - top, bottom := cursor/gcSweepBlockEntries, cursor%gcSweepBlockEntries - var spans []*mspan - if uintptr(i) < top { - spans = block.spans[:] - } else { - spans = block.spans[:bottom] - } - return spans -} diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 6341375160..0807726863 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -52,7 +52,7 @@ const ( // The definition of this flag helps ensure that if there's a problem with // the new markroot spans implementation and it gets turned off, that the new // mcentral implementation also gets turned off so the runtime isn't broken. - go115NewMCentralImpl = true && go115NewMarkrootSpans + go115NewMCentralImpl = true ) // Main malloc heap. @@ -1705,9 +1705,7 @@ func addspecial(p unsafe.Pointer, s *special) bool { s.offset = uint16(offset) s.next = *t *t = s - if go115NewMarkrootSpans { - spanHasSpecials(span) - } + spanHasSpecials(span) unlock(&span.speciallock) releasem(mp) @@ -1748,7 +1746,7 @@ func removespecial(p unsafe.Pointer, kind uint8) *special { } t = &s.next } - if go115NewMarkrootSpans && span.specials == nil { + if span.specials == nil { spanHasNoSpecials(span) } unlock(&span.speciallock) -- cgit v1.3-5-g9baa From e6d0bd2b8951bde6f0ac6421f20e18efc7ba0cdb Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 19 Feb 2020 16:37:48 +0000 Subject: runtime: clean up old mcentral code This change deletes the old mcentral implementation from the code base and the newMCentralImpl feature flag along with it. Updates #37487. Change-Id: Ibca8f722665f0865051f649ffe699cbdbfdcfcf2 Reviewed-on: https://go-review.googlesource.com/c/go/+/221184 Run-TryBot: Michael Knyszek TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements Reviewed-by: Michael Pratt --- src/runtime/lockrank.go | 16 +-- src/runtime/malloc.go | 8 +- src/runtime/mcache.go | 6 +- src/runtime/mcentral.go | 239 +-------------------------------------------- src/runtime/mgc.go | 10 +- src/runtime/mgcsweep.go | 237 ++------------------------------------------ src/runtime/mgcsweepbuf.go | 138 -------------------------- src/runtime/mheap.go | 36 +------ 8 files changed, 25 insertions(+), 665 deletions(-) delete mode 100644 src/runtime/mgcsweepbuf.go diff --git a/src/runtime/lockrank.go b/src/runtime/lockrank.go index 000193585d..b23cf767be 100644 --- a/src/runtime/lockrank.go +++ b/src/runtime/lockrank.go @@ -67,8 +67,6 @@ const ( lockRankRwmutexW lockRankRwmutexR - lockRankMcentral // For !go115NewMCentralImpl - lockRankSpine // For !go115NewMCentralImpl lockRankSpanSetSpine lockRankGscan lockRankStackpool @@ -149,8 +147,6 @@ var lockNames = []string{ lockRankRwmutexW: "rwmutexW", lockRankRwmutexR: "rwmutexR", - lockRankMcentral: "mcentral", - lockRankSpine: "spine", lockRankSpanSetSpine: "spanSetSpine", lockRankGscan: "gscan", lockRankStackpool: "stackpool", @@ -228,18 +224,16 @@ var lockPartialOrder [][]lockRank = [][]lockRank{ lockRankRwmutexW: {}, lockRankRwmutexR: {lockRankRwmutexW}, - lockRankMcentral: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankHchan}, - lockRankSpine: {lockRankSysmon, lockRankScavenge, lockRankAssistQueue, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankHchan}, lockRankSpanSetSpine: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankHchan}, - lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankHchan, lockRankFin, lockRankTraceBuf, lockRankTraceStrings, lockRankRoot, lockRankNotifyList, lockRankProf, lockRankGcBitsArenas, lockRankTrace, lockRankTraceStackTab, lockRankNetpollInit, lockRankMcentral, lockRankSpine, lockRankSpanSetSpine}, - lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankSweepWaiters, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankPollDesc, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankHchan, lockRankFin, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankProf, lockRankGcBitsArenas, lockRankRoot, lockRankTrace, lockRankTraceStackTab, lockRankNetpollInit, lockRankRwmutexR, lockRankMcentral, lockRankSpine, lockRankSpanSetSpine, lockRankGscan}, - lockRankStackLarge: {lockRankSysmon, lockRankAssistQueue, lockRankSched, lockRankItab, lockRankHchan, lockRankProf, lockRankGcBitsArenas, lockRankRoot, lockRankMcentral, lockRankSpanSetSpine, lockRankGscan}, + lockRankGscan: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankHchan, lockRankFin, lockRankTraceBuf, lockRankTraceStrings, lockRankRoot, lockRankNotifyList, lockRankProf, lockRankGcBitsArenas, lockRankTrace, lockRankTraceStackTab, lockRankNetpollInit, lockRankSpanSetSpine}, + lockRankStackpool: {lockRankSysmon, lockRankScavenge, lockRankSweepWaiters, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankPollDesc, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankHchan, lockRankFin, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankProf, lockRankGcBitsArenas, lockRankRoot, lockRankTrace, lockRankTraceStackTab, lockRankNetpollInit, lockRankRwmutexR, lockRankSpanSetSpine, lockRankGscan}, + lockRankStackLarge: {lockRankSysmon, lockRankAssistQueue, lockRankSched, lockRankItab, lockRankHchan, lockRankProf, lockRankGcBitsArenas, lockRankRoot, lockRankSpanSetSpine, lockRankGscan}, lockRankDefer: {}, lockRankSudog: {lockRankNotifyList, lockRankHchan}, lockRankWbufSpans: {lockRankSysmon, lockRankScavenge, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankSched, lockRankAllg, lockRankPollDesc, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankHchan, lockRankNotifyList, lockRankTraceStrings, lockRankMspanSpecial, lockRankProf, lockRankRoot, lockRankGscan, lockRankDefer, lockRankSudog}, - lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankSweepWaiters, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankAllg, lockRankAllp, lockRankPollDesc, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankHchan, lockRankMspanSpecial, lockRankProf, lockRankGcBitsArenas, lockRankRoot, lockRankMcentral, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankDefer, lockRankSudog, lockRankWbufSpans, lockRankSpanSetSpine}, + lockRankMheap: {lockRankSysmon, lockRankScavenge, lockRankSweepWaiters, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankAllg, lockRankAllp, lockRankPollDesc, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankHchan, lockRankMspanSpecial, lockRankProf, lockRankGcBitsArenas, lockRankRoot, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankDefer, lockRankSudog, lockRankWbufSpans, lockRankSpanSetSpine}, lockRankMheapSpecial: {lockRankSysmon, lockRankScavenge, lockRankAssistQueue, lockRankCpuprof, lockRankSweep, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankItab, lockRankReflectOffs, lockRankNotifyList, lockRankTraceBuf, lockRankTraceStrings, lockRankHchan}, - lockRankGlobalAlloc: {lockRankProf, lockRankSpine, lockRankSpanSetSpine, lockRankMheap, lockRankMheapSpecial}, + lockRankGlobalAlloc: {lockRankProf, lockRankSpanSetSpine, lockRankMheap, lockRankMheapSpecial}, lockRankGFree: {lockRankSched}, lockRankHchanLeaf: {lockRankGscan, lockRankHchanLeaf}, diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index b3fac3de24..e46327f9ce 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -1178,11 +1178,9 @@ func largeAlloc(size uintptr, needzero bool, noscan bool) *mspan { if s == nil { throw("out of memory") } - if go115NewMCentralImpl { - // Put the large span in the mcentral swept list so that it's - // visible to the background sweeper. - mheap_.central[spc].mcentral.fullSwept(mheap_.sweepgen).push(s) - } + // Put the large span in the mcentral swept list so that it's + // visible to the background sweeper. + mheap_.central[spc].mcentral.fullSwept(mheap_.sweepgen).push(s) s.limit = s.base() + size heapBitsForAddr(s.base()).initSpan(s) return s diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index 5bceb51ac9..7a7d33ccae 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -131,11 +131,7 @@ func (c *mcache) refill(spc spanClass) { if s.sweepgen != mheap_.sweepgen+3 { throw("bad sweepgen in refill") } - if go115NewMCentralImpl { - mheap_.central[spc].mcentral.uncacheSpan(s) - } else { - atomic.Store(&s.sweepgen, mheap_.sweepgen) - } + mheap_.central[spc].mcentral.uncacheSpan(s) } // Get a new cached span from the central lists. diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go index ed49d86d0c..ed49e01677 100644 --- a/src/runtime/mcentral.go +++ b/src/runtime/mcentral.go @@ -18,7 +18,6 @@ import "runtime/internal/atomic" // //go:notinheap type mcentral struct { - lock mutex spanclass spanClass // For !go115NewMCentralImpl. @@ -55,16 +54,10 @@ type mcentral struct { // Initialize a single central free list. func (c *mcentral) init(spc spanClass) { c.spanclass = spc - if go115NewMCentralImpl { - lockInit(&c.partial[0].spineLock, lockRankSpanSetSpine) - lockInit(&c.partial[1].spineLock, lockRankSpanSetSpine) - lockInit(&c.full[0].spineLock, lockRankSpanSetSpine) - lockInit(&c.full[1].spineLock, lockRankSpanSetSpine) - } else { - c.nonempty.init() - c.empty.init() - lockInit(&c.lock, lockRankMcentral) - } + lockInit(&c.partial[0].spineLock, lockRankSpanSetSpine) + lockInit(&c.partial[1].spineLock, lockRankSpanSetSpine) + lockInit(&c.full[0].spineLock, lockRankSpanSetSpine) + lockInit(&c.full[1].spineLock, lockRankSpanSetSpine) } // partialUnswept returns the spanSet which holds partially-filled @@ -93,9 +86,6 @@ func (c *mcentral) fullSwept(sweepgen uint32) *spanSet { // Allocate a span to use in an mcache. func (c *mcentral) cacheSpan() *mspan { - if !go115NewMCentralImpl { - return c.oldCacheSpan() - } // Deduct credit for this span allocation and sweep if necessary. spanBytes := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) * _PageSize deductSweepCredit(spanBytes, 0) @@ -213,127 +203,11 @@ havespan: return s } -// Allocate a span to use in an mcache. -// -// For !go115NewMCentralImpl. -func (c *mcentral) oldCacheSpan() *mspan { - // Deduct credit for this span allocation and sweep if necessary. - spanBytes := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) * _PageSize - deductSweepCredit(spanBytes, 0) - - lock(&c.lock) - traceDone := false - if trace.enabled { - traceGCSweepStart() - } - sg := mheap_.sweepgen -retry: - var s *mspan - for s = c.nonempty.first; s != nil; s = s.next { - if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) { - c.nonempty.remove(s) - c.empty.insertBack(s) - unlock(&c.lock) - s.sweep(true) - goto havespan - } - if s.sweepgen == sg-1 { - // the span is being swept by background sweeper, skip - continue - } - // we have a nonempty span that does not require sweeping, allocate from it - c.nonempty.remove(s) - c.empty.insertBack(s) - unlock(&c.lock) - goto havespan - } - - for s = c.empty.first; s != nil; s = s.next { - if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) { - // we have an empty span that requires sweeping, - // sweep it and see if we can free some space in it - c.empty.remove(s) - // swept spans are at the end of the list - c.empty.insertBack(s) - unlock(&c.lock) - s.sweep(true) - freeIndex := s.nextFreeIndex() - if freeIndex != s.nelems { - s.freeindex = freeIndex - goto havespan - } - lock(&c.lock) - // the span is still empty after sweep - // it is already in the empty list, so just retry - goto retry - } - if s.sweepgen == sg-1 { - // the span is being swept by background sweeper, skip - continue - } - // already swept empty span, - // all subsequent ones must also be either swept or in process of sweeping - break - } - if trace.enabled { - traceGCSweepDone() - traceDone = true - } - unlock(&c.lock) - - // Replenish central list if empty. - s = c.grow() - if s == nil { - return nil - } - lock(&c.lock) - c.empty.insertBack(s) - unlock(&c.lock) - - // At this point s is a non-empty span, queued at the end of the empty list, - // c is unlocked. -havespan: - if trace.enabled && !traceDone { - traceGCSweepDone() - } - n := int(s.nelems) - int(s.allocCount) - if n == 0 || s.freeindex == s.nelems || uintptr(s.allocCount) == s.nelems { - throw("span has no free objects") - } - // Assume all objects from this span will be allocated in the - // mcache. If it gets uncached, we'll adjust this. - atomic.Xadd64(&c.nmalloc, int64(n)) - usedBytes := uintptr(s.allocCount) * s.elemsize - atomic.Xadd64(&memstats.heap_live, int64(spanBytes)-int64(usedBytes)) - if trace.enabled { - // heap_live changed. - traceHeapAlloc() - } - if gcBlackenEnabled != 0 { - // heap_live changed. - gcController.revise() - } - freeByteBase := s.freeindex &^ (64 - 1) - whichByte := freeByteBase / 8 - // Init alloc bits cache. - s.refillAllocCache(whichByte) - - // Adjust the allocCache so that s.freeindex corresponds to the low bit in - // s.allocCache. - s.allocCache >>= s.freeindex % 64 - - return s -} - // Return span from an mcache. // // s must have a span class corresponding to this // mcentral and it must not be empty. func (c *mcentral) uncacheSpan(s *mspan) { - if !go115NewMCentralImpl { - c.oldUncacheSpan(s) - return - } if s.allocCount == 0 { throw("uncaching span but s.allocCount == 0") } @@ -393,111 +267,6 @@ func (c *mcentral) uncacheSpan(s *mspan) { } } -// Return span from an mcache. -// -// For !go115NewMCentralImpl. -func (c *mcentral) oldUncacheSpan(s *mspan) { - if s.allocCount == 0 { - throw("uncaching span but s.allocCount == 0") - } - - sg := mheap_.sweepgen - stale := s.sweepgen == sg+1 - if stale { - // Span was cached before sweep began. It's our - // responsibility to sweep it. - // - // Set sweepgen to indicate it's not cached but needs - // sweeping and can't be allocated from. sweep will - // set s.sweepgen to indicate s is swept. - atomic.Store(&s.sweepgen, sg-1) - } else { - // Indicate that s is no longer cached. - atomic.Store(&s.sweepgen, sg) - } - - n := int(s.nelems) - int(s.allocCount) - if n > 0 { - // cacheSpan updated alloc assuming all objects on s - // were going to be allocated. Adjust for any that - // weren't. We must do this before potentially - // sweeping the span. - atomic.Xadd64(&c.nmalloc, -int64(n)) - - lock(&c.lock) - c.empty.remove(s) - c.nonempty.insert(s) - if !stale { - // mCentral_CacheSpan conservatively counted - // unallocated slots in heap_live. Undo this. - // - // If this span was cached before sweep, then - // heap_live was totally recomputed since - // caching this span, so we don't do this for - // stale spans. - atomic.Xadd64(&memstats.heap_live, -int64(n)*int64(s.elemsize)) - } - unlock(&c.lock) - } - - if stale { - // Now that s is in the right mcentral list, we can - // sweep it. - s.sweep(false) - } -} - -// freeSpan updates c and s after sweeping s. -// It sets s's sweepgen to the latest generation, -// and, based on the number of free objects in s, -// moves s to the appropriate list of c or returns it -// to the heap. -// freeSpan reports whether s was returned to the heap. -// If preserve=true, it does not move s (the caller -// must take care of it). -// -// For !go115NewMCentralImpl. -func (c *mcentral) freeSpan(s *mspan, preserve bool, wasempty bool) bool { - if sg := mheap_.sweepgen; s.sweepgen == sg+1 || s.sweepgen == sg+3 { - throw("freeSpan given cached span") - } - s.needzero = 1 - - if preserve { - // preserve is set only when called from (un)cacheSpan above, - // the span must be in the empty list. - if !s.inList() { - throw("can't preserve unlinked span") - } - atomic.Store(&s.sweepgen, mheap_.sweepgen) - return false - } - - lock(&c.lock) - - // Move to nonempty if necessary. - if wasempty { - c.empty.remove(s) - c.nonempty.insert(s) - } - - // delay updating sweepgen until here. This is the signal that - // the span may be used in an mcache, so it must come after the - // linked list operations above (actually, just after the - // lock of c above.) - atomic.Store(&s.sweepgen, mheap_.sweepgen) - - if s.allocCount != 0 { - unlock(&c.lock) - return false - } - - c.nonempty.remove(s) - unlock(&c.lock) - mheap_.freeSpan(s) - return true -} - // grow allocates a new empty span from the heap and initializes it for c's size class. func (c *mcentral) grow() *mspan { npages := uintptr(class_to_allocnpages[c.spanclass.sizeclass()]) diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index c8c4a4c758..bd87144355 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -2149,21 +2149,13 @@ func gcSweep(mode gcMode) { lock(&mheap_.lock) mheap_.sweepgen += 2 mheap_.sweepdone = 0 - if !go115NewMCentralImpl && mheap_.sweepSpans[mheap_.sweepgen/2%2].index != 0 { - // We should have drained this list during the last - // sweep phase. We certainly need to start this phase - // with an empty swept list. - throw("non-empty swept list") - } mheap_.pagesSwept = 0 mheap_.sweepArenas = mheap_.allArenas mheap_.reclaimIndex = 0 mheap_.reclaimCredit = 0 unlock(&mheap_.lock) - if go115NewMCentralImpl { - sweep.centralIndex.clear() - } + sweep.centralIndex.clear() if !_ConcurrentSweep || mode == gcForceBlockMode { // Special case synchronous sweep. diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index 9244174403..6b8c56ce35 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -132,17 +132,15 @@ func finishsweep_m() { sweep.npausesweep++ } - if go115NewMCentralImpl { - // Reset all the unswept buffers, which should be empty. - // Do this in sweep termination as opposed to mark termination - // so that we can catch unswept spans and reclaim blocks as - // soon as possible. - sg := mheap_.sweepgen - for i := range mheap_.central { - c := &mheap_.central[i].mcentral - c.partialUnswept(sg).reset() - c.fullUnswept(sg).reset() - } + // Reset all the unswept buffers, which should be empty. + // Do this in sweep termination as opposed to mark termination + // so that we can catch unswept spans and reclaim blocks as + // soon as possible. + sg := mheap_.sweepgen + for i := range mheap_.central { + c := &mheap_.central[i].mcentral + c.partialUnswept(sg).reset() + c.fullUnswept(sg).reset() } // Sweeping is done, so if the scavenger isn't already awake, @@ -202,11 +200,7 @@ func sweepone() uintptr { var s *mspan sg := mheap_.sweepgen for { - if go115NewMCentralImpl { - s = mheap_.nextSpanForSweep() - } else { - s = mheap_.sweepSpans[1-sg/2%2].pop() - } + s = mheap_.nextSpanForSweep() if s == nil { atomic.Store(&mheap_.sweepdone, 1) break @@ -322,9 +316,6 @@ func (s *mspan) ensureSwept() { // If preserve=true, don't return it to heap nor relink in mcentral lists; // caller takes care of it. func (s *mspan) sweep(preserve bool) bool { - if !go115NewMCentralImpl { - return s.oldSweep(preserve) - } // It's critical that we enter this function with preemption disabled, // GC must not start while we are in the middle of this function. _g_ := getg() @@ -568,214 +559,6 @@ func (s *mspan) sweep(preserve bool) bool { return false } -// Sweep frees or collects finalizers for blocks not marked in the mark phase. -// It clears the mark bits in preparation for the next GC round. -// Returns true if the span was returned to heap. -// If preserve=true, don't return it to heap nor relink in mcentral lists; -// caller takes care of it. -// -// For !go115NewMCentralImpl. -func (s *mspan) oldSweep(preserve bool) bool { - // It's critical that we enter this function with preemption disabled, - // GC must not start while we are in the middle of this function. - _g_ := getg() - if _g_.m.locks == 0 && _g_.m.mallocing == 0 && _g_ != _g_.m.g0 { - throw("mspan.sweep: m is not locked") - } - sweepgen := mheap_.sweepgen - if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 { - print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") - throw("mspan.sweep: bad span state") - } - - if trace.enabled { - traceGCSweepSpan(s.npages * _PageSize) - } - - atomic.Xadd64(&mheap_.pagesSwept, int64(s.npages)) - - spc := s.spanclass - size := s.elemsize - res := false - - c := _g_.m.p.ptr().mcache - freeToHeap := false - - // The allocBits indicate which unmarked objects don't need to be - // processed since they were free at the end of the last GC cycle - // and were not allocated since then. - // If the allocBits index is >= s.freeindex and the bit - // is not marked then the object remains unallocated - // since the last GC. - // This situation is analogous to being on a freelist. - - // Unlink & free special records for any objects we're about to free. - // Two complications here: - // 1. An object can have both finalizer and profile special records. - // In such case we need to queue finalizer for execution, - // mark the object as live and preserve the profile special. - // 2. A tiny object can have several finalizers setup for different offsets. - // If such object is not marked, we need to queue all finalizers at once. - // Both 1 and 2 are possible at the same time. - hadSpecials := s.specials != nil - specialp := &s.specials - special := *specialp - for special != nil { - // A finalizer can be set for an inner byte of an object, find object beginning. - objIndex := uintptr(special.offset) / size - p := s.base() + objIndex*size - mbits := s.markBitsForIndex(objIndex) - if !mbits.isMarked() { - // This object is not marked and has at least one special record. - // Pass 1: see if it has at least one finalizer. - hasFin := false - endOffset := p - s.base() + size - for tmp := special; tmp != nil && uintptr(tmp.offset) < endOffset; tmp = tmp.next { - if tmp.kind == _KindSpecialFinalizer { - // Stop freeing of object if it has a finalizer. - mbits.setMarkedNonAtomic() - hasFin = true - break - } - } - // Pass 2: queue all finalizers _or_ handle profile record. - for special != nil && uintptr(special.offset) < endOffset { - // Find the exact byte for which the special was setup - // (as opposed to object beginning). - p := s.base() + uintptr(special.offset) - if special.kind == _KindSpecialFinalizer || !hasFin { - // Splice out special record. - y := special - special = special.next - *specialp = special - freespecial(y, unsafe.Pointer(p), size) - } else { - // This is profile record, but the object has finalizers (so kept alive). - // Keep special record. - specialp = &special.next - special = *specialp - } - } - } else { - // object is still live: keep special record - specialp = &special.next - special = *specialp - } - } - if hadSpecials && s.specials == nil { - spanHasNoSpecials(s) - } - - if debug.allocfreetrace != 0 || debug.clobberfree != 0 || raceenabled || msanenabled { - // Find all newly freed objects. This doesn't have to - // efficient; allocfreetrace has massive overhead. - mbits := s.markBitsForBase() - abits := s.allocBitsForIndex(0) - for i := uintptr(0); i < s.nelems; i++ { - if !mbits.isMarked() && (abits.index < s.freeindex || abits.isMarked()) { - x := s.base() + i*s.elemsize - if debug.allocfreetrace != 0 { - tracefree(unsafe.Pointer(x), size) - } - if debug.clobberfree != 0 { - clobberfree(unsafe.Pointer(x), size) - } - if raceenabled { - racefree(unsafe.Pointer(x), size) - } - if msanenabled { - msanfree(unsafe.Pointer(x), size) - } - } - mbits.advance() - abits.advance() - } - } - - // Count the number of free objects in this span. - nalloc := uint16(s.countAlloc()) - if spc.sizeclass() == 0 && nalloc == 0 { - s.needzero = 1 - freeToHeap = true - } - nfreed := s.allocCount - nalloc - if nalloc > s.allocCount { - print("runtime: nelems=", s.nelems, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n") - throw("sweep increased allocation count") - } - - s.allocCount = nalloc - wasempty := s.nextFreeIndex() == s.nelems - s.freeindex = 0 // reset allocation index to start of span. - if trace.enabled { - getg().m.p.ptr().traceReclaimed += uintptr(nfreed) * s.elemsize - } - - // gcmarkBits becomes the allocBits. - // get a fresh cleared gcmarkBits in preparation for next GC - s.allocBits = s.gcmarkBits - s.gcmarkBits = newMarkBits(s.nelems) - - // Initialize alloc bits cache. - s.refillAllocCache(0) - - // We need to set s.sweepgen = h.sweepgen only when all blocks are swept, - // because of the potential for a concurrent free/SetFinalizer. - // But we need to set it before we make the span available for allocation - // (return it to heap or mcentral), because allocation code assumes that a - // span is already swept if available for allocation. - if freeToHeap || nfreed == 0 { - // The span must be in our exclusive ownership until we update sweepgen, - // check for potential races. - if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 { - print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n") - throw("mspan.sweep: bad span state after sweep") - } - // Serialization point. - // At this point the mark bits are cleared and allocation ready - // to go so release the span. - atomic.Store(&s.sweepgen, sweepgen) - } - - if nfreed > 0 && spc.sizeclass() != 0 { - c.local_nsmallfree[spc.sizeclass()] += uintptr(nfreed) - res = mheap_.central[spc].mcentral.freeSpan(s, preserve, wasempty) - // mcentral.freeSpan updates sweepgen - } else if freeToHeap { - // Free large span to heap - - // NOTE(rsc,dvyukov): The original implementation of efence - // in CL 22060046 used sysFree instead of sysFault, so that - // the operating system would eventually give the memory - // back to us again, so that an efence program could run - // longer without running out of memory. Unfortunately, - // calling sysFree here without any kind of adjustment of the - // heap data structures means that when the memory does - // come back to us, we have the wrong metadata for it, either in - // the mspan structures or in the garbage collection bitmap. - // Using sysFault here means that the program will run out of - // memory fairly quickly in efence mode, but at least it won't - // have mysterious crashes due to confused memory reuse. - // It should be possible to switch back to sysFree if we also - // implement and then call some kind of mheap.deleteSpan. - if debug.efence > 0 { - s.limit = 0 // prevent mlookup from finding this span - sysFault(unsafe.Pointer(s.base()), size) - } else { - mheap_.freeSpan(s) - } - c.local_nlargefree++ - c.local_largefree += size - res = true - } - if !res { - // The span has been swept and is still in-use, so put - // it on the swept in-use list. - mheap_.sweepSpans[sweepgen/2%2].push(s) - } - return res -} - // reportZombies reports any marked but free objects in s and throws. // // This generally means one of the following: diff --git a/src/runtime/mgcsweepbuf.go b/src/runtime/mgcsweepbuf.go deleted file mode 100644 index 5e5ca3dd2f..0000000000 --- a/src/runtime/mgcsweepbuf.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -import ( - "internal/cpu" - "runtime/internal/atomic" - "runtime/internal/sys" - "unsafe" -) - -// A gcSweepBuf is a set of *mspans. -// -// gcSweepBuf is safe for concurrent push operations *or* concurrent -// pop operations, but not both simultaneously. -type gcSweepBuf struct { - // A gcSweepBuf is a two-level data structure consisting of a - // growable spine that points to fixed-sized blocks. The spine - // can be accessed without locks, but adding a block or - // growing it requires taking the spine lock. - // - // Because each mspan covers at least 8K of heap and takes at - // most 8 bytes in the gcSweepBuf, the growth of the spine is - // quite limited. - // - // The spine and all blocks are allocated off-heap, which - // allows this to be used in the memory manager and avoids the - // need for write barriers on all of these. We never release - // this memory because there could be concurrent lock-free - // access and we're likely to reuse it anyway. (In principle, - // we could do this during STW.) - - spineLock mutex - spine unsafe.Pointer // *[N]*gcSweepBlock, accessed atomically - spineLen uintptr // Spine array length, accessed atomically - spineCap uintptr // Spine array cap, accessed under lock - - // index is the first unused slot in the logical concatenation - // of all blocks. It is accessed atomically. - index uint32 -} - -const ( - gcSweepBlockEntries = 512 // 4KB on 64-bit - gcSweepBufInitSpineCap = 256 // Enough for 1GB heap on 64-bit -) - -type gcSweepBlock struct { - spans [gcSweepBlockEntries]*mspan -} - -// push adds span s to buffer b. push is safe to call concurrently -// with other push operations, but NOT to call concurrently with pop. -func (b *gcSweepBuf) push(s *mspan) { - // Obtain our slot. - cursor := uintptr(atomic.Xadd(&b.index, +1) - 1) - top, bottom := cursor/gcSweepBlockEntries, cursor%gcSweepBlockEntries - - // Do we need to add a block? - spineLen := atomic.Loaduintptr(&b.spineLen) - var block *gcSweepBlock -retry: - if top < spineLen { - spine := atomic.Loadp(unsafe.Pointer(&b.spine)) - blockp := add(spine, sys.PtrSize*top) - block = (*gcSweepBlock)(atomic.Loadp(blockp)) - } else { - // Add a new block to the spine, potentially growing - // the spine. - lock(&b.spineLock) - // spineLen cannot change until we release the lock, - // but may have changed while we were waiting. - spineLen = atomic.Loaduintptr(&b.spineLen) - if top < spineLen { - unlock(&b.spineLock) - goto retry - } - - if spineLen == b.spineCap { - // Grow the spine. - newCap := b.spineCap * 2 - if newCap == 0 { - newCap = gcSweepBufInitSpineCap - } - newSpine := persistentalloc(newCap*sys.PtrSize, cpu.CacheLineSize, &memstats.gc_sys) - if b.spineCap != 0 { - // Blocks are allocated off-heap, so - // no write barriers. - memmove(newSpine, b.spine, b.spineCap*sys.PtrSize) - } - // Spine is allocated off-heap, so no write barrier. - atomic.StorepNoWB(unsafe.Pointer(&b.spine), newSpine) - b.spineCap = newCap - // We can't immediately free the old spine - // since a concurrent push with a lower index - // could still be reading from it. We let it - // leak because even a 1TB heap would waste - // less than 2MB of memory on old spines. If - // this is a problem, we could free old spines - // during STW. - } - - // Allocate a new block and add it to the spine. - block = (*gcSweepBlock)(persistentalloc(unsafe.Sizeof(gcSweepBlock{}), cpu.CacheLineSize, &memstats.gc_sys)) - blockp := add(b.spine, sys.PtrSize*top) - // Blocks are allocated off-heap, so no write barrier. - atomic.StorepNoWB(blockp, unsafe.Pointer(block)) - atomic.Storeuintptr(&b.spineLen, spineLen+1) - unlock(&b.spineLock) - } - - // We have a block. Insert the span atomically, since there may be - // concurrent readers via the block API. - atomic.StorepNoWB(unsafe.Pointer(&block.spans[bottom]), unsafe.Pointer(s)) -} - -// pop removes and returns a span from buffer b, or nil if b is empty. -// pop is safe to call concurrently with other pop operations, but NOT -// to call concurrently with push. -func (b *gcSweepBuf) pop() *mspan { - cursor := atomic.Xadd(&b.index, -1) - if int32(cursor) < 0 { - atomic.Xadd(&b.index, +1) - return nil - } - - // There are no concurrent spine or block modifications during - // pop, so we can omit the atomics. - top, bottom := cursor/gcSweepBlockEntries, cursor%gcSweepBlockEntries - blockp := (**gcSweepBlock)(add(b.spine, sys.PtrSize*uintptr(top))) - block := *blockp - s := block.spans[bottom] - // Clear the pointer for block(i). - block.spans[bottom] = nil - return s -} diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 0807726863..cb586171c4 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -44,15 +44,6 @@ const ( // Must be a multiple of the pageInUse bitmap element size and // must also evenly divid pagesPerArena. pagesPerReclaimerChunk = 512 - - // go115NewMCentralImpl is a feature flag for the new mcentral implementation. - // - // This flag depends on go115NewMarkrootSpans because the new mcentral - // implementation requires that markroot spans no longer rely on mgcsweepbufs. - // The definition of this flag helps ensure that if there's a problem with - // the new markroot spans implementation and it gets turned off, that the new - // mcentral implementation also gets turned off so the runtime isn't broken. - go115NewMCentralImpl = true ) // Main malloc heap. @@ -85,19 +76,6 @@ type mheap struct { // access (since that may free the backing store). allspans []*mspan // all spans out there - // sweepSpans contains two mspan stacks: one of swept in-use - // spans, and one of unswept in-use spans. These two trade - // roles on each GC cycle. Since the sweepgen increases by 2 - // on each cycle, this means the swept spans are in - // sweepSpans[sweepgen/2%2] and the unswept spans are in - // sweepSpans[1-sweepgen/2%2]. Sweeping pops spans from the - // unswept stack and pushes spans that are still in-use on the - // swept stack. Likewise, allocating an in-use span pushes it - // on the swept stack. - // - // For !go115NewMCentralImpl. - sweepSpans [2]gcSweepBuf - _ uint32 // align uint64 fields on 32-bit for atomics // Proportional sweep @@ -220,7 +198,7 @@ type mheap struct { base, end uintptr } - // _ uint32 // ensure 64-bit alignment of central + _ uint32 // ensure 64-bit alignment of central // central free lists for small size classes. // the padding makes sure that the mcentrals are @@ -719,8 +697,6 @@ func pageIndexOf(p uintptr) (arena *heapArena, pageIdx uintptr, pageMask uint8) // Initialize the heap. func (h *mheap) init() { lockInit(&h.lock, lockRankMheap) - lockInit(&h.sweepSpans[0].spineLock, lockRankSpine) - lockInit(&h.sweepSpans[1].spineLock, lockRankSpine) lockInit(&h.speciallock, lockRankMheapSpecial) h.spanalloc.init(unsafe.Sizeof(mspan{}), recordspan, unsafe.Pointer(h), &memstats.mspan_sys) @@ -1294,16 +1270,6 @@ HaveSpan: h.setSpans(s.base(), npages, s) if !manual { - if !go115NewMCentralImpl { - // Add to swept in-use list. - // - // This publishes the span to root marking. - // - // h.sweepgen is guaranteed to only change during STW, - // and preemption is disabled in the page allocator. - h.sweepSpans[h.sweepgen/2%2].push(s) - } - // Mark in-use span in arena page bitmap. // // This publishes the span to the page sweeper, so -- cgit v1.3-5-g9baa From a61a3c378d9ce71d9b97a1b4fb3320b8b3d6a599 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 11 Jun 2020 11:17:20 +0800 Subject: runtime: use the CBZ instruction in the assembler Use CBZ to replace the comparison and branch of arm64 and the zero instruction in the assembly file. Change-Id: Id6c03e9af13aadafc3ad3953f82d2ffa29c12926 Reviewed-on: https://go-review.googlesource.com/c/go/+/237497 Reviewed-by: Keith Randall Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot --- src/runtime/rt0_freebsd_arm64.s | 3 +-- src/runtime/rt0_netbsd_arm64.s | 3 +-- src/runtime/rt0_openbsd_arm64.s | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/runtime/rt0_freebsd_arm64.s b/src/runtime/rt0_freebsd_arm64.s index 3a348c33e2..a938d98262 100644 --- a/src/runtime/rt0_freebsd_arm64.s +++ b/src/runtime/rt0_freebsd_arm64.s @@ -45,8 +45,7 @@ TEXT _rt0_arm64_freebsd_lib(SB),NOSPLIT,$184 // Create a new thread to do the runtime initialization and return. MOVD _cgo_sys_thread_create(SB), R4 - CMP $0, R4 - BEQ nocgo + CBZ R4, nocgo MOVD $_rt0_arm64_freebsd_lib_go(SB), R0 MOVD $0, R1 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. diff --git a/src/runtime/rt0_netbsd_arm64.s b/src/runtime/rt0_netbsd_arm64.s index 75ecbe5176..2f3b5a5a87 100644 --- a/src/runtime/rt0_netbsd_arm64.s +++ b/src/runtime/rt0_netbsd_arm64.s @@ -44,8 +44,7 @@ TEXT _rt0_arm64_netbsd_lib(SB),NOSPLIT,$184 // Create a new thread to do the runtime initialization and return. MOVD _cgo_sys_thread_create(SB), R4 - CMP $0, R4 - BEQ nocgo + CBZ R4, nocgo MOVD $_rt0_arm64_netbsd_lib_go(SB), R0 MOVD $0, R1 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. diff --git a/src/runtime/rt0_openbsd_arm64.s b/src/runtime/rt0_openbsd_arm64.s index 12408f2eec..722fab6129 100644 --- a/src/runtime/rt0_openbsd_arm64.s +++ b/src/runtime/rt0_openbsd_arm64.s @@ -50,8 +50,7 @@ TEXT _rt0_arm64_openbsd_lib(SB),NOSPLIT,$184 // Create a new thread to do the runtime initialization and return. MOVD _cgo_sys_thread_create(SB), R4 - CMP $0, R4 - BEQ nocgo + CBZ R4, nocgo MOVD $_rt0_arm64_openbsd_lib_go(SB), R0 MOVD $0, R1 SUB $16, RSP // reserve 16 bytes for sp-8 where fp may be saved. -- cgit v1.3-5-g9baa From 99d6e3eec2cad4387b28759322273d774cc94fe5 Mon Sep 17 00:00:00 2001 From: Heisenberg Date: Thu, 11 Jun 2020 10:16:33 +0800 Subject: internal/bytealg: use CBZ instructions Use CBZ to replace the comparison and jump to the zero instruction in the arm64 assembly file. Change-Id: Ie16fb52e27b4d327343e119ebc0f0ca756437bc4 Reviewed-on: https://go-review.googlesource.com/c/go/+/237477 Reviewed-by: Keith Randall Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot --- src/internal/bytealg/compare_arm64.s | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/internal/bytealg/compare_arm64.s b/src/internal/bytealg/compare_arm64.s index 32e2ba200d..56d56f241e 100644 --- a/src/internal/bytealg/compare_arm64.s +++ b/src/internal/bytealg/compare_arm64.s @@ -36,8 +36,7 @@ TEXT cmpbody<>(SB),NOSPLIT|NOFRAME,$0-0 CMP R0, R1 CSEL LT, R1, R0, R6 // R6 is min(R0, R1) - CMP $0, R6 - BEQ samebytes + CBZ R6, samebytes BIC $0xf, R6, R10 CBZ R10, small // length < 16 ADD R2, R10 // end of chunk16 -- cgit v1.3-5-g9baa From 17553c6e7154deab8501595f4fa387c0b718337a Mon Sep 17 00:00:00 2001 From: surechen Date: Mon, 27 Jul 2020 12:00:19 +0800 Subject: cmd/compile: move dumpFileSeq I noticed that there is a Todo comment here. This variable is only used for filename when dump a function's ssa passes result in details. It is no problem to print a function alone, but may be edited by not only one goroutine if dump multiple functions at the same time. Although it looks only dump one function's ssa passes now. As far as I am concerned this variable can be a member variable of the struct Func. I'm not sure if this change is necessary. Looking forward to your advices, thank you very much. Change-Id: I35dd7247889e0cc7f19c0b400b597206592dee75 Reviewed-on: https://go-review.googlesource.com/c/go/+/244918 Reviewed-by: Keith Randall Run-TryBot: Keith Randall --- src/cmd/compile/internal/ssa/compile.go | 7 ++----- src/cmd/compile/internal/ssa/func.go | 7 ++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cmd/compile/internal/ssa/compile.go b/src/cmd/compile/internal/ssa/compile.go index dbdd027716..444475d67a 100644 --- a/src/cmd/compile/internal/ssa/compile.go +++ b/src/cmd/compile/internal/ssa/compile.go @@ -160,15 +160,12 @@ func Compile(f *Func) { phaseName = "" } -// TODO: should be a config field -var dumpFileSeq int - // dumpFile creates a file from the phase name and function name // Dumping is done to files to avoid buffering huge strings before // output. func (f *Func) dumpFile(phaseName string) { - dumpFileSeq++ - fname := fmt.Sprintf("%s_%02d__%s.dump", f.Name, dumpFileSeq, phaseName) + f.dumpFileSeq++ + fname := fmt.Sprintf("%s_%02d__%s.dump", f.Name, int(f.dumpFileSeq), phaseName) fname = strings.Replace(fname, " ", "_", -1) fname = strings.Replace(fname, "/", "_", -1) fname = strings.Replace(fname, ":", "_", -1) diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index 4b9189fb3e..9e40b6214c 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -44,9 +44,10 @@ type Func struct { PrintOrHtmlSSA bool // true if GOSSAFUNC matches, true even if fe.Log() (spew phase results to stdout) is false. ruleMatches map[string]int // number of times countRule was called during compilation for any given string - scheduled bool // Values in Blocks are in final order - laidout bool // Blocks are ordered - NoSplit bool // true if function is marked as nosplit. Used by schedule check pass. + scheduled bool // Values in Blocks are in final order + laidout bool // Blocks are ordered + NoSplit bool // true if function is marked as nosplit. Used by schedule check pass. + dumpFileSeq uint8 // the sequence numbers of dump file. (%s_%02d__%s.dump", funcname, dumpFileSeq, phaseName) // when register allocation is done, maps value ids to locations RegAlloc []Location -- cgit v1.3-5-g9baa From ba97be4b58241bebbc4ff70574bd82152ab19ffe Mon Sep 17 00:00:00 2001 From: liu-xuewen Date: Mon, 13 Jul 2020 09:15:38 +0000 Subject: runtime: remove tracebackinit and unused skipPC CL [152537](https://go-review.googlesource.com/c/go/+/152537/) changed the way inlined frames are represented in tracebacks to no longer use skipPC Change-Id: I42386fdcc5cf72f3c122e789b6af9cbd0c6bed4b GitHub-Last-Rev: 79c26dcd532907eda4ffc30951845c1c01243501 GitHub-Pull-Request: golang/go#39829 Reviewed-on: https://go-review.googlesource.com/c/go/+/239701 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/asm.s | 21 --------------------- src/runtime/proc.go | 1 - src/runtime/traceback.go | 13 ------------- 3 files changed, 35 deletions(-) diff --git a/src/runtime/asm.s b/src/runtime/asm.s index 95a3424de2..27d8df9e06 100644 --- a/src/runtime/asm.s +++ b/src/runtime/asm.s @@ -11,24 +11,3 @@ DATA runtime·no_pointers_stackmap+0x00(SB)/4, $2 DATA runtime·no_pointers_stackmap+0x04(SB)/4, $0 GLOBL runtime·no_pointers_stackmap(SB),RODATA, $8 - -// NaCl requires that these skips be verifiable machine code. -#ifdef GOARCH_amd64 -#define SKIP4 BYTE $0x90; BYTE $0x90; BYTE $0x90; BYTE $0x90 -#endif -#ifdef GOARCH_386 -#define SKIP4 BYTE $0x90; BYTE $0x90; BYTE $0x90; BYTE $0x90 -#endif -#ifdef GOARCH_wasm -#define SKIP4 UNDEF; UNDEF; UNDEF; UNDEF -#endif -#ifndef SKIP4 -#define SKIP4 WORD $0 -#endif - -#define SKIP16 SKIP4; SKIP4; SKIP4; SKIP4 -#define SKIP64 SKIP16; SKIP16; SKIP16; SKIP16 - -// This function must be sizeofSkipFunction bytes. -TEXT runtime·skipPleaseUseCallersFrames(SB),NOSPLIT,$0-0 - SKIP64; SKIP64; SKIP64; SKIP64 diff --git a/src/runtime/proc.go b/src/runtime/proc.go index ed7e2128ae..9a358cd529 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -558,7 +558,6 @@ func schedinit() { sched.maxmcount = 10000 - tracebackinit() moduledataverify() stackinit() mallocinit() diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 96e552524e..7850eceafa 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -36,16 +36,6 @@ import ( const usesLR = sys.MinFrameSize > 0 -var skipPC uintptr - -func tracebackinit() { - // Go variable initialization happens late during runtime startup. - // Instead of initializing the variables above in the declarations, - // schedinit calls this function so that the variables are - // initialized and available earlier in the startup sequence. - skipPC = funcPC(skipPleaseUseCallersFrames) -} - // Traceback over the deferred function calls. // Report them like calls that have been invoked but not started executing yet. func tracebackdefers(gp *g, callback func(*stkframe, unsafe.Pointer) bool, v unsafe.Pointer) { @@ -83,9 +73,6 @@ func tracebackdefers(gp *g, callback func(*stkframe, unsafe.Pointer) bool, v uns const sizeofSkipFunction = 256 -// This function is defined in asm.s to be sizeofSkipFunction bytes long. -func skipPleaseUseCallersFrames() - // Generic traceback. Handles runtime stack prints (pcbuf == nil), // the runtime.Callers function (pcbuf != nil), as well as the garbage // collector (callback != nil). A little clunky to merge these, but avoids -- cgit v1.3-5-g9baa From 88c094c96a164aef2134e548d495c4bc14dc4687 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 25 Jun 2020 09:10:23 -0700 Subject: runtime: print faulting instruction on a SIGFPE Just like SIGILL, it might be useful to see what the instruction that generated the SIGFPE is. Update #39816 Change-Id: I8b2ff692998f0b770289339537dceab96b09d1ee Reviewed-on: https://go-review.googlesource.com/c/go/+/239999 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/signal_unix.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index dd6d79f8ec..6a11c91fb9 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -616,7 +616,7 @@ func sighandler(sig uint32, info *siginfo, ctxt unsafe.Pointer, gp *g) { print("signal arrived during cgo execution\n") gp = _g_.m.lockedg.ptr() } - if sig == _SIGILL { + if sig == _SIGILL || sig == _SIGFPE { // It would be nice to know how long the instruction is. // Unfortunately, that's complicated to do in general (mostly for x86 // and s930x, but other archs have non-standard instruction lengths also). -- cgit v1.3-5-g9baa From 4e5ed83e8d2fbbbc8f6524f40ab3b6733dc57a38 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Fri, 3 Jul 2020 11:28:50 -0700 Subject: runtime: use bit-parallel operations to compute heap bit summaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new implementation is much faster in all cases. name old time/op new time/op delta PallocBitsSummarize/Unpacked00-16 142ns ± 1% 7ns ± 2% -94.75% (p=0.000 n=10+9) PallocBitsSummarize/UnpackedFFFFFFFFFFFFFFFF-16 172ns ± 0% 24ns ± 0% -86.02% (p=0.000 n=9+9) PallocBitsSummarize/UnpackedAA-16 145ns ± 0% 32ns ± 0% -78.16% (p=0.000 n=8+10) PallocBitsSummarize/UnpackedAAAAAAAAAAAAAAAA-16 172ns ± 0% 33ns ± 0% -80.95% (p=0.000 n=9+9) PallocBitsSummarize/Unpacked80000000AAAAAAAA-16 162ns ± 1% 60ns ± 0% -62.69% (p=0.000 n=10+9) PallocBitsSummarize/UnpackedAAAAAAAA00000001-16 163ns ± 0% 68ns ± 1% -58.47% (p=0.000 n=8+10) PallocBitsSummarize/UnpackedBBBBBBBBBBBBBBBB-16 172ns ± 0% 35ns ± 0% -79.70% (p=0.000 n=9+9) PallocBitsSummarize/Unpacked80000000BBBBBBBB-16 161ns ± 0% 63ns ± 0% -60.61% (p=0.000 n=8+10) PallocBitsSummarize/UnpackedBBBBBBBB00000001-16 163ns ± 0% 60ns ± 0% -63.14% (p=0.000 n=9+10) PallocBitsSummarize/UnpackedCCCCCCCCCCCCCCCC-16 172ns ± 0% 39ns ± 0% -77.41% (p=0.000 n=7+10) PallocBitsSummarize/Unpacked4444444444444444-16 172ns ± 0% 39ns ± 0% -77.42% (p=0.000 n=7+10) PallocBitsSummarize/Unpacked4040404040404040-16 173ns ± 2% 51ns ± 1% -70.55% (p=0.000 n=10+10) PallocBitsSummarize/Unpacked4000400040004000-16 160ns ± 1% 53ns ± 0% -66.78% (p=0.000 n=10+10) PallocBitsSummarize/Unpacked1000404044CCAAFF-16 169ns ± 1% 59ns ± 1% -65.28% (p=0.000 n=10+10) Change-Id: I94daa645b76a9cf9c93edeb2058d7132216fcb72 Reviewed-on: https://go-review.googlesource.com/c/go/+/240900 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Michael Knyszek --- src/runtime/mpallocbits.go | 147 +++++++++++++++++++++++----------------- src/runtime/mpallocbits_test.go | 36 ++++++---- 2 files changed, 108 insertions(+), 75 deletions(-) diff --git a/src/runtime/mpallocbits.go b/src/runtime/mpallocbits.go index a8011341bc..ff79bfbc1a 100644 --- a/src/runtime/mpallocbits.go +++ b/src/runtime/mpallocbits.go @@ -120,78 +120,99 @@ func (b *pageBits) popcntRange(i, n uint) (s uint) { // sake of documentation, 0s are free pages and 1s are allocated pages. type pallocBits pageBits -// consec8tab is a table containing the number of consecutive -// zero bits for any uint8 value. -// -// The table is generated by calling consec8(i) for each -// possible uint8 value, which is defined as: -// -// // consec8 counts the maximum number of consecutive 0 bits -// // in a uint8. -// func consec8(n uint8) int { -// n = ^n -// i := 0 -// for n != 0 { -// n &= (n << 1) -// i++ -// } -// return i -// } -var consec8tab = [256]uint{ - 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 5, 4, 3, 3, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, - 4, 3, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, - 6, 5, 4, 4, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, - 4, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 2, 1, 1, 1, - 5, 4, 3, 3, 2, 2, 2, 2, 3, 2, 1, 1, 2, 1, 1, 1, - 4, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 2, 1, 1, 1, - 7, 6, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, - 4, 3, 2, 2, 2, 2, 2, 2, 3, 2, 2, 2, 2, 2, 2, 2, - 5, 4, 3, 3, 2, 2, 2, 2, 3, 2, 1, 1, 2, 1, 1, 1, - 4, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 2, 1, 1, 1, - 6, 5, 4, 4, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, - 4, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 2, 1, 1, 1, - 5, 4, 3, 3, 2, 2, 2, 2, 3, 2, 1, 1, 2, 1, 1, 1, - 4, 3, 2, 2, 2, 1, 1, 1, 3, 2, 1, 1, 2, 1, 1, 0, -} - // summarize returns a packed summary of the bitmap in pallocBits. func (b *pallocBits) summarize() pallocSum { - // TODO(mknyszek): There may be something more clever to be done - // here to make the summarize operation more efficient. For example, - // we can compute start and end with 64-bit wide operations easily, - // but max is a bit more complex. Perhaps there exists some way to - // leverage the 64-bit start and end to our advantage? - var start, max, end uint + var start, max, cur uint + const notSetYet = ^uint(0) // sentinel for start value + start = notSetYet for i := 0; i < len(b); i++ { - a := b[i] - for j := 0; j < 64; j += 8 { - k := uint8(a >> j) - - // Compute start. - si := uint(sys.TrailingZeros8(k)) - if start == uint(i*64+j) { - start += si - } + x := b[i] + if x == 0 { + cur += 64 + continue + } + t := uint(sys.TrailingZeros64(x)) + l := uint(sys.LeadingZeros64(x)) - // Compute max. - if end+si > max { - max = end + si - } - if mi := consec8tab[k]; mi > max { - max = mi + // Finish any region spanning the uint64s + cur += t + if start == notSetYet { + start = cur + } + if cur > max { + max = cur + } + // Final region that might span to next uint64 + cur = l + } + if start == notSetYet { + // Made it all the way through without finding a single 1 bit. + const n = uint(64 * len(b)) + return packPallocSum(n, n, n) + } + if cur > max { + max = cur + } + if max >= 64-2 { + // There is no way an internal run of zeros could beat max. + return packPallocSum(start, max, cur) + } + // Now look inside each uint64 for runs of zeros. + // All uint64s must be nonzero, or we would have aborted above. +outer: + for i := 0; i < len(b); i++ { + x := b[i] + + // Look inside this uint64. We have a pattern like + // 000000 1xxxxx1 000000 + // We need to look inside the 1xxxxx1 for any contiguous + // region of zeros. + + // We already know the trailing zeros are no larger than max. Remove them. + x >>= sys.TrailingZeros64(x) & 63 + if x&(x+1) == 0 { // no more zeros (except at the top). + continue + } + + // Strategy: shrink all runs of zeros by max. If any runs of zero + // remain, then we've identified a larger maxiumum zero run. + p := max // number of zeros we still need to shrink by. + k := uint(1) // current minimum length of runs of ones in x. + for { + // Shrink all runs of zeros by p places (except the top zeros). + for p > 0 { + if p <= k { + // Shift p ones down into the top of each run of zeros. + x |= x >> (p & 63) + if x&(x+1) == 0 { // no more zeros (except at the top). + continue outer + } + break + } + // Shift k ones down into the top of each run of zeros. + x |= x >> (k & 63) + if x&(x+1) == 0 { // no more zeros (except at the top). + continue outer + } + p -= k + // We've just doubled the minimum length of 1-runs. + // This allows us to shift farther in the next iteration. + k *= 2 } - // Compute end. - if k == 0 { - end += 8 - } else { - end = uint(sys.LeadingZeros8(k)) + // The length of the lowest-order zero run is an increment to our maximum. + j := uint(sys.TrailingZeros64(^x)) // count contiguous trailing ones + x >>= j & 63 // remove trailing ones + j = uint(sys.TrailingZeros64(x)) // count contiguous trailing zeros + x >>= j & 63 // remove zeros + max += j // we have a new maximum! + if x&(x+1) == 0 { // no more zeros (except at the top). + continue outer } + p = j // remove j more zeros from each zero run. } } - return packPallocSum(start, max, end) + return packPallocSum(start, max, cur) } // find searches for npages contiguous free pages in pallocBits and returns diff --git a/src/runtime/mpallocbits_test.go b/src/runtime/mpallocbits_test.go index 71a29f3b3a..42268a1698 100644 --- a/src/runtime/mpallocbits_test.go +++ b/src/runtime/mpallocbits_test.go @@ -101,7 +101,7 @@ func invertPallocBits(b *PallocBits) { // Ensures two packed summaries are identical, and reports a detailed description // of the difference if they're not. -func checkPallocSum(t *testing.T, got, want PallocSum) { +func checkPallocSum(t testing.TB, got, want PallocSum) { if got.Start() != want.Start() { t.Errorf("inconsistent start: got %d, want %d", got.Start(), want.Start()) } @@ -297,17 +297,29 @@ func TestPallocBitsSummarize(t *testing.T) { // Benchmarks how quickly we can summarize a PallocBits. func BenchmarkPallocBitsSummarize(b *testing.B) { - buf0 := new(PallocBits) - buf1 := new(PallocBits) - for i := 0; i < len(buf1); i++ { - buf1[i] = ^uint64(0) - } - bufa := new(PallocBits) - for i := 0; i < len(bufa); i++ { - bufa[i] = 0xaa - } - for _, buf := range []*PallocBits{buf0, buf1, bufa} { - b.Run(fmt.Sprintf("Unpacked%02X", buf[0]), func(b *testing.B) { + patterns := []uint64{ + 0, + ^uint64(0), + 0xaa, + 0xaaaaaaaaaaaaaaaa, + 0x80000000aaaaaaaa, + 0xaaaaaaaa00000001, + 0xbbbbbbbbbbbbbbbb, + 0x80000000bbbbbbbb, + 0xbbbbbbbb00000001, + 0xcccccccccccccccc, + 0x4444444444444444, + 0x4040404040404040, + 0x4000400040004000, + 0x1000404044ccaaff, + } + for _, p := range patterns { + buf := new(PallocBits) + for i := 0; i < len(buf); i++ { + buf[i] = p + } + b.Run(fmt.Sprintf("Unpacked%02X", p), func(b *testing.B) { + checkPallocSum(b, buf.Summarize(), SummarizeSlow(buf)) for i := 0; i < b.N; i++ { buf.Summarize() } -- cgit v1.3-5-g9baa From 8b8f926fc3f0e8f002d0a8e97aab9500e4db83a7 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 6 Jul 2020 20:46:31 -0700 Subject: runtime: bit parallel implementation of findBitRange64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use a bit-parallel implementation of findBitRange64. It uses a repeated shift-'N-and technique to erase all the free marks that are too small for the allocation. Also some small improvements to find1. name old time/op new time/op delta FindBitRange64/Pattern00Size2-16 4.19ns ± 0% 2.26ns ± 0% -46.04% (p=0.000 n=10+8) FindBitRange64/Pattern00Size8-16 4.19ns ± 0% 2.12ns ± 0% -49.35% (p=0.000 n=9+10) FindBitRange64/Pattern00Size32-16 4.20ns ± 0% 2.12ns ± 0% -49.49% (p=0.000 n=10+8) FindBitRange64/PatternFFFFFFFFFFFFFFFFSize2-16 2.13ns ± 0% 2.27ns ± 0% +6.28% (p=0.000 n=10+10) FindBitRange64/PatternFFFFFFFFFFFFFFFFSize8-16 2.13ns ± 0% 4.46ns ± 0% +109.39% (p=0.000 n=10+9) FindBitRange64/PatternFFFFFFFFFFFFFFFFSize32-16 2.13ns ± 1% 5.58ns ± 0% +162.37% (p=0.000 n=10+9) FindBitRange64/PatternAASize2-16 22.2ns ± 0% 2.3ns ± 0% -89.82% (p=0.000 n=9+8) FindBitRange64/PatternAASize8-16 22.2ns ± 0% 2.1ns ± 1% -90.41% (p=0.000 n=9+10) FindBitRange64/PatternAASize32-16 22.2ns ± 0% 2.1ns ± 1% -90.43% (p=0.000 n=10+10) FindBitRange64/PatternAAAAAAAAAAAAAAAASize2-16 156ns ± 1% 2ns ± 0% -98.54% (p=0.000 n=10+10) FindBitRange64/PatternAAAAAAAAAAAAAAAASize8-16 155ns ± 1% 2ns ± 0% -98.63% (p=0.000 n=10+8) FindBitRange64/PatternAAAAAAAAAAAAAAAASize32-16 155ns ± 0% 2ns ± 1% -98.63% (p=0.000 n=8+10) FindBitRange64/Pattern80000000AAAAAAAASize2-16 81.2ns ± 0% 2.3ns ± 1% -97.21% (p=0.000 n=10+10) FindBitRange64/Pattern80000000AAAAAAAASize8-16 81.1ns ± 0% 2.1ns ± 0% -97.39% (p=0.000 n=10+9) FindBitRange64/Pattern80000000AAAAAAAASize32-16 81.1ns ± 0% 2.1ns ± 0% -97.38% (p=0.000 n=10+10) FindBitRange64/PatternAAAAAAAA00000001Size2-16 76.8ns ± 1% 2.3ns ± 0% -97.05% (p=0.000 n=10+10) FindBitRange64/PatternAAAAAAAA00000001Size8-16 76.6ns ± 0% 2.1ns ± 0% -97.23% (p=0.000 n=8+10) FindBitRange64/PatternAAAAAAAA00000001Size32-16 76.7ns ± 0% 2.1ns ± 0% -97.23% (p=0.000 n=9+9) FindBitRange64/PatternBBBBBBBBBBBBBBBBSize2-16 2.13ns ± 0% 2.27ns ± 0% +6.57% (p=0.000 n=8+8) FindBitRange64/PatternBBBBBBBBBBBBBBBBSize8-16 76.7ns ± 0% 2.9ns ± 0% -96.20% (p=0.000 n=9+10) FindBitRange64/PatternBBBBBBBBBBBBBBBBSize32-16 76.7ns ± 0% 2.9ns ± 0% -96.20% (p=0.000 n=10+10) FindBitRange64/Pattern80000000BBBBBBBBSize2-16 2.12ns ± 0% 2.27ns ± 1% +6.74% (p=0.000 n=10+10) FindBitRange64/Pattern80000000BBBBBBBBSize8-16 44.8ns ± 0% 2.9ns ± 0% -93.49% (p=0.000 n=9+10) FindBitRange64/Pattern80000000BBBBBBBBSize32-16 44.9ns ± 0% 2.9ns ± 0% -93.49% (p=0.000 n=10+8) FindBitRange64/PatternBBBBBBBB00000001Size2-16 4.20ns ± 1% 2.27ns ± 1% -46.02% (p=0.000 n=10+10) FindBitRange64/PatternBBBBBBBB00000001Size8-16 44.9ns ± 0% 2.9ns ± 1% -93.51% (p=0.000 n=10+9) FindBitRange64/PatternBBBBBBBB00000001Size32-16 44.9ns ± 0% 2.9ns ± 0% -93.51% (p=0.000 n=10+9) FindBitRange64/PatternCCCCCCCCCCCCCCCCSize2-16 4.19ns ± 0% 2.26ns ± 0% -46.10% (p=0.000 n=10+10) FindBitRange64/PatternCCCCCCCCCCCCCCCCSize8-16 76.5ns ± 0% 2.9ns ± 0% -96.19% (p=0.000 n=8+7) FindBitRange64/PatternCCCCCCCCCCCCCCCCSize32-16 76.5ns ± 0% 2.9ns ± 0% -96.19% (p=0.000 n=10+8) FindBitRange64/Pattern4444444444444444Size2-16 76.4ns ± 0% 2.3ns ± 0% -97.04% (p=0.000 n=8+10) FindBitRange64/Pattern4444444444444444Size8-16 76.5ns ± 0% 2.1ns ± 0% -97.23% (p=0.000 n=9+10) FindBitRange64/Pattern4444444444444444Size32-16 76.5ns ± 0% 2.1ns ± 0% -97.23% (p=0.000 n=8+10) FindBitRange64/Pattern4040404040404040Size2-16 40.3ns ± 0% 2.3ns ± 0% -94.38% (p=0.000 n=7+10) FindBitRange64/Pattern4040404040404040Size8-16 40.2ns ± 0% 2.1ns ± 0% -94.75% (p=0.000 n=10+10) FindBitRange64/Pattern4040404040404040Size32-16 40.2ns ± 0% 2.1ns ± 0% -94.76% (p=0.000 n=10+6) FindBitRange64/Pattern4000400040004000Size2-16 22.2ns ± 0% 2.2ns ± 0% -89.86% (p=0.001 n=8+9) FindBitRange64/Pattern4000400040004000Size8-16 22.2ns ± 0% 2.1ns ± 0% -90.52% (p=0.000 n=8+10) FindBitRange64/Pattern4000400040004000Size32-16 22.2ns ± 1% 2.1ns ± 0% -90.50% (p=0.000 n=10+10) The cases that slow down aren't really that slow, and those inputs never actually occur (there's a short circuit before the call to findBitRange64 for that case). Change-Id: I50fae62915098032d8ce7fa57ef29eee9deb01ba Reviewed-on: https://go-review.googlesource.com/c/go/+/241279 Reviewed-by: Michael Knyszek --- src/runtime/mpallocbits.go | 41 ++++++++++++++++++++++++++++++----------- src/runtime/mpallocbits_test.go | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 61 insertions(+), 13 deletions(-) diff --git a/src/runtime/mpallocbits.go b/src/runtime/mpallocbits.go index ff79bfbc1a..ff112300c3 100644 --- a/src/runtime/mpallocbits.go +++ b/src/runtime/mpallocbits.go @@ -218,7 +218,7 @@ outer: // find searches for npages contiguous free pages in pallocBits and returns // the index where that run starts, as well as the index of the first free page // it found in the search. searchIdx represents the first known free page and -// where to begin the search from. +// where to begin the next search from. // // If find fails to find any free space, it returns an index of ^uint(0) and // the new searchIdx should be ignored. @@ -239,9 +239,10 @@ func (b *pallocBits) find(npages uintptr, searchIdx uint) (uint, uint) { // // See find for an explanation of the searchIdx parameter. func (b *pallocBits) find1(searchIdx uint) uint { + _ = b[0] // lift nil check out of loop for i := searchIdx / 64; i < uint(len(b)); i++ { x := b[i] - if x == ^uint64(0) { + if ^x == 0 { continue } return i*64 + uint(sys.TrailingZeros64(^x)) @@ -263,18 +264,18 @@ func (b *pallocBits) findSmallN(npages uintptr, searchIdx uint) (uint, uint) { end, newSearchIdx := uint(0), ^uint(0) for i := searchIdx / 64; i < uint(len(b)); i++ { bi := b[i] - if bi == ^uint64(0) { + if ^bi == 0 { end = 0 continue } // First see if we can pack our allocation in the trailing // zeros plus the end of the last 64 bits. - start := uint(sys.TrailingZeros64(bi)) if newSearchIdx == ^uint(0) { // The new searchIdx is going to be at these 64 bits after any // 1s we file, so count trailing 1s. newSearchIdx = i*64 + uint(sys.TrailingZeros64(^bi)) } + start := uint(sys.TrailingZeros64(bi)) if end+start >= uint(npages) { return i*64 - end, newSearchIdx } @@ -369,15 +370,33 @@ func (b *pallocBits) pages64(i uint) uint64 { // findBitRange64 returns the bit index of the first set of // n consecutive 1 bits. If no consecutive set of 1 bits of // size n may be found in c, then it returns an integer >= 64. +// n must be > 0. func findBitRange64(c uint64, n uint) uint { - i := uint(0) - cont := uint(sys.TrailingZeros64(^c)) - for cont < n && i < 64 { - i += cont - i += uint(sys.TrailingZeros64(c >> i)) - cont = uint(sys.TrailingZeros64(^(c >> i))) + // This implementation is based on shrinking the length of + // runs of contiguous 1 bits. We remove the top n-1 1 bits + // from each run of 1s, then look for the first remaining 1 bit. + p := n - 1 // number of 1s we want to remove. + k := uint(1) // current minimum width of runs of 0 in c. + for p > 0 { + if p <= k { + // Shift p 0s down into the top of each run of 1s. + c &= c >> (p & 63) + break + } + // Shift k 0s down into the top of each run of 1s. + c &= c >> (k & 63) + if c == 0 { + return 64 + } + p -= k + // We've just doubled the minimum length of 0-runs. + // This allows us to shift farther in the next iteration. + k *= 2 } - return i + // Find first remaining 1. + // Since we shrunk from the top down, the first 1 is in + // its correct original position. + return uint(sys.TrailingZeros64(c)) } // pallocData encapsulates pallocBits and a bitmap for diff --git a/src/runtime/mpallocbits_test.go b/src/runtime/mpallocbits_test.go index 42268a1698..5095e24220 100644 --- a/src/runtime/mpallocbits_test.go +++ b/src/runtime/mpallocbits_test.go @@ -504,10 +504,9 @@ func TestFindBitRange64(t *testing.T) { t.Errorf("case (%016x, %d): got %d, want %d", x, n, i, result) } } - for i := uint(0); i <= 64; i++ { + for i := uint(1); i <= 64; i++ { check(^uint64(0), i, 0) } - check(0, 0, 0) for i := uint(1); i <= 64; i++ { check(0, i, ^uint(0)) } @@ -520,3 +519,33 @@ func TestFindBitRange64(t *testing.T) { check(0xffff03ff0107ffff, 16, 0) check(0x0fff03ff01079fff, 16, ^uint(0)) } + +func BenchmarkFindBitRange64(b *testing.B) { + patterns := []uint64{ + 0, + ^uint64(0), + 0xaa, + 0xaaaaaaaaaaaaaaaa, + 0x80000000aaaaaaaa, + 0xaaaaaaaa00000001, + 0xbbbbbbbbbbbbbbbb, + 0x80000000bbbbbbbb, + 0xbbbbbbbb00000001, + 0xcccccccccccccccc, + 0x4444444444444444, + 0x4040404040404040, + 0x4000400040004000, + } + sizes := []uint{ + 2, 8, 32, + } + for _, pattern := range patterns { + for _, size := range sizes { + b.Run(fmt.Sprintf("Pattern%02XSize%d", pattern, size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + FindBitRange64(pattern, size) + } + }) + } + } +} -- cgit v1.3-5-g9baa From ef9c8a38ad177fa7f48dfaad5d0e27f39a03529d Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 7 Jul 2020 09:32:12 -0700 Subject: cmd/compile: don't rewrite (CMP (AND x y) 0) to TEST if AND has other uses If the AND has other uses, we end up saving an argument to the AND in another register, so we can use it for the TEST. No point in doing that. Change-Id: I73444a6aeddd6f55e2328ce04d77c3e6cf4a83e0 Reviewed-on: https://go-review.googlesource.com/c/go/+/241280 Run-TryBot: Keith Randall Reviewed-by: Josh Bleecher Snyder TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/ssa/gen/AMD64.rules | 16 ++-- src/cmd/compile/internal/ssa/rewriteAMD64.go | 128 ++++++++++++++++++++------- test/codegen/logic.go | 24 +++++ 3 files changed, 128 insertions(+), 40 deletions(-) create mode 100644 test/codegen/logic.go diff --git a/src/cmd/compile/internal/ssa/gen/AMD64.rules b/src/cmd/compile/internal/ssa/gen/AMD64.rules index 9967c7b030..5111ef79d3 100644 --- a/src/cmd/compile/internal/ssa/gen/AMD64.rules +++ b/src/cmd/compile/internal/ssa/gen/AMD64.rules @@ -1463,14 +1463,14 @@ (MULQconst [c] (NEGQ x)) && c != -(1<<31) -> (MULQconst [-c] x) // checking AND against 0. -(CMPQconst (ANDQ x y) [0]) -> (TESTQ x y) -(CMPLconst (ANDL x y) [0]) -> (TESTL x y) -(CMPWconst (ANDL x y) [0]) -> (TESTW x y) -(CMPBconst (ANDL x y) [0]) -> (TESTB x y) -(CMPQconst (ANDQconst [c] x) [0]) -> (TESTQconst [c] x) -(CMPLconst (ANDLconst [c] x) [0]) -> (TESTLconst [c] x) -(CMPWconst (ANDLconst [c] x) [0]) -> (TESTWconst [int64(int16(c))] x) -(CMPBconst (ANDLconst [c] x) [0]) -> (TESTBconst [int64(int8(c))] x) +(CMPQconst a:(ANDQ x y) [0]) && a.Uses == 1 -> (TESTQ x y) +(CMPLconst a:(ANDL x y) [0]) && a.Uses == 1 -> (TESTL x y) +(CMPWconst a:(ANDL x y) [0]) && a.Uses == 1 -> (TESTW x y) +(CMPBconst a:(ANDL x y) [0]) && a.Uses == 1 -> (TESTB x y) +(CMPQconst a:(ANDQconst [c] x) [0]) && a.Uses == 1 -> (TESTQconst [c] x) +(CMPLconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 -> (TESTLconst [c] x) +(CMPWconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 -> (TESTWconst [int64(int16(c))] x) +(CMPBconst a:(ANDLconst [c] x) [0]) && a.Uses == 1 -> (TESTBconst [int64(int8(c))] x) // Convert TESTx to TESTxconst if possible. (TESTQ (MOVQconst [c]) x) && is32Bit(c) -> (TESTQconst [c] x) diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index 20eab05e9c..cda9df56f4 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -6924,26 +6924,42 @@ func rewriteValueAMD64_OpAMD64CMPBconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPBconst (ANDL x y) [0]) + // match: (CMPBconst a:(ANDL x y) [0]) + // cond: a.Uses == 1 // result: (TESTB x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDL { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDL { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTB) v.AddArg2(x, y) return true } - // match: (CMPBconst (ANDLconst [c] x) [0]) + // match: (CMPBconst a:(ANDLconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTBconst [int64(int8(c))] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDLconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDLconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTBconst) v.AuxInt = int64(int8(c)) v.AddArg(x) @@ -7309,26 +7325,42 @@ func rewriteValueAMD64_OpAMD64CMPLconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPLconst (ANDL x y) [0]) + // match: (CMPLconst a:(ANDL x y) [0]) + // cond: a.Uses == 1 // result: (TESTL x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDL { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDL { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTL) v.AddArg2(x, y) return true } - // match: (CMPLconst (ANDLconst [c] x) [0]) + // match: (CMPLconst a:(ANDLconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTLconst [c] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDLconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDLconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTLconst) v.AuxInt = c v.AddArg(x) @@ -7874,26 +7906,42 @@ func rewriteValueAMD64_OpAMD64CMPQconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPQconst (ANDQ x y) [0]) + // match: (CMPQconst a:(ANDQ x y) [0]) + // cond: a.Uses == 1 // result: (TESTQ x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDQ { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDQ { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTQ) v.AddArg2(x, y) return true } - // match: (CMPQconst (ANDQconst [c] x) [0]) + // match: (CMPQconst a:(ANDQconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTQconst [c] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDQconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDQconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTQconst) v.AuxInt = c v.AddArg(x) @@ -8244,26 +8292,42 @@ func rewriteValueAMD64_OpAMD64CMPWconst(v *Value) bool { v.reset(OpAMD64FlagLT_ULT) return true } - // match: (CMPWconst (ANDL x y) [0]) + // match: (CMPWconst a:(ANDL x y) [0]) + // cond: a.Uses == 1 // result: (TESTW x y) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDL { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDL { + break + } + y := a.Args[1] + x := a.Args[0] + if !(a.Uses == 1) { break } - y := v_0.Args[1] - x := v_0.Args[0] v.reset(OpAMD64TESTW) v.AddArg2(x, y) return true } - // match: (CMPWconst (ANDLconst [c] x) [0]) + // match: (CMPWconst a:(ANDLconst [c] x) [0]) + // cond: a.Uses == 1 // result: (TESTWconst [int64(int16(c))] x) for { - if v.AuxInt != 0 || v_0.Op != OpAMD64ANDLconst { + if v.AuxInt != 0 { + break + } + a := v_0 + if a.Op != OpAMD64ANDLconst { + break + } + c := a.AuxInt + x := a.Args[0] + if !(a.Uses == 1) { break } - c := v_0.AuxInt - x := v_0.Args[0] v.reset(OpAMD64TESTWconst) v.AuxInt = int64(int16(c)) v.AddArg(x) diff --git a/test/codegen/logic.go b/test/codegen/logic.go new file mode 100644 index 0000000000..9afdfd760f --- /dev/null +++ b/test/codegen/logic.go @@ -0,0 +1,24 @@ +// asmcheck + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package codegen + +var gx, gy int + +// Test to make sure that (CMPQ (ANDQ x y) [0]) does not get rewritten to +// (TESTQ x y) if the ANDQ has other uses. If that rewrite happens, then one +// of the args of the ANDQ needs to be saved so it can be used as the arg to TESTQ. +func andWithUse(x, y int) int { + // Load x,y into registers, so those MOVQ will not appear at the z := x&y line. + gx, gy = x, y + // amd64:-"MOVQ" + z := x & y + if z == 0 { + return 77 + } + // use z by returning it + return z +} -- cgit v1.3-5-g9baa From 77a11c05d6a6f766c75f804ea9b8796f9a9f85a3 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 17 Jun 2020 18:22:38 -0700 Subject: reflect: remove depth from deepequal recursion We aren't using it for anything. The visited map will terminate any recursion for us. Change-Id: I36e6bd8e34952123c2ed46323067e42928ec7168 Reviewed-on: https://go-review.googlesource.com/c/go/+/238759 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Emmanuel Odeke --- src/reflect/deepequal.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/reflect/deepequal.go b/src/reflect/deepequal.go index 8a2bf8b09e..be66464129 100644 --- a/src/reflect/deepequal.go +++ b/src/reflect/deepequal.go @@ -21,7 +21,7 @@ type visit struct { // Tests for deep equality using reflected types. The map argument tracks // comparisons that have already been seen, which allows short circuiting on // recursive types. -func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { +func deepValueEqual(v1, v2 Value, visited map[visit]bool) bool { if !v1.IsValid() || !v2.IsValid() { return v1.IsValid() == v2.IsValid() } @@ -29,8 +29,6 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { return false } - // if depth > 10 { panic("deepValueEqual") } // for debugging - // We want to avoid putting more in the visited map than we need to. // For any possible reference cycle that might be encountered, // hard(v1, v2) needs to return true for at least one of the types in the cycle, @@ -79,7 +77,7 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { switch v1.Kind() { case Array: for i := 0; i < v1.Len(); i++ { - if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) { + if !deepValueEqual(v1.Index(i), v2.Index(i), visited) { return false } } @@ -95,7 +93,7 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { return true } for i := 0; i < v1.Len(); i++ { - if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) { + if !deepValueEqual(v1.Index(i), v2.Index(i), visited) { return false } } @@ -104,15 +102,15 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { if v1.IsNil() || v2.IsNil() { return v1.IsNil() == v2.IsNil() } - return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1) + return deepValueEqual(v1.Elem(), v2.Elem(), visited) case Ptr: if v1.Pointer() == v2.Pointer() { return true } - return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1) + return deepValueEqual(v1.Elem(), v2.Elem(), visited) case Struct: for i, n := 0, v1.NumField(); i < n; i++ { - if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) { + if !deepValueEqual(v1.Field(i), v2.Field(i), visited) { return false } } @@ -130,7 +128,7 @@ func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool { for _, k := range v1.MapKeys() { val1 := v1.MapIndex(k) val2 := v2.MapIndex(k) - if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(val1, val2, visited, depth+1) { + if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(val1, val2, visited) { return false } } @@ -207,5 +205,5 @@ func DeepEqual(x, y interface{}) bool { if v1.Type() != v2.Type() { return false } - return deepValueEqual(v1, v2, make(map[visit]bool), 0) + return deepValueEqual(v1, v2, make(map[visit]bool)) } -- cgit v1.3-5-g9baa From db4cda2ec0955854c8ff556ac19ec5e67d48d090 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Mon, 17 Aug 2020 12:25:49 -0700 Subject: testing/iotest: correct ErrReader signature and remove exported error Corrects ErrReader's signature to what was accepted in the approved proposal, and also removes an exported ErrIO which wasn't part of the proposal and is unnecessary. The new signature allows users to customize their own errors. While here, started examples, with ErrReader leading the way. Updates #38781 Change-Id: Ia7f84721f11061343cfef8b1adc2b7b69bc3f43c Reviewed-on: https://go-review.googlesource.com/c/go/+/248898 Run-TryBot: Emmanuel Odeke Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/testing/iotest/example_test.go | 22 ++++++++++++++++++++++ src/testing/iotest/logger_test.go | 4 ++-- src/testing/iotest/reader.go | 17 ++++++++--------- src/testing/iotest/reader_test.go | 27 +++++++++++++++++++++------ 4 files changed, 53 insertions(+), 17 deletions(-) create mode 100644 src/testing/iotest/example_test.go diff --git a/src/testing/iotest/example_test.go b/src/testing/iotest/example_test.go new file mode 100644 index 0000000000..10f6bd38f7 --- /dev/null +++ b/src/testing/iotest/example_test.go @@ -0,0 +1,22 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package iotest_test + +import ( + "errors" + "fmt" + "testing/iotest" +) + +func ExampleErrReader() { + // A reader that always returns a custom error. + r := iotest.ErrReader(errors.New("custom error")) + n, err := r.Read(nil) + fmt.Printf("n: %d\nerr: %q\n", n, err) + + // Output: + // n: 0 + // err: "custom error" +} diff --git a/src/testing/iotest/logger_test.go b/src/testing/iotest/logger_test.go index 575f37e05c..fec4467cc6 100644 --- a/src/testing/iotest/logger_test.go +++ b/src/testing/iotest/logger_test.go @@ -138,14 +138,14 @@ func TestReadLogger_errorOnRead(t *testing.T) { data := []byte("Hello, World!") p := make([]byte, len(data)) - lr := ErrReader() + lr := ErrReader(errors.New("io failure")) rl := NewReadLogger("read", lr) n, err := rl.Read(p) if err == nil { t.Fatalf("Unexpectedly succeeded to read: %v", err) } - wantLogWithHex := fmt.Sprintf("lr: read %x: %v\n", p[:n], "io") + wantLogWithHex := fmt.Sprintf("lr: read %x: io failure\n", p[:n]) if g, w := lOut.String(), wantLogWithHex; g != w { t.Errorf("ReadLogger mismatch\n\tgot: %q\n\twant: %q", g, w) } diff --git a/src/testing/iotest/reader.go b/src/testing/iotest/reader.go index b18e912f27..bc2f72a911 100644 --- a/src/testing/iotest/reader.go +++ b/src/testing/iotest/reader.go @@ -88,16 +88,15 @@ func (r *timeoutReader) Read(p []byte) (int, error) { return r.r.Read(p) } -// ErrIO is a fake IO error. -var ErrIO = errors.New("io") - -// ErrReader returns a fake error every time it is read from. -func ErrReader() io.Reader { - return errReader(0) +// ErrReader returns an io.Reader that returns 0, err from all Read calls. +func ErrReader(err error) io.Reader { + return &alwaysErrReader{err: err} } -type errReader int +type alwaysErrReader struct { + err error +} -func (r errReader) Read(p []byte) (int, error) { - return 0, ErrIO +func (aer *alwaysErrReader) Read(p []byte) (int, error) { + return 0, aer.err } diff --git a/src/testing/iotest/reader_test.go b/src/testing/iotest/reader_test.go index ccba22ee29..6004e841e5 100644 --- a/src/testing/iotest/reader_test.go +++ b/src/testing/iotest/reader_test.go @@ -6,6 +6,7 @@ package iotest import ( "bytes" + "errors" "io" "testing" ) @@ -226,11 +227,25 @@ func TestDataErrReader_emptyReader(t *testing.T) { } func TestErrReader(t *testing.T) { - n, err := ErrReader().Read([]byte{}) - if err != ErrIO { - t.Errorf("ErrReader.Read(any) should have returned ErrIO, returned %v", err) - } - if n != 0 { - t.Errorf("ErrReader.Read(any) should have read 0 bytes, read %v", n) + cases := []struct { + name string + err error + }{ + {"nil error", nil}, + {"non-nil error", errors.New("io failure")}, + {"io.EOF", io.EOF}, + } + + for _, tt := range cases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + n, err := ErrReader(tt.err).Read(nil) + if err != tt.err { + t.Fatalf("Error mismatch\nGot: %v\nWant: %v", err, tt.err) + } + if n != 0 { + t.Fatalf("Byte count mismatch: got %d want 0", n) + } + }) } } -- cgit v1.3-5-g9baa From 7fbd8c75c6c57e713069a3a405e5cde26cfae090 Mon Sep 17 00:00:00 2001 From: lihaowei Date: Fri, 14 Aug 2020 10:35:46 +0000 Subject: all: fix spelling mistakes Change-Id: I7d512281d8442d306594b57b5deaecd132b5ea9e GitHub-Last-Rev: 251e1d6857516b21fd71f654133f81f23ffec654 GitHub-Pull-Request: golang/go#40793 Reviewed-on: https://go-review.googlesource.com/c/go/+/248441 Reviewed-by: Dave Cheney --- src/bufio/bufio.go | 2 +- src/net/http/client.go | 2 +- src/runtime/mheap.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bufio/bufio.go b/src/bufio/bufio.go index 7cbd5424ea..6baf9b9e40 100644 --- a/src/bufio/bufio.go +++ b/src/bufio/bufio.go @@ -425,7 +425,7 @@ func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) { // of bytes in the combined first two elements, error). // The complete result is equal to // `bytes.Join(append(fullBuffers, finalFragment), nil)`, which has a -// length of `totalLen`. The result is strucured in this way to allow callers +// length of `totalLen`. The result is structured in this way to allow callers // to minimize allocations and copies. func (b *Reader) collectFragments(delim byte) (fullBuffers [][]byte, finalFragment []byte, totalLen int, err error) { var frag []byte diff --git a/src/net/http/client.go b/src/net/http/client.go index 3860d97d8f..6ca0d2e6cf 100644 --- a/src/net/http/client.go +++ b/src/net/http/client.go @@ -321,7 +321,7 @@ func knownRoundTripperImpl(rt RoundTripper, req *Request) bool { return true } // There's a very minor chance of a false positive with this. - // Insted of detecting our golang.org/x/net/http2.Transport, + // Instead of detecting our golang.org/x/net/http2.Transport, // it might detect a Transport type in a different http2 // package. But I know of none, and the only problem would be // some temporarily leaked goroutines if the transport didn't diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index cb586171c4..1a57bcd66e 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -42,7 +42,7 @@ const ( // roughly 100µs. // // Must be a multiple of the pageInUse bitmap element size and - // must also evenly divid pagesPerArena. + // must also evenly divide pagesPerArena. pagesPerReclaimerChunk = 512 ) -- cgit v1.3-5-g9baa From 06337823ef4d9c57b1b01f9b97a348a47277a9df Mon Sep 17 00:00:00 2001 From: Junchen Li Date: Fri, 10 Jul 2020 11:39:23 +0800 Subject: cmd/compile: optimize unsigned comparisons to 0/1 on arm64 For an unsigned integer, it's useful to convert its order test with 0/1 to its equality test with 0. We can save a comparison instruction that followed by a conditional branch on arm64 since it supports compare-with-zero-and-branch instructions. For example, if x > 0 { ... } else { ... } the original version: CMP $0, R0 BLS 9 the optimized version: CBZ R0, 8 Updates #21439 Change-Id: Id1de6f865f6aa72c5d45b29f7894818857288425 Reviewed-on: https://go-review.googlesource.com/c/go/+/246857 Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/gen/ARM64.rules | 10 ++ src/cmd/compile/internal/ssa/rewriteARM64.go | 204 +++++++++++++++++++++++++++ test/codegen/comparisons.go | 32 +++++ 3 files changed, 246 insertions(+) diff --git a/src/cmd/compile/internal/ssa/gen/ARM64.rules b/src/cmd/compile/internal/ssa/gen/ARM64.rules index 442d769fdd..27959d01fc 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/gen/ARM64.rules @@ -279,6 +279,16 @@ (Less32F x y) => (LessThanF (FCMPS x y)) (Less64F x y) => (LessThanF (FCMPD x y)) +// For an unsigned integer x, the following rules are useful when combining branch +// 0 < x => x != 0 +// x <= 0 => x == 0 +// x < 1 => x == 0 +// 1 <= x => x != 0 +(Less(8U|16U|32U|64U) zero:(MOVDconst [0]) x) => (Neq(8|16|32|64) zero x) +(Leq(8U|16U|32U|64U) x zero:(MOVDconst [0])) => (Eq(8|16|32|64) x zero) +(Less(8U|16U|32U|64U) x (MOVDconst [1])) => (Eq(8|16|32|64) x (MOVDconst [0])) +(Leq(8U|16U|32U|64U) (MOVDconst [1]) x) => (Neq(8|16|32|64) (MOVDconst [0]) x) + (Less8U x y) => (LessThanU (CMPW (ZeroExt8to32 x) (ZeroExt8to32 y))) (Less16U x y) => (LessThanU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y))) (Less32U x y) => (LessThanU (CMPW x y)) diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go index 8e48b33628..023d9908c2 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM64.go +++ b/src/cmd/compile/internal/ssa/rewriteARM64.go @@ -21976,6 +21976,31 @@ func rewriteValueARM64_OpLeq16U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Leq16U x zero:(MOVDconst [0])) + // result: (Eq16 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq16) + v.AddArg2(x, zero) + return true + } + // match: (Leq16U (MOVDconst [1]) x) + // result: (Neq16 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq16) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq16U x y) // result: (LessEqualU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y))) for { @@ -22028,6 +22053,32 @@ func rewriteValueARM64_OpLeq32U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Leq32U x zero:(MOVDconst [0])) + // result: (Eq32 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq32) + v.AddArg2(x, zero) + return true + } + // match: (Leq32U (MOVDconst [1]) x) + // result: (Neq32 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq32) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq32U x y) // result: (LessEqualU (CMPW x y)) for { @@ -22076,6 +22127,32 @@ func rewriteValueARM64_OpLeq64U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Leq64U x zero:(MOVDconst [0])) + // result: (Eq64 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq64) + v.AddArg2(x, zero) + return true + } + // match: (Leq64U (MOVDconst [1]) x) + // result: (Neq64 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq64) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq64U x y) // result: (LessEqualU (CMP x y)) for { @@ -22114,6 +22191,31 @@ func rewriteValueARM64_OpLeq8U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Leq8U x zero:(MOVDconst [0])) + // result: (Eq8 x zero) + for { + x := v_0 + zero := v_1 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + v.reset(OpEq8) + v.AddArg2(x, zero) + return true + } + // match: (Leq8U (MOVDconst [1]) x) + // result: (Neq8 (MOVDconst [0]) x) + for { + if v_0.Op != OpARM64MOVDconst || auxIntToInt64(v_0.AuxInt) != 1 { + break + } + x := v_1 + v.reset(OpNeq8) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(v0, x) + return true + } // match: (Leq8U x y) // result: (LessEqualU (CMPW (ZeroExt8to32 x) (ZeroExt8to32 y))) for { @@ -22156,6 +22258,31 @@ func rewriteValueARM64_OpLess16U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Less16U zero:(MOVDconst [0]) x) + // result: (Neq16 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq16) + v.AddArg2(zero, x) + return true + } + // match: (Less16U x (MOVDconst [1])) + // result: (Eq16 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq16) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less16U x y) // result: (LessThanU (CMPW (ZeroExt16to32 x) (ZeroExt16to32 y))) for { @@ -22208,6 +22335,32 @@ func rewriteValueARM64_OpLess32U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Less32U zero:(MOVDconst [0]) x) + // result: (Neq32 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq32) + v.AddArg2(zero, x) + return true + } + // match: (Less32U x (MOVDconst [1])) + // result: (Eq32 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq32) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less32U x y) // result: (LessThanU (CMPW x y)) for { @@ -22256,6 +22409,32 @@ func rewriteValueARM64_OpLess64U(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] b := v.Block + typ := &b.Func.Config.Types + // match: (Less64U zero:(MOVDconst [0]) x) + // result: (Neq64 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq64) + v.AddArg2(zero, x) + return true + } + // match: (Less64U x (MOVDconst [1])) + // result: (Eq64 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq64) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less64U x y) // result: (LessThanU (CMP x y)) for { @@ -22294,6 +22473,31 @@ func rewriteValueARM64_OpLess8U(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (Less8U zero:(MOVDconst [0]) x) + // result: (Neq8 zero x) + for { + zero := v_0 + if zero.Op != OpARM64MOVDconst || auxIntToInt64(zero.AuxInt) != 0 { + break + } + x := v_1 + v.reset(OpNeq8) + v.AddArg2(zero, x) + return true + } + // match: (Less8U x (MOVDconst [1])) + // result: (Eq8 x (MOVDconst [0])) + for { + x := v_0 + if v_1.Op != OpARM64MOVDconst || auxIntToInt64(v_1.AuxInt) != 1 { + break + } + v.reset(OpEq8) + v0 := b.NewValue0(v.Pos, OpARM64MOVDconst, typ.UInt64) + v0.AuxInt = int64ToAuxInt(0) + v.AddArg2(x, v0) + return true + } // match: (Less8U x y) // result: (LessThanU (CMPW (ZeroExt8to32 x) (ZeroExt8to32 y))) for { diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index f3c15538a8..3c2dcb7eba 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -424,3 +424,35 @@ func UintGeqZero(a uint8, b uint16, c uint32, d uint64) int { } return 0 } + +func UintGtZero(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBZW`, `CBNZW`, `CBNZ`, -`(CMPW|CMP|BLS|BHI)` + if a > 0 || b > 0 || c > 0 || d > 0 { + return 1 + } + return 0 +} + +func UintLeqZero(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBNZW`, `CBZW`, `CBZ`, -`(CMPW|CMP|BHI|BLS)` + if a <= 0 || b <= 0 || c <= 0 || d <= 0 { + return 1 + } + return 0 +} + +func UintLtOne(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBNZW`, `CBZW`, `CBZW`, `CBZ`, -`(CMPW|CMP|BHS|BLO)` + if a < 1 || b < 1 || c < 1 || d < 1 { + return 1 + } + return 0 +} + +func UintGeqOne(a uint8, b uint16, c uint32, d uint64) int { + // arm64: `CBZW`, `CBNZW`, `CBNZ`, -`(CMPW|CMP|BLO|BHS)` + if a >= 1 || b >= 1 || c >= 1 || d >= 1 { + return 1 + } + return 0 +} -- cgit v1.3-5-g9baa From 6dad1b4c9bb63cad432bcddb7aec0d36f03f8341 Mon Sep 17 00:00:00 2001 From: alex-semenyuk Date: Sun, 17 May 2020 16:05:05 +0000 Subject: cmd/cgo: close file Change-Id: Ia70edc8ba22e31e498fe07946db41882804bd39f GitHub-Last-Rev: 280232e879965b34fae3aa02e6215eff5bd7be46 GitHub-Pull-Request: golang/go#39120 Reviewed-on: https://go-review.googlesource.com/c/go/+/234317 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/cgo/out.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cmd/cgo/out.go b/src/cmd/cgo/out.go index 4064f0ae41..50d2811f1b 100644 --- a/src/cmd/cgo/out.go +++ b/src/cmd/cgo/out.go @@ -243,6 +243,7 @@ func (p *Package) writeDefs() { if err != nil { fatalf("%s", err) } + defer fgcch.Close() _, err = io.Copy(fexp, fgcch) if err != nil { fatalf("%s", err) -- cgit v1.3-5-g9baa From 613388315e29d4e906805e602602500ca1e7e334 Mon Sep 17 00:00:00 2001 From: Cholerae Hu Date: Mon, 11 May 2020 11:18:57 +0800 Subject: runtime: reduce critical path in injectglist Change-Id: Ia3fb30ac9add39c803f11f69d967c6604fdeacf8 Reviewed-on: https://go-review.googlesource.com/c/go/+/233217 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/proc.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 9a358cd529..5e38b3194c 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2575,15 +2575,20 @@ func injectglist(glist *gList) { return } - lock(&sched.lock) - npidle := int(sched.npidle) + npidle := int(atomic.Load(&sched.npidle)) + var globq gQueue var n int for n = 0; n < npidle && !q.empty(); n++ { - globrunqput(q.pop()) + g := q.pop() + globq.pushBack(g) + } + if n > 0 { + lock(&sched.lock) + globrunqputbatch(&globq, int32(n)) + unlock(&sched.lock) + startIdle(n) + qsize -= n } - unlock(&sched.lock) - startIdle(n) - qsize -= n if !q.empty() { runqputbatch(pp, &q, qsize) -- cgit v1.3-5-g9baa From a745171e6b30394b661a040d04e8807b4bd0c7da Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 17 Aug 2020 21:59:07 -0700 Subject: cmd/compile: fix SSA type comparison A typo in the conversion code caused comparisons of SSA types to report CMPeq when they were not in fact equal. Fixes #40837 Change-Id: I0627eee51d524a585908b34a4590bc533c8415fc Reviewed-on: https://go-review.googlesource.com/c/go/+/248781 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Cuong Manh Le Reviewed-by: Emmanuel Odeke --- src/cmd/compile/internal/types/type.go | 3 ++- src/cmd/compile/internal/types/type_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 src/cmd/compile/internal/types/type_test.go diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index 3b7b31c5d6..91b54b43d4 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -131,6 +131,7 @@ type Type struct { // TPTR: Ptr // TARRAY: *Array // TSLICE: Slice + // TSSA: string Extra interface{} // Width is the width of this Type in bytes. @@ -1026,7 +1027,7 @@ func (t *Type) cmp(x *Type) Cmp { case TSSA: tname := t.Extra.(string) - xname := t.Extra.(string) + xname := x.Extra.(string) // desire fast sorting, not pretty sorting. if len(tname) == len(xname) { if tname == xname { diff --git a/src/cmd/compile/internal/types/type_test.go b/src/cmd/compile/internal/types/type_test.go new file mode 100644 index 0000000000..fe3f380b21 --- /dev/null +++ b/src/cmd/compile/internal/types/type_test.go @@ -0,0 +1,28 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package types_test + +import ( + "cmd/compile/internal/types" + "testing" +) + +func TestSSACompare(t *testing.T) { + a := []*types.Type{ + types.TypeInvalid, + types.TypeMem, + types.TypeFlags, + types.TypeVoid, + types.TypeInt128, + } + for _, x := range a { + for _, y := range a { + c := x.Compare(y) + if x == y && c != types.CMPeq || x != y && c == types.CMPeq { + t.Errorf("%s compare %s == %d\n", x.Extra, y.Extra, c) + } + } + } +} -- cgit v1.3-5-g9baa From 4b94e881611890c6d6cbda6f542a94ab08de17e0 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Thu, 7 May 2020 20:48:37 +0200 Subject: net: simplify error return in *RawConn test helpers No need to check operr before returning. Change-Id: I64b849f7c102de01180823e3f0123d438ce7608a Reviewed-on: https://go-review.googlesource.com/c/go/+/232797 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/net/rawconn_unix_test.go | 20 ++++---------------- src/net/rawconn_windows_test.go | 20 ++++---------------- 2 files changed, 8 insertions(+), 32 deletions(-) diff --git a/src/net/rawconn_unix_test.go b/src/net/rawconn_unix_test.go index a71b6f3041..0194ba67c8 100644 --- a/src/net/rawconn_unix_test.go +++ b/src/net/rawconn_unix_test.go @@ -24,10 +24,7 @@ func readRawConn(c syscall.RawConn, b []byte) (int, error) { if err != nil { return n, err } - if operr != nil { - return n, operr - } - return n, nil + return n, operr } func writeRawConn(c syscall.RawConn, b []byte) error { @@ -42,10 +39,7 @@ func writeRawConn(c syscall.RawConn, b []byte) error { if err != nil { return err } - if operr != nil { - return operr - } - return nil + return operr } func controlRawConn(c syscall.RawConn, addr Addr) error { @@ -87,10 +81,7 @@ func controlRawConn(c syscall.RawConn, addr Addr) error { if err := c.Control(fn); err != nil { return err } - if operr != nil { - return operr - } - return nil + return operr } func controlOnConnSetup(network string, address string, c syscall.RawConn) error { @@ -120,8 +111,5 @@ func controlOnConnSetup(network string, address string, c syscall.RawConn) error if err := c.Control(fn); err != nil { return err } - if operr != nil { - return operr - } - return nil + return operr } diff --git a/src/net/rawconn_windows_test.go b/src/net/rawconn_windows_test.go index 2774c97e5c..5febf08f77 100644 --- a/src/net/rawconn_windows_test.go +++ b/src/net/rawconn_windows_test.go @@ -26,10 +26,7 @@ func readRawConn(c syscall.RawConn, b []byte) (int, error) { if err != nil { return n, err } - if operr != nil { - return n, operr - } - return n, nil + return n, operr } func writeRawConn(c syscall.RawConn, b []byte) error { @@ -45,10 +42,7 @@ func writeRawConn(c syscall.RawConn, b []byte) error { if err != nil { return err } - if operr != nil { - return operr - } - return nil + return operr } func controlRawConn(c syscall.RawConn, addr Addr) error { @@ -92,10 +86,7 @@ func controlRawConn(c syscall.RawConn, addr Addr) error { if err := c.Control(fn); err != nil { return err } - if operr != nil { - return operr - } - return nil + return operr } func controlOnConnSetup(network string, address string, c syscall.RawConn) error { @@ -121,8 +112,5 @@ func controlOnConnSetup(network string, address string, c syscall.RawConn) error if err := c.Control(fn); err != nil { return err } - if operr != nil { - return operr - } - return nil + return operr } -- cgit v1.3-5-g9baa From 4149493443f09c14d9f0fad7030704ed57149b55 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Mon, 17 Aug 2020 16:32:33 +0200 Subject: runtime: move startupRandomData declaration to os_linux.go startupRandomData is only used in sysauxv and getRandomData on linux, thus move it closer to where it is used. Also adjust its godoc comment. Change-Id: Ice51d579ec33436adbfdf247caf4ba00bae865e0 Reviewed-on: https://go-review.googlesource.com/c/go/+/248761 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/os_linux.go | 4 ++++ src/runtime/runtime2.go | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index 22931b4d5c..9702920bcf 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -249,6 +249,10 @@ func sysargs(argc int32, argv **byte) { sysauxv(buf[:]) } +// startupRandomData holds random bytes initialized at startup. These come from +// the ELF AT_RANDOM auxiliary vector. +var startupRandomData []byte + func sysauxv(auxv []uintptr) int { var i int for ; auxv[i] != _AT_NULL; i += 2 { diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 0bddcaa789..959878400d 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -845,10 +845,6 @@ type forcegcstate struct { idle uint32 } -// startup_random_data holds random bytes initialized at startup. These come from -// the ELF AT_RANDOM auxiliary vector (vdso_linux_amd64.go or os_linux_386.go). -var startupRandomData []byte - // extendRandom extends the random numbers in r[:n] to the whole slice r. // Treats n<0 as n==0. func extendRandom(r []byte, n int) { -- cgit v1.3-5-g9baa From 6e876f19857a8fbd259571080f7f91bc03276559 Mon Sep 17 00:00:00 2001 From: Michael Munday Date: Thu, 4 Jun 2020 10:55:01 -0700 Subject: cmd/compile: clean up and optimize s390x multiplication rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Some of the existing optimizations aren't triggered because they are handled by the generic rules so this CL removes them. Also some constraints were copied without much thought from the amd64 rules and they don't make sense on s390x, so we remove those constraints. Finally, add a 'multiply by the sum of two powers of two' optimization. This makes sense on s390x as shifts are low latency and can also sometimes be optimized further (especially if we add support for RISBG instructions). name old time/op new time/op delta IntMulByConst/3-8 1.70ns ±11% 1.10ns ± 5% -35.26% (p=0.000 n=10+10) IntMulByConst/5-8 1.64ns ± 7% 1.10ns ± 4% -32.94% (p=0.000 n=10+9) IntMulByConst/12-8 1.65ns ± 6% 1.20ns ± 4% -27.16% (p=0.000 n=10+9) IntMulByConst/120-8 1.66ns ± 4% 1.22ns ±13% -26.43% (p=0.000 n=10+10) IntMulByConst/-120-8 1.65ns ± 7% 1.19ns ± 4% -28.06% (p=0.000 n=9+10) IntMulByConst/65537-8 0.86ns ± 9% 1.12ns ±12% +30.41% (p=0.000 n=10+10) IntMulByConst/65538-8 1.65ns ± 5% 1.23ns ± 5% -25.11% (p=0.000 n=10+10) Change-Id: Ib196e6bff1e97febfd266134d0a2b2a62897989f Reviewed-on: https://go-review.googlesource.com/c/go/+/248937 Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/ssa/gen/S390X.rules | 51 +++-- src/cmd/compile/internal/ssa/rewriteS390X.go | 268 ++++++++++++++----------- src/cmd/compile/internal/test/mulconst_test.go | 242 ++++++++++++++++++++++ test/codegen/arithmetic.go | 6 + 4 files changed, 441 insertions(+), 126 deletions(-) create mode 100644 src/cmd/compile/internal/test/mulconst_test.go diff --git a/src/cmd/compile/internal/ssa/gen/S390X.rules b/src/cmd/compile/internal/ssa/gen/S390X.rules index d3234c1a00..5e4c436ca1 100644 --- a/src/cmd/compile/internal/ssa/gen/S390X.rules +++ b/src/cmd/compile/internal/ssa/gen/S390X.rules @@ -716,20 +716,40 @@ (ANDWconst [0xFF] x) => (MOVBZreg x) (ANDWconst [0xFFFF] x) => (MOVHZreg x) -// strength reduction -(MULLDconst [-1] x) => (NEG x) -(MULLDconst [0] _) => (MOVDconst [0]) -(MULLDconst [1] x) => x -(MULLDconst [c] x) && isPowerOfTwo(c) -> (SLDconst [log2(c)] x) -(MULLDconst [c] x) && isPowerOfTwo(c+1) && c >= 15 -> (SUB (SLDconst [log2(c+1)] x) x) -(MULLDconst [c] x) && isPowerOfTwo(c-1) && c >= 17 -> (ADD (SLDconst [log2(c-1)] x) x) - -(MULLWconst [-1] x) => (NEGW x) -(MULLWconst [0] _) => (MOVDconst [0]) -(MULLWconst [1] x) => x -(MULLWconst [c] x) && isPowerOfTwo(c) -> (SLWconst [log2(c)] x) -(MULLWconst [c] x) && isPowerOfTwo(c+1) && c >= 15 -> (SUBW (SLWconst [log2(c+1)] x) x) -(MULLWconst [c] x) && isPowerOfTwo(c-1) && c >= 17 -> (ADDW (SLWconst [log2(c-1)] x) x) +// Strength reduce multiplication to the sum (or difference) of two powers of two. +// +// Examples: +// 5x -> 4x + 1x +// 10x -> 8x + 2x +// 120x -> 128x - 8x +// -120x -> 8x - 128x +// +// We know that the rightmost bit of any positive value, once isolated, must either +// be a power of 2 (because it is a single bit) or 0 (if the original value is 0). +// In all of these rules we use a rightmost bit calculation to determine one operand +// for the addition or subtraction. We then just need to calculate if the other +// operand is a valid power of 2 before we can match the rule. +// +// Notes: +// - the generic rules have already matched single powers of two so we ignore them here +// - isPowerOfTwo32 asserts that its argument is greater than 0 +// - c&(c-1) = clear rightmost bit +// - c&^(c-1) = isolate rightmost bit + +// c = 2ˣ + 2ʸ => c - 2ˣ = 2ʸ +(MULL(D|W)const x [c]) && isPowerOfTwo32(c&(c-1)) + => ((ADD|ADDW) (SL(D|W)const x [int8(log32(c&(c-1)))]) + (SL(D|W)const x [int8(log32(c&^(c-1)))])) + +// c = 2ʸ - 2ˣ => c + 2ˣ = 2ʸ +(MULL(D|W)const x [c]) && isPowerOfTwo32(c+(c&^(c-1))) + => ((SUB|SUBW) (SL(D|W)const x [int8(log32(c+(c&^(c-1))))]) + (SL(D|W)const x [int8(log32(c&^(c-1)))])) + +// c = 2ˣ - 2ʸ => -c + 2ˣ = 2ʸ +(MULL(D|W)const x [c]) && isPowerOfTwo32(-c+(-c&^(-c-1))) + => ((SUB|SUBW) (SL(D|W)const x [int8(log32(-c&^(-c-1)))]) + (SL(D|W)const x [int8(log32(-c+(-c&^(-c-1))))])) // Fold ADD into MOVDaddr. Odd offsets from SB shouldn't be folded (LARL can't handle them). (ADDconst [c] (MOVDaddr [d] {s} x:(SB))) && ((c+d)&1 == 0) && is32Bit(c+d) -> (MOVDaddr [c+d] {s} x) @@ -1133,6 +1153,9 @@ (XORconst [0] x) => x (XORWconst [c] x) && int32(c)==0 => x +// Shifts by zero (may be inserted during multiplication strength reduction). +((SLD|SLW|SRD|SRW|SRAD|SRAW)const x [0]) => x + // Convert constant subtracts to constant adds. (SUBconst [c] x) && c != -(1<<31) => (ADDconst [-c] x) (SUBWconst [c] x) -> (ADDWconst [int64(int32(-c))] x) diff --git a/src/cmd/compile/internal/ssa/rewriteS390X.go b/src/cmd/compile/internal/ssa/rewriteS390X.go index dc9b143562..536f8db320 100644 --- a/src/cmd/compile/internal/ssa/rewriteS390X.go +++ b/src/cmd/compile/internal/ssa/rewriteS390X.go @@ -732,8 +732,12 @@ func rewriteValueS390X(v *Value) bool { return rewriteValueS390X_OpS390XRLLG(v) case OpS390XSLD: return rewriteValueS390X_OpS390XSLD(v) + case OpS390XSLDconst: + return rewriteValueS390X_OpS390XSLDconst(v) case OpS390XSLW: return rewriteValueS390X_OpS390XSLW(v) + case OpS390XSLWconst: + return rewriteValueS390X_OpS390XSLWconst(v) case OpS390XSRAD: return rewriteValueS390X_OpS390XSRAD(v) case OpS390XSRADconst: @@ -748,6 +752,8 @@ func rewriteValueS390X(v *Value) bool { return rewriteValueS390X_OpS390XSRDconst(v) case OpS390XSRW: return rewriteValueS390X_OpS390XSRW(v) + case OpS390XSRWconst: + return rewriteValueS390X_OpS390XSRWconst(v) case OpS390XSTM2: return rewriteValueS390X_OpS390XSTM2(v) case OpS390XSTMG2: @@ -13853,81 +13859,64 @@ func rewriteValueS390X_OpS390XMULLD(v *Value) bool { func rewriteValueS390X_OpS390XMULLDconst(v *Value) bool { v_0 := v.Args[0] b := v.Block - // match: (MULLDconst [-1] x) - // result: (NEG x) + // match: (MULLDconst x [c]) + // cond: isPowerOfTwo32(c&(c-1)) + // result: (ADD (SLDconst x [int8(log32(c&(c-1)))]) (SLDconst x [int8(log32(c&^(c-1)))])) for { - if auxIntToInt32(v.AuxInt) != -1 { - break - } - x := v_0 - v.reset(OpS390XNEG) - v.AddArg(x) - return true - } - // match: (MULLDconst [0] _) - // result: (MOVDconst [0]) - for { - if auxIntToInt32(v.AuxInt) != 0 { - break - } - v.reset(OpS390XMOVDconst) - v.AuxInt = int64ToAuxInt(0) - return true - } - // match: (MULLDconst [1] x) - // result: x - for { - if auxIntToInt32(v.AuxInt) != 1 { - break - } - x := v_0 - v.copyOf(x) - return true - } - // match: (MULLDconst [c] x) - // cond: isPowerOfTwo(c) - // result: (SLDconst [log2(c)] x) - for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c)) { + if !(isPowerOfTwo32(c & (c - 1))) { break } - v.reset(OpS390XSLDconst) - v.AuxInt = log2(c) - v.AddArg(x) + v.reset(OpS390XADD) + v0 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c & (c - 1)))) + v0.AddArg(x) + v1 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLDconst [c] x) - // cond: isPowerOfTwo(c+1) && c >= 15 - // result: (SUB (SLDconst [log2(c+1)] x) x) + // match: (MULLDconst x [c]) + // cond: isPowerOfTwo32(c+(c&^(c-1))) + // result: (SUB (SLDconst x [int8(log32(c+(c&^(c-1))))]) (SLDconst x [int8(log32(c&^(c-1)))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c+1) && c >= 15) { + if !(isPowerOfTwo32(c + (c &^ (c - 1)))) { break } v.reset(OpS390XSUB) - v0 := b.NewValue0(v.Pos, OpS390XSLDconst, v.Type) - v0.AuxInt = log2(c + 1) + v0 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c + (c &^ (c - 1))))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLDconst [c] x) - // cond: isPowerOfTwo(c-1) && c >= 17 - // result: (ADD (SLDconst [log2(c-1)] x) x) + // match: (MULLDconst x [c]) + // cond: isPowerOfTwo32(-c+(-c&^(-c-1))) + // result: (SUB (SLDconst x [int8(log32(-c&^(-c-1)))]) (SLDconst x [int8(log32(-c+(-c&^(-c-1))))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c-1) && c >= 17) { + if !(isPowerOfTwo32(-c + (-c &^ (-c - 1)))) { break } - v.reset(OpS390XADD) - v0 := b.NewValue0(v.Pos, OpS390XSLDconst, v.Type) - v0.AuxInt = log2(c - 1) + v.reset(OpS390XSUB) + v0 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(-c &^ (-c - 1)))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLDconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(-c + (-c &^ (-c - 1))))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } // match: (MULLDconst [c] (MOVDconst [d])) @@ -14097,81 +14086,64 @@ func rewriteValueS390X_OpS390XMULLW(v *Value) bool { func rewriteValueS390X_OpS390XMULLWconst(v *Value) bool { v_0 := v.Args[0] b := v.Block - // match: (MULLWconst [-1] x) - // result: (NEGW x) - for { - if auxIntToInt32(v.AuxInt) != -1 { - break - } - x := v_0 - v.reset(OpS390XNEGW) - v.AddArg(x) - return true - } - // match: (MULLWconst [0] _) - // result: (MOVDconst [0]) - for { - if auxIntToInt32(v.AuxInt) != 0 { - break - } - v.reset(OpS390XMOVDconst) - v.AuxInt = int64ToAuxInt(0) - return true - } - // match: (MULLWconst [1] x) - // result: x + // match: (MULLWconst x [c]) + // cond: isPowerOfTwo32(c&(c-1)) + // result: (ADDW (SLWconst x [int8(log32(c&(c-1)))]) (SLWconst x [int8(log32(c&^(c-1)))])) for { - if auxIntToInt32(v.AuxInt) != 1 { - break - } - x := v_0 - v.copyOf(x) - return true - } - // match: (MULLWconst [c] x) - // cond: isPowerOfTwo(c) - // result: (SLWconst [log2(c)] x) - for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c)) { + if !(isPowerOfTwo32(c & (c - 1))) { break } - v.reset(OpS390XSLWconst) - v.AuxInt = log2(c) - v.AddArg(x) + v.reset(OpS390XADDW) + v0 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c & (c - 1)))) + v0.AddArg(x) + v1 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLWconst [c] x) - // cond: isPowerOfTwo(c+1) && c >= 15 - // result: (SUBW (SLWconst [log2(c+1)] x) x) + // match: (MULLWconst x [c]) + // cond: isPowerOfTwo32(c+(c&^(c-1))) + // result: (SUBW (SLWconst x [int8(log32(c+(c&^(c-1))))]) (SLWconst x [int8(log32(c&^(c-1)))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c+1) && c >= 15) { + if !(isPowerOfTwo32(c + (c &^ (c - 1)))) { break } v.reset(OpS390XSUBW) - v0 := b.NewValue0(v.Pos, OpS390XSLWconst, v.Type) - v0.AuxInt = log2(c + 1) + v0 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(c + (c &^ (c - 1))))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(c &^ (c - 1)))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } - // match: (MULLWconst [c] x) - // cond: isPowerOfTwo(c-1) && c >= 17 - // result: (ADDW (SLWconst [log2(c-1)] x) x) + // match: (MULLWconst x [c]) + // cond: isPowerOfTwo32(-c+(-c&^(-c-1))) + // result: (SUBW (SLWconst x [int8(log32(-c&^(-c-1)))]) (SLWconst x [int8(log32(-c+(-c&^(-c-1))))])) for { - c := v.AuxInt + t := v.Type + c := auxIntToInt32(v.AuxInt) x := v_0 - if !(isPowerOfTwo(c-1) && c >= 17) { + if !(isPowerOfTwo32(-c + (-c &^ (-c - 1)))) { break } - v.reset(OpS390XADDW) - v0 := b.NewValue0(v.Pos, OpS390XSLWconst, v.Type) - v0.AuxInt = log2(c - 1) + v.reset(OpS390XSUBW) + v0 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v0.AuxInt = int8ToAuxInt(int8(log32(-c &^ (-c - 1)))) v0.AddArg(x) - v.AddArg2(v0, x) + v1 := b.NewValue0(v.Pos, OpS390XSLWconst, t) + v1.AuxInt = int8ToAuxInt(int8(log32(-c + (-c &^ (-c - 1))))) + v1.AddArg(x) + v.AddArg2(v0, v1) return true } // match: (MULLWconst [c] (MOVDconst [d])) @@ -16826,6 +16798,20 @@ func rewriteValueS390X_OpS390XSLD(v *Value) bool { } return false } +func rewriteValueS390X_OpS390XSLDconst(v *Value) bool { + v_0 := v.Args[0] + // match: (SLDconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } + return false +} func rewriteValueS390X_OpS390XSLW(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -16960,6 +16946,20 @@ func rewriteValueS390X_OpS390XSLW(v *Value) bool { } return false } +func rewriteValueS390X_OpS390XSLWconst(v *Value) bool { + v_0 := v.Args[0] + // match: (SLWconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } + return false +} func rewriteValueS390X_OpS390XSRAD(v *Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] @@ -17096,6 +17096,16 @@ func rewriteValueS390X_OpS390XSRAD(v *Value) bool { } func rewriteValueS390X_OpS390XSRADconst(v *Value) bool { v_0 := v.Args[0] + // match: (SRADconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } // match: (SRADconst [c] (MOVDconst [d])) // result: (MOVDconst [d>>uint64(c)]) for { @@ -17246,6 +17256,16 @@ func rewriteValueS390X_OpS390XSRAW(v *Value) bool { } func rewriteValueS390X_OpS390XSRAWconst(v *Value) bool { v_0 := v.Args[0] + // match: (SRAWconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } // match: (SRAWconst [c] (MOVDconst [d])) // result: (MOVDconst [int64(int32(d))>>uint64(c)]) for { @@ -17416,6 +17436,16 @@ func rewriteValueS390X_OpS390XSRDconst(v *Value) bool { v.AddArg(v0) return true } + // match: (SRDconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } return false } func rewriteValueS390X_OpS390XSRW(v *Value) bool { @@ -17552,6 +17582,20 @@ func rewriteValueS390X_OpS390XSRW(v *Value) bool { } return false } +func rewriteValueS390X_OpS390XSRWconst(v *Value) bool { + v_0 := v.Args[0] + // match: (SRWconst x [0]) + // result: x + for { + if auxIntToInt8(v.AuxInt) != 0 { + break + } + x := v_0 + v.copyOf(x) + return true + } + return false +} func rewriteValueS390X_OpS390XSTM2(v *Value) bool { v_3 := v.Args[3] v_2 := v.Args[2] diff --git a/src/cmd/compile/internal/test/mulconst_test.go b/src/cmd/compile/internal/test/mulconst_test.go new file mode 100644 index 0000000000..314cab32de --- /dev/null +++ b/src/cmd/compile/internal/test/mulconst_test.go @@ -0,0 +1,242 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package test + +import "testing" + +// Benchmark multiplication of an integer by various constants. +// +// The comment above each sub-benchmark provides an example of how the +// target multiplication operation might be implemented using shift +// (multiplication by a power of 2), addition and subtraction +// operations. It is platform-dependent whether these transformations +// are actually applied. + +var ( + mulSinkI32 int32 + mulSinkI64 int64 + mulSinkU32 uint32 + mulSinkU64 uint64 +) + +func BenchmarkMulconstI32(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkI32 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkI32 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkI32 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkI32 = x + }) + // -120x = 8x - 120x + b.Run("-120", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= -120 + } + mulSinkI32 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkI32 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := int32(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkI32 = x + }) +} + +func BenchmarkMulconstI64(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkI64 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkI64 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkI64 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkI64 = x + }) + // -120x = 8x - 120x + b.Run("-120", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= -120 + } + mulSinkI64 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkI64 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := int64(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkI64 = x + }) +} + +func BenchmarkMulconstU32(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkU32 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkU32 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkU32 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkU32 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkU32 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := uint32(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkU32 = x + }) +} + +func BenchmarkMulconstU64(b *testing.B) { + // 3x = 2x + x + b.Run("3", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 3 + } + mulSinkU64 = x + }) + // 5x = 4x + x + b.Run("5", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 5 + } + mulSinkU64 = x + }) + // 12x = 8x + 4x + b.Run("12", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 12 + } + mulSinkU64 = x + }) + // 120x = 128x - 8x + b.Run("120", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 120 + } + mulSinkU64 = x + }) + // 65537x = 65536x + x + b.Run("65537", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 65537 + } + mulSinkU64 = x + }) + // 65538x = 65536x + 2x + b.Run("65538", func(b *testing.B) { + x := uint64(1) + for i := 0; i < b.N; i++ { + x *= 65538 + } + mulSinkU64 = x + }) +} diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index 8f25974376..9f30ec8ce4 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -71,9 +71,15 @@ func Mul_96(n int) int { // 386:`SHLL\t[$]5`,`LEAL\t\(.*\)\(.*\*2\),`,-`IMULL` // arm64:`LSL\t[$]5`,`ADD\sR[0-9]+<<1,\sR[0-9]+`,-`MUL` // arm:`SLL\t[$]5`,`ADD\sR[0-9]+<<1,\sR[0-9]+`,-`MUL` + // s390x:`SLD\t[$]5`,`SLD\t[$]6`,-`MULLD` return n * 96 } +func Mul_n120(n int) int { + // s390x:`SLD\t[$]3`,`SLD\t[$]7`,-`MULLD` + return n * -120 +} + func MulMemSrc(a []uint32, b []float32) { // 386:`IMULL\s4\([A-Z]+\),\s[A-Z]+` a[0] *= a[1] -- cgit v1.3-5-g9baa From c12d9ed87596688aeeeb61111e408f6a176aa287 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 18 Aug 2020 10:54:11 -0400 Subject: cmd/go: revert 3 CLs affecting par.Work, context propagation, tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts the following changes: • cmd/go: add tracing for querying and downloading from the proxy CL 242786, commit 1a3558341860357c2400e37773e5076bb3a51628 • cmd/go: do context propagation for tracing downloads CL 248327, commit c0cf190d226cc3defb71d17c01d0b45bf49a8a85 • cmd/go/internal: remove some users of par.Work CL 248326, commit f30044a03bc7cf107dbec03c02fb6d0072878252 Reason for revert: broke linux 386 and amd64 longtest builders. The problem started with CL 248326, but CL 248327 and CL 242786 are reverted as well due to conflicts. Updates #38714. Fixes #40861. Change-Id: I68496b4e5a27e47a42183553c3a645b288edac83 Reviewed-on: https://go-review.googlesource.com/c/go/+/249017 Run-TryBot: Dmitri Shuralyov Reviewed-by: Bryan C. Mills Reviewed-by: Carlos Amedee TryBot-Result: Gobot Gobot --- src/cmd/go/internal/get/get.go | 4 +- src/cmd/go/internal/list/list.go | 5 +- src/cmd/go/internal/load/pkg.go | 40 ++++++------ src/cmd/go/internal/load/test.go | 6 +- src/cmd/go/internal/modcmd/download.go | 75 ++++++++++------------ src/cmd/go/internal/modcmd/graph.go | 19 +++--- src/cmd/go/internal/modcmd/init.go | 2 +- src/cmd/go/internal/modcmd/tidy.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modcmd/why.go | 6 +- src/cmd/go/internal/modconv/convert.go | 59 ++++++++--------- src/cmd/go/internal/modconv/convert_test.go | 5 +- src/cmd/go/internal/modfetch/fetch.go | 24 ++----- .../internal/modfetch/zip_sum_test/zip_sum_test.go | 3 +- src/cmd/go/internal/modget/get.go | 59 +++++++---------- src/cmd/go/internal/modload/build.go | 17 +++-- src/cmd/go/internal/modload/import.go | 9 ++- src/cmd/go/internal/modload/import_test.go | 5 +- src/cmd/go/internal/modload/init.go | 9 ++- src/cmd/go/internal/modload/list.go | 49 ++++++-------- src/cmd/go/internal/modload/load.go | 39 ++++++----- src/cmd/go/internal/modload/mvs.go | 5 +- src/cmd/go/internal/modload/query.go | 55 ++++++---------- src/cmd/go/internal/modload/query_test.go | 5 +- src/cmd/go/internal/modload/search.go | 5 +- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- 28 files changed, 216 insertions(+), 299 deletions(-) diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index e5bacadaa3..ef43602aca 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index e68c39f392..7303e6c866 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -20,7 +20,6 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" - "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -350,7 +349,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, + "module": modload.ModuleInfo, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -390,7 +389,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. + modload.InitMod() // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 71fd9b5538..32c2ba7912 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -42,10 +42,10 @@ var ( ModBinDir func() string // return effective bin directory ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths + ModImportPaths func(args []string) []*search.Match // expand import paths ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files + ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files ModDirImportPath func(string) string // return effective import path for directory ) @@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -605,11 +605,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -657,7 +657,7 @@ func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent * // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(ctx, path, stk, importPos, bp, err) + p.load(path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1800,7 +1800,7 @@ func (p *Package) load(ctx context.Context, path string, stk *ImportStack, impor if path == "C" { continue } - p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(ctx context.Context, roots []*Package) []*Package { +func TestPackageList(roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2089,7 +2089,7 @@ func TestPackageList(ctx context.Context, roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2112,7 +2112,7 @@ func TestPackageList(ctx context.Context, roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) + p := LoadImport(path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } @@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(ctx, patterns)} + return []*Package{GoFilesPackage(patterns)} } } } - matches := ImportPaths(ctx, patterns) + matches := ImportPaths(patterns) var ( pkgs []*Package stk ImportStack @@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(ctx context.Context, args []string) []*search.Match { +func ImportPaths(args []string) []*search.Match { if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(ctx, args) + return ModImportPaths(args) } return search.ImportPaths(args) } @@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(ctx context.Context, gofiles []string) *Package { +func GoFilesPackage(gofiles []string) *Package { ModInit() for _, f := range gofiles { @@ -2329,7 +2329,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(ctx, gofiles) + ModImportFromFiles(gofiles) } var err error @@ -2345,7 +2345,7 @@ func GoFilesPackage(ctx context.Context, gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) + pkg.load("command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index a0e275095b..6db8a00245 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 13e5cb066b..946e8ed3cf 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -5,15 +5,15 @@ package modcmd import ( - "cmd/go/internal/modfetch" "context" "encoding/json" "os" - "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" + "cmd/go/internal/modfetch" "cmd/go/internal/modload" + "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod(ctx) // to fill Target + modload.InitMod() // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" @@ -102,7 +102,33 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } } - downloadModule := func(m *moduleJSON) { + var mods []*moduleJSON + var work par.Work + listU := false + listVersions := false + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + work.Add(m) + } + + work.Do(10, func(item interface{}) { + m := item.(*moduleJSON) var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -120,53 +146,18 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(ctx, mod) + m.Zip, err = modfetch.DownloadZip(mod) if err != nil { m.Error = err.Error() return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(ctx, mod) + m.Dir, err = modfetch.Download(mod) if err != nil { m.Error = err.Error() return } - } - - var mods []*moduleJSON - listU := false - listVersions := false - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - sem <- token{} - go func() { - downloadModule(m) - <-sem - }() - } - - // Fill semaphore channel to wait for goroutines to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} - } + }) if *downloadJSON { for _, m := range mods { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 6da12b9cab..4853503fd4 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,6 +15,7 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" + "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -58,25 +59,23 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { return m.Path + "@" + m.Version } + // Note: using par.Work only to manage work queue. + // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - seen := map[module.Version]bool{modload.Target: true} - queue := []module.Version{modload.Target} - for len(queue) > 0 { - var m module.Version - m, queue = queue[0], queue[1:] + var work par.Work + work.Add(modload.Target) + work.Do(1, func(item interface{}) { + m := item.(module.Version) list, _ := reqs.Required(m) for _, r := range list { - if !seen[r] { - queue = append(queue, r) - seen[r] = true - } + work.Add(r) out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - } + }) sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index b6cffd332d..95063e62f4 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if strings.Contains(modload.CmdModModule, "@") { base.Fatalf("go mod init: module path must not contain '@'") } - modload.InitMod(ctx) // does all the hard work + modload.InitMod() // does all the hard work modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index c7c53d7c0c..769cd11fe8 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL(ctx) + modload.LoadALL() modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index e5353b5c7f..257d1cd0ef 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor(ctx) + pkgs := modload.LoadVendor() vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index da33fff89e..f400339b25 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) - for _, path := range loadALL(ctx) { + for _, path := range loadALL() { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(ctx, args) // resolve to packages - loadALL(ctx) // rebuild graph, from main module (not from named packages) + matches := modload.ImportPaths(args) // resolve to packages + loadALL() // rebuild graph, from main module (not from named packages) sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modconv/convert.go b/src/cmd/go/internal/modconv/convert.go index d5a0bc21e9..f465a9f395 100644 --- a/src/cmd/go/internal/modconv/convert.go +++ b/src/cmd/go/internal/modconv/convert.go @@ -7,12 +7,13 @@ package modconv import ( "fmt" "os" - "runtime" "sort" "strings" + "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" + "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -41,52 +42,46 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - versions := make([]*module.Version, len(mf.Require)) - replace := make(map[string]*modfile.Replace) + var ( + work par.Work + mu sync.Mutex + need = make(map[string]string) + replace = make(map[string]*modfile.Replace) + ) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) - for i, r := range mf.Require { + for _, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - m = re.New + work.Add(re.New) + continue } - sem <- token{} - go func(i int, m module.Version) { - repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) - return - } - - path := repo.ModulePath() - versions[i].Path = path - versions[i].Version = info.Version - - <-sem - }(i, m) - } - // Fill semaphore channel to wait for all tasks to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} + work.Add(r.Mod) } - need := map[string]string{} - for _, v := range versions { + work.Do(10, func(item interface{}) { + r := item.(module.Version) + repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) + return + } + mu.Lock() + path := repo.ModulePath() // Don't use semver.Max here; need to preserve +incompatible suffix. - if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { - need[v.Path] = v.Version + if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { + need[path] = info.Version } - } - paths := make([]string, 0, len(need)) + mu.Unlock() + }) + + var paths []string for path := range need { paths = append(paths, path) } diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index faa2b4c606..a04a13b14f 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -6,7 +6,6 @@ package modconv import ( "bytes" - "context" "fmt" "internal/testenv" "io/ioutil" @@ -147,8 +146,6 @@ func TestConvertLegacyConfig(t *testing.T) { }, } - ctx := context.Background() - for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -160,7 +157,7 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e29eb0a942..e40158b535 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -7,7 +7,6 @@ package modfetch import ( "archive/zip" "bytes" - "context" "errors" "fmt" "io" @@ -24,7 +23,6 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" - "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -36,7 +34,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(ctx context.Context, mod module.Version) (dir string, err error) { +func Download(mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. @@ -49,7 +47,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(ctx, mod) + dir, err := download(mod) if err != nil { return cached{"", err} } @@ -59,10 +57,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(ctx context.Context, mod module.Version) (dir string, err error) { - ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) - defer span.Done() - +func download(mod module.Version) (dir string, err error) { // If the directory exists, and no .partial file exists, the module has // already been completely extracted. .partial files may be created when a // module zip directory is extracted in place instead of being extracted to a @@ -77,7 +72,7 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(ctx, mod) + zipfile, err := DownloadZip(mod) if err != nil { return "", err } @@ -147,7 +142,6 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { return "", err } - ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) if unzipInPlace { if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { return "", err @@ -177,7 +171,6 @@ func download(ctx context.Context, mod module.Version) (dir string, err error) { return "", err } } - defer span.Done() if !cfg.ModCacheRW { // Make dir read-only only *after* renaming it. @@ -202,7 +195,7 @@ var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { +func DownloadZip(mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -237,7 +230,7 @@ func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err e if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(ctx, mod, zipfile); err != nil { + if err := downloadZip(mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -245,10 +238,7 @@ func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err e return c.zipfile, c.err } -func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { - ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) - defer span.Done() - +func downloadZip(mod module.Version, zipfile string) (err error) { // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. diff --git a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index 82398ebfed..eac9b32fa8 100644 --- a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,7 +16,6 @@ package zip_sum_test import ( - "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -120,7 +119,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(context.Background(), test.m) + zipPath, err := modfetch.DownloadZip(test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index ee9757912b..93a6bb54d5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "runtime" "sort" "strings" "sync" @@ -22,6 +21,7 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/mvs" + "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" @@ -353,7 +353,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if !strings.Contains(path, "...") { m := search.NewMatch(path) if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(ctx, pkgPath) + m = modload.TargetPackages(pkgPath) } if len(m.Pkgs) == 0 { for _, err := range m.Errs { @@ -399,7 +399,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { default: // The argument is a package or module path. if modload.HasModRoot() { - if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { + if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { // The path is in the main module. Nothing to query. if vers != "upgrade" && vers != "patch" { base.Errorf("go get %s: can't request explicit version of path in main module", arg) @@ -491,7 +491,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if q.path == q.m.Path { wg.Add(1) go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil { + if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { base.Errorf("go get: %v", err) } else if !hasPkg { modOnlyMu.Lock() @@ -536,7 +536,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Don't load packages if pkgPatterns is empty. Both // modload.ImportPathsQuiet and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) + matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -725,44 +725,29 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // reported. A map from module paths to queries is returned, which includes // queries and modOnly. func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { + var lookup par.Work + for _, q := range queries { + if cached := cache[q.querySpec]; cached != nil { + *q = *cached + } else { + cache[q.querySpec] = q + lookup.Add(q) + } + } - runQuery := func(q *query) { + lookup.Do(10, func(item interface{}) { + q := item.(*query) if q.vers == "none" { // Wait for downgrade step. q.m = module.Version{Path: q.path, Version: "none"} return } - m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath) + m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) if err != nil { base.Errorf("go get %s: %v", q.arg, err) } q.m = m - } - - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - sem <- token{} - go func(q *query) { - runQuery(q) - <-sem - }(q) - } - } - - // Fill semaphore channel to wait for goroutines to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} - } - - // Add to cache after concurrent section to avoid races... - for _, q := range queries { - cache[q.querySpec] = q - } - + }) base.ExitIfErrors() byPath := make(map[string]*query) @@ -790,7 +775,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer // to determine the underlying module version being requested. // If forceModulePath is set, getQuery must interpret path // as a module path. -func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { +func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { if (prevM.Version != "") != forceModulePath { // We resolve package patterns by calling QueryPattern, which does not // accept a previous version and therefore cannot take it into account for @@ -812,7 +797,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc } } - info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +823,7 @@ func getQuery(ctx context.Context, path, vers string, prevM module.Version, forc // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) + results, err := modload.QueryPattern(path, vers, modload.Allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -994,7 +979,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) + info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index a101681a1f..5f8a2e7e05 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -6,7 +6,6 @@ package modload import ( "bytes" - "context" "encoding/hex" "fmt" "internal/goroot" @@ -58,21 +57,21 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(context.TODO(), m, true) + return moduleInfo(m, true) } -func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { +func ModuleInfo(path string) *modinfo.ModulePublic { if !Enabled() { return nil } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) + return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(ctx, m, true) + return moduleInfo(m, true) } } @@ -85,12 +84,12 @@ func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { +func addUpdate(m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -104,7 +103,7 @@ func addVersions(m *modinfo.ModulePublic) { m.Versions, _ = versions(m.Path) } -func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -133,7 +132,7 @@ func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modi // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { + if q, err := Query(m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 5c51a79124..4d2bc805e2 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -5,7 +5,6 @@ package modload import ( - "context" "errors" "fmt" "go/build" @@ -111,7 +110,7 @@ var _ load.ImportPathError = &AmbiguousImportError{} // Import returns an ImportMissingError as the error. // If Import can identify a module that could be added to supply the package, // the ImportMissingError records that module. -func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { +func Import(path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -166,7 +165,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(ctx, m) + root, isLocal, err := fetch(m) if err != nil { // Report fetch error. // Note that we don't know for sure this module is necessary, @@ -249,7 +248,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(ctx, m) + root, isLocal, err := fetch(m) if err != nil { // Report fetch error as above. return module.Version{}, "", err @@ -286,7 +285,7 @@ func Import(ctx context.Context, path string) (m module.Version, dir string, err fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(ctx, path, "latest", Allowed) + candidates, err := QueryPackage(path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index 47ce89a084..accc60eecd 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -5,7 +5,6 @@ package modload import ( - "context" "internal/testenv" "regexp" "strings" @@ -50,12 +49,10 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() - ctx := context.Background() - for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(ctx, tt.path) + m, dir, err := Import(tt.path) if err == nil { t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index 93027c44c4..fff060e665 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -6,7 +6,6 @@ package modload import ( "bytes" - "context" "encoding/json" "errors" "fmt" @@ -333,7 +332,7 @@ func die() { // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod(ctx context.Context) { +func InitMod() { if len(buildList) > 0 { return } @@ -360,7 +359,7 @@ func InitMod(ctx context.Context) { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -398,7 +397,7 @@ func InitMod(ctx context.Context) { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { +func fixVersion(fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -430,7 +429,7 @@ func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { } } - info, err := Query(ctx, path, vers, "", nil) + info, err := Query(path, vers, "", nil) if err != nil { return "", err } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 7bf4e86c8d..4768516e90 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "os" - "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" + "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" @@ -22,42 +22,31 @@ import ( func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { mods := listModules(ctx, args, listVersions) - - type token struct{} - sem := make(chan token, runtime.GOMAXPROCS(0)) if listU || listVersions { + var work par.Work for _, m := range mods { - add := func(m *modinfo.ModulePublic) { - sem <- token{} - go func() { - if listU { - addUpdate(ctx, m) - } - if listVersions { - addVersions(m) - } - <-sem - }() - } - - add(m) + work.Add(m) if m.Replace != nil { - add(m.Replace) + work.Add(m.Replace) } } + work.Do(10, func(item interface{}) { + m := item.(*modinfo.ModulePublic) + if listU { + addUpdate(m) + } + if listVersions { + addVersions(m) + } + }) } - // Fill semaphore channel to wait for all tasks to finish. - for n := cap(sem); n > 0; n-- { - sem <- token{} - } - return mods } func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { LoadBuildList(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -83,7 +72,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(ctx, path, vers, current, nil) + info, err := Query(path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +81,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) + mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) continue } @@ -117,7 +106,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(ctx, m, true)) + mods = append(mods, moduleInfo(m, true)) } } } @@ -127,9 +116,9 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(ctx, arg, "latest", "", nil) + info, err := Query(arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) + mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 686d491219..8190009b23 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -52,8 +52,8 @@ var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), // on the target platform. Modules may be added to the build list // to satisfy new imports. -func ImportPaths(ctx context.Context, patterns []string) []*search.Match { - matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) +func ImportPaths(patterns []string) []*search.Match { + matches := ImportPathsQuiet(patterns, imports.Tags()) search.WarnUnmatched(matches) return matches } @@ -62,7 +62,7 @@ func ImportPaths(ctx context.Context, patterns []string) []*search.Match { // no matches. It also lets the caller specify a set of build tags to match // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { +func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { updateMatches := func(matches []*search.Match, iterating bool) { for _, m := range matches { switch { @@ -103,7 +103,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, includeStd, buildList) + matchPackages(m, loaded.tags, includeStd, buildList) case m.Pattern() == "all": loaded.testAll = true @@ -111,7 +111,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo // Enumerate the packages in the main module. // We'll load the dependencies as we find them. m.Errs = m.Errs[:0] - matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -129,7 +129,7 @@ func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bo } } - InitMod(ctx) + InitMod() var matches []*search.Match for _, pattern := range search.CleanPatterns(patterns) { @@ -338,8 +338,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(ctx context.Context, gofiles []string) { - InitMod(ctx) +func ImportFromFiles(gofiles []string) { + InitMod() tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -391,7 +391,7 @@ func DirImportPath(dir string) string { func LoadBuildList(ctx context.Context) []module.Version { ctx, span := trace.StartSpan(ctx, "LoadBuildList") defer span.Done() - InitMod(ctx) + InitMod() ReloadBuildList() WriteGoMod() return buildList @@ -409,20 +409,20 @@ func ReloadBuildList() []module.Version { // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. -func LoadALL(ctx context.Context) []string { - return loadAll(ctx, true) +func LoadALL() []string { + return loadAll(true) } // LoadVendor is like LoadALL but only follows test dependencies // for tests in the main module. Tests in dependency modules are // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor(ctx context.Context) []string { - return loadAll(ctx, false) +func LoadVendor() []string { + return loadAll(false) } -func loadAll(ctx context.Context, testAll bool) []string { - InitMod(ctx) +func loadAll(testAll bool) []string { + InitMod() loaded = newLoader(imports.AnyTags()) loaded.isALL = true @@ -430,7 +430,7 @@ func loadAll(ctx context.Context, testAll bool) []string { if !testAll { loaded.testRoots = true } - all := TargetPackages(ctx, "...") + all := TargetPackages("...") loaded.load(func() []string { return all.Pkgs }) checkMultiplePaths() WriteGoMod() @@ -453,13 +453,13 @@ func loadAll(ctx context.Context, testAll bool) []string { // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(ctx context.Context, pattern string) *search.Match { +func TargetPackages(pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. ModRoot() m := search.NewMatch(pattern) - matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } @@ -817,8 +817,7 @@ func (ld *loader) doPkg(item interface{}) { return } - // TODO(matloob): Handle TODO context. This needs to be threaded through Do. - pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) + pkg.mod, pkg.dir, pkg.err = Import(pkg.path) if pkg.dir == "" { return } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 67eb2c2e19..5dd009d31d 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -5,7 +5,6 @@ package modload import ( - "context" "errors" "fmt" "os" @@ -225,7 +224,7 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { // // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { +func fetch(mod module.Version) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -255,6 +254,6 @@ func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, e mod = r } - dir, err = modfetch.Download(ctx, mod) + dir, err = modfetch.Download(mod) return dir, false, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index e82eb1506f..acc886bf21 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -5,7 +5,6 @@ package modload import ( - "context" "errors" "fmt" "os" @@ -19,7 +18,6 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/search" "cmd/go/internal/str" - "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -57,10 +55,10 @@ import ( // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(ctx, proxy, path, query, current, allowed) + info, err = queryProxy(proxy, path, query, current, allowed) return err }) return info, err @@ -77,10 +75,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { - ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) - defer span.Done() - +func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -248,7 +243,7 @@ func queryProxy(ctx context.Context, proxy, path, query, current string, allowed if err != nil { return nil, err } - releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) + releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) if err != nil { return nil, err } @@ -332,7 +327,7 @@ func matchSemverPrefix(p, v string) bool { // 1. versions that do not satisfy the 'ok' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { if !ok(module.Version{Path: path, Version: v}) { @@ -348,7 +343,7 @@ func filterVersions(ctx context.Context, path string, versions []string, ok func // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) if err != nil { return nil, nil, err } @@ -385,12 +380,12 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) } - return QueryPattern(ctx, path, query, allowed) + return QueryPattern(path, query, allowed) } // QueryPattern looks up the module(s) containing at least one package matching @@ -406,10 +401,7 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { - ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) - defer span.Done() - +func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { base := pattern firstError := func(m *search.Match) error { @@ -425,7 +417,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul base = pathpkg.Dir(pattern[:i+3]) match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -477,18 +469,15 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { - ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) - defer span.Done() - + queryModule := func(path string) (r QueryResult, err error) { current := findCurrentVersion(path) r.Mod.Path = path - r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) + r.Rev, err = queryProxy(proxy, path, query, current, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(ctx, r.Mod) + root, isLocal, err := fetch(r.Mod) if err != nil { return r, err } @@ -509,7 +498,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } var err error - results, err = queryPrefixModules(ctx, candidateModules, queryModule) + results, err = queryPrefixModules(candidateModules, queryModule) return err }) @@ -553,10 +542,7 @@ type prefixResult struct { err error } -func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { - ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") - defer span.Done() - +func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -569,9 +555,8 @@ func queryPrefixModules(ctx context.Context, candidateModules []string, queryMod var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { - ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(ctx, p) + r.QueryResult, r.err = queryModule(p) wg.Done() }(p, &results[i]) } @@ -713,8 +698,8 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { - root, isLocal, err := fetch(ctx, m) +func ModuleHasRootPackage(m module.Version) (bool, error) { + root, isLocal, err := fetch(m) if err != nil { return false, err } @@ -722,8 +707,8 @@ func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { - root, _, err := fetch(ctx, m) +func versionHasGoMod(m module.Version) (bool, error) { + root, _, err := fetch(m) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 77080e9b5b..247e4c40d2 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -5,7 +5,6 @@ package modload import ( - "context" "internal/testenv" "io/ioutil" "log" @@ -180,8 +179,6 @@ func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") - ctx := context.Background() - for _, tt := range queryTests { allow := tt.allow if allow == "" { @@ -195,7 +192,7 @@ func TestQuery(t *testing.T) { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) + info, err := Query(tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index a9bee0af4e..c28e7c0c1e 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -5,7 +5,6 @@ package modload import ( - "context" "fmt" "os" "path/filepath" @@ -28,7 +27,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -154,7 +153,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f isLocal = true } else { var err error - root, isLocal, err = fetch(ctx, mod) + root, isLocal, err = fetch(mod) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index 99578b244c..deec5106ff 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -77,7 +77,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(ctx, files) + p = load.GoFilesPackage(files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 3aee6939d2..9cef8cf89c 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -702,7 +702,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(ctx, pkgs) { + for _, p := range load.TestPackageList(pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index d975c36306..681ecd7646 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2900,7 +2900,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { } srcs := []string{src} - p := load.GoFilesPackage(context.TODO(), srcs) + p := load.GoFilesPackage(srcs) if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { return "32", nil -- cgit v1.3-5-g9baa From b58d29741650c7bf10b17f455666e2727e1cdd2e Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Mon, 17 Aug 2020 19:06:19 -0400 Subject: cmd/compile, runtime: mark R12 clobbered for write barrier call on PPC64 When external linking, for large binaries, the external linker may insert a trampoline for the write barrier call, which looks 0000000005a98cc8 <__long_branch_runtime.gcWriteBarrier>: 5a98cc8: 86 01 82 3d addis r12,r2,390 5a98ccc: d8 bd 8c e9 ld r12,-16936(r12) 5a98cd0: a6 03 89 7d mtctr r12 5a98cd4: 20 04 80 4e bctr It clobbers R12 (and CTR, which is never live across a call). As at compile time we don't know whether the binary is big and what link mode will be used, I think we need to mark R12 as clobbered for write barrier call. For extra safety (future-proof) we mark caller-saved register that cannot be used for function arguments, which includes R11, as potentially clobbered as well. Fixes #40851. Change-Id: Iedd901c5072f1127cc59b0a48cfeb4aaec81b519 Reviewed-on: https://go-review.googlesource.com/c/go/+/248917 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements --- src/cmd/compile/internal/ssa/gen/PPC64Ops.go | 4 +-- src/cmd/compile/internal/ssa/opGen.go | 2 +- src/runtime/asm_ppc64x.s | 41 ++++++++++++++-------------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go index f8bc6cb20b..0261dc283b 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go @@ -645,9 +645,9 @@ func init() { {name: "LoweredAtomicOr8", argLength: 3, reg: gpstore, asm: "OR", faultOnNilArg0: true, hasSideEffects: true}, // LoweredWB invokes runtime.gcWriteBarrier. arg0=destptr, arg1=srcptr, arg2=mem, aux=runtime.gcWriteBarrier - // It preserves R0 through R15, g, and its arguments R20 and R21, + // It preserves R0 through R17 (except special registers R1, R2, R11, R12, R13), g, and its arguments R20 and R21, // but may clobber anything else, including R31 (REGTMP). - {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("R20"), buildReg("R21")}, clobbers: (callerSave &^ buildReg("R0 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R20 R21 g")) | buildReg("R31")}, clobberFlags: true, aux: "Sym", symEffect: "None"}, + {name: "LoweredWB", argLength: 3, reg: regInfo{inputs: []regMask{buildReg("R20"), buildReg("R21")}, clobbers: (callerSave &^ buildReg("R0 R3 R4 R5 R6 R7 R8 R9 R10 R14 R15 R16 R17 R20 R21 g")) | buildReg("R31")}, clobberFlags: true, aux: "Sym", symEffect: "None"}, // There are three of these functions so that they can have three different register inputs. // When we check 0 <= c <= cap (A), then 0 <= b <= c (B), then 0 <= a <= b (C), we want the diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 408c855dbd..df2a27368b 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -26889,7 +26889,7 @@ var opcodeTable = [...]opInfo{ {0, 1048576}, // R20 {1, 2097152}, // R21 }, - clobbers: 576460746931503104, // R16 R17 R18 R19 R22 R23 R24 R25 R26 R27 R28 R29 R31 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 + clobbers: 576460746931312640, // R11 R12 R18 R19 R22 R23 R24 R25 R26 R27 R28 R29 R31 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 }, }, { diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 11d2f2f51a..23387a2165 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -916,23 +916,23 @@ TEXT ·checkASM(SB),NOSPLIT,$0-1 // - R20 is the destination of the write // - R21 is the value being written at R20. // It clobbers condition codes. -// It does not clobber R0 through R15, +// It does not clobber R0 through R17 (except special registers), // but may clobber any other register, *including* R31. TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112 // The standard prologue clobbers R31. - // We use R16 and R17 as scratch registers. - MOVD g_m(g), R16 - MOVD m_p(R16), R16 - MOVD (p_wbBuf+wbBuf_next)(R16), R17 + // We use R18 and R19 as scratch registers. + MOVD g_m(g), R18 + MOVD m_p(R18), R18 + MOVD (p_wbBuf+wbBuf_next)(R18), R19 // Increment wbBuf.next position. - ADD $16, R17 - MOVD R17, (p_wbBuf+wbBuf_next)(R16) - MOVD (p_wbBuf+wbBuf_end)(R16), R16 - CMP R16, R17 + ADD $16, R19 + MOVD R19, (p_wbBuf+wbBuf_next)(R18) + MOVD (p_wbBuf+wbBuf_end)(R18), R18 + CMP R18, R19 // Record the write. - MOVD R21, -16(R17) // Record value - MOVD (R20), R16 // TODO: This turns bad writes into bad reads. - MOVD R16, -8(R17) // Record *slot + MOVD R21, -16(R19) // Record value + MOVD (R20), R18 // TODO: This turns bad writes into bad reads. + MOVD R18, -8(R19) // Record *slot // Is the buffer full? (flags set in CMP above) BEQ flush ret: @@ -956,11 +956,12 @@ flush: MOVD R8, (FIXED_FRAME+56)(R1) MOVD R9, (FIXED_FRAME+64)(R1) MOVD R10, (FIXED_FRAME+72)(R1) - MOVD R11, (FIXED_FRAME+80)(R1) - MOVD R12, (FIXED_FRAME+88)(R1) + // R11, R12 may be clobbered by external-linker-inserted trampoline // R13 is REGTLS - MOVD R14, (FIXED_FRAME+96)(R1) - MOVD R15, (FIXED_FRAME+104)(R1) + MOVD R14, (FIXED_FRAME+80)(R1) + MOVD R15, (FIXED_FRAME+88)(R1) + MOVD R16, (FIXED_FRAME+96)(R1) + MOVD R17, (FIXED_FRAME+104)(R1) // This takes arguments R20 and R21. CALL runtime·wbBufFlush(SB) @@ -975,10 +976,10 @@ flush: MOVD (FIXED_FRAME+56)(R1), R8 MOVD (FIXED_FRAME+64)(R1), R9 MOVD (FIXED_FRAME+72)(R1), R10 - MOVD (FIXED_FRAME+80)(R1), R11 - MOVD (FIXED_FRAME+88)(R1), R12 - MOVD (FIXED_FRAME+96)(R1), R14 - MOVD (FIXED_FRAME+104)(R1), R15 + MOVD (FIXED_FRAME+80)(R1), R14 + MOVD (FIXED_FRAME+88)(R1), R15 + MOVD (FIXED_FRAME+96)(R1), R16 + MOVD (FIXED_FRAME+104)(R1), R17 JMP ret // Note: these functions use a special calling convention to save generated code space. -- cgit v1.3-5-g9baa From 20b4987d3e658a1cd93472185d7998f745bd062e Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Wed, 13 May 2020 13:57:36 -0400 Subject: cmd/dist: use GO_TEST_SHORT value more consistently There were two places where the -short flag was added in order to speed up tests when run in short mode, in CL 178399 and CL 177417. It appears viable to re-use the GO_TEST_SHORT value so that -short flag is not used when the tests are executed on a longtest builder, where it is not a goal to skip slow tests for improved performance. Do so, in order to make the testing configurations simpler and more predictable. Factor out the flag name out of the string returned by short, so that it can be used in context of 'go test' which can accept a -short flag, and a test binary which requires the use of a longer -test.short flag. For #39054. For #29252. Change-Id: I52dfbef73cc8307735c52e2ebaa609305fb05933 Reviewed-on: https://go-review.googlesource.com/c/go/+/233898 Run-TryBot: Dmitri Shuralyov TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/dist/test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/cmd/dist/test.go b/src/cmd/dist/test.go index 2dc9459215..a83ae35293 100644 --- a/src/cmd/dist/test.go +++ b/src/cmd/dist/test.go @@ -241,13 +241,15 @@ func (t *tester) shouldRunTest(name string) bool { return false } -// short returns a -short flag to pass to 'go test'. -// It returns "-short", unless the environment variable +// short returns a -short flag value to use with 'go test' +// or a test binary for tests intended to run in short mode. +// It returns "true", unless the environment variable // GO_TEST_SHORT is set to a non-empty, false-ish string. // // This environment variable is meant to be an internal -// detail between the Go build system and cmd/dist -// and is not intended for use by users. +// detail between the Go build system and cmd/dist for +// the purpose of longtest builders, and is not intended +// for use by users. See golang.org/issue/12508. func short() string { if v := os.Getenv("GO_TEST_SHORT"); v != "" { short, err := strconv.ParseBool(v) @@ -255,10 +257,10 @@ func short() string { fatalf("invalid GO_TEST_SHORT %q: %v", v, err) } if !short { - return "-short=false" + return "false" } } - return "-short" + return "true" } // goTest returns the beginning of the go test command line. @@ -266,7 +268,7 @@ func short() string { // defaults as later arguments in the command line. func (t *tester) goTest() []string { return []string{ - "go", "test", short(), "-count=1", t.tags(), t.runFlag(""), + "go", "test", "-short=" + short(), "-count=1", t.tags(), t.runFlag(""), } } @@ -335,7 +337,7 @@ func (t *tester) registerStdTest(pkg string) { } args := []string{ "test", - short(), + "-short=" + short(), t.tags(), t.timeout(timeoutSec), "-gcflags=all=" + gogcflags, @@ -373,7 +375,7 @@ func (t *tester) registerRaceBenchTest(pkg string) { ranGoBench = true args := []string{ "test", - short(), + "-short=" + short(), "-race", t.timeout(1200), // longer timeout for race with benchmarks "-run=^$", // nothing. only benchmarks. @@ -1069,7 +1071,7 @@ func (t *tester) runHostTest(dir, pkg string) error { if err := cmd.Run(); err != nil { return err } - return t.dirCmd(dir, f.Name(), "-test.short").Run() + return t.dirCmd(dir, f.Name(), "-test.short="+short()).Run() } func (t *tester) cgoTest(dt *distTest) error { @@ -1570,7 +1572,7 @@ func (t *tester) prebuiltGoPackageTestBinary() string { func (t *tester) runPrecompiledStdTest(timeout time.Duration) error { bin := t.prebuiltGoPackageTestBinary() fmt.Fprintf(os.Stderr, "# %s: using pre-built %s...\n", stdMatches[0], bin) - cmd := exec.Command(bin, "-test.short", "-test.timeout="+timeout.String()) + cmd := exec.Command(bin, "-test.short="+short(), "-test.timeout="+timeout.String()) cmd.Dir = filepath.Dir(bin) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr -- cgit v1.3-5-g9baa From 93eeb819cab491d4e429b7aa85a864a045979a18 Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Mon, 8 Jun 2020 13:58:53 +0800 Subject: cmd/asm: Add SHA512 hardware instructions for ARM64 ARMv8.2-SHA add SHA512 intructions: 1. SHA512H Vm.D2, Vn, Vd 2. SHA512H2 Vm.D2, Vn, Vd 3. SHA512SU0 Vn.D2, Vd.D2 4. SHA512SU1 Vm.D2, Vn.D2, Vd.D2 ARMv8 Architecture Reference Manual C7.2.234-C7.2.234 Change-Id: Ie970fef1bba5312ad466f246035da4c40a1bbb39 Reviewed-on: https://go-review.googlesource.com/c/go/+/180057 Reviewed-by: Cherry Zhang Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot --- src/cmd/asm/internal/asm/testdata/arm64.s | 4 ++++ src/cmd/internal/obj/arm64/a.out.go | 4 ++++ src/cmd/internal/obj/arm64/anames.go | 4 ++++ src/cmd/internal/obj/arm64/asm7.go | 16 ++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/src/cmd/asm/internal/asm/testdata/arm64.s b/src/cmd/asm/internal/asm/testdata/arm64.s index 69267bfa63..5a6db05074 100644 --- a/src/cmd/asm/internal/asm/testdata/arm64.s +++ b/src/cmd/asm/internal/asm/testdata/arm64.s @@ -77,6 +77,10 @@ TEXT foo(SB), DUPOK|NOSPLIT, $-8 SHA1H V5, V4 // a408285e SHA1M V8.S4, V7, V6 // e620085e SHA1P V11.S4, V10, V9 // 49110b5e + SHA512H V2.D2, V1, V0 // 208062ce + SHA512H2 V4.D2, V3, V2 // 628464ce + SHA512SU0 V9.D2, V8.D2 // 2881c0ce + SHA512SU1 V7.D2, V6.D2, V5.D2 // c58867ce VADDV V0.S4, V0 // 00b8b14e VMOVI $82, V0.B16 // 40e6024f VUADDLV V6.B16, V6 // c638306e diff --git a/src/cmd/internal/obj/arm64/a.out.go b/src/cmd/internal/obj/arm64/a.out.go index 152c493a65..03e0278a33 100644 --- a/src/cmd/internal/obj/arm64/a.out.go +++ b/src/cmd/internal/obj/arm64/a.out.go @@ -946,6 +946,10 @@ const ( ASHA256H2 ASHA256SU0 ASHA256SU1 + ASHA512H + ASHA512H2 + ASHA512SU0 + ASHA512SU1 AVADD AVADDP AVAND diff --git a/src/cmd/internal/obj/arm64/anames.go b/src/cmd/internal/obj/arm64/anames.go index 565f70aaf9..65ecd007ea 100644 --- a/src/cmd/internal/obj/arm64/anames.go +++ b/src/cmd/internal/obj/arm64/anames.go @@ -453,6 +453,10 @@ var Anames = []string{ "SHA256H2", "SHA256SU0", "SHA256SU1", + "SHA512H", + "SHA512H2", + "SHA512SU0", + "SHA512SU1", "VADD", "VADDP", "VAND", diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index df17729a76..8f8981479b 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -2747,6 +2747,7 @@ func buildop(ctxt *obj.Link) { oprangeset(AAESIMC, t) oprangeset(ASHA1SU1, t) oprangeset(ASHA256SU0, t) + oprangeset(ASHA512SU0, t) case ASHA1C: oprangeset(ASHA1P, t) @@ -2754,9 +2755,12 @@ func buildop(ctxt *obj.Link) { case ASHA256H: oprangeset(ASHA256H2, t) + oprangeset(ASHA512H, t) + oprangeset(ASHA512H2, t) case ASHA1SU0: oprangeset(ASHA256SU1, t) + oprangeset(ASHA512SU1, t) case AVADDV: oprangeset(AVUADDLV, t) @@ -5391,6 +5395,18 @@ func (c *ctxt7) oprrr(p *obj.Prog, a obj.As) uint32 { case ASHA256SU0: return 0x5E<<24 | 2<<20 | 8<<16 | 2<<12 | 2<<10 + case ASHA512H: + return 0xCE<<24 | 3<<21 | 8<<12 + + case ASHA512H2: + return 0xCE<<24 | 3<<21 | 8<<12 | 4<<8 + + case ASHA512SU1: + return 0xCE<<24 | 3<<21 | 8<<12 | 8<<8 + + case ASHA512SU0: + return 0xCE<<24 | 3<<22 | 8<<12 + case AFCVTZSD: return FPCVTI(1, 0, 1, 3, 0) -- cgit v1.3-5-g9baa From cdc77d34d7770ed02d84b9193380f9646017dce6 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 17 Aug 2020 17:31:21 -0400 Subject: testing: treat PAUSE lines as changing the active test name We could instead fix cmd/test2json to treat PAUSE lines as *not* changing the active test name, but that seems like it would be more confusing to humans, and also wouldn't fix tools that parse output using existing builds of cmd/test2json. Fixes #40657 Change-Id: I937611778f5b1e7dd1d6e9f44424d7e725a589ed Reviewed-on: https://go-review.googlesource.com/c/go/+/248727 Run-TryBot: Bryan C. Mills Reviewed-by: Jean de Klerk --- .../go/testdata/script/test_json_interleaved.txt | 27 ++++++++++++++++++++++ src/testing/testing.go | 25 ++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 src/cmd/go/testdata/script/test_json_interleaved.txt diff --git a/src/cmd/go/testdata/script/test_json_interleaved.txt b/src/cmd/go/testdata/script/test_json_interleaved.txt new file mode 100644 index 0000000000..e2d349e3fb --- /dev/null +++ b/src/cmd/go/testdata/script/test_json_interleaved.txt @@ -0,0 +1,27 @@ +# Regression test for https://golang.org/issue/40657: output from the main test +# function should be attributed correctly even if interleaved with the PAUSE +# line for a new parallel subtest. + +[short] skip + +go test -json +stdout '"Test":"TestWeirdTiming","Output":"[^"]* logging to outer again\\n"' + +-- go.mod -- +module example.com +go 1.15 +-- main_test.go -- +package main + +import ( + "testing" +) + +func TestWeirdTiming(outer *testing.T) { + outer.Run("pauser", func(pauser *testing.T) { + outer.Logf("logging to outer") + pauser.Parallel() + }) + + outer.Logf("logging to outer again") +} diff --git a/src/testing/testing.go b/src/testing/testing.go index 061142b9ab..6fc8c4fa9f 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -357,10 +357,19 @@ func (p *testPrinter) Fprint(w io.Writer, testName, out string) { defer p.lastNameMu.Unlock() if !p.chatty || - strings.HasPrefix(out, "--- PASS") || - strings.HasPrefix(out, "--- FAIL") || - strings.HasPrefix(out, "=== CONT") || - strings.HasPrefix(out, "=== RUN") { + strings.HasPrefix(out, "--- PASS: ") || + strings.HasPrefix(out, "--- FAIL: ") || + strings.HasPrefix(out, "--- SKIP: ") || + strings.HasPrefix(out, "=== RUN ") || + strings.HasPrefix(out, "=== CONT ") || + strings.HasPrefix(out, "=== PAUSE ") { + // If we're buffering test output (!p.chatty), we don't really care which + // test is emitting which line so long as they are serialized. + // + // If the message already implies an association with a specific new test, + // we don't need to check what the old test name was or log an extra CONT + // line for it. (We're updating it anyway, and the current message already + // includes the test name.) p.lastName = testName fmt.Fprint(w, out) return @@ -976,7 +985,13 @@ func (t *T) Parallel() { for ; root.parent != nil; root = root.parent { } root.mu.Lock() - fmt.Fprintf(root.w, "=== PAUSE %s\n", t.name) + // Unfortunately, even though PAUSE indicates that the named test is *no + // longer* running, cmd/test2json interprets it as changing the active test + // for the purpose of log parsing. We could fix cmd/test2json, but that + // won't fix existing deployments of third-party tools that already shell + // out to older builds of cmd/test2json — so merely fixing cmd/test2json + // isn't enough for now. + printer.Fprint(root.w, t.name, fmt.Sprintf("=== PAUSE %s\n", t.name)) root.mu.Unlock() } -- cgit v1.3-5-g9baa From 861a9483357a1a13609430ec6684b3dc9209e80c Mon Sep 17 00:00:00 2001 From: Martin Möhrmann Date: Sun, 3 May 2020 19:58:10 +0200 Subject: encoding/asn1: speed up marshal by reducing allocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace strings.Split by strings.IndexByte and explicit slicing to avoid the allocation of the return slice of strings.Split. name old time/op new time/op delta Marshal 43.3µs ± 1% 36.7µs ± 1% -15.23% (p=0.000 n=9+9) name old alloc/op new alloc/op delta Marshal 10.7kB ± 0% 9.2kB ± 0% -13.96% (p=0.000 n=10+10) name old allocs/op new allocs/op delta Marshal 444 ± 0% 366 ± 0% -17.57% (p=0.000 n=10+10) Change-Id: I9e727defa23f7e5fc684f246de0136fe28cf8d25 Reviewed-on: https://go-review.googlesource.com/c/go/+/231738 TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/encoding/asn1/common.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/encoding/asn1/common.go b/src/encoding/asn1/common.go index e2aa8bd9c5..1c712e1eff 100644 --- a/src/encoding/asn1/common.go +++ b/src/encoding/asn1/common.go @@ -92,7 +92,16 @@ type fieldParameters struct { // parseFieldParameters will parse it into a fieldParameters structure, // ignoring unknown parts of the string. func parseFieldParameters(str string) (ret fieldParameters) { - for _, part := range strings.Split(str, ",") { + var part string + for len(str) > 0 { + // This loop uses IndexByte and explicit slicing + // instead of strings.Split(str, ",") to reduce allocations. + i := strings.IndexByte(str, ',') + if i < 0 { + part, str = str, "" + } else { + part, str = str[:i], str[i+1:] + } switch { case part == "optional": ret.optional = true -- cgit v1.3-5-g9baa From 30a68bfb806b5217932e280f5a5f521237e69077 Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 12:40:18 -0700 Subject: runtime: add "success" field to sudog The current wakeup protocol for channel communications is that the second goroutine sets gp.param to the sudog when a value is successfully communicated over the channel, and to nil when the wakeup is due to closing the channel. Setting nil to indicate channel closure works okay for chansend and chanrecv, because they're only communicating with one channel, so they know it must be the channel that was closed. However, it means selectgo has to re-poll all of the channels to figure out which one was closed. This commit adds a "success" field to sudog, and changes the wakeup protocol to always set gp.param to sg, and to use sg.success to indicate successful communication vs channel closure. While here, this also reorganizes the chansend code slightly so that the sudog is still released to the pool if the send blocks and then is awoken because the channel closed. Updates #40410. Change-Id: I6cd9a20ebf9febe370a15af1b8afe24c5539efc6 Reviewed-on: https://go-review.googlesource.com/c/go/+/245019 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Cuong Manh Le Reviewed-by: Keith Randall --- src/runtime/chan.go | 25 +++++++++++++++---------- src/runtime/runtime2.go | 6 ++++++ src/runtime/select.go | 19 +++++++------------ 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/runtime/chan.go b/src/runtime/chan.go index f6f4ffd02e..0afe5d962b 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -263,18 +263,19 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { } gp.waiting = nil gp.activeStackChans = false - if gp.param == nil { - if c.closed == 0 { - throw("chansend: spurious wakeup") - } - panic(plainError("send on closed channel")) - } + closed := !mysg.success gp.param = nil if mysg.releasetime > 0 { blockevent(mysg.releasetime-t0, 2) } mysg.c = nil releaseSudog(mysg) + if closed { + if c.closed == 0 { + throw("chansend: spurious wakeup") + } + panic(plainError("send on closed channel")) + } return true } @@ -311,6 +312,7 @@ func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { gp := sg.g unlockf() gp.param = unsafe.Pointer(sg) + sg.success = true if sg.releasetime != 0 { sg.releasetime = cputicks() } @@ -384,7 +386,8 @@ func closechan(c *hchan) { sg.releasetime = cputicks() } gp := sg.g - gp.param = nil + gp.param = unsafe.Pointer(sg) + sg.success = false if raceenabled { raceacquireg(gp, c.raceaddr()) } @@ -402,7 +405,8 @@ func closechan(c *hchan) { sg.releasetime = cputicks() } gp := sg.g - gp.param = nil + gp.param = unsafe.Pointer(sg) + sg.success = false if raceenabled { raceacquireg(gp, c.raceaddr()) } @@ -575,11 +579,11 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) if mysg.releasetime > 0 { blockevent(mysg.releasetime-t0, 2) } - closed := gp.param == nil + success := mysg.success gp.param = nil mysg.c = nil releaseSudog(mysg) - return true, !closed + return true, success } // recv processes a receive operation on a full channel c. @@ -632,6 +636,7 @@ func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func(), skip int) { gp := sg.g unlockf() gp.param = unsafe.Pointer(sg) + sg.success = true if sg.releasetime != 0 { sg.releasetime = cputicks() } diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 959878400d..b7d0739e54 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -366,6 +366,12 @@ type sudog struct { // g.selectDone must be CAS'd to win the wake-up race. isSelect bool + // success indicates whether communication over channel c + // succeeded. It is true if the goroutine was awoken because a + // value was delivered over channel c, and false if awoken + // because c was closed. + success bool + parent *sudog // semaRoot binary tree waitlink *sudog // g.waiting list or semaRoot waittail *sudog // semaRoot diff --git a/src/runtime/select.go b/src/runtime/select.go index a069e3e050..081db7bad4 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -221,12 +221,12 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { nextp **sudog ) -loop: // pass 1 - look for something already waiting var dfli int var dfl *scase var casi int var cas *scase + var caseSuccess bool var recvOK bool for i := 0; i < ncases; i++ { casi = int(pollorder[i]) @@ -331,6 +331,7 @@ loop: // We singly-linked up the SudoGs in lock order. casi = -1 cas = nil + caseSuccess = false sglist = gp.waiting // Clear all elem before unlinking from gp.waiting. for sg1 := gp.waiting; sg1 != nil; sg1 = sg1.waitlink { @@ -352,6 +353,7 @@ loop: // sg has already been dequeued by the G that woke us up. casi = int(casei) cas = k + caseSuccess = sglist.success } else { c = k.c if k.kind == caseSend { @@ -367,16 +369,7 @@ loop: } if cas == nil { - // We can wake up with gp.param == nil (so cas == nil) - // when a channel involved in the select has been closed. - // It is easiest to loop and re-run the operation; - // we'll see that it's now closed. - // Maybe some day we can signal the close explicitly, - // but we'd have to distinguish close-on-reader from close-on-writer. - // It's easiest not to duplicate the code and just recheck above. - // We know that something closed, and things never un-close, - // so we won't block again. - goto loop + throw("selectgo: bad wakeup") } c = cas.c @@ -386,7 +379,9 @@ loop: } if cas.kind == caseRecv { - recvOK = true + recvOK = caseSuccess + } else if cas.kind == caseSend && !caseSuccess { + goto sclose } if raceenabled { -- cgit v1.3-5-g9baa From 78a1064d5dd05fc669342df3a6a5e11d49749d85 Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 14:05:05 -0700 Subject: runtime: remove scase.releasetime field selectgo will report at most one block event, so there's no need to keep a releasetime for every select case. It suffices to simply track the releasetime of the case responsible for the wakeup. Updates #40410. Change-Id: I72679cd43dde80d7e6dbab21a78952a4372d1e79 Reviewed-on: https://go-review.googlesource.com/c/go/+/245122 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/select.go | 1 - src/runtime/select.go | 23 ++++++++++------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index 49cc23cd3d..eb5ff8469b 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -386,7 +386,6 @@ func scasetype() *types.Type { namedfield("elem", types.Types[TUNSAFEPTR]), namedfield("kind", types.Types[TUINT16]), namedfield("pc", types.Types[TUINTPTR]), - namedfield("releasetime", types.Types[TINT64]), }) scase.SetNoalg(true) } diff --git a/src/runtime/select.go b/src/runtime/select.go index 081db7bad4..2f8b139155 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -26,11 +26,10 @@ const ( // Known to compiler. // Changes here must also be made in src/cmd/internal/gc/select.go's scasetype. type scase struct { - c *hchan // chan - elem unsafe.Pointer // data element - kind uint16 - pc uintptr // race pc (for race detector / msan) - releasetime int64 + c *hchan // chan + elem unsafe.Pointer // data element + kind uint16 + pc uintptr // race pc (for race detector / msan) } var ( @@ -142,9 +141,6 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { var t0 int64 if blockprofilerate > 0 { t0 = cputicks() - for i := 0; i < ncases; i++ { - scases[i].releasetime = -1 - } } // The compiler rewrites selects that statically have @@ -227,6 +223,7 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { var casi int var cas *scase var caseSuccess bool + var caseReleaseTime int64 = -1 var recvOK bool for i := 0; i < ncases; i++ { casi = int(pollorder[i]) @@ -346,14 +343,14 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { if k.kind == caseNil { continue } - if sglist.releasetime > 0 { - k.releasetime = sglist.releasetime - } if sg == sglist { // sg has already been dequeued by the G that woke us up. casi = int(casei) cas = k caseSuccess = sglist.success + if sglist.releasetime > 0 { + caseReleaseTime = sglist.releasetime + } } else { c = k.c if k.kind == caseSend { @@ -483,8 +480,8 @@ send: goto retc retc: - if cas.releasetime > 0 { - blockevent(cas.releasetime-t0, 1) + if caseReleaseTime > 0 { + blockevent(caseReleaseTime-t0, 1) } return casi, recvOK -- cgit v1.3-5-g9baa From 8a984e8e3f2cf4101f448ea9b9d9880b9e83c11e Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 14:47:47 -0700 Subject: runtime: omit nil-channel cases from selectgo's orders Currently, selectgo does an initial pass over the cases array to look for entries with nil channels, so they can be easily recognized and skipped later on. But this still involves actually visiting the cases. This commit changes selectgo to omit cases with nil channels when constructing pollorder, so that they'll be skipped over entirely later on. It also checks for caseDefault up front, which will facilitate changing it to use a "block bool" parameter instead. Updates #40410. Change-Id: Icaebcb8f08df03cc33b6d8087616fb5585f7fedd Reviewed-on: https://go-review.googlesource.com/c/go/+/245123 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/select.go | 66 +++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 39 deletions(-) diff --git a/src/runtime/select.go b/src/runtime/select.go index 2f8b139155..d540dd2e69 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -45,7 +45,7 @@ func sellock(scases []scase, lockorder []uint16) { var c *hchan for _, o := range lockorder { c0 := scases[o].c - if c0 != nil && c0 != c { + if c0 != c { c = c0 lock(&c.lock) } @@ -61,11 +61,8 @@ func selunlock(scases []scase, lockorder []uint16) { // the G that calls select runnable again and schedules it for execution. // When the G runs on another M, it locks all the locks and frees sel. // Now if the first M touches sel, it will access freed memory. - for i := len(scases) - 1; i >= 0; i-- { + for i := len(lockorder) - 1; i >= 0; i-- { c := scases[lockorder[i]].c - if c == nil { - break - } if i > 0 && c == scases[lockorder[i-1]].c { continue // will unlock it on the next iteration } @@ -129,15 +126,6 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { pollorder := order1[:ncases:ncases] lockorder := order1[ncases:][:ncases:ncases] - // Replace send/receive cases involving nil channels with - // caseNil so logic below can assume non-nil channel. - for i := range scases { - cas := &scases[i] - if cas.c == nil && cas.kind != caseDefault { - *cas = scase{} - } - } - var t0 int64 if blockprofilerate > 0 { t0 = cputicks() @@ -152,15 +140,31 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { // optimizing (and needing to test). // generate permuted order - for i := 1; i < ncases; i++ { - j := fastrandn(uint32(i + 1)) - pollorder[i] = pollorder[j] + dfli := -1 + norder := 0 + for i := range scases { + cas := &scases[i] + + // Omit cases without channels from the poll and lock orders. + if cas.c == nil { + if cas.kind == caseDefault { + dfli = i + } + cas.elem = nil // allow GC + continue + } + + j := fastrandn(uint32(norder + 1)) + pollorder[norder] = pollorder[j] pollorder[j] = uint16(i) + norder++ } + pollorder = pollorder[:norder] + lockorder = lockorder[:norder] // sort the cases by Hchan address to get the locking order. // simple heap sort, to guarantee n log n time and constant stack footprint. - for i := 0; i < ncases; i++ { + for i := range lockorder { j := i // Start with the pollorder to permute cases on the same channel. c := scases[pollorder[i]].c @@ -171,7 +175,7 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { } lockorder[j] = pollorder[i] } - for i := ncases - 1; i >= 0; i-- { + for i := len(lockorder) - 1; i >= 0; i-- { o := lockorder[i] c := scases[o].c lockorder[i] = lockorder[0] @@ -195,7 +199,7 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { } if debugSelect { - for i := 0; i+1 < ncases; i++ { + for i := 0; i+1 < len(lockorder); i++ { if scases[lockorder[i]].c.sortkey() > scases[lockorder[i+1]].c.sortkey() { print("i=", i, " x=", lockorder[i], " y=", lockorder[i+1], "\n") throw("select: broken sort") @@ -218,22 +222,17 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { ) // pass 1 - look for something already waiting - var dfli int - var dfl *scase var casi int var cas *scase var caseSuccess bool var caseReleaseTime int64 = -1 var recvOK bool - for i := 0; i < ncases; i++ { - casi = int(pollorder[i]) + for _, casei := range pollorder { + casi = int(casei) cas = &scases[casi] c = cas.c switch cas.kind { - case caseNil: - continue - case caseRecv: sg = c.sendq.dequeue() if sg != nil { @@ -260,17 +259,12 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { if c.qcount < c.dataqsiz { goto bufsend } - - case caseDefault: - dfli = casi - dfl = cas } } - if dfl != nil { + if dfli >= 0 { selunlock(scases, lockorder) casi = dfli - cas = dfl goto retc } @@ -283,9 +277,6 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { for _, casei := range lockorder { casi = int(casei) cas = &scases[casi] - if cas.kind == caseNil { - continue - } c = cas.c sg := acquireSudog() sg.g = gp @@ -340,9 +331,6 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { for _, casei := range lockorder { k = &scases[casei] - if k.kind == caseNil { - continue - } if sg == sglist { // sg has already been dequeued by the G that woke us up. casi = int(casei) -- cgit v1.3-5-g9baa From d36bc7d78ad226b20056c08fb8bca041e25b3d1d Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 15:20:18 -0700 Subject: runtime: split PCs out of scase Per-case PCs are only needed for race detector builds, so this allows skipping allocating stack space for them for non-race builds. It's possible to arrange the PCs and order arrays consecutively in memory so that we could just reuse the order0 pointer to identify both. However, there's more risk of that silently going wrong, so this commit passes them as separate arguments for now. We can revisit this in the future. Updates #40410. Change-Id: I8468bc25749e559891cb0cb007d1cc4a40fdd0f8 Reviewed-on: https://go-review.googlesource.com/c/go/+/245124 Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/builtin.go | 199 +++++++++++++------------ src/cmd/compile/internal/gc/builtin/runtime.go | 4 +- src/cmd/compile/internal/gc/select.go | 18 ++- src/runtime/select.go | 50 +++++-- 4 files changed, 153 insertions(+), 118 deletions(-) diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index 2cf2f4687e..eafdb0ebe7 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -126,74 +126,74 @@ var runtimeDecls = [...]struct { {"selectnbsend", funcTag, 94}, {"selectnbrecv", funcTag, 95}, {"selectnbrecv2", funcTag, 97}, - {"selectsetpc", funcTag, 62}, - {"selectgo", funcTag, 98}, + {"selectsetpc", funcTag, 98}, + {"selectgo", funcTag, 99}, {"block", funcTag, 9}, - {"makeslice", funcTag, 99}, - {"makeslice64", funcTag, 100}, - {"makeslicecopy", funcTag, 101}, - {"growslice", funcTag, 103}, - {"memmove", funcTag, 104}, - {"memclrNoHeapPointers", funcTag, 105}, - {"memclrHasPointers", funcTag, 105}, - {"memequal", funcTag, 106}, - {"memequal0", funcTag, 107}, - {"memequal8", funcTag, 107}, - {"memequal16", funcTag, 107}, - {"memequal32", funcTag, 107}, - {"memequal64", funcTag, 107}, - {"memequal128", funcTag, 107}, - {"f32equal", funcTag, 108}, - {"f64equal", funcTag, 108}, - {"c64equal", funcTag, 108}, - {"c128equal", funcTag, 108}, - {"strequal", funcTag, 108}, - {"interequal", funcTag, 108}, - {"nilinterequal", funcTag, 108}, - {"memhash", funcTag, 109}, - {"memhash0", funcTag, 110}, - {"memhash8", funcTag, 110}, - {"memhash16", funcTag, 110}, - {"memhash32", funcTag, 110}, - {"memhash64", funcTag, 110}, - {"memhash128", funcTag, 110}, - {"f32hash", funcTag, 110}, - {"f64hash", funcTag, 110}, - {"c64hash", funcTag, 110}, - {"c128hash", funcTag, 110}, - {"strhash", funcTag, 110}, - {"interhash", funcTag, 110}, - {"nilinterhash", funcTag, 110}, - {"int64div", funcTag, 111}, - {"uint64div", funcTag, 112}, - {"int64mod", funcTag, 111}, - {"uint64mod", funcTag, 112}, - {"float64toint64", funcTag, 113}, - {"float64touint64", funcTag, 114}, - {"float64touint32", funcTag, 115}, - {"int64tofloat64", funcTag, 116}, - {"uint64tofloat64", funcTag, 117}, - {"uint32tofloat64", funcTag, 118}, - {"complex128div", funcTag, 119}, - {"racefuncenter", funcTag, 120}, + {"makeslice", funcTag, 100}, + {"makeslice64", funcTag, 101}, + {"makeslicecopy", funcTag, 102}, + {"growslice", funcTag, 104}, + {"memmove", funcTag, 105}, + {"memclrNoHeapPointers", funcTag, 106}, + {"memclrHasPointers", funcTag, 106}, + {"memequal", funcTag, 107}, + {"memequal0", funcTag, 108}, + {"memequal8", funcTag, 108}, + {"memequal16", funcTag, 108}, + {"memequal32", funcTag, 108}, + {"memequal64", funcTag, 108}, + {"memequal128", funcTag, 108}, + {"f32equal", funcTag, 109}, + {"f64equal", funcTag, 109}, + {"c64equal", funcTag, 109}, + {"c128equal", funcTag, 109}, + {"strequal", funcTag, 109}, + {"interequal", funcTag, 109}, + {"nilinterequal", funcTag, 109}, + {"memhash", funcTag, 110}, + {"memhash0", funcTag, 111}, + {"memhash8", funcTag, 111}, + {"memhash16", funcTag, 111}, + {"memhash32", funcTag, 111}, + {"memhash64", funcTag, 111}, + {"memhash128", funcTag, 111}, + {"f32hash", funcTag, 111}, + {"f64hash", funcTag, 111}, + {"c64hash", funcTag, 111}, + {"c128hash", funcTag, 111}, + {"strhash", funcTag, 111}, + {"interhash", funcTag, 111}, + {"nilinterhash", funcTag, 111}, + {"int64div", funcTag, 112}, + {"uint64div", funcTag, 113}, + {"int64mod", funcTag, 112}, + {"uint64mod", funcTag, 113}, + {"float64toint64", funcTag, 114}, + {"float64touint64", funcTag, 115}, + {"float64touint32", funcTag, 116}, + {"int64tofloat64", funcTag, 117}, + {"uint64tofloat64", funcTag, 118}, + {"uint32tofloat64", funcTag, 119}, + {"complex128div", funcTag, 120}, + {"racefuncenter", funcTag, 121}, {"racefuncenterfp", funcTag, 9}, {"racefuncexit", funcTag, 9}, - {"raceread", funcTag, 120}, - {"racewrite", funcTag, 120}, - {"racereadrange", funcTag, 121}, - {"racewriterange", funcTag, 121}, - {"msanread", funcTag, 121}, - {"msanwrite", funcTag, 121}, - {"checkptrAlignment", funcTag, 122}, - {"checkptrArithmetic", funcTag, 124}, - {"libfuzzerTraceCmp1", funcTag, 126}, - {"libfuzzerTraceCmp2", funcTag, 128}, - {"libfuzzerTraceCmp4", funcTag, 129}, - {"libfuzzerTraceCmp8", funcTag, 130}, - {"libfuzzerTraceConstCmp1", funcTag, 126}, - {"libfuzzerTraceConstCmp2", funcTag, 128}, - {"libfuzzerTraceConstCmp4", funcTag, 129}, - {"libfuzzerTraceConstCmp8", funcTag, 130}, + {"raceread", funcTag, 121}, + {"racewrite", funcTag, 121}, + {"racereadrange", funcTag, 122}, + {"racewriterange", funcTag, 122}, + {"msanread", funcTag, 122}, + {"msanwrite", funcTag, 122}, + {"checkptrAlignment", funcTag, 123}, + {"checkptrArithmetic", funcTag, 125}, + {"libfuzzerTraceCmp1", funcTag, 127}, + {"libfuzzerTraceCmp2", funcTag, 129}, + {"libfuzzerTraceCmp4", funcTag, 130}, + {"libfuzzerTraceCmp8", funcTag, 131}, + {"libfuzzerTraceConstCmp1", funcTag, 127}, + {"libfuzzerTraceConstCmp2", funcTag, 129}, + {"libfuzzerTraceConstCmp4", funcTag, 130}, + {"libfuzzerTraceConstCmp8", funcTag, 131}, {"x86HasPOPCNT", varTag, 6}, {"x86HasSSE41", varTag, 6}, {"x86HasFMA", varTag, 6}, @@ -202,7 +202,7 @@ var runtimeDecls = [...]struct { } func runtimeTypes() []*types.Type { - var typs [131]*types.Type + var typs [132]*types.Type typs[0] = types.Bytetype typs[1] = types.NewPtr(typs[0]) typs[2] = types.Types[TANY] @@ -301,38 +301,39 @@ func runtimeTypes() []*types.Type { typs[95] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[84])}, []*Node{anonfield(typs[6])}) typs[96] = types.NewPtr(typs[6]) typs[97] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[96]), anonfield(typs[84])}, []*Node{anonfield(typs[6])}) - typs[98] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) - typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])}) - typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])}) - typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])}) - typs[102] = types.NewSlice(typs[2]) - typs[103] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[102]), anonfield(typs[15])}, []*Node{anonfield(typs[102])}) - typs[104] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, nil) - typs[105] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, nil) - typs[106] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, []*Node{anonfield(typs[6])}) - typs[107] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[6])}) - typs[108] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[7])}, []*Node{anonfield(typs[6])}) - typs[109] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) - typs[110] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) - typs[111] = functype(nil, []*Node{anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[22])}) - typs[112] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, []*Node{anonfield(typs[24])}) - typs[113] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[22])}) - typs[114] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[24])}) - typs[115] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[65])}) - typs[116] = functype(nil, []*Node{anonfield(typs[22])}, []*Node{anonfield(typs[20])}) - typs[117] = functype(nil, []*Node{anonfield(typs[24])}, []*Node{anonfield(typs[20])}) - typs[118] = functype(nil, []*Node{anonfield(typs[65])}, []*Node{anonfield(typs[20])}) - typs[119] = functype(nil, []*Node{anonfield(typs[26]), anonfield(typs[26])}, []*Node{anonfield(typs[26])}) - typs[120] = functype(nil, []*Node{anonfield(typs[5])}, nil) - typs[121] = functype(nil, []*Node{anonfield(typs[5]), anonfield(typs[5])}, nil) - typs[122] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[1]), anonfield(typs[5])}, nil) - typs[123] = types.NewSlice(typs[7]) - typs[124] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[123])}, nil) - typs[125] = types.Types[TUINT8] - typs[126] = functype(nil, []*Node{anonfield(typs[125]), anonfield(typs[125])}, nil) - typs[127] = types.Types[TUINT16] - typs[128] = functype(nil, []*Node{anonfield(typs[127]), anonfield(typs[127])}, nil) - typs[129] = functype(nil, []*Node{anonfield(typs[65]), anonfield(typs[65])}, nil) - typs[130] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, nil) + typs[98] = functype(nil, []*Node{anonfield(typs[63])}, nil) + typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) + typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])}) + typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])}) + typs[102] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])}) + typs[103] = types.NewSlice(typs[2]) + typs[104] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[103]), anonfield(typs[15])}, []*Node{anonfield(typs[103])}) + typs[105] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, nil) + typs[106] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, nil) + typs[107] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3]), anonfield(typs[5])}, []*Node{anonfield(typs[6])}) + typs[108] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[3])}, []*Node{anonfield(typs[6])}) + typs[109] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[7])}, []*Node{anonfield(typs[6])}) + typs[110] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) + typs[111] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[5])}, []*Node{anonfield(typs[5])}) + typs[112] = functype(nil, []*Node{anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[22])}) + typs[113] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, []*Node{anonfield(typs[24])}) + typs[114] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[22])}) + typs[115] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[24])}) + typs[116] = functype(nil, []*Node{anonfield(typs[20])}, []*Node{anonfield(typs[65])}) + typs[117] = functype(nil, []*Node{anonfield(typs[22])}, []*Node{anonfield(typs[20])}) + typs[118] = functype(nil, []*Node{anonfield(typs[24])}, []*Node{anonfield(typs[20])}) + typs[119] = functype(nil, []*Node{anonfield(typs[65])}, []*Node{anonfield(typs[20])}) + typs[120] = functype(nil, []*Node{anonfield(typs[26]), anonfield(typs[26])}, []*Node{anonfield(typs[26])}) + typs[121] = functype(nil, []*Node{anonfield(typs[5])}, nil) + typs[122] = functype(nil, []*Node{anonfield(typs[5]), anonfield(typs[5])}, nil) + typs[123] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[1]), anonfield(typs[5])}, nil) + typs[124] = types.NewSlice(typs[7]) + typs[125] = functype(nil, []*Node{anonfield(typs[7]), anonfield(typs[124])}, nil) + typs[126] = types.Types[TUINT8] + typs[127] = functype(nil, []*Node{anonfield(typs[126]), anonfield(typs[126])}, nil) + typs[128] = types.Types[TUINT16] + typs[129] = functype(nil, []*Node{anonfield(typs[128]), anonfield(typs[128])}, nil) + typs[130] = functype(nil, []*Node{anonfield(typs[65]), anonfield(typs[65])}, nil) + typs[131] = functype(nil, []*Node{anonfield(typs[24]), anonfield(typs[24])}, nil) return typs[:] } diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index 00448272c5..25f86efdd6 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -169,8 +169,8 @@ func selectnbsend(hchan chan<- any, elem *any) bool func selectnbrecv(elem *any, hchan <-chan any) bool func selectnbrecv2(elem *any, received *bool, hchan <-chan any) bool -func selectsetpc(cas *byte) -func selectgo(cas0 *byte, order0 *byte, ncases int) (int, bool) +func selectsetpc(pc *uintptr) +func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, ncases int) (int, bool) func block() func makeslice(typ *byte, len int, cap int) unsafe.Pointer diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index eb5ff8469b..8eb31eb5c1 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -271,6 +271,14 @@ func walkselectcases(cases *Nodes) []*Node { r = typecheck(r, ctxStmt) init = append(init, r) + var pc0, pcs *Node + if flag_race { + pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(n))) + pc0 = typecheck(nod(OADDR, nod(OINDEX, pcs, nodintconst(0)), nil), ctxExpr) + } else { + pc0 = nodnil() + } + // register cases for i, cas := range cases.Slice() { setlineno(cas) @@ -324,8 +332,8 @@ func walkselectcases(cases *Nodes) []*Node { // TODO(mdempsky): There should be a cleaner way to // handle this. - if instrumenting { - r = mkcall("selectsetpc", nil, nil, bytePtrToIndex(selv, int64(i))) + if flag_race { + r = mkcall("selectsetpc", nil, nil, nod(OADDR, nod(OINDEX, pcs, nodintconst(int64(i))), nil)) init = append(init, r) } } @@ -337,13 +345,16 @@ func walkselectcases(cases *Nodes) []*Node { r = nod(OAS2, nil, nil) r.List.Set2(chosen, recvOK) fn := syslook("selectgo") - r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), nodintconst(int64(n)))) + r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(n)))) r = typecheck(r, ctxStmt) init = append(init, r) // selv and order are no longer alive after selectgo. init = append(init, nod(OVARKILL, selv, nil)) init = append(init, nod(OVARKILL, order, nil)) + if flag_race { + init = append(init, nod(OVARKILL, pcs, nil)) + } // dispatch cases for i, cas := range cases.Slice() { @@ -385,7 +396,6 @@ func scasetype() *types.Type { namedfield("c", types.Types[TUNSAFEPTR]), namedfield("elem", types.Types[TUNSAFEPTR]), namedfield("kind", types.Types[TUINT16]), - namedfield("pc", types.Types[TUINTPTR]), }) scase.SetNoalg(true) } diff --git a/src/runtime/select.go b/src/runtime/select.go index d540dd2e69..d7c7d9f26f 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -29,7 +29,6 @@ type scase struct { c *hchan // chan elem unsafe.Pointer // data element kind uint16 - pc uintptr // race pc (for race detector / msan) } var ( @@ -37,8 +36,8 @@ var ( chanrecvpc = funcPC(chanrecv) ) -func selectsetpc(cas *scase) { - cas.pc = getcallerpc() +func selectsetpc(pc *uintptr) { + *pc = getcallerpc() } func sellock(scases []scase, lockorder []uint16) { @@ -108,11 +107,15 @@ func block() { // Both reside on the goroutine's stack (regardless of any escaping in // selectgo). // +// For race detector builds, pc0 points to an array of type +// [ncases]uintptr (also on the stack); for other builds, it's set to +// nil. +// // selectgo returns the index of the chosen scase, which matches the // ordinal position of its respective select{recv,send,default} call. // Also, if the chosen scase was a receive operation, it reports whether // a value was received. -func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { +func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) { if debugSelect { print("select: cas0=", cas0, "\n") } @@ -126,6 +129,21 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { pollorder := order1[:ncases:ncases] lockorder := order1[ncases:][:ncases:ncases] + // Even when raceenabled is true, there might be select + // statements in packages compiled without -race (e.g., + // ensureSigM in runtime/signal_unix.go). + var pcs []uintptr + if raceenabled && pc0 != nil { + pc1 := (*[1 << 16]uintptr)(unsafe.Pointer(pc0)) + pcs = pc1[:ncases:ncases] + } + casePC := func(casi int) uintptr { + if pcs == nil { + return 0 + } + return pcs[casi] + } + var t0 int64 if blockprofilerate > 0 { t0 = cputicks() @@ -247,7 +265,7 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { case caseSend: if raceenabled { - racereadpc(c.raceaddr(), cas.pc, chansendpc) + racereadpc(c.raceaddr(), casePC(casi), chansendpc) } if c.closed != 0 { goto sclose @@ -371,9 +389,9 @@ func selectgo(cas0 *scase, order0 *uint16, ncases int) (int, bool) { if raceenabled { if cas.kind == caseRecv && cas.elem != nil { - raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) } else if cas.kind == caseSend { - raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) } } if msanenabled { @@ -391,7 +409,7 @@ bufrecv: // can receive from buffer if raceenabled { if cas.elem != nil { - raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc) + raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) } raceacquire(chanbuf(c, c.recvx)) racerelease(chanbuf(c, c.recvx)) @@ -418,7 +436,7 @@ bufsend: if raceenabled { raceacquire(chanbuf(c, c.sendx)) racerelease(chanbuf(c, c.sendx)) - raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) } if msanenabled { msanread(cas.elem, c.elemtype.size) @@ -456,7 +474,7 @@ rclose: send: // can send to a sleeping receiver (sg) if raceenabled { - raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc) + raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) } if msanenabled { msanread(cas.elem, c.elemtype.size) @@ -519,12 +537,18 @@ func reflect_rselect(cases []runtimeSelect) (int, bool) { case selectRecv: sel[i] = scase{kind: caseRecv, c: rc.ch, elem: rc.val} } - if raceenabled || msanenabled { - selectsetpc(&sel[i]) + } + + var pc0 *uintptr + if raceenabled { + pcs := make([]uintptr, len(cases)) + for i := range pcs { + selectsetpc(&pcs[i]) } + pc0 = &pcs[0] } - return selectgo(&sel[0], &order[0], len(cases)) + return selectgo(&sel[0], &order[0], pc0, len(cases)) } func (q *waitq) dequeueSudoG(sgp *sudog) { -- cgit v1.3-5-g9baa From fe23ba4a145ce8465d16ea2a92b9a7e96e15c28e Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 16:19:15 -0700 Subject: runtime: eliminate scase.kind field Currently, we include a "kind" field on scase to distinguish the three kinds of cases in a select statement: sends, receives, and defaults. This commit removes by kind field by instead arranging for the compiler to always place sends before receives, and to provide their counts separately. It also passes an explicit "block bool" parameter to avoid needing to include a default case in the array. It's safe to shuffle cases like this because the runtime will randomize the order they're polled in anyway. Fixes #40410. Change-Id: Iaeaed4cf7bddd576d78f2c863bd91a03a5c82df2 Reviewed-on: https://go-review.googlesource.com/c/go/+/245125 Reviewed-by: Keith Randall --- src/cmd/compile/internal/gc/builtin.go | 2 +- src/cmd/compile/internal/gc/builtin/runtime.go | 2 +- src/cmd/compile/internal/gc/select.go | 108 +++++++++++++------------ src/reflect/all_test.go | 8 ++ src/runtime/select.go | 104 +++++++++++++----------- 5 files changed, 123 insertions(+), 101 deletions(-) diff --git a/src/cmd/compile/internal/gc/builtin.go b/src/cmd/compile/internal/gc/builtin.go index eafdb0ebe7..861ffaaa5b 100644 --- a/src/cmd/compile/internal/gc/builtin.go +++ b/src/cmd/compile/internal/gc/builtin.go @@ -302,7 +302,7 @@ func runtimeTypes() []*types.Type { typs[96] = types.NewPtr(typs[6]) typs[97] = functype(nil, []*Node{anonfield(typs[3]), anonfield(typs[96]), anonfield(typs[84])}, []*Node{anonfield(typs[6])}) typs[98] = functype(nil, []*Node{anonfield(typs[63])}, nil) - typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) + typs[99] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[1]), anonfield(typs[63]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[6])}, []*Node{anonfield(typs[15]), anonfield(typs[6])}) typs[100] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15])}, []*Node{anonfield(typs[7])}) typs[101] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[22]), anonfield(typs[22])}, []*Node{anonfield(typs[7])}) typs[102] = functype(nil, []*Node{anonfield(typs[1]), anonfield(typs[15]), anonfield(typs[15]), anonfield(typs[7])}, []*Node{anonfield(typs[7])}) diff --git a/src/cmd/compile/internal/gc/builtin/runtime.go b/src/cmd/compile/internal/gc/builtin/runtime.go index 25f86efdd6..635da80f7c 100644 --- a/src/cmd/compile/internal/gc/builtin/runtime.go +++ b/src/cmd/compile/internal/gc/builtin/runtime.go @@ -170,7 +170,7 @@ func selectnbrecv(elem *any, hchan <-chan any) bool func selectnbrecv2(elem *any, received *bool, hchan <-chan any) bool func selectsetpc(pc *uintptr) -func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, ncases int) (int, bool) +func selectgo(cas0 *byte, order0 *byte, pc0 *uintptr, nsends int, nrecvs int, block bool) (int, bool) func block() func makeslice(typ *byte, len int, cap int) unsafe.Pointer diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index 8eb31eb5c1..bae7ed30e2 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -106,18 +106,16 @@ func walkselect(sel *Node) { } func walkselectcases(cases *Nodes) []*Node { - n := cases.Len() + ncas := cases.Len() sellineno := lineno // optimization: zero-case select - if n == 0 { + if ncas == 0 { return []*Node{mkcall("block", nil, nil)} } // optimization: one-case select: single op. - // TODO(rsc): Reenable optimization once order.go can handle it. - // golang.org/issue/7672. - if n == 1 { + if ncas == 1 { cas := cases.First() setlineno(cas) l := cas.Ninit.Slice() @@ -178,10 +176,12 @@ func walkselectcases(cases *Nodes) []*Node { // convert case value arguments to addresses. // this rewrite is used by both the general code and the next optimization. + var dflt *Node for _, cas := range cases.Slice() { setlineno(cas) n := cas.Left if n == nil { + dflt = cas continue } switch n.Op { @@ -202,15 +202,10 @@ func walkselectcases(cases *Nodes) []*Node { } // optimization: two-case select but one is default: single non-blocking op. - if n == 2 && (cases.First().Left == nil || cases.Second().Left == nil) { - var cas *Node - var dflt *Node - if cases.First().Left == nil { + if ncas == 2 && dflt != nil { + cas := cases.First() + if cas == dflt { cas = cases.Second() - dflt = cases.First() - } else { - dflt = cases.Second() - cas = cases.First() } n := cas.Left @@ -257,74 +252,73 @@ func walkselectcases(cases *Nodes) []*Node { return []*Node{r, nod(OBREAK, nil, nil)} } + if dflt != nil { + ncas-- + } + casorder := make([]*Node, ncas) + nsends, nrecvs := 0, 0 + var init []*Node // generate sel-struct lineno = sellineno - selv := temp(types.NewArray(scasetype(), int64(n))) + selv := temp(types.NewArray(scasetype(), int64(ncas))) r := nod(OAS, selv, nil) r = typecheck(r, ctxStmt) init = append(init, r) - order := temp(types.NewArray(types.Types[TUINT16], 2*int64(n))) + order := temp(types.NewArray(types.Types[TUINT16], 2*int64(ncas))) r = nod(OAS, order, nil) r = typecheck(r, ctxStmt) init = append(init, r) var pc0, pcs *Node if flag_race { - pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(n))) + pcs = temp(types.NewArray(types.Types[TUINTPTR], int64(ncas))) pc0 = typecheck(nod(OADDR, nod(OINDEX, pcs, nodintconst(0)), nil), ctxExpr) } else { pc0 = nodnil() } // register cases - for i, cas := range cases.Slice() { + for _, cas := range cases.Slice() { setlineno(cas) init = append(init, cas.Ninit.Slice()...) cas.Ninit.Set(nil) - // Keep in sync with runtime/select.go. - const ( - caseNil = iota - caseRecv - caseSend - caseDefault - ) + n := cas.Left + if n == nil { // default: + continue + } + var i int var c, elem *Node - var kind int64 = caseDefault - - if n := cas.Left; n != nil { - init = append(init, n.Ninit.Slice()...) - - switch n.Op { - default: - Fatalf("select %v", n.Op) - case OSEND: - kind = caseSend - c = n.Left - elem = n.Right - case OSELRECV, OSELRECV2: - kind = caseRecv - c = n.Right.Left - elem = n.Left - } + switch n.Op { + default: + Fatalf("select %v", n.Op) + case OSEND: + i = nsends + nsends++ + c = n.Left + elem = n.Right + case OSELRECV, OSELRECV2: + nrecvs++ + i = ncas - nrecvs + c = n.Right.Left + elem = n.Left } + casorder[i] = cas + setField := func(f string, val *Node) { r := nod(OAS, nodSym(ODOT, nod(OINDEX, selv, nodintconst(int64(i))), lookup(f)), val) r = typecheck(r, ctxStmt) init = append(init, r) } - setField("kind", nodintconst(kind)) - if c != nil { - c = convnop(c, types.Types[TUNSAFEPTR]) - setField("c", c) - } + c = convnop(c, types.Types[TUNSAFEPTR]) + setField("c", c) if elem != nil { elem = convnop(elem, types.Types[TUNSAFEPTR]) setField("elem", elem) @@ -337,6 +331,9 @@ func walkselectcases(cases *Nodes) []*Node { init = append(init, r) } } + if nsends+nrecvs != ncas { + Fatalf("walkselectcases: miscount: %v + %v != %v", nsends, nrecvs, ncas) + } // run the select lineno = sellineno @@ -345,7 +342,7 @@ func walkselectcases(cases *Nodes) []*Node { r = nod(OAS2, nil, nil) r.List.Set2(chosen, recvOK) fn := syslook("selectgo") - r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(n)))) + r.Rlist.Set1(mkcall1(fn, fn.Type.Results(), nil, bytePtrToIndex(selv, 0), bytePtrToIndex(order, 0), pc0, nodintconst(int64(nsends)), nodintconst(int64(nrecvs)), nodbool(dflt == nil))) r = typecheck(r, ctxStmt) init = append(init, r) @@ -357,14 +354,11 @@ func walkselectcases(cases *Nodes) []*Node { } // dispatch cases - for i, cas := range cases.Slice() { - setlineno(cas) - - cond := nod(OEQ, chosen, nodintconst(int64(i))) + dispatch := func(cond, cas *Node) { cond = typecheck(cond, ctxExpr) cond = defaultlit(cond, nil) - r = nod(OIF, cond, nil) + r := nod(OIF, cond, nil) if n := cas.Left; n != nil && n.Op == OSELRECV2 { x := nod(OAS, n.List.First(), recvOK) @@ -377,6 +371,15 @@ func walkselectcases(cases *Nodes) []*Node { init = append(init, r) } + if dflt != nil { + setlineno(dflt) + dispatch(nod(OLT, chosen, nodintconst(0)), dflt) + } + for i, cas := range casorder { + setlineno(cas) + dispatch(nod(OEQ, chosen, nodintconst(int64(i))), cas) + } + return init } @@ -395,7 +398,6 @@ func scasetype() *types.Type { scase = tostruct([]*Node{ namedfield("c", types.Types[TUNSAFEPTR]), namedfield("elem", types.Types[TUNSAFEPTR]), - namedfield("kind", types.Types[TUINT16]), }) scase.SetNoalg(true) } diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index ed2f225077..5a12699472 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -1725,6 +1725,14 @@ func TestSelectMaxCases(t *testing.T) { _, _, _ = Select(sCases) } +func TestSelectNop(t *testing.T) { + // "select { default: }" should always return the default case. + chosen, _, _ := Select([]SelectCase{{Dir: SelectDefault}}) + if chosen != 0 { + t.Fatalf("expected Select to return 0, but got %#v", chosen) + } +} + func BenchmarkSelect(b *testing.B) { channel := make(chan int) close(channel) diff --git a/src/runtime/select.go b/src/runtime/select.go index d7c7d9f26f..80768b285b 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -12,23 +12,12 @@ import ( const debugSelect = false -// scase.kind values. -// Known to compiler. -// Changes here must also be made in src/cmd/compile/internal/gc/select.go's walkselectcases. -const ( - caseNil = iota - caseRecv - caseSend - caseDefault -) - // Select case descriptor. // Known to compiler. // Changes here must also be made in src/cmd/internal/gc/select.go's scasetype. type scase struct { c *hchan // chan elem unsafe.Pointer // data element - kind uint16 } var ( @@ -115,7 +104,7 @@ func block() { // ordinal position of its respective select{recv,send,default} call. // Also, if the chosen scase was a receive operation, it reports whether // a value was received. -func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) { +func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, nsends, nrecvs int, block bool) (int, bool) { if debugSelect { print("select: cas0=", cas0, "\n") } @@ -125,6 +114,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) cas1 := (*[1 << 16]scase)(unsafe.Pointer(cas0)) order1 := (*[1 << 17]uint16)(unsafe.Pointer(order0)) + ncases := nsends + nrecvs scases := cas1[:ncases:ncases] pollorder := order1[:ncases:ncases] lockorder := order1[ncases:][:ncases:ncases] @@ -158,16 +148,12 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) // optimizing (and needing to test). // generate permuted order - dfli := -1 norder := 0 for i := range scases { cas := &scases[i] // Omit cases without channels from the poll and lock orders. if cas.c == nil { - if cas.kind == caseDefault { - dfli = i - } cas.elem = nil // allow GC continue } @@ -250,8 +236,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) cas = &scases[casi] c = cas.c - switch cas.kind { - case caseRecv: + if casi >= nsends { sg = c.sendq.dequeue() if sg != nil { goto recv @@ -262,8 +247,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) if c.closed != 0 { goto rclose } - - case caseSend: + } else { if raceenabled { racereadpc(c.raceaddr(), casePC(casi), chansendpc) } @@ -280,9 +264,9 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) } } - if dfli >= 0 { + if !block { selunlock(scases, lockorder) - casi = dfli + casi = -1 goto retc } @@ -311,12 +295,10 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) *nextp = sg nextp = &sg.waitlink - switch cas.kind { - case caseRecv: - c.recvq.enqueue(sg) - - case caseSend: + if casi < nsends { c.sendq.enqueue(sg) + } else { + c.recvq.enqueue(sg) } } @@ -359,7 +341,7 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) } } else { c = k.c - if k.kind == caseSend { + if int(casei) < nsends { c.sendq.dequeueSudoG(sglist) } else { c.recvq.dequeueSudoG(sglist) @@ -378,27 +360,29 @@ func selectgo(cas0 *scase, order0 *uint16, pc0 *uintptr, ncases int) (int, bool) c = cas.c if debugSelect { - print("wait-return: cas0=", cas0, " c=", c, " cas=", cas, " kind=", cas.kind, "\n") + print("wait-return: cas0=", cas0, " c=", c, " cas=", cas, " send=", casi < nsends, "\n") } - if cas.kind == caseRecv { + if casi < nsends { + if !caseSuccess { + goto sclose + } + } else { recvOK = caseSuccess - } else if cas.kind == caseSend && !caseSuccess { - goto sclose } if raceenabled { - if cas.kind == caseRecv && cas.elem != nil { - raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) - } else if cas.kind == caseSend { + if casi < nsends { raceReadObjectPC(c.elemtype, cas.elem, casePC(casi), chansendpc) + } else if cas.elem != nil { + raceWriteObjectPC(c.elemtype, cas.elem, casePC(casi), chanrecvpc) } } if msanenabled { - if cas.kind == caseRecv && cas.elem != nil { - msanwrite(cas.elem, c.elemtype.size) - } else if cas.kind == caseSend { + if casi < nsends { msanread(cas.elem, c.elemtype.size) + } else if cas.elem != nil { + msanwrite(cas.elem, c.elemtype.size) } } @@ -526,29 +510,57 @@ func reflect_rselect(cases []runtimeSelect) (int, bool) { block() } sel := make([]scase, len(cases)) - order := make([]uint16, 2*len(cases)) - for i := range cases { - rc := &cases[i] + orig := make([]int, len(cases)) + nsends, nrecvs := 0, 0 + dflt := -1 + for i, rc := range cases { + var j int switch rc.dir { case selectDefault: - sel[i] = scase{kind: caseDefault} + dflt = i + continue case selectSend: - sel[i] = scase{kind: caseSend, c: rc.ch, elem: rc.val} + j = nsends + nsends++ case selectRecv: - sel[i] = scase{kind: caseRecv, c: rc.ch, elem: rc.val} + nrecvs++ + j = len(cases) - nrecvs } + + sel[j] = scase{c: rc.ch, elem: rc.val} + orig[j] = i } + // Only a default case. + if nsends+nrecvs == 0 { + return dflt, false + } + + // Compact sel and orig if necessary. + if nsends+nrecvs < len(cases) { + copy(sel[nsends:], sel[len(cases)-nrecvs:]) + copy(orig[nsends:], orig[len(cases)-nrecvs:]) + } + + order := make([]uint16, 2*(nsends+nrecvs)) var pc0 *uintptr if raceenabled { - pcs := make([]uintptr, len(cases)) + pcs := make([]uintptr, nsends+nrecvs) for i := range pcs { selectsetpc(&pcs[i]) } pc0 = &pcs[0] } - return selectgo(&sel[0], &order[0], pc0, len(cases)) + chosen, recvOK := selectgo(&sel[0], &order[0], pc0, nsends, nrecvs, dflt == -1) + + // Translate chosen back to caller's ordering. + if chosen < 0 { + chosen = dflt + } else { + chosen = orig[chosen] + } + return chosen, recvOK } func (q *waitq) dequeueSudoG(sgp *sudog) { -- cgit v1.3-5-g9baa From a40171857595db60e95a04b64aad8ba262cf64a7 Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Mon, 27 Jul 2020 16:45:48 -0700 Subject: cmd/compile/internal/gc: cleanup walkselectcases slightly Remove some unnecessary code. Most significantly, we can skip testing "if ch == nil { block() }", because this is already the semantics implied by normal send/receive operations. Updates #40410. Change-Id: I4acd33383cc876719fc3b998d85244d4ac1ff9d9 Reviewed-on: https://go-review.googlesource.com/c/go/+/245126 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Cuong Manh Le --- src/cmd/compile/internal/gc/select.go | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/src/cmd/compile/internal/gc/select.go b/src/cmd/compile/internal/gc/select.go index bae7ed30e2..3812a0e1fa 100644 --- a/src/cmd/compile/internal/gc/select.go +++ b/src/cmd/compile/internal/gc/select.go @@ -123,17 +123,14 @@ func walkselectcases(cases *Nodes) []*Node { n := cas.Left l = append(l, n.Ninit.Slice()...) n.Ninit.Set(nil) - var ch *Node switch n.Op { default: Fatalf("select %v", n.Op) - // ok already case OSEND: - ch = n.Left + // already ok case OSELRECV, OSELRECV2: - ch = n.Right.Left if n.Op == OSELRECV || n.List.Len() == 0 { if n.Left == nil { n = n.Right @@ -157,16 +154,7 @@ func walkselectcases(cases *Nodes) []*Node { n = typecheck(n, ctxStmt) } - // if ch == nil { block() }; n; - a := nod(OIF, nil, nil) - - a.Left = nod(OEQ, ch, nodnil()) - var ln Nodes - ln.Set(l) - a.Nbody.Set1(mkcall("block", nil, &ln)) - l = ln.Slice() - a = typecheck(a, ctxStmt) - l = append(l, a, n) + l = append(l, n) } l = append(l, cas.Nbody.Slice()...) @@ -223,8 +211,6 @@ func walkselectcases(cases *Nodes) []*Node { case OSELRECV: // if selectnbrecv(&v, c) { body } else { default body } - r = nod(OIF, nil, nil) - r.Ninit.Set(cas.Ninit.Slice()) ch := n.Right.Left elem := n.Left if elem == nil { @@ -234,8 +220,6 @@ func walkselectcases(cases *Nodes) []*Node { case OSELRECV2: // if selectnbrecv2(&v, &received, c) { body } else { default body } - r = nod(OIF, nil, nil) - r.Ninit.Set(cas.Ninit.Slice()) ch := n.Right.Left elem := n.Left if elem == nil { -- cgit v1.3-5-g9baa From 216714e44f703470f102cf248f7e9097160093d4 Mon Sep 17 00:00:00 2001 From: Lynn Boger Date: Mon, 17 Aug 2020 10:57:54 -0400 Subject: math/big: improve performance of mulAddVWW on ppc64x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This changes the assembly implementation on ppc64x to improve performance by reordering some instructions. It also eliminates an unnecessary move by changing an ADDZE to use the correct target register. Improvement on power9: MulAddVWW/1 6.89ns ± 0% 7.30ns ± 0% +5.95% (p=1.000 n=1+1) MulAddVWW/2 8.04ns ± 0% 8.06ns ± 0% +0.25% (p=1.000 n=1+1) MulAddVWW/3 9.39ns ± 0% 9.39ns ± 0% ~ (all equal) MulAddVWW/4 9.76ns ± 0% 9.48ns ± 0% -2.87% (p=1.000 n=1+1) MulAddVWW/5 10.5ns ± 0% 10.3ns ± 0% -1.90% (p=1.000 n=1+1) MulAddVWW/10 15.4ns ± 0% 14.9ns ± 0% -3.25% (p=1.000 n=1+1) MulAddVWW/100 149ns ± 0% 125ns ± 0% -16.11% (p=1.000 n=1+1) MulAddVWW/1000 1.42µs ± 0% 1.28µs ± 0% -9.74% (p=1.000 n=1+1) MulAddVWW/10000 14.2µs ± 0% 12.8µs ± 0% -9.73% (p=1.000 n=1+1) MulAddVWW/100000 144µs ± 0% 129µs ± 0% -10.10% (p=1.000 n=1+1) Change-Id: I0ae7002a69783ca19d7a4e3e42042ae75dc60069 Reviewed-on: https://go-review.googlesource.com/c/go/+/248721 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Carlos Eduardo Seo Reviewed-by: Paul Murphy --- src/math/big/arith_ppc64x.s | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/math/big/arith_ppc64x.s b/src/math/big/arith_ppc64x.s index dbb168a376..409e10ab48 100644 --- a/src/math/big/arith_ppc64x.s +++ b/src/math/big/arith_ppc64x.s @@ -394,17 +394,16 @@ loop: ADDZE R21 MULLD R9, R22, R26 MULHDU R9, R22, R22 - ADDC R21, R26 - ADDZE R22 MULLD R9, R23, R27 MULHDU R9, R23, R23 - ADDC R22, R27 - ADDZE R23 + ADDC R21, R26 + ADDZE R22 MOVD R24, 8(R10) // z[i] MOVD R25, 16(R10) // z[i+1] + ADDC R22, R27 + ADDZE R23,R4 // update carry MOVD R26, 24(R10) // z[i+2] MOVDU R27, 32(R10) // z[i+3] - MOVD R23, R4 // R4 = c ADD $-4, R11 // R11 = z_len - 4 BC 16, 0, loop // bdnz -- cgit v1.3-5-g9baa From e7c7ce646f37b260fe5a5635bc52243d28125dd8 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Mon, 17 Aug 2020 16:14:48 -0500 Subject: cmd/compile: combine multiply/add into maddld on ppc64le/power9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new lowering rule to match and replace such instances with the MADDLD instruction available on power9 where possible. Likewise, this plumbs in a new ppc64 ssa opcode to house the newly generated MADDLD instructions. When testing ed25519, this reduced binary size by 936B. Similarly, MADDLD combination occcurs in a few other less obvious cases such as division by constant. Testing of golang.org/x/crypto/ed25519 shows non-trivial speedup during keygeneration: name old time/op new time/op delta KeyGeneration 65.2µs ± 0% 63.1µs ± 0% -3.19% Signing 64.3µs ± 0% 64.4µs ± 0% +0.16% Verification 147µs ± 0% 147µs ± 0% +0.11% Similarly, this test binary has shrunk by 66488B. Change-Id: I077aeda7943119b41f07e4e62e44a648f16e4ad0 Reviewed-on: https://go-review.googlesource.com/c/go/+/248723 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Lynn Boger --- src/cmd/compile/internal/ppc64/ssa.go | 14 ++++++++++++++ src/cmd/compile/internal/ssa/gen/PPC64.rules | 3 +++ src/cmd/compile/internal/ssa/gen/PPC64Ops.go | 2 ++ src/cmd/compile/internal/ssa/opGen.go | 16 ++++++++++++++++ src/cmd/compile/internal/ssa/rewritePPC64.go | 21 +++++++++++++++++++++ test/codegen/arithmetic.go | 12 ++++++++---- 6 files changed, 64 insertions(+), 4 deletions(-) diff --git a/src/cmd/compile/internal/ppc64/ssa.go b/src/cmd/compile/internal/ppc64/ssa.go index 0efdd710fb..4d2ad48135 100644 --- a/src/cmd/compile/internal/ppc64/ssa.go +++ b/src/cmd/compile/internal/ppc64/ssa.go @@ -601,6 +601,20 @@ func ssaGenValue(s *gc.SSAGenState, v *ssa.Value) { p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() + case ssa.OpPPC64MADDLD: + r := v.Reg() + r1 := v.Args[0].Reg() + r2 := v.Args[1].Reg() + r3 := v.Args[2].Reg() + // r = r1*r2 ± r3 + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = r1 + p.Reg = r2 + p.SetFrom3(obj.Addr{Type: obj.TYPE_REG, Reg: r3}) + p.To.Type = obj.TYPE_REG + p.To.Reg = r + case ssa.OpPPC64FMADD, ssa.OpPPC64FMADDS, ssa.OpPPC64FMSUB, ssa.OpPPC64FMSUBS: r := v.Reg() r1 := v.Args[0].Reg() diff --git a/src/cmd/compile/internal/ssa/gen/PPC64.rules b/src/cmd/compile/internal/ssa/gen/PPC64.rules index fd28e10098..14942d50f9 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64.rules +++ b/src/cmd/compile/internal/ssa/gen/PPC64.rules @@ -11,6 +11,9 @@ (Sub32F ...) => (FSUBS ...) (Sub64F ...) => (FSUB ...) +// Combine 64 bit integer multiply and adds +(ADD l:(MULLD x y) z) && objabi.GOPPC64 >= 9 && l.Uses == 1 && clobber(l) => (MADDLD x y z) + (Mod16 x y) => (Mod32 (SignExt16to32 x) (SignExt16to32 y)) (Mod16u x y) => (Mod32u (ZeroExt16to32 x) (ZeroExt16to32 y)) (Mod8 x y) => (Mod32 (SignExt8to32 x) (SignExt8to32 y)) diff --git a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go index 0261dc283b..825d0faf34 100644 --- a/src/cmd/compile/internal/ssa/gen/PPC64Ops.go +++ b/src/cmd/compile/internal/ssa/gen/PPC64Ops.go @@ -137,6 +137,7 @@ func init() { gp01 = regInfo{inputs: nil, outputs: []regMask{gp}} gp11 = regInfo{inputs: []regMask{gp | sp | sb}, outputs: []regMask{gp}} gp21 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp}} + gp31 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp}} gp22 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp, gp}} gp32 = regInfo{inputs: []regMask{gp | sp | sb, gp | sp | sb, gp | sp | sb}, outputs: []regMask{gp, gp}} gp1cr = regInfo{inputs: []regMask{gp | sp | sb}} @@ -179,6 +180,7 @@ func init() { {name: "MULLD", argLength: 2, reg: gp21, asm: "MULLD", typ: "Int64", commutative: true}, // arg0*arg1 (signed 64-bit) {name: "MULLW", argLength: 2, reg: gp21, asm: "MULLW", typ: "Int32", commutative: true}, // arg0*arg1 (signed 32-bit) + {name: "MADDLD", argLength: 3, reg: gp31, asm: "MADDLD", typ: "Int64"}, // (arg0*arg1)+arg2 (signed 64-bit) {name: "MULHD", argLength: 2, reg: gp21, asm: "MULHD", commutative: true}, // (arg0 * arg1) >> 64, signed {name: "MULHW", argLength: 2, reg: gp21, asm: "MULHW", commutative: true}, // (arg0 * arg1) >> 32, signed diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index df2a27368b..4cd72799e8 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -1832,6 +1832,7 @@ const ( OpPPC64FSUBS OpPPC64MULLD OpPPC64MULLW + OpPPC64MADDLD OpPPC64MULHD OpPPC64MULHW OpPPC64MULHDU @@ -24374,6 +24375,21 @@ var opcodeTable = [...]opInfo{ }, }, }, + { + name: "MADDLD", + argLen: 3, + asm: ppc64.AMADDLD, + reg: regInfo{ + inputs: []inputInfo{ + {0, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + {1, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + {2, 1073733630}, // SP SB R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + }, + outputs: []outputInfo{ + {0, 1073733624}, // R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R14 R15 R16 R17 R18 R19 R20 R21 R22 R23 R24 R25 R26 R27 R28 R29 + }, + }, + }, { name: "MULHD", argLen: 2, diff --git a/src/cmd/compile/internal/ssa/rewritePPC64.go b/src/cmd/compile/internal/ssa/rewritePPC64.go index 37b75cc58a..7704b80dc6 100644 --- a/src/cmd/compile/internal/ssa/rewritePPC64.go +++ b/src/cmd/compile/internal/ssa/rewritePPC64.go @@ -3852,6 +3852,27 @@ func rewriteValuePPC64_OpPPC64ADD(v *Value) bool { v_0 := v.Args[0] b := v.Block typ := &b.Func.Config.Types + // match: (ADD l:(MULLD x y) z) + // cond: objabi.GOPPC64 >= 9 && l.Uses == 1 && clobber(l) + // result: (MADDLD x y z) + for { + for _i0 := 0; _i0 <= 1; _i0, v_0, v_1 = _i0+1, v_1, v_0 { + l := v_0 + if l.Op != OpPPC64MULLD { + continue + } + y := l.Args[1] + x := l.Args[0] + z := v_1 + if !(objabi.GOPPC64 >= 9 && l.Uses == 1 && clobber(l)) { + continue + } + v.reset(OpPPC64MADDLD) + v.AddArg3(x, y, z) + return true + } + break + } // match: (ADD (SLDconst x [c]) (SRDconst x [d])) // cond: d == 64-c // result: (ROTLconst [c] x) diff --git a/test/codegen/arithmetic.go b/test/codegen/arithmetic.go index 9f30ec8ce4..45fdb68903 100644 --- a/test/codegen/arithmetic.go +++ b/test/codegen/arithmetic.go @@ -253,16 +253,20 @@ func Divisible(n1 uint, n2 int) (bool, bool, bool, bool) { // 386:"IMUL3L\t[$]-1431655765","ADDL\t[$]715827882","ROLL\t[$]31",-"DIVQ" // arm64:"MUL","ADD\t[$]3074457345618258602","ROR",-"DIV" // arm:"MUL","ADD\t[$]715827882",-".*udiv" - // ppc64:"MULLD","ADD","ROTL\t[$]63" - // ppc64le:"MULLD","ADD","ROTL\t[$]63" + // ppc64/power8:"MULLD","ADD","ROTL\t[$]63" + // ppc64le/power8:"MULLD","ADD","ROTL\t[$]63" + // ppc64/power9:"MADDLD","ROTL\t[$]63" + // ppc64le/power9:"MADDLD","ROTL\t[$]63" evenS := n2%6 == 0 // amd64:"IMULQ","ADD",-"ROLQ",-"DIVQ" // 386:"IMUL3L\t[$]678152731","ADDL\t[$]113025455",-"ROLL",-"DIVQ" // arm64:"MUL","ADD\t[$]485440633518672410",-"ROR",-"DIV" // arm:"MUL","ADD\t[$]113025455",-".*udiv" - // ppc64:"MULLD","ADD",-"ROTL" - // ppc64le:"MULLD","ADD",-"ROTL" + // ppc64/power8:"MULLD","ADD",-"ROTL" + // ppc64/power9:"MADDLD",-"ROTL" + // ppc64le/power8:"MULLD","ADD",-"ROTL" + // ppc64le/power9:"MADDLD",-"ROTL" oddS := n2%19 == 0 return evenU, oddU, evenS, oddS -- cgit v1.3-5-g9baa From 84a62453e5c01df3f7d0c48d9aca32832c2052c1 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Fri, 8 May 2020 17:12:45 -0400 Subject: cmd/compile: remove unnecessary error condition on reading fingerprint io.ReadFull guarantees n == len(buf) if and only if err == nil, so the length check is redundant. Change-Id: I15bff97868e27a65648acd791883cac8dab77630 Reviewed-on: https://go-review.googlesource.com/c/go/+/232988 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Dmitri Shuralyov --- src/cmd/compile/internal/gc/iimport.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cmd/compile/internal/gc/iimport.go b/src/cmd/compile/internal/gc/iimport.go index 0eeb047c06..4169222c14 100644 --- a/src/cmd/compile/internal/gc/iimport.go +++ b/src/cmd/compile/internal/gc/iimport.go @@ -191,9 +191,9 @@ func iimport(pkg *types.Pkg, in *bio.Reader) (fingerprint goobj.FingerprintType) } } - // Fingerprint - n, err := io.ReadFull(in, fingerprint[:]) - if err != nil || n != len(fingerprint) { + // Fingerprint. + _, err = io.ReadFull(in, fingerprint[:]) + if err != nil { yyerror("import %s: error reading fingerprint", pkg.Path) errorexit() } -- cgit v1.3-5-g9baa From 0941fc3f9ff43598d25fa6e964e7829a268102bf Mon Sep 17 00:00:00 2001 From: cui Date: Wed, 12 Aug 2020 17:33:41 +0000 Subject: runtime: reduce syscall when call runtime.clone Change-Id: I3ea398fd86aae4c86557dd6fff65d90a6f756890 GitHub-Last-Rev: 4c295388f7b5e6768ffd2530337f78b4c75a9310 GitHub-Pull-Request: golang/go#40392 Reviewed-on: https://go-review.googlesource.com/c/go/+/244626 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/sys_linux_amd64.s | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 621c01b365..8d90813589 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -592,13 +592,25 @@ TEXT runtime·clone(SB),NOSPLIT,$0 MOVQ stk+8(FP), SI MOVQ $0, DX MOVQ $0, R10 - + MOVQ $0, R8 // Copy mp, gp, fn off parent stack for use by child. // Careful: Linux system call clobbers CX and R11. - MOVQ mp+16(FP), R8 + MOVQ mp+16(FP), R13 MOVQ gp+24(FP), R9 MOVQ fn+32(FP), R12 - + CMPQ R13, $0 // m + JEQ nog1 + CMPQ R9, $0 // g + JEQ nog1 + LEAQ m_tls(R13), R8 +#ifdef GOOS_android + // Android stores the TLS offset in runtime·tls_g. + SUBQ runtime·tls_g(SB), R8 +#else + ADDQ $8, R8 // ELF wants to use -8(FS) +#endif + ORQ $0x00080000, DI //add flag CLONE_SETTLS(0x00080000) to call clone +nog1: MOVL $SYS_clone, AX SYSCALL @@ -612,27 +624,23 @@ TEXT runtime·clone(SB),NOSPLIT,$0 MOVQ SI, SP // If g or m are nil, skip Go-related setup. - CMPQ R8, $0 // m - JEQ nog + CMPQ R13, $0 // m + JEQ nog2 CMPQ R9, $0 // g - JEQ nog + JEQ nog2 // Initialize m->procid to Linux tid MOVL $SYS_gettid, AX SYSCALL - MOVQ AX, m_procid(R8) - - // Set FS to point at m->tls. - LEAQ m_tls(R8), DI - CALL runtime·settls(SB) + MOVQ AX, m_procid(R13) // In child, set up new stack get_tls(CX) - MOVQ R8, g_m(R9) + MOVQ R13, g_m(R9) MOVQ R9, g(CX) CALL runtime·stackcheck(SB) -nog: +nog2: // Call fn CALL R12 -- cgit v1.3-5-g9baa From 98a0071a5363e307c2e284034f810378de3883dd Mon Sep 17 00:00:00 2001 From: Andrew Ekstedt Date: Fri, 24 Jul 2020 12:48:30 -0700 Subject: path,path/filepath: add Join examples with ".." components People sometimes expect Join to trim .. components from its arguments before joining, and are surprised that it doesn't. This is bad if they were relying on that assumed behaviour to prevent directory traversal attacks. While a careful reading of the documentation for Join and Clean might dispel this notion, it is not obvious at first glance. Add a case to the examples to nudge people in the right direction. Updates #40373 Change-Id: Ib5792c12ba1000811a0c0eb77048196d0b26da60 Reviewed-on: https://go-review.googlesource.com/c/go/+/249177 Run-TryBot: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Reviewed-by: Rob Pike --- src/path/example_test.go | 5 +++++ src/path/filepath/example_unix_test.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/src/path/example_test.go b/src/path/example_test.go index 67b9718664..e30ebd13dc 100644 --- a/src/path/example_test.go +++ b/src/path/example_test.go @@ -79,13 +79,18 @@ func ExampleJoin() { fmt.Println(path.Join("a", "b", "c")) fmt.Println(path.Join("a", "b/c")) fmt.Println(path.Join("a/b", "c")) + + fmt.Println(path.Join("a/b", "../../../xyz")) + fmt.Println(path.Join("", "")) fmt.Println(path.Join("a", "")) fmt.Println(path.Join("", "a")) + // Output: // a/b/c // a/b/c // a/b/c + // ../xyz // // a // a diff --git a/src/path/filepath/example_unix_test.go b/src/path/filepath/example_unix_test.go index 23f21380d0..c9d6944518 100644 --- a/src/path/filepath/example_unix_test.go +++ b/src/path/filepath/example_unix_test.go @@ -72,12 +72,16 @@ func ExampleJoin() { fmt.Println(filepath.Join("a", "b/c")) fmt.Println(filepath.Join("a/b", "c")) fmt.Println(filepath.Join("a/b", "/c")) + + fmt.Println(filepath.Join("a/b", "../../../xyz")) + // Output: // On Unix: // a/b/c // a/b/c // a/b/c // a/b/c + // ../xyz } func ExampleMatch() { -- cgit v1.3-5-g9baa From ac875bc923db2b7350f244f06a06557e6fd97e05 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Sat, 15 Aug 2020 00:44:58 +0700 Subject: cmd/compile: don't bother to declare closure inside redeclared func Fixes #17758 Change-Id: I75f5dc5be85fd8a6791ac89dfc0681be759cca36 Reviewed-on: https://go-review.googlesource.com/c/go/+/248517 Run-TryBot: Cuong Manh Le Reviewed-by: Matthew Dempsky TryBot-Result: Gobot Gobot --- src/cmd/compile/internal/gc/closure.go | 12 +++++++++++- test/fixedbugs/issue17758.go | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 test/fixedbugs/issue17758.go diff --git a/src/cmd/compile/internal/gc/closure.go b/src/cmd/compile/internal/gc/closure.go index 04fb7d5495..23e48939b4 100644 --- a/src/cmd/compile/internal/gc/closure.go +++ b/src/cmd/compile/internal/gc/closure.go @@ -108,7 +108,17 @@ func typecheckclosure(clo *Node, top int) { xfunc.Func.Nname.Sym = closurename(Curfn) disableExport(xfunc.Func.Nname.Sym) - declare(xfunc.Func.Nname, PFUNC) + if xfunc.Func.Nname.Sym.Def != nil { + // The only case we can reach here is when the outer function was redeclared. + // In that case, don't bother to redeclare the closure. Otherwise, we will get + // a spurious error message, see #17758. While we are here, double check that + // we already reported other error. + if nsavederrors+nerrors == 0 { + Fatalf("unexpected symbol collision %v", xfunc.Func.Nname.Sym) + } + } else { + declare(xfunc.Func.Nname, PFUNC) + } xfunc = typecheck(xfunc, ctxStmt) // Type check the body now, but only if we're inside a function. diff --git a/test/fixedbugs/issue17758.go b/test/fixedbugs/issue17758.go new file mode 100644 index 0000000000..e7f2f3af91 --- /dev/null +++ b/test/fixedbugs/issue17758.go @@ -0,0 +1,17 @@ +// errorcheck + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +func foo() { + _ = func() {} +} + +func foo() { // ERROR "foo redeclared in this block" + _ = func() {} +} + +func main() {} -- cgit v1.3-5-g9baa From 31da1d993a498acefcf3dd5fdfbefa8eec132791 Mon Sep 17 00:00:00 2001 From: Tao Qingyun Date: Wed, 19 Aug 2020 01:38:43 +0000 Subject: cmd/internal/objfile: cache computation of goobj.Arch Change-Id: I23774cf185e5fa6b89398001cd0655fb0c5bdb46 GitHub-Last-Rev: ca8cae2469b5fad84bd636a3305a484dfdcb0db2 GitHub-Pull-Request: golang/go#40877 Reviewed-on: https://go-review.googlesource.com/c/go/+/249180 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/internal/archive/archive.go | 6 ++++++ src/cmd/internal/objfile/goobj.go | 32 +++++++++++++------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/cmd/internal/archive/archive.go b/src/cmd/internal/archive/archive.go index db67ce424b..c1661d7711 100644 --- a/src/cmd/internal/archive/archive.go +++ b/src/cmd/internal/archive/archive.go @@ -17,6 +17,7 @@ import ( "log" "os" "strconv" + "strings" "time" "unicode/utf8" ) @@ -83,6 +84,7 @@ func (e *Entry) String() string { type GoObj struct { TextHeader []byte + Arch string Data } @@ -404,6 +406,10 @@ func (r *objReader) parseObject(o *GoObj, size int64) error { } } o.TextHeader = h + hs := strings.Fields(string(h)) + if len(hs) >= 4 { + o.Arch = hs[3] + } o.Offset = r.offset o.Size = size - int64(len(h)) diff --git a/src/cmd/internal/objfile/goobj.go b/src/cmd/internal/objfile/goobj.go index e838f58aed..af9ada3324 100644 --- a/src/cmd/internal/objfile/goobj.go +++ b/src/cmd/internal/objfile/goobj.go @@ -17,13 +17,13 @@ import ( "fmt" "io" "os" - "strings" ) type goobjFile struct { goobj *archive.GoObj r *goobj.Reader f *os.File + arch *sys.Arch } func openGoFile(f *os.File) (*File, error) { @@ -45,9 +45,16 @@ L: return nil, err } r := goobj.NewReaderFromBytes(b, false) + var arch *sys.Arch + for _, a := range sys.Archs { + if a.Name == e.Obj.Arch { + arch = a + break + } + } entries = append(entries, &Entry{ name: e.Name, - raw: &goobjFile{e.Obj, r, f}, + raw: &goobjFile{e.Obj, r, f, arch}, }) continue case archive.EntryNativeObj: @@ -223,17 +230,8 @@ func (f *goobjFile) pcln() (textStart uint64, symtab, pclntab []byte, err error) // Returns "",0,nil if unknown. // This function implements the Liner interface in preference to pcln() above. func (f *goobjFile) PCToLine(pc uint64) (string, int, *gosym.Func) { - // TODO: this is really inefficient. Binary search? Memoize last result? r := f.r - var arch *sys.Arch - archname := f.goarch() - for _, a := range sys.Archs { - if a.Name == archname { - arch = a - break - } - } - if arch == nil { + if f.arch == nil { return "", 0, nil } pcdataBase := r.PcdataBase() @@ -264,10 +262,10 @@ func (f *goobjFile) PCToLine(pc uint64) (string, int, *gosym.Func) { lengths := info.ReadFuncInfoLengths(b) off, end := info.ReadPcline(b) pcline := r.BytesAt(pcdataBase+off, int(end-off)) - line := int(pcValue(pcline, pc-addr, arch)) + line := int(pcValue(pcline, pc-addr, f.arch)) off, end = info.ReadPcfile(b) pcfile := r.BytesAt(pcdataBase+off, int(end-off)) - fileID := pcValue(pcfile, pc-addr, arch) + fileID := pcValue(pcfile, pc-addr, f.arch) globalFileID := info.ReadFile(b, lengths.FileOff, uint32(fileID)) fileName := r.File(int(globalFileID)) // Note: we provide only the name in the Func structure. @@ -332,11 +330,7 @@ func (f *goobjFile) text() (textStart uint64, text []byte, err error) { } func (f *goobjFile) goarch() string { - hs := strings.Fields(string(f.goobj.TextHeader)) - if len(hs) >= 4 { - return hs[3] - } - return "" + return f.goobj.Arch } func (f *goobjFile) loadAddress() (uint64, error) { -- cgit v1.3-5-g9baa From bd519d0c8734c3e30cb1a8b8217dd9934cd61e25 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Thu, 25 Jun 2020 14:50:10 -0700 Subject: runtime: don't call setitimer for each thread Previously, on Unix systems, when the profiler was enabled or disabled, we called setitimer once per thread. With this change we instead call it once per process. Change-Id: I90f0189b562e11232816390dc7d55ed154bd836d Reviewed-on: https://go-review.googlesource.com/c/go/+/240003 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/signal_unix.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 6a11c91fb9..064a0ea100 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -272,6 +272,12 @@ func setProcessCPUProfiler(hz int32) { atomic.Storeuintptr(&fwdSig[_SIGPROF], getsig(_SIGPROF)) setsig(_SIGPROF, funcPC(sighandler)) } + + var it itimerval + it.it_interval.tv_sec = 0 + it.it_interval.set_usec(1000000 / hz) + it.it_value = it.it_interval + setitimer(_ITIMER_PROF, &it, nil) } else { // If the Go signal handler should be disabled by default, // switch back to the signal handler that was installed @@ -296,23 +302,16 @@ func setProcessCPUProfiler(hz int32) { setsig(_SIGPROF, h) } } + + setitimer(_ITIMER_PROF, &itimerval{}, nil) } } // setThreadCPUProfiler makes any thread-specific changes required to // implement profiling at a rate of hz. +// No changes required on Unix systems. func setThreadCPUProfiler(hz int32) { - var it itimerval - if hz == 0 { - setitimer(_ITIMER_PROF, &it, nil) - } else { - it.it_interval.tv_sec = 0 - it.it_interval.set_usec(1000000 / hz) - it.it_value = it.it_interval - setitimer(_ITIMER_PROF, &it, nil) - } - _g_ := getg() - _g_.m.profilehz = hz + getg().m.profilehz = hz } func sigpipe() { -- cgit v1.3-5-g9baa From d3a411b6debccb665da3497e7fa597c9a5ff16f1 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 12 Aug 2020 12:09:35 -0700 Subject: internal/poll: treat copy_file_range EOPNOTSUPP as not-handled Fixes #40731 Change-Id: I3e29878d597318acf5edcc38497aa2624f72be35 Reviewed-on: https://go-review.googlesource.com/c/go/+/248258 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Emmanuel Odeke Reviewed-by: Tobias Klauser --- src/internal/poll/copy_file_range_linux.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/internal/poll/copy_file_range_linux.go b/src/internal/poll/copy_file_range_linux.go index 604607f774..7e67125818 100644 --- a/src/internal/poll/copy_file_range_linux.go +++ b/src/internal/poll/copy_file_range_linux.go @@ -41,7 +41,7 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // use copy_file_range(2) again. atomic.StoreInt32(©FileRangeSupported, 0) return 0, false, nil - case syscall.EXDEV, syscall.EINVAL: + case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP: // Prior to Linux 5.3, it was not possible to // copy_file_range across file systems. Similarly to // the ENOSYS case above, if we see EXDEV, we have @@ -52,6 +52,9 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // dst or src refer to a pipe rather than a regular // file. This is another case where no data has been // transfered, so we consider it unhandled. + // + // If the file is on NFS, we can see EOPNOTSUPP. + // See issue #40731. return 0, false, nil case nil: if n == 0 { -- cgit v1.3-5-g9baa From 8d91d736e2eb518e0a95eb47ae463c0dd80827c4 Mon Sep 17 00:00:00 2001 From: Cherry Zhang Date: Tue, 18 Aug 2020 12:01:41 -0400 Subject: cmd/dist: ignore _test.s files in bootstrap Ignore all _test.s files in bootstrap, not only the ARM64 one. They are for testing only. Fixes #40855. Change-Id: I00e6b4ab5349e317c9ad3a503997de85aed49373 Reviewed-on: https://go-review.googlesource.com/c/go/+/249018 Reviewed-by: Ian Lance Taylor --- src/cmd/dist/buildtool.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index 40d28b535b..0b50f66c32 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -113,14 +113,15 @@ var ignorePrefixes = []string{ // File suffixes that use build tags introduced since Go 1.4. // These must not be copied into the bootstrap build directory. +// Also igonore test files. var ignoreSuffixes = []string{ "_arm64.s", - "_arm64_test.s", "_arm64.go", "_riscv64.s", "_riscv64.go", "_wasm.s", "_wasm.go", + "_test.s", } func bootstrapBuildTools() { -- cgit v1.3-5-g9baa From 01aad9ea939fed313d5c51778485349435302ead Mon Sep 17 00:00:00 2001 From: diaxu01 Date: Fri, 5 Jun 2020 03:53:53 +0000 Subject: cmd/compile: Optimize ARM64's code with EON This patch fuses pattern '(MVN (XOR x y))' into '(EON x y)'. Change-Id: I269c98ce198d51a4945ce8bd0e1024acbd1b7609 Reviewed-on: https://go-review.googlesource.com/c/go/+/239638 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/ssa/gen/ARM64.rules | 1 + src/cmd/compile/internal/ssa/rewriteARM64.go | 12 ++++++++++++ test/codegen/bits.go | 13 +++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/cmd/compile/internal/ssa/gen/ARM64.rules b/src/cmd/compile/internal/ssa/gen/ARM64.rules index 27959d01fc..80e8c7137b 100644 --- a/src/cmd/compile/internal/ssa/gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/gen/ARM64.rules @@ -1323,6 +1323,7 @@ (AND x (MVN y)) -> (BIC x y) (XOR x (MVN y)) -> (EON x y) (OR x (MVN y)) -> (ORN x y) +(MVN (XOR x y)) -> (EON x y) (CSEL {cc} x (MOVDconst [0]) flag) -> (CSEL0 {cc} x flag) (CSEL {cc} (MOVDconst [0]) y flag) -> (CSEL0 {arm64Negate(cc.(Op))} y flag) (SUB x (SUB y z)) -> (SUB (ADD x z) y) diff --git a/src/cmd/compile/internal/ssa/rewriteARM64.go b/src/cmd/compile/internal/ssa/rewriteARM64.go index 023d9908c2..842eddbf4a 100644 --- a/src/cmd/compile/internal/ssa/rewriteARM64.go +++ b/src/cmd/compile/internal/ssa/rewriteARM64.go @@ -14593,6 +14593,18 @@ func rewriteValueARM64_OpARM64MULW(v *Value) bool { } func rewriteValueARM64_OpARM64MVN(v *Value) bool { v_0 := v.Args[0] + // match: (MVN (XOR x y)) + // result: (EON x y) + for { + if v_0.Op != OpARM64XOR { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + v.reset(OpARM64EON) + v.AddArg2(x, y) + return true + } // match: (MVN (MOVDconst [c])) // result: (MOVDconst [^c]) for { diff --git a/test/codegen/bits.go b/test/codegen/bits.go index 0a5428b55a..398dd84e9e 100644 --- a/test/codegen/bits.go +++ b/test/codegen/bits.go @@ -310,9 +310,18 @@ func op_bic(x, y uint32) uint32 { return x &^ y } -func op_eon(x, y uint32) uint32 { +func op_eon(x, y, z uint32, a []uint32, n, m uint64) uint64 { + // arm64:`EON\t`,-`EOR`,-`MVN` + a[0] = x ^ (y ^ 0xffffffff) + + // arm64:`EON\t`,-`EOR`,-`MVN` + a[1] = ^(y ^ z) + // arm64:`EON\t`,-`XOR` - return x ^ ^y + a[2] = x ^ ^z + + // arm64:`EON\t`,-`EOR`,-`MVN` + return n ^ (m ^ 0xffffffffffffffff) } func op_orn(x, y uint32) uint32 { -- cgit v1.3-5-g9baa From 64350f1eabeb688e997c6cd0c103e21c02ab2a46 Mon Sep 17 00:00:00 2001 From: "Paul E. Murphy" Date: Tue, 18 Aug 2020 16:26:42 -0500 Subject: cmd/asm,cmd/internal/obj/ppc64: add {l,st}xvx power9 instructions These are the indexed vsx load operations with the same endian and alignment benefits of {l,st}vx. Likewise, cleanup redundant comments in op{load,store}x and fix ISA 3.0 typos nearby. Change-Id: Ie1ace17c6150cf9168a834e435114028ff6eb07c Reviewed-on: https://go-review.googlesource.com/c/go/+/249025 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Lynn Boger --- src/cmd/asm/internal/asm/testdata/ppc64.s | 2 ++ src/cmd/asm/internal/asm/testdata/ppc64enc.s | 2 ++ src/cmd/internal/obj/ppc64/asm9.go | 30 ++++++++++++++++------------ 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/cmd/asm/internal/asm/testdata/ppc64.s b/src/cmd/asm/internal/asm/testdata/ppc64.s index b3736bf6a4..ba64d84a35 100644 --- a/src/cmd/asm/internal/asm/testdata/ppc64.s +++ b/src/cmd/asm/internal/asm/testdata/ppc64.s @@ -1037,6 +1037,7 @@ label1: // VSX load with length X-form (also left-justified) LXVL R3,R4, VS0 LXVLL R3,R4, VS0 + LXVX R3,R4, VS0 // VSX load, DQ-form // DQ(RA), XS produces // XS, DQ(RA) @@ -1060,6 +1061,7 @@ label1: // VSX store with length, X-form (also left-justified) STXVL VS0, R3,R4 STXVLL VS0, R3,R4 + STXVX VS0, R3,R4 // VSX move from VSR, XX1-form // XS,RA produces diff --git a/src/cmd/asm/internal/asm/testdata/ppc64enc.s b/src/cmd/asm/internal/asm/testdata/ppc64enc.s index 07a8a540cd..10a05ec402 100644 --- a/src/cmd/asm/internal/asm/testdata/ppc64enc.s +++ b/src/cmd/asm/internal/asm/testdata/ppc64enc.s @@ -595,11 +595,13 @@ TEXT asmtest(SB),DUPOK|NOSPLIT,$0 LXV 16(R3), VS1 // f4230011 LXVL R3, R4, VS1 // 7c23221a LXVLL R3, R4, VS1 // 7c23225a + LXVX R3, R4, VS1 // 7c232218 LXSDX (R3)(R4), VS1 // 7c241c98 STXVD2X VS1, (R3)(R4) // 7c241f98 STXV VS1,16(R3) // f4230015 STXVL VS1, R3, R4 // 7c23231a STXVLL VS1, R3, R4 // 7c23235a + STXVX VS1, R3, R4 // 7c232318 STXSDX VS1, (R3)(R4) // 7c241d98 LXSIWAX (R3)(R4), VS1 // 7c241898 STXSIWX VS1, (R3)(R4) // 7c241918 diff --git a/src/cmd/internal/obj/ppc64/asm9.go b/src/cmd/internal/obj/ppc64/asm9.go index 238ca8f0b7..3c82477fc4 100644 --- a/src/cmd/internal/obj/ppc64/asm9.go +++ b/src/cmd/internal/obj/ppc64/asm9.go @@ -1584,8 +1584,9 @@ func buildop(ctxt *obj.Link) { case ALXV: /* lxv */ opset(ALXV, r0) - case ALXVL: /* lxvl */ + case ALXVL: /* lxvl, lxvll, lxvx */ opset(ALXVLL, r0) + opset(ALXVX, r0) case ASTXVD2X: /* stxvd2x, stxvdsx, stxvw4x, stxvh8x, stxvb16x */ opset(ASTXVW4X, r0) @@ -1595,8 +1596,9 @@ func buildop(ctxt *obj.Link) { case ASTXV: /* stxv */ opset(ASTXV, r0) - case ASTXVL: /* stxvl, stxvll */ + case ASTXVL: /* stxvl, stxvll, stvx */ opset(ASTXVLL, r0) + opset(ASTXVX, r0) case ALXSDX: /* lxsdx */ opset(ALXSDX, r0) @@ -5020,11 +5022,13 @@ func (c *ctxt9) opload(a obj.As) uint32 { case AMOVW: return OPVCC(58, 0, 0, 0) | 1<<1 /* lwa */ case ALXV: - return OPDQ(61, 1, 0) /* lxv - ISA v3.00 */ + return OPDQ(61, 1, 0) /* lxv - ISA v3.0 */ case ALXVL: - return OPVXX1(31, 269, 0) /* lxvl - ISA v3.00 */ + return OPVXX1(31, 269, 0) /* lxvl - ISA v3.0 */ case ALXVLL: - return OPVXX1(31, 301, 0) /* lxvll - ISA v3.00 */ + return OPVXX1(31, 301, 0) /* lxvll - ISA v3.0 */ + case ALXVX: + return OPVXX1(31, 268, 0) /* lxvx - ISA v3.0 */ /* no AMOVWU */ case AMOVB, AMOVBZ: @@ -5122,8 +5126,6 @@ func (c *ctxt9) oploadx(a obj.As) uint32 { return OPVCC(31, 309, 0, 0) /* ldmx */ /* Vector (VMX/Altivec) instructions */ - /* ISA 2.03 enables these for PPC970. For POWERx processors, these */ - /* are enabled starting at POWER6 (ISA 2.05). */ case ALVEBX: return OPVCC(31, 7, 0, 0) /* lvebx - v2.03 */ case ALVEHX: @@ -5141,7 +5143,8 @@ func (c *ctxt9) oploadx(a obj.As) uint32 { /* End of vector instructions */ /* Vector scalar (VSX) instructions */ - /* ISA 2.06 enables these for POWER7. */ + case ALXVX: + return OPVXX1(31, 268, 0) /* lxvx - ISA v3.0 */ case ALXVD2X: return OPVXX1(31, 844, 0) /* lxvd2x - v2.06 */ case ALXVW4X: @@ -5208,6 +5211,8 @@ func (c *ctxt9) opstore(a obj.As) uint32 { return OPVXX1(31, 397, 0) /* stxvl ISA 3.0 */ case ASTXVLL: return OPVXX1(31, 429, 0) /* stxvll ISA 3.0 */ + case ASTXVX: + return OPVXX1(31, 396, 0) /* stxvx - ISA v3.0 */ } @@ -5271,8 +5276,6 @@ func (c *ctxt9) opstorex(a obj.As) uint32 { return OPVCC(31, 181, 0, 0) /* stdux */ /* Vector (VMX/Altivec) instructions */ - /* ISA 2.03 enables these for PPC970. For POWERx processors, these */ - /* are enabled starting at POWER6 (ISA 2.05). */ case ASTVEBX: return OPVCC(31, 135, 0, 0) /* stvebx - v2.03 */ case ASTVEHX: @@ -5286,15 +5289,16 @@ func (c *ctxt9) opstorex(a obj.As) uint32 { /* End of vector instructions */ /* Vector scalar (VSX) instructions */ - /* ISA 2.06 enables these for POWER7. */ + case ASTXVX: + return OPVXX1(31, 396, 0) /* stxvx - v3.0 */ case ASTXVD2X: return OPVXX1(31, 972, 0) /* stxvd2x - v2.06 */ case ASTXVW4X: return OPVXX1(31, 908, 0) /* stxvw4x - v2.06 */ case ASTXVH8X: - return OPVXX1(31, 940, 0) /* stxvh8x - v3.00 */ + return OPVXX1(31, 940, 0) /* stxvh8x - v3.0 */ case ASTXVB16X: - return OPVXX1(31, 1004, 0) /* stxvb16x - v3.00 */ + return OPVXX1(31, 1004, 0) /* stxvb16x - v3.0 */ case ASTXSDX: return OPVXX1(31, 716, 0) /* stxsdx - v2.06 */ -- cgit v1.3-5-g9baa From 18239be10a0b5caa1b3222b228ff590b1c036382 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 25 Jun 2020 19:11:28 -0400 Subject: cmd/go/internal: remove some users of par.Work par.Work is used in a number of places as a parallel work queue. This change replaces it with goroutines and channels in a number of simpler places where it's used. This is the same CL as golang.org/cl/240062 and golang.org/cl/248326 except for the following changes in convert.go (all line numbers from this CL), as well as fixing up imports in download.go: - On line 44, the "*" before modules.Versions is removed (we were trying to assign to a nil value on lines 72 and 73). - Line 64 is new, and ensures that we receive on the semaphore channel once the goroutine function exits. (The previous versions of this CL only received at the end of the function, ignoring the return point in the branch in the middle of the function.) - The semaphore channel receive right before line 74 is gone, replaced with the deferred receive above. - The if block at line 83 is new, accounting for cases where modfetch.ImportRepoRev returned an error in the goroutine, so that versions[i] is ignored. Change-Id: I0e33670bb2eb0a1e4d7a5fa693a471e61ffbc8b7 Reviewed-on: https://go-review.googlesource.com/c/go/+/249020 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 69 +++++++++++++++++++--------------- src/cmd/go/internal/modcmd/graph.go | 19 +++++----- src/cmd/go/internal/modconv/convert.go | 59 ++++++++++++++++------------- src/cmd/go/internal/modget/get.go | 41 +++++++++++++------- src/cmd/go/internal/modload/list.go | 37 +++++++++++------- 5 files changed, 134 insertions(+), 91 deletions(-) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 946e8ed3cf..2f42d4306b 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -8,12 +8,12 @@ import ( "context" "encoding/json" "os" + "runtime" "cmd/go/internal/base" "cmd/go/internal/cfg" - "cmd/go/internal/modfetch" "cmd/go/internal/modload" - "cmd/go/internal/par" + "cmd/go/internal/modfetch" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -102,33 +102,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { } } - var mods []*moduleJSON - var work par.Work - listU := false - listVersions := false - for _, info := range modload.ListModules(ctx, args, listU, listVersions) { - if info.Replace != nil { - info = info.Replace - } - if info.Version == "" && info.Error == nil { - // main module or module replaced with file path. - // Nothing to download. - continue - } - m := &moduleJSON{ - Path: info.Path, - Version: info.Version, - } - mods = append(mods, m) - if info.Error != nil { - m.Error = info.Error.Err - continue - } - work.Add(m) - } - - work.Do(10, func(item interface{}) { - m := item.(*moduleJSON) + downloadModule := func(m *moduleJSON) { var err error m.Info, err = modfetch.InfoFile(m.Path, m.Version) if err != nil { @@ -157,7 +131,42 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { m.Error = err.Error() return } - }) + } + + var mods []*moduleJSON + listU := false + listVersions := false + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, info := range modload.ListModules(ctx, args, listU, listVersions) { + if info.Replace != nil { + info = info.Replace + } + if info.Version == "" && info.Error == nil { + // main module or module replaced with file path. + // Nothing to download. + continue + } + m := &moduleJSON{ + Path: info.Path, + Version: info.Version, + } + mods = append(mods, m) + if info.Error != nil { + m.Error = info.Error.Err + continue + } + sem <- token{} + go func() { + downloadModule(m) + <-sem + }() + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } if *downloadJSON { for _, m := range mods { diff --git a/src/cmd/go/internal/modcmd/graph.go b/src/cmd/go/internal/modcmd/graph.go index 4853503fd4..6da12b9cab 100644 --- a/src/cmd/go/internal/modcmd/graph.go +++ b/src/cmd/go/internal/modcmd/graph.go @@ -15,7 +15,6 @@ import ( "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modload" - "cmd/go/internal/par" "cmd/go/internal/work" "golang.org/x/mod/module" @@ -59,23 +58,25 @@ func runGraph(ctx context.Context, cmd *base.Command, args []string) { return m.Path + "@" + m.Version } - // Note: using par.Work only to manage work queue. - // No parallelism here, so no locking. var out []string var deps int // index in out where deps start - var work par.Work - work.Add(modload.Target) - work.Do(1, func(item interface{}) { - m := item.(module.Version) + seen := map[module.Version]bool{modload.Target: true} + queue := []module.Version{modload.Target} + for len(queue) > 0 { + var m module.Version + m, queue = queue[0], queue[1:] list, _ := reqs.Required(m) for _, r := range list { - work.Add(r) + if !seen[r] { + queue = append(queue, r) + seen[r] = true + } out = append(out, format(m)+" "+format(r)+"\n") } if m == modload.Target { deps = len(out) } - }) + } sort.Slice(out[deps:], func(i, j int) bool { return out[deps+i][0] < out[deps+j][0] diff --git a/src/cmd/go/internal/modconv/convert.go b/src/cmd/go/internal/modconv/convert.go index f465a9f395..5d4165c944 100644 --- a/src/cmd/go/internal/modconv/convert.go +++ b/src/cmd/go/internal/modconv/convert.go @@ -7,13 +7,12 @@ package modconv import ( "fmt" "os" + "runtime" "sort" "strings" - "sync" "cmd/go/internal/base" "cmd/go/internal/modfetch" - "cmd/go/internal/par" "golang.org/x/mod/modfile" "golang.org/x/mod/module" @@ -42,46 +41,54 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error { // Convert requirements block, which may use raw SHA1 hashes as versions, // to valid semver requirement list, respecting major versions. - var ( - work par.Work - mu sync.Mutex - need = make(map[string]string) - replace = make(map[string]*modfile.Replace) - ) + versions := make([]module.Version, len(mf.Require)) + replace := make(map[string]*modfile.Replace) for _, r := range mf.Replace { replace[r.New.Path] = r replace[r.Old.Path] = r } - for _, r := range mf.Require { + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for i, r := range mf.Require { m := r.Mod if m.Path == "" { continue } if re, ok := replace[m.Path]; ok { - work.Add(re.New) - continue + m = re.New } - work.Add(r.Mod) + sem <- token{} + go func(i int, m module.Version) { + defer func() { <-sem }() + repo, info, err := modfetch.ImportRepoRev(m.Path, m.Version) + if err != nil { + fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), m.Path, m.Version, err) + return + } + + path := repo.ModulePath() + versions[i].Path = path + versions[i].Version = info.Version + }(i, m) + } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} } - work.Do(10, func(item interface{}) { - r := item.(module.Version) - repo, info, err := modfetch.ImportRepoRev(r.Path, r.Version) - if err != nil { - fmt.Fprintf(os.Stderr, "go: converting %s: stat %s@%s: %v\n", base.ShortPath(file), r.Path, r.Version, err) - return + need := map[string]string{} + for _, v := range versions { + if v.Path == "" { + continue } - mu.Lock() - path := repo.ModulePath() // Don't use semver.Max here; need to preserve +incompatible suffix. - if v, ok := need[path]; !ok || semver.Compare(v, info.Version) < 0 { - need[path] = info.Version + if needv, ok := need[v.Path]; !ok || semver.Compare(needv, v.Version) < 0 { + need[v.Path] = v.Version } - mu.Unlock() - }) - - var paths []string + } + paths := make([]string, 0, len(need)) for path := range need { paths = append(paths, path) } diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index 93a6bb54d5..d02c9a8da5 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -21,7 +22,6 @@ import ( "cmd/go/internal/load" "cmd/go/internal/modload" "cmd/go/internal/mvs" - "cmd/go/internal/par" "cmd/go/internal/search" "cmd/go/internal/work" @@ -725,18 +725,8 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // reported. A map from module paths to queries is returned, which includes // queries and modOnly. func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*query, modOnly map[string]*query) map[string]*query { - var lookup par.Work - for _, q := range queries { - if cached := cache[q.querySpec]; cached != nil { - *q = *cached - } else { - cache[q.querySpec] = q - lookup.Add(q) - } - } - lookup.Do(10, func(item interface{}) { - q := item.(*query) + runQuery := func(q *query) { if q.vers == "none" { // Wait for downgrade step. q.m = module.Version{Path: q.path, Version: "none"} @@ -747,7 +737,32 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer base.Errorf("go get %s: %v", q.arg, err) } q.m = m - }) + } + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) + for _, q := range queries { + if cached := cache[q.querySpec]; cached != nil { + *q = *cached + } else { + sem <- token{} + go func(q *query) { + runQuery(q) + <-sem + }(q) + } + } + + // Fill semaphore channel to wait for goroutines to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + + // Add to cache after concurrent section to avoid races... + for _, q := range queries { + cache[q.querySpec] = q + } + base.ExitIfErrors() byPath := make(map[string]*query) diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 4768516e90..8db4d64706 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "os" + "runtime" "strings" "cmd/go/internal/base" "cmd/go/internal/cfg" "cmd/go/internal/modinfo" - "cmd/go/internal/par" "cmd/go/internal/search" "golang.org/x/mod/module" @@ -22,24 +22,35 @@ import ( func ListModules(ctx context.Context, args []string, listU, listVersions bool) []*modinfo.ModulePublic { mods := listModules(ctx, args, listVersions) + + type token struct{} + sem := make(chan token, runtime.GOMAXPROCS(0)) if listU || listVersions { - var work par.Work for _, m := range mods { - work.Add(m) + add := func(m *modinfo.ModulePublic) { + sem <- token{} + go func() { + if listU { + addUpdate(m) + } + if listVersions { + addVersions(m) + } + <-sem + }() + } + + add(m) if m.Replace != nil { - work.Add(m.Replace) + add(m.Replace) } } - work.Do(10, func(item interface{}) { - m := item.(*modinfo.ModulePublic) - if listU { - addUpdate(m) - } - if listVersions { - addVersions(m) - } - }) } + // Fill semaphore channel to wait for all tasks to finish. + for n := cap(sem); n > 0; n-- { + sem <- token{} + } + return mods } -- cgit v1.3-5-g9baa From 6b420169d798c7ebe733487b56ea5c3fa4aab5ce Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Tue, 18 Aug 2020 16:46:24 -0700 Subject: os, internal/poll: loop on EINTR for all file syscalls When using a FUSE file system, any system call that touches the file system can return EINTR. Fixes #40846 Change-Id: I25d32da22cec08dea81ab297291a85ad72db2df7 Reviewed-on: https://go-review.googlesource.com/c/go/+/249178 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/internal/poll/fd_fsync_posix.go | 4 +++- src/internal/poll/fd_opendir_darwin.go | 8 ++++++- src/internal/poll/fd_posix.go | 28 +++++++++++++++++++++--- src/internal/poll/fd_unix.go | 22 ++++++++----------- src/os/dir_darwin.go | 3 +++ src/os/file.go | 5 ++++- src/os/file_plan9.go | 4 ++++ src/os/file_posix.go | 32 ++++++++++++++++++++++++--- src/os/file_unix.go | 40 +++++++++++++++++++++++++++------- src/os/getwd.go | 11 +++++++++- src/os/stat_unix.go | 8 +++++-- src/runtime/trace/trace_stack_test.go | 2 +- 12 files changed, 133 insertions(+), 34 deletions(-) diff --git a/src/internal/poll/fd_fsync_posix.go b/src/internal/poll/fd_fsync_posix.go index 69358297f4..dd7956f14d 100644 --- a/src/internal/poll/fd_fsync_posix.go +++ b/src/internal/poll/fd_fsync_posix.go @@ -14,5 +14,7 @@ func (fd *FD) Fsync() error { return err } defer fd.decref() - return syscall.Fsync(fd.Sysfd) + return ignoringEINTR(func() error { + return syscall.Fsync(fd.Sysfd) + }) } diff --git a/src/internal/poll/fd_opendir_darwin.go b/src/internal/poll/fd_opendir_darwin.go index c7d3318c72..8eb770c358 100644 --- a/src/internal/poll/fd_opendir_darwin.go +++ b/src/internal/poll/fd_opendir_darwin.go @@ -19,7 +19,13 @@ func (fd *FD) OpenDir() (uintptr, string, error) { if err != nil { return 0, call, err } - dir, err := fdopendir(fd2) + var dir uintptr + for { + dir, err = fdopendir(fd2) + if err != syscall.EINTR { + break + } + } if err != nil { syscall.Close(fd2) return 0, "fdopendir", err diff --git a/src/internal/poll/fd_posix.go b/src/internal/poll/fd_posix.go index 54747b4c99..e5fb05c9c2 100644 --- a/src/internal/poll/fd_posix.go +++ b/src/internal/poll/fd_posix.go @@ -35,7 +35,9 @@ func (fd *FD) Fchmod(mode uint32) error { return err } defer fd.decref() - return syscall.Fchmod(fd.Sysfd, mode) + return ignoringEINTR(func() error { + return syscall.Fchmod(fd.Sysfd, mode) + }) } // Fchown wraps syscall.Fchown. @@ -44,7 +46,9 @@ func (fd *FD) Fchown(uid, gid int) error { return err } defer fd.decref() - return syscall.Fchown(fd.Sysfd, uid, gid) + return ignoringEINTR(func() error { + return syscall.Fchown(fd.Sysfd, uid, gid) + }) } // Ftruncate wraps syscall.Ftruncate. @@ -53,7 +57,9 @@ func (fd *FD) Ftruncate(size int64) error { return err } defer fd.decref() - return syscall.Ftruncate(fd.Sysfd, size) + return ignoringEINTR(func() error { + return syscall.Ftruncate(fd.Sysfd, size) + }) } // RawControl invokes the user-defined function f for a non-IO @@ -66,3 +72,19 @@ func (fd *FD) RawControl(f func(uintptr)) error { f(uintptr(fd.Sysfd)) return nil } + +// ignoringEINTR makes a function call and repeats it if it returns +// an EINTR error. This appears to be required even though we install all +// signal handlers with SA_RESTART: see #22838, #38033, #38836, #40846. +// Also #20400 and #36644 are issues in which a signal handler is +// installed without setting SA_RESTART. None of these are the common case, +// but there are enough of them that it seems that we can't avoid +// an EINTR loop. +func ignoringEINTR(fn func() error) error { + for { + err := fn() + if err != syscall.EINTR { + return err + } + } +} diff --git a/src/internal/poll/fd_unix.go b/src/internal/poll/fd_unix.go index 4872fa9851..1d5101eac3 100644 --- a/src/internal/poll/fd_unix.go +++ b/src/internal/poll/fd_unix.go @@ -152,7 +152,7 @@ func (fd *FD) Read(p []byte) (int, error) { p = p[:maxRW] } for { - n, err := ignoringEINTR(syscall.Read, fd.Sysfd, p) + n, err := ignoringEINTRIO(syscall.Read, fd.Sysfd, p) if err != nil { n = 0 if err == syscall.EAGAIN && fd.pd.pollable() { @@ -264,7 +264,7 @@ func (fd *FD) Write(p []byte) (int, error) { if fd.IsStream && max-nn > maxRW { max = nn + maxRW } - n, err := ignoringEINTR(syscall.Write, fd.Sysfd, p[nn:max]) + n, err := ignoringEINTRIO(syscall.Write, fd.Sysfd, p[nn:max]) if n > 0 { nn += n } @@ -423,7 +423,7 @@ func (fd *FD) ReadDirent(buf []byte) (int, error) { } defer fd.decref() for { - n, err := ignoringEINTR(syscall.ReadDirent, fd.Sysfd, buf) + n, err := ignoringEINTRIO(syscall.ReadDirent, fd.Sysfd, buf) if err != nil { n = 0 if err == syscall.EAGAIN && fd.pd.pollable() { @@ -452,7 +452,9 @@ func (fd *FD) Fstat(s *syscall.Stat_t) error { return err } defer fd.decref() - return syscall.Fstat(fd.Sysfd, s) + return ignoringEINTR(func() error { + return syscall.Fstat(fd.Sysfd, s) + }) } // tryDupCloexec indicates whether F_DUPFD_CLOEXEC should be used. @@ -514,7 +516,7 @@ func (fd *FD) WriteOnce(p []byte) (int, error) { return 0, err } defer fd.writeUnlock() - return ignoringEINTR(syscall.Write, fd.Sysfd, p) + return ignoringEINTRIO(syscall.Write, fd.Sysfd, p) } // RawRead invokes the user-defined function f for a read operation. @@ -555,14 +557,8 @@ func (fd *FD) RawWrite(f func(uintptr) bool) error { } } -// ignoringEINTR makes a function call and repeats it if it returns -// an EINTR error. This appears to be required even though we install -// all signal handlers with SA_RESTART: see #22838, #38033, #38836. -// Also #20400 and #36644 are issues in which a signal handler is -// installed without setting SA_RESTART. None of these are the common case, -// but there are enough of them that it seems that we can't avoid -// an EINTR loop. -func ignoringEINTR(fn func(fd int, p []byte) (int, error), fd int, p []byte) (int, error) { +// ignoringEINTRIO is like ignoringEINTR, but just for IO calls. +func ignoringEINTRIO(fn func(fd int, p []byte) (int, error), fd int, p []byte) (int, error) { for { n, err := fn(fd, p) if err != syscall.EINTR { diff --git a/src/os/dir_darwin.go b/src/os/dir_darwin.go index 2f9ba78d68..87797e2dda 100644 --- a/src/os/dir_darwin.go +++ b/src/os/dir_darwin.go @@ -47,6 +47,9 @@ func (f *File) readdirnames(n int) (names []string, err error) { var entptr *syscall.Dirent for len(names) < size || n == -1 { if res := readdir_r(d.dir, &dirent, &entptr); res != 0 { + if syscall.Errno(res) == syscall.EINTR { + continue + } return names, wrapSyscallError("readdir", syscall.Errno(res)) } if entptr == nil { // EOF diff --git a/src/os/file.go b/src/os/file.go index a2b71cb61a..05d2f83283 100644 --- a/src/os/file.go +++ b/src/os/file.go @@ -255,7 +255,10 @@ func Mkdir(name string, perm FileMode) error { if runtime.GOOS == "windows" && isWindowsNulName(name) { return &PathError{"mkdir", name, syscall.ENOTDIR} } - e := syscall.Mkdir(fixLongPath(name), syscallMode(perm)) + longName := fixLongPath(name) + e := ignoringEINTR(func() error { + return syscall.Mkdir(longName, syscallMode(perm)) + }) if e != nil { return &PathError{"mkdir", name, e} diff --git a/src/os/file_plan9.go b/src/os/file_plan9.go index eb158905ab..043500744b 100644 --- a/src/os/file_plan9.go +++ b/src/os/file_plan9.go @@ -558,3 +558,7 @@ func (c *rawConn) Write(f func(uintptr) bool) error { func newRawConn(file *File) (*rawConn, error) { return nil, syscall.EPLAN9 } + +func ignoringEINTR(fn func() error) error { + return fn() +} diff --git a/src/os/file_posix.go b/src/os/file_posix.go index 24ea554b62..ae23d22d0a 100644 --- a/src/os/file_posix.go +++ b/src/os/file_posix.go @@ -76,7 +76,11 @@ func syscallMode(i FileMode) (o uint32) { // See docs in file.go:Chmod. func chmod(name string, mode FileMode) error { - if e := syscall.Chmod(fixLongPath(name), syscallMode(mode)); e != nil { + longName := fixLongPath(name) + e := ignoringEINTR(func() error { + return syscall.Chmod(longName, syscallMode(mode)) + }) + if e != nil { return &PathError{"chmod", name, e} } return nil @@ -101,7 +105,10 @@ func (f *File) chmod(mode FileMode) error { // On Windows or Plan 9, Chown always returns the syscall.EWINDOWS or // EPLAN9 error, wrapped in *PathError. func Chown(name string, uid, gid int) error { - if e := syscall.Chown(name, uid, gid); e != nil { + e := ignoringEINTR(func() error { + return syscall.Chown(name, uid, gid) + }) + if e != nil { return &PathError{"chown", name, e} } return nil @@ -114,7 +121,10 @@ func Chown(name string, uid, gid int) error { // On Windows, it always returns the syscall.EWINDOWS error, wrapped // in *PathError. func Lchown(name string, uid, gid int) error { - if e := syscall.Lchown(name, uid, gid); e != nil { + e := ignoringEINTR(func() error { + return syscall.Lchown(name, uid, gid) + }) + if e != nil { return &PathError{"lchown", name, e} } return nil @@ -222,3 +232,19 @@ func (f *File) checkValid(op string) error { } return nil } + +// ignoringEINTR makes a function call and repeats it if it returns an +// EINTR error. This appears to be required even though we install all +// signal handlers with SA_RESTART: see #22838, #38033, #38836, #40846. +// Also #20400 and #36644 are issues in which a signal handler is +// installed without setting SA_RESTART. None of these are the common case, +// but there are enough of them that it seems that we can't avoid +// an EINTR loop. +func ignoringEINTR(fn func() error) error { + for { + err := fn() + if err != syscall.EINTR { + return err + } + } +} diff --git a/src/os/file_unix.go b/src/os/file_unix.go index f2c00ae0cb..5446dd5003 100644 --- a/src/os/file_unix.go +++ b/src/os/file_unix.go @@ -39,7 +39,9 @@ func rename(oldname, newname string) error { return &LinkError{"rename", oldname, newname, syscall.EEXIST} } } - err = syscall.Rename(oldname, newname) + err = ignoringEINTR(func() error { + return syscall.Rename(oldname, newname) + }) if err != nil { return &LinkError{"rename", oldname, newname, err} } @@ -129,7 +131,9 @@ func newFile(fd uintptr, name string, kind newFileKind) *File { switch runtime.GOOS { case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd": var st syscall.Stat_t - err := syscall.Fstat(fdi, &st) + err := ignoringEINTR(func() error { + return syscall.Fstat(fdi, &st) + }) typ := st.Mode & syscall.S_IFMT // Don't try to use kqueue with regular files on *BSDs. // On FreeBSD a regular file is always @@ -264,7 +268,10 @@ func (f *File) seek(offset int64, whence int) (ret int64, err error) { // If the file is a symbolic link, it changes the size of the link's target. // If there is an error, it will be of type *PathError. func Truncate(name string, size int64) error { - if e := syscall.Truncate(name, size); e != nil { + e := ignoringEINTR(func() error { + return syscall.Truncate(name, size) + }) + if e != nil { return &PathError{"truncate", name, e} } return nil @@ -277,11 +284,15 @@ func Remove(name string) error { // whether name is a file or directory. // Try both: it is cheaper on average than // doing a Stat plus the right one. - e := syscall.Unlink(name) + e := ignoringEINTR(func() error { + return syscall.Unlink(name) + }) if e == nil { return nil } - e1 := syscall.Rmdir(name) + e1 := ignoringEINTR(func() error { + return syscall.Rmdir(name) + }) if e1 == nil { return nil } @@ -316,7 +327,9 @@ func tempDir() string { // Link creates newname as a hard link to the oldname file. // If there is an error, it will be of type *LinkError. func Link(oldname, newname string) error { - e := syscall.Link(oldname, newname) + e := ignoringEINTR(func() error { + return syscall.Link(oldname, newname) + }) if e != nil { return &LinkError{"link", oldname, newname, e} } @@ -326,7 +339,9 @@ func Link(oldname, newname string) error { // Symlink creates newname as a symbolic link to oldname. // If there is an error, it will be of type *LinkError. func Symlink(oldname, newname string) error { - e := syscall.Symlink(oldname, newname) + e := ignoringEINTR(func() error { + return syscall.Symlink(oldname, newname) + }) if e != nil { return &LinkError{"symlink", oldname, newname, e} } @@ -365,7 +380,16 @@ func (f *File) readdir(n int) (fi []FileInfo, err error) { func Readlink(name string) (string, error) { for len := 128; ; len *= 2 { b := make([]byte, len) - n, e := fixCount(syscall.Readlink(name, b)) + var ( + n int + e error + ) + for { + n, e = fixCount(syscall.Readlink(name, b)) + if e != syscall.EINTR { + break + } + } // buffer too small if runtime.GOOS == "aix" && e == syscall.ERANGE { continue diff --git a/src/os/getwd.go b/src/os/getwd.go index 6d25466bb4..f3afd8c06c 100644 --- a/src/os/getwd.go +++ b/src/os/getwd.go @@ -45,7 +45,16 @@ func Getwd() (dir string, err error) { // If the operating system provides a Getwd call, use it. // Otherwise, we're trying to find our way back to ".". if syscall.ImplementsGetwd { - s, e := syscall.Getwd() + var ( + s string + e error + ) + for { + s, e = syscall.Getwd() + if e != syscall.EINTR { + break + } + } if useSyscallwd(e) { return s, NewSyscallError("getwd", e) } diff --git a/src/os/stat_unix.go b/src/os/stat_unix.go index 0a7e6029ac..ef74a43758 100644 --- a/src/os/stat_unix.go +++ b/src/os/stat_unix.go @@ -28,7 +28,9 @@ func (f *File) Stat() (FileInfo, error) { // statNolog stats a file with no test logging. func statNolog(name string) (FileInfo, error) { var fs fileStat - err := syscall.Stat(name, &fs.sys) + err := ignoringEINTR(func() error { + return syscall.Stat(name, &fs.sys) + }) if err != nil { return nil, &PathError{"stat", name, err} } @@ -39,7 +41,9 @@ func statNolog(name string) (FileInfo, error) { // lstatNolog lstats a file with no test logging. func lstatNolog(name string) (FileInfo, error) { var fs fileStat - err := syscall.Lstat(name, &fs.sys) + err := ignoringEINTR(func() error { + return syscall.Lstat(name, &fs.sys) + }) if err != nil { return nil, &PathError{"lstat", name, err} } diff --git a/src/runtime/trace/trace_stack_test.go b/src/runtime/trace/trace_stack_test.go index cfc0419b72..be3adc9801 100644 --- a/src/runtime/trace/trace_stack_test.go +++ b/src/runtime/trace/trace_stack_test.go @@ -252,7 +252,7 @@ func TestTraceSymbolize(t *testing.T) { {trace.EvGoSysCall, []frame{ {"syscall.read", 0}, {"syscall.Read", 0}, - {"internal/poll.ignoringEINTR", 0}, + {"internal/poll.ignoringEINTRIO", 0}, {"internal/poll.(*FD).Read", 0}, {"os.(*File).read", 0}, {"os.(*File).Read", 0}, -- cgit v1.3-5-g9baa From 2d34f3d023586fc001d64b372bd709efaab54a18 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 19 Aug 2020 14:42:11 -0700 Subject: cmd/dist: fix tipo in comment Change-Id: I98fbf0dc94cf182adec5d414a56b9cc5126c38f4 Reviewed-on: https://go-review.googlesource.com/c/go/+/249437 Run-TryBot: Ian Lance Taylor Reviewed-by: Cherry Zhang TryBot-Result: Gobot Gobot --- src/cmd/dist/buildtool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index 0b50f66c32..79eab24d29 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -113,7 +113,7 @@ var ignorePrefixes = []string{ // File suffixes that use build tags introduced since Go 1.4. // These must not be copied into the bootstrap build directory. -// Also igonore test files. +// Also ignore test files. var ignoreSuffixes = []string{ "_arm64.s", "_arm64.go", -- cgit v1.3-5-g9baa From b0cc02e8c2bdba5401838d9d70a859191af9bfa5 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 19 Aug 2020 16:54:36 +0200 Subject: internal/poll: treat copy_file_range EPERM as not-handled Fixes #40893 Change-Id: I938ea4796c1e1d1e136117fe78b06ad6da8e40de Reviewed-on: https://go-review.googlesource.com/c/go/+/249257 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Antonio Troina Reviewed-by: Ian Lance Taylor --- src/internal/poll/copy_file_range_linux.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/internal/poll/copy_file_range_linux.go b/src/internal/poll/copy_file_range_linux.go index 7e67125818..09de299ff7 100644 --- a/src/internal/poll/copy_file_range_linux.go +++ b/src/internal/poll/copy_file_range_linux.go @@ -41,7 +41,7 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // use copy_file_range(2) again. atomic.StoreInt32(©FileRangeSupported, 0) return 0, false, nil - case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP: + case syscall.EXDEV, syscall.EINVAL, syscall.EOPNOTSUPP, syscall.EPERM: // Prior to Linux 5.3, it was not possible to // copy_file_range across file systems. Similarly to // the ENOSYS case above, if we see EXDEV, we have @@ -55,6 +55,11 @@ func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err // // If the file is on NFS, we can see EOPNOTSUPP. // See issue #40731. + // + // If the process is running inside a Docker container, + // we might see EPERM instead of ENOSYS. See issue + // #40893. Since EPERM might also be a legitimate error, + // don't mark copy_file_range(2) as unsupported. return 0, false, nil case nil: if n == 0 { -- cgit v1.3-5-g9baa From 46ca7b5ee2a8582736f1ddac27d8660e1104c345 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 15 Aug 2020 08:08:37 -0700 Subject: cmd/internal/obj: stop removing NOPs from instruction stream This has already been done for s390x, ppc64. This CL is for all the other architectures. Fixes #40796 Change-Id: Idd1816e057df63022d47e99fa06617811d8c8489 Reviewed-on: https://go-review.googlesource.com/c/go/+/248684 Run-TryBot: Cherry Zhang TryBot-Result: Gobot Gobot Reviewed-by: Cherry Zhang --- src/cmd/internal/obj/arm/asm5.go | 3 ++ src/cmd/internal/obj/arm/obj5.go | 46 ------------------------------- src/cmd/internal/obj/arm64/asm7.go | 3 ++ src/cmd/internal/obj/arm64/obj7.go | 56 ++------------------------------------ src/cmd/internal/obj/mips/asm0.go | 3 ++ src/cmd/internal/obj/mips/obj0.go | 26 ++---------------- 6 files changed, 14 insertions(+), 123 deletions(-) diff --git a/src/cmd/internal/obj/arm/asm5.go b/src/cmd/internal/obj/arm/asm5.go index f66f8aaf84..7b7e42ee2e 100644 --- a/src/cmd/internal/obj/arm/asm5.go +++ b/src/cmd/internal/obj/arm/asm5.go @@ -327,6 +327,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_LCON, 0, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_LCON, C_NONE, C_ADDR, 0, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, + {obj.ANOP, C_FREG, C_NONE, C_NONE, 0, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0, 0}, // same as ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0, 0}, // same as ABL {obj.AXXX, C_NONE, C_NONE, C_NONE, 0, 4, 0, 0, 0, 0}, diff --git a/src/cmd/internal/obj/arm/obj5.go b/src/cmd/internal/obj/arm/obj5.go index 008118c47b..86831f2b44 100644 --- a/src/cmd/internal/obj/arm/obj5.go +++ b/src/cmd/internal/obj/arm/obj5.go @@ -276,67 +276,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs - * expand RET - * expand BECOME pseudo */ - var q1 *obj.Prog - var q *obj.Prog for p := cursym.Func.Text; p != nil; p = p.Link { switch p.As { case obj.ATEXT: p.Mark |= LEAF - case obj.ARET: - break - case ADIV, ADIVU, AMOD, AMODU: - q = p cursym.Func.Text.Mark &^= LEAF - continue - - case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - if q1 != nil { - q1.Mark |= p.Mark - } - continue case ABL, ABX, obj.ADUFFZERO, obj.ADUFFCOPY: cursym.Func.Text.Mark &^= LEAF - fallthrough - - case AB, - ABEQ, - ABNE, - ABCS, - ABHS, - ABCC, - ABLO, - ABMI, - ABPL, - ABVS, - ABVC, - ABHI, - ABLS, - ABGE, - ABLT, - ABGT, - ABLE: - q1 = p.Pcond - if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } - } } - - q = p } var q2 *obj.Prog diff --git a/src/cmd/internal/obj/arm64/asm7.go b/src/cmd/internal/obj/arm64/asm7.go index 8f8981479b..7a5a8ff38c 100644 --- a/src/cmd/internal/obj/arm64/asm7.go +++ b/src/cmd/internal/obj/arm64/asm7.go @@ -837,6 +837,9 @@ var optab = []Optab{ {obj.APCDATA, C_VCON, C_NONE, C_NONE, C_VCON, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_VCON, C_NONE, C_NONE, C_ADDR, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_VREG, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0}, // same as AB/ABL {obj.ADUFFCOPY, C_NONE, C_NONE, C_NONE, C_SBRA, 5, 4, 0, 0, 0}, // same as AB/ABL {obj.APCALIGN, C_LCON, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // align code diff --git a/src/cmd/internal/obj/arm64/obj7.go b/src/cmd/internal/obj/arm64/obj7.go index b046685ada..0d74430053 100644 --- a/src/cmd/internal/obj/arm64/obj7.go +++ b/src/cmd/internal/obj/arm64/obj7.go @@ -468,73 +468,21 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs - * expand RET */ - q := (*obj.Prog)(nil) - var q1 *obj.Prog for p := c.cursym.Func.Text; p != nil; p = p.Link { switch p.As { case obj.ATEXT: p.Mark |= LEAF - case obj.ARET: - break - - case obj.ANOP: - if p.Link != nil { - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark - } - continue - case ABL, obj.ADUFFZERO, obj.ADUFFCOPY: c.cursym.Func.Text.Mark &^= LEAF - fallthrough - - case ACBNZ, - ACBZ, - ACBNZW, - ACBZW, - ATBZ, - ATBNZ, - AB, - ABEQ, - ABNE, - ABCS, - ABHS, - ABCC, - ABLO, - ABMI, - ABPL, - ABVS, - ABVC, - ABHI, - ABLS, - ABGE, - ABLT, - ABGT, - ABLE, - AADR, /* strange */ - AADRP: - q1 = p.Pcond - - if q1 != nil { - for q1.As == obj.ANOP { - q1 = q1.Link - p.Pcond = q1 - } - } - - break } - - q = p } + var q *obj.Prog + var q1 *obj.Prog var retjmp *obj.LSym for p := c.cursym.Func.Text; p != nil; p = p.Link { o := p.As diff --git a/src/cmd/internal/obj/mips/asm0.go b/src/cmd/internal/obj/mips/asm0.go index faa12bf133..faa827da9f 100644 --- a/src/cmd/internal/obj/mips/asm0.go +++ b/src/cmd/internal/obj/mips/asm0.go @@ -391,6 +391,9 @@ var optab = []Optab{ {obj.APCDATA, C_LCON, C_NONE, C_LCON, 0, 0, 0, 0, 0}, {obj.AFUNCDATA, C_SCON, C_NONE, C_ADDR, 0, 0, 0, 0, 0}, {obj.ANOP, C_NONE, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_LCON, C_NONE, C_NONE, 0, 0, 0, 0, 0}, // nop variants, see #40689 + {obj.ANOP, C_REG, C_NONE, C_NONE, 0, 0, 0, 0, 0}, + {obj.ANOP, C_FREG, C_NONE, C_NONE, 0, 0, 0, 0, 0}, {obj.ADUFFZERO, C_NONE, C_NONE, C_LBRA, 11, 4, 0, 0, 0}, // same as AJMP {obj.ADUFFCOPY, C_NONE, C_NONE, C_LBRA, 11, 4, 0, 0, 0}, // same as AJMP diff --git a/src/cmd/internal/obj/mips/obj0.go b/src/cmd/internal/obj/mips/obj0.go index 3106143844..77cad979a6 100644 --- a/src/cmd/internal/obj/mips/obj0.go +++ b/src/cmd/internal/obj/mips/obj0.go @@ -158,19 +158,14 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* * find leaf subroutines - * strip NOPs * expand RET * expand BECOME pseudo */ - var q *obj.Prog - var q1 *obj.Prog for p := c.cursym.Func.Text; p != nil; p = p.Link { switch p.As { /* too hard, just leave alone */ case obj.ATEXT: - q = p - p.Mark |= LABEL | LEAF | SYNC if p.Link != nil { p.Link.Mark |= LABEL @@ -179,7 +174,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { /* too hard, just leave alone */ case AMOVW, AMOVV: - q = p if p.To.Type == obj.TYPE_REG && p.To.Reg >= REG_SPECIAL { p.Mark |= LABEL | SYNC break @@ -195,11 +189,9 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ATLBWI, ATLBP, ATLBR: - q = p p.Mark |= LABEL | SYNC case ANOR: - q = p if p.To.Type == obj.TYPE_REG { if p.To.Reg == REGZERO { p.Mark |= LABEL | SYNC @@ -235,8 +227,7 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { } else { p.Mark |= BRANCH } - q = p - q1 = p.Pcond + q1 := p.Pcond if q1 != nil { for q1.As == obj.ANOP { q1 = q1.Link @@ -254,24 +245,11 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { if q1 != nil { q1.Mark |= LABEL } - continue case ARET: - q = p if p.Link != nil { p.Link.Mark |= LABEL } - continue - - case obj.ANOP: - q1 = p.Link - q.Link = q1 /* q is non-nop */ - q1.Mark |= p.Mark - continue - - default: - q = p - continue } } @@ -284,6 +262,8 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { mov = AMOVW } + var q *obj.Prog + var q1 *obj.Prog autosize := int32(0) var p1 *obj.Prog var p2 *obj.Prog -- cgit v1.3-5-g9baa From 8ec5a052ec8d4e628353fb5099b29341fae9b3a4 Mon Sep 17 00:00:00 2001 From: Marcel van Lohuizen Date: Mon, 17 Aug 2020 19:10:12 +0200 Subject: unicode: upgrade to Unicode 13.0.0 Fixes #40755 Change-Id: I14b3977317994095db8ae1bd873c174641209356 Reviewed-on: https://go-review.googlesource.com/c/go/+/248765 Run-TryBot: Marcel van Lohuizen TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- api/except.txt | 1 + api/next.txt | 5 + doc/go1.16.html | 11 + src/strconv/isprint.go | 98 +++++---- src/unicode/tables.go | 544 +++++++++++++++++++++++++++++++------------------ 5 files changed, 416 insertions(+), 243 deletions(-) diff --git a/api/except.txt b/api/except.txt index ccfdf06c55..962bb14271 100644 --- a/api/except.txt +++ b/api/except.txt @@ -456,3 +456,4 @@ pkg syscall (freebsd-arm-cgo), type Statfs_t struct, Mntonname [88]int8 pkg text/scanner, const GoTokens = 1012 pkg unicode, const Version = "10.0.0" pkg unicode, const Version = "11.0.0" +pkg unicode, const Version = "12.0.0" diff --git a/api/next.txt b/api/next.txt index e69de29bb2..fe7509bf82 100644 --- a/api/next.txt +++ b/api/next.txt @@ -0,0 +1,5 @@ +pkg unicode, const Version = "13.0.0" +pkg unicode, var Chorasmian *RangeTable +pkg unicode, var Dives_Akuru *RangeTable +pkg unicode, var Khitan_Small_Script *RangeTable +pkg unicode, var Yezidi *RangeTable diff --git a/doc/go1.16.html b/doc/go1.16.html index 1fea359436..4753cf914d 100644 --- a/doc/go1.16.html +++ b/doc/go1.16.html @@ -90,6 +90,17 @@ Do not send CLs removing the interior tags from such phrases. TODO

+

unicode

+ +

+ The unicode package and associated + support throughout the system has been upgraded from Unicode 12.0.0 to + Unicode 13.0.0, + which adds 5,930 new characters, including four new scripts, and 55 new emoji. + Unicode 13.0.0 also designates plane 3 (U+30000-U+3FFFF) as the tertiary + ideographic plane. +

+

Minor changes to the library

diff --git a/src/strconv/isprint.go b/src/strconv/isprint.go index 7ada2d1b04..994a8e423c 100644 --- a/src/strconv/isprint.go +++ b/src/strconv/isprint.go @@ -6,7 +6,7 @@ package strconv -// (442+132+90)*2 + (450)*4 = 3128 bytes +// (434+132+95)*2 + (468)*4 = 3194 bytes var isPrint16 = []uint16{ 0x0020, 0x007e, @@ -25,7 +25,7 @@ var isPrint16 = []uint16{ 0x07fd, 0x082d, 0x0830, 0x085b, 0x085e, 0x086a, - 0x08a0, 0x08bd, + 0x08a0, 0x08c7, 0x08d3, 0x098c, 0x098f, 0x0990, 0x0993, 0x09b2, @@ -56,7 +56,7 @@ var isPrint16 = []uint16{ 0x0b3c, 0x0b44, 0x0b47, 0x0b48, 0x0b4b, 0x0b4d, - 0x0b56, 0x0b57, + 0x0b55, 0x0b57, 0x0b5c, 0x0b63, 0x0b66, 0x0b77, 0x0b82, 0x0b8a, @@ -82,8 +82,7 @@ var isPrint16 = []uint16{ 0x0ce6, 0x0cf2, 0x0d00, 0x0d4f, 0x0d54, 0x0d63, - 0x0d66, 0x0d7f, - 0x0d82, 0x0d96, + 0x0d66, 0x0d96, 0x0d9a, 0x0dbd, 0x0dc0, 0x0dc6, 0x0dca, 0x0dca, @@ -138,7 +137,7 @@ var isPrint16 = []uint16{ 0x1a7f, 0x1a89, 0x1a90, 0x1a99, 0x1aa0, 0x1aad, - 0x1ab0, 0x1abe, + 0x1ab0, 0x1ac0, 0x1b00, 0x1b4b, 0x1b50, 0x1b7c, 0x1b80, 0x1bf3, @@ -166,30 +165,27 @@ var isPrint16 = []uint16{ 0x2190, 0x2426, 0x2440, 0x244a, 0x2460, 0x2b73, - 0x2b76, 0x2b95, - 0x2b98, 0x2cf3, + 0x2b76, 0x2cf3, 0x2cf9, 0x2d27, 0x2d2d, 0x2d2d, 0x2d30, 0x2d67, 0x2d6f, 0x2d70, 0x2d7f, 0x2d96, - 0x2da0, 0x2e4f, + 0x2da0, 0x2e52, 0x2e80, 0x2ef3, 0x2f00, 0x2fd5, 0x2ff0, 0x2ffb, 0x3001, 0x3096, 0x3099, 0x30ff, - 0x3105, 0x31ba, - 0x31c0, 0x31e3, - 0x31f0, 0x4db5, - 0x4dc0, 0x9fef, + 0x3105, 0x31e3, + 0x31f0, 0x9ffc, 0xa000, 0xa48c, 0xa490, 0xa4c6, 0xa4d0, 0xa62b, 0xa640, 0xa6f7, 0xa700, 0xa7bf, - 0xa7c2, 0xa7c6, - 0xa7f7, 0xa82b, + 0xa7c2, 0xa7ca, + 0xa7f5, 0xa82c, 0xa830, 0xa839, 0xa840, 0xa877, 0xa880, 0xa8c5, @@ -205,7 +201,7 @@ var isPrint16 = []uint16{ 0xab01, 0xab06, 0xab09, 0xab0e, 0xab11, 0xab16, - 0xab20, 0xab67, + 0xab20, 0xab6b, 0xab70, 0xabed, 0xabf0, 0xabf9, 0xac00, 0xd7a3, @@ -288,11 +284,11 @@ var isNotPrint16 = []uint16{ 0x0cc9, 0x0cdf, 0x0cf0, - 0x0d04, 0x0d0d, 0x0d11, 0x0d45, 0x0d49, + 0x0d80, 0x0d84, 0x0db2, 0x0dbc, @@ -335,6 +331,7 @@ var isNotPrint16 = []uint16{ 0x1fdc, 0x1ff5, 0x208f, + 0x2b96, 0x2c2f, 0x2c5f, 0x2d26, @@ -351,7 +348,6 @@ var isNotPrint16 = []uint16{ 0x3130, 0x318f, 0x321f, - 0x32ff, 0xa9ce, 0xa9ff, 0xab27, @@ -373,7 +369,7 @@ var isPrint32 = []uint32{ 0x010080, 0x0100fa, 0x010100, 0x010102, 0x010107, 0x010133, - 0x010137, 0x01019b, + 0x010137, 0x01019c, 0x0101a0, 0x0101a0, 0x0101d0, 0x0101fd, 0x010280, 0x01029c, @@ -424,19 +420,20 @@ var isPrint32 = []uint32{ 0x010cc0, 0x010cf2, 0x010cfa, 0x010d27, 0x010d30, 0x010d39, - 0x010e60, 0x010e7e, + 0x010e60, 0x010ead, + 0x010eb0, 0x010eb1, 0x010f00, 0x010f27, 0x010f30, 0x010f59, + 0x010fb0, 0x010fcb, 0x010fe0, 0x010ff6, 0x011000, 0x01104d, 0x011052, 0x01106f, 0x01107f, 0x0110c1, 0x0110d0, 0x0110e8, 0x0110f0, 0x0110f9, - 0x011100, 0x011146, + 0x011100, 0x011147, 0x011150, 0x011176, - 0x011180, 0x0111cd, - 0x0111d0, 0x0111f4, + 0x011180, 0x0111f4, 0x011200, 0x01123e, 0x011280, 0x0112a9, 0x0112b0, 0x0112ea, @@ -451,7 +448,7 @@ var isPrint32 = []uint32{ 0x01135d, 0x011363, 0x011366, 0x01136c, 0x011370, 0x011374, - 0x011400, 0x01145f, + 0x011400, 0x011461, 0x011480, 0x0114c7, 0x0114d0, 0x0114d9, 0x011580, 0x0115b5, @@ -466,7 +463,11 @@ var isPrint32 = []uint32{ 0x011730, 0x01173f, 0x011800, 0x01183b, 0x0118a0, 0x0118f2, - 0x0118ff, 0x0118ff, + 0x0118ff, 0x011906, + 0x011909, 0x011909, + 0x01190c, 0x011938, + 0x01193b, 0x011946, + 0x011950, 0x011959, 0x0119a0, 0x0119a7, 0x0119aa, 0x0119d7, 0x0119da, 0x0119e4, @@ -483,6 +484,7 @@ var isPrint32 = []uint32{ 0x011d60, 0x011d98, 0x011da0, 0x011da9, 0x011ee0, 0x011ef8, + 0x011fb0, 0x011fb0, 0x011fc0, 0x011ff1, 0x011fff, 0x012399, 0x012400, 0x012474, @@ -501,9 +503,11 @@ var isPrint32 = []uint32{ 0x016f00, 0x016f4a, 0x016f4f, 0x016f87, 0x016f8f, 0x016f9f, - 0x016fe0, 0x016fe3, + 0x016fe0, 0x016fe4, + 0x016ff0, 0x016ff1, 0x017000, 0x0187f7, - 0x018800, 0x018af2, + 0x018800, 0x018cd5, + 0x018d00, 0x018d08, 0x01b000, 0x01b11e, 0x01b150, 0x01b152, 0x01b164, 0x01b167, @@ -557,17 +561,15 @@ var isPrint32 = []uint32{ 0x01f030, 0x01f093, 0x01f0a0, 0x01f0ae, 0x01f0b1, 0x01f0f5, - 0x01f100, 0x01f10c, - 0x01f110, 0x01f16c, - 0x01f170, 0x01f1ac, + 0x01f100, 0x01f1ad, 0x01f1e6, 0x01f202, 0x01f210, 0x01f23b, 0x01f240, 0x01f248, 0x01f250, 0x01f251, 0x01f260, 0x01f265, - 0x01f300, 0x01f6d5, + 0x01f300, 0x01f6d7, 0x01f6e0, 0x01f6ec, - 0x01f6f0, 0x01f6fa, + 0x01f6f0, 0x01f6fc, 0x01f700, 0x01f773, 0x01f780, 0x01f7d8, 0x01f7e0, 0x01f7eb, @@ -576,22 +578,25 @@ var isPrint32 = []uint32{ 0x01f850, 0x01f859, 0x01f860, 0x01f887, 0x01f890, 0x01f8ad, - 0x01f900, 0x01f976, - 0x01f97a, 0x01f9a2, - 0x01f9a5, 0x01f9aa, - 0x01f9ae, 0x01f9ca, - 0x01f9cd, 0x01fa53, + 0x01f8b0, 0x01f8b1, + 0x01f900, 0x01fa53, 0x01fa60, 0x01fa6d, - 0x01fa70, 0x01fa73, + 0x01fa70, 0x01fa74, 0x01fa78, 0x01fa7a, - 0x01fa80, 0x01fa82, - 0x01fa90, 0x01fa95, - 0x020000, 0x02a6d6, + 0x01fa80, 0x01fa86, + 0x01fa90, 0x01faa8, + 0x01fab0, 0x01fab6, + 0x01fac0, 0x01fac2, + 0x01fad0, 0x01fad6, + 0x01fb00, 0x01fbca, + 0x01fbf0, 0x01fbf9, + 0x020000, 0x02a6dd, 0x02a700, 0x02b734, 0x02b740, 0x02b81d, 0x02b820, 0x02cea1, 0x02ceb0, 0x02ebe0, 0x02f800, 0x02fa1d, + 0x030000, 0x03134a, 0x0e0100, 0x0e01ef, } @@ -609,6 +614,8 @@ var isNotPrint32 = []uint16{ // add 0x10000 to each entry 0x0a04, 0x0a14, 0x0a18, + 0x0e7f, + 0x0eaa, 0x10bd, 0x1135, 0x11e0, @@ -622,8 +629,10 @@ var isNotPrint32 = []uint16{ // add 0x10000 to each entry 0x1331, 0x1334, 0x133a, - 0x145a, 0x145c, + 0x1914, + 0x1917, + 0x1936, 0x1c09, 0x1c37, 0x1ca8, @@ -684,8 +693,9 @@ var isNotPrint32 = []uint16{ // add 0x10000 to each entry 0xeeaa, 0xf0c0, 0xf0d0, - 0xf90c, - 0xf972, + 0xf979, + 0xf9cc, + 0xfb93, } // isGraphic lists the graphic runes not matched by IsPrint. diff --git a/src/unicode/tables.go b/src/unicode/tables.go index 12441682d0..a9b23bfacd 100644 --- a/src/unicode/tables.go +++ b/src/unicode/tables.go @@ -3,7 +3,7 @@ package unicode // Version is the Unicode edition from which the tables are derived. -const Version = "12.0.0" +const Version = "13.0.0" // Categories is the set of Unicode category tables. var Categories = map[string]*RangeTable{ @@ -170,7 +170,7 @@ var _L = &RangeTable{ {0x0841, 0x0858, 1}, {0x0860, 0x086a, 1}, {0x08a0, 0x08b4, 1}, - {0x08b6, 0x08bd, 1}, + {0x08b6, 0x08c7, 1}, {0x0904, 0x0939, 1}, {0x093d, 0x0950, 19}, {0x0958, 0x0961, 1}, @@ -241,7 +241,7 @@ var _L = &RangeTable{ {0x0cbd, 0x0cde, 33}, {0x0ce0, 0x0ce1, 1}, {0x0cf1, 0x0cf2, 1}, - {0x0d05, 0x0d0c, 1}, + {0x0d04, 0x0d0c, 1}, {0x0d0e, 0x0d10, 1}, {0x0d12, 0x0d3a, 1}, {0x0d3d, 0x0d4e, 17}, @@ -402,10 +402,10 @@ var _L = &RangeTable{ {0x30fc, 0x30ff, 1}, {0x3105, 0x312f, 1}, {0x3131, 0x318e, 1}, - {0x31a0, 0x31ba, 1}, + {0x31a0, 0x31bf, 1}, {0x31f0, 0x31ff, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fef, 1}, + {0x3400, 0x4dbf, 1}, + {0x4e00, 0x9ffc, 1}, {0xa000, 0xa48c, 1}, {0xa4d0, 0xa4fd, 1}, {0xa500, 0xa60c, 1}, @@ -417,8 +417,8 @@ var _L = &RangeTable{ {0xa717, 0xa71f, 1}, {0xa722, 0xa788, 1}, {0xa78b, 0xa7bf, 1}, - {0xa7c2, 0xa7c6, 1}, - {0xa7f7, 0xa801, 1}, + {0xa7c2, 0xa7ca, 1}, + {0xa7f5, 0xa801, 1}, {0xa803, 0xa805, 1}, {0xa807, 0xa80a, 1}, {0xa80c, 0xa822, 1}, @@ -454,7 +454,7 @@ var _L = &RangeTable{ {0xab20, 0xab26, 1}, {0xab28, 0xab2e, 1}, {0xab30, 0xab5a, 1}, - {0xab5c, 0xab67, 1}, + {0xab5c, 0xab69, 1}, {0xab70, 0xabe2, 1}, {0xac00, 0xd7a3, 1}, {0xd7b0, 0xd7c6, 1}, @@ -540,16 +540,19 @@ var _L = &RangeTable{ {0x10c80, 0x10cb2, 1}, {0x10cc0, 0x10cf2, 1}, {0x10d00, 0x10d23, 1}, + {0x10e80, 0x10ea9, 1}, + {0x10eb0, 0x10eb1, 1}, {0x10f00, 0x10f1c, 1}, {0x10f27, 0x10f30, 9}, {0x10f31, 0x10f45, 1}, + {0x10fb0, 0x10fc4, 1}, {0x10fe0, 0x10ff6, 1}, {0x11003, 0x11037, 1}, {0x11083, 0x110af, 1}, {0x110d0, 0x110e8, 1}, {0x11103, 0x11126, 1}, - {0x11144, 0x11150, 12}, - {0x11151, 0x11172, 1}, + {0x11144, 0x11147, 3}, + {0x11150, 0x11172, 1}, {0x11176, 0x11183, 13}, {0x11184, 0x111b2, 1}, {0x111c1, 0x111c4, 1}, @@ -572,8 +575,8 @@ var _L = &RangeTable{ {0x1135d, 0x11361, 1}, {0x11400, 0x11434, 1}, {0x11447, 0x1144a, 1}, - {0x1145f, 0x11480, 33}, - {0x11481, 0x114af, 1}, + {0x1145f, 0x11461, 1}, + {0x11480, 0x114af, 1}, {0x114c4, 0x114c5, 1}, {0x114c7, 0x11580, 185}, {0x11581, 0x115ae, 1}, @@ -585,8 +588,13 @@ var _L = &RangeTable{ {0x11701, 0x1171a, 1}, {0x11800, 0x1182b, 1}, {0x118a0, 0x118df, 1}, - {0x118ff, 0x119a0, 161}, - {0x119a1, 0x119a7, 1}, + {0x118ff, 0x11906, 1}, + {0x11909, 0x1190c, 3}, + {0x1190d, 0x11913, 1}, + {0x11915, 0x11916, 1}, + {0x11918, 0x1192f, 1}, + {0x1193f, 0x11941, 2}, + {0x119a0, 0x119a7, 1}, {0x119aa, 0x119d0, 1}, {0x119e1, 0x119e3, 2}, {0x11a00, 0x11a0b, 11}, @@ -608,7 +616,8 @@ var _L = &RangeTable{ {0x11d6a, 0x11d89, 1}, {0x11d98, 0x11ee0, 328}, {0x11ee1, 0x11ef2, 1}, - {0x12000, 0x12399, 1}, + {0x11fb0, 0x12000, 80}, + {0x12001, 0x12399, 1}, {0x12480, 0x12543, 1}, {0x13000, 0x1342e, 1}, {0x14400, 0x14646, 1}, @@ -626,7 +635,8 @@ var _L = &RangeTable{ {0x16fe0, 0x16fe1, 1}, {0x16fe3, 0x17000, 29}, {0x17001, 0x187f7, 1}, - {0x18800, 0x18af2, 1}, + {0x18800, 0x18cd5, 1}, + {0x18d00, 0x18d08, 1}, {0x1b000, 0x1b11e, 1}, {0x1b150, 0x1b152, 1}, {0x1b164, 0x1b167, 1}, @@ -696,12 +706,13 @@ var _L = &RangeTable{ {0x1eea1, 0x1eea3, 1}, {0x1eea5, 0x1eea9, 1}, {0x1eeab, 0x1eebb, 1}, - {0x20000, 0x2a6d6, 1}, + {0x20000, 0x2a6dd, 1}, {0x2a700, 0x2b734, 1}, {0x2b740, 0x2b81d, 1}, {0x2b820, 0x2cea1, 1}, {0x2ceb0, 0x2ebe0, 1}, {0x2f800, 0x2fa1d, 1}, + {0x30000, 0x3134a, 1}, }, LatinOffset: 6, } @@ -821,9 +832,11 @@ var _Ll = &RangeTable{ {0xa797, 0xa7a9, 2}, {0xa7af, 0xa7b5, 6}, {0xa7b7, 0xa7bf, 2}, - {0xa7c3, 0xa7fa, 55}, - {0xab30, 0xab5a, 1}, - {0xab60, 0xab67, 1}, + {0xa7c3, 0xa7c8, 5}, + {0xa7ca, 0xa7f6, 44}, + {0xa7fa, 0xab30, 822}, + {0xab31, 0xab5a, 1}, + {0xab60, 0xab68, 1}, {0xab70, 0xabbf, 1}, {0xfb00, 0xfb06, 1}, {0xfb13, 0xfb17, 1}, @@ -908,8 +921,8 @@ var _Lm = &RangeTable{ {0xaa70, 0xaadd, 109}, {0xaaf3, 0xaaf4, 1}, {0xab5c, 0xab5f, 1}, - {0xff70, 0xff9e, 46}, - {0xff9f, 0xff9f, 1}, + {0xab69, 0xff70, 21511}, + {0xff9e, 0xff9f, 1}, }, R32: []Range32{ {0x16b40, 0x16b43, 1}, @@ -945,7 +958,7 @@ var _Lo = &RangeTable{ {0x0840, 0x0858, 1}, {0x0860, 0x086a, 1}, {0x08a0, 0x08b4, 1}, - {0x08b6, 0x08bd, 1}, + {0x08b6, 0x08c7, 1}, {0x0904, 0x0939, 1}, {0x093d, 0x0950, 19}, {0x0958, 0x0961, 1}, @@ -1016,7 +1029,7 @@ var _Lo = &RangeTable{ {0x0cbd, 0x0cde, 33}, {0x0ce0, 0x0ce1, 1}, {0x0cf1, 0x0cf2, 1}, - {0x0d05, 0x0d0c, 1}, + {0x0d04, 0x0d0c, 1}, {0x0d0e, 0x0d10, 1}, {0x0d12, 0x0d3a, 1}, {0x0d3d, 0x0d4e, 17}, @@ -1127,10 +1140,10 @@ var _Lo = &RangeTable{ {0x30ff, 0x3105, 6}, {0x3106, 0x312f, 1}, {0x3131, 0x318e, 1}, - {0x31a0, 0x31ba, 1}, + {0x31a0, 0x31bf, 1}, {0x31f0, 0x31ff, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fef, 1}, + {0x3400, 0x4dbf, 1}, + {0x4e00, 0x9ffc, 1}, {0xa000, 0xa014, 1}, {0xa016, 0xa48c, 1}, {0xa4d0, 0xa4f7, 1}, @@ -1254,16 +1267,19 @@ var _Lo = &RangeTable{ {0x10b80, 0x10b91, 1}, {0x10c00, 0x10c48, 1}, {0x10d00, 0x10d23, 1}, + {0x10e80, 0x10ea9, 1}, + {0x10eb0, 0x10eb1, 1}, {0x10f00, 0x10f1c, 1}, {0x10f27, 0x10f30, 9}, {0x10f31, 0x10f45, 1}, + {0x10fb0, 0x10fc4, 1}, {0x10fe0, 0x10ff6, 1}, {0x11003, 0x11037, 1}, {0x11083, 0x110af, 1}, {0x110d0, 0x110e8, 1}, {0x11103, 0x11126, 1}, - {0x11144, 0x11150, 12}, - {0x11151, 0x11172, 1}, + {0x11144, 0x11147, 3}, + {0x11150, 0x11172, 1}, {0x11176, 0x11183, 13}, {0x11184, 0x111b2, 1}, {0x111c1, 0x111c4, 1}, @@ -1286,8 +1302,8 @@ var _Lo = &RangeTable{ {0x1135d, 0x11361, 1}, {0x11400, 0x11434, 1}, {0x11447, 0x1144a, 1}, - {0x1145f, 0x11480, 33}, - {0x11481, 0x114af, 1}, + {0x1145f, 0x11461, 1}, + {0x11480, 0x114af, 1}, {0x114c4, 0x114c5, 1}, {0x114c7, 0x11580, 185}, {0x11581, 0x115ae, 1}, @@ -1298,8 +1314,13 @@ var _Lo = &RangeTable{ {0x116b8, 0x11700, 72}, {0x11701, 0x1171a, 1}, {0x11800, 0x1182b, 1}, - {0x118ff, 0x119a0, 161}, - {0x119a1, 0x119a7, 1}, + {0x118ff, 0x11906, 1}, + {0x11909, 0x1190c, 3}, + {0x1190d, 0x11913, 1}, + {0x11915, 0x11916, 1}, + {0x11918, 0x1192f, 1}, + {0x1193f, 0x11941, 2}, + {0x119a0, 0x119a7, 1}, {0x119aa, 0x119d0, 1}, {0x119e1, 0x119e3, 2}, {0x11a00, 0x11a0b, 11}, @@ -1321,7 +1342,8 @@ var _Lo = &RangeTable{ {0x11d6a, 0x11d89, 1}, {0x11d98, 0x11ee0, 328}, {0x11ee1, 0x11ef2, 1}, - {0x12000, 0x12399, 1}, + {0x11fb0, 0x12000, 80}, + {0x12001, 0x12399, 1}, {0x12480, 0x12543, 1}, {0x13000, 0x1342e, 1}, {0x14400, 0x14646, 1}, @@ -1334,7 +1356,8 @@ var _Lo = &RangeTable{ {0x16f00, 0x16f4a, 1}, {0x16f50, 0x17000, 176}, {0x17001, 0x187f7, 1}, - {0x18800, 0x18af2, 1}, + {0x18800, 0x18cd5, 1}, + {0x18d00, 0x18d08, 1}, {0x1b000, 0x1b11e, 1}, {0x1b150, 0x1b152, 1}, {0x1b164, 0x1b167, 1}, @@ -1371,12 +1394,13 @@ var _Lo = &RangeTable{ {0x1eea1, 0x1eea3, 1}, {0x1eea5, 0x1eea9, 1}, {0x1eeab, 0x1eebb, 1}, - {0x20000, 0x2a6d6, 1}, + {0x20000, 0x2a6dd, 1}, {0x2a700, 0x2b734, 1}, {0x2b740, 0x2b81d, 1}, {0x2b820, 0x2cea1, 1}, {0x2ceb0, 0x2ebe0, 1}, {0x2f800, 0x2fa1d, 1}, + {0x30000, 0x3134a, 1}, }, LatinOffset: 1, } @@ -1500,7 +1524,8 @@ var _Lu = &RangeTable{ {0xa7b0, 0xa7b4, 1}, {0xa7b6, 0xa7be, 2}, {0xa7c2, 0xa7c4, 2}, - {0xa7c5, 0xa7c6, 1}, + {0xa7c5, 0xa7c7, 1}, + {0xa7c9, 0xa7f5, 44}, {0xff21, 0xff3a, 1}, }, R32: []Range32{ @@ -1601,7 +1626,7 @@ var _M = &RangeTable{ {0x0b3f, 0x0b44, 1}, {0x0b47, 0x0b48, 1}, {0x0b4b, 0x0b4d, 1}, - {0x0b56, 0x0b57, 1}, + {0x0b55, 0x0b57, 1}, {0x0b62, 0x0b63, 1}, {0x0b82, 0x0bbe, 60}, {0x0bbf, 0x0bc2, 1}, @@ -1627,9 +1652,10 @@ var _M = &RangeTable{ {0x0d46, 0x0d48, 1}, {0x0d4a, 0x0d4d, 1}, {0x0d57, 0x0d62, 11}, - {0x0d63, 0x0d82, 31}, - {0x0d83, 0x0dca, 71}, - {0x0dcf, 0x0dd4, 1}, + {0x0d63, 0x0d81, 30}, + {0x0d82, 0x0d83, 1}, + {0x0dca, 0x0dcf, 5}, + {0x0dd0, 0x0dd4, 1}, {0x0dd6, 0x0dd8, 2}, {0x0dd9, 0x0ddf, 1}, {0x0df2, 0x0df3, 1}, @@ -1672,7 +1698,7 @@ var _M = &RangeTable{ {0x1a55, 0x1a5e, 1}, {0x1a60, 0x1a7c, 1}, {0x1a7f, 0x1ab0, 49}, - {0x1ab1, 0x1abe, 1}, + {0x1ab1, 0x1ac0, 1}, {0x1b00, 0x1b04, 1}, {0x1b34, 0x1b44, 1}, {0x1b6b, 0x1b73, 1}, @@ -1699,8 +1725,9 @@ var _M = &RangeTable{ {0xa802, 0xa806, 4}, {0xa80b, 0xa823, 24}, {0xa824, 0xa827, 1}, - {0xa880, 0xa881, 1}, - {0xa8b4, 0xa8c5, 1}, + {0xa82c, 0xa880, 84}, + {0xa881, 0xa8b4, 51}, + {0xa8b5, 0xa8c5, 1}, {0xa8e0, 0xa8f1, 1}, {0xa8ff, 0xa926, 39}, {0xa927, 0xa92d, 1}, @@ -1735,6 +1762,7 @@ var _M = &RangeTable{ {0x10a3f, 0x10ae5, 166}, {0x10ae6, 0x10d24, 574}, {0x10d25, 0x10d27, 1}, + {0x10eab, 0x10eac, 1}, {0x10f46, 0x10f50, 1}, {0x11000, 0x11002, 1}, {0x11038, 0x11046, 1}, @@ -1747,6 +1775,7 @@ var _M = &RangeTable{ {0x11181, 0x11182, 1}, {0x111b3, 0x111c0, 1}, {0x111c9, 0x111cc, 1}, + {0x111ce, 0x111cf, 1}, {0x1122c, 0x11237, 1}, {0x1123e, 0x112df, 161}, {0x112e0, 0x112ea, 1}, @@ -1769,7 +1798,12 @@ var _M = &RangeTable{ {0x116ab, 0x116b7, 1}, {0x1171d, 0x1172b, 1}, {0x1182c, 0x1183a, 1}, - {0x119d1, 0x119d7, 1}, + {0x11930, 0x11935, 1}, + {0x11937, 0x11938, 1}, + {0x1193b, 0x1193e, 1}, + {0x11940, 0x11942, 2}, + {0x11943, 0x119d1, 142}, + {0x119d2, 0x119d7, 1}, {0x119da, 0x119e0, 1}, {0x119e4, 0x11a01, 29}, {0x11a02, 0x11a0a, 1}, @@ -1796,8 +1830,10 @@ var _M = &RangeTable{ {0x16f4f, 0x16f51, 2}, {0x16f52, 0x16f87, 1}, {0x16f8f, 0x16f92, 1}, - {0x1bc9d, 0x1bc9e, 1}, - {0x1d165, 0x1d169, 1}, + {0x16fe4, 0x16ff0, 12}, + {0x16ff1, 0x1bc9d, 19628}, + {0x1bc9e, 0x1d165, 5319}, + {0x1d166, 0x1d169, 1}, {0x1d16d, 0x1d172, 1}, {0x1d17b, 0x1d182, 1}, {0x1d185, 0x1d18b, 1}, @@ -1929,7 +1965,8 @@ var _Mc = &RangeTable{ {0x11146, 0x11182, 60}, {0x111b3, 0x111b5, 1}, {0x111bf, 0x111c0, 1}, - {0x1122c, 0x1122e, 1}, + {0x111ce, 0x1122c, 94}, + {0x1122d, 0x1122e, 1}, {0x11232, 0x11233, 1}, {0x11235, 0x112e0, 171}, {0x112e1, 0x112e2, 1}, @@ -1957,7 +1994,11 @@ var _Mc = &RangeTable{ {0x116b6, 0x11720, 106}, {0x11721, 0x11726, 5}, {0x1182c, 0x1182e, 1}, - {0x11838, 0x119d1, 409}, + {0x11838, 0x11930, 248}, + {0x11931, 0x11935, 1}, + {0x11937, 0x11938, 1}, + {0x1193d, 0x11940, 3}, + {0x11942, 0x119d1, 143}, {0x119d2, 0x119d3, 1}, {0x119dc, 0x119df, 1}, {0x119e4, 0x11a39, 85}, @@ -1970,6 +2011,7 @@ var _Mc = &RangeTable{ {0x11d96, 0x11ef5, 351}, {0x11ef6, 0x16f51, 20571}, {0x16f52, 0x16f87, 1}, + {0x16ff0, 0x16ff1, 1}, {0x1d165, 0x1d166, 1}, {0x1d16d, 0x1d172, 1}, }, @@ -2038,12 +2080,12 @@ var _Mn = &RangeTable{ {0x0b01, 0x0b3c, 59}, {0x0b3f, 0x0b41, 2}, {0x0b42, 0x0b44, 1}, - {0x0b4d, 0x0b56, 9}, - {0x0b62, 0x0b63, 1}, - {0x0b82, 0x0bc0, 62}, - {0x0bcd, 0x0c00, 51}, - {0x0c04, 0x0c3e, 58}, - {0x0c3f, 0x0c40, 1}, + {0x0b4d, 0x0b55, 8}, + {0x0b56, 0x0b62, 12}, + {0x0b63, 0x0b82, 31}, + {0x0bc0, 0x0bcd, 13}, + {0x0c00, 0x0c04, 4}, + {0x0c3e, 0x0c40, 1}, {0x0c46, 0x0c48, 1}, {0x0c4a, 0x0c4d, 1}, {0x0c55, 0x0c56, 1}, @@ -2056,8 +2098,9 @@ var _Mn = &RangeTable{ {0x0d3b, 0x0d3c, 1}, {0x0d41, 0x0d44, 1}, {0x0d4d, 0x0d62, 21}, - {0x0d63, 0x0dca, 103}, - {0x0dd2, 0x0dd4, 1}, + {0x0d63, 0x0d81, 30}, + {0x0dca, 0x0dd2, 8}, + {0x0dd3, 0x0dd4, 1}, {0x0dd6, 0x0e31, 91}, {0x0e34, 0x0e3a, 1}, {0x0e47, 0x0e4e, 1}, @@ -2107,6 +2150,7 @@ var _Mn = &RangeTable{ {0x1a73, 0x1a7c, 1}, {0x1a7f, 0x1ab0, 49}, {0x1ab1, 0x1abd, 1}, + {0x1abf, 0x1ac0, 1}, {0x1b00, 0x1b03, 1}, {0x1b34, 0x1b36, 2}, {0x1b37, 0x1b3a, 1}, @@ -2142,9 +2186,9 @@ var _Mn = &RangeTable{ {0xa6f0, 0xa6f1, 1}, {0xa802, 0xa806, 4}, {0xa80b, 0xa825, 26}, - {0xa826, 0xa8c4, 158}, - {0xa8c5, 0xa8e0, 27}, - {0xa8e1, 0xa8f1, 1}, + {0xa826, 0xa82c, 6}, + {0xa8c4, 0xa8c5, 1}, + {0xa8e0, 0xa8f1, 1}, {0xa8ff, 0xa926, 39}, {0xa927, 0xa92d, 1}, {0xa947, 0xa951, 1}, @@ -2178,6 +2222,7 @@ var _Mn = &RangeTable{ {0x10a3f, 0x10ae5, 166}, {0x10ae6, 0x10d24, 574}, {0x10d25, 0x10d27, 1}, + {0x10eab, 0x10eac, 1}, {0x10f46, 0x10f50, 1}, {0x11001, 0x11038, 55}, {0x11039, 0x11046, 1}, @@ -2191,7 +2236,8 @@ var _Mn = &RangeTable{ {0x11181, 0x111b6, 53}, {0x111b7, 0x111be, 1}, {0x111c9, 0x111cc, 1}, - {0x1122f, 0x11231, 1}, + {0x111cf, 0x1122f, 96}, + {0x11230, 0x11231, 1}, {0x11234, 0x11236, 2}, {0x11237, 0x1123e, 7}, {0x112df, 0x112e3, 4}, @@ -2223,6 +2269,8 @@ var _Mn = &RangeTable{ {0x11727, 0x1172b, 1}, {0x1182f, 0x11837, 1}, {0x11839, 0x1183a, 1}, + {0x1193b, 0x1193c, 1}, + {0x1193e, 0x11943, 5}, {0x119d4, 0x119d7, 1}, {0x119da, 0x119db, 1}, {0x119e0, 0x11a01, 33}, @@ -2253,8 +2301,9 @@ var _Mn = &RangeTable{ {0x16b30, 0x16b36, 1}, {0x16f4f, 0x16f8f, 64}, {0x16f90, 0x16f92, 1}, - {0x1bc9d, 0x1bc9e, 1}, - {0x1d167, 0x1d169, 1}, + {0x16fe4, 0x1bc9d, 19641}, + {0x1bc9e, 0x1d167, 5321}, + {0x1d168, 0x1d169, 1}, {0x1d17b, 0x1d182, 1}, {0x1d185, 0x1d18b, 1}, {0x1d1aa, 0x1d1ad, 1}, @@ -2375,6 +2424,7 @@ var _N = &RangeTable{ {0x10e60, 0x10e7e, 1}, {0x10f1d, 0x10f26, 1}, {0x10f51, 0x10f54, 1}, + {0x10fc5, 0x10fcb, 1}, {0x11052, 0x1106f, 1}, {0x110f0, 0x110f9, 1}, {0x11136, 0x1113f, 1}, @@ -2387,6 +2437,7 @@ var _N = &RangeTable{ {0x116c0, 0x116c9, 1}, {0x11730, 0x1173b, 1}, {0x118e0, 0x118f2, 1}, + {0x11950, 0x11959, 1}, {0x11c50, 0x11c6c, 1}, {0x11d50, 0x11d59, 1}, {0x11da0, 0x11da9, 1}, @@ -2409,6 +2460,7 @@ var _N = &RangeTable{ {0x1ed01, 0x1ed2d, 1}, {0x1ed2f, 0x1ed3d, 1}, {0x1f100, 0x1f10c, 1}, + {0x1fbf0, 0x1fbf9, 1}, }, LatinOffset: 4, } @@ -2467,6 +2519,7 @@ var _Nd = &RangeTable{ {0x116c0, 0x116c9, 1}, {0x11730, 0x11739, 1}, {0x118e0, 0x118e9, 1}, + {0x11950, 0x11959, 1}, {0x11c50, 0x11c59, 1}, {0x11d50, 0x11d59, 1}, {0x11da0, 0x11da9, 1}, @@ -2476,6 +2529,7 @@ var _Nd = &RangeTable{ {0x1e140, 0x1e149, 1}, {0x1e2f0, 0x1e2f9, 1}, {0x1e950, 0x1e959, 1}, + {0x1fbf0, 0x1fbf9, 1}, }, LatinOffset: 1, } @@ -2554,6 +2608,7 @@ var _No = &RangeTable{ {0x10e60, 0x10e7e, 1}, {0x10f1d, 0x10f26, 1}, {0x10f51, 0x10f54, 1}, + {0x10fc5, 0x10fcb, 1}, {0x11052, 0x11065, 1}, {0x111e1, 0x111f4, 1}, {0x1173a, 0x1173b, 1}, @@ -2655,7 +2710,8 @@ var _P = &RangeTable{ {0x2d70, 0x2e00, 144}, {0x2e01, 0x2e2e, 1}, {0x2e30, 0x2e4f, 1}, - {0x3001, 0x3003, 1}, + {0x2e52, 0x3001, 431}, + {0x3002, 0x3003, 1}, {0x3008, 0x3011, 1}, {0x3014, 0x301f, 1}, {0x3030, 0x303d, 13}, @@ -2701,7 +2757,8 @@ var _P = &RangeTable{ {0x10af1, 0x10af6, 1}, {0x10b39, 0x10b3f, 1}, {0x10b99, 0x10b9c, 1}, - {0x10f55, 0x10f59, 1}, + {0x10ead, 0x10f55, 168}, + {0x10f56, 0x10f59, 1}, {0x11047, 0x1104d, 1}, {0x110bb, 0x110bc, 1}, {0x110be, 0x110c1, 1}, @@ -2713,14 +2770,16 @@ var _P = &RangeTable{ {0x11238, 0x1123d, 1}, {0x112a9, 0x1144b, 418}, {0x1144c, 0x1144f, 1}, - {0x1145b, 0x1145d, 2}, - {0x114c6, 0x115c1, 251}, - {0x115c2, 0x115d7, 1}, + {0x1145a, 0x1145b, 1}, + {0x1145d, 0x114c6, 105}, + {0x115c1, 0x115d7, 1}, {0x11641, 0x11643, 1}, {0x11660, 0x1166c, 1}, {0x1173c, 0x1173e, 1}, - {0x1183b, 0x119e2, 423}, - {0x11a3f, 0x11a46, 1}, + {0x1183b, 0x11944, 265}, + {0x11945, 0x11946, 1}, + {0x119e2, 0x11a3f, 93}, + {0x11a40, 0x11a46, 1}, {0x11a9a, 0x11a9c, 1}, {0x11a9e, 0x11aa2, 1}, {0x11c41, 0x11c45, 1}, @@ -2764,6 +2823,9 @@ var _Pd = &RangeTable{ {0xfe58, 0xfe63, 11}, {0xff0d, 0xff0d, 1}, }, + R32: []Range32{ + {0x10ead, 0x10ead, 1}, + }, } var _Pe = &RangeTable{ @@ -2894,7 +2956,8 @@ var _Po = &RangeTable{ {0x2e3c, 0x2e3f, 1}, {0x2e41, 0x2e43, 2}, {0x2e44, 0x2e4f, 1}, - {0x3001, 0x3003, 1}, + {0x2e52, 0x3001, 431}, + {0x3002, 0x3003, 1}, {0x303d, 0x30fb, 190}, {0xa4fe, 0xa4ff, 1}, {0xa60d, 0xa60f, 1}, @@ -2951,14 +3014,16 @@ var _Po = &RangeTable{ {0x11238, 0x1123d, 1}, {0x112a9, 0x1144b, 418}, {0x1144c, 0x1144f, 1}, - {0x1145b, 0x1145d, 2}, - {0x114c6, 0x115c1, 251}, - {0x115c2, 0x115d7, 1}, + {0x1145a, 0x1145b, 1}, + {0x1145d, 0x114c6, 105}, + {0x115c1, 0x115d7, 1}, {0x11641, 0x11643, 1}, {0x11660, 0x1166c, 1}, {0x1173c, 0x1173e, 1}, - {0x1183b, 0x119e2, 423}, - {0x11a3f, 0x11a46, 1}, + {0x1183b, 0x11944, 265}, + {0x11945, 0x11946, 1}, + {0x119e2, 0x11a3f, 93}, + {0x11a40, 0x11a46, 1}, {0x11a9a, 0x11a9c, 1}, {0x11a9e, 0x11aa2, 1}, {0x11c41, 0x11c45, 1}, @@ -3094,8 +3159,9 @@ var _S = &RangeTable{ {0x29dc, 0x29fb, 1}, {0x29fe, 0x2b73, 1}, {0x2b76, 0x2b95, 1}, - {0x2b98, 0x2bff, 1}, + {0x2b97, 0x2bff, 1}, {0x2ce5, 0x2cea, 1}, + {0x2e50, 0x2e51, 1}, {0x2e80, 0x2e99, 1}, {0x2e9b, 0x2ef3, 1}, {0x2f00, 0x2fd5, 1}, @@ -3113,8 +3179,7 @@ var _S = &RangeTable{ {0x3250, 0x3260, 16}, {0x3261, 0x327f, 1}, {0x328a, 0x32b0, 1}, - {0x32c0, 0x32fe, 1}, - {0x3300, 0x33ff, 1}, + {0x32c0, 0x33ff, 1}, {0x4dc0, 0x4dff, 1}, {0xa490, 0xa4c6, 1}, {0xa700, 0xa716, 1}, @@ -3123,7 +3188,8 @@ var _S = &RangeTable{ {0xa828, 0xa82b, 1}, {0xa836, 0xa839, 1}, {0xaa77, 0xaa79, 1}, - {0xab5b, 0xfb29, 20430}, + {0xab5b, 0xab6a, 15}, + {0xab6b, 0xfb29, 20414}, {0xfbb2, 0xfbc1, 1}, {0xfdfc, 0xfdfd, 1}, {0xfe62, 0xfe64, 2}, @@ -3141,7 +3207,7 @@ var _S = &RangeTable{ {0x10137, 0x1013f, 1}, {0x10179, 0x10189, 1}, {0x1018c, 0x1018e, 1}, - {0x10190, 0x1019b, 1}, + {0x10190, 0x1019c, 1}, {0x101a0, 0x101d0, 48}, {0x101d1, 0x101fc, 1}, {0x10877, 0x10878, 1}, @@ -3179,16 +3245,15 @@ var _S = &RangeTable{ {0x1f0b1, 0x1f0bf, 1}, {0x1f0c1, 0x1f0cf, 1}, {0x1f0d1, 0x1f0f5, 1}, - {0x1f110, 0x1f16c, 1}, - {0x1f170, 0x1f1ac, 1}, + {0x1f10d, 0x1f1ad, 1}, {0x1f1e6, 0x1f202, 1}, {0x1f210, 0x1f23b, 1}, {0x1f240, 0x1f248, 1}, {0x1f250, 0x1f251, 1}, {0x1f260, 0x1f265, 1}, - {0x1f300, 0x1f6d5, 1}, + {0x1f300, 0x1f6d7, 1}, {0x1f6e0, 0x1f6ec, 1}, - {0x1f6f0, 0x1f6fa, 1}, + {0x1f6f0, 0x1f6fc, 1}, {0x1f700, 0x1f773, 1}, {0x1f780, 0x1f7d8, 1}, {0x1f7e0, 0x1f7eb, 1}, @@ -3197,18 +3262,20 @@ var _S = &RangeTable{ {0x1f850, 0x1f859, 1}, {0x1f860, 0x1f887, 1}, {0x1f890, 0x1f8ad, 1}, - {0x1f900, 0x1f90b, 1}, - {0x1f90d, 0x1f971, 1}, - {0x1f973, 0x1f976, 1}, - {0x1f97a, 0x1f9a2, 1}, - {0x1f9a5, 0x1f9aa, 1}, - {0x1f9ae, 0x1f9ca, 1}, + {0x1f8b0, 0x1f8b1, 1}, + {0x1f900, 0x1f978, 1}, + {0x1f97a, 0x1f9cb, 1}, {0x1f9cd, 0x1fa53, 1}, {0x1fa60, 0x1fa6d, 1}, - {0x1fa70, 0x1fa73, 1}, + {0x1fa70, 0x1fa74, 1}, {0x1fa78, 0x1fa7a, 1}, - {0x1fa80, 0x1fa82, 1}, - {0x1fa90, 0x1fa95, 1}, + {0x1fa80, 0x1fa86, 1}, + {0x1fa90, 0x1faa8, 1}, + {0x1fab0, 0x1fab6, 1}, + {0x1fac0, 0x1fac2, 1}, + {0x1fad0, 0x1fad6, 1}, + {0x1fb00, 0x1fb92, 1}, + {0x1fb94, 0x1fbca, 1}, }, LatinOffset: 10, } @@ -3257,7 +3324,8 @@ var _Sk = &RangeTable{ {0xa700, 0xa716, 1}, {0xa720, 0xa721, 1}, {0xa789, 0xa78a, 1}, - {0xab5b, 0xfbb2, 20567}, + {0xab5b, 0xab6a, 15}, + {0xab6b, 0xfbb2, 20551}, {0xfbb3, 0xfbc1, 1}, {0xff3e, 0xff40, 2}, {0xffe3, 0xffe3, 1}, @@ -3394,8 +3462,9 @@ var _So = &RangeTable{ {0x2b45, 0x2b46, 1}, {0x2b4d, 0x2b73, 1}, {0x2b76, 0x2b95, 1}, - {0x2b98, 0x2bff, 1}, + {0x2b97, 0x2bff, 1}, {0x2ce5, 0x2cea, 1}, + {0x2e50, 0x2e51, 1}, {0x2e80, 0x2e99, 1}, {0x2e9b, 0x2ef3, 1}, {0x2f00, 0x2fd5, 1}, @@ -3412,8 +3481,7 @@ var _So = &RangeTable{ {0x3250, 0x3260, 16}, {0x3261, 0x327f, 1}, {0x328a, 0x32b0, 1}, - {0x32c0, 0x32fe, 1}, - {0x3300, 0x33ff, 1}, + {0x32c0, 0x33ff, 1}, {0x4dc0, 0x4dff, 1}, {0xa490, 0xa4c6, 1}, {0xa828, 0xa82b, 1}, @@ -3429,7 +3497,7 @@ var _So = &RangeTable{ {0x10137, 0x1013f, 1}, {0x10179, 0x10189, 1}, {0x1018c, 0x1018e, 1}, - {0x10190, 0x1019b, 1}, + {0x10190, 0x1019c, 1}, {0x101a0, 0x101d0, 48}, {0x101d1, 0x101fc, 1}, {0x10877, 0x10878, 1}, @@ -3461,17 +3529,16 @@ var _So = &RangeTable{ {0x1f0b1, 0x1f0bf, 1}, {0x1f0c1, 0x1f0cf, 1}, {0x1f0d1, 0x1f0f5, 1}, - {0x1f110, 0x1f16c, 1}, - {0x1f170, 0x1f1ac, 1}, + {0x1f10d, 0x1f1ad, 1}, {0x1f1e6, 0x1f202, 1}, {0x1f210, 0x1f23b, 1}, {0x1f240, 0x1f248, 1}, {0x1f250, 0x1f251, 1}, {0x1f260, 0x1f265, 1}, {0x1f300, 0x1f3fa, 1}, - {0x1f400, 0x1f6d5, 1}, + {0x1f400, 0x1f6d7, 1}, {0x1f6e0, 0x1f6ec, 1}, - {0x1f6f0, 0x1f6fa, 1}, + {0x1f6f0, 0x1f6fc, 1}, {0x1f700, 0x1f773, 1}, {0x1f780, 0x1f7d8, 1}, {0x1f7e0, 0x1f7eb, 1}, @@ -3480,18 +3547,20 @@ var _So = &RangeTable{ {0x1f850, 0x1f859, 1}, {0x1f860, 0x1f887, 1}, {0x1f890, 0x1f8ad, 1}, - {0x1f900, 0x1f90b, 1}, - {0x1f90d, 0x1f971, 1}, - {0x1f973, 0x1f976, 1}, - {0x1f97a, 0x1f9a2, 1}, - {0x1f9a5, 0x1f9aa, 1}, - {0x1f9ae, 0x1f9ca, 1}, + {0x1f8b0, 0x1f8b1, 1}, + {0x1f900, 0x1f978, 1}, + {0x1f97a, 0x1f9cb, 1}, {0x1f9cd, 0x1fa53, 1}, {0x1fa60, 0x1fa6d, 1}, - {0x1fa70, 0x1fa73, 1}, + {0x1fa70, 0x1fa74, 1}, {0x1fa78, 0x1fa7a, 1}, - {0x1fa80, 0x1fa82, 1}, - {0x1fa90, 0x1fa95, 1}, + {0x1fa80, 0x1fa86, 1}, + {0x1fa90, 0x1faa8, 1}, + {0x1fab0, 0x1fab6, 1}, + {0x1fac0, 0x1fac2, 1}, + {0x1fad0, 0x1fad6, 1}, + {0x1fb00, 0x1fb92, 1}, + {0x1fb94, 0x1fbca, 1}, }, LatinOffset: 2, } @@ -3607,6 +3676,7 @@ var Scripts = map[string]*RangeTable{ "Chakma": Chakma, "Cham": Cham, "Cherokee": Cherokee, + "Chorasmian": Chorasmian, "Common": Common, "Coptic": Coptic, "Cuneiform": Cuneiform, @@ -3614,6 +3684,7 @@ var Scripts = map[string]*RangeTable{ "Cyrillic": Cyrillic, "Deseret": Deseret, "Devanagari": Devanagari, + "Dives_Akuru": Dives_Akuru, "Dogra": Dogra, "Duployan": Duployan, "Egyptian_Hieroglyphs": Egyptian_Hieroglyphs, @@ -3645,6 +3716,7 @@ var Scripts = map[string]*RangeTable{ "Katakana": Katakana, "Kayah_Li": Kayah_Li, "Kharoshthi": Kharoshthi, + "Khitan_Small_Script": Khitan_Small_Script, "Khmer": Khmer, "Khojki": Khojki, "Khudawadi": Khudawadi, @@ -3734,6 +3806,7 @@ var Scripts = map[string]*RangeTable{ "Vai": Vai, "Wancho": Wancho, "Warang_Citi": Warang_Citi, + "Yezidi": Yezidi, "Yi": Yi, "Zanabazar_Square": Zanabazar_Square, } @@ -3776,7 +3849,7 @@ var _Arabic = &RangeTable{ {0x06de, 0x06ff, 1}, {0x0750, 0x077f, 1}, {0x08a0, 0x08b4, 1}, - {0x08b6, 0x08bd, 1}, + {0x08b6, 0x08c7, 1}, {0x08d3, 0x08e1, 1}, {0x08e3, 0x08ff, 1}, {0xfb50, 0xfbc1, 1}, @@ -3820,9 +3893,8 @@ var _Arabic = &RangeTable{ var _Armenian = &RangeTable{ R16: []Range16{ {0x0531, 0x0556, 1}, - {0x0559, 0x0588, 1}, - {0x058a, 0x058d, 3}, - {0x058e, 0x058f, 1}, + {0x0559, 0x058a, 1}, + {0x058d, 0x058f, 1}, {0xfb13, 0xfb17, 1}, }, } @@ -3899,7 +3971,7 @@ var _Bopomofo = &RangeTable{ R16: []Range16{ {0x02ea, 0x02eb, 1}, {0x3105, 0x312f, 1}, - {0x31a0, 0x31ba, 1}, + {0x31a0, 0x31bf, 1}, }, } @@ -3957,7 +4029,7 @@ var _Chakma = &RangeTable{ R16: []Range16{}, R32: []Range32{ {0x11100, 0x11134, 1}, - {0x11136, 0x11146, 1}, + {0x11136, 0x11147, 1}, }, } @@ -3978,6 +4050,13 @@ var _Cherokee = &RangeTable{ }, } +var _Chorasmian = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10fb0, 0x10fcb, 1}, + }, +} + var _Common = &RangeTable{ R16: []Range16{ {0x0000, 0x0040, 1}, @@ -3991,13 +4070,12 @@ var _Common = &RangeTable{ {0x02ec, 0x02ff, 1}, {0x0374, 0x037e, 10}, {0x0385, 0x0387, 2}, - {0x0589, 0x0605, 124}, - {0x060c, 0x061b, 15}, - {0x061f, 0x0640, 33}, - {0x06dd, 0x08e2, 517}, - {0x0964, 0x0965, 1}, - {0x0e3f, 0x0fd5, 406}, - {0x0fd6, 0x0fd8, 1}, + {0x0605, 0x060c, 7}, + {0x061b, 0x061f, 4}, + {0x0640, 0x06dd, 157}, + {0x08e2, 0x0964, 130}, + {0x0965, 0x0e3f, 1242}, + {0x0fd5, 0x0fd8, 1}, {0x10fb, 0x16eb, 1520}, {0x16ec, 0x16ed, 1}, {0x1735, 0x1736, 1}, @@ -4025,8 +4103,8 @@ var _Common = &RangeTable{ {0x2460, 0x27ff, 1}, {0x2900, 0x2b73, 1}, {0x2b76, 0x2b95, 1}, - {0x2b98, 0x2bff, 1}, - {0x2e00, 0x2e4f, 1}, + {0x2b97, 0x2bff, 1}, + {0x2e00, 0x2e52, 1}, {0x2ff0, 0x2ffb, 1}, {0x3000, 0x3004, 1}, {0x3006, 0x3008, 2}, @@ -4040,13 +4118,15 @@ var _Common = &RangeTable{ {0x31c0, 0x31e3, 1}, {0x3220, 0x325f, 1}, {0x327f, 0x32cf, 1}, - {0x3358, 0x33ff, 1}, + {0x32ff, 0x3358, 89}, + {0x3359, 0x33ff, 1}, {0x4dc0, 0x4dff, 1}, {0xa700, 0xa721, 1}, {0xa788, 0xa78a, 1}, {0xa830, 0xa839, 1}, {0xa92e, 0xa9cf, 161}, - {0xab5b, 0xfd3e, 20963}, + {0xab5b, 0xab6a, 15}, + {0xab6b, 0xfd3e, 20947}, {0xfd3f, 0xfe10, 209}, {0xfe11, 0xfe19, 1}, {0xfe30, 0xfe52, 1}, @@ -4066,7 +4146,7 @@ var _Common = &RangeTable{ {0x10100, 0x10102, 1}, {0x10107, 0x10133, 1}, {0x10137, 0x1013f, 1}, - {0x10190, 0x1019b, 1}, + {0x10190, 0x1019c, 1}, {0x101d0, 0x101fc, 1}, {0x102e1, 0x102fb, 1}, {0x16fe2, 0x16fe3, 1}, @@ -4110,18 +4190,16 @@ var _Common = &RangeTable{ {0x1f0b1, 0x1f0bf, 1}, {0x1f0c1, 0x1f0cf, 1}, {0x1f0d1, 0x1f0f5, 1}, - {0x1f100, 0x1f10c, 1}, - {0x1f110, 0x1f16c, 1}, - {0x1f170, 0x1f1ac, 1}, + {0x1f100, 0x1f1ad, 1}, {0x1f1e6, 0x1f1ff, 1}, {0x1f201, 0x1f202, 1}, {0x1f210, 0x1f23b, 1}, {0x1f240, 0x1f248, 1}, {0x1f250, 0x1f251, 1}, {0x1f260, 0x1f265, 1}, - {0x1f300, 0x1f6d5, 1}, + {0x1f300, 0x1f6d7, 1}, {0x1f6e0, 0x1f6ec, 1}, - {0x1f6f0, 0x1f6fa, 1}, + {0x1f6f0, 0x1f6fc, 1}, {0x1f700, 0x1f773, 1}, {0x1f780, 0x1f7d8, 1}, {0x1f7e0, 0x1f7eb, 1}, @@ -4130,18 +4208,21 @@ var _Common = &RangeTable{ {0x1f850, 0x1f859, 1}, {0x1f860, 0x1f887, 1}, {0x1f890, 0x1f8ad, 1}, - {0x1f900, 0x1f90b, 1}, - {0x1f90d, 0x1f971, 1}, - {0x1f973, 0x1f976, 1}, - {0x1f97a, 0x1f9a2, 1}, - {0x1f9a5, 0x1f9aa, 1}, - {0x1f9ae, 0x1f9ca, 1}, + {0x1f8b0, 0x1f8b1, 1}, + {0x1f900, 0x1f978, 1}, + {0x1f97a, 0x1f9cb, 1}, {0x1f9cd, 0x1fa53, 1}, {0x1fa60, 0x1fa6d, 1}, - {0x1fa70, 0x1fa73, 1}, + {0x1fa70, 0x1fa74, 1}, {0x1fa78, 0x1fa7a, 1}, - {0x1fa80, 0x1fa82, 1}, - {0x1fa90, 0x1fa95, 1}, + {0x1fa80, 0x1fa86, 1}, + {0x1fa90, 0x1faa8, 1}, + {0x1fab0, 0x1fab6, 1}, + {0x1fac0, 0x1fac2, 1}, + {0x1fad0, 0x1fad6, 1}, + {0x1fb00, 0x1fb92, 1}, + {0x1fb94, 0x1fbca, 1}, + {0x1fbf0, 0x1fbf9, 1}, {0xe0001, 0xe0020, 31}, {0xe0021, 0xe007f, 1}, }, @@ -4205,6 +4286,20 @@ var _Devanagari = &RangeTable{ }, } +var _Dives_Akuru = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x11900, 0x11906, 1}, + {0x11909, 0x1190c, 3}, + {0x1190d, 0x11913, 1}, + {0x11915, 0x11916, 1}, + {0x11918, 0x11935, 1}, + {0x11937, 0x11938, 1}, + {0x1193b, 0x11946, 1}, + {0x11950, 0x11959, 1}, + }, +} + var _Dogra = &RangeTable{ R16: []Range16{}, R32: []Range32{ @@ -4435,18 +4530,20 @@ var _Han = &RangeTable{ {0x3005, 0x3007, 2}, {0x3021, 0x3029, 1}, {0x3038, 0x303b, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fef, 1}, + {0x3400, 0x4dbf, 1}, + {0x4e00, 0x9ffc, 1}, {0xf900, 0xfa6d, 1}, {0xfa70, 0xfad9, 1}, }, R32: []Range32{ - {0x20000, 0x2a6d6, 1}, + {0x16ff0, 0x16ff1, 1}, + {0x20000, 0x2a6dd, 1}, {0x2a700, 0x2b734, 1}, {0x2b740, 0x2b81d, 1}, {0x2b820, 0x2cea1, 1}, {0x2ceb0, 0x2ebe0, 1}, {0x2f800, 0x2fa1d, 1}, + {0x30000, 0x3134a, 1}, }, } @@ -4533,7 +4630,7 @@ var _Inherited = &RangeTable{ {0x064b, 0x0655, 1}, {0x0670, 0x0951, 737}, {0x0952, 0x0954, 1}, - {0x1ab0, 0x1abe, 1}, + {0x1ab0, 0x1ac0, 1}, {0x1cd0, 0x1cd2, 1}, {0x1cd4, 0x1ce0, 1}, {0x1ce2, 0x1ce8, 1}, @@ -4646,6 +4743,14 @@ var _Kharoshthi = &RangeTable{ }, } +var _Khitan_Small_Script = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x16fe4, 0x18b00, 6940}, + {0x18b01, 0x18cd5, 1}, + }, +} + var _Khmer = &RangeTable{ R16: []Range16{ {0x1780, 0x17dd, 1}, @@ -4710,11 +4815,11 @@ var _Latin = &RangeTable{ {0x2c60, 0x2c7f, 1}, {0xa722, 0xa787, 1}, {0xa78b, 0xa7bf, 1}, - {0xa7c2, 0xa7c6, 1}, - {0xa7f7, 0xa7ff, 1}, + {0xa7c2, 0xa7ca, 1}, + {0xa7f5, 0xa7ff, 1}, {0xab30, 0xab5a, 1}, {0xab5c, 0xab64, 1}, - {0xab66, 0xab67, 1}, + {0xab66, 0xab69, 1}, {0xfb00, 0xfb06, 1}, {0xff21, 0xff3a, 1}, {0xff41, 0xff5a, 1}, @@ -4766,6 +4871,9 @@ var _Lisu = &RangeTable{ R16: []Range16{ {0xa4d0, 0xa4ff, 1}, }, + R32: []Range32{ + {0x11fb0, 0x11fb0, 1}, + }, } var _Lycian = &RangeTable{ @@ -4799,8 +4907,7 @@ var _Makasar = &RangeTable{ var _Malayalam = &RangeTable{ R16: []Range16{ - {0x0d00, 0x0d03, 1}, - {0x0d05, 0x0d0c, 1}, + {0x0d00, 0x0d0c, 1}, {0x0d0e, 0x0d10, 1}, {0x0d12, 0x0d44, 1}, {0x0d46, 0x0d48, 1}, @@ -4974,9 +5081,8 @@ var _New_Tai_Lue = &RangeTable{ var _Newa = &RangeTable{ R16: []Range16{}, R32: []Range32{ - {0x11400, 0x11459, 1}, - {0x1145b, 0x1145d, 2}, - {0x1145e, 0x1145f, 1}, + {0x11400, 0x1145b, 1}, + {0x1145d, 0x11461, 1}, }, } @@ -5089,7 +5195,7 @@ var _Oriya = &RangeTable{ {0x0b3c, 0x0b44, 1}, {0x0b47, 0x0b48, 1}, {0x0b4b, 0x0b4d, 1}, - {0x0b56, 0x0b57, 1}, + {0x0b55, 0x0b57, 1}, {0x0b5c, 0x0b5d, 1}, {0x0b5f, 0x0b63, 1}, {0x0b66, 0x0b77, 1}, @@ -5191,8 +5297,7 @@ var _Saurashtra = &RangeTable{ var _Sharada = &RangeTable{ R16: []Range16{}, R32: []Range32{ - {0x11180, 0x111cd, 1}, - {0x111d0, 0x111df, 1}, + {0x11180, 0x111df, 1}, }, } @@ -5222,7 +5327,7 @@ var _SignWriting = &RangeTable{ var _Sinhala = &RangeTable{ R16: []Range16{ - {0x0d82, 0x0d83, 1}, + {0x0d81, 0x0d83, 1}, {0x0d85, 0x0d96, 1}, {0x0d9a, 0x0db1, 1}, {0x0db3, 0x0dbb, 1}, @@ -5271,7 +5376,7 @@ var _Sundanese = &RangeTable{ var _Syloti_Nagri = &RangeTable{ R16: []Range16{ - {0xa800, 0xa82b, 1}, + {0xa800, 0xa82c, 1}, }, } @@ -5360,7 +5465,8 @@ var _Tangut = &RangeTable{ R32: []Range32{ {0x16fe0, 0x17000, 32}, {0x17001, 0x187f7, 1}, - {0x18800, 0x18af2, 1}, + {0x18800, 0x18aff, 1}, + {0x18d00, 0x18d08, 1}, }, } @@ -5452,6 +5558,15 @@ var _Warang_Citi = &RangeTable{ }, } +var _Yezidi = &RangeTable{ + R16: []Range16{}, + R32: []Range32{ + {0x10e80, 0x10ea9, 1}, + {0x10eab, 0x10ead, 1}, + {0x10eb0, 0x10eb1, 1}, + }, +} + var _Yi = &RangeTable{ R16: []Range16{ {0xa000, 0xa48c, 1}, @@ -5491,6 +5606,7 @@ var ( Chakma = _Chakma // Chakma is the set of Unicode characters in script Chakma. Cham = _Cham // Cham is the set of Unicode characters in script Cham. Cherokee = _Cherokee // Cherokee is the set of Unicode characters in script Cherokee. + Chorasmian = _Chorasmian // Chorasmian is the set of Unicode characters in script Chorasmian. Common = _Common // Common is the set of Unicode characters in script Common. Coptic = _Coptic // Coptic is the set of Unicode characters in script Coptic. Cuneiform = _Cuneiform // Cuneiform is the set of Unicode characters in script Cuneiform. @@ -5498,6 +5614,7 @@ var ( Cyrillic = _Cyrillic // Cyrillic is the set of Unicode characters in script Cyrillic. Deseret = _Deseret // Deseret is the set of Unicode characters in script Deseret. Devanagari = _Devanagari // Devanagari is the set of Unicode characters in script Devanagari. + Dives_Akuru = _Dives_Akuru // Dives_Akuru is the set of Unicode characters in script Dives_Akuru. Dogra = _Dogra // Dogra is the set of Unicode characters in script Dogra. Duployan = _Duployan // Duployan is the set of Unicode characters in script Duployan. Egyptian_Hieroglyphs = _Egyptian_Hieroglyphs // Egyptian_Hieroglyphs is the set of Unicode characters in script Egyptian_Hieroglyphs. @@ -5529,6 +5646,7 @@ var ( Katakana = _Katakana // Katakana is the set of Unicode characters in script Katakana. Kayah_Li = _Kayah_Li // Kayah_Li is the set of Unicode characters in script Kayah_Li. Kharoshthi = _Kharoshthi // Kharoshthi is the set of Unicode characters in script Kharoshthi. + Khitan_Small_Script = _Khitan_Small_Script // Khitan_Small_Script is the set of Unicode characters in script Khitan_Small_Script. Khmer = _Khmer // Khmer is the set of Unicode characters in script Khmer. Khojki = _Khojki // Khojki is the set of Unicode characters in script Khojki. Khudawadi = _Khudawadi // Khudawadi is the set of Unicode characters in script Khudawadi. @@ -5618,6 +5736,7 @@ var ( Vai = _Vai // Vai is the set of Unicode characters in script Vai. Wancho = _Wancho // Wancho is the set of Unicode characters in script Wancho. Warang_Citi = _Warang_Citi // Warang_Citi is the set of Unicode characters in script Warang_Citi. + Yezidi = _Yezidi // Yezidi is the set of Unicode characters in script Yezidi. Yi = _Yi // Yi is the set of Unicode characters in script Yi. Zanabazar_Square = _Zanabazar_Square // Zanabazar_Square is the set of Unicode characters in script Zanabazar_Square. ) @@ -5695,6 +5814,9 @@ var _Dash = &RangeTable{ {0xfe58, 0xfe63, 11}, {0xff0d, 0xff0d, 1}, }, + R32: []Range32{ + {0x10ead, 0x10ead, 1}, + }, } var _Deprecated = &RangeTable{ @@ -5746,10 +5868,11 @@ var _Diacritic = &RangeTable{ {0x0acd, 0x0afd, 48}, {0x0afe, 0x0aff, 1}, {0x0b3c, 0x0b4d, 17}, - {0x0bcd, 0x0c4d, 128}, - {0x0cbc, 0x0ccd, 17}, - {0x0d3b, 0x0d3c, 1}, - {0x0d4d, 0x0e47, 125}, + {0x0b55, 0x0bcd, 120}, + {0x0c4d, 0x0cbc, 111}, + {0x0ccd, 0x0d3b, 110}, + {0x0d3c, 0x0d4d, 17}, + {0x0dca, 0x0e47, 125}, {0x0e48, 0x0e4c, 1}, {0x0e4e, 0x0eba, 108}, {0x0ec8, 0x0ecc, 1}, @@ -5811,6 +5934,7 @@ var _Diacritic = &RangeTable{ {0xaabf, 0xaac2, 1}, {0xaaf6, 0xab5b, 101}, {0xab5c, 0xab5f, 1}, + {0xab69, 0xab6b, 1}, {0xabec, 0xabed, 1}, {0xfb1e, 0xfe20, 770}, {0xfe21, 0xfe2f, 1}, @@ -5838,14 +5962,16 @@ var _Diacritic = &RangeTable{ {0x1163f, 0x116b6, 119}, {0x116b7, 0x1172b, 116}, {0x11839, 0x1183a, 1}, - {0x119e0, 0x11a34, 84}, - {0x11a47, 0x11a99, 82}, - {0x11c3f, 0x11d42, 259}, - {0x11d44, 0x11d45, 1}, - {0x11d97, 0x16af0, 19801}, - {0x16af1, 0x16af4, 1}, + {0x1193d, 0x1193e, 1}, + {0x11943, 0x119e0, 157}, + {0x11a34, 0x11a47, 19}, + {0x11a99, 0x11c3f, 422}, + {0x11d42, 0x11d44, 2}, + {0x11d45, 0x11d97, 82}, + {0x16af0, 0x16af4, 1}, {0x16b30, 0x16b36, 1}, {0x16f8f, 0x16f9f, 1}, + {0x16ff0, 0x16ff1, 1}, {0x1d167, 0x1d169, 1}, {0x1d16d, 0x1d172, 1}, {0x1d17b, 0x1d182, 1}, @@ -5864,12 +5990,12 @@ var _Extender = &RangeTable{ R16: []Range16{ {0x00b7, 0x02d0, 537}, {0x02d1, 0x0640, 879}, - {0x07fa, 0x0e46, 1612}, - {0x0ec6, 0x180a, 2372}, - {0x1843, 0x1aa7, 612}, - {0x1c36, 0x1c7b, 69}, - {0x3005, 0x3031, 44}, - {0x3032, 0x3035, 1}, + {0x07fa, 0x0b55, 859}, + {0x0e46, 0x0ec6, 128}, + {0x180a, 0x1843, 57}, + {0x1aa7, 0x1c36, 399}, + {0x1c7b, 0x3005, 5002}, + {0x3031, 0x3035, 1}, {0x309d, 0x309e, 1}, {0x30fc, 0x30fe, 1}, {0xa015, 0xa60c, 1527}, @@ -5931,21 +6057,24 @@ var _Ideographic = &RangeTable{ {0x3006, 0x3007, 1}, {0x3021, 0x3029, 1}, {0x3038, 0x303a, 1}, - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fef, 1}, + {0x3400, 0x4dbf, 1}, + {0x4e00, 0x9ffc, 1}, {0xf900, 0xfa6d, 1}, {0xfa70, 0xfad9, 1}, }, R32: []Range32{ - {0x17000, 0x187f7, 1}, - {0x18800, 0x18af2, 1}, + {0x16fe4, 0x17000, 28}, + {0x17001, 0x187f7, 1}, + {0x18800, 0x18cd5, 1}, + {0x18d00, 0x18d08, 1}, {0x1b170, 0x1b2fb, 1}, - {0x20000, 0x2a6d6, 1}, + {0x20000, 0x2a6dd, 1}, {0x2a700, 0x2b734, 1}, {0x2b740, 0x2b81d, 1}, {0x2b820, 0x2cea1, 1}, {0x2ceb0, 0x2ebe0, 1}, {0x2f800, 0x2fa1d, 1}, + {0x30000, 0x3134a, 1}, }, } @@ -6066,9 +6195,9 @@ var _Other_Alphabetic = &RangeTable{ {0x0d46, 0x0d48, 1}, {0x0d4a, 0x0d4c, 1}, {0x0d57, 0x0d62, 11}, - {0x0d63, 0x0d82, 31}, - {0x0d83, 0x0dcf, 76}, - {0x0dd0, 0x0dd4, 1}, + {0x0d63, 0x0d81, 30}, + {0x0d82, 0x0d83, 1}, + {0x0dcf, 0x0dd4, 1}, {0x0dd6, 0x0dd8, 2}, {0x0dd9, 0x0ddf, 1}, {0x0df2, 0x0df3, 1}, @@ -6104,6 +6233,7 @@ var _Other_Alphabetic = &RangeTable{ {0x1a17, 0x1a1b, 1}, {0x1a55, 0x1a5e, 1}, {0x1a61, 0x1a74, 1}, + {0x1abf, 0x1ac0, 1}, {0x1b00, 0x1b04, 1}, {0x1b35, 0x1b43, 1}, {0x1b80, 0x1b82, 1}, @@ -6145,6 +6275,7 @@ var _Other_Alphabetic = &RangeTable{ {0x10a05, 0x10a06, 1}, {0x10a0c, 0x10a0f, 1}, {0x10d24, 0x10d27, 1}, + {0x10eab, 0x10eac, 1}, {0x11000, 0x11002, 1}, {0x11038, 0x11045, 1}, {0x11082, 0x110b0, 46}, @@ -6154,6 +6285,7 @@ var _Other_Alphabetic = &RangeTable{ {0x11145, 0x11146, 1}, {0x11180, 0x11182, 1}, {0x111b3, 0x111bf, 1}, + {0x111ce, 0x111cf, 1}, {0x1122c, 0x11234, 1}, {0x11237, 0x1123e, 7}, {0x112df, 0x112e8, 1}, @@ -6174,6 +6306,10 @@ var _Other_Alphabetic = &RangeTable{ {0x116ac, 0x116b5, 1}, {0x1171d, 0x1172a, 1}, {0x1182c, 0x11838, 1}, + {0x11930, 0x11935, 1}, + {0x11937, 0x11938, 1}, + {0x1193b, 0x1193c, 1}, + {0x11940, 0x11942, 2}, {0x119d1, 0x119d7, 1}, {0x119da, 0x119df, 1}, {0x119e4, 0x11a01, 29}, @@ -6198,6 +6334,7 @@ var _Other_Alphabetic = &RangeTable{ {0x16f4f, 0x16f51, 2}, {0x16f52, 0x16f87, 1}, {0x16f8f, 0x16f92, 1}, + {0x16ff0, 0x16ff1, 1}, {0x1bc9e, 0x1e000, 9058}, {0x1e001, 0x1e006, 1}, {0x1e008, 0x1e018, 1}, @@ -6243,8 +6380,9 @@ var _Other_Grapheme_Extend = &RangeTable{ R32: []Range32{ {0x1133e, 0x11357, 25}, {0x114b0, 0x114bd, 13}, - {0x115af, 0x1d165, 48054}, - {0x1d16e, 0x1d172, 1}, + {0x115af, 0x11930, 897}, + {0x1d165, 0x1d16e, 9}, + {0x1d16f, 0x1d172, 1}, {0xe0020, 0xe007f, 1}, }, } @@ -6562,6 +6700,7 @@ var _Sentence_Terminal = &RangeTable{ {0x115c9, 0x115d7, 1}, {0x11641, 0x11642, 1}, {0x1173c, 0x1173e, 1}, + {0x11944, 0x11946, 2}, {0x11a42, 0x11a43, 1}, {0x11a9b, 0x11a9c, 1}, {0x11c41, 0x11c42, 1}, @@ -6679,11 +6818,12 @@ var _Terminal_Punctuation = &RangeTable{ {0x11239, 0x1123c, 1}, {0x112a9, 0x1144b, 418}, {0x1144c, 0x1144d, 1}, - {0x1145b, 0x115c2, 359}, - {0x115c3, 0x115c5, 1}, + {0x1145a, 0x1145b, 1}, + {0x115c2, 0x115c5, 1}, {0x115c9, 0x115d7, 1}, {0x11641, 0x11642, 1}, {0x1173c, 0x1173e, 1}, + {0x11944, 0x11946, 2}, {0x11a42, 0x11a43, 1}, {0x11a9b, 0x11a9c, 1}, {0x11aa1, 0x11aa2, 1}, @@ -6703,8 +6843,8 @@ var _Terminal_Punctuation = &RangeTable{ var _Unified_Ideograph = &RangeTable{ R16: []Range16{ - {0x3400, 0x4db5, 1}, - {0x4e00, 0x9fef, 1}, + {0x3400, 0x4dbf, 1}, + {0x4e00, 0x9ffc, 1}, {0xfa0e, 0xfa0f, 1}, {0xfa11, 0xfa13, 2}, {0xfa14, 0xfa1f, 11}, @@ -6713,11 +6853,12 @@ var _Unified_Ideograph = &RangeTable{ {0xfa28, 0xfa29, 1}, }, R32: []Range32{ - {0x20000, 0x2a6d6, 1}, + {0x20000, 0x2a6dd, 1}, {0x2a700, 0x2b734, 1}, {0x2b740, 0x2b81d, 1}, {0x2b820, 0x2cea1, 1}, {0x2ceb0, 0x2ebe0, 1}, + {0x30000, 0x3134a, 1}, }, } @@ -7088,6 +7229,8 @@ var _CaseRanges = []CaseRange{ {0xA7C4, 0xA7C4, d{0, -48, 0}}, {0xA7C5, 0xA7C5, d{0, -42307, 0}}, {0xA7C6, 0xA7C6, d{0, -35384, 0}}, + {0xA7C7, 0xA7CA, d{UpperLower, UpperLower, UpperLower}}, + {0xA7F5, 0xA7F6, d{UpperLower, UpperLower, UpperLower}}, {0xAB53, 0xAB53, d{-928, 0, -928}}, {0xAB70, 0xABBF, d{-38864, 0, -38864}}, {0xFF21, 0xFF3A, d{0, 32, 0}}, @@ -7711,7 +7854,8 @@ var foldLl = &RangeTable{ {0xa7b0, 0xa7b4, 1}, {0xa7b6, 0xa7be, 2}, {0xa7c2, 0xa7c4, 2}, - {0xa7c5, 0xa7c6, 1}, + {0xa7c5, 0xa7c7, 1}, + {0xa7c9, 0xa7f5, 44}, {0xff21, 0xff3a, 1}, }, R32: []Range32{ @@ -7844,8 +7988,10 @@ var foldLu = &RangeTable{ {0xa793, 0xa794, 1}, {0xa797, 0xa7a9, 2}, {0xa7b5, 0xa7bf, 2}, - {0xa7c3, 0xab53, 912}, - {0xab70, 0xabbf, 1}, + {0xa7c3, 0xa7c8, 5}, + {0xa7ca, 0xa7f6, 44}, + {0xab53, 0xab70, 29}, + {0xab71, 0xabbf, 1}, {0xff41, 0xff5a, 1}, }, R32: []Range32{ @@ -7902,7 +8048,7 @@ var foldInherited = &RangeTable{ }, } -// Range entries: 3483 16-bit, 1730 32-bit, 5213 total. -// Range bytes: 20898 16-bit, 20760 32-bit, 41658 total. +// Range entries: 3499 16-bit, 1820 32-bit, 5319 total. +// Range bytes: 20994 16-bit, 21840 32-bit, 42834 total. // Fold orbit bytes: 88 pairs, 352 bytes -- cgit v1.3-5-g9baa From c57c0212ebf44b7896477177c8e35be79ff3a586 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 19 Aug 2020 23:01:58 -0700 Subject: cmd/compile: define starting block before opening html writer Opening the html writer can fail, and the failure printer wants to use the entry block's line number. So make sure we set up the entry block first. Fixes #40919 Change-Id: I4ffa2839b45a721bbaf04ff84418e8108fa1cc37 Reviewed-on: https://go-review.googlesource.com/c/go/+/249497 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: David Chase --- src/cmd/compile/internal/gc/ssa.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 5d0098b4e6..4124655b79 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -338,6 +338,10 @@ func buildssa(fn *Node, worker int) *ssa.Func { s.panics = map[funcLine]*ssa.Block{} s.softFloat = s.config.SoftFloat + // Allocate starting block + s.f.Entry = s.f.NewBlock(ssa.BlockPlain) + s.f.Entry.Pos = fn.Pos + if printssa { s.f.HTMLWriter = ssa.NewHTMLWriter(ssaDumpFile, s.f, ssaDumpCFG) // TODO: generate and print a mapping from nodes to values and blocks @@ -345,9 +349,6 @@ func buildssa(fn *Node, worker int) *ssa.Func { s.f.HTMLWriter.WriteAST("AST", astBuf) } - // Allocate starting block - s.f.Entry = s.f.NewBlock(ssa.BlockPlain) - // Allocate starting values s.labels = map[string]*ssaLabel{} s.labeledNodes = map[*Node]*ssaLabel{} -- cgit v1.3-5-g9baa From 3f568625980f6e23a603cfd1cb4fcd2bf5c895d7 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Tue, 30 Jun 2020 17:51:52 -0400 Subject: cmd/go: do context propagation for tracing downloads This change does context propagation (and only context propagation) necessary to add context to modfetch.Download and pkg.LoadImport. This was done by adding context to their callers, and then adding context to all call-sites, and then repeating adding context to callers of those enclosing functions and their callers until none were left. In some cases the call graph expansion was pruned by using context.TODOs. The next CL will add a span to Download. I kept it out of this change to avoid making it any larger (and harder to review) than it needs to be. Updates #38714 Change-Id: I7a03416e04a14ca71636d96f2c1bda2c4c30d348 Reviewed-on: https://go-review.googlesource.com/c/go/+/249021 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/get/get.go | 4 +-- src/cmd/go/internal/list/list.go | 5 ++-- src/cmd/go/internal/load/pkg.go | 40 ++++++++++++++--------------- src/cmd/go/internal/load/test.go | 6 ++--- src/cmd/go/internal/modcmd/download.go | 4 +-- src/cmd/go/internal/modcmd/init.go | 2 +- src/cmd/go/internal/modcmd/tidy.go | 2 +- src/cmd/go/internal/modcmd/vendor.go | 2 +- src/cmd/go/internal/modcmd/why.go | 6 ++--- src/cmd/go/internal/modconv/convert_test.go | 5 +++- src/cmd/go/internal/modfetch/fetch.go | 3 ++- src/cmd/go/internal/modget/get.go | 18 ++++++------- src/cmd/go/internal/modload/build.go | 17 ++++++------ src/cmd/go/internal/modload/import.go | 9 ++++--- src/cmd/go/internal/modload/import_test.go | 5 +++- src/cmd/go/internal/modload/init.go | 9 ++++--- src/cmd/go/internal/modload/list.go | 14 +++++----- src/cmd/go/internal/modload/load.go | 39 ++++++++++++++-------------- src/cmd/go/internal/modload/mvs.go | 5 ++-- src/cmd/go/internal/modload/query.go | 33 ++++++++++++------------ src/cmd/go/internal/modload/query_test.go | 5 +++- src/cmd/go/internal/modload/search.go | 5 ++-- src/cmd/go/internal/run/run.go | 2 +- src/cmd/go/internal/test/test.go | 2 +- src/cmd/go/internal/work/exec.go | 2 +- 25 files changed, 131 insertions(+), 113 deletions(-) diff --git a/src/cmd/go/internal/get/get.go b/src/cmd/go/internal/get/get.go index ef43602aca..e5bacadaa3 100644 --- a/src/cmd/go/internal/get/get.go +++ b/src/cmd/go/internal/get/get.go @@ -246,9 +246,9 @@ func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) load1 := func(path string, mode int) *load.Package { if parent == nil { mode := 0 // don't do module or vendor resolution - return load.LoadImport(path, base.Cwd, nil, stk, nil, mode) + return load.LoadImport(context.TODO(), path, base.Cwd, nil, stk, nil, mode) } - return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) + return load.LoadImport(context.TODO(), path, parent.Dir, parent, stk, nil, mode|load.ResolveModule) } p := load1(arg, mode) diff --git a/src/cmd/go/internal/list/list.go b/src/cmd/go/internal/list/list.go index 7303e6c866..e68c39f392 100644 --- a/src/cmd/go/internal/list/list.go +++ b/src/cmd/go/internal/list/list.go @@ -20,6 +20,7 @@ import ( "cmd/go/internal/cache" "cmd/go/internal/cfg" "cmd/go/internal/load" + "cmd/go/internal/modinfo" "cmd/go/internal/modload" "cmd/go/internal/str" "cmd/go/internal/work" @@ -349,7 +350,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { fm := template.FuncMap{ "join": strings.Join, "context": context, - "module": modload.ModuleInfo, + "module": func(path string) *modinfo.ModulePublic { return modload.ModuleInfo(ctx, path) }, } tmpl, err := template.New("main").Funcs(fm).Parse(*listFmt) if err != nil { @@ -389,7 +390,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go list -m: not using modules") } - modload.InitMod() // Parses go.mod and sets cfg.BuildMod. + modload.InitMod(ctx) // Parses go.mod and sets cfg.BuildMod. if cfg.BuildMod == "vendor" { const actionDisabledFormat = "go list -m: can't %s using the vendor directory\n\t(Use -mod=mod or -mod=readonly to bypass.)" diff --git a/src/cmd/go/internal/load/pkg.go b/src/cmd/go/internal/load/pkg.go index 32c2ba7912..71fd9b5538 100644 --- a/src/cmd/go/internal/load/pkg.go +++ b/src/cmd/go/internal/load/pkg.go @@ -42,10 +42,10 @@ var ( ModBinDir func() string // return effective bin directory ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct - ModImportPaths func(args []string) []*search.Match // expand import paths + ModImportPaths func(ctx context.Context, args []string) []*search.Match // expand import paths ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary ModInfoProg func(info string, isgccgo bool) []byte // wrap module info in .go code for binary - ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files + ModImportFromFiles func(context.Context, []string) // update go.mod to add modules for imports in these files ModDirImportPath func(string) string // return effective import path for directory ) @@ -553,7 +553,7 @@ func ReloadPackageNoFlags(arg string, stk *ImportStack) *Package { }) packageDataCache.Delete(p.ImportPath) } - return LoadImport(arg, base.Cwd, nil, stk, nil, 0) + return LoadImport(context.TODO(), arg, base.Cwd, nil, stk, nil, 0) } // dirToImportPath returns the pseudo-import path we use for a package @@ -605,11 +605,11 @@ const ( // LoadImport does not set tool flags and should only be used by // this package, as part of a bigger load operation, and by GOPATH-based "go get". // TODO(rsc): When GOPATH-based "go get" is removed, unexport this function. -func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - return loadImport(nil, path, srcDir, parent, stk, importPos, mode) +func LoadImport(ctx context.Context, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { + return loadImport(ctx, nil, path, srcDir, parent, stk, importPos, mode) } -func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { +func loadImport(ctx context.Context, pre *preload, path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { if path == "" { panic("LoadImport called with empty package path") } @@ -657,7 +657,7 @@ func loadImport(pre *preload, path, srcDir string, parent *Package, stk *ImportS // Load package. // loadPackageData may return bp != nil even if an error occurs, // in order to return partial information. - p.load(path, stk, importPos, bp, err) + p.load(ctx, path, stk, importPos, bp, err) if !cfg.ModulesEnabled && path != cleanImport(path) { p.Error = &PackageError{ @@ -1591,7 +1591,7 @@ func (p *Package) DefaultExecName() string { // load populates p using information from bp, err, which should // be the result of calling build.Context.Import. // stk contains the import stack, not including path itself. -func (p *Package) load(path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { +func (p *Package) load(ctx context.Context, path string, stk *ImportStack, importPos []token.Position, bp *build.Package, err error) { p.copyBuild(bp) // The localPrefix is the path we interpret ./ imports relative to. @@ -1800,7 +1800,7 @@ func (p *Package) load(path string, stk *ImportStack, importPos []token.Position if path == "C" { continue } - p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport) path = p1.ImportPath importPaths[i] = path @@ -2073,7 +2073,7 @@ func PackageList(roots []*Package) []*Package { // TestPackageList returns the list of packages in the dag rooted at roots // as visited in a depth-first post-order traversal, including the test // imports of the roots. This ignores errors in test packages. -func TestPackageList(roots []*Package) []*Package { +func TestPackageList(ctx context.Context, roots []*Package) []*Package { seen := map[*Package]bool{} all := []*Package{} var walk func(*Package) @@ -2089,7 +2089,7 @@ func TestPackageList(roots []*Package) []*Package { } walkTest := func(root *Package, path string) { var stk ImportStack - p1 := LoadImport(path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) + p1 := LoadImport(ctx, path, root.Dir, root, &stk, root.Internal.Build.TestImportPos[path], ResolveImport) if p1.Error == nil { walk(p1) } @@ -2112,7 +2112,7 @@ func TestPackageList(roots []*Package) []*Package { // TODO(jayconrod): delete this function and set flags automatically // in LoadImport instead. func LoadImportWithFlags(path, srcDir string, parent *Package, stk *ImportStack, importPos []token.Position, mode int) *Package { - p := LoadImport(path, srcDir, parent, stk, importPos, mode) + p := LoadImport(context.TODO(), path, srcDir, parent, stk, importPos, mode) setToolFlags(p) return p } @@ -2153,12 +2153,12 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { // We need to test whether the path is an actual Go file and not a // package path or pattern ending in '.go' (see golang.org/issue/34653). if fi, err := os.Stat(p); err == nil && !fi.IsDir() { - return []*Package{GoFilesPackage(patterns)} + return []*Package{GoFilesPackage(ctx, patterns)} } } } - matches := ImportPaths(patterns) + matches := ImportPaths(ctx, patterns) var ( pkgs []*Package stk ImportStack @@ -2174,7 +2174,7 @@ func PackagesAndErrors(ctx context.Context, patterns []string) []*Package { if pkg == "" { panic(fmt.Sprintf("ImportPaths returned empty package for pattern %s", m.Pattern())) } - p := loadImport(pre, pkg, base.Cwd, nil, &stk, nil, 0) + p := loadImport(ctx, pre, pkg, base.Cwd, nil, &stk, nil, 0) p.Match = append(p.Match, m.Pattern()) p.Internal.CmdlinePkg = true if m.IsLiteral() { @@ -2228,9 +2228,9 @@ func setToolFlags(pkgs ...*Package) { } } -func ImportPaths(args []string) []*search.Match { +func ImportPaths(ctx context.Context, args []string) []*search.Match { if ModInit(); cfg.ModulesEnabled { - return ModImportPaths(args) + return ModImportPaths(ctx, args) } return search.ImportPaths(args) } @@ -2281,7 +2281,7 @@ func PackagesForBuild(ctx context.Context, args []string) []*Package { // GoFilesPackage creates a package for building a collection of Go files // (typically named on the command line). The target is named p.a for // package p or named after the first Go file for package main. -func GoFilesPackage(gofiles []string) *Package { +func GoFilesPackage(ctx context.Context, gofiles []string) *Package { ModInit() for _, f := range gofiles { @@ -2329,7 +2329,7 @@ func GoFilesPackage(gofiles []string) *Package { ctxt.ReadDir = func(string) ([]os.FileInfo, error) { return dirent, nil } if cfg.ModulesEnabled { - ModImportFromFiles(gofiles) + ModImportFromFiles(ctx, gofiles) } var err error @@ -2345,7 +2345,7 @@ func GoFilesPackage(gofiles []string) *Package { pkg := new(Package) pkg.Internal.Local = true pkg.Internal.CmdlineFiles = true - pkg.load("command-line-arguments", &stk, nil, bp, err) + pkg.load(ctx, "command-line-arguments", &stk, nil, bp, err) pkg.Internal.LocalPrefix = dirToImportPath(dir) pkg.ImportPath = "command-line-arguments" pkg.Target = "" diff --git a/src/cmd/go/internal/load/test.go b/src/cmd/go/internal/load/test.go index 6db8a00245..a0e275095b 100644 --- a/src/cmd/go/internal/load/test.go +++ b/src/cmd/go/internal/load/test.go @@ -108,7 +108,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p stk.Push(p.ImportPath + " (test)") rawTestImports := str.StringList(p.TestImports) for i, path := range p.TestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.TestImportPos[path], ResolveImport) if str.Contains(p1.Deps, p.ImportPath) || p1.ImportPath == p.ImportPath { // Same error that loadPackage returns (via reusePackage) in pkg.go. // Can't change that code, because that code is only for loading the @@ -127,7 +127,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p pxtestNeedsPtest := false rawXTestImports := str.StringList(p.XTestImports) for i, path := range p.XTestImports { - p1 := loadImport(pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) + p1 := loadImport(ctx, pre, path, p.Dir, p, &stk, p.Internal.Build.XTestImportPos[path], ResolveImport) if p1.ImportPath == p.ImportPath { pxtestNeedsPtest = true } else { @@ -244,7 +244,7 @@ func TestPackagesAndErrors(ctx context.Context, p *Package, cover *TestCover) (p if dep == ptest.ImportPath { pmain.Internal.Imports = append(pmain.Internal.Imports, ptest) } else { - p1 := loadImport(pre, dep, "", nil, &stk, nil, 0) + p1 := loadImport(ctx, pre, dep, "", nil, &stk, nil, 0) pmain.Internal.Imports = append(pmain.Internal.Imports, p1) } } diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 2f42d4306b..6c4ec572fc 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -90,7 +90,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { if len(args) == 0 { args = []string{"all"} } else if modload.HasModRoot() { - modload.InitMod() // to fill Target + modload.InitMod(ctx) // to fill Target targetAtLatest := modload.Target.Path + "@latest" targetAtUpgrade := modload.Target.Path + "@upgrade" targetAtPatch := modload.Target.Path + "@patch" @@ -126,7 +126,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } m.Sum = modfetch.Sum(mod) - m.Dir, err = modfetch.Download(mod) + m.Dir, err = modfetch.Download(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modcmd/init.go b/src/cmd/go/internal/modcmd/init.go index 95063e62f4..b6cffd332d 100644 --- a/src/cmd/go/internal/modcmd/init.go +++ b/src/cmd/go/internal/modcmd/init.go @@ -51,6 +51,6 @@ func runInit(ctx context.Context, cmd *base.Command, args []string) { if strings.Contains(modload.CmdModModule, "@") { base.Fatalf("go mod init: module path must not contain '@'") } - modload.InitMod() // does all the hard work + modload.InitMod(ctx) // does all the hard work modload.WriteGoMod() } diff --git a/src/cmd/go/internal/modcmd/tidy.go b/src/cmd/go/internal/modcmd/tidy.go index 769cd11fe8..c7c53d7c0c 100644 --- a/src/cmd/go/internal/modcmd/tidy.go +++ b/src/cmd/go/internal/modcmd/tidy.go @@ -40,7 +40,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go mod tidy: no arguments allowed") } - modload.LoadALL() + modload.LoadALL(ctx) modload.TidyBuildList() modload.TrimGoSum() modload.WriteGoMod() diff --git a/src/cmd/go/internal/modcmd/vendor.go b/src/cmd/go/internal/modcmd/vendor.go index 257d1cd0ef..e5353b5c7f 100644 --- a/src/cmd/go/internal/modcmd/vendor.go +++ b/src/cmd/go/internal/modcmd/vendor.go @@ -48,7 +48,7 @@ func runVendor(ctx context.Context, cmd *base.Command, args []string) { if len(args) != 0 { base.Fatalf("go mod vendor: vendor takes no arguments") } - pkgs := modload.LoadVendor() + pkgs := modload.LoadVendor(ctx) vdir := filepath.Join(modload.ModRoot(), "vendor") if err := os.RemoveAll(vdir); err != nil { diff --git a/src/cmd/go/internal/modcmd/why.go b/src/cmd/go/internal/modcmd/why.go index f400339b25..da33fff89e 100644 --- a/src/cmd/go/internal/modcmd/why.go +++ b/src/cmd/go/internal/modcmd/why.go @@ -76,7 +76,7 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { } mods := modload.ListModules(ctx, args, listU, listVersions) byModule := make(map[module.Version][]string) - for _, path := range loadALL() { + for _, path := range loadALL(ctx) { m := modload.PackageModule(path) if m.Path != "" { byModule[m] = append(byModule[m], path) @@ -105,8 +105,8 @@ func runWhy(ctx context.Context, cmd *base.Command, args []string) { sep = "\n" } } else { - matches := modload.ImportPaths(args) // resolve to packages - loadALL() // rebuild graph, from main module (not from named packages) + matches := modload.ImportPaths(ctx, args) // resolve to packages + loadALL(ctx) // rebuild graph, from main module (not from named packages) sep := "" for _, m := range matches { for _, path := range m.Pkgs { diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index a04a13b14f..faa2b4c606 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -6,6 +6,7 @@ package modconv import ( "bytes" + "context" "fmt" "internal/testenv" "io/ioutil" @@ -146,6 +147,8 @@ func TestConvertLegacyConfig(t *testing.T) { }, } + ctx := context.Background() + for _, tt := range tests { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) @@ -157,7 +160,7 @@ func TestConvertLegacyConfig(t *testing.T) { t.Fatal(err) } - dir, err := modfetch.Download(module.Version{Path: tt.path, Version: tt.vers}) + dir, err := modfetch.Download(ctx, module.Version{Path: tt.path, Version: tt.vers}) if err != nil { t.Fatal(err) } diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index e40158b535..6606612658 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -7,6 +7,7 @@ package modfetch import ( "archive/zip" "bytes" + "context" "errors" "fmt" "io" @@ -34,7 +35,7 @@ var downloadCache par.Cache // Download downloads the specific module version to the // local download cache and returns the name of the directory // corresponding to the root of the module's file tree. -func Download(mod module.Version) (dir string, err error) { +func Download(ctx context.Context, mod module.Version) (dir string, err error) { if cfg.GOMODCACHE == "" { // modload.Init exits if GOPATH[0] is empty, and cfg.GOMODCACHE // is set to GOPATH[0]/pkg/mod if GOMODCACHE is empty, so this should never happen. diff --git a/src/cmd/go/internal/modget/get.go b/src/cmd/go/internal/modget/get.go index d02c9a8da5..ee9757912b 100644 --- a/src/cmd/go/internal/modget/get.go +++ b/src/cmd/go/internal/modget/get.go @@ -353,7 +353,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if !strings.Contains(path, "...") { m := search.NewMatch(path) if pkgPath := modload.DirImportPath(path); pkgPath != "." { - m = modload.TargetPackages(pkgPath) + m = modload.TargetPackages(ctx, pkgPath) } if len(m.Pkgs) == 0 { for _, err := range m.Errs { @@ -399,7 +399,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { default: // The argument is a package or module path. if modload.HasModRoot() { - if m := modload.TargetPackages(path); len(m.Pkgs) != 0 { + if m := modload.TargetPackages(ctx, path); len(m.Pkgs) != 0 { // The path is in the main module. Nothing to query. if vers != "upgrade" && vers != "patch" { base.Errorf("go get %s: can't request explicit version of path in main module", arg) @@ -491,7 +491,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { if q.path == q.m.Path { wg.Add(1) go func(q *query) { - if hasPkg, err := modload.ModuleHasRootPackage(q.m); err != nil { + if hasPkg, err := modload.ModuleHasRootPackage(ctx, q.m); err != nil { base.Errorf("go get: %v", err) } else if !hasPkg { modOnlyMu.Lock() @@ -536,7 +536,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) { // Don't load packages if pkgPatterns is empty. Both // modload.ImportPathsQuiet and ModulePackages convert an empty list // of patterns to []string{"."}, which is not what we want. - matches = modload.ImportPathsQuiet(pkgPatterns, imports.AnyTags()) + matches = modload.ImportPathsQuiet(ctx, pkgPatterns, imports.AnyTags()) seenPkgs = make(map[string]bool) for i, match := range matches { arg := pkgGets[i] @@ -732,7 +732,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer q.m = module.Version{Path: q.path, Version: "none"} return } - m, err := getQuery(q.path, q.vers, q.prevM, q.forceModulePath) + m, err := getQuery(ctx, q.path, q.vers, q.prevM, q.forceModulePath) if err != nil { base.Errorf("go get %s: %v", q.arg, err) } @@ -790,7 +790,7 @@ func runQueries(ctx context.Context, cache map[querySpec]*query, queries []*quer // to determine the underlying module version being requested. // If forceModulePath is set, getQuery must interpret path // as a module path. -func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { +func getQuery(ctx context.Context, path, vers string, prevM module.Version, forceModulePath bool) (module.Version, error) { if (prevM.Version != "") != forceModulePath { // We resolve package patterns by calling QueryPattern, which does not // accept a previous version and therefore cannot take it into account for @@ -812,7 +812,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo } } - info, err := modload.Query(path, vers, prevM.Version, modload.Allowed) + info, err := modload.Query(ctx, path, vers, prevM.Version, modload.Allowed) if err == nil { if info.Version != vers && info.Version != prevM.Version { logOncef("go: %s %s => %s", path, vers, info.Version) @@ -838,7 +838,7 @@ func getQuery(path, vers string, prevM module.Version, forceModulePath bool) (mo // If it turns out to only exist as a module, we can detect the resulting // PackageNotInModuleError and avoid a second round-trip through (potentially) // all of the configured proxies. - results, err := modload.QueryPattern(path, vers, modload.Allowed) + results, err := modload.QueryPattern(ctx, path, vers, modload.Allowed) if err != nil { // If the path doesn't contain a wildcard, check whether it was actually a // module path instead. If so, return that. @@ -994,7 +994,7 @@ func (u *upgrader) Upgrade(m module.Version) (module.Version, error) { // If we're querying "upgrade" or "patch", Query will compare the current // version against the chosen version and will return the current version // if it is newer. - info, err := modload.Query(m.Path, string(getU), m.Version, modload.Allowed) + info, err := modload.Query(context.TODO(), m.Path, string(getU), m.Version, modload.Allowed) if err != nil { // Report error but return m, to let version selection continue. // (Reporting the error will fail the command at the next base.ExitIfErrors.) diff --git a/src/cmd/go/internal/modload/build.go b/src/cmd/go/internal/modload/build.go index 5f8a2e7e05..a101681a1f 100644 --- a/src/cmd/go/internal/modload/build.go +++ b/src/cmd/go/internal/modload/build.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/hex" "fmt" "internal/goroot" @@ -57,21 +58,21 @@ func PackageModuleInfo(pkgpath string) *modinfo.ModulePublic { if !ok { return nil } - return moduleInfo(m, true) + return moduleInfo(context.TODO(), m, true) } -func ModuleInfo(path string) *modinfo.ModulePublic { +func ModuleInfo(ctx context.Context, path string) *modinfo.ModulePublic { if !Enabled() { return nil } if i := strings.Index(path, "@"); i >= 0 { - return moduleInfo(module.Version{Path: path[:i], Version: path[i+1:]}, false) + return moduleInfo(ctx, module.Version{Path: path[:i], Version: path[i+1:]}, false) } for _, m := range BuildList() { if m.Path == path { - return moduleInfo(m, true) + return moduleInfo(ctx, m, true) } } @@ -84,12 +85,12 @@ func ModuleInfo(path string) *modinfo.ModulePublic { } // addUpdate fills in m.Update if an updated version is available. -func addUpdate(m *modinfo.ModulePublic) { +func addUpdate(ctx context.Context, m *modinfo.ModulePublic) { if m.Version == "" { return } - if info, err := Query(m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { + if info, err := Query(ctx, m.Path, "upgrade", m.Version, Allowed); err == nil && semver.Compare(info.Version, m.Version) > 0 { m.Update = &modinfo.ModulePublic{ Path: m.Path, Version: info.Version, @@ -103,7 +104,7 @@ func addVersions(m *modinfo.ModulePublic) { m.Versions, _ = versions(m.Path) } -func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { +func moduleInfo(ctx context.Context, m module.Version, fromBuildList bool) *modinfo.ModulePublic { if m == Target { info := &modinfo.ModulePublic{ Path: m.Path, @@ -132,7 +133,7 @@ func moduleInfo(m module.Version, fromBuildList bool) *modinfo.ModulePublic { // completeFromModCache fills in the extra fields in m using the module cache. completeFromModCache := func(m *modinfo.ModulePublic) { if m.Version != "" { - if q, err := Query(m.Path, m.Version, "", nil); err != nil { + if q, err := Query(ctx, m.Path, m.Version, "", nil); err != nil { m.Error = &modinfo.ModuleError{Err: err.Error()} } else { m.Version = q.Version diff --git a/src/cmd/go/internal/modload/import.go b/src/cmd/go/internal/modload/import.go index 4d2bc805e2..5c51a79124 100644 --- a/src/cmd/go/internal/modload/import.go +++ b/src/cmd/go/internal/modload/import.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "go/build" @@ -110,7 +111,7 @@ var _ load.ImportPathError = &AmbiguousImportError{} // Import returns an ImportMissingError as the error. // If Import can identify a module that could be added to supply the package, // the ImportMissingError records that module. -func Import(path string) (m module.Version, dir string, err error) { +func Import(ctx context.Context, path string) (m module.Version, dir string, err error) { if strings.Contains(path, "@") { return module.Version{}, "", fmt.Errorf("import path should not have @version") } @@ -165,7 +166,7 @@ func Import(path string) (m module.Version, dir string, err error) { // Avoid possibly downloading irrelevant modules. continue } - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error. // Note that we don't know for sure this module is necessary, @@ -248,7 +249,7 @@ func Import(path string) (m module.Version, dir string, err error) { return len(mods[i].Path) > len(mods[j].Path) }) for _, m := range mods { - root, isLocal, err := fetch(m) + root, isLocal, err := fetch(ctx, m) if err != nil { // Report fetch error as above. return module.Version{}, "", err @@ -285,7 +286,7 @@ func Import(path string) (m module.Version, dir string, err error) { fmt.Fprintf(os.Stderr, "go: finding module for package %s\n", path) - candidates, err := QueryPackage(path, "latest", Allowed) + candidates, err := QueryPackage(ctx, path, "latest", Allowed) if err != nil { if errors.Is(err, os.ErrNotExist) { // Return "cannot find module providing package […]" instead of whatever diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index accc60eecd..47ce89a084 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "regexp" "strings" @@ -49,10 +50,12 @@ func TestImport(t *testing.T) { }(allowMissingModuleImports) AllowMissingModuleImports() + ctx := context.Background() + for _, tt := range importTests { t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. - m, dir, err := Import(tt.path) + m, dir, err := Import(ctx, tt.path) if err == nil { t.Fatalf("Import(%q) = %v, %v, nil; expected error", tt.path, m, dir) } diff --git a/src/cmd/go/internal/modload/init.go b/src/cmd/go/internal/modload/init.go index fff060e665..93027c44c4 100644 --- a/src/cmd/go/internal/modload/init.go +++ b/src/cmd/go/internal/modload/init.go @@ -6,6 +6,7 @@ package modload import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -332,7 +333,7 @@ func die() { // // As a side-effect, InitMod sets a default for cfg.BuildMod if it does not // already have an explicit value. -func InitMod() { +func InitMod(ctx context.Context) { if len(buildList) > 0 { return } @@ -359,7 +360,7 @@ func InitMod() { } var fixed bool - f, err := modfile.Parse(gomod, data, fixVersion(&fixed)) + f, err := modfile.Parse(gomod, data, fixVersion(ctx, &fixed)) if err != nil { // Errors returned by modfile.Parse begin with file:line. base.Fatalf("go: errors parsing go.mod:\n%s\n", err) @@ -397,7 +398,7 @@ func InitMod() { // and does nothing for versions that already appear to be canonical. // // The VersionFixer sets 'fixed' if it ever returns a non-canonical version. -func fixVersion(fixed *bool) modfile.VersionFixer { +func fixVersion(ctx context.Context, fixed *bool) modfile.VersionFixer { return func(path, vers string) (resolved string, err error) { defer func() { if err == nil && resolved != vers { @@ -429,7 +430,7 @@ func fixVersion(fixed *bool) modfile.VersionFixer { } } - info, err := Query(path, vers, "", nil) + info, err := Query(ctx, path, vers, "", nil) if err != nil { return "", err } diff --git a/src/cmd/go/internal/modload/list.go b/src/cmd/go/internal/modload/list.go index 8db4d64706..7bf4e86c8d 100644 --- a/src/cmd/go/internal/modload/list.go +++ b/src/cmd/go/internal/modload/list.go @@ -31,7 +31,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ sem <- token{} go func() { if listU { - addUpdate(m) + addUpdate(ctx, m) } if listVersions { addVersions(m) @@ -57,7 +57,7 @@ func ListModules(ctx context.Context, args []string, listU, listVersions bool) [ func listModules(ctx context.Context, args []string, listVersions bool) []*modinfo.ModulePublic { LoadBuildList(ctx) if len(args) == 0 { - return []*modinfo.ModulePublic{moduleInfo(buildList[0], true)} + return []*modinfo.ModulePublic{moduleInfo(ctx, buildList[0], true)} } var mods []*modinfo.ModulePublic @@ -83,7 +83,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin } } - info, err := Query(path, vers, current, nil) + info, err := Query(ctx, path, vers, current, nil) if err != nil { mods = append(mods, &modinfo.ModulePublic{ Path: path, @@ -92,7 +92,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin }) continue } - mods = append(mods, moduleInfo(module.Version{Path: path, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: path, Version: info.Version}, false)) continue } @@ -117,7 +117,7 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin matched = true if !matchedBuildList[i] { matchedBuildList[i] = true - mods = append(mods, moduleInfo(m, true)) + mods = append(mods, moduleInfo(ctx, m, true)) } } } @@ -127,9 +127,9 @@ func listModules(ctx context.Context, args []string, listVersions bool) []*modin // Don't make the user provide an explicit '@latest' when they're // explicitly asking what the available versions are. // Instead, resolve the module, even if it isn't an existing dependency. - info, err := Query(arg, "latest", "", nil) + info, err := Query(ctx, arg, "latest", "", nil) if err == nil { - mods = append(mods, moduleInfo(module.Version{Path: arg, Version: info.Version}, false)) + mods = append(mods, moduleInfo(ctx, module.Version{Path: arg, Version: info.Version}, false)) } else { mods = append(mods, &modinfo.ModulePublic{ Path: arg, diff --git a/src/cmd/go/internal/modload/load.go b/src/cmd/go/internal/modload/load.go index 8190009b23..686d491219 100644 --- a/src/cmd/go/internal/modload/load.go +++ b/src/cmd/go/internal/modload/load.go @@ -52,8 +52,8 @@ var loaded *loader // ImportPaths returns the set of packages matching the args (patterns), // on the target platform. Modules may be added to the build list // to satisfy new imports. -func ImportPaths(patterns []string) []*search.Match { - matches := ImportPathsQuiet(patterns, imports.Tags()) +func ImportPaths(ctx context.Context, patterns []string) []*search.Match { + matches := ImportPathsQuiet(ctx, patterns, imports.Tags()) search.WarnUnmatched(matches) return matches } @@ -62,7 +62,7 @@ func ImportPaths(patterns []string) []*search.Match { // no matches. It also lets the caller specify a set of build tags to match // packages. The build tags should typically be imports.Tags() or // imports.AnyTags(); a nil map has no special meaning. -func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { +func ImportPathsQuiet(ctx context.Context, patterns []string, tags map[string]bool) []*search.Match { updateMatches := func(matches []*search.Match, iterating bool) { for _, m := range matches { switch { @@ -103,7 +103,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { case strings.Contains(m.Pattern(), "..."): m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, includeStd, buildList) + matchPackages(ctx, m, loaded.tags, includeStd, buildList) case m.Pattern() == "all": loaded.testAll = true @@ -111,7 +111,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { // Enumerate the packages in the main module. // We'll load the dependencies as we find them. m.Errs = m.Errs[:0] - matchPackages(m, loaded.tags, omitStd, []module.Version{Target}) + matchPackages(ctx, m, loaded.tags, omitStd, []module.Version{Target}) } else { // Starting with the packages in the main module, // enumerate the full list of "all". @@ -129,7 +129,7 @@ func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match { } } - InitMod() + InitMod(ctx) var matches []*search.Match for _, pattern := range search.CleanPatterns(patterns) { @@ -338,8 +338,8 @@ func pathInModuleCache(dir string) string { // ImportFromFiles adds modules to the build list as needed // to satisfy the imports in the named Go source files. -func ImportFromFiles(gofiles []string) { - InitMod() +func ImportFromFiles(ctx context.Context, gofiles []string) { + InitMod(ctx) tags := imports.Tags() imports, testImports, err := imports.ScanFiles(gofiles, tags) @@ -391,7 +391,7 @@ func DirImportPath(dir string) string { func LoadBuildList(ctx context.Context) []module.Version { ctx, span := trace.StartSpan(ctx, "LoadBuildList") defer span.Done() - InitMod() + InitMod(ctx) ReloadBuildList() WriteGoMod() return buildList @@ -409,20 +409,20 @@ func ReloadBuildList() []module.Version { // It adds modules to the build list as needed to satisfy new imports. // This set is useful for deciding whether a particular import is needed // anywhere in a module. -func LoadALL() []string { - return loadAll(true) +func LoadALL(ctx context.Context) []string { + return loadAll(ctx, true) } // LoadVendor is like LoadALL but only follows test dependencies // for tests in the main module. Tests in dependency modules are // ignored completely. // This set is useful for identifying the which packages to include in a vendor directory. -func LoadVendor() []string { - return loadAll(false) +func LoadVendor(ctx context.Context) []string { + return loadAll(ctx, false) } -func loadAll(testAll bool) []string { - InitMod() +func loadAll(ctx context.Context, testAll bool) []string { + InitMod(ctx) loaded = newLoader(imports.AnyTags()) loaded.isALL = true @@ -430,7 +430,7 @@ func loadAll(testAll bool) []string { if !testAll { loaded.testRoots = true } - all := TargetPackages("...") + all := TargetPackages(ctx, "...") loaded.load(func() []string { return all.Pkgs }) checkMultiplePaths() WriteGoMod() @@ -453,13 +453,13 @@ func loadAll(testAll bool) []string { // TargetPackages returns the list of packages in the target (top-level) module // matching pattern, which may be relative to the working directory, under all // build tag settings. -func TargetPackages(pattern string) *search.Match { +func TargetPackages(ctx context.Context, pattern string) *search.Match { // TargetPackages is relative to the main module, so ensure that the main // module is a thing that can contain packages. ModRoot() m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{Target}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{Target}) return m } @@ -817,7 +817,8 @@ func (ld *loader) doPkg(item interface{}) { return } - pkg.mod, pkg.dir, pkg.err = Import(pkg.path) + // TODO(matloob): Handle TODO context. This needs to be threaded through Do. + pkg.mod, pkg.dir, pkg.err = Import(context.TODO(), pkg.path) if pkg.dir == "" { return } diff --git a/src/cmd/go/internal/modload/mvs.go b/src/cmd/go/internal/modload/mvs.go index 5dd009d31d..67eb2c2e19 100644 --- a/src/cmd/go/internal/modload/mvs.go +++ b/src/cmd/go/internal/modload/mvs.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -224,7 +225,7 @@ func (*mvsReqs) next(m module.Version) (module.Version, error) { // // The isLocal return value reports whether the replacement, // if any, is local to the filesystem. -func fetch(mod module.Version) (dir string, isLocal bool, err error) { +func fetch(ctx context.Context, mod module.Version) (dir string, isLocal bool, err error) { if mod == Target { return ModRoot(), true, nil } @@ -254,6 +255,6 @@ func fetch(mod module.Version) (dir string, isLocal bool, err error) { mod = r } - dir, err = modfetch.Download(mod) + dir, err = modfetch.Download(ctx, mod) return dir, false, err } diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index acc886bf21..39a813447c 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -5,6 +5,7 @@ package modload import ( + "context" "errors" "fmt" "os" @@ -55,10 +56,10 @@ import ( // // If path is the path of the main module and the query is "latest", // Query returns Target.Version as the version. -func Query(path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func Query(ctx context.Context, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { var info *modfetch.RevInfo err := modfetch.TryProxies(func(proxy string) (err error) { - info, err = queryProxy(proxy, path, query, current, allowed) + info, err = queryProxy(ctx, proxy, path, query, current, allowed) return err }) return info, err @@ -75,7 +76,7 @@ func (queryDisabledError) Error() string { return fmt.Sprintf("cannot query module due to -mod=%s\n\t(%s)", cfg.BuildMod, cfg.BuildModReason) } -func queryProxy(proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { +func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -243,7 +244,7 @@ func queryProxy(proxy, path, query, current string, allowed func(module.Version) if err != nil { return nil, err } - releases, prereleases, err := filterVersions(path, versions, ok, preferIncompatible) + releases, prereleases, err := filterVersions(ctx, path, versions, ok, preferIncompatible) if err != nil { return nil, err } @@ -327,7 +328,7 @@ func matchSemverPrefix(p, v string) bool { // 1. versions that do not satisfy the 'ok' predicate, and // 2. "+incompatible" versions, if a compatible one satisfies the predicate // and the incompatible version is not preferred. -func filterVersions(path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { +func filterVersions(ctx context.Context, path string, versions []string, ok func(module.Version) bool, preferIncompatible bool) (releases, prereleases []string, err error) { var lastCompatible string for _, v := range versions { if !ok(module.Version{Path: path, Version: v}) { @@ -343,7 +344,7 @@ func filterVersions(path string, versions []string, ok func(module.Version) bool // https://golang.org/issue/34165.) Note that we even prefer a // compatible pre-release over an incompatible release. - ok, err := versionHasGoMod(module.Version{Path: path, Version: lastCompatible}) + ok, err := versionHasGoMod(ctx, module.Version{Path: path, Version: lastCompatible}) if err != nil { return nil, nil, err } @@ -380,12 +381,12 @@ type QueryResult struct { // If the package is in the main module, QueryPackage considers only the main // module and only the version "latest", without checking for other possible // modules. -func QueryPackage(path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPackage(ctx context.Context, path, query string, allowed func(module.Version) bool) ([]QueryResult, error) { m := search.NewMatch(path) if m.IsLocal() || !m.IsLiteral() { return nil, fmt.Errorf("pattern %s is not an importable package", path) } - return QueryPattern(path, query, allowed) + return QueryPattern(ctx, path, query, allowed) } // QueryPattern looks up the module(s) containing at least one package matching @@ -401,7 +402,7 @@ func QueryPackage(path, query string, allowed func(module.Version) bool) ([]Quer // If any matching package is in the main module, QueryPattern considers only // the main module and only the version "latest", without checking for other // possible modules. -func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { +func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { base := pattern firstError := func(m *search.Match) error { @@ -417,7 +418,7 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q base = pathpkg.Dir(pattern[:i+3]) match = func(mod module.Version, root string, isLocal bool) *search.Match { m := search.NewMatch(pattern) - matchPackages(m, imports.AnyTags(), omitStd, []module.Version{mod}) + matchPackages(ctx, m, imports.AnyTags(), omitStd, []module.Version{mod}) return m } } else { @@ -472,12 +473,12 @@ func QueryPattern(pattern, query string, allowed func(module.Version) bool) ([]Q queryModule := func(path string) (r QueryResult, err error) { current := findCurrentVersion(path) r.Mod.Path = path - r.Rev, err = queryProxy(proxy, path, query, current, allowed) + r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) if err != nil { return r, err } r.Mod.Version = r.Rev.Version - root, isLocal, err := fetch(r.Mod) + root, isLocal, err := fetch(ctx, r.Mod) if err != nil { return r, err } @@ -698,8 +699,8 @@ func (e *PackageNotInModuleError) ImportPath() string { } // ModuleHasRootPackage returns whether module m contains a package m.Path. -func ModuleHasRootPackage(m module.Version) (bool, error) { - root, isLocal, err := fetch(m) +func ModuleHasRootPackage(ctx context.Context, m module.Version) (bool, error) { + root, isLocal, err := fetch(ctx, m) if err != nil { return false, err } @@ -707,8 +708,8 @@ func ModuleHasRootPackage(m module.Version) (bool, error) { return ok, err } -func versionHasGoMod(m module.Version) (bool, error) { - root, _, err := fetch(m) +func versionHasGoMod(ctx context.Context, m module.Version) (bool, error) { + root, _, err := fetch(ctx, m) if err != nil { return false, err } diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 247e4c40d2..77080e9b5b 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -5,6 +5,7 @@ package modload import ( + "context" "internal/testenv" "io/ioutil" "log" @@ -179,6 +180,8 @@ func TestQuery(t *testing.T) { testenv.MustHaveExternalNetwork(t) testenv.MustHaveExecPath(t, "git") + ctx := context.Background() + for _, tt := range queryTests { allow := tt.allow if allow == "" { @@ -192,7 +195,7 @@ func TestQuery(t *testing.T) { t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+tt.current+"/"+allow, func(t *testing.T) { t.Parallel() - info, err := Query(tt.path, tt.query, tt.current, allowed) + info, err := Query(ctx, tt.path, tt.query, tt.current, allowed) if tt.err != "" { if err == nil { t.Errorf("Query(%q, %q, %v) = %v, want error %q", tt.path, tt.query, allow, info.Version, tt.err) diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go index c28e7c0c1e..a9bee0af4e 100644 --- a/src/cmd/go/internal/modload/search.go +++ b/src/cmd/go/internal/modload/search.go @@ -5,6 +5,7 @@ package modload import ( + "context" "fmt" "os" "path/filepath" @@ -27,7 +28,7 @@ const ( // matchPackages is like m.MatchPackages, but uses a local variable (rather than // a global) for tags, can include or exclude packages in the standard library, // and is restricted to the given list of modules. -func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { +func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) { m.Pkgs = []string{} isMatch := func(string) bool { return true } @@ -153,7 +154,7 @@ func matchPackages(m *search.Match, tags map[string]bool, filter stdFilter, modu isLocal = true } else { var err error - root, isLocal, err = fetch(mod) + root, isLocal, err = fetch(ctx, mod) if err != nil { m.AddError(err) continue diff --git a/src/cmd/go/internal/run/run.go b/src/cmd/go/internal/run/run.go index deec5106ff..99578b244c 100644 --- a/src/cmd/go/internal/run/run.go +++ b/src/cmd/go/internal/run/run.go @@ -77,7 +77,7 @@ func runRun(ctx context.Context, cmd *base.Command, args []string) { base.Fatalf("go run: cannot run *_test.go files (%s)", file) } } - p = load.GoFilesPackage(files) + p = load.GoFilesPackage(ctx, files) } else if len(args) > 0 && !strings.HasPrefix(args[0], "-") { pkgs := load.PackagesAndErrors(ctx, args[:1]) if len(pkgs) == 0 { diff --git a/src/cmd/go/internal/test/test.go b/src/cmd/go/internal/test/test.go index 9cef8cf89c..3aee6939d2 100644 --- a/src/cmd/go/internal/test/test.go +++ b/src/cmd/go/internal/test/test.go @@ -702,7 +702,7 @@ func runTest(ctx context.Context, cmd *base.Command, args []string) { } // Select for coverage all dependencies matching the testCoverPaths patterns. - for _, p := range load.TestPackageList(pkgs) { + for _, p := range load.TestPackageList(ctx, pkgs) { haveMatch := false for i := range testCoverPaths { if match[i](p) { diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 681ecd7646..d975c36306 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -2900,7 +2900,7 @@ func (b *Builder) swigDoIntSize(objdir string) (intsize string, err error) { } srcs := []string{src} - p := load.GoFilesPackage(srcs) + p := load.GoFilesPackage(context.TODO(), srcs) if _, _, e := BuildToolchain.gc(b, &Action{Mode: "swigDoIntSize", Package: p, Objdir: objdir}, "", nil, "", false, srcs); e != nil { return "32", nil -- cgit v1.3-5-g9baa From 822dca4b383611335385ef05c2882d23f6f89d88 Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Wed, 15 Jul 2020 17:38:36 -0400 Subject: cmd/go: add tracing for querying and downloading from the proxy This CL adds tracing spans for modload.queryPattern, modload.queryProxy, modload.QueryPattern, modload.QueryPattern.queryModule, modload.queryPrefixModules and modfetch.Download. Updates #38714 Change-Id: I91af3f022a6e18ab8d9c1d9b0ccf4928b6c236bd Reviewed-on: https://go-review.googlesource.com/c/go/+/249022 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modcmd/download.go | 2 +- src/cmd/go/internal/modfetch/fetch.go | 21 +++++++++++++++------ .../internal/modfetch/zip_sum_test/zip_sum_test.go | 3 ++- src/cmd/go/internal/modload/query.go | 22 ++++++++++++++++++---- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/cmd/go/internal/modcmd/download.go b/src/cmd/go/internal/modcmd/download.go index 6c4ec572fc..d4c161fca1 100644 --- a/src/cmd/go/internal/modcmd/download.go +++ b/src/cmd/go/internal/modcmd/download.go @@ -120,7 +120,7 @@ func runDownload(ctx context.Context, cmd *base.Command, args []string) { return } mod := module.Version{Path: m.Path, Version: m.Version} - m.Zip, err = modfetch.DownloadZip(mod) + m.Zip, err = modfetch.DownloadZip(ctx, mod) if err != nil { m.Error = err.Error() return diff --git a/src/cmd/go/internal/modfetch/fetch.go b/src/cmd/go/internal/modfetch/fetch.go index 6606612658..e29eb0a942 100644 --- a/src/cmd/go/internal/modfetch/fetch.go +++ b/src/cmd/go/internal/modfetch/fetch.go @@ -24,6 +24,7 @@ import ( "cmd/go/internal/par" "cmd/go/internal/renameio" "cmd/go/internal/robustio" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/sumdb/dirhash" @@ -48,7 +49,7 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { err error } c := downloadCache.Do(mod, func() interface{} { - dir, err := download(mod) + dir, err := download(ctx, mod) if err != nil { return cached{"", err} } @@ -58,7 +59,10 @@ func Download(ctx context.Context, mod module.Version) (dir string, err error) { return c.dir, c.err } -func download(mod module.Version) (dir string, err error) { +func download(ctx context.Context, mod module.Version) (dir string, err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.download "+mod.String()) + defer span.Done() + // If the directory exists, and no .partial file exists, the module has // already been completely extracted. .partial files may be created when a // module zip directory is extracted in place instead of being extracted to a @@ -73,7 +77,7 @@ func download(mod module.Version) (dir string, err error) { // To avoid cluttering the cache with extraneous files, // DownloadZip uses the same lockfile as Download. // Invoke DownloadZip before locking the file. - zipfile, err := DownloadZip(mod) + zipfile, err := DownloadZip(ctx, mod) if err != nil { return "", err } @@ -143,6 +147,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } + ctx, span = trace.StartSpan(ctx, "unzip "+zipfile) if unzipInPlace { if err := ioutil.WriteFile(partialPath, nil, 0666); err != nil { return "", err @@ -172,6 +177,7 @@ func download(mod module.Version) (dir string, err error) { return "", err } } + defer span.Done() if !cfg.ModCacheRW { // Make dir read-only only *after* renaming it. @@ -196,7 +202,7 @@ var downloadZipCache par.Cache // DownloadZip downloads the specific module version to the // local zip cache and returns the name of the zip file. -func DownloadZip(mod module.Version) (zipfile string, err error) { +func DownloadZip(ctx context.Context, mod module.Version) (zipfile string, err error) { // The par.Cache here avoids duplicate work. type cached struct { zipfile string @@ -231,7 +237,7 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil { return cached{"", err} } - if err := downloadZip(mod, zipfile); err != nil { + if err := downloadZip(ctx, mod, zipfile); err != nil { return cached{"", err} } return cached{zipfile, nil} @@ -239,7 +245,10 @@ func DownloadZip(mod module.Version) (zipfile string, err error) { return c.zipfile, c.err } -func downloadZip(mod module.Version, zipfile string) (err error) { +func downloadZip(ctx context.Context, mod module.Version, zipfile string) (err error) { + ctx, span := trace.StartSpan(ctx, "modfetch.downloadZip "+zipfile) + defer span.Done() + // Clean up any remaining tempfiles from previous runs. // This is only safe to do because the lock file ensures that their // writers are no longer active. diff --git a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go index eac9b32fa8..82398ebfed 100644 --- a/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go +++ b/src/cmd/go/internal/modfetch/zip_sum_test/zip_sum_test.go @@ -16,6 +16,7 @@ package zip_sum_test import ( + "context" "crypto/sha256" "encoding/csv" "encoding/hex" @@ -119,7 +120,7 @@ func TestZipSums(t *testing.T) { name := fmt.Sprintf("%s@%s", strings.ReplaceAll(test.m.Path, "/", "_"), test.m.Version) t.Run(name, func(t *testing.T) { t.Parallel() - zipPath, err := modfetch.DownloadZip(test.m) + zipPath, err := modfetch.DownloadZip(context.Background(), test.m) if err != nil { if *updateTestData { t.Logf("%s: could not download module: %s (will remove from testdata)", test.m, err) diff --git a/src/cmd/go/internal/modload/query.go b/src/cmd/go/internal/modload/query.go index 39a813447c..e82eb1506f 100644 --- a/src/cmd/go/internal/modload/query.go +++ b/src/cmd/go/internal/modload/query.go @@ -19,6 +19,7 @@ import ( "cmd/go/internal/modfetch" "cmd/go/internal/search" "cmd/go/internal/str" + "cmd/go/internal/trace" "golang.org/x/mod/module" "golang.org/x/mod/semver" @@ -77,6 +78,9 @@ func (queryDisabledError) Error() string { } func queryProxy(ctx context.Context, proxy, path, query, current string, allowed func(module.Version) bool) (*modfetch.RevInfo, error) { + ctx, span := trace.StartSpan(ctx, "modload.queryProxy "+path+" "+query) + defer span.Done() + if current != "" && !semver.IsValid(current) { return nil, fmt.Errorf("invalid previous version %q", current) } @@ -403,6 +407,9 @@ func QueryPackage(ctx context.Context, path, query string, allowed func(module.V // the main module and only the version "latest", without checking for other // possible modules. func QueryPattern(ctx context.Context, pattern, query string, allowed func(module.Version) bool) ([]QueryResult, error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern "+pattern+" "+query) + defer span.Done() + base := pattern firstError := func(m *search.Match) error { @@ -470,7 +477,10 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } err := modfetch.TryProxies(func(proxy string) error { - queryModule := func(path string) (r QueryResult, err error) { + queryModule := func(ctx context.Context, path string) (r QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.QueryPattern.queryModule ["+proxy+"] "+path) + defer span.Done() + current := findCurrentVersion(path) r.Mod.Path = path r.Rev, err = queryProxy(ctx, proxy, path, query, current, allowed) @@ -499,7 +509,7 @@ func QueryPattern(ctx context.Context, pattern, query string, allowed func(modul } var err error - results, err = queryPrefixModules(candidateModules, queryModule) + results, err = queryPrefixModules(ctx, candidateModules, queryModule) return err }) @@ -543,7 +553,10 @@ type prefixResult struct { err error } -func queryPrefixModules(candidateModules []string, queryModule func(path string) (QueryResult, error)) (found []QueryResult, err error) { +func queryPrefixModules(ctx context.Context, candidateModules []string, queryModule func(ctx context.Context, path string) (QueryResult, error)) (found []QueryResult, err error) { + ctx, span := trace.StartSpan(ctx, "modload.queryPrefixModules") + defer span.Done() + // If the path we're attempting is not in the module cache and we don't have a // fetch result cached either, we'll end up making a (potentially slow) // request to the proxy or (often even slower) the origin server. @@ -556,8 +569,9 @@ func queryPrefixModules(candidateModules []string, queryModule func(path string) var wg sync.WaitGroup wg.Add(len(candidateModules)) for i, p := range candidateModules { + ctx := trace.StartGoroutine(ctx) go func(p string, r *result) { - r.QueryResult, r.err = queryModule(p) + r.QueryResult, r.err = queryModule(ctx, p) wg.Done() }(p, &results[i]) } -- cgit v1.3-5-g9baa From 268dd2e5a7a8919bd26f0a59c847f8268a2437d1 Mon Sep 17 00:00:00 2001 From: Michael Munday Date: Mon, 10 Aug 2020 08:01:21 -0700 Subject: cmd/internal/obj: fix inline marker issue on s390x The optimization that replaces inline markers with pre-existing instructions assumes that 'Prog' values produced by the compiler are still reachable after the assembler has run. This was not true on s390x where the assembler was removing NOP instructions from the linked list of 'Prog' values. This led to broken inlining data which in turn caused an infinite loop in the runtime traceback code. Fix this by stopping the s390x assembler backend removing NOP values. It does not make any difference to the output of the assembler because NOP instructions are 0 bytes long anyway. Fixes #40473. Change-Id: I9b97c494afaae2d5ed6bca4cd428b4132b5f8133 Reviewed-on: https://go-review.googlesource.com/c/go/+/249448 Run-TryBot: Michael Munday TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/cmd/internal/obj/pcln.go | 15 +++++++++++++++ src/cmd/internal/obj/s390x/objz.go | 11 ----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/cmd/internal/obj/pcln.go b/src/cmd/internal/obj/pcln.go index 1f7ccf47ef..bffeda041d 100644 --- a/src/cmd/internal/obj/pcln.go +++ b/src/cmd/internal/obj/pcln.go @@ -278,6 +278,21 @@ func linkpcln(ctxt *Link, cursym *LSym) { funcpctab(ctxt, &pcln.Pcfile, cursym, "pctofile", pctofileline, pcln) funcpctab(ctxt, &pcln.Pcline, cursym, "pctoline", pctofileline, nil) + // Check that all the Progs used as inline markers are still reachable. + // See issue #40473. + inlMarkProgs := make(map[*Prog]struct{}, len(cursym.Func.InlMarks)) + for _, inlMark := range cursym.Func.InlMarks { + inlMarkProgs[inlMark.p] = struct{}{} + } + for p := cursym.Func.Text; p != nil; p = p.Link { + if _, ok := inlMarkProgs[p]; ok { + delete(inlMarkProgs, p) + } + } + if len(inlMarkProgs) > 0 { + ctxt.Diag("one or more instructions used as inline markers are no longer reachable") + } + pcinlineState := new(pcinlineState) funcpctab(ctxt, &pcln.Pcinline, cursym, "pctoinline", pcinlineState.pctoinline, nil) for _, inlMark := range cursym.Func.InlMarks { diff --git a/src/cmd/internal/obj/s390x/objz.go b/src/cmd/internal/obj/s390x/objz.go index b14dc810fa..ef6335d849 100644 --- a/src/cmd/internal/obj/s390x/objz.go +++ b/src/cmd/internal/obj/s390x/objz.go @@ -283,17 +283,6 @@ func preprocess(ctxt *obj.Link, cursym *obj.LSym, newprog obj.ProgAlloc) { ACMPUBNE: q = p p.Mark |= BRANCH - if p.Pcond != nil { - q := p.Pcond - for q.As == obj.ANOP { - q = q.Link - p.Pcond = q - } - } - - case obj.ANOP: - q.Link = p.Link /* q is non-nop */ - p.Link.Mark |= p.Mark default: q = p -- cgit v1.3-5-g9baa From e94544cf012535da6b3c9e735bc4026e2db1c99c Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Wed, 19 Aug 2020 21:39:12 -0700 Subject: cmd/compile: fix checkptr handling of &^ checkptr has code to recognize &^ expressions, but it didn't take into account that "p &^ x" gets rewritten to "p & ^x" during walk, which resulted in false positive diagnostics. This CL changes walkexpr to mark OANDNOT expressions with Implicit when they're rewritten to OAND, so that walkCheckPtrArithmetic can still recognize them later. It would be slightly more idiomatic to instead mark the OBITNOT expression as Implicit (as it's a compiler-generated Node), but the OBITNOT expression might get constant folded. It's not worth the extra complexity/subtlety of relying on n.Right.Orig, so we set Implicit on the OAND node instead. To atone for this transgression, I add documentation for nodeImplicit. Fixes #40917. Change-Id: I386304171ad299c530e151e5924f179e9a5fd5b8 Reviewed-on: https://go-review.googlesource.com/c/go/+/249477 Run-TryBot: Matthew Dempsky TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall Reviewed-by: Cuong Manh Le --- src/cmd/compile/internal/gc/syntax.go | 4 ++-- src/cmd/compile/internal/gc/walk.go | 7 ++++++- src/runtime/checkptr_test.go | 1 + src/runtime/testdata/testprog/checkptr.go | 8 ++++++++ test/fixedbugs/issue40917.go | 23 +++++++++++++++++++++++ 5 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 test/fixedbugs/issue40917.go diff --git a/src/cmd/compile/internal/gc/syntax.go b/src/cmd/compile/internal/gc/syntax.go index b658410c53..47e5e59156 100644 --- a/src/cmd/compile/internal/gc/syntax.go +++ b/src/cmd/compile/internal/gc/syntax.go @@ -141,8 +141,8 @@ const ( nodeInitorder, _ // tracks state during init1; two bits _, _ // second nodeInitorder bit _, nodeHasBreak - _, nodeNoInline // used internally by inliner to indicate that a function call should not be inlined; set for OCALLFUNC and OCALLMETH only - _, nodeImplicit + _, nodeNoInline // used internally by inliner to indicate that a function call should not be inlined; set for OCALLFUNC and OCALLMETH only + _, nodeImplicit // implicit OADDR or ODEREF; ++/-- statement represented as OASOP; or ANDNOT lowered to OAND _, nodeIsDDD // is the argument variadic _, nodeDiag // already printed error about this _, nodeColas // OAS resulting from := diff --git a/src/cmd/compile/internal/gc/walk.go b/src/cmd/compile/internal/gc/walk.go index 8ae3d9a5c7..74ed0411bd 100644 --- a/src/cmd/compile/internal/gc/walk.go +++ b/src/cmd/compile/internal/gc/walk.go @@ -973,6 +973,7 @@ opswitch: case OANDNOT: n.Left = walkexpr(n.Left, init) n.Op = OAND + n.SetImplicit(true) // for walkCheckPtrArithmetic n.Right = nod(OBITNOT, n.Right, nil) n.Right = typecheck(n.Right, ctxExpr) n.Right = walkexpr(n.Right, init) @@ -4003,8 +4004,12 @@ func walkCheckPtrArithmetic(n *Node, init *Nodes) *Node { case OADD: walk(n.Left) walk(n.Right) - case OSUB, OANDNOT: + case OSUB: walk(n.Left) + case OAND: + if n.Implicit() { // was OANDNOT + walk(n.Left) + } case OCONVNOP: if n.Left.Type.Etype == TUNSAFEPTR { n.Left = cheapexpr(n.Left, init) diff --git a/src/runtime/checkptr_test.go b/src/runtime/checkptr_test.go index 8ab8a4937c..194cc1243a 100644 --- a/src/runtime/checkptr_test.go +++ b/src/runtime/checkptr_test.go @@ -27,6 +27,7 @@ func TestCheckPtr(t *testing.T) { {"CheckPtrAlignmentPtr", "fatal error: checkptr: misaligned pointer conversion\n"}, {"CheckPtrAlignmentNoPtr", ""}, {"CheckPtrArithmetic", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, + {"CheckPtrArithmetic2", "fatal error: checkptr: pointer arithmetic result points to invalid allocation\n"}, {"CheckPtrSize", "fatal error: checkptr: converted pointer straddles multiple allocations\n"}, {"CheckPtrSmall", "fatal error: checkptr: pointer arithmetic computed bad pointer value\n"}, } diff --git a/src/runtime/testdata/testprog/checkptr.go b/src/runtime/testdata/testprog/checkptr.go index 45e6fb1aa5..e0a2794f4c 100644 --- a/src/runtime/testdata/testprog/checkptr.go +++ b/src/runtime/testdata/testprog/checkptr.go @@ -10,6 +10,7 @@ func init() { register("CheckPtrAlignmentNoPtr", CheckPtrAlignmentNoPtr) register("CheckPtrAlignmentPtr", CheckPtrAlignmentPtr) register("CheckPtrArithmetic", CheckPtrArithmetic) + register("CheckPtrArithmetic2", CheckPtrArithmetic2) register("CheckPtrSize", CheckPtrSize) register("CheckPtrSmall", CheckPtrSmall) } @@ -32,6 +33,13 @@ func CheckPtrArithmetic() { sink2 = (*int)(unsafe.Pointer(i)) } +func CheckPtrArithmetic2() { + var x [2]int64 + p := unsafe.Pointer(&x[1]) + var one uintptr = 1 + sink2 = unsafe.Pointer(uintptr(p) & ^one) +} + func CheckPtrSize() { p := new(int64) sink2 = p diff --git a/test/fixedbugs/issue40917.go b/test/fixedbugs/issue40917.go new file mode 100644 index 0000000000..2128be5eca --- /dev/null +++ b/test/fixedbugs/issue40917.go @@ -0,0 +1,23 @@ +// run -gcflags=-d=checkptr + +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "unsafe" + +func main() { + var x [2]uint64 + a := unsafe.Pointer(&x[1]) + + b := a + b = unsafe.Pointer(uintptr(b) + 2) + b = unsafe.Pointer(uintptr(b) - 1) + b = unsafe.Pointer(uintptr(b) &^ 1) + + if a != b { + panic("pointer arithmetic failed") + } +} -- cgit v1.3-5-g9baa From cbc0a7906cfb4c82f850d35ffdb7eb0d674cc9b1 Mon Sep 17 00:00:00 2001 From: Robert Griesemer Date: Thu, 20 Aug 2020 16:48:21 -0700 Subject: go/types: update flag documentation for gotype command The documentation refers to a non longer existing flag (-seq). Remove those references. Change-Id: I480b6259f9199b47761dc655a90911eabfe07427 Reviewed-on: https://go-review.googlesource.com/c/go/+/249738 Run-TryBot: Robert Griesemer TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/go/types/gotype.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/go/types/gotype.go b/src/go/types/gotype.go index 19dd702c45..eacf68f52f 100644 --- a/src/go/types/gotype.go +++ b/src/go/types/gotype.go @@ -48,9 +48,9 @@ The flags are: Flags controlling additional output: -ast - print AST (forces -seq) + print AST -trace - print parse trace (forces -seq) + print parse trace -comments parse comments (ignored unless -ast or -trace is provided) -panic @@ -104,8 +104,8 @@ var ( compiler = flag.String("c", "source", "compiler used for installed packages (gc, gccgo, or source)") // additional output control - printAST = flag.Bool("ast", false, "print AST (forces -seq)") - printTrace = flag.Bool("trace", false, "print parse trace (forces -seq)") + printAST = flag.Bool("ast", false, "print AST") + printTrace = flag.Bool("trace", false, "print parse trace") parseComments = flag.Bool("comments", false, "parse comments (ignored unless -ast or -trace is provided)") panicOnError = flag.Bool("panic", false, "panic on first error") ) -- cgit v1.3-5-g9baa From 454300a617b35a417e7595ff93c6e2e059fe1374 Mon Sep 17 00:00:00 2001 From: Meng Zhuo Date: Fri, 21 Aug 2020 11:11:32 +0800 Subject: hash/maphash: adding benchmarks for maphash goos: linux goarch: arm64 pkg: hash/maphash BenchmarkHash8Bytes BenchmarkHash8Bytes 22568919 46.0 ns/op 173.80 MB/s BenchmarkHash320Bytes BenchmarkHash320Bytes 5243858 230 ns/op 1393.30 MB/s BenchmarkHash1K BenchmarkHash1K 1755870 660 ns/op 1550.60 MB/s BenchmarkHash8K BenchmarkHash8K 225688 5313 ns/op 1541.90 MB/s PASS ok hash/maphash 6.465s Change-Id: I5a909042a542135ebc47d639fea02dc46c900c1c Reviewed-on: https://go-review.googlesource.com/c/go/+/249079 Run-TryBot: Meng Zhuo Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/hash/maphash/maphash_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/hash/maphash/maphash_test.go b/src/hash/maphash/maphash_test.go index caea43a8c8..daf6eb4786 100644 --- a/src/hash/maphash/maphash_test.go +++ b/src/hash/maphash/maphash_test.go @@ -165,3 +165,32 @@ func TestSeedFromReset(t *testing.T) { // Make sure a Hash implements the hash.Hash and hash.Hash64 interfaces. var _ hash.Hash = &Hash{} var _ hash.Hash64 = &Hash{} + +func benchmarkSize(b *testing.B, size int) { + h := &Hash{} + buf := make([]byte, size) + b.SetBytes(int64(size)) + b.ResetTimer() + + for i := 0; i < b.N; i++ { + h.Reset() + h.Write(buf) + h.Sum64() + } +} + +func BenchmarkHash8Bytes(b *testing.B) { + benchmarkSize(b, 8) +} + +func BenchmarkHash320Bytes(b *testing.B) { + benchmarkSize(b, 320) +} + +func BenchmarkHash1K(b *testing.B) { + benchmarkSize(b, 1024) +} + +func BenchmarkHash8K(b *testing.B) { + benchmarkSize(b, 8192) +} -- cgit v1.3-5-g9baa From 8535008765b4fcd5c7dc3fb2b73a856af4d51f9b Mon Sep 17 00:00:00 2001 From: Michał Łowicki Date: Mon, 11 May 2020 22:57:26 +0100 Subject: cmd/go: clarify error for invalid proxy responses Add information that error comes from parsing module proxy responses. Fixes #38680 Change-Id: Ic318b9cdbca789c1b0d983e083e692a914892fbd Reviewed-on: https://go-review.googlesource.com/c/go/+/233437 Run-TryBot: Jay Conrod TryBot-Result: Gobot Gobot Reviewed-by: Jay Conrod --- src/cmd/go/internal/modfetch/proxy.go | 13 +++++++------ src/cmd/go/proxy_test.go | 6 ++++++ src/cmd/go/testdata/script/mod_proxy_invalid.txt | 8 ++++++++ src/cmd/go/testdata/script/mod_query_empty.txt | 2 +- 4 files changed, 22 insertions(+), 7 deletions(-) create mode 100644 src/cmd/go/testdata/script/mod_proxy_invalid.txt diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index 1c35d0b99b..4ac26650a9 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -242,8 +242,9 @@ func TryProxies(f func(proxy string) error) error { } type proxyRepo struct { - url *url.URL - path string + url *url.URL + path string + redactedURL string } func newProxyRepo(baseURL, path string) (Repo, error) { @@ -268,10 +269,10 @@ func newProxyRepo(baseURL, path string) (Repo, error) { if err != nil { return nil, err } - + redactedURL := base.Redacted() base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc) - return &proxyRepo{base, path}, nil + return &proxyRepo{base, path, redactedURL}, nil } func (p *proxyRepo) ModulePath() string { @@ -413,7 +414,7 @@ func (p *proxyRepo) Stat(rev string) (*RevInfo, error) { } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError(rev, err) + return nil, p.versionError(rev, fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil { // If we request a correct, appropriate version for the module path, the @@ -434,7 +435,7 @@ func (p *proxyRepo) Latest() (*RevInfo, error) { } info := new(RevInfo) if err := json.Unmarshal(data, info); err != nil { - return nil, p.versionError("", err) + return nil, p.versionError("", fmt.Errorf("invalid response from proxy %q: %w", p.redactedURL, err)) } return info, nil } diff --git a/src/cmd/go/proxy_test.go b/src/cmd/go/proxy_test.go index 2a4d2935b3..7f58fb8ce4 100644 --- a/src/cmd/go/proxy_test.go +++ b/src/cmd/go/proxy_test.go @@ -131,6 +131,12 @@ func proxyHandler(w http.ResponseWriter, r *http.Request) { } path := r.URL.Path[len("/mod/"):] + // /mod/invalid returns faulty responses. + if strings.HasPrefix(path, "invalid/") { + w.Write([]byte("invalid")) + return + } + // /mod/quiet/ does not print errors. quiet := false if strings.HasPrefix(path, "quiet/") { diff --git a/src/cmd/go/testdata/script/mod_proxy_invalid.txt b/src/cmd/go/testdata/script/mod_proxy_invalid.txt new file mode 100644 index 0000000000..6427cc1527 --- /dev/null +++ b/src/cmd/go/testdata/script/mod_proxy_invalid.txt @@ -0,0 +1,8 @@ +env GO111MODULE=on +env GOPROXY=$GOPROXY/invalid + +! go list -m rsc.io/quote@latest +stderr '^go list -m: module rsc.io/quote: invalid response from proxy "'$GOPROXY'": invalid character ''i'' looking for beginning of value$' + +! go list -m rsc.io/quote@1.5.2 +stderr '^go list -m: rsc.io/quote@1.5.2: invalid version: invalid response from proxy "'$GOPROXY'": invalid character ''i'' looking for beginning of value$' diff --git a/src/cmd/go/testdata/script/mod_query_empty.txt b/src/cmd/go/testdata/script/mod_query_empty.txt index 4d8259b40f..b3ea3e3de0 100644 --- a/src/cmd/go/testdata/script/mod_query_empty.txt +++ b/src/cmd/go/testdata/script/mod_query_empty.txt @@ -40,7 +40,7 @@ env GOPROXY=file:///$WORK/gatekeeper chmod 0000 $WORK/gatekeeper/example.com/join/subpkg/@latest cp go.mod.orig go.mod ! go get -d example.com/join/subpkg -stderr 'go get example.com/join/subpkg: module example.com/join/subpkg: (invalid character .+|reading file://.*/gatekeeper/example.com/join/subpkg/@latest: .+)' +stderr 'go get example.com/join/subpkg: module example.com/join/subpkg: (invalid response from proxy ".+": invalid character .+|reading file://.*/gatekeeper/example.com/join/subpkg/@latest: .+)' -- go.mod.orig -- module example.com/othermodule -- cgit v1.3-5-g9baa From 9679b307334bce77cc6e50751956a4c717e9458c Mon Sep 17 00:00:00 2001 From: Michael Matloob Date: Thu, 23 Apr 2020 18:06:21 -0400 Subject: cmd/go/testdata/script: make list_case_collision's behavior more clear Implementing the suggestion made by bcmills on a comment on golang.org/cl/228783. Change-Id: I314a24a002c65b582ea51610dcc1a54a69afbb8c Reviewed-on: https://go-review.googlesource.com/c/go/+/229705 Run-TryBot: Michael Matloob TryBot-Result: Gobot Gobot Reviewed-by: Bryan C. Mills --- src/cmd/go/script_test.go | 38 ++++++++++++++++++++++ src/cmd/go/testdata/script/README | 1 + src/cmd/go/testdata/script/list_case_collision.txt | 25 +++++--------- 3 files changed, 48 insertions(+), 16 deletions(-) diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 2e8f18a897..986646252a 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -22,6 +22,7 @@ import ( "runtime" "strconv" "strings" + "sync" "testing" "time" @@ -296,6 +297,8 @@ Script: ok = os.Geteuid() == 0 case "symlink": ok = testenv.HasSymlink() + case "case-sensitive": + ok = isCaseSensitive(ts.t) default: if strings.HasPrefix(cond.tag, "exec:") { prog := cond.tag[len("exec:"):] @@ -364,6 +367,41 @@ Script: } } +var ( + onceCaseSensitive sync.Once + caseSensitive bool +) + +func isCaseSensitive(t *testing.T) bool { + onceCaseSensitive.Do(func() { + tmpdir, err := ioutil.TempDir("", "case-sensitive") + if err != nil { + t.Fatal("failed to create directory to determine case-sensitivity:", err) + } + defer os.RemoveAll(tmpdir) + + fcap := filepath.Join(tmpdir, "FILE") + if err := ioutil.WriteFile(fcap, []byte{}, 0644); err != nil { + t.Fatal("error writing file to determine case-sensitivity:", err) + } + + flow := filepath.Join(tmpdir, "file") + _, err = ioutil.ReadFile(flow) + switch { + case err == nil: + caseSensitive = false + return + case os.IsNotExist(err): + caseSensitive = true + return + default: + t.Fatal("unexpected error reading file when determining case-sensitivity:", err) + } + }) + + return caseSensitive +} + // scriptCmds are the script command implementations. // Keep list and the implementations below sorted by name. // diff --git a/src/cmd/go/testdata/script/README b/src/cmd/go/testdata/script/README index 76d6651718..d658cebfce 100644 --- a/src/cmd/go/testdata/script/README +++ b/src/cmd/go/testdata/script/README @@ -85,6 +85,7 @@ should only run when the condition is satisfied. The available conditions are: - [link] for testenv.HasLink() - [root] for os.Geteuid() == 0 - [symlink] for testenv.HasSymlink() + - [case-sensitive] for whether the file system is case-sensitive - [exec:prog] for whether prog is available for execution (found by exec.LookPath) - [GODEBUG:value] for whether value is one of the comma-separated entries in the GODEBUG variable - [buildmode:value] for whether -buildmode=value is supported diff --git a/src/cmd/go/testdata/script/list_case_collision.txt b/src/cmd/go/testdata/script/list_case_collision.txt index 1b5f305587..73f44b63a0 100644 --- a/src/cmd/go/testdata/script/list_case_collision.txt +++ b/src/cmd/go/testdata/script/list_case_collision.txt @@ -6,23 +6,20 @@ stdout 'case-insensitive import collision' ! go build example/a stderr 'case-insensitive import collision' -# If we're not guaranteed to have a case-sensitive file system, list files explicitly on command line. -# Otherwise, let directory read find both files. -[darwin] ! go list example/b/file.go example/b/FILE.go -[windows] ! go list example/b/file.go example/b/FILE.go -[!darwin] [!windows] ! go list example/b +# List files explicitly on command line, to encounter case-checking +# logic even on case-insensitive filesystems. +cp example/b/file.go example/b/FILE.go # no-op on case-insensitive filesystems +! go list example/b/file.go example/b/FILE.go stderr 'case-insensitive file name collision' +mkdir example/a/Pkg # no-op on case-insensitive filesystems +cp example/a/pkg/pkg.go example/a/Pkg/pkg.go # no-op on case-insensitive filesystems ! go list example/a/pkg example/a/Pkg -stderr 'case-insensitive import collision' -go list -json -e example/a/pkg example/a/Pkg -stdout 'case-insensitive import collision' -! go build example/a/pkg example/a/Pkg -stderr 'case-insensitive import collision' # Test that the path reported with an indirect import is correct. -[!darwin] [!windows] ! go build example/c -[!darwin] [!windows] stderr '^package example/c\n\timports example/b: case-insensitive file name collision: "FILE.go" and "file.go"$' +cp example/b/file.go example/b/FILE.go +[case-sensitive] ! go build example/c +[case-sensitive] stderr '^package example/c\n\timports example/b: case-insensitive file name collision: "FILE.go" and "file.go"$' -- example/a/a.go -- package p @@ -32,12 +29,8 @@ import ( ) -- example/a/pkg/pkg.go -- package pkg --- example/a/Pkg/pkg.go -- -package pkg -- example/b/file.go -- package b --- example/b/FILE.go -- -package b -- example/c/c.go -- package c -- cgit v1.3-5-g9baa