diff options
| author | Matthew Dempsky <mdempsky@google.com> | 2021-01-22 12:00:45 -0800 |
|---|---|---|
| committer | Matthew Dempsky <mdempsky@google.com> | 2021-01-22 12:01:13 -0800 |
| commit | 7e0a81d2806b073c6455f73b10fbf2c811703f46 (patch) | |
| tree | 4b7bc9664df2a88ba01afcc06b25da026037940f /src/runtime | |
| parent | d7e71c01ad1c8edd568380ce9276c265dfd3635b (diff) | |
| parent | dab3e5affefb7e0b157ad27fe4797c4c6f3c4ea6 (diff) | |
| download | go-7e0a81d2806b073c6455f73b10fbf2c811703f46.tar.xz | |
[dev.regabi] all: merge master (dab3e5a) into dev.regabi
This merge had two conflicts to resolve:
1. The embed code on master had somewhat substantially diverged, so
this CL tediously backported the changes to dev.regabi. In particular,
I went through all of the embed changes to gc/{embed,noder,syntax}.go
and made sure the analogous code on dev.regabi in noder/noder.go and
staticdata/embed.go mirrors it.
2. The init-cycle reporting code on master was extended slightly to
track already visited declarations to avoid exponential behavior. The
same fix is applied on dev.regabi, just using ir.NameSet instead of
map[ir.Node]bool.
Conflicts:
- src/cmd/compile/internal/gc/embed.go
- src/cmd/compile/internal/gc/noder.go
- src/cmd/compile/internal/gc/syntax.go
- src/cmd/compile/internal/pkginit/initorder.go
- src/embed/internal/embedtest/embed_test.go
- src/go/types/stdlib_test.go
Merge List:
+ 2021-01-22 dab3e5affe runtime: switch runtime to libc for openbsd/amd64
+ 2021-01-22 a1b53d85da cmd/go: add documentation for test and xtest fields output by go list
+ 2021-01-22 b268b60774 runtime: remove pthread_kill/pthread_self for openbsd
+ 2021-01-22 ec4051763d runtime: fix typo in mgcscavenge.go
+ 2021-01-22 7ece3a7b17 net/http: fix flaky TestDisableKeepAliveUpgrade
+ 2021-01-22 50cba0506f time: clarify Timer.Reset behavior on AfterFunc Timers
+ 2021-01-22 cf10e69f17 doc/go1.16: mention net/http.Transport.GetProxyConnectHeader
+ 2021-01-22 ec1b945265 doc/go1.16: mention path/filepath.WalkDir
+ 2021-01-22 11def3d40b doc/go1.16: mention syscall.AllThreadsSyscall
+ 2021-01-21 07b0235609 doc/go1.16: add notes about package-specific fs.FS changes
+ 2021-01-21 e2b4f1fea5 doc/go1.16: minor formatting fix
+ 2021-01-21 9f43a9e07b doc/go1.16: mention new debug/elf constants
+ 2021-01-21 3c2f11ba5b cmd/go: overwrite program name with full path
+ 2021-01-21 953d1feca9 all: introduce and use internal/execabs
+ 2021-01-21 b186e4d70d cmd/go: add test case for cgo CC setting
+ 2021-01-21 5a8a2265fb cmd/cgo: report exec errors a bit more clearly
+ 2021-01-21 46e2e2e9d9 cmd/go: pass resolved CC, GCCGO to cgo
+ 2021-01-21 3d40895e36 runtime: switch openbsd/arm64 to pthreads
+ 2021-01-21 d95ca91380 crypto/elliptic: fix P-224 field reduction
+ 2021-01-20 ecf4ebf100 cmd/internal/moddeps: check content of all modules in GOROOT
+ 2021-01-20 d2d155d1ae runtime: don't adjust timer pp field in timerWaiting status
+ 2021-01-20 803d18fc6c cmd/go: set Incomplete field on go list output if no files match embed
+ 2021-01-20 6e243ce71d cmd/go: have go mod vendor copy embedded files in subdirs
+ 2021-01-20 be28e5abc5 cmd/go: fix mod_get_fallback test
+ 2021-01-20 928bda4f4a runtime: convert openbsd/amd64 locking to libc
+ 2021-01-19 824f2d635c cmd/go: allow go fmt to complete when embedded file is missing
+ 2021-01-19 0575e35e50 cmd/compile: require 'go 1.16' go.mod line for //go:embed
+ 2021-01-19 ccb2e90688 cmd/link: exit before Asmb2 if error
+ 2021-01-19 ca5774a5a5 embed: treat uninitialized FS as empty
+ 2021-01-19 d047c91a6c cmd/link,runtime: switch openbsd/amd64 to pthreads
+ 2021-01-19 61debffd97 runtime: factor out usesLibcall
+ 2021-01-19 9fed39d281 runtime: factor out mStackIsSystemAllocated
+ 2021-01-18 dbab079835 runtime: free Windows event handles after last lock is dropped
+ 2021-01-18 5a8fbb0d2d os: do not close syscall.Stdin in TestReadStdin
+ 2021-01-15 682a1d2176 runtime: detect errors in DuplicateHandle
+ 2021-01-15 9f83418b83 cmd/link: remove GOROOT write in TestBuildForTvOS
+ 2021-01-15 ec9470162f cmd/compile: allow embed into any string or byte slice type
+ 2021-01-15 54198b04db cmd/compile: disallow embed of var inside func
+ 2021-01-15 b386c735e7 cmd/go: fix go generate docs
+ 2021-01-15 bb5075a525 syscall: remove RtlGenRandom and move it into internal/syscall
+ 2021-01-15 1deae0b597 os: invoke processKiller synchronously in testKillProcess
+ 2021-01-15 ff196c3e84 crypto/x509: update iOS bundled roots to version 55188.40.9
+ 2021-01-14 e125ccd10e cmd/go: in 'go mod edit', validate versions given to -retract and -exclude
+ 2021-01-14 eb330020dc cmd/dist, cmd/go: pass -arch for C compilation on Darwin
+ 2021-01-14 84e8a06f62 cmd/cgo: remove unnecessary space in cgo export header
+ 2021-01-14 0c86b999c3 cmd/test2json: document passing -test.paniconexit0
+ 2021-01-14 9135795891 cmd/go/internal/load: report positions for embed errors
+ 2021-01-14 d9b79e53bb cmd/compile: fix wrong complement for arm64 floating-point comparisons
+ 2021-01-14 c73232d08f cmd/go/internal/load: refactor setErrorPos to PackageError.setPos
+ 2021-01-14 6aa28d3e06 go/build: report positions for go:embed directives
+ 2021-01-13 7eb31d999c cmd/go: add hints to more missing sum error messages
+ 2021-01-12 ba76567bc2 cmd/go/internal/modload: delete unused *mvsReqs.next method
+ 2021-01-12 665def2c11 encoding/asn1: document unmarshaling behavior for IMPLICIT string fields
+ 2021-01-11 81ea89adf3 cmd/go: fix non-script staleness checks interacting badly with GOFLAGS
+ 2021-01-11 759309029f doc: update editors.html for Go 1.16
+ 2021-01-11 c3b4c7093a cmd/internal/objfile: don't require runtime.symtab symbol for XCOFF
+ 2021-01-08 59bfc18e34 cmd/go: add hint to read 'go help vcs' to GOVCS errors
+ 2021-01-08 cd6f3a54e4 cmd/go: revise 'go help' documentation for modules
+ 2021-01-08 6192b98751 cmd/go: make hints in error messages more consistent
+ 2021-01-08 25886cf4bd cmd/go: preserve sums for indirect deps fetched by 'go mod download'
+ 2021-01-08 6250833911 runtime/metrics: mark histogram metrics as cumulative
+ 2021-01-08 8f6a9acbb3 runtime/metrics: remove unused StopTheWorld Description field
+ 2021-01-08 6598c65646 cmd/compile: fix exponential-time init-cycle reporting
+ 2021-01-08 fefad1dc85 test: fix timeout code for invoking compiler
+ 2021-01-08 6728118e0a cmd/go: pass signals forward during "go tool"
+ 2021-01-08 e65c543f3c go/build/constraint: add parser for build tag constraint expressions
+ 2021-01-08 0c5afc4fb7 testing/fstest,os: clarify racy behavior of TestFS
+ 2021-01-08 32afcc9436 runtime/metrics: change unit on *-by-size metrics to match bucket unit
+ 2021-01-08 c6513bca5a io/fs: minor corrections to Glob doc
+ 2021-01-08 304f769ffc cmd/compile: don't short-circuit copies whose source is volatile
+ 2021-01-08 ae97717133 runtime,runtime/metrics: use explicit histogram boundaries
+ 2021-01-08 a9ccd2d795 go/build: skip string literal while findEmbed
+ 2021-01-08 d92f8add32 archive/tar: fix typo in comment
+ 2021-01-08 cab1202183 cmd/link: accept extra blocks in TestFallocate
+ 2021-01-08 ee4d32249b io/fs: minor corrections to Glob release date
+ 2021-01-08 54bd1ccce2 cmd: update to latest golang.org/x/tools
+ 2021-01-07 9ec21a8f34 Revert "reflect: support multiple keys in struct tags"
+ 2021-01-07 091414b5b7 io/fs: correct WalkDirFunc documentation
+ 2021-01-07 9b55088d6b doc/go1.16: add release note for disallowing non-ASCII import paths
+ 2021-01-07 fa90aaca7d cmd/compile: fix late expand_calls leaf type for OpStructSelect/OpArraySelect
+ 2021-01-07 7cee66d4cb cmd/go: add documentation for Embed fields in go list output
+ 2021-01-07 e60cffa4ca html/template: attach functions to namespace
+ 2021-01-07 6da2d3b7d7 cmd/link: fix typo in asm.go
+ 2021-01-07 df81a15819 runtime: check mips64 VDSO clock_gettime return code
+ 2021-01-06 4787e906cf crypto/x509: rollback new CertificateRequest fields
+ 2021-01-06 c9658bee93 cmd/go: make module suggestion more friendly
+ 2021-01-06 4c668b25c6 runtime/metrics: fix panic message for Float64Histogram
+ 2021-01-06 d2131704a6 net/http/httputil: fix deadlock in DumpRequestOut
+ 2021-01-05 3e1e13ce6d cmd/go: set cfg.BuildMod to "readonly" by default with no module root
+ 2021-01-05 0b0d004983 cmd/go: pass embedcfg to gccgo if supported
+ 2021-01-05 1b85e7c057 cmd/go: don't scan gccgo standard library packages for imports
+ 2021-01-05 6b37b15d95 runtime: don't take allglock in tracebackothers
+ 2021-01-04 9eef49cfa6 math/rand: fix typo in comment
+ 2021-01-04 b01fb2af9e testing/fstest: fix typo in error message
+ 2021-01-01 3dd5867605 doc: 2021 is the Year of the Gopher
+ 2020-12-31 95ce805d14 io/fs: remove darwin/arm64 special condition
+ 2020-12-30 20d0991b86 lib/time, time/tzdata: update tzdata to 2020f
+ 2020-12-30 ed301733bb misc/cgo/testcarchive: remove special flags for Darwin/ARM
+ 2020-12-30 0ae2e032f2 misc/cgo/test: enable TestCrossPackageTests on darwin/arm64
+ 2020-12-29 780b4de16b misc/ios: fix wording for command line instructions
+ 2020-12-29 b4a71c95d2 doc/go1.16: reference misc/ios/README for how to build iOS programs
+ 2020-12-29 f83e0f6616 misc/ios: add to README how to build ios executables
+ 2020-12-28 4fd9455882 io/fs: fix typo in comment
Change-Id: I2f257bbc5fbb05f15c2d959f8cfe0ce13b083538
Diffstat (limited to 'src/runtime')
46 files changed, 1515 insertions, 656 deletions
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 53d1f8e358..b5d01ba73c 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -181,6 +181,10 @@ needtls: // skip TLS setup on Darwin JMP ok #endif +#ifdef GOOS_openbsd + // skip TLS setup on OpenBSD + JMP ok +#endif LEAQ runtime·m0+m_tls(SB), DI CALL runtime·settls(SB) diff --git a/src/runtime/defs_openbsd.go b/src/runtime/defs_openbsd.go index 53e9d59a3c..ff7e21c71e 100644 --- a/src/runtime/defs_openbsd.go +++ b/src/runtime/defs_openbsd.go @@ -54,6 +54,13 @@ const ( SA_RESTART = C.SA_RESTART SA_ONSTACK = C.SA_ONSTACK + PTHREAD_CREATE_DETACHED = C.PTHREAD_CREATE_DETACHED + + F_SETFD = C.F_SETFD + F_GETFL = C.F_GETFL + F_SETFL = C.F_SETFL + FD_CLOEXEC = C.FD_CLOEXEC + SIGHUP = C.SIGHUP SIGINT = C.SIGINT SIGQUIT = C.SIGQUIT @@ -129,3 +136,10 @@ type Timeval C.struct_timeval type Itimerval C.struct_itimerval type KeventT C.struct_kevent + +type Pthread C.pthread_t +type PthreadAttr C.pthread_attr_t +type PthreadCond C.pthread_cond_t +type PthreadCondAttr C.pthread_condattr_t +type PthreadMutex C.pthread_mutex_t +type PthreadMutexAttr C.pthread_mutexattr_t diff --git a/src/runtime/defs_openbsd_amd64.go b/src/runtime/defs_openbsd_amd64.go index c187a98ae0..46f1245201 100644 --- a/src/runtime/defs_openbsd_amd64.go +++ b/src/runtime/defs_openbsd_amd64.go @@ -30,6 +30,13 @@ const ( _SA_RESTART = 0x2 _SA_ONSTACK = 0x1 + _PTHREAD_CREATE_DETACHED = 0x1 + + _F_SETFD = 0x2 + _F_GETFL = 0x3 + _F_SETFL = 0x4 + _FD_CLOEXEC = 0x1 + _SIGHUP = 0x1 _SIGINT = 0x2 _SIGQUIT = 0x3 @@ -177,3 +184,10 @@ type keventt struct { data int64 udata *byte } + +type pthread uintptr +type pthreadattr uintptr +type pthreadcond uintptr +type pthreadcondattr uintptr +type pthreadmutex uintptr +type pthreadmutexattr uintptr diff --git a/src/runtime/defs_openbsd_arm64.go b/src/runtime/defs_openbsd_arm64.go index 628f4bc5a5..63ea8dfecc 100644 --- a/src/runtime/defs_openbsd_arm64.go +++ b/src/runtime/defs_openbsd_arm64.go @@ -31,6 +31,8 @@ const ( _SA_RESTART = 0x2 _SA_ONSTACK = 0x1 + _PTHREAD_CREATE_DETACHED = 0x1 + _SIGHUP = 0x1 _SIGINT = 0x2 _SIGQUIT = 0x3 @@ -157,3 +159,10 @@ type keventt struct { data int64 udata *byte } + +type pthread uintptr +type pthreadattr uintptr +type pthreadcond uintptr +type pthreadcondattr uintptr +type pthreadmutex uintptr +type pthreadmutexattr uintptr diff --git a/src/runtime/histogram.go b/src/runtime/histogram.go index d48e856cd0..42baa6c5e2 100644 --- a/src/runtime/histogram.go +++ b/src/runtime/histogram.go @@ -7,6 +7,7 @@ package runtime import ( "runtime/internal/atomic" "runtime/internal/sys" + "unsafe" ) const ( @@ -69,7 +70,13 @@ const ( // for concurrent use. It is also safe to read all the values // atomically. type timeHistogram struct { - counts [timeHistNumSuperBuckets * timeHistNumSubBuckets]uint64 + counts [timeHistNumSuperBuckets * timeHistNumSubBuckets]uint64 + + // underflow counts all the times we got a negative duration + // sample. Because of how time works on some platforms, it's + // possible to measure negative durations. We could ignore them, + // but we record them anyway because it's better to have some + // signal that it's happening than just missing samples. underflow uint64 } @@ -107,14 +114,30 @@ func (h *timeHistogram) record(duration int64) { atomic.Xadd64(&h.counts[superBucket*timeHistNumSubBuckets+subBucket], 1) } +const ( + fInf = 0x7FF0000000000000 + fNegInf = 0xFFF0000000000000 +) + +func float64Inf() float64 { + inf := uint64(fInf) + return *(*float64)(unsafe.Pointer(&inf)) +} + +func float64NegInf() float64 { + inf := uint64(fNegInf) + return *(*float64)(unsafe.Pointer(&inf)) +} + // timeHistogramMetricsBuckets generates a slice of boundaries for // the timeHistogram. These boundaries are represented in seconds, // not nanoseconds like the timeHistogram represents durations. func timeHistogramMetricsBuckets() []float64 { - b := make([]float64, timeHistTotalBuckets-1) + b := make([]float64, timeHistTotalBuckets+1) + b[0] = float64NegInf() for i := 0; i < timeHistNumSuperBuckets; i++ { superBucketMin := uint64(0) - // The (inclusive) minimum for the first bucket is 0. + // The (inclusive) minimum for the first non-negative bucket is 0. if i > 0 { // The minimum for the second bucket will be // 1 << timeHistSubBucketBits, indicating that all @@ -141,8 +164,9 @@ func timeHistogramMetricsBuckets() []float64 { // Convert the subBucketMin which is in nanoseconds to a float64 seconds value. // These values will all be exactly representable by a float64. - b[i*timeHistNumSubBuckets+j] = float64(subBucketMin) / 1e9 + b[i*timeHistNumSubBuckets+j+1] = float64(subBucketMin) / 1e9 } } + b[len(b)-1] = float64Inf() return b } diff --git a/src/runtime/metrics.go b/src/runtime/metrics.go index 1d191e6298..3e8dbda0ca 100644 --- a/src/runtime/metrics.go +++ b/src/runtime/metrics.go @@ -41,8 +41,13 @@ func initMetrics() { if metricsInit { return } - sizeClassBuckets = make([]float64, _NumSizeClasses) - for i := range sizeClassBuckets { + + sizeClassBuckets = make([]float64, _NumSizeClasses, _NumSizeClasses+1) + // Skip size class 0 which is a stand-in for large objects, but large + // objects are tracked separately (and they actually get placed in + // the last bucket, not the first). + sizeClassBuckets[0] = 1 // The smallest allocation is 1 byte in size. + for i := 1; i < _NumSizeClasses; i++ { // Size classes have an inclusive upper-bound // and exclusive lower bound (e.g. 48-byte size class is // (32, 48]) whereas we want and inclusive lower-bound @@ -56,6 +61,8 @@ func initMetrics() { // boundaries. sizeClassBuckets[i] = float64(class_to_size[i] + 1) } + sizeClassBuckets = append(sizeClassBuckets, float64Inf()) + timeHistBuckets = timeHistogramMetricsBuckets() metrics = map[string]metricData{ "/gc/cycles/automatic:gc-cycles": { @@ -79,23 +86,27 @@ func initMetrics() { out.scalar = in.sysStats.gcCyclesDone }, }, - "/gc/heap/allocs-by-size:objects": { + "/gc/heap/allocs-by-size:bytes": { deps: makeStatDepSet(heapStatsDep), compute: func(in *statAggregate, out *metricValue) { hist := out.float64HistOrInit(sizeClassBuckets) hist.counts[len(hist.counts)-1] = uint64(in.heapStats.largeAllocCount) - for i := range hist.buckets { - hist.counts[i] = uint64(in.heapStats.smallAllocCount[i]) + // Cut off the first index which is ostensibly for size class 0, + // but large objects are tracked separately so it's actually unused. + for i, count := range in.heapStats.smallAllocCount[1:] { + hist.counts[i] = uint64(count) } }, }, - "/gc/heap/frees-by-size:objects": { + "/gc/heap/frees-by-size:bytes": { deps: makeStatDepSet(heapStatsDep), compute: func(in *statAggregate, out *metricValue) { hist := out.float64HistOrInit(sizeClassBuckets) hist.counts[len(hist.counts)-1] = uint64(in.heapStats.largeFreeCount) - for i := range hist.buckets { - hist.counts[i] = uint64(in.heapStats.smallFreeCount[i]) + // Cut off the first index which is ostensibly for size class 0, + // but large objects are tracked separately so it's actually unused. + for i, count := range in.heapStats.smallFreeCount[1:] { + hist.counts[i] = uint64(count) } }, }, @@ -116,8 +127,11 @@ func initMetrics() { "/gc/pauses:seconds": { compute: func(_ *statAggregate, out *metricValue) { hist := out.float64HistOrInit(timeHistBuckets) + // The bottom-most bucket, containing negative values, is tracked + // as a separately as underflow, so fill that in manually and then + // iterate over the rest. hist.counts[0] = atomic.Load64(&memstats.gcPauseDist.underflow) - for i := range hist.buckets { + for i := range memstats.gcPauseDist.counts { hist.counts[i+1] = atomic.Load64(&memstats.gcPauseDist.counts[i]) } }, @@ -437,8 +451,8 @@ func (v *metricValue) float64HistOrInit(buckets []float64) *metricFloat64Histogr v.pointer = unsafe.Pointer(hist) } hist.buckets = buckets - if len(hist.counts) != len(hist.buckets)+1 { - hist.counts = make([]uint64, len(buckets)+1) + if len(hist.counts) != len(hist.buckets)-1 { + hist.counts = make([]uint64, len(buckets)-1) } return hist } diff --git a/src/runtime/metrics/description.go b/src/runtime/metrics/description.go index 32af5d1727..1175156104 100644 --- a/src/runtime/metrics/description.go +++ b/src/runtime/metrics/description.go @@ -23,6 +23,11 @@ type Description struct { // Examples of units might be "seconds", "bytes", "bytes/second", "cpu-seconds", // "byte*cpu-seconds", and "bytes/second/second". // + // For histograms, multiple units may apply. For instance, the units of the buckets and + // the count. By convention, for histograms, the units of the count are always "samples" + // with the type of sample evident by the metric's name, while the unit in the name + // specifies the buckets' unit. + // // A complete name might look like "/memory/heap/free:bytes". Name string @@ -41,10 +46,6 @@ type Description struct { // // This flag thus indicates whether or not it's useful to compute a rate from this value. Cumulative bool - - // StopTheWorld is whether or not the metric requires a stop-the-world - // event in order to collect it. - StopTheWorld bool } // The English language descriptions below must be kept in sync with the @@ -69,14 +70,16 @@ var allDesc = []Description{ Cumulative: true, }, { - Name: "/gc/heap/allocs-by-size:objects", + Name: "/gc/heap/allocs-by-size:bytes", Description: "Distribution of all objects allocated by approximate size.", Kind: KindFloat64Histogram, + Cumulative: true, }, { - Name: "/gc/heap/frees-by-size:objects", + Name: "/gc/heap/frees-by-size:bytes", Description: "Distribution of all objects freed by approximate size.", Kind: KindFloat64Histogram, + Cumulative: true, }, { Name: "/gc/heap/goal:bytes", @@ -92,6 +95,7 @@ var allDesc = []Description{ Name: "/gc/pauses:seconds", Description: "Distribution individual GC-related stop-the-world pause latencies.", Kind: KindFloat64Histogram, + Cumulative: true, }, { Name: "/memory/classes/heap/free:bytes", diff --git a/src/runtime/metrics/doc.go b/src/runtime/metrics/doc.go index a68184ee82..021a0bddca 100644 --- a/src/runtime/metrics/doc.go +++ b/src/runtime/metrics/doc.go @@ -61,10 +61,10 @@ Below is the full list of supported metrics, ordered lexicographically. /gc/cycles/total:gc-cycles Count of all completed GC cycles. - /gc/heap/allocs-by-size:objects + /gc/heap/allocs-by-size:bytes Distribution of all objects allocated by approximate size. - /gc/heap/frees-by-size:objects + /gc/heap/frees-by-size:bytes Distribution of all objects freed by approximate size. /gc/heap/goal:bytes diff --git a/src/runtime/metrics/histogram.go b/src/runtime/metrics/histogram.go index e1364e1e26..956422bf84 100644 --- a/src/runtime/metrics/histogram.go +++ b/src/runtime/metrics/histogram.go @@ -6,25 +6,28 @@ package metrics // Float64Histogram represents a distribution of float64 values. type Float64Histogram struct { - // Counts contains the weights for each histogram bucket. The length of - // Counts is equal to the length of Buckets (in the metric description) - // plus one to account for the implicit minimum bucket. + // Counts contains the weights for each histogram bucket. // - // Given N buckets, the following is the mathematical relationship between - // Counts and Buckets. - // count[0] is the weight of the range (-inf, bucket[0]) - // count[n] is the weight of the range [bucket[n], bucket[n+1]), for 0 < n < N-1 - // count[N-1] is the weight of the range [bucket[N-1], inf) + // Given N buckets, Count[n] is the weight of the range + // [bucket[n], bucket[n+1]), for 0 <= n < N. Counts []uint64 - // Buckets contains the boundaries between histogram buckets, in increasing order. + // Buckets contains the boundaries of the histogram buckets, in increasing order. // - // Because this slice contains boundaries, there are len(Buckets)+1 counts: - // a count for all values less than the first boundary, a count covering each - // [slice[i], slice[i+1]) interval, and a count for all values greater than or - // equal to the last boundary. + // Buckets[0] is the inclusive lower bound of the minimum bucket while + // Buckets[len(Buckets)-1] is the exclusive upper bound of the maximum bucket. + // Hence, there are len(Buckets)-1 counts. Furthermore, len(Buckets) != 1, always, + // since at least two boundaries are required to describe one bucket (and 0 + // boundaries are used to describe 0 buckets). + // + // Buckets[0] is permitted to have value -Inf and Buckets[len(Buckets)-1] is + // permitted to have value Inf. // // For a given metric name, the value of Buckets is guaranteed not to change // between calls until program exit. + // + // This slice value is permitted to alias with other Float64Histograms' Buckets + // fields, so the values within should only ever be read. If they need to be + // modified, the user must make a copy. Buckets []float64 } diff --git a/src/runtime/metrics/value.go b/src/runtime/metrics/value.go index 0b056b4ea8..61e8a192a3 100644 --- a/src/runtime/metrics/value.go +++ b/src/runtime/metrics/value.go @@ -63,7 +63,7 @@ func (v Value) Float64() float64 { // If v.Kind() != KindFloat64Histogram, this method panics. func (v Value) Float64Histogram() *Float64Histogram { if v.kind != KindFloat64Histogram { - panic("called Float64 on non-float64 metric value") + panic("called Float64Histogram on non-Float64Histogram metric value") } return (*Float64Histogram)(v.pointer) } diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go index 0ee469ae29..8a3cf019bd 100644 --- a/src/runtime/metrics_test.go +++ b/src/runtime/metrics_test.go @@ -70,6 +70,34 @@ func TestReadMetrics(t *testing.T) { checkUint64(t, name, samples[i].Value.Uint64(), mstats.BuckHashSys) case "/memory/classes/total:bytes": checkUint64(t, name, samples[i].Value.Uint64(), mstats.Sys) + case "/gc/heap/allocs-by-size:bytes": + hist := samples[i].Value.Float64Histogram() + // Skip size class 0 in BySize, because it's always empty and not represented + // in the histogram. + for i, sc := range mstats.BySize[1:] { + if b, s := hist.Buckets[i+1], float64(sc.Size+1); b != s { + t.Errorf("bucket does not match size class: got %f, want %f", b, s) + // The rest of the checks aren't expected to work anyway. + continue + } + if c, m := hist.Counts[i], sc.Mallocs; c != m { + t.Errorf("histogram counts do not much BySize for class %d: got %d, want %d", i, c, m) + } + } + case "/gc/heap/frees-by-size:bytes": + hist := samples[i].Value.Float64Histogram() + // Skip size class 0 in BySize, because it's always empty and not represented + // in the histogram. + for i, sc := range mstats.BySize[1:] { + if b, s := hist.Buckets[i+1], float64(sc.Size+1); b != s { + t.Errorf("bucket does not match size class: got %f, want %f", b, s) + // The rest of the checks aren't expected to work anyway. + continue + } + if c, f := hist.Counts[i], sc.Frees; c != f { + t.Errorf("histogram counts do not much BySize for class %d: got %d, want %d", i, c, f) + } + } case "/gc/heap/objects:objects": checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapObjects) case "/gc/heap/goal:bytes": @@ -133,9 +161,9 @@ func TestReadMetricsConsistency(t *testing.T) { totalVirtual.got = samples[i].Value.Uint64() case "/gc/heap/objects:objects": objects.total = samples[i].Value.Uint64() - case "/gc/heap/allocs-by-size:objects": + case "/gc/heap/allocs-by-size:bytes": objects.alloc = samples[i].Value.Float64Histogram() - case "/gc/heap/frees-by-size:objects": + case "/gc/heap/frees-by-size:bytes": objects.free = samples[i].Value.Float64Histogram() case "/gc/cycles:gc-cycles": gc.numGC = samples[i].Value.Uint64() @@ -154,11 +182,11 @@ func TestReadMetricsConsistency(t *testing.T) { if totalVirtual.got != totalVirtual.want { t.Errorf(`"/memory/classes/total:bytes" does not match sum of /memory/classes/**: got %d, want %d`, totalVirtual.got, totalVirtual.want) } - if objects.alloc.Counts[0] > 0 { - t.Error("found counts for objects of non-positive size in allocs-by-size") + if b, c := len(objects.alloc.Buckets), len(objects.alloc.Counts); b != c+1 { + t.Errorf("allocs-by-size has wrong bucket or counts length: %d buckets, %d counts", b, c) } - if objects.free.Counts[0] > 0 { - t.Error("found counts for objects of non-positive size in frees-by-size") + if b, c := len(objects.free.Buckets), len(objects.free.Counts); b != c+1 { + t.Errorf("frees-by-size has wrong bucket or counts length: %d buckets, %d counts", b, c) } if len(objects.alloc.Buckets) != len(objects.free.Buckets) { t.Error("allocs-by-size and frees-by-size buckets don't match in length") diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 52267e6fb0..46fae5de72 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -132,7 +132,6 @@ fail: println("gp", gp, "goid", gp.goid, "status", readgstatus(gp), "gcscandone", gp.gcscandone) - unlock(&allglock) // Avoid self-deadlock with traceback. throw("scan missed a g") } diff --git a/src/runtime/mgcscavenge.go b/src/runtime/mgcscavenge.go index 38f09309dc..a7c5bc49b8 100644 --- a/src/runtime/mgcscavenge.go +++ b/src/runtime/mgcscavenge.go @@ -562,7 +562,7 @@ func (p *pageAlloc) scavengeUnreserve(r addrRange, gen uint32) { func (p *pageAlloc) scavengeOne(work addrRange, max uintptr, mayUnlock bool) (uintptr, addrRange) { assertLockHeld(p.mheapLock) - // Defensively check if we've recieved an empty address range. + // Defensively check if we've received an empty address range. // If so, just return. if work.size() == 0 { // Nothing to do. diff --git a/src/runtime/mmap.go b/src/runtime/mmap.go index 9fe31cb416..1b1848b79e 100644 --- a/src/runtime/mmap.go +++ b/src/runtime/mmap.go @@ -2,14 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build !aix +// +build !darwin +// +build !js +// +build !linux !amd64 +// +build !linux !arm64 +// +build !openbsd // +build !plan9 // +build !solaris // +build !windows -// +build !linux !amd64 -// +build !linux !arm64 -// +build !js -// +build !darwin -// +build !aix package runtime diff --git a/src/runtime/os3_solaris.go b/src/runtime/os3_solaris.go index d6e36fbfbb..6ba11afd93 100644 --- a/src/runtime/os3_solaris.go +++ b/src/runtime/os3_solaris.go @@ -227,6 +227,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + func sigtramp() //go:nosplit diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go index 0c501be96a..303f0876de 100644 --- a/src/runtime/os_aix.go +++ b/src/runtime/os_aix.go @@ -180,6 +180,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + // tstart is a function descriptor to _tstart defined in assembly. var tstart funcDescriptor diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index e0a43c28aa..9ca17c20df 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -325,6 +325,11 @@ func unminit() { } } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + //go:nosplit func osyield() { usleep(1) diff --git a/src/runtime/os_dragonfly.go b/src/runtime/os_dragonfly.go index 6578fcbeb1..383df54bd4 100644 --- a/src/runtime/os_dragonfly.go +++ b/src/runtime/os_dragonfly.go @@ -203,6 +203,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + func sigtramp() type sigactiont struct { diff --git a/src/runtime/os_freebsd.go b/src/runtime/os_freebsd.go index 1c60ee2a57..09065ccb68 100644 --- a/src/runtime/os_freebsd.go +++ b/src/runtime/os_freebsd.go @@ -319,6 +319,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + func sigtramp() type sigactiont struct { diff --git a/src/runtime/os_js.go b/src/runtime/os_js.go index 91d18a078f..24261e88a2 100644 --- a/src/runtime/os_js.go +++ b/src/runtime/os_js.go @@ -84,6 +84,11 @@ func minit() { func unminit() { } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + func osinit() { ncpu = 1 getg().m.procid = 2 diff --git a/src/runtime/os_linux.go b/src/runtime/os_linux.go index f122d2c2ef..058c7daf9c 100644 --- a/src/runtime/os_linux.go +++ b/src/runtime/os_linux.go @@ -375,6 +375,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + //#ifdef GOARCH_386 //#define sa_handler k_sa_handler //#endif diff --git a/src/runtime/os_netbsd.go b/src/runtime/os_netbsd.go index f7f90cedc1..2b742a3711 100644 --- a/src/runtime/os_netbsd.go +++ b/src/runtime/os_netbsd.go @@ -290,6 +290,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + func sigtramp() type sigactiont struct { diff --git a/src/runtime/os_openbsd.go b/src/runtime/os_openbsd.go index d7960f4c91..6259b96c22 100644 --- a/src/runtime/os_openbsd.go +++ b/src/runtime/os_openbsd.go @@ -6,7 +6,6 @@ package runtime import ( "runtime/internal/atomic" - "runtime/internal/sys" "unsafe" ) @@ -14,60 +13,6 @@ type mOS struct { waitsemacount uint32 } -//go:noescape -func setitimer(mode int32, new, old *itimerval) - -//go:noescape -func sigaction(sig uint32, new, old *sigactiont) - -//go:noescape -func sigaltstack(new, old *stackt) - -//go:noescape -func obsdsigprocmask(how int32, new sigset) sigset - -//go:nosplit -//go:nowritebarrierrec -func sigprocmask(how int32, new, old *sigset) { - n := sigset(0) - if new != nil { - n = *new - } - r := obsdsigprocmask(how, n) - if old != nil { - *old = r - } -} - -//go:noescape -func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 - -func raiseproc(sig uint32) - -func getthrid() int32 -func thrkill(tid int32, sig int) - -//go:noescape -func tfork(param *tforkt, psize uintptr, mm *m, gg *g, fn uintptr) int32 - -//go:noescape -func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 - -//go:noescape -func thrwakeup(ident uintptr, n int32) int32 - -func osyield() - -func kqueue() int32 - -//go:noescape -func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 - -func pipe() (r, w int32, errno int32) -func pipe2(flags int32) (r, w int32, errno int32) -func closeonexec(fd int32) -func setNonblock(fd int32) - const ( _ESRCH = 3 _EWOULDBLOCK = _EAGAIN @@ -183,36 +128,6 @@ func semawakeup(mp *m) { } } -// May run with m.p==nil, so write barriers are not allowed. -//go:nowritebarrier -func newosproc(mp *m) { - stk := unsafe.Pointer(mp.g0.stack.hi) - if false { - print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n") - } - - // Stack pointer must point inside stack area (as marked with MAP_STACK), - // rather than at the top of it. - param := tforkt{ - tf_tcb: unsafe.Pointer(&mp.tls[0]), - tf_tid: nil, // minit will record tid - tf_stack: uintptr(stk) - sys.PtrSize, - } - - var oset sigset - sigprocmask(_SIG_SETMASK, &sigset_all, &oset) - ret := tfork(¶m, unsafe.Sizeof(param), mp, mp.g0, funcPC(mstart)) - sigprocmask(_SIG_SETMASK, &oset, nil) - - if ret < 0 { - print("runtime: failed to create new OS thread (have ", mcount()-1, " already; errno=", -ret, ")\n") - if ret == -_EAGAIN { - println("runtime: may need to increase max user processes (ulimit -p)") - } - throw("runtime.newosproc") - } -} - func osinit() { ncpu = getncpu() physPageSize = getPageSize() @@ -257,6 +172,11 @@ func unminit() { unminitSignals() } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + func sigtramp() type sigactiont struct { diff --git a/src/runtime/os_openbsd_libc.go b/src/runtime/os_openbsd_libc.go new file mode 100644 index 0000000000..2edb0358b0 --- /dev/null +++ b/src/runtime/os_openbsd_libc.go @@ -0,0 +1,58 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,amd64 openbsd,arm64 + +package runtime + +import ( + "unsafe" +) + +var failThreadCreate = []byte("runtime: failed to create new OS thread\n") + +// mstart_stub provides glue code to call mstart from pthread_create. +func mstart_stub() + +// May run with m.p==nil, so write barriers are not allowed. +//go:nowritebarrierrec +func newosproc(mp *m) { + if false { + print("newosproc m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n") + } + + // Initialize an attribute object. + var attr pthreadattr + if err := pthread_attr_init(&attr); err != 0 { + write(2, unsafe.Pointer(&failThreadCreate[0]), int32(len(failThreadCreate))) + exit(1) + } + + // Find out OS stack size for our own stack guard. + var stacksize uintptr + if pthread_attr_getstacksize(&attr, &stacksize) != 0 { + write(2, unsafe.Pointer(&failThreadCreate[0]), int32(len(failThreadCreate))) + exit(1) + } + mp.g0.stack.hi = stacksize // for mstart + + // Tell the pthread library we won't join with this thread. + if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { + write(2, unsafe.Pointer(&failThreadCreate[0]), int32(len(failThreadCreate))) + exit(1) + } + + // Finally, create the thread. It starts at mstart_stub, which does some low-level + // setup and then calls mstart. + var oset sigset + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + err := pthread_create(&attr, funcPC(mstart_stub), unsafe.Pointer(mp)) + sigprocmask(_SIG_SETMASK, &oset, nil) + if err != 0 { + write(2, unsafe.Pointer(&failThreadCreate[0]), int32(len(failThreadCreate))) + exit(1) + } + + pthread_attr_destroy(&attr) +} diff --git a/src/runtime/os_openbsd_syscall.go b/src/runtime/os_openbsd_syscall.go new file mode 100644 index 0000000000..16ff2b8e25 --- /dev/null +++ b/src/runtime/os_openbsd_syscall.go @@ -0,0 +1,46 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,!amd64 +// +build openbsd,!arm64 + +package runtime + +import ( + "runtime/internal/sys" + "unsafe" +) + +//go:noescape +func tfork(param *tforkt, psize uintptr, mm *m, gg *g, fn uintptr) int32 + +// May run with m.p==nil, so write barriers are not allowed. +//go:nowritebarrier +func newosproc(mp *m) { + stk := unsafe.Pointer(mp.g0.stack.hi) + if false { + print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, " ostk=", &mp, "\n") + } + + // Stack pointer must point inside stack area (as marked with MAP_STACK), + // rather than at the top of it. + param := tforkt{ + tf_tcb: unsafe.Pointer(&mp.tls[0]), + tf_tid: nil, // minit will record tid + tf_stack: uintptr(stk) - sys.PtrSize, + } + + var oset sigset + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + ret := tfork(¶m, unsafe.Sizeof(param), mp, mp.g0, funcPC(mstart)) + sigprocmask(_SIG_SETMASK, &oset, nil) + + if ret < 0 { + print("runtime: failed to create new OS thread (have ", mcount()-1, " already; errno=", -ret, ")\n") + if ret == -_EAGAIN { + println("runtime: may need to increase max user processes (ulimit -p)") + } + throw("runtime.newosproc") + } +} diff --git a/src/runtime/os_openbsd_syscall1.go b/src/runtime/os_openbsd_syscall1.go new file mode 100644 index 0000000000..08928cfef4 --- /dev/null +++ b/src/runtime/os_openbsd_syscall1.go @@ -0,0 +1,15 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,!amd64 + +package runtime + +//go:noescape +func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 + +//go:noescape +func thrwakeup(ident uintptr, n int32) int32 + +func osyield() diff --git a/src/runtime/os_openbsd_syscall2.go b/src/runtime/os_openbsd_syscall2.go new file mode 100644 index 0000000000..74eb271c2c --- /dev/null +++ b/src/runtime/os_openbsd_syscall2.go @@ -0,0 +1,95 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,!amd64 + +package runtime + +import ( + "unsafe" +) + +//go:noescape +func sigaction(sig uint32, new, old *sigactiont) + +func kqueue() int32 + +//go:noescape +func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 + +func raiseproc(sig uint32) + +func getthrid() int32 +func thrkill(tid int32, sig int) + +// read calls the read system call. +// It returns a non-negative number of bytes written or a negative errno value. +func read(fd int32, p unsafe.Pointer, n int32) int32 + +func closefd(fd int32) int32 + +func exit(code int32) +func usleep(usec uint32) + +// write calls the write system call. +// It returns a non-negative number of bytes written or a negative errno value. +//go:noescape +func write1(fd uintptr, p unsafe.Pointer, n int32) int32 + +//go:noescape +func open(name *byte, mode, perm int32) int32 + +// return value is only set on linux to be used in osinit() +func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 + +// exitThread terminates the current thread, writing *wait = 0 when +// the stack is safe to reclaim. +// +//go:noescape +func exitThread(wait *uint32) + +//go:noescape +func obsdsigprocmask(how int32, new sigset) sigset + +//go:nosplit +//go:nowritebarrierrec +func sigprocmask(how int32, new, old *sigset) { + n := sigset(0) + if new != nil { + n = *new + } + r := obsdsigprocmask(how, n) + if old != nil { + *old = r + } +} + +func pipe() (r, w int32, errno int32) +func pipe2(flags int32) (r, w int32, errno int32) + +//go:noescape +func setitimer(mode int32, new, old *itimerval) + +//go:noescape +func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 + +// mmap calls the mmap system call. It is implemented in assembly. +// We only pass the lower 32 bits of file offset to the +// assembly routine; the higher bits (if required), should be provided +// by the assembly routine as 0. +// The err result is an OS error code such as ENOMEM. +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int) + +// munmap calls the munmap system call. It is implemented in assembly. +func munmap(addr unsafe.Pointer, n uintptr) + +func nanotime1() int64 + +//go:noescape +func sigaltstack(new, old *stackt) + +func closeonexec(fd int32) +func setNonblock(fd int32) + +func walltime1() (sec int64, nsec int32) diff --git a/src/runtime/os_plan9.go b/src/runtime/os_plan9.go index a035526937..2a84a73716 100644 --- a/src/runtime/os_plan9.go +++ b/src/runtime/os_plan9.go @@ -213,6 +213,11 @@ func minit() { func unminit() { } +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +func mdestroy(mp *m) { +} + var sysstat = []byte("/dev/sysstat\x00") func getproccount() int32 { diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index d389d38ab9..83d0d63e5d 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -893,22 +893,23 @@ func sigblock(exiting bool) { // Called on the new thread, cannot allocate memory. func minit() { var thandle uintptr - stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) + if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { + print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n") + throw("runtime.minit: duplicatehandle failed") + } + + mp := getg().m + lock(&mp.threadLock) + mp.thread = thandle // Configure usleep timer, if possible. - var timer uintptr - if haveHighResTimer { - timer = createHighResTimer() - if timer == 0 { + if mp.highResTimer == 0 && haveHighResTimer { + mp.highResTimer = createHighResTimer() + if mp.highResTimer == 0 { print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") throw("CreateWaitableTimerEx when creating timer failed") } } - - mp := getg().m - lock(&mp.threadLock) - mp.thread = thandle - mp.highResTimer = timer unlock(&mp.threadLock) // Query the true stack base from the OS. Currently we're @@ -944,13 +945,29 @@ func minit() { func unminit() { mp := getg().m lock(&mp.threadLock) - stdcall1(_CloseHandle, mp.thread) - mp.thread = 0 + if mp.thread != 0 { + stdcall1(_CloseHandle, mp.thread) + mp.thread = 0 + } + unlock(&mp.threadLock) +} + +// Called from exitm, but not from drop, to undo the effect of thread-owned +// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. +//go:nosplit +func mdestroy(mp *m) { if mp.highResTimer != 0 { stdcall1(_CloseHandle, mp.highResTimer) mp.highResTimer = 0 } - unlock(&mp.threadLock) + if mp.waitsema != 0 { + stdcall1(_CloseHandle, mp.waitsema) + mp.waitsema = 0 + } + if mp.resumesema != 0 { + stdcall1(_CloseHandle, mp.resumesema) + mp.resumesema = 0 + } } // Calling stdcall on os stack. @@ -1134,8 +1151,12 @@ func profileloop1(param uintptr) uint32 { } // Acquire our own handle to the thread. var thread uintptr - stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) + if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { + print("runtime.profileloop1: duplicatehandle failed; errno=", getlasterror(), "\n") + throw("runtime.profileloop1: duplicatehandle failed") + } unlock(&mp.threadLock) + // mp may exit between the DuplicateHandle // above and the SuspendThread. The handle // will remain valid, but SuspendThread may @@ -1214,7 +1235,10 @@ func preemptM(mp *m) { return } var thread uintptr - stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) + if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { + print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n") + throw("runtime.preemptM: duplicatehandle failed") + } unlock(&mp.threadLock) // Prepare thread context buffer. This must be aligned to 16 bytes. diff --git a/src/runtime/proc.go b/src/runtime/proc.go index ca78587aad..aa44c625c5 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -490,8 +490,29 @@ func lockedOSThread() bool { } var ( - allgs []*g + // allgs contains all Gs ever created (including dead Gs), and thus + // never shrinks. + // + // Access via the slice is protected by allglock or stop-the-world. + // Readers that cannot take the lock may (carefully!) use the atomic + // variables below. allglock mutex + allgs []*g + + // allglen and allgptr are atomic variables that contain len(allg) and + // &allg[0] respectively. Proper ordering depends on totally-ordered + // loads and stores. Writes are protected by allglock. + // + // allgptr is updated before allglen. Readers should read allglen + // before allgptr to ensure that allglen is always <= len(allgptr). New + // Gs appended during the race can be missed. For a consistent view of + // all Gs, allglock must be held. + // + // allgptr copies should always be stored as a concrete type or + // unsafe.Pointer, not uintptr, to ensure that GC can still reach it + // even if it points to a stale array. + allglen uintptr + allgptr **g ) func allgadd(gp *g) { @@ -501,10 +522,25 @@ func allgadd(gp *g) { lock(&allglock) allgs = append(allgs, gp) - allglen = uintptr(len(allgs)) + if &allgs[0] != allgptr { + atomicstorep(unsafe.Pointer(&allgptr), unsafe.Pointer(&allgs[0])) + } + atomic.Storeuintptr(&allglen, uintptr(len(allgs))) unlock(&allglock) } +// atomicAllG returns &allgs[0] and len(allgs) for use with atomicAllGIndex. +func atomicAllG() (**g, uintptr) { + length := atomic.Loaduintptr(&allglen) + ptr := (**g)(atomic.Loadp(unsafe.Pointer(&allgptr))) + return ptr, length +} + +// atomicAllGIndex returns ptr[i] with the allgptr returned from atomicAllG. +func atomicAllGIndex(ptr **g, i uintptr) *g { + return *(**g)(add(unsafe.Pointer(ptr), i*sys.PtrSize)) +} + const ( // Number of goroutine ids to grab from sched.goidgen to local per-P cache at once. // 16 seems to provide enough amortization, but other than that it's mostly arbitrary number. @@ -1170,6 +1206,33 @@ func startTheWorldWithSema(emitTraceEvent bool) int64 { return startTime } +// usesLibcall indicates whether this runtime performs system calls +// via libcall. +func usesLibcall() bool { + switch GOOS { + case "aix", "darwin", "illumos", "ios", "solaris", "windows": + return true + case "openbsd": + return GOARCH == "amd64" + } + return false +} + +// mStackIsSystemAllocated indicates whether this runtime starts on a +// system-allocated stack. +func mStackIsSystemAllocated() bool { + switch GOOS { + case "aix", "darwin", "plan9", "illumos", "ios", "solaris", "windows": + return true + case "openbsd": + switch GOARCH { + case "amd64", "arm64": + return true + } + } + return false +} + // mstart is the entry-point for new Ms. // // This must not split the stack because we may not even have stack @@ -1204,8 +1267,7 @@ func mstart() { mstart1() // Exit this thread. - switch GOOS { - case "windows", "solaris", "illumos", "plan9", "darwin", "ios", "aix": + if mStackIsSystemAllocated() { // Windows, Solaris, illumos, Darwin, AIX and Plan 9 always system-allocate // the stack, but put it in _g_.stack before mstart, // so the logic above hasn't set osStack yet. @@ -1371,6 +1433,10 @@ found: } } + // Destroy all allocated resources. After this is called, we may no + // longer take any locks. + mdestroy(m) + if osStack { // Return from mstart and let the system thread // library free the g0 stack and terminate the thread. @@ -1684,7 +1750,7 @@ func allocm(_p_ *p, fn func(), id int64) *m { // In case of cgo or Solaris or illumos or Darwin, pthread_create will make us a stack. // Windows and Plan 9 will layout sched stack on OS stack. - if iscgo || GOOS == "solaris" || GOOS == "illumos" || GOOS == "windows" || GOOS == "plan9" || GOOS == "darwin" || GOOS == "ios" { + if iscgo || mStackIsSystemAllocated() { mp.g0 = malg(-1) } else { mp.g0 = malg(8192 * sys.StackGuardMultiplier) @@ -4266,7 +4332,7 @@ func badunlockosthread() { } func gcount() int32 { - n := int32(allglen) - sched.gFree.n - int32(atomic.Load(&sched.ngsys)) + n := int32(atomic.Loaduintptr(&allglen)) - sched.gFree.n - int32(atomic.Load(&sched.ngsys)) for _, _p_ := range allp { n -= _p_.gFree.n } @@ -4432,7 +4498,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { // Normal traceback is impossible or has failed. // See if it falls into several common cases. n = 0 - if (GOOS == "windows" || GOOS == "solaris" || GOOS == "illumos" || GOOS == "darwin" || GOOS == "ios" || GOOS == "aix") && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 { + if usesLibcall() && mp.libcallg != 0 && mp.libcallpc != 0 && mp.libcallsp != 0 { // Libcall, i.e. runtime syscall on windows. // Collect Go stack that leads to the call. n = gentraceback(mp.libcallpc, mp.libcallsp, 0, mp.libcallg.ptr(), 0, &stk[0], len(stk), nil, nil, 0) @@ -4970,7 +5036,6 @@ func checkdead() { case _Grunnable, _Grunning, _Gsyscall: - unlock(&allglock) print("runtime: checkdead: find g ", gp.goid, " in status ", s, "\n") throw("checkdead: runnable g") } diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 9c3ceabd18..b7c7b4cff2 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -1052,7 +1052,6 @@ func (w waitReason) String() string { } var ( - allglen uintptr allm *m gomaxprocs int32 ncpu int32 diff --git a/src/runtime/signal_openbsd.go b/src/runtime/signal_openbsd.go index 99c601ce58..d2c5c5e39a 100644 --- a/src/runtime/signal_openbsd.go +++ b/src/runtime/signal_openbsd.go @@ -37,5 +37,5 @@ var sigtable = [...]sigTabT{ /* 29 */ {_SigNotify, "SIGINFO: status request from keyboard"}, /* 30 */ {_SigNotify, "SIGUSR1: user-defined signal 1"}, /* 31 */ {_SigNotify, "SIGUSR2: user-defined signal 2"}, - /* 32 */ {_SigNotify, "SIGTHR: reserved"}, + /* 32 */ {0, "SIGTHR: reserved"}, // thread AST - cannot be registered. } diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index 4a1a5cc3d9..85088b3ab9 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -2,12 +2,13 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build !aix +// +build !darwin +// +build !js +// +build !openbsd // +build !plan9 // +build !solaris // +build !windows -// +build !js -// +build !darwin -// +build !aix package runtime diff --git a/src/runtime/stubs3.go b/src/runtime/stubs3.go index 95eecc7eca..1885d32051 100644 --- a/src/runtime/stubs3.go +++ b/src/runtime/stubs3.go @@ -2,11 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build !aix +// +build !darwin +// +build !freebsd +// +build !openbsd // +build !plan9 // +build !solaris -// +build !freebsd -// +build !darwin -// +build !aix package runtime diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 55845bf2e5..4a3f2fc453 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -6,50 +6,6 @@ package runtime import "unsafe" -// Call fn with arg as its argument. Return what fn returns. -// fn is the raw pc value of the entry point of the desired function. -// Switches to the system stack, if not already there. -// Preserves the calling point as the location where a profiler traceback will begin. -//go:nosplit -func libcCall(fn, arg unsafe.Pointer) int32 { - // Leave caller's PC/SP/G around for traceback. - gp := getg() - var mp *m - if gp != nil { - mp = gp.m - } - if mp != nil && mp.libcallsp == 0 { - mp.libcallg.set(gp) - mp.libcallpc = getcallerpc() - // sp must be the last, because once async cpu profiler finds - // all three values to be non-zero, it will use them - mp.libcallsp = getcallersp() - } else { - // Make sure we don't reset libcallsp. This makes - // libcCall reentrant; We remember the g/pc/sp for the - // first call on an M, until that libcCall instance - // returns. Reentrance only matters for signals, as - // libc never calls back into Go. The tricky case is - // where we call libcX from an M and record g/pc/sp. - // Before that call returns, a signal arrives on the - // same M and the signal handling code calls another - // libc function. We don't want that second libcCall - // from within the handler to be recorded, and we - // don't want that call's completion to zero - // libcallsp. - // We don't need to set libcall* while we're in a sighandler - // (even if we're not currently in libc) because we block all - // signals while we're handling a signal. That includes the - // profile signal, which is the one that uses the libcall* info. - mp = nil - } - res := asmcgocall(fn, arg) - if mp != nil { - mp.libcallsp = 0 - } - return res -} - // The X versions of syscall expect the libc call to return a 64-bit result. // Otherwise (the non-X version) expects a 32-bit result. // This distinction is required because an error is indicated by returning -1, diff --git a/src/runtime/sys_libc.go b/src/runtime/sys_libc.go new file mode 100644 index 0000000000..996c032105 --- /dev/null +++ b/src/runtime/sys_libc.go @@ -0,0 +1,53 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin openbsd,amd64 openbsd,arm64 + +package runtime + +import "unsafe" + +// Call fn with arg as its argument. Return what fn returns. +// fn is the raw pc value of the entry point of the desired function. +// Switches to the system stack, if not already there. +// Preserves the calling point as the location where a profiler traceback will begin. +//go:nosplit +func libcCall(fn, arg unsafe.Pointer) int32 { + // Leave caller's PC/SP/G around for traceback. + gp := getg() + var mp *m + if gp != nil { + mp = gp.m + } + if mp != nil && mp.libcallsp == 0 { + mp.libcallg.set(gp) + mp.libcallpc = getcallerpc() + // sp must be the last, because once async cpu profiler finds + // all three values to be non-zero, it will use them + mp.libcallsp = getcallersp() + } else { + // Make sure we don't reset libcallsp. This makes + // libcCall reentrant; We remember the g/pc/sp for the + // first call on an M, until that libcCall instance + // returns. Reentrance only matters for signals, as + // libc never calls back into Go. The tricky case is + // where we call libcX from an M and record g/pc/sp. + // Before that call returns, a signal arrives on the + // same M and the signal handling code calls another + // libc function. We don't want that second libcCall + // from within the handler to be recorded, and we + // don't want that call's completion to zero + // libcallsp. + // We don't need to set libcall* while we're in a sighandler + // (even if we're not currently in libc) because we block all + // signals while we're handling a signal. That includes the + // profile signal, which is the one that uses the libcall* info. + mp = nil + } + res := asmcgocall(fn, arg) + if mp != nil { + mp.libcallsp = 0 + } + return res +} diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index afad056d06..c3e9f37694 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -250,6 +250,14 @@ noswitch: BEQ R25, fallback JAL (R25) + // check on vdso call return for kernel compatibility + // see https://golang.org/issues/39046 + // if we get any error make fallback permanent. + BEQ R2, R0, finish + MOVV R0, runtime·vdsoClockgettimeSym(SB) + MOVW $0, R4 // CLOCK_REALTIME + MOVV $0(R29), R5 + JMP fallback finish: MOVV 0(R29), R3 // sec @@ -311,6 +319,12 @@ noswitch: BEQ R25, fallback JAL (R25) + // see walltime1 for detail + BEQ R2, R0, finish + MOVV R0, runtime·vdsoClockgettimeSym(SB) + MOVW $1, R4 // CLOCK_MONOTONIC + MOVV $0(R29), R5 + JMP fallback finish: MOVV 0(R29), R3 // sec diff --git a/src/runtime/sys_openbsd.go b/src/runtime/sys_openbsd.go new file mode 100644 index 0000000000..2d41ed0d46 --- /dev/null +++ b/src/runtime/sys_openbsd.go @@ -0,0 +1,59 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,amd64 openbsd,arm64 + +package runtime + +import "unsafe" + +// The *_trampoline functions convert from the Go calling convention to the C calling convention +// and then call the underlying libc function. These are defined in sys_openbsd_$ARCH.s. + +//go:nosplit +//go:cgo_unsafe_args +func pthread_attr_init(attr *pthreadattr) int32 { + return libcCall(unsafe.Pointer(funcPC(pthread_attr_init_trampoline)), unsafe.Pointer(&attr)) +} +func pthread_attr_init_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func pthread_attr_destroy(attr *pthreadattr) int32 { + return libcCall(unsafe.Pointer(funcPC(pthread_attr_destroy_trampoline)), unsafe.Pointer(&attr)) +} +func pthread_attr_destroy_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func pthread_attr_getstacksize(attr *pthreadattr, size *uintptr) int32 { + return libcCall(unsafe.Pointer(funcPC(pthread_attr_getstacksize_trampoline)), unsafe.Pointer(&attr)) +} +func pthread_attr_getstacksize_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func pthread_attr_setdetachstate(attr *pthreadattr, state int) int32 { + return libcCall(unsafe.Pointer(funcPC(pthread_attr_setdetachstate_trampoline)), unsafe.Pointer(&attr)) +} +func pthread_attr_setdetachstate_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func pthread_create(attr *pthreadattr, start uintptr, arg unsafe.Pointer) int32 { + return libcCall(unsafe.Pointer(funcPC(pthread_create_trampoline)), unsafe.Pointer(&attr)) +} +func pthread_create_trampoline() + +// Tell the linker that the libc_* functions are to be found +// in a system library, with the libc_ prefix missing. + +//go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "libpthread.so" +//go:cgo_import_dynamic libc_pthread_attr_destroy pthread_attr_destroy "libpthread.so" +//go:cgo_import_dynamic libc_pthread_attr_getstacksize pthread_attr_getstacksize "libpthread.so" +//go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "libpthread.so" +//go:cgo_import_dynamic libc_pthread_create pthread_create "libpthread.so" +//go:cgo_import_dynamic libc_pthread_sigmask pthread_sigmask "libpthread.so" + +//go:cgo_import_dynamic _ _ "libpthread.so" diff --git a/src/runtime/sys_openbsd1.go b/src/runtime/sys_openbsd1.go new file mode 100644 index 0000000000..a201a16c53 --- /dev/null +++ b/src/runtime/sys_openbsd1.go @@ -0,0 +1,34 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,amd64 + +package runtime + +import "unsafe" + +//go:nosplit +//go:cgo_unsafe_args +func thrsleep(ident uintptr, clock_id int32, tsp *timespec, lock uintptr, abort *uint32) int32 { + return libcCall(unsafe.Pointer(funcPC(thrsleep_trampoline)), unsafe.Pointer(&ident)) +} +func thrsleep_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func thrwakeup(ident uintptr, n int32) int32 { + return libcCall(unsafe.Pointer(funcPC(thrwakeup_trampoline)), unsafe.Pointer(&ident)) +} +func thrwakeup_trampoline() + +func osyield() { + libcCall(unsafe.Pointer(funcPC(sched_yield_trampoline)), unsafe.Pointer(nil)) +} +func sched_yield_trampoline() + +//go:cgo_import_dynamic libc_thrsleep __thrsleep "libc.so" +//go:cgo_import_dynamic libc_thrwakeup __thrwakeup "libc.so" +//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" + +//go:cgo_import_dynamic _ _ "libc.so" diff --git a/src/runtime/sys_openbsd2.go b/src/runtime/sys_openbsd2.go new file mode 100644 index 0000000000..73592df226 --- /dev/null +++ b/src/runtime/sys_openbsd2.go @@ -0,0 +1,250 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build openbsd,amd64 + +package runtime + +import "unsafe" + +// This is exported via linkname to assembly in runtime/cgo. +//go:linkname exit +//go:nosplit +//go:cgo_unsafe_args +func exit(code int32) { + libcCall(unsafe.Pointer(funcPC(exit_trampoline)), unsafe.Pointer(&code)) +} +func exit_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func getthrid() (tid int32) { + libcCall(unsafe.Pointer(funcPC(getthrid_trampoline)), unsafe.Pointer(&tid)) + return +} +func getthrid_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func raiseproc(sig uint32) { + libcCall(unsafe.Pointer(funcPC(raiseproc_trampoline)), unsafe.Pointer(&sig)) +} +func raiseproc_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func thrkill(tid int32, sig int) { + libcCall(unsafe.Pointer(funcPC(thrkill_trampoline)), unsafe.Pointer(&tid)) +} +func thrkill_trampoline() + +// mmap is used to do low-level memory allocation via mmap. Don't allow stack +// splits, since this function (used by sysAlloc) is called in a lot of low-level +// parts of the runtime and callers often assume it won't acquire any locks. +// go:nosplit +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) { + args := struct { + addr unsafe.Pointer + n uintptr + prot, flags, fd int32 + off uint32 + ret1 unsafe.Pointer + ret2 int + }{addr, n, prot, flags, fd, off, nil, 0} + libcCall(unsafe.Pointer(funcPC(mmap_trampoline)), unsafe.Pointer(&args)) + return args.ret1, args.ret2 +} +func mmap_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func munmap(addr unsafe.Pointer, n uintptr) { + libcCall(unsafe.Pointer(funcPC(munmap_trampoline)), unsafe.Pointer(&addr)) +} +func munmap_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func madvise(addr unsafe.Pointer, n uintptr, flags int32) { + libcCall(unsafe.Pointer(funcPC(madvise_trampoline)), unsafe.Pointer(&addr)) +} +func madvise_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func open(name *byte, mode, perm int32) (ret int32) { + return libcCall(unsafe.Pointer(funcPC(open_trampoline)), unsafe.Pointer(&name)) +} +func open_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func closefd(fd int32) int32 { + return libcCall(unsafe.Pointer(funcPC(close_trampoline)), unsafe.Pointer(&fd)) +} +func close_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func read(fd int32, p unsafe.Pointer, n int32) int32 { + return libcCall(unsafe.Pointer(funcPC(read_trampoline)), unsafe.Pointer(&fd)) +} +func read_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func write1(fd uintptr, p unsafe.Pointer, n int32) int32 { + return libcCall(unsafe.Pointer(funcPC(write_trampoline)), unsafe.Pointer(&fd)) +} +func write_trampoline() + +func pipe() (r, w int32, errno int32) { + return pipe2(0) +} + +func pipe2(flags int32) (r, w int32, errno int32) { + var p [2]int32 + args := struct { + p unsafe.Pointer + flags int32 + }{noescape(unsafe.Pointer(&p)), flags} + errno = libcCall(unsafe.Pointer(funcPC(pipe2_trampoline)), unsafe.Pointer(&args)) + return p[0], p[1], errno +} +func pipe2_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func setitimer(mode int32, new, old *itimerval) { + libcCall(unsafe.Pointer(funcPC(setitimer_trampoline)), unsafe.Pointer(&mode)) +} +func setitimer_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func usleep(usec uint32) { + libcCall(unsafe.Pointer(funcPC(usleep_trampoline)), unsafe.Pointer(&usec)) +} +func usleep_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sysctl(mib *uint32, miblen uint32, out *byte, size *uintptr, dst *byte, ndst uintptr) int32 { + return libcCall(unsafe.Pointer(funcPC(sysctl_trampoline)), unsafe.Pointer(&mib)) +} +func sysctl_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func fcntl(fd, cmd, arg int32) int32 { + return libcCall(unsafe.Pointer(funcPC(fcntl_trampoline)), unsafe.Pointer(&fd)) +} +func fcntl_trampoline() + +//go:nosplit +func nanotime1() int64 { + var ts timespec + args := struct { + clock_id int32 + tp unsafe.Pointer + }{_CLOCK_MONOTONIC, unsafe.Pointer(&ts)} + libcCall(unsafe.Pointer(funcPC(clock_gettime_trampoline)), unsafe.Pointer(&args)) + return ts.tv_sec*1e9 + int64(ts.tv_nsec) +} +func clock_gettime_trampoline() + +//go:nosplit +func walltime1() (int64, int32) { + var ts timespec + args := struct { + clock_id int32 + tp unsafe.Pointer + }{_CLOCK_REALTIME, unsafe.Pointer(&ts)} + libcCall(unsafe.Pointer(funcPC(clock_gettime_trampoline)), unsafe.Pointer(&args)) + return ts.tv_sec, int32(ts.tv_nsec) +} + +//go:nosplit +//go:cgo_unsafe_args +func kqueue() int32 { + return libcCall(unsafe.Pointer(funcPC(kqueue_trampoline)), nil) +} +func kqueue_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func kevent(kq int32, ch *keventt, nch int32, ev *keventt, nev int32, ts *timespec) int32 { + return libcCall(unsafe.Pointer(funcPC(kevent_trampoline)), unsafe.Pointer(&kq)) +} +func kevent_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sigaction(sig uint32, new *sigactiont, old *sigactiont) { + libcCall(unsafe.Pointer(funcPC(sigaction_trampoline)), unsafe.Pointer(&sig)) +} +func sigaction_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sigprocmask(how uint32, new *sigset, old *sigset) { + libcCall(unsafe.Pointer(funcPC(sigprocmask_trampoline)), unsafe.Pointer(&how)) +} +func sigprocmask_trampoline() + +//go:nosplit +//go:cgo_unsafe_args +func sigaltstack(new *stackt, old *stackt) { + libcCall(unsafe.Pointer(funcPC(sigaltstack_trampoline)), unsafe.Pointer(&new)) +} +func sigaltstack_trampoline() + +// Not used on OpenBSD, but must be defined. +func exitThread(wait *uint32) { +} + +//go:nosplit +func closeonexec(fd int32) { + fcntl(fd, _F_SETFD, _FD_CLOEXEC) +} + +//go:nosplit +func setNonblock(fd int32) { + flags := fcntl(fd, _F_GETFL, 0) + fcntl(fd, _F_SETFL, flags|_O_NONBLOCK) +} + +// Tell the linker that the libc_* functions are to be found +// in a system library, with the libc_ prefix missing. + +//go:cgo_import_dynamic libc_errno __errno "libc.so" +//go:cgo_import_dynamic libc_exit exit "libc.so" +//go:cgo_import_dynamic libc_getthrid getthrid "libc.so" +//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so" +//go:cgo_import_dynamic libc_thrkill thrkill "libc.so" + +//go:cgo_import_dynamic libc_mmap mmap "libc.so" +//go:cgo_import_dynamic libc_munmap munmap "libc.so" +//go:cgo_import_dynamic libc_madvise madvise "libc.so" + +//go:cgo_import_dynamic libc_open open "libc.so" +//go:cgo_import_dynamic libc_close close "libc.so" +//go:cgo_import_dynamic libc_read read "libc.so" +//go:cgo_import_dynamic libc_write write "libc.so" +//go:cgo_import_dynamic libc_pipe2 pipe2 "libc.so" + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" +//go:cgo_import_dynamic libc_setitimer setitimer "libc.so" +//go:cgo_import_dynamic libc_usleep usleep "libc.so" +//go:cgo_import_dynamic libc_sysctl sysctl "libc.so" +//go:cgo_import_dynamic libc_fcntl fcntl "libc.so" +//go:cgo_import_dynamic libc_getpid getpid "libc.so" +//go:cgo_import_dynamic libc_kill kill "libc.so" +//go:cgo_import_dynamic libc_kqueue kqueue "libc.so" +//go:cgo_import_dynamic libc_kevent kevent "libc.so" + +//go:cgo_import_dynamic libc_sigaction sigaction "libc.so" +//go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so" + +//go:cgo_import_dynamic _ _ "libc.so" diff --git a/src/runtime/sys_openbsd_amd64.s b/src/runtime/sys_openbsd_amd64.s index 37d70ab9aa..4680a7f7aa 100644 --- a/src/runtime/sys_openbsd_amd64.s +++ b/src/runtime/sys_openbsd_amd64.s @@ -2,8 +2,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // -// System calls and other sys.stuff for AMD64, OpenBSD -// /usr/src/sys/kern/syscalls.master for syscall numbers. +// System calls and other sys.stuff for AMD64, OpenBSD. +// System calls are implemented in libc/libpthread, this file +// contains trampolines that convert from Go to C calling convention. +// Some direct system call implementations currently remain. // #include "go_asm.h" @@ -12,405 +14,434 @@ #define CLOCK_MONOTONIC $3 -// int32 tfork(void *param, uintptr psize, M *mp, G *gp, void (*fn)(void)); -TEXT runtime·tfork(SB),NOSPLIT,$32 - - // Copy mp, gp and fn off parent stack for use by child. - MOVQ mm+16(FP), R8 - MOVQ gg+24(FP), R9 - MOVQ fn+32(FP), R12 - - MOVQ param+0(FP), DI - MOVQ psize+8(FP), SI - MOVL $8, AX // sys___tfork - SYSCALL - - // Return if tfork syscall failed. - JCC 4(PC) - NEGQ AX - MOVL AX, ret+40(FP) +TEXT runtime·settls(SB),NOSPLIT,$0 + // Nothing to do, pthread already set thread-local storage up. RET - // In parent, return. - CMPL AX, $0 - JEQ 3(PC) - MOVL AX, ret+40(FP) - RET +// mstart_stub is the first function executed on a new thread started by pthread_create. +// It just does some low-level setup and then calls mstart. +// Note: called with the C calling convention. +TEXT runtime·mstart_stub(SB),NOSPLIT,$0 + // DI points to the m. + // We are already on m's g0 stack. - // Set FS to point at m->tls. - LEAQ m_tls(R8), DI - CALL runtime·settls(SB) + // Save callee-save registers. + SUBQ $48, SP + MOVQ BX, 0(SP) + MOVQ BP, 8(SP) + MOVQ R12, 16(SP) + MOVQ R13, 24(SP) + MOVQ R14, 32(SP) + MOVQ R15, 40(SP) - // In child, set up new stack. - get_tls(CX) - MOVQ R8, g_m(R9) - MOVQ R9, g(CX) - CALL runtime·stackcheck(SB) + // Load g and save to TLS entry. + // See cmd/link/internal/ld/sym.go:computeTLSOffset. + MOVQ m_g0(DI), DX // g + MOVQ DX, -8(FS) - // Call fn - CALL R12 + // Someday the convention will be D is always cleared. + CLD - // It shouldn't return. If it does, exit - MOVQ $0, DI // arg 1 - notdead - MOVL $302, AX // sys___threxit - SYSCALL - JMP -3(PC) // keep exiting + CALL runtime·mstart(SB) -TEXT runtime·osyield(SB),NOSPLIT,$0 - MOVL $298, AX // sys_sched_yield - SYSCALL - RET + // Restore callee-save registers. + MOVQ 0(SP), BX + MOVQ 8(SP), BP + MOVQ 16(SP), R12 + MOVQ 24(SP), R13 + MOVQ 32(SP), R14 + MOVQ 40(SP), R15 -TEXT runtime·thrsleep(SB),NOSPLIT,$0 - MOVQ ident+0(FP), DI // arg 1 - ident - MOVL clock_id+8(FP), SI // arg 2 - clock_id - MOVQ tsp+16(FP), DX // arg 3 - tp - MOVQ lock+24(FP), R10 // arg 4 - lock - MOVQ abort+32(FP), R8 // arg 5 - abort - MOVL $94, AX // sys___thrsleep - SYSCALL - MOVL AX, ret+40(FP) - RET + // Go is all done with this OS thread. + // Tell pthread everything is ok (we never join with this thread, so + // the value here doesn't really matter). + XORL AX, AX -TEXT runtime·thrwakeup(SB),NOSPLIT,$0 - MOVQ ident+0(FP), DI // arg 1 - ident - MOVL n+8(FP), SI // arg 2 - n - MOVL $301, AX // sys___thrwakeup - SYSCALL - MOVL AX, ret+16(FP) + ADDQ $48, SP RET -// Exit the entire program (like C exit) -TEXT runtime·exit(SB),NOSPLIT,$-8 - MOVL code+0(FP), DI // arg 1 - exit status - MOVL $1, AX // sys_exit - SYSCALL - MOVL $0xf1, 0xf1 // crash +TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 + MOVQ fn+0(FP), AX + MOVL sig+8(FP), DI + MOVQ info+16(FP), SI + MOVQ ctx+24(FP), DX + PUSHQ BP + MOVQ SP, BP + ANDQ $~15, SP // alignment for x86_64 ABI + CALL AX + MOVQ BP, SP + POPQ BP RET -// func exitThread(wait *uint32) -TEXT runtime·exitThread(SB),NOSPLIT,$0-8 - MOVQ wait+0(FP), DI // arg 1 - notdead - MOVL $302, AX // sys___threxit - SYSCALL - MOVL $0xf1, 0xf1 // crash - JMP 0(PC) - -TEXT runtime·open(SB),NOSPLIT,$-8 - MOVQ name+0(FP), DI // arg 1 pathname - MOVL mode+8(FP), SI // arg 2 flags - MOVL perm+12(FP), DX // arg 3 mode - MOVL $5, AX - SYSCALL - JCC 2(PC) - MOVL $-1, AX - MOVL AX, ret+16(FP) - RET +TEXT runtime·sigtramp(SB),NOSPLIT,$72 + // Save callee-saved C registers, since the caller may be a C signal handler. + MOVQ BX, bx-8(SP) + MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set + MOVQ R12, r12-24(SP) + MOVQ R13, r13-32(SP) + MOVQ R14, r14-40(SP) + MOVQ R15, r15-48(SP) + // We don't save mxcsr or the x87 control word because sigtrampgo doesn't + // modify them. -TEXT runtime·closefd(SB),NOSPLIT,$-8 - MOVL fd+0(FP), DI // arg 1 fd - MOVL $6, AX - SYSCALL - JCC 2(PC) - MOVL $-1, AX - MOVL AX, ret+8(FP) - RET + MOVQ DX, ctx-56(SP) + MOVQ SI, info-64(SP) + MOVQ DI, signum-72(SP) + CALL runtime·sigtrampgo(SB) -TEXT runtime·read(SB),NOSPLIT,$-8 - MOVL fd+0(FP), DI // arg 1 fd - MOVQ p+8(FP), SI // arg 2 buf - MOVL n+16(FP), DX // arg 3 count - MOVL $3, AX - SYSCALL - JCC 2(PC) - NEGQ AX // caller expects negative errno - MOVL AX, ret+24(FP) + MOVQ r15-48(SP), R15 + MOVQ r14-40(SP), R14 + MOVQ r13-32(SP), R13 + MOVQ r12-24(SP), R12 + MOVQ bp-16(SP), BP + MOVQ bx-8(SP), BX RET -// func pipe() (r, w int32, errno int32) -TEXT runtime·pipe(SB),NOSPLIT,$0-12 - LEAQ r+0(FP), DI - MOVL $263, AX - SYSCALL - MOVL AX, errno+8(FP) +// +// These trampolines help convert from Go calling convention to C calling convention. +// They should be called with asmcgocall. +// A pointer to the arguments is passed in DI. +// A single int32 result is returned in AX. +// (For more results, make an args/results structure.) +TEXT runtime·pthread_attr_init_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 0(DI), DI // arg 1 - attr + CALL libc_pthread_attr_init(SB) + POPQ BP RET -// func pipe2(flags int32) (r, w int32, errno int32) -TEXT runtime·pipe2(SB),NOSPLIT,$0-20 - LEAQ r+8(FP), DI - MOVL flags+0(FP), SI - MOVL $101, AX - SYSCALL - MOVL AX, errno+16(FP) +TEXT runtime·pthread_attr_destroy_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 0(DI), DI // arg 1 - attr + CALL libc_pthread_attr_destroy(SB) + POPQ BP RET -TEXT runtime·write1(SB),NOSPLIT,$-8 - MOVQ fd+0(FP), DI // arg 1 - fd - MOVQ p+8(FP), SI // arg 2 - buf - MOVL n+16(FP), DX // arg 3 - nbyte - MOVL $4, AX // sys_write - SYSCALL - JCC 2(PC) - NEGQ AX // caller expects negative errno - MOVL AX, ret+24(FP) +TEXT runtime·pthread_attr_getstacksize_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 - stacksize + MOVQ 0(DI), DI // arg 1 - attr + CALL libc_pthread_attr_getstacksize(SB) + POPQ BP RET -TEXT runtime·usleep(SB),NOSPLIT,$16 - MOVL $0, DX - MOVL usec+0(FP), AX - MOVL $1000000, CX - DIVL CX - MOVQ AX, 0(SP) // tv_sec - MOVL $1000, AX - MULL DX - MOVQ AX, 8(SP) // tv_nsec - - MOVQ SP, DI // arg 1 - rqtp - MOVQ $0, SI // arg 2 - rmtp - MOVL $91, AX // sys_nanosleep - SYSCALL +TEXT runtime·pthread_attr_setdetachstate_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 - detachstate + MOVQ 0(DI), DI // arg 1 - attr + CALL libc_pthread_attr_setdetachstate(SB) + POPQ BP RET -TEXT runtime·getthrid(SB),NOSPLIT,$0-4 - MOVL $299, AX // sys_getthrid - SYSCALL - MOVL AX, ret+0(FP) +TEXT runtime·pthread_create_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + SUBQ $16, SP + MOVQ 0(DI), SI // arg 2 - attr + MOVQ 8(DI), DX // arg 3 - start + MOVQ 16(DI), CX // arg 4 - arg + MOVQ SP, DI // arg 1 - &thread (discarded) + CALL libc_pthread_create(SB) + MOVQ BP, SP + POPQ BP RET -TEXT runtime·thrkill(SB),NOSPLIT,$0-16 - MOVL tid+0(FP), DI // arg 1 - tid - MOVQ sig+8(FP), SI // arg 2 - signum +TEXT runtime·thrkill_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 8(DI), SI // arg 2 - signal MOVQ $0, DX // arg 3 - tcb - MOVL $119, AX // sys_thrkill - SYSCALL + MOVL 0(DI), DI // arg 1 - tid + CALL libc_thrkill(SB) + POPQ BP RET -TEXT runtime·raiseproc(SB),NOSPLIT,$16 - MOVL $20, AX // sys_getpid - SYSCALL - MOVQ AX, DI // arg 1 - pid - MOVL sig+0(FP), SI // arg 2 - signum - MOVL $122, AX // sys_kill - SYSCALL +TEXT runtime·thrsleep_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 8(DI), SI // arg 2 - clock_id + MOVQ 16(DI), DX // arg 3 - abstime + MOVQ 24(DI), CX // arg 4 - lock + MOVQ 32(DI), R8 // arg 5 - abort + MOVQ 0(DI), DI // arg 1 - id + CALL libc_thrsleep(SB) + POPQ BP RET -TEXT runtime·setitimer(SB),NOSPLIT,$-8 - MOVL mode+0(FP), DI // arg 1 - which - MOVQ new+8(FP), SI // arg 2 - itv - MOVQ old+16(FP), DX // arg 3 - oitv - MOVL $69, AX // sys_setitimer - SYSCALL +TEXT runtime·thrwakeup_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 8(DI), SI // arg 2 - count + MOVQ 0(DI), DI // arg 1 - id + CALL libc_thrwakeup(SB) + POPQ BP RET -// func walltime1() (sec int64, nsec int32) -TEXT runtime·walltime1(SB), NOSPLIT, $32 - MOVQ $0, DI // arg 1 - clock_id - LEAQ 8(SP), SI // arg 2 - tp - MOVL $87, AX // sys_clock_gettime - SYSCALL - MOVQ 8(SP), AX // sec - MOVQ 16(SP), DX // nsec +TEXT runtime·exit_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 0(DI), DI // arg 1 exit status + CALL libc_exit(SB) + MOVL $0xf1, 0xf1 // crash + POPQ BP + RET - // sec is in AX, nsec in DX - MOVQ AX, sec+0(FP) - MOVL DX, nsec+8(FP) +TEXT runtime·getthrid_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ DI, BX // BX is caller-save + CALL libc_getthrid(SB) + MOVL AX, 0(BX) // return value + POPQ BP RET -TEXT runtime·nanotime1(SB),NOSPLIT,$24 - MOVQ CLOCK_MONOTONIC, DI // arg 1 - clock_id - LEAQ 8(SP), SI // arg 2 - tp - MOVL $87, AX // sys_clock_gettime - SYSCALL - MOVQ 8(SP), AX // sec - MOVQ 16(SP), DX // nsec +TEXT runtime·raiseproc_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 0(DI), BX // signal + CALL libc_getpid(SB) + MOVL AX, DI // arg 1 pid + MOVL BX, SI // arg 2 signal + CALL libc_kill(SB) + POPQ BP + RET - // sec is in AX, nsec in DX - // return nsec in AX - IMULQ $1000000000, AX - ADDQ DX, AX - MOVQ AX, ret+0(FP) +TEXT runtime·sched_yield_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + CALL libc_sched_yield(SB) + POPQ BP RET -TEXT runtime·sigaction(SB),NOSPLIT,$-8 - MOVL sig+0(FP), DI // arg 1 - signum - MOVQ new+8(FP), SI // arg 2 - nsa - MOVQ old+16(FP), DX // arg 3 - osa - MOVL $46, AX - SYSCALL - JCC 2(PC) - MOVL $0xf1, 0xf1 // crash +TEXT runtime·mmap_trampoline(SB),NOSPLIT,$0 + PUSHQ BP // make a frame; keep stack aligned + MOVQ SP, BP + MOVQ DI, BX + MOVQ 0(BX), DI // arg 1 addr + MOVQ 8(BX), SI // arg 2 len + MOVL 16(BX), DX // arg 3 prot + MOVL 20(BX), CX // arg 4 flags + MOVL 24(BX), R8 // arg 5 fid + MOVL 28(BX), R9 // arg 6 offset + CALL libc_mmap(SB) + XORL DX, DX + CMPQ AX, $-1 + JNE ok + CALL libc_errno(SB) + MOVLQSX (AX), DX // errno + XORQ AX, AX +ok: + MOVQ AX, 32(BX) + MOVQ DX, 40(BX) + POPQ BP RET -TEXT runtime·obsdsigprocmask(SB),NOSPLIT,$0 - MOVL how+0(FP), DI // arg 1 - how - MOVL new+4(FP), SI // arg 2 - set - MOVL $48, AX // sys_sigprocmask - SYSCALL - JCC 2(PC) - MOVL $0xf1, 0xf1 // crash - MOVL AX, ret+8(FP) +TEXT runtime·munmap_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 len + MOVQ 0(DI), DI // arg 1 addr + CALL libc_munmap(SB) + TESTQ AX, AX + JEQ 2(PC) + MOVL $0xf1, 0xf1 // crash + POPQ BP RET -TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 - MOVQ fn+0(FP), AX - MOVL sig+8(FP), DI - MOVQ info+16(FP), SI - MOVQ ctx+24(FP), DX +TEXT runtime·madvise_trampoline(SB), NOSPLIT, $0 PUSHQ BP MOVQ SP, BP - ANDQ $~15, SP // alignment for x86_64 ABI - CALL AX - MOVQ BP, SP + MOVQ 8(DI), SI // arg 2 len + MOVL 16(DI), DX // arg 3 advice + MOVQ 0(DI), DI // arg 1 addr + CALL libc_madvise(SB) + // ignore failure - maybe pages are locked POPQ BP RET -TEXT runtime·sigtramp(SB),NOSPLIT,$72 - // Save callee-saved C registers, since the caller may be a C signal handler. - MOVQ BX, bx-8(SP) - MOVQ BP, bp-16(SP) // save in case GOEXPERIMENT=noframepointer is set - MOVQ R12, r12-24(SP) - MOVQ R13, r13-32(SP) - MOVQ R14, r14-40(SP) - MOVQ R15, r15-48(SP) - // We don't save mxcsr or the x87 control word because sigtrampgo doesn't - // modify them. +TEXT runtime·open_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 8(DI), SI // arg 2 - flags + MOVL 12(DI), DX // arg 3 - mode + MOVQ 0(DI), DI // arg 1 - path + XORL AX, AX // vararg: say "no float args" + CALL libc_open(SB) + POPQ BP + RET - MOVQ DX, ctx-56(SP) - MOVQ SI, info-64(SP) - MOVQ DI, signum-72(SP) - CALL runtime·sigtrampgo(SB) +TEXT runtime·close_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 0(DI), DI // arg 1 - fd + CALL libc_close(SB) + POPQ BP + RET - MOVQ r15-48(SP), R15 - MOVQ r14-40(SP), R14 - MOVQ r13-32(SP), R13 - MOVQ r12-24(SP), R12 - MOVQ bp-16(SP), BP - MOVQ bx-8(SP), BX +TEXT runtime·read_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 - buf + MOVL 16(DI), DX // arg 3 - count + MOVL 0(DI), DI // arg 1 - fd + CALL libc_read(SB) + TESTL AX, AX + JGE noerr + CALL libc_errno(SB) + MOVL (AX), AX // errno + NEGL AX // caller expects negative errno value +noerr: + POPQ BP RET -TEXT runtime·mmap(SB),NOSPLIT,$0 - MOVQ addr+0(FP), DI // arg 1 - addr - MOVQ n+8(FP), SI // arg 2 - len - MOVL prot+16(FP), DX // arg 3 - prot - MOVL flags+20(FP), R10 // arg 4 - flags - MOVL fd+24(FP), R8 // arg 5 - fd - MOVL off+28(FP), R9 - SUBQ $16, SP - MOVQ R9, 8(SP) // arg 7 - offset (passed on stack) - MOVQ $0, R9 // arg 6 - pad - MOVL $197, AX - SYSCALL - JCC ok - ADDQ $16, SP - MOVQ $0, p+32(FP) - MOVQ AX, err+40(FP) +TEXT runtime·write_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 buf + MOVL 16(DI), DX // arg 3 count + MOVL 0(DI), DI // arg 1 fd + CALL libc_write(SB) + TESTL AX, AX + JGE noerr + CALL libc_errno(SB) + MOVL (AX), AX // errno + NEGL AX // caller expects negative errno value +noerr: + POPQ BP RET -ok: - ADDQ $16, SP - MOVQ AX, p+32(FP) - MOVQ $0, err+40(FP) + +TEXT runtime·pipe2_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 8(DI), SI // arg 2 flags + MOVQ 0(DI), DI // arg 1 filedes + CALL libc_pipe2(SB) + TESTL AX, AX + JEQ 3(PC) + CALL libc_errno(SB) + MOVL (AX), AX // errno + NEGL AX // caller expects negative errno value + POPQ BP RET -TEXT runtime·munmap(SB),NOSPLIT,$0 - MOVQ addr+0(FP), DI // arg 1 - addr - MOVQ n+8(FP), SI // arg 2 - len - MOVL $73, AX // sys_munmap - SYSCALL - JCC 2(PC) - MOVL $0xf1, 0xf1 // crash +TEXT runtime·setitimer_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 new + MOVQ 16(DI), DX // arg 3 old + MOVL 0(DI), DI // arg 1 which + CALL libc_setitimer(SB) + POPQ BP RET -TEXT runtime·madvise(SB),NOSPLIT,$0 - MOVQ addr+0(FP), DI // arg 1 - addr - MOVQ n+8(FP), SI // arg 2 - len - MOVL flags+16(FP), DX // arg 3 - behav - MOVQ $75, AX // sys_madvise - SYSCALL - JCC 2(PC) - MOVL $-1, AX - MOVL AX, ret+24(FP) +TEXT runtime·usleep_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 0(DI), DI // arg 1 usec + CALL libc_usleep(SB) + POPQ BP RET -TEXT runtime·sigaltstack(SB),NOSPLIT,$-8 - MOVQ new+0(FP), DI // arg 1 - nss - MOVQ old+8(FP), SI // arg 2 - oss - MOVQ $288, AX // sys_sigaltstack - SYSCALL - JCC 2(PC) - MOVL $0xf1, 0xf1 // crash +TEXT runtime·sysctl_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 8(DI), SI // arg 2 miblen + MOVQ 16(DI), DX // arg 3 out + MOVQ 24(DI), CX // arg 4 size + MOVQ 32(DI), R8 // arg 5 dst + MOVQ 40(DI), R9 // arg 6 ndst + MOVQ 0(DI), DI // arg 1 mib + CALL libc_sysctl(SB) + POPQ BP RET -// set tls base to DI -TEXT runtime·settls(SB),NOSPLIT,$0 - // adjust for ELF: wants to use -8(FS) for g - ADDQ $8, DI - MOVQ $329, AX // sys___settcb - SYSCALL - JCC 2(PC) - MOVL $0xf1, 0xf1 // crash +TEXT runtime·kqueue_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + CALL libc_kqueue(SB) + POPQ BP RET -TEXT runtime·sysctl(SB),NOSPLIT,$0 - MOVQ mib+0(FP), DI // arg 1 - name - MOVL miblen+8(FP), SI // arg 2 - namelen - MOVQ out+16(FP), DX // arg 3 - oldp - MOVQ size+24(FP), R10 // arg 4 - oldlenp - MOVQ dst+32(FP), R8 // arg 5 - newp - MOVQ ndst+40(FP), R9 // arg 6 - newlen - MOVQ $202, AX // sys___sysctl - SYSCALL - JCC 4(PC) - NEGQ AX - MOVL AX, ret+48(FP) +TEXT runtime·kevent_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 keventt + MOVL 16(DI), DX // arg 3 nch + MOVQ 24(DI), CX // arg 4 ev + MOVL 32(DI), R8 // arg 5 nev + MOVQ 40(DI), R9 // arg 6 ts + MOVL 0(DI), DI // arg 1 kq + CALL libc_kevent(SB) + CMPL AX, $-1 + JNE ok + CALL libc_errno(SB) + MOVL (AX), AX // errno + NEGL AX // caller expects negative errno value +ok: + POPQ BP RET - MOVL $0, AX - MOVL AX, ret+48(FP) + +TEXT runtime·clock_gettime_trampoline(SB),NOSPLIT,$0 + PUSHQ BP // make a frame; keep stack aligned + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 tp + MOVL 0(DI), DI // arg 1 clock_id + CALL libc_clock_gettime(SB) + TESTL AX, AX + JEQ 2(PC) + MOVL $0xf1, 0xf1 // crash + POPQ BP RET -// int32 runtime·kqueue(void); -TEXT runtime·kqueue(SB),NOSPLIT,$0 - MOVL $269, AX - SYSCALL - JCC 2(PC) - NEGQ AX - MOVL AX, ret+0(FP) +TEXT runtime·fcntl_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVL 4(DI), SI // arg 2 cmd + MOVL 8(DI), DX // arg 3 arg + MOVL 0(DI), DI // arg 1 fd + XORL AX, AX // vararg: say "no float args" + CALL libc_fcntl(SB) + POPQ BP RET -// int32 runtime·kevent(int kq, Kevent *changelist, int nchanges, Kevent *eventlist, int nevents, Timespec *timeout); -TEXT runtime·kevent(SB),NOSPLIT,$0 - MOVL kq+0(FP), DI - MOVQ ch+8(FP), SI - MOVL nch+16(FP), DX - MOVQ ev+24(FP), R10 - MOVL nev+32(FP), R8 - MOVQ ts+40(FP), R9 - MOVL $72, AX - SYSCALL - JCC 2(PC) - NEGQ AX - MOVL AX, ret+48(FP) +TEXT runtime·sigaction_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 new + MOVQ 16(DI), DX // arg 3 old + MOVL 0(DI), DI // arg 1 sig + CALL libc_sigaction(SB) + TESTL AX, AX + JEQ 2(PC) + MOVL $0xf1, 0xf1 // crash + POPQ BP RET -// void runtime·closeonexec(int32 fd); -TEXT runtime·closeonexec(SB),NOSPLIT,$0 - MOVL fd+0(FP), DI // fd - MOVQ $2, SI // F_SETFD - MOVQ $1, DX // FD_CLOEXEC - MOVL $92, AX // fcntl - SYSCALL +TEXT runtime·sigprocmask_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 new + MOVQ 16(DI), DX // arg 3 old + MOVL 0(DI), DI // arg 1 how + CALL libc_pthread_sigmask(SB) + TESTL AX, AX + JEQ 2(PC) + MOVL $0xf1, 0xf1 // crash + POPQ BP RET -// func runtime·setNonblock(int32 fd) -TEXT runtime·setNonblock(SB),NOSPLIT,$0-4 - MOVL fd+0(FP), DI // fd - MOVQ $3, SI // F_GETFL - MOVQ $0, DX - MOVL $92, AX // fcntl - SYSCALL - MOVL fd+0(FP), DI // fd - MOVQ $4, SI // F_SETFL - MOVQ $4, DX // O_NONBLOCK - ORL AX, DX - MOVL $92, AX // fcntl - SYSCALL +TEXT runtime·sigaltstack_trampoline(SB),NOSPLIT,$0 + PUSHQ BP + MOVQ SP, BP + MOVQ 8(DI), SI // arg 2 old + MOVQ 0(DI), DI // arg 1 new + CALL libc_sigaltstack(SB) + TESTQ AX, AX + JEQ 2(PC) + MOVL $0xf1, 0xf1 // crash + POPQ BP RET diff --git a/src/runtime/sys_openbsd_arm64.s b/src/runtime/sys_openbsd_arm64.s index 621b1b1a42..2ec9d038ba 100644 --- a/src/runtime/sys_openbsd_arm64.s +++ b/src/runtime/sys_openbsd_arm64.s @@ -3,7 +3,9 @@ // license that can be found in the LICENSE file. // // System calls and other sys.stuff for arm64, OpenBSD -// /usr/src/sys/kern/syscalls.master for syscall numbers. +// System calls are implemented in libc/libpthread, this file +// contains trampolines that convert from Go to C calling convention. +// Some direct system call implementations currently remain. // #include "go_asm.h" @@ -24,6 +26,168 @@ NOOP; \ NOOP +// mstart_stub is the first function executed on a new thread started by pthread_create. +// It just does some low-level setup and then calls mstart. +// Note: called with the C calling convention. +TEXT runtime·mstart_stub(SB),NOSPLIT,$160 + // R0 points to the m. + // We are already on m's g0 stack. + + // Save callee-save registers. + MOVD R19, 8(RSP) + MOVD R20, 16(RSP) + MOVD R21, 24(RSP) + MOVD R22, 32(RSP) + MOVD R23, 40(RSP) + MOVD R24, 48(RSP) + MOVD R25, 56(RSP) + MOVD R26, 64(RSP) + MOVD R27, 72(RSP) + MOVD g, 80(RSP) + MOVD R29, 88(RSP) + FMOVD F8, 96(RSP) + FMOVD F9, 104(RSP) + FMOVD F10, 112(RSP) + FMOVD F11, 120(RSP) + FMOVD F12, 128(RSP) + FMOVD F13, 136(RSP) + FMOVD F14, 144(RSP) + FMOVD F15, 152(RSP) + + MOVD m_g0(R0), g + BL runtime·save_g(SB) + + BL runtime·mstart(SB) + + // Restore callee-save registers. + MOVD 8(RSP), R19 + MOVD 16(RSP), R20 + MOVD 24(RSP), R21 + MOVD 32(RSP), R22 + MOVD 40(RSP), R23 + MOVD 48(RSP), R24 + MOVD 56(RSP), R25 + MOVD 64(RSP), R26 + MOVD 72(RSP), R27 + MOVD 80(RSP), g + MOVD 88(RSP), R29 + FMOVD 96(RSP), F8 + FMOVD 104(RSP), F9 + FMOVD 112(RSP), F10 + FMOVD 120(RSP), F11 + FMOVD 128(RSP), F12 + FMOVD 136(RSP), F13 + FMOVD 144(RSP), F14 + FMOVD 152(RSP), F15 + + // Go is all done with this OS thread. + // Tell pthread everything is ok (we never join with this thread, so + // the value here doesn't really matter). + MOVD $0, R0 + + RET + +TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 + MOVW sig+8(FP), R0 + MOVD info+16(FP), R1 + MOVD ctx+24(FP), R2 + MOVD fn+0(FP), R11 + BL (R11) // Alignment for ELF ABI? + RET + +TEXT runtime·sigtramp(SB),NOSPLIT,$192 + // Save callee-save registers in the case of signal forwarding. + // Please refer to https://golang.org/issue/31827 . + MOVD R19, 8*4(RSP) + MOVD R20, 8*5(RSP) + MOVD R21, 8*6(RSP) + MOVD R22, 8*7(RSP) + MOVD R23, 8*8(RSP) + MOVD R24, 8*9(RSP) + MOVD R25, 8*10(RSP) + MOVD R26, 8*11(RSP) + MOVD R27, 8*12(RSP) + MOVD g, 8*13(RSP) + MOVD R29, 8*14(RSP) + FMOVD F8, 8*15(RSP) + FMOVD F9, 8*16(RSP) + FMOVD F10, 8*17(RSP) + FMOVD F11, 8*18(RSP) + FMOVD F12, 8*19(RSP) + FMOVD F13, 8*20(RSP) + FMOVD F14, 8*21(RSP) + FMOVD F15, 8*22(RSP) + + // If called from an external code context, g will not be set. + // Save R0, since runtime·load_g will clobber it. + MOVW R0, 8(RSP) // signum + BL runtime·load_g(SB) + + MOVD R1, 16(RSP) + MOVD R2, 24(RSP) + BL runtime·sigtrampgo(SB) + + // Restore callee-save registers. + MOVD 8*4(RSP), R19 + MOVD 8*5(RSP), R20 + MOVD 8*6(RSP), R21 + MOVD 8*7(RSP), R22 + MOVD 8*8(RSP), R23 + MOVD 8*9(RSP), R24 + MOVD 8*10(RSP), R25 + MOVD 8*11(RSP), R26 + MOVD 8*12(RSP), R27 + MOVD 8*13(RSP), g + MOVD 8*14(RSP), R29 + FMOVD 8*15(RSP), F8 + FMOVD 8*16(RSP), F9 + FMOVD 8*17(RSP), F10 + FMOVD 8*18(RSP), F11 + FMOVD 8*19(RSP), F12 + FMOVD 8*20(RSP), F13 + FMOVD 8*21(RSP), F14 + FMOVD 8*22(RSP), F15 + + RET + +// +// These trampolines help convert from Go calling convention to C calling convention. +// They should be called with asmcgocall. +// A pointer to the arguments is passed in R0. +// A single int32 result is returned in R0. +// (For more results, make an args/results structure.) +TEXT runtime·pthread_attr_init_trampoline(SB),NOSPLIT,$0 + MOVD 0(R0), R0 // arg 1 - attr + CALL libc_pthread_attr_init(SB) + RET + +TEXT runtime·pthread_attr_destroy_trampoline(SB),NOSPLIT,$0 + MOVD 0(R0), R0 // arg 1 - attr + CALL libc_pthread_attr_destroy(SB) + RET + +TEXT runtime·pthread_attr_getstacksize_trampoline(SB),NOSPLIT,$0 + MOVD 8(R0), R1 // arg 2 - size + MOVD 0(R0), R0 // arg 1 - attr + CALL libc_pthread_attr_getstacksize(SB) + RET + +TEXT runtime·pthread_attr_setdetachstate_trampoline(SB),NOSPLIT,$0 + MOVD 8(R0), R1 // arg 2 - state + MOVD 0(R0), R0 // arg 1 - attr + CALL libc_pthread_attr_setdetachstate(SB) + RET + +TEXT runtime·pthread_create_trampoline(SB),NOSPLIT,$0 + MOVD 0(R0), R1 // arg 2 - attr + MOVD 8(R0), R2 // arg 3 - start + MOVD 16(R0), R3 // arg 4 - arg + SUB $16, RSP + MOVD RSP, R0 // arg 1 - &threadid (discard) + CALL libc_pthread_create(SB) + ADD $16, RSP + RET + // Exit the entire program (like C exit) TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0 MOVW code+0(FP), R0 // arg 1 - status @@ -248,109 +412,6 @@ TEXT runtime·obsdsigprocmask(SB),NOSPLIT,$0 MOVW R0, ret+8(FP) RET -TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 - MOVW sig+8(FP), R0 - MOVD info+16(FP), R1 - MOVD ctx+24(FP), R2 - MOVD fn+0(FP), R11 - BL (R11) // Alignment for ELF ABI? - RET - -TEXT runtime·sigtramp(SB),NOSPLIT,$192 - // Save callee-save registers in the case of signal forwarding. - // Please refer to https://golang.org/issue/31827 . - MOVD R19, 8*4(RSP) - MOVD R20, 8*5(RSP) - MOVD R21, 8*6(RSP) - MOVD R22, 8*7(RSP) - MOVD R23, 8*8(RSP) - MOVD R24, 8*9(RSP) - MOVD R25, 8*10(RSP) - MOVD R26, 8*11(RSP) - MOVD R27, 8*12(RSP) - MOVD g, 8*13(RSP) - MOVD R29, 8*14(RSP) - FMOVD F8, 8*15(RSP) - FMOVD F9, 8*16(RSP) - FMOVD F10, 8*17(RSP) - FMOVD F11, 8*18(RSP) - FMOVD F12, 8*19(RSP) - FMOVD F13, 8*20(RSP) - FMOVD F14, 8*21(RSP) - FMOVD F15, 8*22(RSP) - - // If called from an external code context, g will not be set. - // Save R0, since runtime·load_g will clobber it. - MOVW R0, 8(RSP) // signum - MOVB runtime·iscgo(SB), R0 - CMP $0, R0 - BEQ 2(PC) - BL runtime·load_g(SB) - - MOVD R1, 16(RSP) - MOVD R2, 24(RSP) - BL runtime·sigtrampgo(SB) - - // Restore callee-save registers. - MOVD 8*4(RSP), R19 - MOVD 8*5(RSP), R20 - MOVD 8*6(RSP), R21 - MOVD 8*7(RSP), R22 - MOVD 8*8(RSP), R23 - MOVD 8*9(RSP), R24 - MOVD 8*10(RSP), R25 - MOVD 8*11(RSP), R26 - MOVD 8*12(RSP), R27 - MOVD 8*13(RSP), g - MOVD 8*14(RSP), R29 - FMOVD 8*15(RSP), F8 - FMOVD 8*16(RSP), F9 - FMOVD 8*17(RSP), F10 - FMOVD 8*18(RSP), F11 - FMOVD 8*19(RSP), F12 - FMOVD 8*20(RSP), F13 - FMOVD 8*21(RSP), F14 - FMOVD 8*22(RSP), F15 - - RET - -// int32 tfork(void *param, uintptr psize, M *mp, G *gp, void (*fn)(void)); -TEXT runtime·tfork(SB),NOSPLIT,$0 - - // Copy mp, gp and fn off parent stack for use by child. - MOVD mm+16(FP), R4 - MOVD gg+24(FP), R5 - MOVD fn+32(FP), R6 - - MOVD param+0(FP), R0 // arg 1 - param - MOVD psize+8(FP), R1 // arg 2 - psize - MOVD $8, R8 // sys___tfork - INVOKE_SYSCALL - - // Return if syscall failed. - BCC 4(PC) - NEG R0, R0 - MOVW R0, ret+40(FP) - RET - - // In parent, return. - CMP $0, R0 - BEQ 3(PC) - MOVW R0, ret+40(FP) - RET - - // Initialise m, g. - MOVD R5, g - MOVD R4, g_m(g) - - // Call fn. - BL (R6) - - // fn should never return. - MOVD $2, R8 // crash if reached - MOVD R8, (R8) - RET - TEXT runtime·sigaltstack(SB),NOSPLIT,$0 MOVD new+0(FP), R0 // arg 1 - new sigaltstack MOVD old+8(FP), R1 // arg 2 - old sigaltstack diff --git a/src/runtime/time.go b/src/runtime/time.go index d338705b7c..8ab2a03430 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -609,8 +609,14 @@ func moveTimers(pp *p, timers []*timer) { for { switch s := atomic.Load(&t.status); s { case timerWaiting: + if !atomic.Cas(&t.status, s, timerMoving) { + continue + } t.pp = 0 doaddtimer(pp, t) + if !atomic.Cas(&t.status, timerMoving, timerWaiting) { + badTimer() + } break loop case timerModifiedEarlier, timerModifiedLater: if !atomic.Cas(&t.status, s, timerMoving) { diff --git a/src/runtime/timestub2.go b/src/runtime/timestub2.go index 6d73aabc35..68777ee4a9 100644 --- a/src/runtime/timestub2.go +++ b/src/runtime/timestub2.go @@ -2,11 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build !aix // +build !darwin -// +build !windows // +build !freebsd -// +build !aix +// +build !openbsd // +build !solaris +// +build !windows package runtime diff --git a/src/runtime/tls_arm64.s b/src/runtime/tls_arm64.s index 3f02974d5b..085012f791 100644 --- a/src/runtime/tls_arm64.s +++ b/src/runtime/tls_arm64.s @@ -10,9 +10,11 @@ TEXT runtime·load_g(SB),NOSPLIT,$0 #ifndef TLS_darwin +#ifndef GOOS_openbsd MOVB runtime·iscgo(SB), R0 CBZ R0, nocgo #endif +#endif MRS_TPIDR_R0 #ifdef TLS_darwin @@ -27,9 +29,11 @@ nocgo: TEXT runtime·save_g(SB),NOSPLIT,$0 #ifndef TLS_darwin +#ifndef GOOS_openbsd MOVB runtime·iscgo(SB), R0 CBZ R0, nocgo #endif +#endif MRS_TPIDR_R0 #ifdef TLS_darwin diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 0825e9e707..2601cd697f 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -917,17 +917,25 @@ func tracebackothers(me *g) { level, _, _ := gotraceback() // Show the current goroutine first, if we haven't already. - g := getg() - gp := g.m.curg - if gp != nil && gp != me { + curgp := getg().m.curg + if curgp != nil && curgp != me { print("\n") - goroutineheader(gp) - traceback(^uintptr(0), ^uintptr(0), 0, gp) + goroutineheader(curgp) + traceback(^uintptr(0), ^uintptr(0), 0, curgp) } - lock(&allglock) - for _, gp := range allgs { - if gp == me || gp == g.m.curg || readgstatus(gp) == _Gdead || isSystemGoroutine(gp, false) && level < 2 { + // We can't take allglock here because this may be during fatal + // throw/panic, where locking allglock could be out-of-order or a + // direct deadlock. + // + // Instead, use atomic access to allgs which requires no locking. We + // don't lock against concurrent creation of new Gs, but even with + // allglock we may miss Gs created after this loop. + ptr, length := atomicAllG() + for i := uintptr(0); i < length; i++ { + gp := atomicAllGIndex(ptr, i) + + if gp == me || gp == curgp || readgstatus(gp) == _Gdead || isSystemGoroutine(gp, false) && level < 2 { continue } print("\n") @@ -936,14 +944,13 @@ func tracebackothers(me *g) { // called from a signal handler initiated during a // systemstack call. The original G is still in the // running state, and we want to print its stack. - if gp.m != g.m && readgstatus(gp)&^_Gscan == _Grunning { + if gp.m != getg().m && readgstatus(gp)&^_Gscan == _Grunning { print("\tgoroutine running on other thread; stack unavailable\n") printcreatedby(gp) } else { traceback(^uintptr(0), ^uintptr(0), 0, gp) } } - unlock(&allglock) } // tracebackHexdump hexdumps part of stk around frame.sp and frame.fp |
