From 73b8e5f81b992597462d39978bad09c0f88a530e Mon Sep 17 00:00:00 2001 From: Marko Kevac Date: Tue, 5 Jun 2018 22:41:18 +0300 Subject: runtime/pprof: remove "deleted" suffix while parsing maps file If binary file of a running program was deleted or moved, maps file (/proc/pid/maps) will contain lines that have this binary filename suffixed with "(deleted)" string. This suffix stayed as a part of the filename and made remote profiling slightly more difficult by requiring from a user to rename binary file to include this suffix. This change cleans up the filename and removes this suffix and thus simplify debugging. Fixes #25740 Change-Id: Ib3c8c3b9ef536c2ac037fcc14e8037fa5c960036 Reviewed-on: https://go-review.googlesource.com/116395 Run-TryBot: Hyang-Ah Hana Kim TryBot-Result: Gobot Gobot Reviewed-by: Hyang-Ah Hana Kim --- src/runtime/pprof/proto.go | 8 ++++ src/runtime/pprof/proto_test.go | 95 ++++++++++++++++++++++++++++++++++------- 2 files changed, 88 insertions(+), 15 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/pprof/proto.go b/src/runtime/pprof/proto.go index cbd0b83376..bd5c8f7afc 100644 --- a/src/runtime/pprof/proto.go +++ b/src/runtime/pprof/proto.go @@ -524,6 +524,14 @@ func parseProcSelfMaps(data []byte, addMapping func(lo, hi, offset uint64, file, continue } file := string(line) + + // Trim deleted file marker. + deletedStr := " (deleted)" + deletedLen := len(deletedStr) + if len(file) >= deletedLen && file[len(file)-deletedLen:] == deletedStr { + file = file[:len(file)-deletedLen] + } + if len(inode) == 1 && inode[0] == '0' && file == "" { // Huge-page text mappings list the initial fragment of // mapped but unpopulated memory as being inode 0. diff --git a/src/runtime/pprof/proto_test.go b/src/runtime/pprof/proto_test.go index 76bd46da02..4452d51231 100644 --- a/src/runtime/pprof/proto_test.go +++ b/src/runtime/pprof/proto_test.go @@ -216,24 +216,89 @@ c000000000-c000036000 rw-p 00000000 00:00 0 07000000 07093000 06c00000 /path/to/gobench_server_main ` +var profSelfMapsTestsWithDeleted = ` +00400000-0040b000 r-xp 00000000 fc:01 787766 /bin/cat (deleted) +0060a000-0060b000 r--p 0000a000 fc:01 787766 /bin/cat (deleted) +0060b000-0060c000 rw-p 0000b000 fc:01 787766 /bin/cat (deleted) +014ab000-014cc000 rw-p 00000000 00:00 0 [heap] +7f7d76af8000-7f7d7797c000 r--p 00000000 fc:01 1318064 /usr/lib/locale/locale-archive +7f7d7797c000-7f7d77b36000 r-xp 00000000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77b36000-7f7d77d36000 ---p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d36000-7f7d77d3a000 r--p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d3a000-7f7d77d3c000 rw-p 001be000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d3c000-7f7d77d41000 rw-p 00000000 00:00 0 +7f7d77d41000-7f7d77d64000 r-xp 00000000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f3f000-7f7d77f42000 rw-p 00000000 00:00 0 +7f7d77f61000-7f7d77f63000 rw-p 00000000 00:00 0 +7f7d77f63000-7f7d77f64000 r--p 00022000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f64000-7f7d77f65000 rw-p 00023000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f65000-7f7d77f66000 rw-p 00000000 00:00 0 +7ffc342a2000-7ffc342c3000 rw-p 00000000 00:00 0 [stack] +7ffc34343000-7ffc34345000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 r-xp 00000090 00:00 0 [vsyscall] +-> +00400000 0040b000 00000000 /bin/cat +7f7d7797c000 7f7d77b36000 00000000 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d41000 7f7d77d64000 00000000 /lib/x86_64-linux-gnu/ld-2.19.so +7ffc34343000 7ffc34345000 00000000 [vdso] +ffffffffff600000 ffffffffff601000 00000090 [vsyscall] + +00400000-0040b000 r-xp 00000000 fc:01 787766 /bin/cat with space +0060a000-0060b000 r--p 0000a000 fc:01 787766 /bin/cat with space +0060b000-0060c000 rw-p 0000b000 fc:01 787766 /bin/cat with space +014ab000-014cc000 rw-p 00000000 00:00 0 [heap] +7f7d76af8000-7f7d7797c000 r--p 00000000 fc:01 1318064 /usr/lib/locale/locale-archive +7f7d7797c000-7f7d77b36000 r-xp 00000000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77b36000-7f7d77d36000 ---p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d36000-7f7d77d3a000 r--p 001ba000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d3a000-7f7d77d3c000 rw-p 001be000 fc:01 1180226 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d3c000-7f7d77d41000 rw-p 00000000 00:00 0 +7f7d77d41000-7f7d77d64000 r-xp 00000000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f3f000-7f7d77f42000 rw-p 00000000 00:00 0 +7f7d77f61000-7f7d77f63000 rw-p 00000000 00:00 0 +7f7d77f63000-7f7d77f64000 r--p 00022000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f64000-7f7d77f65000 rw-p 00023000 fc:01 1180217 /lib/x86_64-linux-gnu/ld-2.19.so +7f7d77f65000-7f7d77f66000 rw-p 00000000 00:00 0 +7ffc342a2000-7ffc342c3000 rw-p 00000000 00:00 0 [stack] +7ffc34343000-7ffc34345000 r-xp 00000000 00:00 0 [vdso] +ffffffffff600000-ffffffffff601000 r-xp 00000090 00:00 0 [vsyscall] +-> +00400000 0040b000 00000000 /bin/cat with space +7f7d7797c000 7f7d77b36000 00000000 /lib/x86_64-linux-gnu/libc-2.19.so +7f7d77d41000 7f7d77d64000 00000000 /lib/x86_64-linux-gnu/ld-2.19.so +7ffc34343000 7ffc34345000 00000000 [vdso] +ffffffffff600000 ffffffffff601000 00000090 [vsyscall] +` + func TestProcSelfMaps(t *testing.T) { - for tx, tt := range strings.Split(profSelfMapsTests, "\n\n") { - i := strings.Index(tt, "->\n") - if i < 0 { - t.Fatal("malformed test case") - } - in, out := tt[:i], tt[i+len("->\n"):] - if len(out) > 0 && out[len(out)-1] != '\n' { - out += "\n" - } - var buf bytes.Buffer - parseProcSelfMaps([]byte(in), func(lo, hi, offset uint64, file, buildID string) { - fmt.Fprintf(&buf, "%08x %08x %08x %s\n", lo, hi, offset, file) - }) - if buf.String() != out { - t.Errorf("#%d: have:\n%s\nwant:\n%s\n%q\n%q", tx, buf.String(), out, buf.String(), out) + + f := func(t *testing.T, input string) { + for tx, tt := range strings.Split(input, "\n\n") { + i := strings.Index(tt, "->\n") + if i < 0 { + t.Fatal("malformed test case") + } + in, out := tt[:i], tt[i+len("->\n"):] + if len(out) > 0 && out[len(out)-1] != '\n' { + out += "\n" + } + var buf bytes.Buffer + parseProcSelfMaps([]byte(in), func(lo, hi, offset uint64, file, buildID string) { + fmt.Fprintf(&buf, "%08x %08x %08x %s\n", lo, hi, offset, file) + }) + if buf.String() != out { + t.Errorf("#%d: have:\n%s\nwant:\n%s\n%q\n%q", tx, buf.String(), out, buf.String(), out) + } } } + + t.Run("Normal", func(t *testing.T) { + f(t, profSelfMapsTests) + }) + + t.Run("WithDeletedFile", func(t *testing.T) { + f(t, profSelfMapsTestsWithDeleted) + }) } // TestMapping checkes the mapping section of CPU profiles -- cgit v1.3-5-g9baa From 42257a262c94d839364113f2dbf4057731971fc1 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Wed, 5 Sep 2018 14:36:20 -0700 Subject: runtime: in semasleep, subtract time spent so far from timeout When pthread_cond_timedwait_relative_np gets a spurious wakeup (due to a signal, typically), we used to retry with the same relative timeout. That's incorrect, we should lower the timeout by the time we've spent in this function so far. In the worst case, signals come in and cause spurious wakeups faster than the timeout, causing semasleep to never time out. Also fix nacl and netbsd while we're here. They have similar issues. Fixes #27520 Change-Id: I6601e120e44a4b8ef436eef75a1e7c8cf1d39e39 Reviewed-on: https://go-review.googlesource.com/133655 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/os_darwin.go | 11 ++++++++++- src/runtime/os_nacl.go | 12 ++++++------ src/runtime/os_netbsd.go | 31 ++++++++++++++----------------- 3 files changed, 30 insertions(+), 24 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/os_darwin.go b/src/runtime/os_darwin.go index d2144edf2e..26b02820cd 100644 --- a/src/runtime/os_darwin.go +++ b/src/runtime/os_darwin.go @@ -34,6 +34,10 @@ func semacreate(mp *m) { //go:nosplit func semasleep(ns int64) int32 { + var start int64 + if ns >= 0 { + start = nanotime() + } mp := getg().m pthread_mutex_lock(&mp.mutex) for { @@ -43,8 +47,13 @@ func semasleep(ns int64) int32 { return 0 } if ns >= 0 { + spent := nanotime() - start + if spent >= ns { + pthread_mutex_unlock(&mp.mutex) + return -1 + } var t timespec - t.set_nsec(ns) + t.set_nsec(ns - spent) err := pthread_cond_timedwait_relative_np(&mp.cond, &mp.mutex, &t) if err == _ETIMEDOUT { pthread_mutex_unlock(&mp.mutex) diff --git a/src/runtime/os_nacl.go b/src/runtime/os_nacl.go index 23ab03b953..ac7bf69582 100644 --- a/src/runtime/os_nacl.go +++ b/src/runtime/os_nacl.go @@ -197,23 +197,23 @@ func semacreate(mp *m) { //go:nosplit func semasleep(ns int64) int32 { var ret int32 - systemstack(func() { _g_ := getg() if nacl_mutex_lock(_g_.m.waitsemalock) < 0 { throw("semasleep") } - + var ts timespec + if ns >= 0 { + end := ns + nanotime() + ts.tv_sec = end / 1e9 + ts.tv_nsec = int32(end % 1e9) + } for _g_.m.waitsemacount == 0 { if ns < 0 { if nacl_cond_wait(_g_.m.waitsema, _g_.m.waitsemalock) < 0 { throw("semasleep") } } else { - var ts timespec - end := ns + nanotime() - ts.tv_sec = end / 1e9 - ts.tv_nsec = int32(end % 1e9) r := nacl_cond_timed_wait_abs(_g_.m.waitsema, _g_.m.waitsemalock, &ts) if r == -_ETIMEDOUT { nacl_mutex_unlock(_g_.m.waitsemalock) diff --git a/src/runtime/os_netbsd.go b/src/runtime/os_netbsd.go index a9bf407a36..7deab3ed03 100644 --- a/src/runtime/os_netbsd.go +++ b/src/runtime/os_netbsd.go @@ -126,15 +126,9 @@ func semacreate(mp *m) { //go:nosplit func semasleep(ns int64) int32 { _g_ := getg() - - // Compute sleep deadline. - var tsp *timespec - var ts timespec + var deadline int64 if ns >= 0 { - var nsec int32 - ts.set_sec(timediv(ns, 1000000000, &nsec)) - ts.set_nsec(nsec) - tsp = &ts + deadline = nanotime() + ns } for { @@ -147,18 +141,21 @@ func semasleep(ns int64) int32 { } // Sleep until unparked by semawakeup or timeout. + var tsp *timespec + var ts timespec + if ns >= 0 { + wait := deadline - nanotime() + if wait <= 0 { + return -1 + } + var nsec int32 + ts.set_sec(timediv(wait, 1000000000, &nsec)) + ts.set_nsec(nsec) + tsp = &ts + } ret := lwp_park(_CLOCK_MONOTONIC, _TIMER_RELTIME, tsp, 0, unsafe.Pointer(&_g_.m.waitsemacount), nil) if ret == _ETIMEDOUT { return -1 - } else if ret == _EINTR && ns >= 0 { - // Avoid sleeping forever if we keep getting - // interrupted (for example by the profiling - // timer). It would be if tsp upon return had the - // remaining time to sleep, but this is good enough. - var nsec int32 - ns /= 2 - ts.set_sec(timediv(ns, 1000000000, &nsec)) - ts.set_nsec(nsec) } } } -- cgit v1.3-5-g9baa From 43f54c8d2e3bddfc6ad7887286eb6564986cb6ad Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Thu, 6 Sep 2018 17:21:59 -0700 Subject: runtime: use tgkill for raise raise uses tkill to send a signal to the current thread. For this use, tgkill is functionally equivalent to tkill expect that it also takes the pid as the first argument. Using tgkill makes it simpler to run a Go program in a strict sandbox. With kill and tgkill, the sandbox policy (e.g., seccomp) can prevent the program from sending signals to other processes by checking that the first argument == getpid(). With tkill, the policy must whitelist all tids in the process, which is effectively impossible given Go's dynamic thread creation. Fixes #27548 Change-Id: I8ed282ef1f7215b02ef46de144493e36454029ea Reviewed-on: https://go-review.googlesource.com/133975 Run-TryBot: Michael Pratt TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/sys_linux_386.s | 11 +++++++---- src/runtime/sys_linux_amd64.s | 12 ++++++++---- src/runtime/sys_linux_arm.s | 12 ++++++++---- src/runtime/sys_linux_arm64.s | 12 ++++++++---- src/runtime/sys_linux_mips64x.s | 12 ++++++++---- src/runtime/sys_linux_mipsx.s | 12 ++++++++---- src/runtime/sys_linux_ppc64x.s | 11 +++++++---- src/runtime/sys_linux_s390x.s | 12 ++++++++---- 8 files changed, 62 insertions(+), 32 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 8d5a4ff977..4e914f3e60 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -48,7 +48,6 @@ #define SYS_mincore 218 #define SYS_madvise 219 #define SYS_gettid 224 -#define SYS_tkill 238 #define SYS_futex 240 #define SYS_sched_getaffinity 242 #define SYS_set_thread_area 243 @@ -57,6 +56,7 @@ #define SYS_epoll_ctl 255 #define SYS_epoll_wait 256 #define SYS_clock_gettime 265 +#define SYS_tgkill 270 #define SYS_epoll_create1 329 TEXT runtime·exit(SB),NOSPLIT,$0 @@ -155,11 +155,14 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT,$12 + MOVL $SYS_getpid, AX + INVOKE_SYSCALL + MOVL AX, BX // arg 1 pid MOVL $SYS_gettid, AX INVOKE_SYSCALL - MOVL AX, BX // arg 1 tid - MOVL sig+0(FP), CX // arg 2 signal - MOVL $SYS_tkill, AX + MOVL AX, CX // arg 2 tid + MOVL sig+0(FP), DX // arg 3 signal + MOVL $SYS_tgkill, AX INVOKE_SYSCALL RET diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 62d80247be..4492dad02e 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -36,12 +36,12 @@ #define SYS_sigaltstack 131 #define SYS_arch_prctl 158 #define SYS_gettid 186 -#define SYS_tkill 200 #define SYS_futex 202 #define SYS_sched_getaffinity 204 #define SYS_epoll_create 213 #define SYS_exit_group 231 #define SYS_epoll_ctl 233 +#define SYS_tgkill 234 #define SYS_openat 257 #define SYS_faccessat 269 #define SYS_epoll_pwait 281 @@ -137,11 +137,15 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT,$0 + MOVL $SYS_getpid, AX + SYSCALL + MOVL AX, R12 MOVL $SYS_gettid, AX SYSCALL - MOVL AX, DI // arg 1 tid - MOVL sig+0(FP), SI // arg 2 - MOVL $SYS_tkill, AX + MOVL AX, SI // arg 2 tid + MOVL R12, DI // arg 1 pid + MOVL sig+0(FP), DX // arg 3 + MOVL $SYS_tgkill, AX SYSCALL RET diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index aa39732cfb..a709c4cbd0 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -36,7 +36,7 @@ #define SYS_setitimer (SYS_BASE + 104) #define SYS_mincore (SYS_BASE + 219) #define SYS_gettid (SYS_BASE + 224) -#define SYS_tkill (SYS_BASE + 238) +#define SYS_tgkill (SYS_BASE + 268) #define SYS_sched_yield (SYS_BASE + 158) #define SYS_nanosleep (SYS_BASE + 162) #define SYS_sched_getaffinity (SYS_BASE + 242) @@ -138,11 +138,15 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 + MOVW $SYS_getpid, R7 + SWI $0 + MOVW R0, R4 MOVW $SYS_gettid, R7 SWI $0 - // arg 1 tid already in R0 from gettid - MOVW sig+0(FP), R1 // arg 2 - signal - MOVW $SYS_tkill, R7 + MOVW R0, R1 // arg 2 tid + MOVW R4, R0 // arg 1 pid + MOVW sig+0(FP), R2 // arg 3 + MOVW $SYS_tgkill, R7 SWI $0 RET diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 1c8fce3db6..086c8ddc63 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -36,7 +36,7 @@ #define SYS_getpid 172 #define SYS_gettid 178 #define SYS_kill 129 -#define SYS_tkill 130 +#define SYS_tgkill 131 #define SYS_futex 98 #define SYS_sched_getaffinity 123 #define SYS_exit_group 94 @@ -143,11 +143,15 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 + MOVD $SYS_getpid, R8 + SVC + MOVW R0, R19 MOVD $SYS_gettid, R8 SVC - MOVW R0, R0 // arg 1 tid - MOVW sig+0(FP), R1 // arg 2 - MOVD $SYS_tkill, R8 + MOVW R0, R1 // arg 2 tid + MOVW R19, R0 // arg 1 pid + MOVW sig+0(FP), R2 // arg 3 + MOVD $SYS_tgkill, R8 SVC RET diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 8e64f1c562..337299ba5f 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -35,12 +35,12 @@ #define SYS_madvise 5027 #define SYS_mincore 5026 #define SYS_gettid 5178 -#define SYS_tkill 5192 #define SYS_futex 5194 #define SYS_sched_getaffinity 5196 #define SYS_exit_group 5205 #define SYS_epoll_create 5207 #define SYS_epoll_ctl 5208 +#define SYS_tgkill 5225 #define SYS_openat 5247 #define SYS_epoll_pwait 5272 #define SYS_clock_gettime 5222 @@ -137,11 +137,15 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 + MOVV $SYS_getpid, R2 + SYSCALL + MOVW R2, R16 MOVV $SYS_gettid, R2 SYSCALL - MOVW R2, R4 // arg 1 tid - MOVW sig+0(FP), R5 // arg 2 - MOVV $SYS_tkill, R2 + MOVW R2, R5 // arg 2 tid + MOVW R16, R4 // arg 1 pid + MOVW sig+0(FP), R6 // arg 3 + MOVV $SYS_tgkill, R2 SYSCALL RET diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index a6bca3bebd..dca5f1ee45 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -35,7 +35,6 @@ #define SYS_madvise 4218 #define SYS_mincore 4217 #define SYS_gettid 4222 -#define SYS_tkill 4236 #define SYS_futex 4238 #define SYS_sched_getaffinity 4240 #define SYS_exit_group 4246 @@ -43,6 +42,7 @@ #define SYS_epoll_ctl 4249 #define SYS_epoll_wait 4250 #define SYS_clock_gettime 4263 +#define SYS_tgkill 4266 #define SYS_epoll_create1 4326 TEXT runtime·exit(SB),NOSPLIT,$0-4 @@ -135,11 +135,15 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT,$0-4 + MOVW $SYS_getpid, R2 + SYSCALL + MOVW R2, R16 MOVW $SYS_gettid, R2 SYSCALL - MOVW R2, R4 // arg 1 tid - MOVW sig+0(FP), R5 // arg 2 - MOVW $SYS_tkill, R2 + MOVW R2, R5 // arg 2 tid + MOVW R16, R4 // arg 1 pid + MOVW sig+0(FP), R6 // arg 3 + MOVW $SYS_tgkill, R2 SYSCALL RET diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index 075adf2368..7c2f8ea637 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -36,7 +36,6 @@ #define SYS_madvise 205 #define SYS_mincore 206 #define SYS_gettid 207 -#define SYS_tkill 208 #define SYS_futex 221 #define SYS_sched_getaffinity 223 #define SYS_exit_group 234 @@ -44,6 +43,7 @@ #define SYS_epoll_ctl 237 #define SYS_epoll_wait 238 #define SYS_clock_gettime 246 +#define SYS_tgkill 250 #define SYS_epoll_create1 315 TEXT runtime·exit(SB),NOSPLIT|NOFRAME,$0-4 @@ -123,10 +123,13 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 + SYSCALL $SYS_getpid + MOVW R3, R14 SYSCALL $SYS_gettid - MOVW R3, R3 // arg 1 tid - MOVW sig+0(FP), R4 // arg 2 - SYSCALL $SYS_tkill + MOVW R3, R4 // arg 2 tid + MOVW R14, R3 // arg 1 pid + MOVW sig+0(FP), R5 // arg 3 + SYSCALL $SYS_tgkill RET TEXT runtime·raiseproc(SB),NOSPLIT|NOFRAME,$0 diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index 1ff110c232..95401af62e 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -31,9 +31,9 @@ #define SYS_madvise 219 #define SYS_mincore 218 #define SYS_gettid 236 -#define SYS_tkill 237 #define SYS_futex 238 #define SYS_sched_getaffinity 240 +#define SYS_tgkill 241 #define SYS_exit_group 248 #define SYS_epoll_create 249 #define SYS_epoll_ctl 250 @@ -129,11 +129,15 @@ TEXT runtime·gettid(SB),NOSPLIT,$0-4 RET TEXT runtime·raise(SB),NOSPLIT|NOFRAME,$0 + MOVW $SYS_getpid, R1 + SYSCALL + MOVW R2, R10 MOVW $SYS_gettid, R1 SYSCALL - MOVW R2, R2 // arg 1 tid - MOVW sig+0(FP), R3 // arg 2 - MOVW $SYS_tkill, R1 + MOVW R2, R3 // arg 2 tid + MOVW R10, R2 // arg 1 pid + MOVW sig+0(FP), R4 // arg 2 + MOVW $SYS_tgkill, R1 SYSCALL RET -- cgit v1.3-5-g9baa From 178a609fed5fba5abaeead485f7b2795b8c4ea3c Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Mon, 10 Sep 2018 01:05:09 -0600 Subject: runtime: convert initial timediv quotient increments to bitsets At the very beginning of timediv, inside a for loop, we reduce the base value by at most (1<<31)-1, while incrementing the quotient result by 1< TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/runtime1.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index d5f78baded..85a9ba2521 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -416,7 +416,9 @@ func timediv(v int64, div int32, rem *int32) int32 { for bit := 30; bit >= 0; bit-- { if v >= int64(div)<= int64(div) { -- cgit v1.3-5-g9baa From b07f60b97ff5ea9ae4cf21b549e9d25ccd695f36 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 12 Sep 2018 14:18:45 +0200 Subject: runtime: use functions from mem_bsd.go on Darwin The implementations of the functions in mem_darwin.go is identical to the ones defined in mem_bsd.go for all other BSD-like GOOSes. Also use them on Darwin. Change-Id: Ie7c170c1a50666475e79599471081cd85f0837ad Reviewed-on: https://go-review.googlesource.com/134875 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/mem_bsd.go | 2 +- src/runtime/mem_darwin.go | 62 ----------------------------------------------- 2 files changed, 1 insertion(+), 63 deletions(-) delete mode 100644 src/runtime/mem_darwin.go (limited to 'src/runtime') diff --git a/src/runtime/mem_bsd.go b/src/runtime/mem_bsd.go index cc70e806ea..13065b61d4 100644 --- a/src/runtime/mem_bsd.go +++ b/src/runtime/mem_bsd.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build dragonfly freebsd nacl netbsd openbsd solaris +// +build darwin dragonfly freebsd nacl netbsd openbsd solaris package runtime diff --git a/src/runtime/mem_darwin.go b/src/runtime/mem_darwin.go deleted file mode 100644 index 75c59f9cdd..0000000000 --- a/src/runtime/mem_darwin.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package runtime - -import "unsafe" - -// Don't split the stack as this function may be invoked without a valid G, -// which prevents us from allocating more stack. -//go:nosplit -func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer { - v, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, -1, 0) - if err != 0 { - return nil - } - mSysStatInc(sysStat, n) - return v -} - -func sysUnused(v unsafe.Pointer, n uintptr) { - // Linux's MADV_DONTNEED is like BSD's MADV_FREE. - madvise(v, n, _MADV_FREE) -} - -func sysUsed(v unsafe.Pointer, n uintptr) { -} - -// Don't split the stack as this function may be invoked without a valid G, -// which prevents us from allocating more stack. -//go:nosplit -func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) { - mSysStatDec(sysStat, n) - munmap(v, n) -} - -func sysFault(v unsafe.Pointer, n uintptr) { - mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, -1, 0) -} - -func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer { - p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, -1, 0) - if err != 0 { - return nil - } - return p -} - -const ( - _ENOMEM = 12 -) - -func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) { - mSysStatInc(sysStat, n) - p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, -1, 0) - if err == _ENOMEM { - throw("runtime: out of memory") - } - if p != v || err != 0 { - throw("runtime: cannot map pages in arena address space") - } -} -- cgit v1.3-5-g9baa From 1b9374450bbf3372f18915deb58ed11a072eef4a Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Wed, 12 Sep 2018 12:22:42 -0600 Subject: runtime: regression test for semasleep indefinite hang A regression test in which: for a program that invokes semasleep, we send non-terminal signals such as SIGIO. Since the signal wakes up pthread_cond_timedwait_relative_np, after CL 133655, we should only re-spin for the amount of time left, instead of re-spinning with the original duration which would cause an indefinite spin. Updates #27520 Change-Id: I744a6d04cf8923bc4e13649446aff5e42b7de5d8 Reviewed-on: https://go-review.googlesource.com/135015 Run-TryBot: Emmanuel Odeke TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/semasleep_test.go | 88 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/runtime/semasleep_test.go (limited to 'src/runtime') diff --git a/src/runtime/semasleep_test.go b/src/runtime/semasleep_test.go new file mode 100644 index 0000000000..4a8b4db338 --- /dev/null +++ b/src/runtime/semasleep_test.go @@ -0,0 +1,88 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//+build !nacl,!windows,!js + +package runtime_test + +import ( + "internal/testenv" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" +) + +// Issue #27250. Spurious wakeups to pthread_cond_timedwait_relative_np +// shouldn't cause semasleep to retry with the same timeout which would +// cause indefinite spinning. +func TestSpuriousWakeupsNeverHangSemasleep(t *testing.T) { + testenv.MustHaveGoBuild(t) + tempDir, err := ioutil.TempDir("", "issue-27250") + if err != nil { + t.Fatalf("Failed to create the temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + repro := ` + package main + + import "time" + + func main() { + <-time.After(1 * time.Second) + } + ` + mainPath := filepath.Join(tempDir, "main.go") + if err := ioutil.WriteFile(mainPath, []byte(repro), 0644); err != nil { + t.Fatalf("Failed to create temp file for repro.go: %v", err) + } + binaryPath := filepath.Join(tempDir, "binary") + + // Build the binary so that we can send the signal to its PID. + out, err := exec.Command(testenv.GoToolPath(t), "build", "-o", binaryPath, mainPath).CombinedOutput() + if err != nil { + t.Fatalf("Failed to compile the binary: err: %v\nOutput: %s\n", err, out) + } + if err := os.Chmod(binaryPath, 0755); err != nil { + t.Fatalf("Failed to chmod binary: %v", err) + } + + // Now run the binary. + cmd := exec.Command(binaryPath) + if err := cmd.Start(); err != nil { + t.Fatalf("Failed to start command: %v", err) + } + doneCh := make(chan error, 1) + go func() { + doneCh <- cmd.Wait() + }() + + // With the repro running, we can continuously send to it + // a non-terminal signal such as SIGIO, to spuriously + // wakeup pthread_cond_timedwait_relative_np. + unfixedTimer := time.NewTimer(2 * time.Second) + for { + select { + case <-time.After(200 * time.Millisecond): + // Send the pesky signal that toggles spinning + // indefinitely if #27520 is not fixed. + cmd.Process.Signal(syscall.SIGIO) + + case <-unfixedTimer.C: + t.Error("Program failed to return on time and has to be killed, issue #27520 still exists") + cmd.Process.Signal(syscall.SIGKILL) + return + + case err := <-doneCh: + if err != nil { + t.Fatalf("The program returned but unfortunately with an error: %v", err) + } + return + } + } +} -- cgit v1.3-5-g9baa From 0ef42f4dd6b12821641fe415f16e425bb094137b Mon Sep 17 00:00:00 2001 From: erifan01 Date: Wed, 12 Sep 2018 10:31:04 +0000 Subject: runtime: skip TestGcSys on arm64 This failure occurs randomly on arm64. 13:10:32 --- FAIL: TestGcSys (0.06s) 13:10:32 gc_test.go:30: expected "OK\n", but got "using too much memory: 71401472 bytes\n" 13:10:32 FAIL Updates #27636 Change-Id: Ifd4cfce167d8054dc6f037bd34368d63c7f68ed4 Reviewed-on: https://go-review.googlesource.com/135155 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/gc_test.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/gc_test.go b/src/runtime/gc_test.go index 0da19cdf34..1f7715c672 100644 --- a/src/runtime/gc_test.go +++ b/src/runtime/gc_test.go @@ -24,6 +24,9 @@ func TestGcSys(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("skipping test; GOOS=windows http://golang.org/issue/27156") } + if runtime.GOOS == "linux" && runtime.GOARCH == "arm64" { + t.Skip("skipping test; GOOS=linux GOARCH=arm64 https://github.com/golang/go/issues/27636") + } got := runTestProg(t, "testprog", "GCSys") want := "OK\n" if got != want { -- cgit v1.3-5-g9baa From 7c95703c090757c4ca3f0792357e1595b03d6bca Mon Sep 17 00:00:00 2001 From: David du Colombier <0intro@gmail.com> Date: Thu, 13 Sep 2018 21:15:34 +0200 Subject: runtime: don't build semasleep_test on Plan 9 CL 135015 added TestSpuriousWakeupsNeverHangSemasleep. However, this test is failing on Plan 9 because syscall.SIGIO is not defined. This change excludes semasleep_test.go on Plan 9 Fixes #27662. Change-Id: I52f9f0fe9ec3c70da5d2f586a95debbc1fe568a1 Reviewed-on: https://go-review.googlesource.com/135315 Run-TryBot: David du Colombier <0intro@gmail.com> Reviewed-by: Emmanuel Odeke TryBot-Result: Gobot Gobot --- src/runtime/semasleep_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/semasleep_test.go b/src/runtime/semasleep_test.go index 4a8b4db338..5b2cc64483 100644 --- a/src/runtime/semasleep_test.go +++ b/src/runtime/semasleep_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//+build !nacl,!windows,!js +//+build !nacl,!plan9,!windows,!js package runtime_test -- cgit v1.3-5-g9baa From 0e21cc2ba0823f2130d950eccf7c023b161d1331 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Fri, 14 Sep 2018 10:04:12 -0700 Subject: runtime: use CleanCmdEnv in TestRuntimePanic This makes TestRuntimePanic keep most of the existing environment, just as the other runtime tests do. Change-Id: I7944abfeee292d41716dca14483134a50d75f081 Reviewed-on: https://go-review.googlesource.com/135376 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Brad Fitzpatrick --- src/runtime/crash_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/crash_test.go b/src/runtime/crash_test.go index 2766b8850a..6835cacb3f 100644 --- a/src/runtime/crash_test.go +++ b/src/runtime/crash_test.go @@ -686,7 +686,7 @@ func init() { func TestRuntimePanic(t *testing.T) { testenv.MustHaveExec(t) - cmd := exec.Command(os.Args[0], "-test.run=TestRuntimePanic") + cmd := testenv.CleanCmdEnv(exec.Command(os.Args[0], "-test.run=TestRuntimePanic")) cmd.Env = append(cmd.Env, "GO_TEST_RUNTIME_PANIC=1") out, err := cmd.CombinedOutput() t.Logf("%s", out) -- cgit v1.3-5-g9baa From 0670b28bf031c5f33ca9831ee894e67bcd79c64c Mon Sep 17 00:00:00 2001 From: Koichi Shiraishi Date: Sat, 15 Sep 2018 17:06:08 +0900 Subject: runtime: fix TODO comment filepath The cmd/compile/internal/ld/go.go file not exist, actually cmd/link/internal/ld/go.go. Also, write line number is not good because it changes every commit of the file. Change-Id: Id2b9f2c9904390adb011dab357716ee8e2fe84fc Reviewed-on: https://go-review.googlesource.com/135516 Reviewed-by: Ian Lance Taylor --- src/runtime/sys_darwin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/sys_darwin.go b/src/runtime/sys_darwin.go index 7efbef746c..9b0cc6f935 100644 --- a/src/runtime/sys_darwin.go +++ b/src/runtime/sys_darwin.go @@ -370,5 +370,5 @@ func closeonexec(fd int32) { //go:cgo_import_dynamic libc_pthread_cond_signal pthread_cond_signal "/usr/lib/libSystem.B.dylib" // Magic incantation to get libSystem actually dynamically linked. -// TODO: Why does the code require this? See cmd/compile/internal/ld/go.go:210 +// TODO: Why does the code require this? See cmd/link/internal/ld/go.go //go:cgo_import_dynamic _ _ "/usr/lib/libSystem.B.dylib" -- cgit v1.3-5-g9baa From 19ac6a82d3be818572881d60026109946a5a69e6 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Tue, 18 Sep 2018 07:58:11 -0700 Subject: runtime: ignore EAGAIN from exec in TestCgoExecSignalMask Fixes #27731 Change-Id: Ifb4d57923b1bba0210ec1f623d779d7b5f442812 Reviewed-on: https://go-review.googlesource.com/135995 Run-TryBot: Ian Lance Taylor Reviewed-by: Michael Munday TryBot-Result: Gobot Gobot --- src/runtime/testdata/testprogcgo/exec.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/testdata/testprogcgo/exec.go b/src/runtime/testdata/testprogcgo/exec.go index 2e948401c8..94da5dc526 100644 --- a/src/runtime/testdata/testprogcgo/exec.go +++ b/src/runtime/testdata/testprogcgo/exec.go @@ -75,6 +75,14 @@ func CgoExecSignalMask() { cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { + // An overloaded system + // may fail with EAGAIN. + // This doesn't tell us + // anything useful; ignore it. + // Issue #27731. + if isEAGAIN(err) { + return + } fmt.Printf("iteration %d: %v\n", j, err) os.Exit(1) } @@ -87,3 +95,11 @@ func CgoExecSignalMask() { fmt.Println("OK") } + +// isEAGAIN reports whether err is an EAGAIN error from a process execution. +func isEAGAIN(err error) bool { + if p, ok := err.(*os.PathError); ok { + err = p.Err + } + return err == syscall.EAGAIN +} -- cgit v1.3-5-g9baa From 77f9b2728eb08456899e6500328e00ec4829dddf Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Fri, 14 Sep 2018 09:57:06 +0200 Subject: runtime: use MADV_FREE on Linux if available On Linux, sysUnused currently uses madvise(MADV_DONTNEED) to signal the kernel that a range of allocated memory contains unneeded data. After a successful call, the range (but not the data it contained before the call to madvise) is still available but the first access to that range will unconditionally incur a page fault (needed to 0-fill the range). A faster alternative is MADV_FREE, available since Linux 4.5. The mechanism is very similar, but the page fault will only be incurred if the kernel, between the call to madvise and the first access, decides to reuse that memory for something else. In sysUnused, test whether MADV_FREE is supported and fall back to MADV_DONTNEED in case it isn't. This requires making the return value of the madvise syscall available to the caller, so change runtime.madvise to return it. Fixes #23687 Change-Id: I962c3429000dd9f4a00846461ad128b71201bb04 Reviewed-on: https://go-review.googlesource.com/135395 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/defs2_linux.go | 5 ++++- src/runtime/defs_linux.go | 5 ++++- src/runtime/defs_linux_386.go | 1 + src/runtime/defs_linux_amd64.go | 1 + src/runtime/defs_linux_arm.go | 1 + src/runtime/defs_linux_arm64.go | 1 + src/runtime/defs_linux_mips64x.go | 1 + src/runtime/defs_linux_mipsx.go | 1 + src/runtime/defs_linux_ppc64.go | 1 + src/runtime/defs_linux_ppc64le.go | 1 + src/runtime/defs_linux_s390x.go | 1 + src/runtime/mem_linux.go | 13 +++++++++++-- src/runtime/stubs2.go | 3 ++- src/runtime/sys_dragonfly_amd64.s | 6 ++++-- src/runtime/sys_freebsd_386.s | 4 +++- src/runtime/sys_freebsd_amd64.s | 6 ++++-- src/runtime/sys_freebsd_arm.s | 15 ++++++++------- src/runtime/sys_linux_386.s | 2 +- src/runtime/sys_linux_amd64.s | 2 +- src/runtime/sys_linux_arm.s | 2 +- src/runtime/sys_linux_arm64.s | 2 +- src/runtime/sys_linux_mips64x.s | 2 +- src/runtime/sys_linux_mipsx.s | 4 ++-- src/runtime/sys_linux_ppc64x.s | 2 +- src/runtime/sys_linux_s390x.s | 2 +- src/runtime/sys_netbsd_386.s | 4 +++- src/runtime/sys_netbsd_amd64.s | 4 +++- src/runtime/sys_netbsd_arm.s | 11 ++++++----- src/runtime/sys_openbsd_386.s | 3 ++- src/runtime/sys_openbsd_amd64.s | 4 +++- src/runtime/sys_openbsd_arm.s | 4 ++-- 31 files changed, 77 insertions(+), 37 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/defs2_linux.go b/src/runtime/defs2_linux.go index c10dfb8624..b08c0dafe1 100644 --- a/src/runtime/defs2_linux.go +++ b/src/runtime/defs2_linux.go @@ -58,7 +58,10 @@ const ( MAP_PRIVATE = C.MAP_PRIVATE MAP_FIXED = C.MAP_FIXED - MADV_DONTNEED = C.MADV_DONTNEED + MADV_DONTNEED = C.MADV_DONTNEED + MADV_FREE = C.MADV_FREE + MADV_HUGEPAGE = C.MADV_HUGEPAGE + MADV_NOHUGEPAGE = C.MADV_HNOUGEPAGE SA_RESTART = C.SA_RESTART SA_ONSTACK = C.SA_ONSTACK diff --git a/src/runtime/defs_linux.go b/src/runtime/defs_linux.go index 553366a50b..2d810136d9 100644 --- a/src/runtime/defs_linux.go +++ b/src/runtime/defs_linux.go @@ -47,7 +47,10 @@ const ( MAP_PRIVATE = C.MAP_PRIVATE MAP_FIXED = C.MAP_FIXED - MADV_DONTNEED = C.MADV_DONTNEED + MADV_DONTNEED = C.MADV_DONTNEED + MADV_FREE = C.MADV_FREE + MADV_HUGEPAGE = C.MADV_HUGEPAGE + MADV_NOHUGEPAGE = C.MADV_HNOUGEPAGE SA_RESTART = C.SA_RESTART SA_ONSTACK = C.SA_ONSTACK diff --git a/src/runtime/defs_linux_386.go b/src/runtime/defs_linux_386.go index a7e435f854..0ebac17aef 100644 --- a/src/runtime/defs_linux_386.go +++ b/src/runtime/defs_linux_386.go @@ -18,6 +18,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_amd64.go b/src/runtime/defs_linux_amd64.go index e8c6a212db..c0a0ef0dd4 100644 --- a/src/runtime/defs_linux_amd64.go +++ b/src/runtime/defs_linux_amd64.go @@ -18,6 +18,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_arm.go b/src/runtime/defs_linux_arm.go index 62ec8fab5e..43946bb79c 100644 --- a/src/runtime/defs_linux_arm.go +++ b/src/runtime/defs_linux_arm.go @@ -16,6 +16,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_arm64.go b/src/runtime/defs_linux_arm64.go index c295bc0257..c2cc281ab4 100644 --- a/src/runtime/defs_linux_arm64.go +++ b/src/runtime/defs_linux_arm64.go @@ -18,6 +18,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_mips64x.go b/src/runtime/defs_linux_mips64x.go index df11cb0965..9dacd5d1e9 100644 --- a/src/runtime/defs_linux_mips64x.go +++ b/src/runtime/defs_linux_mips64x.go @@ -18,6 +18,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_mipsx.go b/src/runtime/defs_linux_mipsx.go index 702fbb51c8..9532ac54ee 100644 --- a/src/runtime/defs_linux_mipsx.go +++ b/src/runtime/defs_linux_mipsx.go @@ -22,6 +22,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_ppc64.go b/src/runtime/defs_linux_ppc64.go index 45363d1285..5a4326da07 100644 --- a/src/runtime/defs_linux_ppc64.go +++ b/src/runtime/defs_linux_ppc64.go @@ -18,6 +18,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_ppc64le.go b/src/runtime/defs_linux_ppc64le.go index 45363d1285..5a4326da07 100644 --- a/src/runtime/defs_linux_ppc64le.go +++ b/src/runtime/defs_linux_ppc64le.go @@ -18,6 +18,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/defs_linux_s390x.go b/src/runtime/defs_linux_s390x.go index ab90723f75..a6cc9c48e9 100644 --- a/src/runtime/defs_linux_s390x.go +++ b/src/runtime/defs_linux_s390x.go @@ -19,6 +19,7 @@ const ( _MAP_FIXED = 0x10 _MADV_DONTNEED = 0x4 + _MADV_FREE = 0x8 _MADV_HUGEPAGE = 0xe _MADV_NOHUGEPAGE = 0xf diff --git a/src/runtime/mem_linux.go b/src/runtime/mem_linux.go index 7aa48170a1..845f72ded2 100644 --- a/src/runtime/mem_linux.go +++ b/src/runtime/mem_linux.go @@ -5,6 +5,7 @@ package runtime import ( + "runtime/internal/atomic" "runtime/internal/sys" "unsafe" ) @@ -34,10 +35,12 @@ func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer { return p } +var adviseUnused = uint32(_MADV_FREE) + func sysUnused(v unsafe.Pointer, n uintptr) { // By default, Linux's "transparent huge page" support will // merge pages into a huge page if there's even a single - // present regular page, undoing the effects of the DONTNEED + // present regular page, undoing the effects of madvise(adviseUnused) // below. On amd64, that means khugepaged can turn a single // 4KB page to 2MB, bloating the process's RSS by as much as // 512X. (See issue #8832 and Linux kernel bug @@ -102,7 +105,13 @@ func sysUnused(v unsafe.Pointer, n uintptr) { throw("unaligned sysUnused") } - madvise(v, n, _MADV_DONTNEED) + advise := atomic.Load(&adviseUnused) + if errno := madvise(v, n, int32(advise)); advise == _MADV_FREE && errno != 0 { + // MADV_FREE was added in Linux 4.5. Fall back to MADV_DONTNEED if it is + // not supported. + atomic.Store(&adviseUnused, _MADV_DONTNEED) + madvise(v, n, _MADV_DONTNEED) + } } func sysUsed(v unsafe.Pointer, n uintptr) { diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index 02249d0aad..c14db74003 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -25,7 +25,8 @@ func write(fd uintptr, p unsafe.Pointer, n int32) int32 //go:noescape func open(name *byte, mode, perm int32) int32 -func madvise(addr unsafe.Pointer, n uintptr, flags int32) +// return value is only set on linux to be used in osinit() +func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 // exitThread terminates the current thread, writing *wait = 0 when // the stack is safe to reclaim. diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index f0eb5f4e21..b18e967651 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -260,9 +260,11 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL flags+16(FP), DX MOVQ $75, AX // madvise SYSCALL - // ignore failure - maybe pages are locked + JCC 2(PC) + MOVL $-1, AX + MOVL AX, ret+24(FP) RET - + TEXT runtime·sigaltstack(SB),NOSPLIT,$-8 MOVQ new+0(FP), DI MOVQ old+8(FP), SI diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s index b8f685a323..754689ba05 100644 --- a/src/runtime/sys_freebsd_386.s +++ b/src/runtime/sys_freebsd_386.s @@ -163,7 +163,9 @@ TEXT runtime·munmap(SB),NOSPLIT,$-4 TEXT runtime·madvise(SB),NOSPLIT,$-4 MOVL $75, AX // madvise INT $0x80 - // ignore failure - maybe pages are locked + JAE 2(PC) + MOVL $-1, AX + MOVL AX, ret+12(FP) RET TEXT runtime·setitimer(SB), NOSPLIT, $-4 diff --git a/src/runtime/sys_freebsd_amd64.s b/src/runtime/sys_freebsd_amd64.s index be191a0784..55959b3e3a 100644 --- a/src/runtime/sys_freebsd_amd64.s +++ b/src/runtime/sys_freebsd_amd64.s @@ -337,9 +337,11 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL flags+16(FP), DX MOVQ $75, AX // madvise SYSCALL - // ignore failure - maybe pages are locked + JCC 2(PC) + MOVL $-1, AX + MOVL AX, ret+24(FP) RET - + TEXT runtime·sigaltstack(SB),NOSPLIT,$-8 MOVQ new+0(FP), DI MOVQ old+8(FP), SI diff --git a/src/runtime/sys_freebsd_arm.s b/src/runtime/sys_freebsd_arm.s index 93bf569367..f347b9fa96 100644 --- a/src/runtime/sys_freebsd_arm.s +++ b/src/runtime/sys_freebsd_arm.s @@ -264,14 +264,15 @@ TEXT runtime·munmap(SB),NOSPLIT,$0 RET TEXT runtime·madvise(SB),NOSPLIT,$0 - MOVW addr+0(FP), R0 // arg 1 addr - MOVW n+4(FP), R1 // arg 2 len - MOVW flags+8(FP), R2 // arg 3 flags - MOVW $SYS_madvise, R7 - SWI $0 - // ignore failure - maybe pages are locked + MOVW addr+0(FP), R0 // arg 1 addr + MOVW n+4(FP), R1 // arg 2 len + MOVW flags+8(FP), R2 // arg 3 flags + MOVW $SYS_madvise, R7 + SWI $0 + MOVW.CS $-1, R0 + MOVW R0, ret+12(FP) RET - + TEXT runtime·sigaltstack(SB),NOSPLIT|NOFRAME,$0 MOVW new+0(FP), R0 MOVW old+4(FP), R1 diff --git a/src/runtime/sys_linux_386.s b/src/runtime/sys_linux_386.s index 4e914f3e60..40b55a67eb 100644 --- a/src/runtime/sys_linux_386.s +++ b/src/runtime/sys_linux_386.s @@ -427,7 +427,7 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL n+4(FP), CX MOVL flags+8(FP), DX INVOKE_SYSCALL - // ignore failure - maybe pages are locked + MOVL AX, ret+12(FP) RET // int32 futex(int32 *uaddr, int32 op, int32 val, diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 4492dad02e..7e846371e5 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -519,7 +519,7 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL flags+16(FP), DX MOVQ $SYS_madvise, AX SYSCALL - // ignore failure - maybe pages are locked + MOVL AX, ret+24(FP) RET // int64 futex(int32 *uaddr, int32 op, int32 val, diff --git a/src/runtime/sys_linux_arm.s b/src/runtime/sys_linux_arm.s index a709c4cbd0..43a58335c8 100644 --- a/src/runtime/sys_linux_arm.s +++ b/src/runtime/sys_linux_arm.s @@ -195,7 +195,7 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVW flags+8(FP), R2 MOVW $SYS_madvise, R7 SWI $0 - // ignore failure - maybe pages are locked + MOVW R0, ret+12(FP) RET TEXT runtime·setitimer(SB),NOSPLIT,$0 diff --git a/src/runtime/sys_linux_arm64.s b/src/runtime/sys_linux_arm64.s index 086c8ddc63..8b344be8f8 100644 --- a/src/runtime/sys_linux_arm64.s +++ b/src/runtime/sys_linux_arm64.s @@ -401,7 +401,7 @@ TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 MOVW flags+16(FP), R2 MOVD $SYS_madvise, R8 SVC - // ignore failure - maybe pages are locked + MOVW R0, ret+24(FP) RET // int64 futex(int32 *uaddr, int32 op, int32 val, diff --git a/src/runtime/sys_linux_mips64x.s b/src/runtime/sys_linux_mips64x.s index 337299ba5f..c45703d228 100644 --- a/src/runtime/sys_linux_mips64x.s +++ b/src/runtime/sys_linux_mips64x.s @@ -291,7 +291,7 @@ TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 MOVW flags+16(FP), R6 MOVV $SYS_madvise, R2 SYSCALL - // ignore failure - maybe pages are locked + MOVW R2, ret+24(FP) RET // int64 futex(int32 *uaddr, int32 op, int32 val, diff --git a/src/runtime/sys_linux_mipsx.s b/src/runtime/sys_linux_mipsx.s index dca5f1ee45..f362b0f3f1 100644 --- a/src/runtime/sys_linux_mipsx.s +++ b/src/runtime/sys_linux_mipsx.s @@ -302,13 +302,13 @@ TEXT runtime·munmap(SB),NOSPLIT,$0-8 UNDEF // crash RET -TEXT runtime·madvise(SB),NOSPLIT,$0-12 +TEXT runtime·madvise(SB),NOSPLIT,$0-16 MOVW addr+0(FP), R4 MOVW n+4(FP), R5 MOVW flags+8(FP), R6 MOVW $SYS_madvise, R2 SYSCALL - // ignore failure - maybe pages are locked + MOVW R2, ret+12(FP) RET // int32 futex(int32 *uaddr, int32 op, int32 val, struct timespec *timeout, int32 *uaddr2, int32 val2); diff --git a/src/runtime/sys_linux_ppc64x.s b/src/runtime/sys_linux_ppc64x.s index 7c2f8ea637..ed79b69257 100644 --- a/src/runtime/sys_linux_ppc64x.s +++ b/src/runtime/sys_linux_ppc64x.s @@ -454,7 +454,7 @@ TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 MOVD n+8(FP), R4 MOVW flags+16(FP), R5 SYSCALL $SYS_madvise - // ignore failure - maybe pages are locked + MOVW R3, ret+24(FP) RET // int64 futex(int32 *uaddr, int32 op, int32 val, diff --git a/src/runtime/sys_linux_s390x.s b/src/runtime/sys_linux_s390x.s index 95401af62e..c79ceea751 100644 --- a/src/runtime/sys_linux_s390x.s +++ b/src/runtime/sys_linux_s390x.s @@ -290,7 +290,7 @@ TEXT runtime·madvise(SB),NOSPLIT|NOFRAME,$0 MOVW flags+16(FP), R4 MOVW $SYS_madvise, R1 SYSCALL - // ignore failure - maybe pages are locked + MOVW R2, ret+24(FP) RET // int64 futex(int32 *uaddr, int32 op, int32 val, diff --git a/src/runtime/sys_netbsd_386.s b/src/runtime/sys_netbsd_386.s index 4042ab4f8a..66f4620cab 100644 --- a/src/runtime/sys_netbsd_386.s +++ b/src/runtime/sys_netbsd_386.s @@ -135,7 +135,9 @@ TEXT runtime·munmap(SB),NOSPLIT,$-4 TEXT runtime·madvise(SB),NOSPLIT,$-4 MOVL $75, AX // sys_madvise INT $0x80 - // ignore failure - maybe pages are locked + JAE 2(PC) + MOVL $-1, AX + MOVL AX, ret+12(FP) RET TEXT runtime·setitimer(SB),NOSPLIT,$-4 diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index 11b9c1b417..5523659196 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -319,7 +319,9 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL flags+16(FP), DX // arg 3 - behav MOVQ $75, AX // sys_madvise SYSCALL - // ignore failure - maybe pages are locked + JCC 2(PC) + MOVL $-1, AX + MOVL AX, ret+24(FP) RET TEXT runtime·sigaltstack(SB),NOSPLIT,$-8 diff --git a/src/runtime/sys_netbsd_arm.s b/src/runtime/sys_netbsd_arm.s index 6b2c5a8357..304075f295 100644 --- a/src/runtime/sys_netbsd_arm.s +++ b/src/runtime/sys_netbsd_arm.s @@ -284,11 +284,12 @@ TEXT runtime·munmap(SB),NOSPLIT,$0 RET TEXT runtime·madvise(SB),NOSPLIT,$0 - MOVW addr+0(FP), R0 // arg 1 - addr - MOVW n+4(FP), R1 // arg 2 - len - MOVW flags+8(FP), R2 // arg 3 - behav - SWI $0xa0004b // sys_madvise - // ignore failure - maybe pages are locked + MOVW addr+0(FP), R0 // arg 1 - addr + MOVW n+4(FP), R1 // arg 2 - len + MOVW flags+8(FP), R2 // arg 3 - behav + SWI $0xa0004b // sys_madvise + MOVW.CS $-1, R0 + MOVW R0, ret+12(FP) RET TEXT runtime·sigaltstack(SB),NOSPLIT|NOFRAME,$0 diff --git a/src/runtime/sys_openbsd_386.s b/src/runtime/sys_openbsd_386.s index 21f13c806e..8e34ab497a 100644 --- a/src/runtime/sys_openbsd_386.s +++ b/src/runtime/sys_openbsd_386.s @@ -136,7 +136,8 @@ TEXT runtime·madvise(SB),NOSPLIT,$-4 MOVL $75, AX // sys_madvise INT $0x80 JAE 2(PC) - MOVL $0xf1, 0xf1 // crash + MOVL $-1, AX + MOVL AX, ret+12(FP) RET TEXT runtime·setitimer(SB),NOSPLIT,$-4 diff --git a/src/runtime/sys_openbsd_amd64.s b/src/runtime/sys_openbsd_amd64.s index 38ac38d9bf..227e81869c 100644 --- a/src/runtime/sys_openbsd_amd64.s +++ b/src/runtime/sys_openbsd_amd64.s @@ -305,7 +305,9 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVL flags+16(FP), DX // arg 3 - behav MOVQ $75, AX // sys_madvise SYSCALL - // ignore failure - maybe pages are locked + JCC 2(PC) + MOVL $-1, AX + MOVL AX, ret+24(FP) RET TEXT runtime·sigaltstack(SB),NOSPLIT,$-8 diff --git a/src/runtime/sys_openbsd_arm.s b/src/runtime/sys_openbsd_arm.s index ff1c1da9b9..52d3638bc1 100644 --- a/src/runtime/sys_openbsd_arm.s +++ b/src/runtime/sys_openbsd_arm.s @@ -143,8 +143,8 @@ TEXT runtime·madvise(SB),NOSPLIT,$0 MOVW flags+8(FP), R2 // arg 2 - flags MOVW $75, R12 // sys_madvise SWI $0 - MOVW.CS $0, R8 // crash on syscall failure - MOVW.CS R8, (R8) + MOVW.CS $-1, R0 + MOVW R0, ret+12(FP) RET TEXT runtime·setitimer(SB),NOSPLIT,$0 -- cgit v1.3-5-g9baa From 83dfc3b001245f0b725afdc94c0b540fe1952d21 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Mon, 17 Sep 2018 12:25:36 -0700 Subject: runtime: ignore races between close and len/cap They aren't really races, or at least they don't have any observable effect. The spec is silent on whether these are actually races or not. Fix this problem by not using the address of len (or of cap) as the location where channel operations are recorded to occur. Use a random other field of hchan for that. I'm not 100% sure we should in fact fix this. Opinions welcome. Fixes #27070 Change-Id: Ib4efd4b62e0d1ef32fa51e373035ef207a655084 Reviewed-on: https://go-review.googlesource.com/135698 Reviewed-by: Dmitry Vyukov --- src/runtime/chan.go | 23 ++++++++++++++++------- src/runtime/race/testdata/chan_test.go | 30 ++++++++++++++++++++++-------- src/runtime/select.go | 4 ++-- 3 files changed, 40 insertions(+), 17 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/chan.go b/src/runtime/chan.go index 615643e6a6..a4ee51ca39 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -92,7 +92,7 @@ func makechan(t *chantype, size int) *hchan { // Queue or element size is zero. c = (*hchan)(mallocgc(hchanSize, nil, true)) // Race detector uses this location for synchronization. - c.buf = unsafe.Pointer(c) + c.buf = c.raceaddr() case elem.kind&kindNoPointers != 0: // Elements do not contain pointers. // Allocate hchan and buf in one call. @@ -151,7 +151,7 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { } if raceenabled { - racereadpc(unsafe.Pointer(c), callerpc, funcPC(chansend)) + racereadpc(c.raceaddr(), callerpc, funcPC(chansend)) } // Fast path: check for failed non-blocking operation without acquiring the lock. @@ -337,8 +337,8 @@ func closechan(c *hchan) { if raceenabled { callerpc := getcallerpc() - racewritepc(unsafe.Pointer(c), callerpc, funcPC(closechan)) - racerelease(unsafe.Pointer(c)) + racewritepc(c.raceaddr(), callerpc, funcPC(closechan)) + racerelease(c.raceaddr()) } c.closed = 1 @@ -361,7 +361,7 @@ func closechan(c *hchan) { gp := sg.g gp.param = nil if raceenabled { - raceacquireg(gp, unsafe.Pointer(c)) + raceacquireg(gp, c.raceaddr()) } glist.push(gp) } @@ -379,7 +379,7 @@ func closechan(c *hchan) { gp := sg.g gp.param = nil if raceenabled { - raceacquireg(gp, unsafe.Pointer(c)) + raceacquireg(gp, c.raceaddr()) } glist.push(gp) } @@ -454,7 +454,7 @@ func chanrecv(c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) if c.closed != 0 && c.qcount == 0 { if raceenabled { - raceacquire(unsafe.Pointer(c)) + raceacquire(c.raceaddr()) } unlock(&c.lock) if ep != nil { @@ -732,6 +732,15 @@ func (q *waitq) dequeue() *sudog { } } +func (c *hchan) raceaddr() unsafe.Pointer { + // Treat read-like and write-like operations on the channel to + // happen at this address. Avoid using the address of qcount + // or dataqsiz, because the len() and cap() builtins read + // those addresses, and we don't want them racing with + // operations like close(). + return unsafe.Pointer(&c.buf) +} + func racesync(c *hchan, sg *sudog) { racerelease(chanbuf(c, 0)) raceacquireg(sg.g, chanbuf(c, 0)) diff --git a/src/runtime/race/testdata/chan_test.go b/src/runtime/race/testdata/chan_test.go index 7f349c42ed..60e55ed66a 100644 --- a/src/runtime/race/testdata/chan_test.go +++ b/src/runtime/race/testdata/chan_test.go @@ -577,18 +577,32 @@ func TestRaceChanItselfCap(t *testing.T) { <-compl } -func TestRaceChanCloseLen(t *testing.T) { - v := 0 - _ = v +func TestNoRaceChanCloseLen(t *testing.T) { c := make(chan int, 10) - c <- 0 + r := make(chan int, 10) + go func() { + r <- len(c) + }() go func() { - v = 1 close(c) + r <- 0 }() - time.Sleep(1e7) - _ = len(c) - v = 2 + <-r + <-r +} + +func TestNoRaceChanCloseCap(t *testing.T) { + c := make(chan int, 10) + r := make(chan int, 10) + go func() { + r <- cap(c) + }() + go func() { + close(c) + r <- 0 + }() + <-r + <-r } func TestRaceChanCloseSend(t *testing.T) { diff --git a/src/runtime/select.go b/src/runtime/select.go index 3a3ac6b7ac..2729c2ecf9 100644 --- a/src/runtime/select.go +++ b/src/runtime/select.go @@ -245,7 +245,7 @@ loop: case caseSend: if raceenabled { - racereadpc(unsafe.Pointer(c), cas.pc, chansendpc) + racereadpc(c.raceaddr(), cas.pc, chansendpc) } if c.closed != 0 { goto sclose @@ -462,7 +462,7 @@ rclose: typedmemclr(c.elemtype, cas.elem) } if raceenabled { - raceacquire(unsafe.Pointer(c)) + raceacquire(c.raceaddr()) } goto retc -- cgit v1.3-5-g9baa From d24ec86e4f64302e91d8062eabfcd21296bb9b04 Mon Sep 17 00:00:00 2001 From: Jordan Rhee Date: Wed, 8 Aug 2018 14:44:42 -0700 Subject: runtime: support windows/arm Updates #26148 Change-Id: I8f68b2c926c7b11dc86c9664ed7ff2d2f78b64b4 Reviewed-on: https://go-review.googlesource.com/128715 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/vet/all/whitelist/windows_386.txt | 1 - src/cmd/vet/all/whitelist/windows_amd64.txt | 1 - src/runtime/asm_arm.s | 4 + src/runtime/defs_windows_386.go | 3 + src/runtime/defs_windows_amd64.go | 3 + src/runtime/defs_windows_arm.go | 149 + src/runtime/os_windows.go | 30 +- src/runtime/os_windows_arm.go | 18 + src/runtime/rt0_windows_arm.s | 12 + src/runtime/signal_windows.go | 10 +- src/runtime/sys_windows_arm.s | 605 ++++ src/runtime/syscall_windows.go | 24 +- src/runtime/tls_arm.s | 43 +- src/runtime/wincallback.go | 42 +- src/runtime/zcallback_windows.go | 2 +- src/runtime/zcallback_windows.s | 4 +- src/runtime/zcallback_windows_arm.s | 4012 +++++++++++++++++++++++++++ 17 files changed, 4942 insertions(+), 21 deletions(-) create mode 100644 src/runtime/defs_windows_arm.go create mode 100644 src/runtime/os_windows_arm.go create mode 100644 src/runtime/rt0_windows_arm.s create mode 100644 src/runtime/sys_windows_arm.s create mode 100644 src/runtime/zcallback_windows_arm.s (limited to 'src/runtime') diff --git a/src/cmd/vet/all/whitelist/windows_386.txt b/src/cmd/vet/all/whitelist/windows_386.txt index 788684a49d..d910022ef6 100644 --- a/src/cmd/vet/all/whitelist/windows_386.txt +++ b/src/cmd/vet/all/whitelist/windows_386.txt @@ -3,7 +3,6 @@ runtime/sys_windows_386.s: [386] profileloop: use of 4(SP) points beyond argument frame runtime/sys_windows_386.s: [386] ctrlhandler: 4(SP) should be _type+0(FP) runtime/sys_windows_386.s: [386] setldt: function setldt missing Go declaration -runtime/zcallback_windows.s: [386] callbackasm: function callbackasm missing Go declaration runtime/sys_windows_386.s: [386] callbackasm1+0: function callbackasm1+0 missing Go declaration runtime/sys_windows_386.s: [386] tstart: function tstart missing Go declaration runtime/sys_windows_386.s: [386] tstart_stdcall: RET without writing to 4-byte ret+4(FP) diff --git a/src/cmd/vet/all/whitelist/windows_amd64.txt b/src/cmd/vet/all/whitelist/windows_amd64.txt index 3be4602579..676e6baf71 100644 --- a/src/cmd/vet/all/whitelist/windows_amd64.txt +++ b/src/cmd/vet/all/whitelist/windows_amd64.txt @@ -6,4 +6,3 @@ runtime/sys_windows_amd64.s: [amd64] callbackasm1: function callbackasm1 missing runtime/sys_windows_amd64.s: [amd64] tstart_stdcall: RET without writing to 4-byte ret+8(FP) runtime/sys_windows_amd64.s: [amd64] settls: function settls missing Go declaration runtime/sys_windows_amd64.s: [amd64] cannot check cross-package assembly function: now is in package time -runtime/zcallback_windows.s: [amd64] callbackasm: function callbackasm missing Go declaration diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index 6722ba760f..ace2b6def7 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -784,6 +784,9 @@ TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0 MOVW R0, g // Save g to thread-local storage. +#ifdef GOOS_windows + B runtime·save_g(SB) +#else MOVB runtime·iscgo(SB), R0 CMP $0, R0 B.EQ 2(PC) @@ -791,6 +794,7 @@ TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0 MOVW g, R0 RET +#endif TEXT runtime·emptyfunc(SB),0,$0-0 RET diff --git a/src/runtime/defs_windows_386.go b/src/runtime/defs_windows_386.go index 589a7884cd..38b30b70e3 100644 --- a/src/runtime/defs_windows_386.go +++ b/src/runtime/defs_windows_386.go @@ -104,6 +104,9 @@ type context struct { func (c *context) ip() uintptr { return uintptr(c.eip) } func (c *context) sp() uintptr { return uintptr(c.esp) } +// 386 does not have link register, so this returns 0. +func (c *context) lr() uintptr { return 0 } + func (c *context) setip(x uintptr) { c.eip = uint32(x) } func (c *context) setsp(x uintptr) { c.esp = uint32(x) } diff --git a/src/runtime/defs_windows_amd64.go b/src/runtime/defs_windows_amd64.go index 1e173e934d..37508c09be 100644 --- a/src/runtime/defs_windows_amd64.go +++ b/src/runtime/defs_windows_amd64.go @@ -119,6 +119,9 @@ type context struct { func (c *context) ip() uintptr { return uintptr(c.rip) } func (c *context) sp() uintptr { return uintptr(c.rsp) } +// Amd64 does not have link register, so this returns 0. +func (c *context) lr() uintptr { return 0 } + func (c *context) setip(x uintptr) { c.rip = uint64(x) } func (c *context) setsp(x uintptr) { c.rsp = uint64(x) } diff --git a/src/runtime/defs_windows_arm.go b/src/runtime/defs_windows_arm.go new file mode 100644 index 0000000000..1140f61651 --- /dev/null +++ b/src/runtime/defs_windows_arm.go @@ -0,0 +1,149 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +const ( + _PROT_NONE = 0 + _PROT_READ = 1 + _PROT_WRITE = 2 + _PROT_EXEC = 4 + + _MAP_ANON = 1 + _MAP_PRIVATE = 2 + + _DUPLICATE_SAME_ACCESS = 0x2 + _THREAD_PRIORITY_HIGHEST = 0x2 + + _SIGINT = 0x2 + _CTRL_C_EVENT = 0x0 + _CTRL_BREAK_EVENT = 0x1 + + _CONTEXT_CONTROL = 0x10001 + _CONTEXT_FULL = 0x10007 + + _EXCEPTION_ACCESS_VIOLATION = 0xc0000005 + _EXCEPTION_BREAKPOINT = 0x80000003 + _EXCEPTION_FLT_DENORMAL_OPERAND = 0xc000008d + _EXCEPTION_FLT_DIVIDE_BY_ZERO = 0xc000008e + _EXCEPTION_FLT_INEXACT_RESULT = 0xc000008f + _EXCEPTION_FLT_OVERFLOW = 0xc0000091 + _EXCEPTION_FLT_UNDERFLOW = 0xc0000093 + _EXCEPTION_INT_DIVIDE_BY_ZERO = 0xc0000094 + _EXCEPTION_INT_OVERFLOW = 0xc0000095 + + _INFINITE = 0xffffffff + _WAIT_TIMEOUT = 0x102 + + _EXCEPTION_CONTINUE_EXECUTION = -0x1 + _EXCEPTION_CONTINUE_SEARCH = 0x0 +) + +type systeminfo struct { + anon0 [4]byte + dwpagesize uint32 + lpminimumapplicationaddress *byte + lpmaximumapplicationaddress *byte + dwactiveprocessormask uint32 + dwnumberofprocessors uint32 + dwprocessortype uint32 + dwallocationgranularity uint32 + wprocessorlevel uint16 + wprocessorrevision uint16 +} + +type exceptionrecord struct { + exceptioncode uint32 + exceptionflags uint32 + exceptionrecord *exceptionrecord + exceptionaddress *byte + numberparameters uint32 + exceptioninformation [15]uint32 +} + +type neon128 struct { + low uint64 + high int64 +} + +type context struct { + contextflags uint32 + r0 uint32 + r1 uint32 + r2 uint32 + r3 uint32 + r4 uint32 + r5 uint32 + r6 uint32 + r7 uint32 + r8 uint32 + r9 uint32 + r10 uint32 + r11 uint32 + r12 uint32 + + spr uint32 + lrr uint32 + pc uint32 + cpsr uint32 + + fpscr uint32 + padding uint32 + + floatNeon [16]neon128 + + bvr [8]uint32 + bcr [8]uint32 + wvr [1]uint32 + wcr [1]uint32 + padding2 [2]uint32 +} + +func (c *context) ip() uintptr { return uintptr(c.pc) } +func (c *context) sp() uintptr { return uintptr(c.spr) } +func (c *context) lr() uintptr { return uintptr(c.lrr) } + +func (c *context) setip(x uintptr) { c.pc = uint32(x) } +func (c *context) setsp(x uintptr) { c.spr = uint32(x) } + +func dumpregs(r *context) { + print("r0 ", hex(r.r0), "\n") + print("r1 ", hex(r.r1), "\n") + print("r2 ", hex(r.r2), "\n") + print("r3 ", hex(r.r3), "\n") + print("r4 ", hex(r.r4), "\n") + print("r5 ", hex(r.r5), "\n") + print("r6 ", hex(r.r6), "\n") + print("r7 ", hex(r.r7), "\n") + print("r8 ", hex(r.r8), "\n") + print("r9 ", hex(r.r9), "\n") + print("r10 ", hex(r.r10), "\n") + print("r11 ", hex(r.r11), "\n") + print("r12 ", hex(r.r12), "\n") + print("sp ", hex(r.spr), "\n") + print("lr ", hex(r.lrr), "\n") + print("pc ", hex(r.pc), "\n") + print("cpsr ", hex(r.cpsr), "\n") +} + +type overlapped struct { + internal uint32 + internalhigh uint32 + anon0 [8]byte + hevent *byte +} + +type memoryBasicInformation struct { + baseAddress uintptr + allocationBase uintptr + allocationProtect uint32 + regionSize uintptr + state uint32 + protect uint32 + type_ uint32 +} + +func stackcheck() { + // TODO: not implemented on ARM +} diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 5607bf95c1..409d537839 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -43,6 +43,7 @@ const ( //go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll" //go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll" //go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll" +//go:cgo_import_dynamic runtime._TlsAlloc TlsAlloc%0 "kernel32.dll" //go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll" //go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll" //go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll" @@ -91,6 +92,7 @@ var ( _SetWaitableTimer, _SuspendThread, _SwitchToThread, + _TlsAlloc, _VirtualAlloc, _VirtualFree, _VirtualQuery, @@ -860,14 +862,34 @@ func profilem(mp *m) { var r *context rbuf := make([]byte, unsafe.Sizeof(*r)+15) - tls := &mp.tls[0] - gp := *((**g)(unsafe.Pointer(tls))) - // align Context to 16 bytes r = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&rbuf[15]))) &^ 15)) r.contextflags = _CONTEXT_CONTROL stdcall2(_GetThreadContext, mp.thread, uintptr(unsafe.Pointer(r))) - sigprof(r.ip(), r.sp(), 0, gp, mp) + + var gp *g + switch GOARCH { + default: + panic("unsupported architecture") + case "arm": + // TODO(jordanrh1): this is incorrect when Go is executing + // on the system or signal stacks because curg returns + // the current user g. The true g is stored in thread + // local storage, which we cannot access from another CPU. + // We cannot pull R10 from the thread context because + // it might be executing C code, in which case R10 + // would not be g. + gp = mp.curg + case "386", "amd64": + tls := &mp.tls[0] + gp = *((**g)(unsafe.Pointer(tls))) + } + + if gp == nil { + sigprofNonGoPC(r.ip()) + } else { + sigprof(r.ip(), r.sp(), 0, gp, mp) + } } func profileloop1(param uintptr) uint32 { diff --git a/src/runtime/os_windows_arm.go b/src/runtime/os_windows_arm.go new file mode 100644 index 0000000000..3115f7241d --- /dev/null +++ b/src/runtime/os_windows_arm.go @@ -0,0 +1,18 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +//go:nosplit +func cputicks() int64 { + return nanotime() +} + +func checkgoarm() { + if goarm < 7 { + print("Need atomic synchronization instructions, coprocessor ", + "access instructions. Recompile using GOARM=7.\n") + exit(1) + } +} diff --git a/src/runtime/rt0_windows_arm.s b/src/runtime/rt0_windows_arm.s new file mode 100644 index 0000000000..c5787d0dee --- /dev/null +++ b/src/runtime/rt0_windows_arm.s @@ -0,0 +1,12 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" + +// This is the entry point for the program from the +// kernel for an ordinary -buildmode=exe program. +TEXT _rt0_arm_windows(SB),NOSPLIT|NOFRAME,$0 + B ·rt0_go(SB) diff --git a/src/runtime/signal_windows.go b/src/runtime/signal_windows.go index a63450038d..873ce66abe 100644 --- a/src/runtime/signal_windows.go +++ b/src/runtime/signal_windows.go @@ -27,7 +27,7 @@ func lastcontinuetramp() func initExceptionHandler() { stdcall2(_AddVectoredExceptionHandler, 1, funcPC(exceptiontramp)) - if _AddVectoredContinueHandler == nil || unsafe.Sizeof(&_AddVectoredContinueHandler) == 4 { + if _AddVectoredContinueHandler == nil || GOARCH == "386" { // use SetUnhandledExceptionFilter for windows-386 or // if VectoredContinueHandler is unavailable. // note: SetUnhandledExceptionFilter handler won't be called, if debugging. @@ -177,9 +177,15 @@ func lastcontinuehandler(info *exceptionrecord, r *context, gp *g) int32 { } print("\n") + // TODO(jordanrh1): This may be needed for 386/AMD64 as well. + if GOARCH == "arm" { + _g_.m.throwing = 1 + _g_.m.caughtsig.set(gp) + } + level, _, docrash := gotraceback() if level > 0 { - tracebacktrap(r.ip(), r.sp(), 0, gp) + tracebacktrap(r.ip(), r.sp(), r.lr(), gp) tracebackothers(gp) dumpregs(r) } diff --git a/src/runtime/sys_windows_arm.s b/src/runtime/sys_windows_arm.s new file mode 100644 index 0000000000..409c72c554 --- /dev/null +++ b/src/runtime/sys_windows_arm.s @@ -0,0 +1,605 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" + +// void runtime·asmstdcall(void *c); +TEXT runtime·asmstdcall(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R5, R14], (R13) // push {r4, r5, lr} + MOVW R0, R4 // put libcall * in r4 + MOVW R13, R5 // save stack pointer in r5 + + // SetLastError(0) + MOVW $0, R0 + MRC 15, 0, R1, C13, C0, 2 + MOVW R0, 0x34(R1) + + MOVW 8(R4), R12 // libcall->args + + // Do we have more than 4 arguments? + MOVW 4(R4), R0 // libcall->n + SUB.S $4, R0, R2 + BLE loadregs + + // Reserve stack space for remaining args + SUB R2<<2, R13 + BIC $0x7, R13 // alignment for ABI + + // R0: count of arguments + // R1: + // R2: loop counter, from 0 to (n-4) + // R3: scratch + // R4: pointer to libcall struct + // R12: libcall->args + MOVW $0, R2 +stackargs: + ADD $4, R2, R3 // r3 = args[4 + i] + MOVW R3<<2(R12), R3 + MOVW R3, R2<<2(R13) // stack[i] = r3 + + ADD $1, R2 // i++ + SUB $4, R0, R3 // while (i < (n - 4)) + CMP R3, R2 + BLT stackargs + +loadregs: + CMP $3, R0 + MOVW.GT 12(R12), R3 + + CMP $2, R0 + MOVW.GT 8(R12), R2 + + CMP $1, R0 + MOVW.GT 4(R12), R1 + + CMP $0, R0 + MOVW.GT 0(R12), R0 + + BIC $0x7, R13 // alignment for ABI + MOVW 0(R4), R12 // branch to libcall->fn + BL (R12) + + MOVW R5, R13 // free stack space + MOVW R0, 12(R4) // save return value to libcall->r1 + MOVW R1, 16(R4) + + // GetLastError + MRC 15, 0, R1, C13, C0, 2 + MOVW 0x34(R1), R0 + MOVW R0, 20(R4) // store in libcall->err + + MOVM.IA.W (R13), [R4, R5, R15] + +TEXT runtime·badsignal2(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R14], (R13) // push {r4, lr} + MOVW R13, R4 // save original stack pointer + SUB $8, R13 // space for 2 variables + BIC $0x7, R13 // alignment for ABI + + // stderr + MOVW runtime·_GetStdHandle(SB), R1 + MOVW $-12, R0 + BL (R1) + + MOVW $runtime·badsignalmsg(SB), R1 // lpBuffer + MOVW $runtime·badsignallen(SB), R2 // lpNumberOfBytesToWrite + MOVW (R2), R2 + ADD $0x4, R13, R3 // lpNumberOfBytesWritten + MOVW $0, R12 // lpOverlapped + MOVW R12, (R13) + + MOVW runtime·_WriteFile(SB), R12 + BL (R12) + + MOVW R4, R13 // restore SP + MOVM.IA.W (R13), [R4, R15] // pop {r4, pc} + +TEXT runtime·getlasterror(SB),NOSPLIT,$0 + MRC 15, 0, R0, C13, C0, 2 + MOVW 0x34(R0), R0 + MOVW R0, ret+0(FP) + RET + +TEXT runtime·setlasterror(SB),NOSPLIT|NOFRAME,$0 + MRC 15, 0, R1, C13, C0, 2 + MOVW R0, 0x34(R1) + RET + +// Called by Windows as a Vectored Exception Handler (VEH). +// First argument is pointer to struct containing +// exception record and context pointers. +// Handler function is stored in R1 +// Return 0 for 'not handled', -1 for handled. +// int32_t sigtramp( +// PEXCEPTION_POINTERS ExceptionInfo, +// func *GoExceptionHandler); +TEXT runtime·sigtramp(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R0, R4-R11, R14], (R13) // push {r0, r4-r11, lr} (SP-=40) + SUB $(8+20), R13 // reserve space for g, sp, and + // parameters/retval to go call + + MOVW R0, R6 // Save param0 + MOVW R1, R7 // Save param1 + + BL runtime·load_g(SB) + CMP $0, g // is there a current g? + BL.EQ runtime·badsignal2(SB) + + // save g and SP in case of stack switch + MOVW R13, 24(R13) + MOVW g, 20(R13) + + // do we need to switch to the g0 stack? + MOVW g, R5 // R5 = g + MOVW g_m(R5), R2 // R2 = m + MOVW m_g0(R2), R4 // R4 = g0 + CMP R5, R4 // if curg == g0 + BEQ g0 + + // switch to g0 stack + MOVW R4, g // g = g0 + MOVW (g_sched+gobuf_sp)(g), R3 // R3 = g->gobuf.sp + BL runtime·save_g(SB) + + // traceback will think that we've done PUSH and SUB + // on this stack, so subtract them here to match. + // (we need room for sighandler arguments anyway). + // and re-save old SP for restoring later. + SUB $(40+8+20), R3 + MOVW R13, 24(R3) // save old stack pointer + MOVW R3, R13 // switch stack + +g0: + MOVW 0(R6), R2 // R2 = ExceptionPointers->ExceptionRecord + MOVW 4(R6), R3 // R3 = ExceptionPointers->ContextRecord + + // make it look like mstart called us on g0, to stop traceback + MOVW $runtime·mstart(SB), R4 + + MOVW R4, 0(R13) // Save link register for traceback + MOVW R2, 4(R13) // Move arg0 (ExceptionRecord) into position + MOVW R3, 8(R13) // Move arg1 (ContextRecord) into position + MOVW R5, 12(R13) // Move arg2 (original g) into position + BL (R7) // Call the go routine + MOVW 16(R13), R4 // Fetch return value from stack + + // Compute the value of the g0 stack pointer after deallocating + // this frame, then allocating 8 bytes. We may need to store + // the resume SP and PC on the g0 stack to work around + // control flow guard when we resume from the exception. + ADD $(40+20), R13, R12 + + // switch back to original stack and g + MOVW 24(R13), R13 + MOVW 20(R13), g + BL runtime·save_g(SB) + +done: + MOVW R4, R0 // move retval into position + ADD $(8 + 20), R13 // free locals + MOVM.IA.W (R13), [R3, R4-R11, R14] // pop {r3, r4-r11, lr} + + // if return value is CONTINUE_SEARCH, do not set up control + // flow guard workaround + CMP $0, R0 + BEQ return + + // Check if we need to set up the control flow guard workaround. + // On Windows/ARM, the stack pointer must lie within system + // stack limits when we resume from exception. + // Store the resume SP and PC on the g0 stack, + // and return to returntramp on the g0 stack. returntramp + // pops the saved PC and SP from the g0 stack, resuming execution + // at the desired location. + // If returntramp has already been set up by a previous exception + // handler, don't clobber the stored SP and PC on the stack. + MOVW 4(R3), R3 // PEXCEPTION_POINTERS->Context + MOVW 0x40(R3), R2 // load PC from context record + MOVW $runtime·returntramp(SB), R1 + CMP R1, R2 + B.EQ return // do not clobber saved SP/PC + + // Save resume SP and PC on g0 stack + MOVW 0x38(R3), R2 // load SP from context record + MOVW R2, 0(R12) // Store resume SP on g0 stack + MOVW 0x40(R3), R2 // load PC from context record + MOVW R2, 4(R12) // Store resume PC on g0 stack + + // Set up context record to return to returntramp on g0 stack + MOVW R12, 0x38(R3) // save g0 stack pointer + // in context record + MOVW $runtime·returntramp(SB), R2 // save resume address + MOVW R2, 0x40(R3) // in context record + +return: + B (R14) // return + +// +// Trampoline to resume execution from exception handler. +// This is part of the control flow guard workaround. +// It switches stacks and jumps to the continuation address. +// +TEXT runtime·returntramp(SB),NOSPLIT|NOFRAME,$0 + MOVM.IA (R13), [R13, R15] // ldm sp, [sp, pc] + +TEXT runtime·exceptiontramp(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·exceptionhandler(SB), R1 + B runtime·sigtramp(SB) + +TEXT runtime·firstcontinuetramp(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·firstcontinuehandler(SB), R1 + B runtime·sigtramp(SB) + +TEXT runtime·lastcontinuetramp(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·lastcontinuehandler(SB), R1 + B runtime·sigtramp(SB) + +TEXT runtime·ctrlhandler(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·ctrlhandler1(SB), R1 + B runtime·externalthreadhandler(SB) + +TEXT runtime·profileloop(SB),NOSPLIT|NOFRAME,$0 + MOVW $runtime·profileloop1(SB), R1 + B runtime·externalthreadhandler(SB) + +// int32 externalthreadhandler(uint32 arg, int (*func)(uint32)) +// stack layout: +// +----------------+ +// | callee-save | +// | registers | +// +----------------+ +// | m | +// +----------------+ +// 20| g | +// +----------------+ +// 16| func ptr (r1) | +// +----------------+ +// 12| argument (r0) | +//---+----------------+ +// 8 | param1 | +// +----------------+ +// 4 | param0 | +// +----------------+ +// 0 | retval | +// +----------------+ +// +TEXT runtime·externalthreadhandler(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} + SUB $(m__size + g__size + 20), R13 // space for locals + MOVW R0, 12(R13) + MOVW R1, 16(R13) + + // zero out m and g structures + ADD $20, R13, R0 // compute pointer to g + MOVW R0, 4(R13) + MOVW $(m__size + g__size), R0 + MOVW R0, 8(R13) + BL runtime·memclrNoHeapPointers(SB) + + // initialize m and g structures + ADD $20, R13, R2 // R2 = g + ADD $(20 + g__size), R13, R3 // R3 = m + MOVW R2, m_g0(R3) // m->g0 = g + MOVW R3, g_m(R2) // g->m = m + MOVW R2, m_curg(R3) // m->curg = g + + MOVW R2, g + BL runtime·save_g(SB) + + // set up stackguard stuff + MOVW R13, R0 + MOVW R0, g_stack+stack_hi(g) + SUB $(32*1024), R0 + MOVW R0, (g_stack+stack_lo)(g) + MOVW R0, g_stackguard0(g) + MOVW R0, g_stackguard1(g) + + // move argument into position and call function + MOVW 12(R13), R0 + MOVW R0, 4(R13) + MOVW 16(R13), R1 + BL (R1) + + // clear g + MOVW $0, g + BL runtime·save_g(SB) + + MOVW 0(R13), R0 // load return value + ADD $(m__size + g__size + 20), R13 // free locals + MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + +GLOBL runtime·cbctxts(SB), NOPTR, $4 + +TEXT runtime·callbackasm1(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} + SUB $36, R13 // space for locals + + // save callback arguments to stack. We currently support up to 4 arguments + ADD $16, R13, R4 + MOVM.IA [R0-R3], (R4) + + // load cbctxts[i]. The trampoline in zcallback_windows.s puts the callback + // index in R12 + MOVW runtime·cbctxts(SB), R4 + MOVW R12<<2(R4), R4 // R4 holds pointer to wincallbackcontext structure + + // extract callback context + MOVW wincallbackcontext_argsize(R4), R5 + MOVW wincallbackcontext_gobody(R4), R4 + + // we currently support up to 4 arguments + CMP $(4 * 4), R5 + BL.GT runtime·abort(SB) + + // extend argsize by size of return value + ADD $4, R5 + + // Build 'type args struct' + MOVW R4, 4(R13) // fn + ADD $16, R13, R0 // arg (points to r0-r3, ret on stack) + MOVW R0, 8(R13) + MOVW R5, 12(R13) // argsize + + BL runtime·load_g(SB) + BL runtime·cgocallback_gofunc(SB) + + ADD $16, R13, R0 // load arg + MOVW 12(R13), R1 // load argsize + SUB $4, R1 // offset to return value + MOVW R1<<0(R0), R0 // load return value + + ADD $36, R13 // free locals + MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + +// uint32 tstart_stdcall(M *newm); +TEXT runtime·tstart_stdcall(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4-R11, R14], (R13) // push {r4-r11, lr} + + MOVW m_g0(R0), g + MOVW R0, g_m(g) + BL runtime·save_g(SB) + + // Layout new m scheduler stack on os stack. + MOVW R13, R0 + MOVW R0, g_stack+stack_hi(g) + SUB $(64*1024), R0 + MOVW R0, (g_stack+stack_lo)(g) + MOVW R0, g_stackguard0(g) + MOVW R0, g_stackguard1(g) + + BL runtime·emptyfunc(SB) // fault if stack check is wrong + BL runtime·mstart(SB) + + // Exit the thread. + MOVW $0, R0 + MOVM.IA.W (R13), [R4-R11, R15] // pop {r4-r11, pc} + +// onosstack calls fn on OS stack. +// adapted from asm_arm.s : systemstack +// func onosstack(fn unsafe.Pointer, arg uint32) +TEXT runtime·onosstack(SB),NOSPLIT,$0 + MOVW fn+0(FP), R5 // R5 = fn + MOVW arg+4(FP), R6 // R6 = arg + + // This function can be called when there is no g, + // for example, when we are handling a callback on a non-go thread. + // In this case we're already on the system stack. + CMP $0, g + BEQ noswitch + + MOVW g_m(g), R1 // R1 = m + + MOVW m_gsignal(R1), R2 // R2 = gsignal + CMP g, R2 + B.EQ noswitch + + MOVW m_g0(R1), R2 // R2 = g0 + CMP g, R2 + B.EQ noswitch + + MOVW m_curg(R1), R3 + CMP g, R3 + B.EQ switch + + // Bad: g is not gsignal, not g0, not curg. What is it? + // Hide call from linker nosplit analysis. + MOVW $runtime·badsystemstack(SB), R0 + BL (R0) + B runtime·abort(SB) + +switch: + // save our state in g->sched. Pretend to + // be systemstack_switch if the G stack is scanned. + MOVW $runtime·systemstack_switch(SB), R3 + ADD $4, R3, R3 // get past push {lr} + MOVW R3, (g_sched+gobuf_pc)(g) + MOVW R13, (g_sched+gobuf_sp)(g) + MOVW LR, (g_sched+gobuf_lr)(g) + MOVW g, (g_sched+gobuf_g)(g) + + // switch to g0 + MOVW R2, g + MOVW (g_sched+gobuf_sp)(R2), R3 + // make it look like mstart called systemstack on g0, to stop traceback + SUB $4, R3, R3 + MOVW $runtime·mstart(SB), R4 + MOVW R4, 0(R3) + MOVW R3, R13 + + // call target function + MOVW R6, R0 // arg + BL (R5) + + // switch back to g + MOVW g_m(g), R1 + MOVW m_curg(R1), g + MOVW (g_sched+gobuf_sp)(g), R13 + MOVW $0, R3 + MOVW R3, (g_sched+gobuf_sp)(g) + RET + +noswitch: + // Using a tail call here cleans up tracebacks since we won't stop + // at an intermediate systemstack. + MOVW.P 4(R13), R14 // restore LR + MOVW R6, R0 // arg + B (R5) + +// Runs on OS stack. Duration (in 100ns units) is in R0. +TEXT runtime·usleep2(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R14], (R13) // push {r4, lr} + MOVW R13, R4 // Save SP + SUB $8, R13 // R13 = R13 - 8 + BIC $0x7, R13 // Align SP for ABI + RSB $0, R0, R3 // R3 = -R0 + MOVW $0, R1 // R1 = FALSE (alertable) + MOVW $-1, R0 // R0 = handle + MOVW R13, R2 // R2 = pTime + MOVW R3, 0(R2) // time_lo + MOVW R0, 4(R2) // time_hi + MOVW runtime·_NtWaitForSingleObject(SB), R3 + BL (R3) + MOVW R4, R13 // Restore SP + MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} + +// Runs on OS stack. +TEXT runtime·switchtothread(SB),NOSPLIT|NOFRAME,$0 + MOVM.DB.W [R4, R14], (R13) // push {R4, lr} + MOVW R13, R4 + BIC $0x7, R13 // alignment for ABI + MOVW runtime·_SwitchToThread(SB), R0 + BL (R0) + MOVW R4, R13 // restore stack pointer + MOVM.IA.W (R13), [R4, R15] // pop {R4, pc} + +TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0 + B runtime·armPublicationBarrier(SB) + +// never called (cgo not supported) +TEXT runtime·read_tls_fallback(SB),NOSPLIT|NOFRAME,$0 + MOVW $0xabcd, R0 + MOVW R0, (R0) + RET + +// See http://www.dcl.hpi.uni-potsdam.de/research/WRK/2007/08/getting-os-information-the-kuser_shared_data-structure/ +// Must read hi1, then lo, then hi2. The snapshot is valid if hi1 == hi2. +#define _INTERRUPT_TIME 0x7ffe0008 +#define _SYSTEM_TIME 0x7ffe0014 +#define time_lo 0 +#define time_hi1 4 +#define time_hi2 8 + +TEXT runtime·nanotime(SB),NOSPLIT,$0-8 + MOVW $0, R0 + MOVB runtime·useQPCTime(SB), R0 + CMP $0, R0 + BNE useQPC + MOVW $_INTERRUPT_TIME, R3 +loop: + MOVW time_hi1(R3), R1 + MOVW time_lo(R3), R0 + MOVW time_hi2(R3), R2 + CMP R1, R2 + BNE loop + + // wintime = R1:R0, multiply by 100 + MOVW $100, R2 + MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 + MULA R1, R2, R4, R4 + + // wintime*100 = R4:R3, subtract startNano and return + MOVW runtime·startNano+0(SB), R0 + MOVW runtime·startNano+4(SB), R1 + SUB.S R0, R3 + SBC R1, R4 + MOVW R3, ret_lo+0(FP) + MOVW R4, ret_hi+4(FP) + RET +useQPC: + B runtime·nanotimeQPC(SB) // tail call + RET + +TEXT time·now(SB),NOSPLIT,$0-20 + MOVW $0, R0 + MOVB runtime·useQPCTime(SB), R0 + CMP $0, R0 + BNE useQPC + MOVW $_INTERRUPT_TIME, R3 +loop: + MOVW time_hi1(R3), R1 + MOVW time_lo(R3), R0 + MOVW time_hi2(R3), R2 + CMP R1, R2 + BNE loop + + // wintime = R1:R0, multiply by 100 + MOVW $100, R2 + MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 + MULA R1, R2, R4, R4 + + // wintime*100 = R4:R3, subtract startNano and return + MOVW runtime·startNano+0(SB), R0 + MOVW runtime·startNano+4(SB), R1 + SUB.S R0, R3 + SBC R1, R4 + MOVW R3, mono+12(FP) + MOVW R4, mono+16(FP) + + MOVW $_SYSTEM_TIME, R3 +wall: + MOVW time_hi1(R3), R1 + MOVW time_lo(R3), R0 + MOVW time_hi2(R3), R2 + CMP R1, R2 + BNE wall + + // w = R1:R0 in 100ns untis + // convert to Unix epoch (but still 100ns units) + #define delta 116444736000000000 + SUB.S $(delta & 0xFFFFFFFF), R0 + SBC $(delta >> 32), R1 + + // Convert to nSec + MOVW $100, R2 + MULLU R0, R2, (R4, R3) // R4:R3 = R1:R0 * R2 + MULA R1, R2, R4, R4 + // w = R2:R1 in nSec + MOVW R3, R1 // R4:R3 -> R2:R1 + MOVW R4, R2 + + // multiply nanoseconds by reciprocal of 10**9 (scaled by 2**61) + // to get seconds (96 bit scaled result) + MOVW $0x89705f41, R3 // 2**61 * 10**-9 + MULLU R1,R3,(R6,R5) // R7:R6:R5 = R2:R1 * R3 + MOVW $0,R7 + MULALU R2,R3,(R7,R6) + + // unscale by discarding low 32 bits, shifting the rest by 29 + MOVW R6>>29,R6 // R7:R6 = (R7:R6:R5 >> 61) + ORR R7<<3,R6 + MOVW R7>>29,R7 + + // subtract (10**9 * sec) from nsec to get nanosecond remainder + MOVW $1000000000, R5 // 10**9 + MULLU R6,R5,(R9,R8) // R9:R8 = R7:R6 * R5 + MULA R7,R5,R9,R9 + SUB.S R8,R1 // R2:R1 -= R9:R8 + SBC R9,R2 + + // because reciprocal was a truncated repeating fraction, quotient + // may be slightly too small -- adjust to make remainder < 10**9 + CMP R5,R1 // if remainder > 10**9 + SUB.HS R5,R1 // remainder -= 10**9 + ADD.HS $1,R6 // sec += 1 + + MOVW R6,sec_lo+0(FP) + MOVW R7,sec_hi+4(FP) + MOVW R1,nsec+8(FP) + RET +useQPC: + B runtime·nanotimeQPC(SB) // tail call + RET + diff --git a/src/runtime/syscall_windows.go b/src/runtime/syscall_windows.go index 8264070569..0858efaf61 100644 --- a/src/runtime/syscall_windows.go +++ b/src/runtime/syscall_windows.go @@ -25,18 +25,32 @@ func (c *wincallbackcontext) setCleanstack(cleanstack bool) { var ( cbs callbacks cbctxts **wincallbackcontext = &cbs.ctxt[0] // to simplify access to cbs.ctxt in sys_windows_*.s - - callbackasm byte // type isn't really byte, it's code in runtime ) +func callbackasm() + // callbackasmAddr returns address of runtime.callbackasm // function adjusted by i. -// runtime.callbackasm is just a series of CALL instructions -// (each is 5 bytes long), and we want callback to arrive at +// On x86 and amd64, runtime.callbackasm is a series of CALL instructions, +// and we want callback to arrive at // correspondent call instruction instead of start of // runtime.callbackasm. +// On ARM, runtime.callbackasm is a series of mov and branch instructions. +// R12 is loaded with the callback index. Each entry is two instructions, +// hence 8 bytes. func callbackasmAddr(i int) uintptr { - return uintptr(add(unsafe.Pointer(&callbackasm), uintptr(i*5))) + var entrySize int + switch GOARCH { + default: + panic("unsupported architecture") + case "386", "amd64": + entrySize = 5 + case "arm": + // On ARM, each entry is a MOV instruction + // followed by a branch instruction + entrySize = 8 + } + return funcPC(callbackasm) + uintptr(i*entrySize) } //go:linkname compileCallback syscall.compileCallback diff --git a/src/runtime/tls_arm.s b/src/runtime/tls_arm.s index cc547a5db1..e2c945d183 100644 --- a/src/runtime/tls_arm.s +++ b/src/runtime/tls_arm.s @@ -23,6 +23,9 @@ #ifdef GOOS_darwin #define TLSG_IS_VARIABLE #endif +#ifdef GOOS_windows +#define TLSG_IS_VARIABLE +#endif // save_g saves the g register into pthread-provided // thread-local memory, so that we can call externally compiled @@ -35,6 +38,17 @@ TEXT runtime·save_g(SB),NOSPLIT|NOFRAME,$0 // nothing to do as nacl/arm does not use TLS at all. MOVW g, R0 // preserve R0 across call to setg<> RET +#else +#ifdef GOOS_windows + // Save the value in the _TEB->TlsSlots array. + // Effectively implements TlsSetValue(). + MRC 15, 0, R0, C13, C0, 2 + ADD $0xe10, R0 + MOVW $runtime·tls_g(SB), R11 + MOVW (R11), R11 + MOVW g, R11<<2(R0) + MOVW g, R0 // preserve R0 accross call to setg<> + RET #else // If the host does not support MRC the linker will replace it with // a call to runtime.read_tls_fallback which jumps to __kuser_get_tls. @@ -47,6 +61,7 @@ TEXT runtime·save_g(SB),NOSPLIT|NOFRAME,$0 MOVW g, R0 // preserve R0 across call to setg<> RET #endif +#endif // load_g loads the g register from pthread-provided // thread-local memory, for use after calling externally compiled @@ -55,6 +70,16 @@ TEXT runtime·load_g(SB),NOSPLIT,$0 #ifdef GOOS_nacl // nothing to do as nacl/arm does not use TLS at all. RET +#else +#ifdef GOOS_windows + // Get the value from the _TEB->TlsSlots array. + // Effectively implements TlsGetValue(). + MRC 15, 0, R0, C13, C0, 2 + ADD $0xe10, R0 + MOVW $runtime·tls_g(SB), g + MOVW (g), g + MOVW g<<2(R0), g + RET #else // See save_g MRC 15, 0, R0, C13, C0, 3 // fetch TLS base pointer @@ -64,6 +89,7 @@ TEXT runtime·load_g(SB),NOSPLIT,$0 MOVW 0(R0), g RET #endif +#endif // This is called from rt0_go, which runs on the system stack // using the initial stack allocated by the OS. @@ -76,6 +102,20 @@ TEXT runtime·load_g(SB),NOSPLIT,$0 // Declare a dummy word ($4, not $0) to make sure the // frame is 8 bytes and stays 8-byte-aligned. TEXT runtime·_initcgo(SB),NOSPLIT,$4 +#ifdef GOOS_windows + MOVW R13, R4 + BIC $0x7, R13 + MOVW $runtime·_TlsAlloc(SB), R0 + MOVW (R0), R0 + BL (R0) + // Assert that slot is less than 64 so we can use _TEB->TlsSlots + CMP $64, R0 + MOVW $runtime·abort(SB), R1 + BL.GE (R1) + MOVW $runtime·tls_g(SB), R1 + MOVW R0, (R1) + MOVW R4, R13 +#else #ifndef GOOS_nacl // if there is an _cgo_init, call it. MOVW _cgo_init(SB), R4 @@ -91,7 +131,8 @@ TEXT runtime·_initcgo(SB),NOSPLIT,$4 MOVW $setg_gcc<>(SB), R1 // arg 1: setg MOVW g, R0 // arg 0: G BL (R4) // will clobber R0-R3 -#endif +#endif // GOOS_nacl +#endif // GOOS_windows nocgo: RET diff --git a/src/runtime/wincallback.go b/src/runtime/wincallback.go index 9f003aed05..c022916422 100644 --- a/src/runtime/wincallback.go +++ b/src/runtime/wincallback.go @@ -17,11 +17,12 @@ import ( const maxCallback = 2000 -func genasm() { +func genasm386Amd64() { var buf bytes.Buffer - buf.WriteString(`// generated by wincallback.go; run go generate + buf.WriteString(`// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. +// +build 386 amd64 // runtime·callbackasm is called by external code to // execute Go implemented callback function. It is not // called from the start, instead runtime·compilecallback @@ -29,13 +30,43 @@ func genasm() { // appropriately so different callbacks start with different // CALL instruction in runtime·callbackasm. This determines // which Go callback function is executed later on. + TEXT runtime·callbackasm(SB),7,$0 `) for i := 0; i < maxCallback; i++ { buf.WriteString("\tCALL\truntime·callbackasm1(SB)\n") } - err := ioutil.WriteFile("zcallback_windows.s", buf.Bytes(), 0666) + filename := fmt.Sprintf("zcallback_windows.s") + err := ioutil.WriteFile(filename, buf.Bytes(), 0666) + if err != nil { + fmt.Fprintf(os.Stderr, "wincallback: %s\n", err) + os.Exit(2) + } +} + +func genasmArm() { + var buf bytes.Buffer + + buf.WriteString(`// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. + +// External code calls into callbackasm at an offset corresponding +// to the callback index. Callbackasm is a table of MOV and B instructions. +// The MOV instruction loads R12 with the callback index, and the +// B instruction branches to callbackasm1. +// callbackasm1 takes the callback index from R12 and +// indexes into an array that stores information about each callback. +// It then calls the Go implementation for that callback. +#include "textflag.h" + +TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 +`) + for i := 0; i < maxCallback; i++ { + buf.WriteString(fmt.Sprintf("\tMOVW\t$%d, R12\n", i)) + buf.WriteString("\tB\truntime·callbackasm1(SB)\n") + } + + err := ioutil.WriteFile("zcallback_windows_arm.s", buf.Bytes(), 0666) if err != nil { fmt.Fprintf(os.Stderr, "wincallback: %s\n", err) os.Exit(2) @@ -45,7 +76,7 @@ TEXT runtime·callbackasm(SB),7,$0 func gengo() { var buf bytes.Buffer - buf.WriteString(fmt.Sprintf(`// generated by wincallback.go; run go generate + buf.WriteString(fmt.Sprintf(`// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. package runtime @@ -59,6 +90,7 @@ const cb_max = %d // maximum number of windows callbacks allowed } func main() { - genasm() + genasm386Amd64() + genasmArm() gengo() } diff --git a/src/runtime/zcallback_windows.go b/src/runtime/zcallback_windows.go index 9908d4ec23..2c3cb28518 100644 --- a/src/runtime/zcallback_windows.go +++ b/src/runtime/zcallback_windows.go @@ -1,4 +1,4 @@ -// generated by wincallback.go; run go generate +// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. package runtime diff --git a/src/runtime/zcallback_windows.s b/src/runtime/zcallback_windows.s index b9a3a30811..7772eef329 100644 --- a/src/runtime/zcallback_windows.s +++ b/src/runtime/zcallback_windows.s @@ -1,5 +1,6 @@ -// generated by wincallback.go; run go generate +// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. +// +build 386 amd64 // runtime·callbackasm is called by external code to // execute Go implemented callback function. It is not // called from the start, instead runtime·compilecallback @@ -7,6 +8,7 @@ // appropriately so different callbacks start with different // CALL instruction in runtime·callbackasm. This determines // which Go callback function is executed later on. + TEXT runtime·callbackasm(SB),7,$0 CALL runtime·callbackasm1(SB) CALL runtime·callbackasm1(SB) diff --git a/src/runtime/zcallback_windows_arm.s b/src/runtime/zcallback_windows_arm.s new file mode 100644 index 0000000000..f943d84cbf --- /dev/null +++ b/src/runtime/zcallback_windows_arm.s @@ -0,0 +1,4012 @@ +// Code generated by wincallback.go using 'go generate'. DO NOT EDIT. + +// External code calls into callbackasm at an offset corresponding +// to the callback index. Callbackasm is a table of MOV and B instructions. +// The MOV instruction loads R12 with the callback index, and the +// B instruction branches to callbackasm1. +// callbackasm1 takes the callback index from R12 and +// indexes into an array that stores information about each callback. +// It then calls the Go implementation for that callback. +#include "textflag.h" + +TEXT runtime·callbackasm(SB),NOSPLIT|NOFRAME,$0 + MOVW $0, R12 + B runtime·callbackasm1(SB) + MOVW $1, R12 + B runtime·callbackasm1(SB) + MOVW $2, R12 + B runtime·callbackasm1(SB) + MOVW $3, R12 + B runtime·callbackasm1(SB) + MOVW $4, R12 + B runtime·callbackasm1(SB) + MOVW $5, R12 + B runtime·callbackasm1(SB) + MOVW $6, R12 + B runtime·callbackasm1(SB) + MOVW $7, R12 + B runtime·callbackasm1(SB) + MOVW $8, R12 + B runtime·callbackasm1(SB) + MOVW $9, R12 + B runtime·callbackasm1(SB) + MOVW $10, R12 + B runtime·callbackasm1(SB) + MOVW $11, R12 + B runtime·callbackasm1(SB) + MOVW $12, R12 + B runtime·callbackasm1(SB) + MOVW $13, R12 + B runtime·callbackasm1(SB) + MOVW $14, R12 + B runtime·callbackasm1(SB) + MOVW $15, R12 + B runtime·callbackasm1(SB) + MOVW $16, R12 + B runtime·callbackasm1(SB) + MOVW $17, R12 + B runtime·callbackasm1(SB) + MOVW $18, R12 + B runtime·callbackasm1(SB) + MOVW $19, R12 + B runtime·callbackasm1(SB) + MOVW $20, R12 + B runtime·callbackasm1(SB) + MOVW $21, R12 + B runtime·callbackasm1(SB) + MOVW $22, R12 + B runtime·callbackasm1(SB) + MOVW $23, R12 + B runtime·callbackasm1(SB) + MOVW $24, R12 + B runtime·callbackasm1(SB) + MOVW $25, R12 + B runtime·callbackasm1(SB) + MOVW $26, R12 + B runtime·callbackasm1(SB) + MOVW $27, R12 + B runtime·callbackasm1(SB) + MOVW $28, R12 + B runtime·callbackasm1(SB) + MOVW $29, R12 + B runtime·callbackasm1(SB) + MOVW $30, R12 + B runtime·callbackasm1(SB) + MOVW $31, R12 + B runtime·callbackasm1(SB) + MOVW $32, R12 + B runtime·callbackasm1(SB) + MOVW $33, R12 + B runtime·callbackasm1(SB) + MOVW $34, R12 + B runtime·callbackasm1(SB) + MOVW $35, R12 + B runtime·callbackasm1(SB) + MOVW $36, R12 + B runtime·callbackasm1(SB) + MOVW $37, R12 + B runtime·callbackasm1(SB) + MOVW $38, R12 + B runtime·callbackasm1(SB) + MOVW $39, R12 + B runtime·callbackasm1(SB) + MOVW $40, R12 + B runtime·callbackasm1(SB) + MOVW $41, R12 + B runtime·callbackasm1(SB) + MOVW $42, R12 + B runtime·callbackasm1(SB) + MOVW $43, R12 + B runtime·callbackasm1(SB) + MOVW $44, R12 + B runtime·callbackasm1(SB) + MOVW $45, R12 + B runtime·callbackasm1(SB) + MOVW $46, R12 + B runtime·callbackasm1(SB) + MOVW $47, R12 + B runtime·callbackasm1(SB) + MOVW $48, R12 + B runtime·callbackasm1(SB) + MOVW $49, R12 + B runtime·callbackasm1(SB) + MOVW $50, R12 + B runtime·callbackasm1(SB) + MOVW $51, R12 + B runtime·callbackasm1(SB) + MOVW $52, R12 + B runtime·callbackasm1(SB) + MOVW $53, R12 + B runtime·callbackasm1(SB) + MOVW $54, R12 + B runtime·callbackasm1(SB) + MOVW $55, R12 + B runtime·callbackasm1(SB) + MOVW $56, R12 + B runtime·callbackasm1(SB) + MOVW $57, R12 + B runtime·callbackasm1(SB) + MOVW $58, R12 + B runtime·callbackasm1(SB) + MOVW $59, R12 + B runtime·callbackasm1(SB) + MOVW $60, R12 + B runtime·callbackasm1(SB) + MOVW $61, R12 + B runtime·callbackasm1(SB) + MOVW $62, R12 + B runtime·callbackasm1(SB) + MOVW $63, R12 + B runtime·callbackasm1(SB) + MOVW $64, R12 + B runtime·callbackasm1(SB) + MOVW $65, R12 + B runtime·callbackasm1(SB) + MOVW $66, R12 + B runtime·callbackasm1(SB) + MOVW $67, R12 + B runtime·callbackasm1(SB) + MOVW $68, R12 + B runtime·callbackasm1(SB) + MOVW $69, R12 + B runtime·callbackasm1(SB) + MOVW $70, R12 + B runtime·callbackasm1(SB) + MOVW $71, R12 + B runtime·callbackasm1(SB) + MOVW $72, R12 + B runtime·callbackasm1(SB) + MOVW $73, R12 + B runtime·callbackasm1(SB) + MOVW $74, R12 + B runtime·callbackasm1(SB) + MOVW $75, R12 + B runtime·callbackasm1(SB) + MOVW $76, R12 + B runtime·callbackasm1(SB) + MOVW $77, R12 + B runtime·callbackasm1(SB) + MOVW $78, R12 + B runtime·callbackasm1(SB) + MOVW $79, R12 + B runtime·callbackasm1(SB) + MOVW $80, R12 + B runtime·callbackasm1(SB) + MOVW $81, R12 + B runtime·callbackasm1(SB) + MOVW $82, R12 + B runtime·callbackasm1(SB) + MOVW $83, R12 + B runtime·callbackasm1(SB) + MOVW $84, R12 + B runtime·callbackasm1(SB) + MOVW $85, R12 + B runtime·callbackasm1(SB) + MOVW $86, R12 + B runtime·callbackasm1(SB) + MOVW $87, R12 + B runtime·callbackasm1(SB) + MOVW $88, R12 + B runtime·callbackasm1(SB) + MOVW $89, R12 + B runtime·callbackasm1(SB) + MOVW $90, R12 + B runtime·callbackasm1(SB) + MOVW $91, R12 + B runtime·callbackasm1(SB) + MOVW $92, R12 + B runtime·callbackasm1(SB) + MOVW $93, R12 + B runtime·callbackasm1(SB) + MOVW $94, R12 + B runtime·callbackasm1(SB) + MOVW $95, R12 + B runtime·callbackasm1(SB) + MOVW $96, R12 + B runtime·callbackasm1(SB) + MOVW $97, R12 + B runtime·callbackasm1(SB) + MOVW $98, R12 + B runtime·callbackasm1(SB) + MOVW $99, R12 + B runtime·callbackasm1(SB) + MOVW $100, R12 + B runtime·callbackasm1(SB) + MOVW $101, R12 + B runtime·callbackasm1(SB) + MOVW $102, R12 + B runtime·callbackasm1(SB) + MOVW $103, R12 + B runtime·callbackasm1(SB) + MOVW $104, R12 + B runtime·callbackasm1(SB) + MOVW $105, R12 + B runtime·callbackasm1(SB) + MOVW $106, R12 + B runtime·callbackasm1(SB) + MOVW $107, R12 + B runtime·callbackasm1(SB) + MOVW $108, R12 + B runtime·callbackasm1(SB) + MOVW $109, R12 + B runtime·callbackasm1(SB) + MOVW $110, R12 + B runtime·callbackasm1(SB) + MOVW $111, R12 + B runtime·callbackasm1(SB) + MOVW $112, R12 + B runtime·callbackasm1(SB) + MOVW $113, R12 + B runtime·callbackasm1(SB) + MOVW $114, R12 + B runtime·callbackasm1(SB) + MOVW $115, R12 + B runtime·callbackasm1(SB) + MOVW $116, R12 + B runtime·callbackasm1(SB) + MOVW $117, R12 + B runtime·callbackasm1(SB) + MOVW $118, R12 + B runtime·callbackasm1(SB) + MOVW $119, R12 + B runtime·callbackasm1(SB) + MOVW $120, R12 + B runtime·callbackasm1(SB) + MOVW $121, R12 + B runtime·callbackasm1(SB) + MOVW $122, R12 + B runtime·callbackasm1(SB) + MOVW $123, R12 + B runtime·callbackasm1(SB) + MOVW $124, R12 + B runtime·callbackasm1(SB) + MOVW $125, R12 + B runtime·callbackasm1(SB) + MOVW $126, R12 + B runtime·callbackasm1(SB) + MOVW $127, R12 + B runtime·callbackasm1(SB) + MOVW $128, R12 + B runtime·callbackasm1(SB) + MOVW $129, R12 + B runtime·callbackasm1(SB) + MOVW $130, R12 + B runtime·callbackasm1(SB) + MOVW $131, R12 + B runtime·callbackasm1(SB) + MOVW $132, R12 + B runtime·callbackasm1(SB) + MOVW $133, R12 + B runtime·callbackasm1(SB) + MOVW $134, R12 + B runtime·callbackasm1(SB) + MOVW $135, R12 + B runtime·callbackasm1(SB) + MOVW $136, R12 + B runtime·callbackasm1(SB) + MOVW $137, R12 + B runtime·callbackasm1(SB) + MOVW $138, R12 + B runtime·callbackasm1(SB) + MOVW $139, R12 + B runtime·callbackasm1(SB) + MOVW $140, R12 + B runtime·callbackasm1(SB) + MOVW $141, R12 + B runtime·callbackasm1(SB) + MOVW $142, R12 + B runtime·callbackasm1(SB) + MOVW $143, R12 + B runtime·callbackasm1(SB) + MOVW $144, R12 + B runtime·callbackasm1(SB) + MOVW $145, R12 + B runtime·callbackasm1(SB) + MOVW $146, R12 + B runtime·callbackasm1(SB) + MOVW $147, R12 + B runtime·callbackasm1(SB) + MOVW $148, R12 + B runtime·callbackasm1(SB) + MOVW $149, R12 + B runtime·callbackasm1(SB) + MOVW $150, R12 + B runtime·callbackasm1(SB) + MOVW $151, R12 + B runtime·callbackasm1(SB) + MOVW $152, R12 + B runtime·callbackasm1(SB) + MOVW $153, R12 + B runtime·callbackasm1(SB) + MOVW $154, R12 + B runtime·callbackasm1(SB) + MOVW $155, R12 + B runtime·callbackasm1(SB) + MOVW $156, R12 + B runtime·callbackasm1(SB) + MOVW $157, R12 + B runtime·callbackasm1(SB) + MOVW $158, R12 + B runtime·callbackasm1(SB) + MOVW $159, R12 + B runtime·callbackasm1(SB) + MOVW $160, R12 + B runtime·callbackasm1(SB) + MOVW $161, R12 + B runtime·callbackasm1(SB) + MOVW $162, R12 + B runtime·callbackasm1(SB) + MOVW $163, R12 + B runtime·callbackasm1(SB) + MOVW $164, R12 + B runtime·callbackasm1(SB) + MOVW $165, R12 + B runtime·callbackasm1(SB) + MOVW $166, R12 + B runtime·callbackasm1(SB) + MOVW $167, R12 + B runtime·callbackasm1(SB) + MOVW $168, R12 + B runtime·callbackasm1(SB) + MOVW $169, R12 + B runtime·callbackasm1(SB) + MOVW $170, R12 + B runtime·callbackasm1(SB) + MOVW $171, R12 + B runtime·callbackasm1(SB) + MOVW $172, R12 + B runtime·callbackasm1(SB) + MOVW $173, R12 + B runtime·callbackasm1(SB) + MOVW $174, R12 + B runtime·callbackasm1(SB) + MOVW $175, R12 + B runtime·callbackasm1(SB) + MOVW $176, R12 + B runtime·callbackasm1(SB) + MOVW $177, R12 + B runtime·callbackasm1(SB) + MOVW $178, R12 + B runtime·callbackasm1(SB) + MOVW $179, R12 + B runtime·callbackasm1(SB) + MOVW $180, R12 + B runtime·callbackasm1(SB) + MOVW $181, R12 + B runtime·callbackasm1(SB) + MOVW $182, R12 + B runtime·callbackasm1(SB) + MOVW $183, R12 + B runtime·callbackasm1(SB) + MOVW $184, R12 + B runtime·callbackasm1(SB) + MOVW $185, R12 + B runtime·callbackasm1(SB) + MOVW $186, R12 + B runtime·callbackasm1(SB) + MOVW $187, R12 + B runtime·callbackasm1(SB) + MOVW $188, R12 + B runtime·callbackasm1(SB) + MOVW $189, R12 + B runtime·callbackasm1(SB) + MOVW $190, R12 + B runtime·callbackasm1(SB) + MOVW $191, R12 + B runtime·callbackasm1(SB) + MOVW $192, R12 + B runtime·callbackasm1(SB) + MOVW $193, R12 + B runtime·callbackasm1(SB) + MOVW $194, R12 + B runtime·callbackasm1(SB) + MOVW $195, R12 + B runtime·callbackasm1(SB) + MOVW $196, R12 + B runtime·callbackasm1(SB) + MOVW $197, R12 + B runtime·callbackasm1(SB) + MOVW $198, R12 + B runtime·callbackasm1(SB) + MOVW $199, R12 + B runtime·callbackasm1(SB) + MOVW $200, R12 + B runtime·callbackasm1(SB) + MOVW $201, R12 + B runtime·callbackasm1(SB) + MOVW $202, R12 + B runtime·callbackasm1(SB) + MOVW $203, R12 + B runtime·callbackasm1(SB) + MOVW $204, R12 + B runtime·callbackasm1(SB) + MOVW $205, R12 + B runtime·callbackasm1(SB) + MOVW $206, R12 + B runtime·callbackasm1(SB) + MOVW $207, R12 + B runtime·callbackasm1(SB) + MOVW $208, R12 + B runtime·callbackasm1(SB) + MOVW $209, R12 + B runtime·callbackasm1(SB) + MOVW $210, R12 + B runtime·callbackasm1(SB) + MOVW $211, R12 + B runtime·callbackasm1(SB) + MOVW $212, R12 + B runtime·callbackasm1(SB) + MOVW $213, R12 + B runtime·callbackasm1(SB) + MOVW $214, R12 + B runtime·callbackasm1(SB) + MOVW $215, R12 + B runtime·callbackasm1(SB) + MOVW $216, R12 + B runtime·callbackasm1(SB) + MOVW $217, R12 + B runtime·callbackasm1(SB) + MOVW $218, R12 + B runtime·callbackasm1(SB) + MOVW $219, R12 + B runtime·callbackasm1(SB) + MOVW $220, R12 + B runtime·callbackasm1(SB) + MOVW $221, R12 + B runtime·callbackasm1(SB) + MOVW $222, R12 + B runtime·callbackasm1(SB) + MOVW $223, R12 + B runtime·callbackasm1(SB) + MOVW $224, R12 + B runtime·callbackasm1(SB) + MOVW $225, R12 + B runtime·callbackasm1(SB) + MOVW $226, R12 + B runtime·callbackasm1(SB) + MOVW $227, R12 + B runtime·callbackasm1(SB) + MOVW $228, R12 + B runtime·callbackasm1(SB) + MOVW $229, R12 + B runtime·callbackasm1(SB) + MOVW $230, R12 + B runtime·callbackasm1(SB) + MOVW $231, R12 + B runtime·callbackasm1(SB) + MOVW $232, R12 + B runtime·callbackasm1(SB) + MOVW $233, R12 + B runtime·callbackasm1(SB) + MOVW $234, R12 + B runtime·callbackasm1(SB) + MOVW $235, R12 + B runtime·callbackasm1(SB) + MOVW $236, R12 + B runtime·callbackasm1(SB) + MOVW $237, R12 + B runtime·callbackasm1(SB) + MOVW $238, R12 + B runtime·callbackasm1(SB) + MOVW $239, R12 + B runtime·callbackasm1(SB) + MOVW $240, R12 + B runtime·callbackasm1(SB) + MOVW $241, R12 + B runtime·callbackasm1(SB) + MOVW $242, R12 + B runtime·callbackasm1(SB) + MOVW $243, R12 + B runtime·callbackasm1(SB) + MOVW $244, R12 + B runtime·callbackasm1(SB) + MOVW $245, R12 + B runtime·callbackasm1(SB) + MOVW $246, R12 + B runtime·callbackasm1(SB) + MOVW $247, R12 + B runtime·callbackasm1(SB) + MOVW $248, R12 + B runtime·callbackasm1(SB) + MOVW $249, R12 + B runtime·callbackasm1(SB) + MOVW $250, R12 + B runtime·callbackasm1(SB) + MOVW $251, R12 + B runtime·callbackasm1(SB) + MOVW $252, R12 + B runtime·callbackasm1(SB) + MOVW $253, R12 + B runtime·callbackasm1(SB) + MOVW $254, R12 + B runtime·callbackasm1(SB) + MOVW $255, R12 + B runtime·callbackasm1(SB) + MOVW $256, R12 + B runtime·callbackasm1(SB) + MOVW $257, R12 + B runtime·callbackasm1(SB) + MOVW $258, R12 + B runtime·callbackasm1(SB) + MOVW $259, R12 + B runtime·callbackasm1(SB) + MOVW $260, R12 + B runtime·callbackasm1(SB) + MOVW $261, R12 + B runtime·callbackasm1(SB) + MOVW $262, R12 + B runtime·callbackasm1(SB) + MOVW $263, R12 + B runtime·callbackasm1(SB) + MOVW $264, R12 + B runtime·callbackasm1(SB) + MOVW $265, R12 + B runtime·callbackasm1(SB) + MOVW $266, R12 + B runtime·callbackasm1(SB) + MOVW $267, R12 + B runtime·callbackasm1(SB) + MOVW $268, R12 + B runtime·callbackasm1(SB) + MOVW $269, R12 + B runtime·callbackasm1(SB) + MOVW $270, R12 + B runtime·callbackasm1(SB) + MOVW $271, R12 + B runtime·callbackasm1(SB) + MOVW $272, R12 + B runtime·callbackasm1(SB) + MOVW $273, R12 + B runtime·callbackasm1(SB) + MOVW $274, R12 + B runtime·callbackasm1(SB) + MOVW $275, R12 + B runtime·callbackasm1(SB) + MOVW $276, R12 + B runtime·callbackasm1(SB) + MOVW $277, R12 + B runtime·callbackasm1(SB) + MOVW $278, R12 + B runtime·callbackasm1(SB) + MOVW $279, R12 + B runtime·callbackasm1(SB) + MOVW $280, R12 + B runtime·callbackasm1(SB) + MOVW $281, R12 + B runtime·callbackasm1(SB) + MOVW $282, R12 + B runtime·callbackasm1(SB) + MOVW $283, R12 + B runtime·callbackasm1(SB) + MOVW $284, R12 + B runtime·callbackasm1(SB) + MOVW $285, R12 + B runtime·callbackasm1(SB) + MOVW $286, R12 + B runtime·callbackasm1(SB) + MOVW $287, R12 + B runtime·callbackasm1(SB) + MOVW $288, R12 + B runtime·callbackasm1(SB) + MOVW $289, R12 + B runtime·callbackasm1(SB) + MOVW $290, R12 + B runtime·callbackasm1(SB) + MOVW $291, R12 + B runtime·callbackasm1(SB) + MOVW $292, R12 + B runtime·callbackasm1(SB) + MOVW $293, R12 + B runtime·callbackasm1(SB) + MOVW $294, R12 + B runtime·callbackasm1(SB) + MOVW $295, R12 + B runtime·callbackasm1(SB) + MOVW $296, R12 + B runtime·callbackasm1(SB) + MOVW $297, R12 + B runtime·callbackasm1(SB) + MOVW $298, R12 + B runtime·callbackasm1(SB) + MOVW $299, R12 + B runtime·callbackasm1(SB) + MOVW $300, R12 + B runtime·callbackasm1(SB) + MOVW $301, R12 + B runtime·callbackasm1(SB) + MOVW $302, R12 + B runtime·callbackasm1(SB) + MOVW $303, R12 + B runtime·callbackasm1(SB) + MOVW $304, R12 + B runtime·callbackasm1(SB) + MOVW $305, R12 + B runtime·callbackasm1(SB) + MOVW $306, R12 + B runtime·callbackasm1(SB) + MOVW $307, R12 + B runtime·callbackasm1(SB) + MOVW $308, R12 + B runtime·callbackasm1(SB) + MOVW $309, R12 + B runtime·callbackasm1(SB) + MOVW $310, R12 + B runtime·callbackasm1(SB) + MOVW $311, R12 + B runtime·callbackasm1(SB) + MOVW $312, R12 + B runtime·callbackasm1(SB) + MOVW $313, R12 + B runtime·callbackasm1(SB) + MOVW $314, R12 + B runtime·callbackasm1(SB) + MOVW $315, R12 + B runtime·callbackasm1(SB) + MOVW $316, R12 + B runtime·callbackasm1(SB) + MOVW $317, R12 + B runtime·callbackasm1(SB) + MOVW $318, R12 + B runtime·callbackasm1(SB) + MOVW $319, R12 + B runtime·callbackasm1(SB) + MOVW $320, R12 + B runtime·callbackasm1(SB) + MOVW $321, R12 + B runtime·callbackasm1(SB) + MOVW $322, R12 + B runtime·callbackasm1(SB) + MOVW $323, R12 + B runtime·callbackasm1(SB) + MOVW $324, R12 + B runtime·callbackasm1(SB) + MOVW $325, R12 + B runtime·callbackasm1(SB) + MOVW $326, R12 + B runtime·callbackasm1(SB) + MOVW $327, R12 + B runtime·callbackasm1(SB) + MOVW $328, R12 + B runtime·callbackasm1(SB) + MOVW $329, R12 + B runtime·callbackasm1(SB) + MOVW $330, R12 + B runtime·callbackasm1(SB) + MOVW $331, R12 + B runtime·callbackasm1(SB) + MOVW $332, R12 + B runtime·callbackasm1(SB) + MOVW $333, R12 + B runtime·callbackasm1(SB) + MOVW $334, R12 + B runtime·callbackasm1(SB) + MOVW $335, R12 + B runtime·callbackasm1(SB) + MOVW $336, R12 + B runtime·callbackasm1(SB) + MOVW $337, R12 + B runtime·callbackasm1(SB) + MOVW $338, R12 + B runtime·callbackasm1(SB) + MOVW $339, R12 + B runtime·callbackasm1(SB) + MOVW $340, R12 + B runtime·callbackasm1(SB) + MOVW $341, R12 + B runtime·callbackasm1(SB) + MOVW $342, R12 + B runtime·callbackasm1(SB) + MOVW $343, R12 + B runtime·callbackasm1(SB) + MOVW $344, R12 + B runtime·callbackasm1(SB) + MOVW $345, R12 + B runtime·callbackasm1(SB) + MOVW $346, R12 + B runtime·callbackasm1(SB) + MOVW $347, R12 + B runtime·callbackasm1(SB) + MOVW $348, R12 + B runtime·callbackasm1(SB) + MOVW $349, R12 + B runtime·callbackasm1(SB) + MOVW $350, R12 + B runtime·callbackasm1(SB) + MOVW $351, R12 + B runtime·callbackasm1(SB) + MOVW $352, R12 + B runtime·callbackasm1(SB) + MOVW $353, R12 + B runtime·callbackasm1(SB) + MOVW $354, R12 + B runtime·callbackasm1(SB) + MOVW $355, R12 + B runtime·callbackasm1(SB) + MOVW $356, R12 + B runtime·callbackasm1(SB) + MOVW $357, R12 + B runtime·callbackasm1(SB) + MOVW $358, R12 + B runtime·callbackasm1(SB) + MOVW $359, R12 + B runtime·callbackasm1(SB) + MOVW $360, R12 + B runtime·callbackasm1(SB) + MOVW $361, R12 + B runtime·callbackasm1(SB) + MOVW $362, R12 + B runtime·callbackasm1(SB) + MOVW $363, R12 + B runtime·callbackasm1(SB) + MOVW $364, R12 + B runtime·callbackasm1(SB) + MOVW $365, R12 + B runtime·callbackasm1(SB) + MOVW $366, R12 + B runtime·callbackasm1(SB) + MOVW $367, R12 + B runtime·callbackasm1(SB) + MOVW $368, R12 + B runtime·callbackasm1(SB) + MOVW $369, R12 + B runtime·callbackasm1(SB) + MOVW $370, R12 + B runtime·callbackasm1(SB) + MOVW $371, R12 + B runtime·callbackasm1(SB) + MOVW $372, R12 + B runtime·callbackasm1(SB) + MOVW $373, R12 + B runtime·callbackasm1(SB) + MOVW $374, R12 + B runtime·callbackasm1(SB) + MOVW $375, R12 + B runtime·callbackasm1(SB) + MOVW $376, R12 + B runtime·callbackasm1(SB) + MOVW $377, R12 + B runtime·callbackasm1(SB) + MOVW $378, R12 + B runtime·callbackasm1(SB) + MOVW $379, R12 + B runtime·callbackasm1(SB) + MOVW $380, R12 + B runtime·callbackasm1(SB) + MOVW $381, R12 + B runtime·callbackasm1(SB) + MOVW $382, R12 + B runtime·callbackasm1(SB) + MOVW $383, R12 + B runtime·callbackasm1(SB) + MOVW $384, R12 + B runtime·callbackasm1(SB) + MOVW $385, R12 + B runtime·callbackasm1(SB) + MOVW $386, R12 + B runtime·callbackasm1(SB) + MOVW $387, R12 + B runtime·callbackasm1(SB) + MOVW $388, R12 + B runtime·callbackasm1(SB) + MOVW $389, R12 + B runtime·callbackasm1(SB) + MOVW $390, R12 + B runtime·callbackasm1(SB) + MOVW $391, R12 + B runtime·callbackasm1(SB) + MOVW $392, R12 + B runtime·callbackasm1(SB) + MOVW $393, R12 + B runtime·callbackasm1(SB) + MOVW $394, R12 + B runtime·callbackasm1(SB) + MOVW $395, R12 + B runtime·callbackasm1(SB) + MOVW $396, R12 + B runtime·callbackasm1(SB) + MOVW $397, R12 + B runtime·callbackasm1(SB) + MOVW $398, R12 + B runtime·callbackasm1(SB) + MOVW $399, R12 + B runtime·callbackasm1(SB) + MOVW $400, R12 + B runtime·callbackasm1(SB) + MOVW $401, R12 + B runtime·callbackasm1(SB) + MOVW $402, R12 + B runtime·callbackasm1(SB) + MOVW $403, R12 + B runtime·callbackasm1(SB) + MOVW $404, R12 + B runtime·callbackasm1(SB) + MOVW $405, R12 + B runtime·callbackasm1(SB) + MOVW $406, R12 + B runtime·callbackasm1(SB) + MOVW $407, R12 + B runtime·callbackasm1(SB) + MOVW $408, R12 + B runtime·callbackasm1(SB) + MOVW $409, R12 + B runtime·callbackasm1(SB) + MOVW $410, R12 + B runtime·callbackasm1(SB) + MOVW $411, R12 + B runtime·callbackasm1(SB) + MOVW $412, R12 + B runtime·callbackasm1(SB) + MOVW $413, R12 + B runtime·callbackasm1(SB) + MOVW $414, R12 + B runtime·callbackasm1(SB) + MOVW $415, R12 + B runtime·callbackasm1(SB) + MOVW $416, R12 + B runtime·callbackasm1(SB) + MOVW $417, R12 + B runtime·callbackasm1(SB) + MOVW $418, R12 + B runtime·callbackasm1(SB) + MOVW $419, R12 + B runtime·callbackasm1(SB) + MOVW $420, R12 + B runtime·callbackasm1(SB) + MOVW $421, R12 + B runtime·callbackasm1(SB) + MOVW $422, R12 + B runtime·callbackasm1(SB) + MOVW $423, R12 + B runtime·callbackasm1(SB) + MOVW $424, R12 + B runtime·callbackasm1(SB) + MOVW $425, R12 + B runtime·callbackasm1(SB) + MOVW $426, R12 + B runtime·callbackasm1(SB) + MOVW $427, R12 + B runtime·callbackasm1(SB) + MOVW $428, R12 + B runtime·callbackasm1(SB) + MOVW $429, R12 + B runtime·callbackasm1(SB) + MOVW $430, R12 + B runtime·callbackasm1(SB) + MOVW $431, R12 + B runtime·callbackasm1(SB) + MOVW $432, R12 + B runtime·callbackasm1(SB) + MOVW $433, R12 + B runtime·callbackasm1(SB) + MOVW $434, R12 + B runtime·callbackasm1(SB) + MOVW $435, R12 + B runtime·callbackasm1(SB) + MOVW $436, R12 + B runtime·callbackasm1(SB) + MOVW $437, R12 + B runtime·callbackasm1(SB) + MOVW $438, R12 + B runtime·callbackasm1(SB) + MOVW $439, R12 + B runtime·callbackasm1(SB) + MOVW $440, R12 + B runtime·callbackasm1(SB) + MOVW $441, R12 + B runtime·callbackasm1(SB) + MOVW $442, R12 + B runtime·callbackasm1(SB) + MOVW $443, R12 + B runtime·callbackasm1(SB) + MOVW $444, R12 + B runtime·callbackasm1(SB) + MOVW $445, R12 + B runtime·callbackasm1(SB) + MOVW $446, R12 + B runtime·callbackasm1(SB) + MOVW $447, R12 + B runtime·callbackasm1(SB) + MOVW $448, R12 + B runtime·callbackasm1(SB) + MOVW $449, R12 + B runtime·callbackasm1(SB) + MOVW $450, R12 + B runtime·callbackasm1(SB) + MOVW $451, R12 + B runtime·callbackasm1(SB) + MOVW $452, R12 + B runtime·callbackasm1(SB) + MOVW $453, R12 + B runtime·callbackasm1(SB) + MOVW $454, R12 + B runtime·callbackasm1(SB) + MOVW $455, R12 + B runtime·callbackasm1(SB) + MOVW $456, R12 + B runtime·callbackasm1(SB) + MOVW $457, R12 + B runtime·callbackasm1(SB) + MOVW $458, R12 + B runtime·callbackasm1(SB) + MOVW $459, R12 + B runtime·callbackasm1(SB) + MOVW $460, R12 + B runtime·callbackasm1(SB) + MOVW $461, R12 + B runtime·callbackasm1(SB) + MOVW $462, R12 + B runtime·callbackasm1(SB) + MOVW $463, R12 + B runtime·callbackasm1(SB) + MOVW $464, R12 + B runtime·callbackasm1(SB) + MOVW $465, R12 + B runtime·callbackasm1(SB) + MOVW $466, R12 + B runtime·callbackasm1(SB) + MOVW $467, R12 + B runtime·callbackasm1(SB) + MOVW $468, R12 + B runtime·callbackasm1(SB) + MOVW $469, R12 + B runtime·callbackasm1(SB) + MOVW $470, R12 + B runtime·callbackasm1(SB) + MOVW $471, R12 + B runtime·callbackasm1(SB) + MOVW $472, R12 + B runtime·callbackasm1(SB) + MOVW $473, R12 + B runtime·callbackasm1(SB) + MOVW $474, R12 + B runtime·callbackasm1(SB) + MOVW $475, R12 + B runtime·callbackasm1(SB) + MOVW $476, R12 + B runtime·callbackasm1(SB) + MOVW $477, R12 + B runtime·callbackasm1(SB) + MOVW $478, R12 + B runtime·callbackasm1(SB) + MOVW $479, R12 + B runtime·callbackasm1(SB) + MOVW $480, R12 + B runtime·callbackasm1(SB) + MOVW $481, R12 + B runtime·callbackasm1(SB) + MOVW $482, R12 + B runtime·callbackasm1(SB) + MOVW $483, R12 + B runtime·callbackasm1(SB) + MOVW $484, R12 + B runtime·callbackasm1(SB) + MOVW $485, R12 + B runtime·callbackasm1(SB) + MOVW $486, R12 + B runtime·callbackasm1(SB) + MOVW $487, R12 + B runtime·callbackasm1(SB) + MOVW $488, R12 + B runtime·callbackasm1(SB) + MOVW $489, R12 + B runtime·callbackasm1(SB) + MOVW $490, R12 + B runtime·callbackasm1(SB) + MOVW $491, R12 + B runtime·callbackasm1(SB) + MOVW $492, R12 + B runtime·callbackasm1(SB) + MOVW $493, R12 + B runtime·callbackasm1(SB) + MOVW $494, R12 + B runtime·callbackasm1(SB) + MOVW $495, R12 + B runtime·callbackasm1(SB) + MOVW $496, R12 + B runtime·callbackasm1(SB) + MOVW $497, R12 + B runtime·callbackasm1(SB) + MOVW $498, R12 + B runtime·callbackasm1(SB) + MOVW $499, R12 + B runtime·callbackasm1(SB) + MOVW $500, R12 + B runtime·callbackasm1(SB) + MOVW $501, R12 + B runtime·callbackasm1(SB) + MOVW $502, R12 + B runtime·callbackasm1(SB) + MOVW $503, R12 + B runtime·callbackasm1(SB) + MOVW $504, R12 + B runtime·callbackasm1(SB) + MOVW $505, R12 + B runtime·callbackasm1(SB) + MOVW $506, R12 + B runtime·callbackasm1(SB) + MOVW $507, R12 + B runtime·callbackasm1(SB) + MOVW $508, R12 + B runtime·callbackasm1(SB) + MOVW $509, R12 + B runtime·callbackasm1(SB) + MOVW $510, R12 + B runtime·callbackasm1(SB) + MOVW $511, R12 + B runtime·callbackasm1(SB) + MOVW $512, R12 + B runtime·callbackasm1(SB) + MOVW $513, R12 + B runtime·callbackasm1(SB) + MOVW $514, R12 + B runtime·callbackasm1(SB) + MOVW $515, R12 + B runtime·callbackasm1(SB) + MOVW $516, R12 + B runtime·callbackasm1(SB) + MOVW $517, R12 + B runtime·callbackasm1(SB) + MOVW $518, R12 + B runtime·callbackasm1(SB) + MOVW $519, R12 + B runtime·callbackasm1(SB) + MOVW $520, R12 + B runtime·callbackasm1(SB) + MOVW $521, R12 + B runtime·callbackasm1(SB) + MOVW $522, R12 + B runtime·callbackasm1(SB) + MOVW $523, R12 + B runtime·callbackasm1(SB) + MOVW $524, R12 + B runtime·callbackasm1(SB) + MOVW $525, R12 + B runtime·callbackasm1(SB) + MOVW $526, R12 + B runtime·callbackasm1(SB) + MOVW $527, R12 + B runtime·callbackasm1(SB) + MOVW $528, R12 + B runtime·callbackasm1(SB) + MOVW $529, R12 + B runtime·callbackasm1(SB) + MOVW $530, R12 + B runtime·callbackasm1(SB) + MOVW $531, R12 + B runtime·callbackasm1(SB) + MOVW $532, R12 + B runtime·callbackasm1(SB) + MOVW $533, R12 + B runtime·callbackasm1(SB) + MOVW $534, R12 + B runtime·callbackasm1(SB) + MOVW $535, R12 + B runtime·callbackasm1(SB) + MOVW $536, R12 + B runtime·callbackasm1(SB) + MOVW $537, R12 + B runtime·callbackasm1(SB) + MOVW $538, R12 + B runtime·callbackasm1(SB) + MOVW $539, R12 + B runtime·callbackasm1(SB) + MOVW $540, R12 + B runtime·callbackasm1(SB) + MOVW $541, R12 + B runtime·callbackasm1(SB) + MOVW $542, R12 + B runtime·callbackasm1(SB) + MOVW $543, R12 + B runtime·callbackasm1(SB) + MOVW $544, R12 + B runtime·callbackasm1(SB) + MOVW $545, R12 + B runtime·callbackasm1(SB) + MOVW $546, R12 + B runtime·callbackasm1(SB) + MOVW $547, R12 + B runtime·callbackasm1(SB) + MOVW $548, R12 + B runtime·callbackasm1(SB) + MOVW $549, R12 + B runtime·callbackasm1(SB) + MOVW $550, R12 + B runtime·callbackasm1(SB) + MOVW $551, R12 + B runtime·callbackasm1(SB) + MOVW $552, R12 + B runtime·callbackasm1(SB) + MOVW $553, R12 + B runtime·callbackasm1(SB) + MOVW $554, R12 + B runtime·callbackasm1(SB) + MOVW $555, R12 + B runtime·callbackasm1(SB) + MOVW $556, R12 + B runtime·callbackasm1(SB) + MOVW $557, R12 + B runtime·callbackasm1(SB) + MOVW $558, R12 + B runtime·callbackasm1(SB) + MOVW $559, R12 + B runtime·callbackasm1(SB) + MOVW $560, R12 + B runtime·callbackasm1(SB) + MOVW $561, R12 + B runtime·callbackasm1(SB) + MOVW $562, R12 + B runtime·callbackasm1(SB) + MOVW $563, R12 + B runtime·callbackasm1(SB) + MOVW $564, R12 + B runtime·callbackasm1(SB) + MOVW $565, R12 + B runtime·callbackasm1(SB) + MOVW $566, R12 + B runtime·callbackasm1(SB) + MOVW $567, R12 + B runtime·callbackasm1(SB) + MOVW $568, R12 + B runtime·callbackasm1(SB) + MOVW $569, R12 + B runtime·callbackasm1(SB) + MOVW $570, R12 + B runtime·callbackasm1(SB) + MOVW $571, R12 + B runtime·callbackasm1(SB) + MOVW $572, R12 + B runtime·callbackasm1(SB) + MOVW $573, R12 + B runtime·callbackasm1(SB) + MOVW $574, R12 + B runtime·callbackasm1(SB) + MOVW $575, R12 + B runtime·callbackasm1(SB) + MOVW $576, R12 + B runtime·callbackasm1(SB) + MOVW $577, R12 + B runtime·callbackasm1(SB) + MOVW $578, R12 + B runtime·callbackasm1(SB) + MOVW $579, R12 + B runtime·callbackasm1(SB) + MOVW $580, R12 + B runtime·callbackasm1(SB) + MOVW $581, R12 + B runtime·callbackasm1(SB) + MOVW $582, R12 + B runtime·callbackasm1(SB) + MOVW $583, R12 + B runtime·callbackasm1(SB) + MOVW $584, R12 + B runtime·callbackasm1(SB) + MOVW $585, R12 + B runtime·callbackasm1(SB) + MOVW $586, R12 + B runtime·callbackasm1(SB) + MOVW $587, R12 + B runtime·callbackasm1(SB) + MOVW $588, R12 + B runtime·callbackasm1(SB) + MOVW $589, R12 + B runtime·callbackasm1(SB) + MOVW $590, R12 + B runtime·callbackasm1(SB) + MOVW $591, R12 + B runtime·callbackasm1(SB) + MOVW $592, R12 + B runtime·callbackasm1(SB) + MOVW $593, R12 + B runtime·callbackasm1(SB) + MOVW $594, R12 + B runtime·callbackasm1(SB) + MOVW $595, R12 + B runtime·callbackasm1(SB) + MOVW $596, R12 + B runtime·callbackasm1(SB) + MOVW $597, R12 + B runtime·callbackasm1(SB) + MOVW $598, R12 + B runtime·callbackasm1(SB) + MOVW $599, R12 + B runtime·callbackasm1(SB) + MOVW $600, R12 + B runtime·callbackasm1(SB) + MOVW $601, R12 + B runtime·callbackasm1(SB) + MOVW $602, R12 + B runtime·callbackasm1(SB) + MOVW $603, R12 + B runtime·callbackasm1(SB) + MOVW $604, R12 + B runtime·callbackasm1(SB) + MOVW $605, R12 + B runtime·callbackasm1(SB) + MOVW $606, R12 + B runtime·callbackasm1(SB) + MOVW $607, R12 + B runtime·callbackasm1(SB) + MOVW $608, R12 + B runtime·callbackasm1(SB) + MOVW $609, R12 + B runtime·callbackasm1(SB) + MOVW $610, R12 + B runtime·callbackasm1(SB) + MOVW $611, R12 + B runtime·callbackasm1(SB) + MOVW $612, R12 + B runtime·callbackasm1(SB) + MOVW $613, R12 + B runtime·callbackasm1(SB) + MOVW $614, R12 + B runtime·callbackasm1(SB) + MOVW $615, R12 + B runtime·callbackasm1(SB) + MOVW $616, R12 + B runtime·callbackasm1(SB) + MOVW $617, R12 + B runtime·callbackasm1(SB) + MOVW $618, R12 + B runtime·callbackasm1(SB) + MOVW $619, R12 + B runtime·callbackasm1(SB) + MOVW $620, R12 + B runtime·callbackasm1(SB) + MOVW $621, R12 + B runtime·callbackasm1(SB) + MOVW $622, R12 + B runtime·callbackasm1(SB) + MOVW $623, R12 + B runtime·callbackasm1(SB) + MOVW $624, R12 + B runtime·callbackasm1(SB) + MOVW $625, R12 + B runtime·callbackasm1(SB) + MOVW $626, R12 + B runtime·callbackasm1(SB) + MOVW $627, R12 + B runtime·callbackasm1(SB) + MOVW $628, R12 + B runtime·callbackasm1(SB) + MOVW $629, R12 + B runtime·callbackasm1(SB) + MOVW $630, R12 + B runtime·callbackasm1(SB) + MOVW $631, R12 + B runtime·callbackasm1(SB) + MOVW $632, R12 + B runtime·callbackasm1(SB) + MOVW $633, R12 + B runtime·callbackasm1(SB) + MOVW $634, R12 + B runtime·callbackasm1(SB) + MOVW $635, R12 + B runtime·callbackasm1(SB) + MOVW $636, R12 + B runtime·callbackasm1(SB) + MOVW $637, R12 + B runtime·callbackasm1(SB) + MOVW $638, R12 + B runtime·callbackasm1(SB) + MOVW $639, R12 + B runtime·callbackasm1(SB) + MOVW $640, R12 + B runtime·callbackasm1(SB) + MOVW $641, R12 + B runtime·callbackasm1(SB) + MOVW $642, R12 + B runtime·callbackasm1(SB) + MOVW $643, R12 + B runtime·callbackasm1(SB) + MOVW $644, R12 + B runtime·callbackasm1(SB) + MOVW $645, R12 + B runtime·callbackasm1(SB) + MOVW $646, R12 + B runtime·callbackasm1(SB) + MOVW $647, R12 + B runtime·callbackasm1(SB) + MOVW $648, R12 + B runtime·callbackasm1(SB) + MOVW $649, R12 + B runtime·callbackasm1(SB) + MOVW $650, R12 + B runtime·callbackasm1(SB) + MOVW $651, R12 + B runtime·callbackasm1(SB) + MOVW $652, R12 + B runtime·callbackasm1(SB) + MOVW $653, R12 + B runtime·callbackasm1(SB) + MOVW $654, R12 + B runtime·callbackasm1(SB) + MOVW $655, R12 + B runtime·callbackasm1(SB) + MOVW $656, R12 + B runtime·callbackasm1(SB) + MOVW $657, R12 + B runtime·callbackasm1(SB) + MOVW $658, R12 + B runtime·callbackasm1(SB) + MOVW $659, R12 + B runtime·callbackasm1(SB) + MOVW $660, R12 + B runtime·callbackasm1(SB) + MOVW $661, R12 + B runtime·callbackasm1(SB) + MOVW $662, R12 + B runtime·callbackasm1(SB) + MOVW $663, R12 + B runtime·callbackasm1(SB) + MOVW $664, R12 + B runtime·callbackasm1(SB) + MOVW $665, R12 + B runtime·callbackasm1(SB) + MOVW $666, R12 + B runtime·callbackasm1(SB) + MOVW $667, R12 + B runtime·callbackasm1(SB) + MOVW $668, R12 + B runtime·callbackasm1(SB) + MOVW $669, R12 + B runtime·callbackasm1(SB) + MOVW $670, R12 + B runtime·callbackasm1(SB) + MOVW $671, R12 + B runtime·callbackasm1(SB) + MOVW $672, R12 + B runtime·callbackasm1(SB) + MOVW $673, R12 + B runtime·callbackasm1(SB) + MOVW $674, R12 + B runtime·callbackasm1(SB) + MOVW $675, R12 + B runtime·callbackasm1(SB) + MOVW $676, R12 + B runtime·callbackasm1(SB) + MOVW $677, R12 + B runtime·callbackasm1(SB) + MOVW $678, R12 + B runtime·callbackasm1(SB) + MOVW $679, R12 + B runtime·callbackasm1(SB) + MOVW $680, R12 + B runtime·callbackasm1(SB) + MOVW $681, R12 + B runtime·callbackasm1(SB) + MOVW $682, R12 + B runtime·callbackasm1(SB) + MOVW $683, R12 + B runtime·callbackasm1(SB) + MOVW $684, R12 + B runtime·callbackasm1(SB) + MOVW $685, R12 + B runtime·callbackasm1(SB) + MOVW $686, R12 + B runtime·callbackasm1(SB) + MOVW $687, R12 + B runtime·callbackasm1(SB) + MOVW $688, R12 + B runtime·callbackasm1(SB) + MOVW $689, R12 + B runtime·callbackasm1(SB) + MOVW $690, R12 + B runtime·callbackasm1(SB) + MOVW $691, R12 + B runtime·callbackasm1(SB) + MOVW $692, R12 + B runtime·callbackasm1(SB) + MOVW $693, R12 + B runtime·callbackasm1(SB) + MOVW $694, R12 + B runtime·callbackasm1(SB) + MOVW $695, R12 + B runtime·callbackasm1(SB) + MOVW $696, R12 + B runtime·callbackasm1(SB) + MOVW $697, R12 + B runtime·callbackasm1(SB) + MOVW $698, R12 + B runtime·callbackasm1(SB) + MOVW $699, R12 + B runtime·callbackasm1(SB) + MOVW $700, R12 + B runtime·callbackasm1(SB) + MOVW $701, R12 + B runtime·callbackasm1(SB) + MOVW $702, R12 + B runtime·callbackasm1(SB) + MOVW $703, R12 + B runtime·callbackasm1(SB) + MOVW $704, R12 + B runtime·callbackasm1(SB) + MOVW $705, R12 + B runtime·callbackasm1(SB) + MOVW $706, R12 + B runtime·callbackasm1(SB) + MOVW $707, R12 + B runtime·callbackasm1(SB) + MOVW $708, R12 + B runtime·callbackasm1(SB) + MOVW $709, R12 + B runtime·callbackasm1(SB) + MOVW $710, R12 + B runtime·callbackasm1(SB) + MOVW $711, R12 + B runtime·callbackasm1(SB) + MOVW $712, R12 + B runtime·callbackasm1(SB) + MOVW $713, R12 + B runtime·callbackasm1(SB) + MOVW $714, R12 + B runtime·callbackasm1(SB) + MOVW $715, R12 + B runtime·callbackasm1(SB) + MOVW $716, R12 + B runtime·callbackasm1(SB) + MOVW $717, R12 + B runtime·callbackasm1(SB) + MOVW $718, R12 + B runtime·callbackasm1(SB) + MOVW $719, R12 + B runtime·callbackasm1(SB) + MOVW $720, R12 + B runtime·callbackasm1(SB) + MOVW $721, R12 + B runtime·callbackasm1(SB) + MOVW $722, R12 + B runtime·callbackasm1(SB) + MOVW $723, R12 + B runtime·callbackasm1(SB) + MOVW $724, R12 + B runtime·callbackasm1(SB) + MOVW $725, R12 + B runtime·callbackasm1(SB) + MOVW $726, R12 + B runtime·callbackasm1(SB) + MOVW $727, R12 + B runtime·callbackasm1(SB) + MOVW $728, R12 + B runtime·callbackasm1(SB) + MOVW $729, R12 + B runtime·callbackasm1(SB) + MOVW $730, R12 + B runtime·callbackasm1(SB) + MOVW $731, R12 + B runtime·callbackasm1(SB) + MOVW $732, R12 + B runtime·callbackasm1(SB) + MOVW $733, R12 + B runtime·callbackasm1(SB) + MOVW $734, R12 + B runtime·callbackasm1(SB) + MOVW $735, R12 + B runtime·callbackasm1(SB) + MOVW $736, R12 + B runtime·callbackasm1(SB) + MOVW $737, R12 + B runtime·callbackasm1(SB) + MOVW $738, R12 + B runtime·callbackasm1(SB) + MOVW $739, R12 + B runtime·callbackasm1(SB) + MOVW $740, R12 + B runtime·callbackasm1(SB) + MOVW $741, R12 + B runtime·callbackasm1(SB) + MOVW $742, R12 + B runtime·callbackasm1(SB) + MOVW $743, R12 + B runtime·callbackasm1(SB) + MOVW $744, R12 + B runtime·callbackasm1(SB) + MOVW $745, R12 + B runtime·callbackasm1(SB) + MOVW $746, R12 + B runtime·callbackasm1(SB) + MOVW $747, R12 + B runtime·callbackasm1(SB) + MOVW $748, R12 + B runtime·callbackasm1(SB) + MOVW $749, R12 + B runtime·callbackasm1(SB) + MOVW $750, R12 + B runtime·callbackasm1(SB) + MOVW $751, R12 + B runtime·callbackasm1(SB) + MOVW $752, R12 + B runtime·callbackasm1(SB) + MOVW $753, R12 + B runtime·callbackasm1(SB) + MOVW $754, R12 + B runtime·callbackasm1(SB) + MOVW $755, R12 + B runtime·callbackasm1(SB) + MOVW $756, R12 + B runtime·callbackasm1(SB) + MOVW $757, R12 + B runtime·callbackasm1(SB) + MOVW $758, R12 + B runtime·callbackasm1(SB) + MOVW $759, R12 + B runtime·callbackasm1(SB) + MOVW $760, R12 + B runtime·callbackasm1(SB) + MOVW $761, R12 + B runtime·callbackasm1(SB) + MOVW $762, R12 + B runtime·callbackasm1(SB) + MOVW $763, R12 + B runtime·callbackasm1(SB) + MOVW $764, R12 + B runtime·callbackasm1(SB) + MOVW $765, R12 + B runtime·callbackasm1(SB) + MOVW $766, R12 + B runtime·callbackasm1(SB) + MOVW $767, R12 + B runtime·callbackasm1(SB) + MOVW $768, R12 + B runtime·callbackasm1(SB) + MOVW $769, R12 + B runtime·callbackasm1(SB) + MOVW $770, R12 + B runtime·callbackasm1(SB) + MOVW $771, R12 + B runtime·callbackasm1(SB) + MOVW $772, R12 + B runtime·callbackasm1(SB) + MOVW $773, R12 + B runtime·callbackasm1(SB) + MOVW $774, R12 + B runtime·callbackasm1(SB) + MOVW $775, R12 + B runtime·callbackasm1(SB) + MOVW $776, R12 + B runtime·callbackasm1(SB) + MOVW $777, R12 + B runtime·callbackasm1(SB) + MOVW $778, R12 + B runtime·callbackasm1(SB) + MOVW $779, R12 + B runtime·callbackasm1(SB) + MOVW $780, R12 + B runtime·callbackasm1(SB) + MOVW $781, R12 + B runtime·callbackasm1(SB) + MOVW $782, R12 + B runtime·callbackasm1(SB) + MOVW $783, R12 + B runtime·callbackasm1(SB) + MOVW $784, R12 + B runtime·callbackasm1(SB) + MOVW $785, R12 + B runtime·callbackasm1(SB) + MOVW $786, R12 + B runtime·callbackasm1(SB) + MOVW $787, R12 + B runtime·callbackasm1(SB) + MOVW $788, R12 + B runtime·callbackasm1(SB) + MOVW $789, R12 + B runtime·callbackasm1(SB) + MOVW $790, R12 + B runtime·callbackasm1(SB) + MOVW $791, R12 + B runtime·callbackasm1(SB) + MOVW $792, R12 + B runtime·callbackasm1(SB) + MOVW $793, R12 + B runtime·callbackasm1(SB) + MOVW $794, R12 + B runtime·callbackasm1(SB) + MOVW $795, R12 + B runtime·callbackasm1(SB) + MOVW $796, R12 + B runtime·callbackasm1(SB) + MOVW $797, R12 + B runtime·callbackasm1(SB) + MOVW $798, R12 + B runtime·callbackasm1(SB) + MOVW $799, R12 + B runtime·callbackasm1(SB) + MOVW $800, R12 + B runtime·callbackasm1(SB) + MOVW $801, R12 + B runtime·callbackasm1(SB) + MOVW $802, R12 + B runtime·callbackasm1(SB) + MOVW $803, R12 + B runtime·callbackasm1(SB) + MOVW $804, R12 + B runtime·callbackasm1(SB) + MOVW $805, R12 + B runtime·callbackasm1(SB) + MOVW $806, R12 + B runtime·callbackasm1(SB) + MOVW $807, R12 + B runtime·callbackasm1(SB) + MOVW $808, R12 + B runtime·callbackasm1(SB) + MOVW $809, R12 + B runtime·callbackasm1(SB) + MOVW $810, R12 + B runtime·callbackasm1(SB) + MOVW $811, R12 + B runtime·callbackasm1(SB) + MOVW $812, R12 + B runtime·callbackasm1(SB) + MOVW $813, R12 + B runtime·callbackasm1(SB) + MOVW $814, R12 + B runtime·callbackasm1(SB) + MOVW $815, R12 + B runtime·callbackasm1(SB) + MOVW $816, R12 + B runtime·callbackasm1(SB) + MOVW $817, R12 + B runtime·callbackasm1(SB) + MOVW $818, R12 + B runtime·callbackasm1(SB) + MOVW $819, R12 + B runtime·callbackasm1(SB) + MOVW $820, R12 + B runtime·callbackasm1(SB) + MOVW $821, R12 + B runtime·callbackasm1(SB) + MOVW $822, R12 + B runtime·callbackasm1(SB) + MOVW $823, R12 + B runtime·callbackasm1(SB) + MOVW $824, R12 + B runtime·callbackasm1(SB) + MOVW $825, R12 + B runtime·callbackasm1(SB) + MOVW $826, R12 + B runtime·callbackasm1(SB) + MOVW $827, R12 + B runtime·callbackasm1(SB) + MOVW $828, R12 + B runtime·callbackasm1(SB) + MOVW $829, R12 + B runtime·callbackasm1(SB) + MOVW $830, R12 + B runtime·callbackasm1(SB) + MOVW $831, R12 + B runtime·callbackasm1(SB) + MOVW $832, R12 + B runtime·callbackasm1(SB) + MOVW $833, R12 + B runtime·callbackasm1(SB) + MOVW $834, R12 + B runtime·callbackasm1(SB) + MOVW $835, R12 + B runtime·callbackasm1(SB) + MOVW $836, R12 + B runtime·callbackasm1(SB) + MOVW $837, R12 + B runtime·callbackasm1(SB) + MOVW $838, R12 + B runtime·callbackasm1(SB) + MOVW $839, R12 + B runtime·callbackasm1(SB) + MOVW $840, R12 + B runtime·callbackasm1(SB) + MOVW $841, R12 + B runtime·callbackasm1(SB) + MOVW $842, R12 + B runtime·callbackasm1(SB) + MOVW $843, R12 + B runtime·callbackasm1(SB) + MOVW $844, R12 + B runtime·callbackasm1(SB) + MOVW $845, R12 + B runtime·callbackasm1(SB) + MOVW $846, R12 + B runtime·callbackasm1(SB) + MOVW $847, R12 + B runtime·callbackasm1(SB) + MOVW $848, R12 + B runtime·callbackasm1(SB) + MOVW $849, R12 + B runtime·callbackasm1(SB) + MOVW $850, R12 + B runtime·callbackasm1(SB) + MOVW $851, R12 + B runtime·callbackasm1(SB) + MOVW $852, R12 + B runtime·callbackasm1(SB) + MOVW $853, R12 + B runtime·callbackasm1(SB) + MOVW $854, R12 + B runtime·callbackasm1(SB) + MOVW $855, R12 + B runtime·callbackasm1(SB) + MOVW $856, R12 + B runtime·callbackasm1(SB) + MOVW $857, R12 + B runtime·callbackasm1(SB) + MOVW $858, R12 + B runtime·callbackasm1(SB) + MOVW $859, R12 + B runtime·callbackasm1(SB) + MOVW $860, R12 + B runtime·callbackasm1(SB) + MOVW $861, R12 + B runtime·callbackasm1(SB) + MOVW $862, R12 + B runtime·callbackasm1(SB) + MOVW $863, R12 + B runtime·callbackasm1(SB) + MOVW $864, R12 + B runtime·callbackasm1(SB) + MOVW $865, R12 + B runtime·callbackasm1(SB) + MOVW $866, R12 + B runtime·callbackasm1(SB) + MOVW $867, R12 + B runtime·callbackasm1(SB) + MOVW $868, R12 + B runtime·callbackasm1(SB) + MOVW $869, R12 + B runtime·callbackasm1(SB) + MOVW $870, R12 + B runtime·callbackasm1(SB) + MOVW $871, R12 + B runtime·callbackasm1(SB) + MOVW $872, R12 + B runtime·callbackasm1(SB) + MOVW $873, R12 + B runtime·callbackasm1(SB) + MOVW $874, R12 + B runtime·callbackasm1(SB) + MOVW $875, R12 + B runtime·callbackasm1(SB) + MOVW $876, R12 + B runtime·callbackasm1(SB) + MOVW $877, R12 + B runtime·callbackasm1(SB) + MOVW $878, R12 + B runtime·callbackasm1(SB) + MOVW $879, R12 + B runtime·callbackasm1(SB) + MOVW $880, R12 + B runtime·callbackasm1(SB) + MOVW $881, R12 + B runtime·callbackasm1(SB) + MOVW $882, R12 + B runtime·callbackasm1(SB) + MOVW $883, R12 + B runtime·callbackasm1(SB) + MOVW $884, R12 + B runtime·callbackasm1(SB) + MOVW $885, R12 + B runtime·callbackasm1(SB) + MOVW $886, R12 + B runtime·callbackasm1(SB) + MOVW $887, R12 + B runtime·callbackasm1(SB) + MOVW $888, R12 + B runtime·callbackasm1(SB) + MOVW $889, R12 + B runtime·callbackasm1(SB) + MOVW $890, R12 + B runtime·callbackasm1(SB) + MOVW $891, R12 + B runtime·callbackasm1(SB) + MOVW $892, R12 + B runtime·callbackasm1(SB) + MOVW $893, R12 + B runtime·callbackasm1(SB) + MOVW $894, R12 + B runtime·callbackasm1(SB) + MOVW $895, R12 + B runtime·callbackasm1(SB) + MOVW $896, R12 + B runtime·callbackasm1(SB) + MOVW $897, R12 + B runtime·callbackasm1(SB) + MOVW $898, R12 + B runtime·callbackasm1(SB) + MOVW $899, R12 + B runtime·callbackasm1(SB) + MOVW $900, R12 + B runtime·callbackasm1(SB) + MOVW $901, R12 + B runtime·callbackasm1(SB) + MOVW $902, R12 + B runtime·callbackasm1(SB) + MOVW $903, R12 + B runtime·callbackasm1(SB) + MOVW $904, R12 + B runtime·callbackasm1(SB) + MOVW $905, R12 + B runtime·callbackasm1(SB) + MOVW $906, R12 + B runtime·callbackasm1(SB) + MOVW $907, R12 + B runtime·callbackasm1(SB) + MOVW $908, R12 + B runtime·callbackasm1(SB) + MOVW $909, R12 + B runtime·callbackasm1(SB) + MOVW $910, R12 + B runtime·callbackasm1(SB) + MOVW $911, R12 + B runtime·callbackasm1(SB) + MOVW $912, R12 + B runtime·callbackasm1(SB) + MOVW $913, R12 + B runtime·callbackasm1(SB) + MOVW $914, R12 + B runtime·callbackasm1(SB) + MOVW $915, R12 + B runtime·callbackasm1(SB) + MOVW $916, R12 + B runtime·callbackasm1(SB) + MOVW $917, R12 + B runtime·callbackasm1(SB) + MOVW $918, R12 + B runtime·callbackasm1(SB) + MOVW $919, R12 + B runtime·callbackasm1(SB) + MOVW $920, R12 + B runtime·callbackasm1(SB) + MOVW $921, R12 + B runtime·callbackasm1(SB) + MOVW $922, R12 + B runtime·callbackasm1(SB) + MOVW $923, R12 + B runtime·callbackasm1(SB) + MOVW $924, R12 + B runtime·callbackasm1(SB) + MOVW $925, R12 + B runtime·callbackasm1(SB) + MOVW $926, R12 + B runtime·callbackasm1(SB) + MOVW $927, R12 + B runtime·callbackasm1(SB) + MOVW $928, R12 + B runtime·callbackasm1(SB) + MOVW $929, R12 + B runtime·callbackasm1(SB) + MOVW $930, R12 + B runtime·callbackasm1(SB) + MOVW $931, R12 + B runtime·callbackasm1(SB) + MOVW $932, R12 + B runtime·callbackasm1(SB) + MOVW $933, R12 + B runtime·callbackasm1(SB) + MOVW $934, R12 + B runtime·callbackasm1(SB) + MOVW $935, R12 + B runtime·callbackasm1(SB) + MOVW $936, R12 + B runtime·callbackasm1(SB) + MOVW $937, R12 + B runtime·callbackasm1(SB) + MOVW $938, R12 + B runtime·callbackasm1(SB) + MOVW $939, R12 + B runtime·callbackasm1(SB) + MOVW $940, R12 + B runtime·callbackasm1(SB) + MOVW $941, R12 + B runtime·callbackasm1(SB) + MOVW $942, R12 + B runtime·callbackasm1(SB) + MOVW $943, R12 + B runtime·callbackasm1(SB) + MOVW $944, R12 + B runtime·callbackasm1(SB) + MOVW $945, R12 + B runtime·callbackasm1(SB) + MOVW $946, R12 + B runtime·callbackasm1(SB) + MOVW $947, R12 + B runtime·callbackasm1(SB) + MOVW $948, R12 + B runtime·callbackasm1(SB) + MOVW $949, R12 + B runtime·callbackasm1(SB) + MOVW $950, R12 + B runtime·callbackasm1(SB) + MOVW $951, R12 + B runtime·callbackasm1(SB) + MOVW $952, R12 + B runtime·callbackasm1(SB) + MOVW $953, R12 + B runtime·callbackasm1(SB) + MOVW $954, R12 + B runtime·callbackasm1(SB) + MOVW $955, R12 + B runtime·callbackasm1(SB) + MOVW $956, R12 + B runtime·callbackasm1(SB) + MOVW $957, R12 + B runtime·callbackasm1(SB) + MOVW $958, R12 + B runtime·callbackasm1(SB) + MOVW $959, R12 + B runtime·callbackasm1(SB) + MOVW $960, R12 + B runtime·callbackasm1(SB) + MOVW $961, R12 + B runtime·callbackasm1(SB) + MOVW $962, R12 + B runtime·callbackasm1(SB) + MOVW $963, R12 + B runtime·callbackasm1(SB) + MOVW $964, R12 + B runtime·callbackasm1(SB) + MOVW $965, R12 + B runtime·callbackasm1(SB) + MOVW $966, R12 + B runtime·callbackasm1(SB) + MOVW $967, R12 + B runtime·callbackasm1(SB) + MOVW $968, R12 + B runtime·callbackasm1(SB) + MOVW $969, R12 + B runtime·callbackasm1(SB) + MOVW $970, R12 + B runtime·callbackasm1(SB) + MOVW $971, R12 + B runtime·callbackasm1(SB) + MOVW $972, R12 + B runtime·callbackasm1(SB) + MOVW $973, R12 + B runtime·callbackasm1(SB) + MOVW $974, R12 + B runtime·callbackasm1(SB) + MOVW $975, R12 + B runtime·callbackasm1(SB) + MOVW $976, R12 + B runtime·callbackasm1(SB) + MOVW $977, R12 + B runtime·callbackasm1(SB) + MOVW $978, R12 + B runtime·callbackasm1(SB) + MOVW $979, R12 + B runtime·callbackasm1(SB) + MOVW $980, R12 + B runtime·callbackasm1(SB) + MOVW $981, R12 + B runtime·callbackasm1(SB) + MOVW $982, R12 + B runtime·callbackasm1(SB) + MOVW $983, R12 + B runtime·callbackasm1(SB) + MOVW $984, R12 + B runtime·callbackasm1(SB) + MOVW $985, R12 + B runtime·callbackasm1(SB) + MOVW $986, R12 + B runtime·callbackasm1(SB) + MOVW $987, R12 + B runtime·callbackasm1(SB) + MOVW $988, R12 + B runtime·callbackasm1(SB) + MOVW $989, R12 + B runtime·callbackasm1(SB) + MOVW $990, R12 + B runtime·callbackasm1(SB) + MOVW $991, R12 + B runtime·callbackasm1(SB) + MOVW $992, R12 + B runtime·callbackasm1(SB) + MOVW $993, R12 + B runtime·callbackasm1(SB) + MOVW $994, R12 + B runtime·callbackasm1(SB) + MOVW $995, R12 + B runtime·callbackasm1(SB) + MOVW $996, R12 + B runtime·callbackasm1(SB) + MOVW $997, R12 + B runtime·callbackasm1(SB) + MOVW $998, R12 + B runtime·callbackasm1(SB) + MOVW $999, R12 + B runtime·callbackasm1(SB) + MOVW $1000, R12 + B runtime·callbackasm1(SB) + MOVW $1001, R12 + B runtime·callbackasm1(SB) + MOVW $1002, R12 + B runtime·callbackasm1(SB) + MOVW $1003, R12 + B runtime·callbackasm1(SB) + MOVW $1004, R12 + B runtime·callbackasm1(SB) + MOVW $1005, R12 + B runtime·callbackasm1(SB) + MOVW $1006, R12 + B runtime·callbackasm1(SB) + MOVW $1007, R12 + B runtime·callbackasm1(SB) + MOVW $1008, R12 + B runtime·callbackasm1(SB) + MOVW $1009, R12 + B runtime·callbackasm1(SB) + MOVW $1010, R12 + B runtime·callbackasm1(SB) + MOVW $1011, R12 + B runtime·callbackasm1(SB) + MOVW $1012, R12 + B runtime·callbackasm1(SB) + MOVW $1013, R12 + B runtime·callbackasm1(SB) + MOVW $1014, R12 + B runtime·callbackasm1(SB) + MOVW $1015, R12 + B runtime·callbackasm1(SB) + MOVW $1016, R12 + B runtime·callbackasm1(SB) + MOVW $1017, R12 + B runtime·callbackasm1(SB) + MOVW $1018, R12 + B runtime·callbackasm1(SB) + MOVW $1019, R12 + B runtime·callbackasm1(SB) + MOVW $1020, R12 + B runtime·callbackasm1(SB) + MOVW $1021, R12 + B runtime·callbackasm1(SB) + MOVW $1022, R12 + B runtime·callbackasm1(SB) + MOVW $1023, R12 + B runtime·callbackasm1(SB) + MOVW $1024, R12 + B runtime·callbackasm1(SB) + MOVW $1025, R12 + B runtime·callbackasm1(SB) + MOVW $1026, R12 + B runtime·callbackasm1(SB) + MOVW $1027, R12 + B runtime·callbackasm1(SB) + MOVW $1028, R12 + B runtime·callbackasm1(SB) + MOVW $1029, R12 + B runtime·callbackasm1(SB) + MOVW $1030, R12 + B runtime·callbackasm1(SB) + MOVW $1031, R12 + B runtime·callbackasm1(SB) + MOVW $1032, R12 + B runtime·callbackasm1(SB) + MOVW $1033, R12 + B runtime·callbackasm1(SB) + MOVW $1034, R12 + B runtime·callbackasm1(SB) + MOVW $1035, R12 + B runtime·callbackasm1(SB) + MOVW $1036, R12 + B runtime·callbackasm1(SB) + MOVW $1037, R12 + B runtime·callbackasm1(SB) + MOVW $1038, R12 + B runtime·callbackasm1(SB) + MOVW $1039, R12 + B runtime·callbackasm1(SB) + MOVW $1040, R12 + B runtime·callbackasm1(SB) + MOVW $1041, R12 + B runtime·callbackasm1(SB) + MOVW $1042, R12 + B runtime·callbackasm1(SB) + MOVW $1043, R12 + B runtime·callbackasm1(SB) + MOVW $1044, R12 + B runtime·callbackasm1(SB) + MOVW $1045, R12 + B runtime·callbackasm1(SB) + MOVW $1046, R12 + B runtime·callbackasm1(SB) + MOVW $1047, R12 + B runtime·callbackasm1(SB) + MOVW $1048, R12 + B runtime·callbackasm1(SB) + MOVW $1049, R12 + B runtime·callbackasm1(SB) + MOVW $1050, R12 + B runtime·callbackasm1(SB) + MOVW $1051, R12 + B runtime·callbackasm1(SB) + MOVW $1052, R12 + B runtime·callbackasm1(SB) + MOVW $1053, R12 + B runtime·callbackasm1(SB) + MOVW $1054, R12 + B runtime·callbackasm1(SB) + MOVW $1055, R12 + B runtime·callbackasm1(SB) + MOVW $1056, R12 + B runtime·callbackasm1(SB) + MOVW $1057, R12 + B runtime·callbackasm1(SB) + MOVW $1058, R12 + B runtime·callbackasm1(SB) + MOVW $1059, R12 + B runtime·callbackasm1(SB) + MOVW $1060, R12 + B runtime·callbackasm1(SB) + MOVW $1061, R12 + B runtime·callbackasm1(SB) + MOVW $1062, R12 + B runtime·callbackasm1(SB) + MOVW $1063, R12 + B runtime·callbackasm1(SB) + MOVW $1064, R12 + B runtime·callbackasm1(SB) + MOVW $1065, R12 + B runtime·callbackasm1(SB) + MOVW $1066, R12 + B runtime·callbackasm1(SB) + MOVW $1067, R12 + B runtime·callbackasm1(SB) + MOVW $1068, R12 + B runtime·callbackasm1(SB) + MOVW $1069, R12 + B runtime·callbackasm1(SB) + MOVW $1070, R12 + B runtime·callbackasm1(SB) + MOVW $1071, R12 + B runtime·callbackasm1(SB) + MOVW $1072, R12 + B runtime·callbackasm1(SB) + MOVW $1073, R12 + B runtime·callbackasm1(SB) + MOVW $1074, R12 + B runtime·callbackasm1(SB) + MOVW $1075, R12 + B runtime·callbackasm1(SB) + MOVW $1076, R12 + B runtime·callbackasm1(SB) + MOVW $1077, R12 + B runtime·callbackasm1(SB) + MOVW $1078, R12 + B runtime·callbackasm1(SB) + MOVW $1079, R12 + B runtime·callbackasm1(SB) + MOVW $1080, R12 + B runtime·callbackasm1(SB) + MOVW $1081, R12 + B runtime·callbackasm1(SB) + MOVW $1082, R12 + B runtime·callbackasm1(SB) + MOVW $1083, R12 + B runtime·callbackasm1(SB) + MOVW $1084, R12 + B runtime·callbackasm1(SB) + MOVW $1085, R12 + B runtime·callbackasm1(SB) + MOVW $1086, R12 + B runtime·callbackasm1(SB) + MOVW $1087, R12 + B runtime·callbackasm1(SB) + MOVW $1088, R12 + B runtime·callbackasm1(SB) + MOVW $1089, R12 + B runtime·callbackasm1(SB) + MOVW $1090, R12 + B runtime·callbackasm1(SB) + MOVW $1091, R12 + B runtime·callbackasm1(SB) + MOVW $1092, R12 + B runtime·callbackasm1(SB) + MOVW $1093, R12 + B runtime·callbackasm1(SB) + MOVW $1094, R12 + B runtime·callbackasm1(SB) + MOVW $1095, R12 + B runtime·callbackasm1(SB) + MOVW $1096, R12 + B runtime·callbackasm1(SB) + MOVW $1097, R12 + B runtime·callbackasm1(SB) + MOVW $1098, R12 + B runtime·callbackasm1(SB) + MOVW $1099, R12 + B runtime·callbackasm1(SB) + MOVW $1100, R12 + B runtime·callbackasm1(SB) + MOVW $1101, R12 + B runtime·callbackasm1(SB) + MOVW $1102, R12 + B runtime·callbackasm1(SB) + MOVW $1103, R12 + B runtime·callbackasm1(SB) + MOVW $1104, R12 + B runtime·callbackasm1(SB) + MOVW $1105, R12 + B runtime·callbackasm1(SB) + MOVW $1106, R12 + B runtime·callbackasm1(SB) + MOVW $1107, R12 + B runtime·callbackasm1(SB) + MOVW $1108, R12 + B runtime·callbackasm1(SB) + MOVW $1109, R12 + B runtime·callbackasm1(SB) + MOVW $1110, R12 + B runtime·callbackasm1(SB) + MOVW $1111, R12 + B runtime·callbackasm1(SB) + MOVW $1112, R12 + B runtime·callbackasm1(SB) + MOVW $1113, R12 + B runtime·callbackasm1(SB) + MOVW $1114, R12 + B runtime·callbackasm1(SB) + MOVW $1115, R12 + B runtime·callbackasm1(SB) + MOVW $1116, R12 + B runtime·callbackasm1(SB) + MOVW $1117, R12 + B runtime·callbackasm1(SB) + MOVW $1118, R12 + B runtime·callbackasm1(SB) + MOVW $1119, R12 + B runtime·callbackasm1(SB) + MOVW $1120, R12 + B runtime·callbackasm1(SB) + MOVW $1121, R12 + B runtime·callbackasm1(SB) + MOVW $1122, R12 + B runtime·callbackasm1(SB) + MOVW $1123, R12 + B runtime·callbackasm1(SB) + MOVW $1124, R12 + B runtime·callbackasm1(SB) + MOVW $1125, R12 + B runtime·callbackasm1(SB) + MOVW $1126, R12 + B runtime·callbackasm1(SB) + MOVW $1127, R12 + B runtime·callbackasm1(SB) + MOVW $1128, R12 + B runtime·callbackasm1(SB) + MOVW $1129, R12 + B runtime·callbackasm1(SB) + MOVW $1130, R12 + B runtime·callbackasm1(SB) + MOVW $1131, R12 + B runtime·callbackasm1(SB) + MOVW $1132, R12 + B runtime·callbackasm1(SB) + MOVW $1133, R12 + B runtime·callbackasm1(SB) + MOVW $1134, R12 + B runtime·callbackasm1(SB) + MOVW $1135, R12 + B runtime·callbackasm1(SB) + MOVW $1136, R12 + B runtime·callbackasm1(SB) + MOVW $1137, R12 + B runtime·callbackasm1(SB) + MOVW $1138, R12 + B runtime·callbackasm1(SB) + MOVW $1139, R12 + B runtime·callbackasm1(SB) + MOVW $1140, R12 + B runtime·callbackasm1(SB) + MOVW $1141, R12 + B runtime·callbackasm1(SB) + MOVW $1142, R12 + B runtime·callbackasm1(SB) + MOVW $1143, R12 + B runtime·callbackasm1(SB) + MOVW $1144, R12 + B runtime·callbackasm1(SB) + MOVW $1145, R12 + B runtime·callbackasm1(SB) + MOVW $1146, R12 + B runtime·callbackasm1(SB) + MOVW $1147, R12 + B runtime·callbackasm1(SB) + MOVW $1148, R12 + B runtime·callbackasm1(SB) + MOVW $1149, R12 + B runtime·callbackasm1(SB) + MOVW $1150, R12 + B runtime·callbackasm1(SB) + MOVW $1151, R12 + B runtime·callbackasm1(SB) + MOVW $1152, R12 + B runtime·callbackasm1(SB) + MOVW $1153, R12 + B runtime·callbackasm1(SB) + MOVW $1154, R12 + B runtime·callbackasm1(SB) + MOVW $1155, R12 + B runtime·callbackasm1(SB) + MOVW $1156, R12 + B runtime·callbackasm1(SB) + MOVW $1157, R12 + B runtime·callbackasm1(SB) + MOVW $1158, R12 + B runtime·callbackasm1(SB) + MOVW $1159, R12 + B runtime·callbackasm1(SB) + MOVW $1160, R12 + B runtime·callbackasm1(SB) + MOVW $1161, R12 + B runtime·callbackasm1(SB) + MOVW $1162, R12 + B runtime·callbackasm1(SB) + MOVW $1163, R12 + B runtime·callbackasm1(SB) + MOVW $1164, R12 + B runtime·callbackasm1(SB) + MOVW $1165, R12 + B runtime·callbackasm1(SB) + MOVW $1166, R12 + B runtime·callbackasm1(SB) + MOVW $1167, R12 + B runtime·callbackasm1(SB) + MOVW $1168, R12 + B runtime·callbackasm1(SB) + MOVW $1169, R12 + B runtime·callbackasm1(SB) + MOVW $1170, R12 + B runtime·callbackasm1(SB) + MOVW $1171, R12 + B runtime·callbackasm1(SB) + MOVW $1172, R12 + B runtime·callbackasm1(SB) + MOVW $1173, R12 + B runtime·callbackasm1(SB) + MOVW $1174, R12 + B runtime·callbackasm1(SB) + MOVW $1175, R12 + B runtime·callbackasm1(SB) + MOVW $1176, R12 + B runtime·callbackasm1(SB) + MOVW $1177, R12 + B runtime·callbackasm1(SB) + MOVW $1178, R12 + B runtime·callbackasm1(SB) + MOVW $1179, R12 + B runtime·callbackasm1(SB) + MOVW $1180, R12 + B runtime·callbackasm1(SB) + MOVW $1181, R12 + B runtime·callbackasm1(SB) + MOVW $1182, R12 + B runtime·callbackasm1(SB) + MOVW $1183, R12 + B runtime·callbackasm1(SB) + MOVW $1184, R12 + B runtime·callbackasm1(SB) + MOVW $1185, R12 + B runtime·callbackasm1(SB) + MOVW $1186, R12 + B runtime·callbackasm1(SB) + MOVW $1187, R12 + B runtime·callbackasm1(SB) + MOVW $1188, R12 + B runtime·callbackasm1(SB) + MOVW $1189, R12 + B runtime·callbackasm1(SB) + MOVW $1190, R12 + B runtime·callbackasm1(SB) + MOVW $1191, R12 + B runtime·callbackasm1(SB) + MOVW $1192, R12 + B runtime·callbackasm1(SB) + MOVW $1193, R12 + B runtime·callbackasm1(SB) + MOVW $1194, R12 + B runtime·callbackasm1(SB) + MOVW $1195, R12 + B runtime·callbackasm1(SB) + MOVW $1196, R12 + B runtime·callbackasm1(SB) + MOVW $1197, R12 + B runtime·callbackasm1(SB) + MOVW $1198, R12 + B runtime·callbackasm1(SB) + MOVW $1199, R12 + B runtime·callbackasm1(SB) + MOVW $1200, R12 + B runtime·callbackasm1(SB) + MOVW $1201, R12 + B runtime·callbackasm1(SB) + MOVW $1202, R12 + B runtime·callbackasm1(SB) + MOVW $1203, R12 + B runtime·callbackasm1(SB) + MOVW $1204, R12 + B runtime·callbackasm1(SB) + MOVW $1205, R12 + B runtime·callbackasm1(SB) + MOVW $1206, R12 + B runtime·callbackasm1(SB) + MOVW $1207, R12 + B runtime·callbackasm1(SB) + MOVW $1208, R12 + B runtime·callbackasm1(SB) + MOVW $1209, R12 + B runtime·callbackasm1(SB) + MOVW $1210, R12 + B runtime·callbackasm1(SB) + MOVW $1211, R12 + B runtime·callbackasm1(SB) + MOVW $1212, R12 + B runtime·callbackasm1(SB) + MOVW $1213, R12 + B runtime·callbackasm1(SB) + MOVW $1214, R12 + B runtime·callbackasm1(SB) + MOVW $1215, R12 + B runtime·callbackasm1(SB) + MOVW $1216, R12 + B runtime·callbackasm1(SB) + MOVW $1217, R12 + B runtime·callbackasm1(SB) + MOVW $1218, R12 + B runtime·callbackasm1(SB) + MOVW $1219, R12 + B runtime·callbackasm1(SB) + MOVW $1220, R12 + B runtime·callbackasm1(SB) + MOVW $1221, R12 + B runtime·callbackasm1(SB) + MOVW $1222, R12 + B runtime·callbackasm1(SB) + MOVW $1223, R12 + B runtime·callbackasm1(SB) + MOVW $1224, R12 + B runtime·callbackasm1(SB) + MOVW $1225, R12 + B runtime·callbackasm1(SB) + MOVW $1226, R12 + B runtime·callbackasm1(SB) + MOVW $1227, R12 + B runtime·callbackasm1(SB) + MOVW $1228, R12 + B runtime·callbackasm1(SB) + MOVW $1229, R12 + B runtime·callbackasm1(SB) + MOVW $1230, R12 + B runtime·callbackasm1(SB) + MOVW $1231, R12 + B runtime·callbackasm1(SB) + MOVW $1232, R12 + B runtime·callbackasm1(SB) + MOVW $1233, R12 + B runtime·callbackasm1(SB) + MOVW $1234, R12 + B runtime·callbackasm1(SB) + MOVW $1235, R12 + B runtime·callbackasm1(SB) + MOVW $1236, R12 + B runtime·callbackasm1(SB) + MOVW $1237, R12 + B runtime·callbackasm1(SB) + MOVW $1238, R12 + B runtime·callbackasm1(SB) + MOVW $1239, R12 + B runtime·callbackasm1(SB) + MOVW $1240, R12 + B runtime·callbackasm1(SB) + MOVW $1241, R12 + B runtime·callbackasm1(SB) + MOVW $1242, R12 + B runtime·callbackasm1(SB) + MOVW $1243, R12 + B runtime·callbackasm1(SB) + MOVW $1244, R12 + B runtime·callbackasm1(SB) + MOVW $1245, R12 + B runtime·callbackasm1(SB) + MOVW $1246, R12 + B runtime·callbackasm1(SB) + MOVW $1247, R12 + B runtime·callbackasm1(SB) + MOVW $1248, R12 + B runtime·callbackasm1(SB) + MOVW $1249, R12 + B runtime·callbackasm1(SB) + MOVW $1250, R12 + B runtime·callbackasm1(SB) + MOVW $1251, R12 + B runtime·callbackasm1(SB) + MOVW $1252, R12 + B runtime·callbackasm1(SB) + MOVW $1253, R12 + B runtime·callbackasm1(SB) + MOVW $1254, R12 + B runtime·callbackasm1(SB) + MOVW $1255, R12 + B runtime·callbackasm1(SB) + MOVW $1256, R12 + B runtime·callbackasm1(SB) + MOVW $1257, R12 + B runtime·callbackasm1(SB) + MOVW $1258, R12 + B runtime·callbackasm1(SB) + MOVW $1259, R12 + B runtime·callbackasm1(SB) + MOVW $1260, R12 + B runtime·callbackasm1(SB) + MOVW $1261, R12 + B runtime·callbackasm1(SB) + MOVW $1262, R12 + B runtime·callbackasm1(SB) + MOVW $1263, R12 + B runtime·callbackasm1(SB) + MOVW $1264, R12 + B runtime·callbackasm1(SB) + MOVW $1265, R12 + B runtime·callbackasm1(SB) + MOVW $1266, R12 + B runtime·callbackasm1(SB) + MOVW $1267, R12 + B runtime·callbackasm1(SB) + MOVW $1268, R12 + B runtime·callbackasm1(SB) + MOVW $1269, R12 + B runtime·callbackasm1(SB) + MOVW $1270, R12 + B runtime·callbackasm1(SB) + MOVW $1271, R12 + B runtime·callbackasm1(SB) + MOVW $1272, R12 + B runtime·callbackasm1(SB) + MOVW $1273, R12 + B runtime·callbackasm1(SB) + MOVW $1274, R12 + B runtime·callbackasm1(SB) + MOVW $1275, R12 + B runtime·callbackasm1(SB) + MOVW $1276, R12 + B runtime·callbackasm1(SB) + MOVW $1277, R12 + B runtime·callbackasm1(SB) + MOVW $1278, R12 + B runtime·callbackasm1(SB) + MOVW $1279, R12 + B runtime·callbackasm1(SB) + MOVW $1280, R12 + B runtime·callbackasm1(SB) + MOVW $1281, R12 + B runtime·callbackasm1(SB) + MOVW $1282, R12 + B runtime·callbackasm1(SB) + MOVW $1283, R12 + B runtime·callbackasm1(SB) + MOVW $1284, R12 + B runtime·callbackasm1(SB) + MOVW $1285, R12 + B runtime·callbackasm1(SB) + MOVW $1286, R12 + B runtime·callbackasm1(SB) + MOVW $1287, R12 + B runtime·callbackasm1(SB) + MOVW $1288, R12 + B runtime·callbackasm1(SB) + MOVW $1289, R12 + B runtime·callbackasm1(SB) + MOVW $1290, R12 + B runtime·callbackasm1(SB) + MOVW $1291, R12 + B runtime·callbackasm1(SB) + MOVW $1292, R12 + B runtime·callbackasm1(SB) + MOVW $1293, R12 + B runtime·callbackasm1(SB) + MOVW $1294, R12 + B runtime·callbackasm1(SB) + MOVW $1295, R12 + B runtime·callbackasm1(SB) + MOVW $1296, R12 + B runtime·callbackasm1(SB) + MOVW $1297, R12 + B runtime·callbackasm1(SB) + MOVW $1298, R12 + B runtime·callbackasm1(SB) + MOVW $1299, R12 + B runtime·callbackasm1(SB) + MOVW $1300, R12 + B runtime·callbackasm1(SB) + MOVW $1301, R12 + B runtime·callbackasm1(SB) + MOVW $1302, R12 + B runtime·callbackasm1(SB) + MOVW $1303, R12 + B runtime·callbackasm1(SB) + MOVW $1304, R12 + B runtime·callbackasm1(SB) + MOVW $1305, R12 + B runtime·callbackasm1(SB) + MOVW $1306, R12 + B runtime·callbackasm1(SB) + MOVW $1307, R12 + B runtime·callbackasm1(SB) + MOVW $1308, R12 + B runtime·callbackasm1(SB) + MOVW $1309, R12 + B runtime·callbackasm1(SB) + MOVW $1310, R12 + B runtime·callbackasm1(SB) + MOVW $1311, R12 + B runtime·callbackasm1(SB) + MOVW $1312, R12 + B runtime·callbackasm1(SB) + MOVW $1313, R12 + B runtime·callbackasm1(SB) + MOVW $1314, R12 + B runtime·callbackasm1(SB) + MOVW $1315, R12 + B runtime·callbackasm1(SB) + MOVW $1316, R12 + B runtime·callbackasm1(SB) + MOVW $1317, R12 + B runtime·callbackasm1(SB) + MOVW $1318, R12 + B runtime·callbackasm1(SB) + MOVW $1319, R12 + B runtime·callbackasm1(SB) + MOVW $1320, R12 + B runtime·callbackasm1(SB) + MOVW $1321, R12 + B runtime·callbackasm1(SB) + MOVW $1322, R12 + B runtime·callbackasm1(SB) + MOVW $1323, R12 + B runtime·callbackasm1(SB) + MOVW $1324, R12 + B runtime·callbackasm1(SB) + MOVW $1325, R12 + B runtime·callbackasm1(SB) + MOVW $1326, R12 + B runtime·callbackasm1(SB) + MOVW $1327, R12 + B runtime·callbackasm1(SB) + MOVW $1328, R12 + B runtime·callbackasm1(SB) + MOVW $1329, R12 + B runtime·callbackasm1(SB) + MOVW $1330, R12 + B runtime·callbackasm1(SB) + MOVW $1331, R12 + B runtime·callbackasm1(SB) + MOVW $1332, R12 + B runtime·callbackasm1(SB) + MOVW $1333, R12 + B runtime·callbackasm1(SB) + MOVW $1334, R12 + B runtime·callbackasm1(SB) + MOVW $1335, R12 + B runtime·callbackasm1(SB) + MOVW $1336, R12 + B runtime·callbackasm1(SB) + MOVW $1337, R12 + B runtime·callbackasm1(SB) + MOVW $1338, R12 + B runtime·callbackasm1(SB) + MOVW $1339, R12 + B runtime·callbackasm1(SB) + MOVW $1340, R12 + B runtime·callbackasm1(SB) + MOVW $1341, R12 + B runtime·callbackasm1(SB) + MOVW $1342, R12 + B runtime·callbackasm1(SB) + MOVW $1343, R12 + B runtime·callbackasm1(SB) + MOVW $1344, R12 + B runtime·callbackasm1(SB) + MOVW $1345, R12 + B runtime·callbackasm1(SB) + MOVW $1346, R12 + B runtime·callbackasm1(SB) + MOVW $1347, R12 + B runtime·callbackasm1(SB) + MOVW $1348, R12 + B runtime·callbackasm1(SB) + MOVW $1349, R12 + B runtime·callbackasm1(SB) + MOVW $1350, R12 + B runtime·callbackasm1(SB) + MOVW $1351, R12 + B runtime·callbackasm1(SB) + MOVW $1352, R12 + B runtime·callbackasm1(SB) + MOVW $1353, R12 + B runtime·callbackasm1(SB) + MOVW $1354, R12 + B runtime·callbackasm1(SB) + MOVW $1355, R12 + B runtime·callbackasm1(SB) + MOVW $1356, R12 + B runtime·callbackasm1(SB) + MOVW $1357, R12 + B runtime·callbackasm1(SB) + MOVW $1358, R12 + B runtime·callbackasm1(SB) + MOVW $1359, R12 + B runtime·callbackasm1(SB) + MOVW $1360, R12 + B runtime·callbackasm1(SB) + MOVW $1361, R12 + B runtime·callbackasm1(SB) + MOVW $1362, R12 + B runtime·callbackasm1(SB) + MOVW $1363, R12 + B runtime·callbackasm1(SB) + MOVW $1364, R12 + B runtime·callbackasm1(SB) + MOVW $1365, R12 + B runtime·callbackasm1(SB) + MOVW $1366, R12 + B runtime·callbackasm1(SB) + MOVW $1367, R12 + B runtime·callbackasm1(SB) + MOVW $1368, R12 + B runtime·callbackasm1(SB) + MOVW $1369, R12 + B runtime·callbackasm1(SB) + MOVW $1370, R12 + B runtime·callbackasm1(SB) + MOVW $1371, R12 + B runtime·callbackasm1(SB) + MOVW $1372, R12 + B runtime·callbackasm1(SB) + MOVW $1373, R12 + B runtime·callbackasm1(SB) + MOVW $1374, R12 + B runtime·callbackasm1(SB) + MOVW $1375, R12 + B runtime·callbackasm1(SB) + MOVW $1376, R12 + B runtime·callbackasm1(SB) + MOVW $1377, R12 + B runtime·callbackasm1(SB) + MOVW $1378, R12 + B runtime·callbackasm1(SB) + MOVW $1379, R12 + B runtime·callbackasm1(SB) + MOVW $1380, R12 + B runtime·callbackasm1(SB) + MOVW $1381, R12 + B runtime·callbackasm1(SB) + MOVW $1382, R12 + B runtime·callbackasm1(SB) + MOVW $1383, R12 + B runtime·callbackasm1(SB) + MOVW $1384, R12 + B runtime·callbackasm1(SB) + MOVW $1385, R12 + B runtime·callbackasm1(SB) + MOVW $1386, R12 + B runtime·callbackasm1(SB) + MOVW $1387, R12 + B runtime·callbackasm1(SB) + MOVW $1388, R12 + B runtime·callbackasm1(SB) + MOVW $1389, R12 + B runtime·callbackasm1(SB) + MOVW $1390, R12 + B runtime·callbackasm1(SB) + MOVW $1391, R12 + B runtime·callbackasm1(SB) + MOVW $1392, R12 + B runtime·callbackasm1(SB) + MOVW $1393, R12 + B runtime·callbackasm1(SB) + MOVW $1394, R12 + B runtime·callbackasm1(SB) + MOVW $1395, R12 + B runtime·callbackasm1(SB) + MOVW $1396, R12 + B runtime·callbackasm1(SB) + MOVW $1397, R12 + B runtime·callbackasm1(SB) + MOVW $1398, R12 + B runtime·callbackasm1(SB) + MOVW $1399, R12 + B runtime·callbackasm1(SB) + MOVW $1400, R12 + B runtime·callbackasm1(SB) + MOVW $1401, R12 + B runtime·callbackasm1(SB) + MOVW $1402, R12 + B runtime·callbackasm1(SB) + MOVW $1403, R12 + B runtime·callbackasm1(SB) + MOVW $1404, R12 + B runtime·callbackasm1(SB) + MOVW $1405, R12 + B runtime·callbackasm1(SB) + MOVW $1406, R12 + B runtime·callbackasm1(SB) + MOVW $1407, R12 + B runtime·callbackasm1(SB) + MOVW $1408, R12 + B runtime·callbackasm1(SB) + MOVW $1409, R12 + B runtime·callbackasm1(SB) + MOVW $1410, R12 + B runtime·callbackasm1(SB) + MOVW $1411, R12 + B runtime·callbackasm1(SB) + MOVW $1412, R12 + B runtime·callbackasm1(SB) + MOVW $1413, R12 + B runtime·callbackasm1(SB) + MOVW $1414, R12 + B runtime·callbackasm1(SB) + MOVW $1415, R12 + B runtime·callbackasm1(SB) + MOVW $1416, R12 + B runtime·callbackasm1(SB) + MOVW $1417, R12 + B runtime·callbackasm1(SB) + MOVW $1418, R12 + B runtime·callbackasm1(SB) + MOVW $1419, R12 + B runtime·callbackasm1(SB) + MOVW $1420, R12 + B runtime·callbackasm1(SB) + MOVW $1421, R12 + B runtime·callbackasm1(SB) + MOVW $1422, R12 + B runtime·callbackasm1(SB) + MOVW $1423, R12 + B runtime·callbackasm1(SB) + MOVW $1424, R12 + B runtime·callbackasm1(SB) + MOVW $1425, R12 + B runtime·callbackasm1(SB) + MOVW $1426, R12 + B runtime·callbackasm1(SB) + MOVW $1427, R12 + B runtime·callbackasm1(SB) + MOVW $1428, R12 + B runtime·callbackasm1(SB) + MOVW $1429, R12 + B runtime·callbackasm1(SB) + MOVW $1430, R12 + B runtime·callbackasm1(SB) + MOVW $1431, R12 + B runtime·callbackasm1(SB) + MOVW $1432, R12 + B runtime·callbackasm1(SB) + MOVW $1433, R12 + B runtime·callbackasm1(SB) + MOVW $1434, R12 + B runtime·callbackasm1(SB) + MOVW $1435, R12 + B runtime·callbackasm1(SB) + MOVW $1436, R12 + B runtime·callbackasm1(SB) + MOVW $1437, R12 + B runtime·callbackasm1(SB) + MOVW $1438, R12 + B runtime·callbackasm1(SB) + MOVW $1439, R12 + B runtime·callbackasm1(SB) + MOVW $1440, R12 + B runtime·callbackasm1(SB) + MOVW $1441, R12 + B runtime·callbackasm1(SB) + MOVW $1442, R12 + B runtime·callbackasm1(SB) + MOVW $1443, R12 + B runtime·callbackasm1(SB) + MOVW $1444, R12 + B runtime·callbackasm1(SB) + MOVW $1445, R12 + B runtime·callbackasm1(SB) + MOVW $1446, R12 + B runtime·callbackasm1(SB) + MOVW $1447, R12 + B runtime·callbackasm1(SB) + MOVW $1448, R12 + B runtime·callbackasm1(SB) + MOVW $1449, R12 + B runtime·callbackasm1(SB) + MOVW $1450, R12 + B runtime·callbackasm1(SB) + MOVW $1451, R12 + B runtime·callbackasm1(SB) + MOVW $1452, R12 + B runtime·callbackasm1(SB) + MOVW $1453, R12 + B runtime·callbackasm1(SB) + MOVW $1454, R12 + B runtime·callbackasm1(SB) + MOVW $1455, R12 + B runtime·callbackasm1(SB) + MOVW $1456, R12 + B runtime·callbackasm1(SB) + MOVW $1457, R12 + B runtime·callbackasm1(SB) + MOVW $1458, R12 + B runtime·callbackasm1(SB) + MOVW $1459, R12 + B runtime·callbackasm1(SB) + MOVW $1460, R12 + B runtime·callbackasm1(SB) + MOVW $1461, R12 + B runtime·callbackasm1(SB) + MOVW $1462, R12 + B runtime·callbackasm1(SB) + MOVW $1463, R12 + B runtime·callbackasm1(SB) + MOVW $1464, R12 + B runtime·callbackasm1(SB) + MOVW $1465, R12 + B runtime·callbackasm1(SB) + MOVW $1466, R12 + B runtime·callbackasm1(SB) + MOVW $1467, R12 + B runtime·callbackasm1(SB) + MOVW $1468, R12 + B runtime·callbackasm1(SB) + MOVW $1469, R12 + B runtime·callbackasm1(SB) + MOVW $1470, R12 + B runtime·callbackasm1(SB) + MOVW $1471, R12 + B runtime·callbackasm1(SB) + MOVW $1472, R12 + B runtime·callbackasm1(SB) + MOVW $1473, R12 + B runtime·callbackasm1(SB) + MOVW $1474, R12 + B runtime·callbackasm1(SB) + MOVW $1475, R12 + B runtime·callbackasm1(SB) + MOVW $1476, R12 + B runtime·callbackasm1(SB) + MOVW $1477, R12 + B runtime·callbackasm1(SB) + MOVW $1478, R12 + B runtime·callbackasm1(SB) + MOVW $1479, R12 + B runtime·callbackasm1(SB) + MOVW $1480, R12 + B runtime·callbackasm1(SB) + MOVW $1481, R12 + B runtime·callbackasm1(SB) + MOVW $1482, R12 + B runtime·callbackasm1(SB) + MOVW $1483, R12 + B runtime·callbackasm1(SB) + MOVW $1484, R12 + B runtime·callbackasm1(SB) + MOVW $1485, R12 + B runtime·callbackasm1(SB) + MOVW $1486, R12 + B runtime·callbackasm1(SB) + MOVW $1487, R12 + B runtime·callbackasm1(SB) + MOVW $1488, R12 + B runtime·callbackasm1(SB) + MOVW $1489, R12 + B runtime·callbackasm1(SB) + MOVW $1490, R12 + B runtime·callbackasm1(SB) + MOVW $1491, R12 + B runtime·callbackasm1(SB) + MOVW $1492, R12 + B runtime·callbackasm1(SB) + MOVW $1493, R12 + B runtime·callbackasm1(SB) + MOVW $1494, R12 + B runtime·callbackasm1(SB) + MOVW $1495, R12 + B runtime·callbackasm1(SB) + MOVW $1496, R12 + B runtime·callbackasm1(SB) + MOVW $1497, R12 + B runtime·callbackasm1(SB) + MOVW $1498, R12 + B runtime·callbackasm1(SB) + MOVW $1499, R12 + B runtime·callbackasm1(SB) + MOVW $1500, R12 + B runtime·callbackasm1(SB) + MOVW $1501, R12 + B runtime·callbackasm1(SB) + MOVW $1502, R12 + B runtime·callbackasm1(SB) + MOVW $1503, R12 + B runtime·callbackasm1(SB) + MOVW $1504, R12 + B runtime·callbackasm1(SB) + MOVW $1505, R12 + B runtime·callbackasm1(SB) + MOVW $1506, R12 + B runtime·callbackasm1(SB) + MOVW $1507, R12 + B runtime·callbackasm1(SB) + MOVW $1508, R12 + B runtime·callbackasm1(SB) + MOVW $1509, R12 + B runtime·callbackasm1(SB) + MOVW $1510, R12 + B runtime·callbackasm1(SB) + MOVW $1511, R12 + B runtime·callbackasm1(SB) + MOVW $1512, R12 + B runtime·callbackasm1(SB) + MOVW $1513, R12 + B runtime·callbackasm1(SB) + MOVW $1514, R12 + B runtime·callbackasm1(SB) + MOVW $1515, R12 + B runtime·callbackasm1(SB) + MOVW $1516, R12 + B runtime·callbackasm1(SB) + MOVW $1517, R12 + B runtime·callbackasm1(SB) + MOVW $1518, R12 + B runtime·callbackasm1(SB) + MOVW $1519, R12 + B runtime·callbackasm1(SB) + MOVW $1520, R12 + B runtime·callbackasm1(SB) + MOVW $1521, R12 + B runtime·callbackasm1(SB) + MOVW $1522, R12 + B runtime·callbackasm1(SB) + MOVW $1523, R12 + B runtime·callbackasm1(SB) + MOVW $1524, R12 + B runtime·callbackasm1(SB) + MOVW $1525, R12 + B runtime·callbackasm1(SB) + MOVW $1526, R12 + B runtime·callbackasm1(SB) + MOVW $1527, R12 + B runtime·callbackasm1(SB) + MOVW $1528, R12 + B runtime·callbackasm1(SB) + MOVW $1529, R12 + B runtime·callbackasm1(SB) + MOVW $1530, R12 + B runtime·callbackasm1(SB) + MOVW $1531, R12 + B runtime·callbackasm1(SB) + MOVW $1532, R12 + B runtime·callbackasm1(SB) + MOVW $1533, R12 + B runtime·callbackasm1(SB) + MOVW $1534, R12 + B runtime·callbackasm1(SB) + MOVW $1535, R12 + B runtime·callbackasm1(SB) + MOVW $1536, R12 + B runtime·callbackasm1(SB) + MOVW $1537, R12 + B runtime·callbackasm1(SB) + MOVW $1538, R12 + B runtime·callbackasm1(SB) + MOVW $1539, R12 + B runtime·callbackasm1(SB) + MOVW $1540, R12 + B runtime·callbackasm1(SB) + MOVW $1541, R12 + B runtime·callbackasm1(SB) + MOVW $1542, R12 + B runtime·callbackasm1(SB) + MOVW $1543, R12 + B runtime·callbackasm1(SB) + MOVW $1544, R12 + B runtime·callbackasm1(SB) + MOVW $1545, R12 + B runtime·callbackasm1(SB) + MOVW $1546, R12 + B runtime·callbackasm1(SB) + MOVW $1547, R12 + B runtime·callbackasm1(SB) + MOVW $1548, R12 + B runtime·callbackasm1(SB) + MOVW $1549, R12 + B runtime·callbackasm1(SB) + MOVW $1550, R12 + B runtime·callbackasm1(SB) + MOVW $1551, R12 + B runtime·callbackasm1(SB) + MOVW $1552, R12 + B runtime·callbackasm1(SB) + MOVW $1553, R12 + B runtime·callbackasm1(SB) + MOVW $1554, R12 + B runtime·callbackasm1(SB) + MOVW $1555, R12 + B runtime·callbackasm1(SB) + MOVW $1556, R12 + B runtime·callbackasm1(SB) + MOVW $1557, R12 + B runtime·callbackasm1(SB) + MOVW $1558, R12 + B runtime·callbackasm1(SB) + MOVW $1559, R12 + B runtime·callbackasm1(SB) + MOVW $1560, R12 + B runtime·callbackasm1(SB) + MOVW $1561, R12 + B runtime·callbackasm1(SB) + MOVW $1562, R12 + B runtime·callbackasm1(SB) + MOVW $1563, R12 + B runtime·callbackasm1(SB) + MOVW $1564, R12 + B runtime·callbackasm1(SB) + MOVW $1565, R12 + B runtime·callbackasm1(SB) + MOVW $1566, R12 + B runtime·callbackasm1(SB) + MOVW $1567, R12 + B runtime·callbackasm1(SB) + MOVW $1568, R12 + B runtime·callbackasm1(SB) + MOVW $1569, R12 + B runtime·callbackasm1(SB) + MOVW $1570, R12 + B runtime·callbackasm1(SB) + MOVW $1571, R12 + B runtime·callbackasm1(SB) + MOVW $1572, R12 + B runtime·callbackasm1(SB) + MOVW $1573, R12 + B runtime·callbackasm1(SB) + MOVW $1574, R12 + B runtime·callbackasm1(SB) + MOVW $1575, R12 + B runtime·callbackasm1(SB) + MOVW $1576, R12 + B runtime·callbackasm1(SB) + MOVW $1577, R12 + B runtime·callbackasm1(SB) + MOVW $1578, R12 + B runtime·callbackasm1(SB) + MOVW $1579, R12 + B runtime·callbackasm1(SB) + MOVW $1580, R12 + B runtime·callbackasm1(SB) + MOVW $1581, R12 + B runtime·callbackasm1(SB) + MOVW $1582, R12 + B runtime·callbackasm1(SB) + MOVW $1583, R12 + B runtime·callbackasm1(SB) + MOVW $1584, R12 + B runtime·callbackasm1(SB) + MOVW $1585, R12 + B runtime·callbackasm1(SB) + MOVW $1586, R12 + B runtime·callbackasm1(SB) + MOVW $1587, R12 + B runtime·callbackasm1(SB) + MOVW $1588, R12 + B runtime·callbackasm1(SB) + MOVW $1589, R12 + B runtime·callbackasm1(SB) + MOVW $1590, R12 + B runtime·callbackasm1(SB) + MOVW $1591, R12 + B runtime·callbackasm1(SB) + MOVW $1592, R12 + B runtime·callbackasm1(SB) + MOVW $1593, R12 + B runtime·callbackasm1(SB) + MOVW $1594, R12 + B runtime·callbackasm1(SB) + MOVW $1595, R12 + B runtime·callbackasm1(SB) + MOVW $1596, R12 + B runtime·callbackasm1(SB) + MOVW $1597, R12 + B runtime·callbackasm1(SB) + MOVW $1598, R12 + B runtime·callbackasm1(SB) + MOVW $1599, R12 + B runtime·callbackasm1(SB) + MOVW $1600, R12 + B runtime·callbackasm1(SB) + MOVW $1601, R12 + B runtime·callbackasm1(SB) + MOVW $1602, R12 + B runtime·callbackasm1(SB) + MOVW $1603, R12 + B runtime·callbackasm1(SB) + MOVW $1604, R12 + B runtime·callbackasm1(SB) + MOVW $1605, R12 + B runtime·callbackasm1(SB) + MOVW $1606, R12 + B runtime·callbackasm1(SB) + MOVW $1607, R12 + B runtime·callbackasm1(SB) + MOVW $1608, R12 + B runtime·callbackasm1(SB) + MOVW $1609, R12 + B runtime·callbackasm1(SB) + MOVW $1610, R12 + B runtime·callbackasm1(SB) + MOVW $1611, R12 + B runtime·callbackasm1(SB) + MOVW $1612, R12 + B runtime·callbackasm1(SB) + MOVW $1613, R12 + B runtime·callbackasm1(SB) + MOVW $1614, R12 + B runtime·callbackasm1(SB) + MOVW $1615, R12 + B runtime·callbackasm1(SB) + MOVW $1616, R12 + B runtime·callbackasm1(SB) + MOVW $1617, R12 + B runtime·callbackasm1(SB) + MOVW $1618, R12 + B runtime·callbackasm1(SB) + MOVW $1619, R12 + B runtime·callbackasm1(SB) + MOVW $1620, R12 + B runtime·callbackasm1(SB) + MOVW $1621, R12 + B runtime·callbackasm1(SB) + MOVW $1622, R12 + B runtime·callbackasm1(SB) + MOVW $1623, R12 + B runtime·callbackasm1(SB) + MOVW $1624, R12 + B runtime·callbackasm1(SB) + MOVW $1625, R12 + B runtime·callbackasm1(SB) + MOVW $1626, R12 + B runtime·callbackasm1(SB) + MOVW $1627, R12 + B runtime·callbackasm1(SB) + MOVW $1628, R12 + B runtime·callbackasm1(SB) + MOVW $1629, R12 + B runtime·callbackasm1(SB) + MOVW $1630, R12 + B runtime·callbackasm1(SB) + MOVW $1631, R12 + B runtime·callbackasm1(SB) + MOVW $1632, R12 + B runtime·callbackasm1(SB) + MOVW $1633, R12 + B runtime·callbackasm1(SB) + MOVW $1634, R12 + B runtime·callbackasm1(SB) + MOVW $1635, R12 + B runtime·callbackasm1(SB) + MOVW $1636, R12 + B runtime·callbackasm1(SB) + MOVW $1637, R12 + B runtime·callbackasm1(SB) + MOVW $1638, R12 + B runtime·callbackasm1(SB) + MOVW $1639, R12 + B runtime·callbackasm1(SB) + MOVW $1640, R12 + B runtime·callbackasm1(SB) + MOVW $1641, R12 + B runtime·callbackasm1(SB) + MOVW $1642, R12 + B runtime·callbackasm1(SB) + MOVW $1643, R12 + B runtime·callbackasm1(SB) + MOVW $1644, R12 + B runtime·callbackasm1(SB) + MOVW $1645, R12 + B runtime·callbackasm1(SB) + MOVW $1646, R12 + B runtime·callbackasm1(SB) + MOVW $1647, R12 + B runtime·callbackasm1(SB) + MOVW $1648, R12 + B runtime·callbackasm1(SB) + MOVW $1649, R12 + B runtime·callbackasm1(SB) + MOVW $1650, R12 + B runtime·callbackasm1(SB) + MOVW $1651, R12 + B runtime·callbackasm1(SB) + MOVW $1652, R12 + B runtime·callbackasm1(SB) + MOVW $1653, R12 + B runtime·callbackasm1(SB) + MOVW $1654, R12 + B runtime·callbackasm1(SB) + MOVW $1655, R12 + B runtime·callbackasm1(SB) + MOVW $1656, R12 + B runtime·callbackasm1(SB) + MOVW $1657, R12 + B runtime·callbackasm1(SB) + MOVW $1658, R12 + B runtime·callbackasm1(SB) + MOVW $1659, R12 + B runtime·callbackasm1(SB) + MOVW $1660, R12 + B runtime·callbackasm1(SB) + MOVW $1661, R12 + B runtime·callbackasm1(SB) + MOVW $1662, R12 + B runtime·callbackasm1(SB) + MOVW $1663, R12 + B runtime·callbackasm1(SB) + MOVW $1664, R12 + B runtime·callbackasm1(SB) + MOVW $1665, R12 + B runtime·callbackasm1(SB) + MOVW $1666, R12 + B runtime·callbackasm1(SB) + MOVW $1667, R12 + B runtime·callbackasm1(SB) + MOVW $1668, R12 + B runtime·callbackasm1(SB) + MOVW $1669, R12 + B runtime·callbackasm1(SB) + MOVW $1670, R12 + B runtime·callbackasm1(SB) + MOVW $1671, R12 + B runtime·callbackasm1(SB) + MOVW $1672, R12 + B runtime·callbackasm1(SB) + MOVW $1673, R12 + B runtime·callbackasm1(SB) + MOVW $1674, R12 + B runtime·callbackasm1(SB) + MOVW $1675, R12 + B runtime·callbackasm1(SB) + MOVW $1676, R12 + B runtime·callbackasm1(SB) + MOVW $1677, R12 + B runtime·callbackasm1(SB) + MOVW $1678, R12 + B runtime·callbackasm1(SB) + MOVW $1679, R12 + B runtime·callbackasm1(SB) + MOVW $1680, R12 + B runtime·callbackasm1(SB) + MOVW $1681, R12 + B runtime·callbackasm1(SB) + MOVW $1682, R12 + B runtime·callbackasm1(SB) + MOVW $1683, R12 + B runtime·callbackasm1(SB) + MOVW $1684, R12 + B runtime·callbackasm1(SB) + MOVW $1685, R12 + B runtime·callbackasm1(SB) + MOVW $1686, R12 + B runtime·callbackasm1(SB) + MOVW $1687, R12 + B runtime·callbackasm1(SB) + MOVW $1688, R12 + B runtime·callbackasm1(SB) + MOVW $1689, R12 + B runtime·callbackasm1(SB) + MOVW $1690, R12 + B runtime·callbackasm1(SB) + MOVW $1691, R12 + B runtime·callbackasm1(SB) + MOVW $1692, R12 + B runtime·callbackasm1(SB) + MOVW $1693, R12 + B runtime·callbackasm1(SB) + MOVW $1694, R12 + B runtime·callbackasm1(SB) + MOVW $1695, R12 + B runtime·callbackasm1(SB) + MOVW $1696, R12 + B runtime·callbackasm1(SB) + MOVW $1697, R12 + B runtime·callbackasm1(SB) + MOVW $1698, R12 + B runtime·callbackasm1(SB) + MOVW $1699, R12 + B runtime·callbackasm1(SB) + MOVW $1700, R12 + B runtime·callbackasm1(SB) + MOVW $1701, R12 + B runtime·callbackasm1(SB) + MOVW $1702, R12 + B runtime·callbackasm1(SB) + MOVW $1703, R12 + B runtime·callbackasm1(SB) + MOVW $1704, R12 + B runtime·callbackasm1(SB) + MOVW $1705, R12 + B runtime·callbackasm1(SB) + MOVW $1706, R12 + B runtime·callbackasm1(SB) + MOVW $1707, R12 + B runtime·callbackasm1(SB) + MOVW $1708, R12 + B runtime·callbackasm1(SB) + MOVW $1709, R12 + B runtime·callbackasm1(SB) + MOVW $1710, R12 + B runtime·callbackasm1(SB) + MOVW $1711, R12 + B runtime·callbackasm1(SB) + MOVW $1712, R12 + B runtime·callbackasm1(SB) + MOVW $1713, R12 + B runtime·callbackasm1(SB) + MOVW $1714, R12 + B runtime·callbackasm1(SB) + MOVW $1715, R12 + B runtime·callbackasm1(SB) + MOVW $1716, R12 + B runtime·callbackasm1(SB) + MOVW $1717, R12 + B runtime·callbackasm1(SB) + MOVW $1718, R12 + B runtime·callbackasm1(SB) + MOVW $1719, R12 + B runtime·callbackasm1(SB) + MOVW $1720, R12 + B runtime·callbackasm1(SB) + MOVW $1721, R12 + B runtime·callbackasm1(SB) + MOVW $1722, R12 + B runtime·callbackasm1(SB) + MOVW $1723, R12 + B runtime·callbackasm1(SB) + MOVW $1724, R12 + B runtime·callbackasm1(SB) + MOVW $1725, R12 + B runtime·callbackasm1(SB) + MOVW $1726, R12 + B runtime·callbackasm1(SB) + MOVW $1727, R12 + B runtime·callbackasm1(SB) + MOVW $1728, R12 + B runtime·callbackasm1(SB) + MOVW $1729, R12 + B runtime·callbackasm1(SB) + MOVW $1730, R12 + B runtime·callbackasm1(SB) + MOVW $1731, R12 + B runtime·callbackasm1(SB) + MOVW $1732, R12 + B runtime·callbackasm1(SB) + MOVW $1733, R12 + B runtime·callbackasm1(SB) + MOVW $1734, R12 + B runtime·callbackasm1(SB) + MOVW $1735, R12 + B runtime·callbackasm1(SB) + MOVW $1736, R12 + B runtime·callbackasm1(SB) + MOVW $1737, R12 + B runtime·callbackasm1(SB) + MOVW $1738, R12 + B runtime·callbackasm1(SB) + MOVW $1739, R12 + B runtime·callbackasm1(SB) + MOVW $1740, R12 + B runtime·callbackasm1(SB) + MOVW $1741, R12 + B runtime·callbackasm1(SB) + MOVW $1742, R12 + B runtime·callbackasm1(SB) + MOVW $1743, R12 + B runtime·callbackasm1(SB) + MOVW $1744, R12 + B runtime·callbackasm1(SB) + MOVW $1745, R12 + B runtime·callbackasm1(SB) + MOVW $1746, R12 + B runtime·callbackasm1(SB) + MOVW $1747, R12 + B runtime·callbackasm1(SB) + MOVW $1748, R12 + B runtime·callbackasm1(SB) + MOVW $1749, R12 + B runtime·callbackasm1(SB) + MOVW $1750, R12 + B runtime·callbackasm1(SB) + MOVW $1751, R12 + B runtime·callbackasm1(SB) + MOVW $1752, R12 + B runtime·callbackasm1(SB) + MOVW $1753, R12 + B runtime·callbackasm1(SB) + MOVW $1754, R12 + B runtime·callbackasm1(SB) + MOVW $1755, R12 + B runtime·callbackasm1(SB) + MOVW $1756, R12 + B runtime·callbackasm1(SB) + MOVW $1757, R12 + B runtime·callbackasm1(SB) + MOVW $1758, R12 + B runtime·callbackasm1(SB) + MOVW $1759, R12 + B runtime·callbackasm1(SB) + MOVW $1760, R12 + B runtime·callbackasm1(SB) + MOVW $1761, R12 + B runtime·callbackasm1(SB) + MOVW $1762, R12 + B runtime·callbackasm1(SB) + MOVW $1763, R12 + B runtime·callbackasm1(SB) + MOVW $1764, R12 + B runtime·callbackasm1(SB) + MOVW $1765, R12 + B runtime·callbackasm1(SB) + MOVW $1766, R12 + B runtime·callbackasm1(SB) + MOVW $1767, R12 + B runtime·callbackasm1(SB) + MOVW $1768, R12 + B runtime·callbackasm1(SB) + MOVW $1769, R12 + B runtime·callbackasm1(SB) + MOVW $1770, R12 + B runtime·callbackasm1(SB) + MOVW $1771, R12 + B runtime·callbackasm1(SB) + MOVW $1772, R12 + B runtime·callbackasm1(SB) + MOVW $1773, R12 + B runtime·callbackasm1(SB) + MOVW $1774, R12 + B runtime·callbackasm1(SB) + MOVW $1775, R12 + B runtime·callbackasm1(SB) + MOVW $1776, R12 + B runtime·callbackasm1(SB) + MOVW $1777, R12 + B runtime·callbackasm1(SB) + MOVW $1778, R12 + B runtime·callbackasm1(SB) + MOVW $1779, R12 + B runtime·callbackasm1(SB) + MOVW $1780, R12 + B runtime·callbackasm1(SB) + MOVW $1781, R12 + B runtime·callbackasm1(SB) + MOVW $1782, R12 + B runtime·callbackasm1(SB) + MOVW $1783, R12 + B runtime·callbackasm1(SB) + MOVW $1784, R12 + B runtime·callbackasm1(SB) + MOVW $1785, R12 + B runtime·callbackasm1(SB) + MOVW $1786, R12 + B runtime·callbackasm1(SB) + MOVW $1787, R12 + B runtime·callbackasm1(SB) + MOVW $1788, R12 + B runtime·callbackasm1(SB) + MOVW $1789, R12 + B runtime·callbackasm1(SB) + MOVW $1790, R12 + B runtime·callbackasm1(SB) + MOVW $1791, R12 + B runtime·callbackasm1(SB) + MOVW $1792, R12 + B runtime·callbackasm1(SB) + MOVW $1793, R12 + B runtime·callbackasm1(SB) + MOVW $1794, R12 + B runtime·callbackasm1(SB) + MOVW $1795, R12 + B runtime·callbackasm1(SB) + MOVW $1796, R12 + B runtime·callbackasm1(SB) + MOVW $1797, R12 + B runtime·callbackasm1(SB) + MOVW $1798, R12 + B runtime·callbackasm1(SB) + MOVW $1799, R12 + B runtime·callbackasm1(SB) + MOVW $1800, R12 + B runtime·callbackasm1(SB) + MOVW $1801, R12 + B runtime·callbackasm1(SB) + MOVW $1802, R12 + B runtime·callbackasm1(SB) + MOVW $1803, R12 + B runtime·callbackasm1(SB) + MOVW $1804, R12 + B runtime·callbackasm1(SB) + MOVW $1805, R12 + B runtime·callbackasm1(SB) + MOVW $1806, R12 + B runtime·callbackasm1(SB) + MOVW $1807, R12 + B runtime·callbackasm1(SB) + MOVW $1808, R12 + B runtime·callbackasm1(SB) + MOVW $1809, R12 + B runtime·callbackasm1(SB) + MOVW $1810, R12 + B runtime·callbackasm1(SB) + MOVW $1811, R12 + B runtime·callbackasm1(SB) + MOVW $1812, R12 + B runtime·callbackasm1(SB) + MOVW $1813, R12 + B runtime·callbackasm1(SB) + MOVW $1814, R12 + B runtime·callbackasm1(SB) + MOVW $1815, R12 + B runtime·callbackasm1(SB) + MOVW $1816, R12 + B runtime·callbackasm1(SB) + MOVW $1817, R12 + B runtime·callbackasm1(SB) + MOVW $1818, R12 + B runtime·callbackasm1(SB) + MOVW $1819, R12 + B runtime·callbackasm1(SB) + MOVW $1820, R12 + B runtime·callbackasm1(SB) + MOVW $1821, R12 + B runtime·callbackasm1(SB) + MOVW $1822, R12 + B runtime·callbackasm1(SB) + MOVW $1823, R12 + B runtime·callbackasm1(SB) + MOVW $1824, R12 + B runtime·callbackasm1(SB) + MOVW $1825, R12 + B runtime·callbackasm1(SB) + MOVW $1826, R12 + B runtime·callbackasm1(SB) + MOVW $1827, R12 + B runtime·callbackasm1(SB) + MOVW $1828, R12 + B runtime·callbackasm1(SB) + MOVW $1829, R12 + B runtime·callbackasm1(SB) + MOVW $1830, R12 + B runtime·callbackasm1(SB) + MOVW $1831, R12 + B runtime·callbackasm1(SB) + MOVW $1832, R12 + B runtime·callbackasm1(SB) + MOVW $1833, R12 + B runtime·callbackasm1(SB) + MOVW $1834, R12 + B runtime·callbackasm1(SB) + MOVW $1835, R12 + B runtime·callbackasm1(SB) + MOVW $1836, R12 + B runtime·callbackasm1(SB) + MOVW $1837, R12 + B runtime·callbackasm1(SB) + MOVW $1838, R12 + B runtime·callbackasm1(SB) + MOVW $1839, R12 + B runtime·callbackasm1(SB) + MOVW $1840, R12 + B runtime·callbackasm1(SB) + MOVW $1841, R12 + B runtime·callbackasm1(SB) + MOVW $1842, R12 + B runtime·callbackasm1(SB) + MOVW $1843, R12 + B runtime·callbackasm1(SB) + MOVW $1844, R12 + B runtime·callbackasm1(SB) + MOVW $1845, R12 + B runtime·callbackasm1(SB) + MOVW $1846, R12 + B runtime·callbackasm1(SB) + MOVW $1847, R12 + B runtime·callbackasm1(SB) + MOVW $1848, R12 + B runtime·callbackasm1(SB) + MOVW $1849, R12 + B runtime·callbackasm1(SB) + MOVW $1850, R12 + B runtime·callbackasm1(SB) + MOVW $1851, R12 + B runtime·callbackasm1(SB) + MOVW $1852, R12 + B runtime·callbackasm1(SB) + MOVW $1853, R12 + B runtime·callbackasm1(SB) + MOVW $1854, R12 + B runtime·callbackasm1(SB) + MOVW $1855, R12 + B runtime·callbackasm1(SB) + MOVW $1856, R12 + B runtime·callbackasm1(SB) + MOVW $1857, R12 + B runtime·callbackasm1(SB) + MOVW $1858, R12 + B runtime·callbackasm1(SB) + MOVW $1859, R12 + B runtime·callbackasm1(SB) + MOVW $1860, R12 + B runtime·callbackasm1(SB) + MOVW $1861, R12 + B runtime·callbackasm1(SB) + MOVW $1862, R12 + B runtime·callbackasm1(SB) + MOVW $1863, R12 + B runtime·callbackasm1(SB) + MOVW $1864, R12 + B runtime·callbackasm1(SB) + MOVW $1865, R12 + B runtime·callbackasm1(SB) + MOVW $1866, R12 + B runtime·callbackasm1(SB) + MOVW $1867, R12 + B runtime·callbackasm1(SB) + MOVW $1868, R12 + B runtime·callbackasm1(SB) + MOVW $1869, R12 + B runtime·callbackasm1(SB) + MOVW $1870, R12 + B runtime·callbackasm1(SB) + MOVW $1871, R12 + B runtime·callbackasm1(SB) + MOVW $1872, R12 + B runtime·callbackasm1(SB) + MOVW $1873, R12 + B runtime·callbackasm1(SB) + MOVW $1874, R12 + B runtime·callbackasm1(SB) + MOVW $1875, R12 + B runtime·callbackasm1(SB) + MOVW $1876, R12 + B runtime·callbackasm1(SB) + MOVW $1877, R12 + B runtime·callbackasm1(SB) + MOVW $1878, R12 + B runtime·callbackasm1(SB) + MOVW $1879, R12 + B runtime·callbackasm1(SB) + MOVW $1880, R12 + B runtime·callbackasm1(SB) + MOVW $1881, R12 + B runtime·callbackasm1(SB) + MOVW $1882, R12 + B runtime·callbackasm1(SB) + MOVW $1883, R12 + B runtime·callbackasm1(SB) + MOVW $1884, R12 + B runtime·callbackasm1(SB) + MOVW $1885, R12 + B runtime·callbackasm1(SB) + MOVW $1886, R12 + B runtime·callbackasm1(SB) + MOVW $1887, R12 + B runtime·callbackasm1(SB) + MOVW $1888, R12 + B runtime·callbackasm1(SB) + MOVW $1889, R12 + B runtime·callbackasm1(SB) + MOVW $1890, R12 + B runtime·callbackasm1(SB) + MOVW $1891, R12 + B runtime·callbackasm1(SB) + MOVW $1892, R12 + B runtime·callbackasm1(SB) + MOVW $1893, R12 + B runtime·callbackasm1(SB) + MOVW $1894, R12 + B runtime·callbackasm1(SB) + MOVW $1895, R12 + B runtime·callbackasm1(SB) + MOVW $1896, R12 + B runtime·callbackasm1(SB) + MOVW $1897, R12 + B runtime·callbackasm1(SB) + MOVW $1898, R12 + B runtime·callbackasm1(SB) + MOVW $1899, R12 + B runtime·callbackasm1(SB) + MOVW $1900, R12 + B runtime·callbackasm1(SB) + MOVW $1901, R12 + B runtime·callbackasm1(SB) + MOVW $1902, R12 + B runtime·callbackasm1(SB) + MOVW $1903, R12 + B runtime·callbackasm1(SB) + MOVW $1904, R12 + B runtime·callbackasm1(SB) + MOVW $1905, R12 + B runtime·callbackasm1(SB) + MOVW $1906, R12 + B runtime·callbackasm1(SB) + MOVW $1907, R12 + B runtime·callbackasm1(SB) + MOVW $1908, R12 + B runtime·callbackasm1(SB) + MOVW $1909, R12 + B runtime·callbackasm1(SB) + MOVW $1910, R12 + B runtime·callbackasm1(SB) + MOVW $1911, R12 + B runtime·callbackasm1(SB) + MOVW $1912, R12 + B runtime·callbackasm1(SB) + MOVW $1913, R12 + B runtime·callbackasm1(SB) + MOVW $1914, R12 + B runtime·callbackasm1(SB) + MOVW $1915, R12 + B runtime·callbackasm1(SB) + MOVW $1916, R12 + B runtime·callbackasm1(SB) + MOVW $1917, R12 + B runtime·callbackasm1(SB) + MOVW $1918, R12 + B runtime·callbackasm1(SB) + MOVW $1919, R12 + B runtime·callbackasm1(SB) + MOVW $1920, R12 + B runtime·callbackasm1(SB) + MOVW $1921, R12 + B runtime·callbackasm1(SB) + MOVW $1922, R12 + B runtime·callbackasm1(SB) + MOVW $1923, R12 + B runtime·callbackasm1(SB) + MOVW $1924, R12 + B runtime·callbackasm1(SB) + MOVW $1925, R12 + B runtime·callbackasm1(SB) + MOVW $1926, R12 + B runtime·callbackasm1(SB) + MOVW $1927, R12 + B runtime·callbackasm1(SB) + MOVW $1928, R12 + B runtime·callbackasm1(SB) + MOVW $1929, R12 + B runtime·callbackasm1(SB) + MOVW $1930, R12 + B runtime·callbackasm1(SB) + MOVW $1931, R12 + B runtime·callbackasm1(SB) + MOVW $1932, R12 + B runtime·callbackasm1(SB) + MOVW $1933, R12 + B runtime·callbackasm1(SB) + MOVW $1934, R12 + B runtime·callbackasm1(SB) + MOVW $1935, R12 + B runtime·callbackasm1(SB) + MOVW $1936, R12 + B runtime·callbackasm1(SB) + MOVW $1937, R12 + B runtime·callbackasm1(SB) + MOVW $1938, R12 + B runtime·callbackasm1(SB) + MOVW $1939, R12 + B runtime·callbackasm1(SB) + MOVW $1940, R12 + B runtime·callbackasm1(SB) + MOVW $1941, R12 + B runtime·callbackasm1(SB) + MOVW $1942, R12 + B runtime·callbackasm1(SB) + MOVW $1943, R12 + B runtime·callbackasm1(SB) + MOVW $1944, R12 + B runtime·callbackasm1(SB) + MOVW $1945, R12 + B runtime·callbackasm1(SB) + MOVW $1946, R12 + B runtime·callbackasm1(SB) + MOVW $1947, R12 + B runtime·callbackasm1(SB) + MOVW $1948, R12 + B runtime·callbackasm1(SB) + MOVW $1949, R12 + B runtime·callbackasm1(SB) + MOVW $1950, R12 + B runtime·callbackasm1(SB) + MOVW $1951, R12 + B runtime·callbackasm1(SB) + MOVW $1952, R12 + B runtime·callbackasm1(SB) + MOVW $1953, R12 + B runtime·callbackasm1(SB) + MOVW $1954, R12 + B runtime·callbackasm1(SB) + MOVW $1955, R12 + B runtime·callbackasm1(SB) + MOVW $1956, R12 + B runtime·callbackasm1(SB) + MOVW $1957, R12 + B runtime·callbackasm1(SB) + MOVW $1958, R12 + B runtime·callbackasm1(SB) + MOVW $1959, R12 + B runtime·callbackasm1(SB) + MOVW $1960, R12 + B runtime·callbackasm1(SB) + MOVW $1961, R12 + B runtime·callbackasm1(SB) + MOVW $1962, R12 + B runtime·callbackasm1(SB) + MOVW $1963, R12 + B runtime·callbackasm1(SB) + MOVW $1964, R12 + B runtime·callbackasm1(SB) + MOVW $1965, R12 + B runtime·callbackasm1(SB) + MOVW $1966, R12 + B runtime·callbackasm1(SB) + MOVW $1967, R12 + B runtime·callbackasm1(SB) + MOVW $1968, R12 + B runtime·callbackasm1(SB) + MOVW $1969, R12 + B runtime·callbackasm1(SB) + MOVW $1970, R12 + B runtime·callbackasm1(SB) + MOVW $1971, R12 + B runtime·callbackasm1(SB) + MOVW $1972, R12 + B runtime·callbackasm1(SB) + MOVW $1973, R12 + B runtime·callbackasm1(SB) + MOVW $1974, R12 + B runtime·callbackasm1(SB) + MOVW $1975, R12 + B runtime·callbackasm1(SB) + MOVW $1976, R12 + B runtime·callbackasm1(SB) + MOVW $1977, R12 + B runtime·callbackasm1(SB) + MOVW $1978, R12 + B runtime·callbackasm1(SB) + MOVW $1979, R12 + B runtime·callbackasm1(SB) + MOVW $1980, R12 + B runtime·callbackasm1(SB) + MOVW $1981, R12 + B runtime·callbackasm1(SB) + MOVW $1982, R12 + B runtime·callbackasm1(SB) + MOVW $1983, R12 + B runtime·callbackasm1(SB) + MOVW $1984, R12 + B runtime·callbackasm1(SB) + MOVW $1985, R12 + B runtime·callbackasm1(SB) + MOVW $1986, R12 + B runtime·callbackasm1(SB) + MOVW $1987, R12 + B runtime·callbackasm1(SB) + MOVW $1988, R12 + B runtime·callbackasm1(SB) + MOVW $1989, R12 + B runtime·callbackasm1(SB) + MOVW $1990, R12 + B runtime·callbackasm1(SB) + MOVW $1991, R12 + B runtime·callbackasm1(SB) + MOVW $1992, R12 + B runtime·callbackasm1(SB) + MOVW $1993, R12 + B runtime·callbackasm1(SB) + MOVW $1994, R12 + B runtime·callbackasm1(SB) + MOVW $1995, R12 + B runtime·callbackasm1(SB) + MOVW $1996, R12 + B runtime·callbackasm1(SB) + MOVW $1997, R12 + B runtime·callbackasm1(SB) + MOVW $1998, R12 + B runtime·callbackasm1(SB) + MOVW $1999, R12 + B runtime·callbackasm1(SB) -- cgit v1.3-5-g9baa From c7ac645d28b99f163c46e2b8622ca7b60dc212ce Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Thu, 20 Sep 2018 11:43:42 +0200 Subject: runtime: fix reference to sys{Fault,Free,Reserve,Unused,Used} in comments Change-Id: Icbaedc49c810c63c51d56ae394d2f70e4d64b3e0 Reviewed-on: https://go-review.googlesource.com/136495 Run-TryBot: Tobias Klauser TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/malloc.go | 18 +++++++++--------- src/runtime/mgcsweep.go | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 07e0a67240..c6c969a3bf 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -328,27 +328,27 @@ var physPageSize uintptr // may use larger alignment, so the caller must be careful to realign the // memory obtained by sysAlloc. // -// SysUnused notifies the operating system that the contents +// sysUnused notifies the operating system that the contents // of the memory region are no longer needed and can be reused // for other purposes. -// SysUsed notifies the operating system that the contents +// sysUsed notifies the operating system that the contents // of the memory region are needed again. // -// SysFree returns it unconditionally; this is only used if +// sysFree returns it unconditionally; this is only used if // an out-of-memory error has been detected midway through -// an allocation. It is okay if SysFree is a no-op. +// an allocation. It is okay if sysFree is a no-op. // -// SysReserve reserves address space without allocating memory. +// sysReserve reserves address space without allocating memory. // If the pointer passed to it is non-nil, the caller wants the -// reservation there, but SysReserve can still choose another +// reservation there, but sysReserve can still choose another // location if that one is unavailable. -// NOTE: SysReserve returns OS-aligned memory, but the heap allocator +// NOTE: sysReserve returns OS-aligned memory, but the heap allocator // may use larger alignment, so the caller must be careful to realign the // memory obtained by sysAlloc. // -// SysMap maps previously reserved address space for use. +// sysMap maps previously reserved address space for use. // -// SysFault marks a (already sysAlloc'd) region to fault +// sysFault marks a (already sysAlloc'd) region to fault // if accessed. Used only for debugging the runtime. func mallocinit() { diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index c7baa455fe..71f5c4b3a9 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -339,18 +339,18 @@ func (s *mspan) sweep(preserve bool) bool { // Free large span to heap // NOTE(rsc,dvyukov): The original implementation of efence - // in CL 22060046 used SysFree instead of SysFault, so that + // in CL 22060046 used sysFree instead of sysFault, so that // the operating system would eventually give the memory // back to us again, so that an efence program could run // longer without running out of memory. Unfortunately, - // calling SysFree here without any kind of adjustment of the + // calling sysFree here without any kind of adjustment of the // heap data structures means that when the memory does // come back to us, we have the wrong metadata for it, either in // the MSpan structures or in the garbage collection bitmap. - // Using SysFault here means that the program will run out of + // Using sysFault here means that the program will run out of // memory fairly quickly in efence mode, but at least it won't // have mysterious crashes due to confused memory reuse. - // It should be possible to switch back to SysFree if we also + // It should be possible to switch back to sysFree if we also // implement and then call some kind of MHeap_DeleteSpan. if debug.efence > 0 { s.limit = 0 // prevent mlookup from finding this span -- cgit v1.3-5-g9baa From e945623930d2c85b9a81476203451ca8f4092875 Mon Sep 17 00:00:00 2001 From: Lynn Boger Date: Tue, 11 Sep 2018 11:42:15 -0400 Subject: runtime: improve CALLFN macro for ppc64x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous CALLFN macro was copying a single byte at a time which is extremely inefficient on ppc64x. This changes the macro so it copies 8 bytes at a time. benchmark in reflect: name old time/op new time/op delta Call-8 177ns ± 0% 165ns ± 0% -6.78% (p=1.000 n=1+1) CallArgCopy/size=128-8 194ns ± 0% 140ns ± 0% -27.84% (p=1.000 n=1+1) CallArgCopy/size=256-8 253ns ± 0% 159ns ± 0% -37.15% (p=1.000 n=1+1) CallArgCopy/size=1024-8 612ns ± 0% 222ns ± 0% -63.73% (p=1.000 n=1+1) CallArgCopy/size=4096-8 2.14µs ± 0% 0.53µs ± 0% -75.01% (p=1.000 n=1+1) CallArgCopy/size=65536-8 33.0µs ± 0% 7.3µs ± 0% -77.72% (p=1.000 n=1+1) Change-Id: I71f6ee788264e61bb072264d21b77b83592c9dca Reviewed-on: https://go-review.googlesource.com/134635 Run-TryBot: Lynn Boger TryBot-Result: Gobot Gobot Reviewed-by: Carlos Eduardo Seo Reviewed-by: Michael Munday --- src/runtime/asm_ppc64x.s | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/asm_ppc64x.s b/src/runtime/asm_ppc64x.s index 57877c0194..b180cb06ab 100644 --- a/src/runtime/asm_ppc64x.s +++ b/src/runtime/asm_ppc64x.s @@ -390,15 +390,36 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \ /* copy arguments to stack */ \ MOVD arg+16(FP), R3; \ MOVWZ argsize+24(FP), R4; \ - MOVD R1, R5; \ - ADD $(FIXED_FRAME-1), R5; \ - SUB $1, R3; \ - ADD R5, R4; \ - CMP R5, R4; \ - BEQ 4(PC); \ - MOVBZU 1(R3), R6; \ - MOVBZU R6, 1(R5); \ - BR -4(PC); \ + MOVD R1, R5; \ + CMP R4, $8; \ + BLT tailsetup; \ + /* copy 8 at a time if possible */ \ + ADD $(FIXED_FRAME-8), R5; \ + SUB $8, R3; \ +top: \ + MOVDU 8(R3), R7; \ + MOVDU R7, 8(R5); \ + SUB $8, R4; \ + CMP R4, $8; \ + BGE top; \ + /* handle remaining bytes */ \ + CMP $0, R4; \ + BEQ callfn; \ + ADD $7, R3; \ + ADD $7, R5; \ + BR tail; \ +tailsetup: \ + CMP $0, R4; \ + BEQ callfn; \ + ADD $(FIXED_FRAME-1), R5; \ + SUB $1, R3; \ +tail: \ + MOVBU 1(R3), R6; \ + MOVBU R6, 1(R5); \ + SUB $1, R4; \ + CMP $0, R4; \ + BGT tail; \ +callfn: \ /* call function */ \ MOVD f+8(FP), R11; \ MOVD (R11), R12; \ -- cgit v1.3-5-g9baa From 5a8c11ce3e7a87485defafb78c7bcf14f9d7b5a2 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 26 Sep 2018 16:39:02 -0400 Subject: runtime: rename _MSpan* constants to mSpan* We already aliased mSpanInUse to _MSpanInUse. The dual constants are getting annoying, so fix all of these to use the mSpan* naming convention. This was done automatically with: sed -i -re 's/_?MSpan(Dead|InUse|Manual|Free)/mSpan\1/g' *.go plus deleting the existing definition of mSpanInUse. Change-Id: I09979d9d491d06c10689cea625dc57faa9cc6767 Reviewed-on: https://go-review.googlesource.com/137875 Run-TryBot: Austin Clements Reviewed-by: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/runtime/cgocheck.go | 2 +- src/runtime/heapdump.go | 8 +++--- src/runtime/malloc.go | 2 -- src/runtime/mbitmap.go | 4 +-- src/runtime/mgcmark.go | 6 ++-- src/runtime/mgcsweep.go | 2 +- src/runtime/mheap.go | 64 ++++++++++++++++++++--------------------- src/runtime/mstats.go | 2 +- src/runtime/runtime-gdb_test.go | 2 +- src/runtime/stack.go | 4 +-- 10 files changed, 47 insertions(+), 49 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/cgocheck.go b/src/runtime/cgocheck.go index 73cb6ecae2..ac57e0344e 100644 --- a/src/runtime/cgocheck.go +++ b/src/runtime/cgocheck.go @@ -126,7 +126,7 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) { } s := spanOfUnchecked(uintptr(src)) - if s.state == _MSpanManual { + if s.state == mSpanManual { // There are no heap bits for value stored on the stack. // For a channel receive src might be on the stack of some // other goroutine, so we can't unwind the stack even if diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index 0fc02a8e80..e2c6f18714 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -430,7 +430,7 @@ func dumproots() { // MSpan.types for _, s := range mheap_.allspans { - if s.state == _MSpanInUse { + if s.state == mSpanInUse { // Finalizers for sp := s.specials; sp != nil; sp = sp.next { if sp.kind != _KindSpecialFinalizer { @@ -453,7 +453,7 @@ var freemark [_PageSize / 8]bool func dumpobjs() { for _, s := range mheap_.allspans { - if s.state != _MSpanInUse { + if s.state != mSpanInUse { continue } p := s.base() @@ -616,7 +616,7 @@ func dumpmemprof_callback(b *bucket, nstk uintptr, pstk *uintptr, size, allocs, func dumpmemprof() { iterate_memprof(dumpmemprof_callback) for _, s := range mheap_.allspans { - if s.state != _MSpanInUse { + if s.state != mSpanInUse { continue } for sp := s.specials; sp != nil; sp = sp.next { @@ -637,7 +637,7 @@ var dumphdr = []byte("go1.7 heap dump\n") func mdump() { // make sure we're done sweeping for _, s := range mheap_.allspans { - if s.state == _MSpanInUse { + if s.state == mSpanInUse { s.ensureSwept() } } diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index c6c969a3bf..dd88d353dd 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -124,8 +124,6 @@ const ( // have the most objects per span. maxObjsPerSpan = pageSize / 8 - mSpanInUse = _MSpanInUse - concurrentSweep = _ConcurrentSweep _PageSize = 1 << _PageShift diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index e217e7695f..553d83f7f2 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -365,7 +365,7 @@ func findObject(p, refBase, refOff uintptr) (base uintptr, s *mspan, objIndex ui s = spanOf(p) // If p is a bad pointer, it may not be in s's bounds. if s == nil || p < s.base() || p >= s.limit || s.state != mSpanInUse { - if s == nil || s.state == _MSpanManual { + if s == nil || s.state == mSpanManual { // If s is nil, the virtual address has never been part of the heap. // This pointer may be to some mmap'd region, so we allow it. // Pointers into stacks are also ok, the runtime manages these explicitly. @@ -611,7 +611,7 @@ func bulkBarrierPreWrite(dst, src, size uintptr) { } } return - } else if s.state != _MSpanInUse || dst < s.base() || s.limit <= dst { + } else if s.state != mSpanInUse || dst < s.base() || s.limit <= dst { // dst was heap memory at some point, but isn't now. // It can't be a global. It must be either our stack, // or in the case of direct channel sends, it could be diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 69ff895512..f713841cfa 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1239,7 +1239,7 @@ func gcDumpObject(label string, obj, off uintptr) { skipped := false size := s.elemsize - if s.state == _MSpanManual && size == 0 { + if s.state == mSpanManual && size == 0 { // We're printing something from a stack frame. We // don't know how big it is, so just show up to an // including off. @@ -1335,7 +1335,7 @@ var useCheckmark = false func initCheckmarks() { useCheckmark = true for _, s := range mheap_.allspans { - if s.state == _MSpanInUse { + if s.state == mSpanInUse { heapBitsForAddr(s.base()).initCheckmarkSpan(s.layout()) } } @@ -1344,7 +1344,7 @@ func initCheckmarks() { func clearCheckmarks() { useCheckmark = false for _, s := range mheap_.allspans { - if s.state == _MSpanInUse { + if s.state == mSpanInUse { heapBitsForAddr(s.base()).clearCheckmarkSpan(s.layout()) } } diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index 71f5c4b3a9..ecfdee59f4 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -159,7 +159,7 @@ func (s *mspan) ensureSwept() { if atomic.Load(&s.sweepgen) == sg { return } - // The caller must be sure that the span is a MSpanInUse span. + // The caller must be sure that the span is a mSpanInUse span. if atomic.Cas(&s.sweepgen, sg-2, sg-1) { s.sweep(false) return diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 00ecfa2d66..65d6b0c7d4 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -82,7 +82,7 @@ type mheap struct { // accounting for current progress. If we could only adjust // the slope, it would create a discontinuity in debt if any // progress has already been made. - pagesInUse uint64 // pages of spans in stats _MSpanInUse; R/W with mheap.lock + pagesInUse uint64 // pages of spans in stats mSpanInUse; R/W with mheap.lock pagesSwept uint64 // pages swept this cycle; updated atomically pagesSweptBasis uint64 // pagesSwept to use as the origin of the sweep ratio; updated atomically sweepHeapLiveBasis uint64 // value of heap_live to use as the origin of sweep ratio; written with lock, read without @@ -199,18 +199,18 @@ type arenaHint struct { // An MSpan is a run of pages. // -// When a MSpan is in the heap free list, state == MSpanFree +// When a MSpan is in the heap free list, state == mSpanFree // and heapmap(s->start) == span, heapmap(s->start+s->npages-1) == span. // -// When a MSpan is allocated, state == MSpanInUse or MSpanManual +// When a MSpan is allocated, state == mSpanInUse or mSpanManual // and heapmap(i) == span for all s->start <= i < s->start+s->npages. // Every MSpan is in one doubly-linked list, // either one of the MHeap's free lists or one of the // MCentral's span lists. -// An MSpan representing actual memory has state _MSpanInUse, -// _MSpanManual, or _MSpanFree. Transitions between these states are +// An MSpan representing actual memory has state mSpanInUse, +// mSpanManual, or mSpanFree. Transitions between these states are // constrained as follows: // // * A span may transition from free to in-use or manual during any GC @@ -226,19 +226,19 @@ type arenaHint struct { type mSpanState uint8 const ( - _MSpanDead mSpanState = iota - _MSpanInUse // allocated for garbage collected heap - _MSpanManual // allocated for manual management (e.g., stack allocator) - _MSpanFree + mSpanDead mSpanState = iota + mSpanInUse // allocated for garbage collected heap + mSpanManual // allocated for manual management (e.g., stack allocator) + mSpanFree ) // mSpanStateNames are the names of the span states, indexed by // mSpanState. var mSpanStateNames = []string{ - "_MSpanDead", - "_MSpanInUse", - "_MSpanManual", - "_MSpanFree", + "mSpanDead", + "mSpanInUse", + "mSpanManual", + "mSpanFree", } // mSpanList heads a linked list of spans. @@ -258,7 +258,7 @@ type mspan struct { startAddr uintptr // address of first byte of span aka s.base() npages uintptr // number of pages in span - manualFreeList gclinkptr // list of free objects in _MSpanManual spans + manualFreeList gclinkptr // list of free objects in mSpanManual spans // freeindex is the slot index between 0 and nelems at which to begin scanning // for the next free object in this span. @@ -458,7 +458,7 @@ func (i arenaIdx) l2() uint { } // inheap reports whether b is a pointer into a (potentially dead) heap object. -// It returns false for pointers into _MSpanManual spans. +// It returns false for pointers into mSpanManual spans. // Non-preemptible because it is used by write barriers. //go:nowritebarrier //go:nosplit @@ -477,7 +477,7 @@ func inHeapOrStack(b uintptr) bool { return false } switch s.state { - case mSpanInUse, _MSpanManual: + case mSpanInUse, mSpanManual: return b < s.limit default: return false @@ -696,7 +696,7 @@ func (h *mheap) alloc_m(npage uintptr, spanclass spanClass, large bool) *mspan { // able to map interior pointer to containing span. atomic.Store(&s.sweepgen, h.sweepgen) h.sweepSpans[h.sweepgen/2%2].push(s) // Add to swept in-use list. - s.state = _MSpanInUse + s.state = mSpanInUse s.allocCount = 0 s.spanclass = spanclass if sizeclass := spanclass.sizeclass(); sizeclass == 0 { @@ -788,7 +788,7 @@ func (h *mheap) allocManual(npage uintptr, stat *uint64) *mspan { lock(&h.lock) s := h.allocSpanLocked(npage, stat) if s != nil { - s.state = _MSpanManual + s.state = mSpanManual s.manualFreeList = 0 s.allocCount = 0 s.spanclass = 0 @@ -829,7 +829,7 @@ func (h *mheap) setSpans(base, npage uintptr, s *mspan) { // Allocates a span of the given size. h must be locked. // The returned span has been removed from the -// free list, but its state is still MSpanFree. +// free list, but its state is still mSpanFree. func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { var list *mSpanList var s *mspan @@ -857,7 +857,7 @@ func (h *mheap) allocSpanLocked(npage uintptr, stat *uint64) *mspan { HaveSpan: // Mark span in use. - if s.state != _MSpanFree { + if s.state != mSpanFree { throw("MHeap_AllocLocked - MSpan not free") } if s.npages < npage { @@ -878,10 +878,10 @@ HaveSpan: h.setSpan(t.base(), t) h.setSpan(t.base()+t.npages*pageSize-1, t) t.needzero = s.needzero - s.state = _MSpanManual // prevent coalescing with s - t.state = _MSpanManual + s.state = mSpanManual // prevent coalescing with s + t.state = mSpanManual h.freeSpanLocked(t, false, false, s.unusedsince) - s.state = _MSpanFree + s.state = mSpanFree } s.unusedsince = 0 @@ -930,7 +930,7 @@ func (h *mheap) grow(npage uintptr) bool { s.init(uintptr(v), size/pageSize) h.setSpans(s.base(), s.npages, s) atomic.Store(&s.sweepgen, h.sweepgen) - s.state = _MSpanInUse + s.state = mSpanInUse h.pagesInUse += uint64(s.npages) h.freeSpanLocked(s, false, true, 0) return true @@ -986,11 +986,11 @@ func (h *mheap) freeManual(s *mspan, stat *uint64) { // s must be on a busy list (h.busy or h.busylarge) or unlinked. func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince int64) { switch s.state { - case _MSpanManual: + case mSpanManual: if s.allocCount != 0 { throw("MHeap_FreeSpanLocked - invalid stack free") } - case _MSpanInUse: + case mSpanInUse: if s.allocCount != 0 || s.sweepgen != h.sweepgen { print("MHeap_FreeSpanLocked - span ", s, " ptr ", hex(s.base()), " allocCount ", s.allocCount, " sweepgen ", s.sweepgen, "/", h.sweepgen, "\n") throw("MHeap_FreeSpanLocked - invalid free") @@ -1006,7 +1006,7 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i if acctidle { memstats.heap_idle += uint64(s.npages << _PageShift) } - s.state = _MSpanFree + s.state = mSpanFree if s.inList() { h.busyList(s.npages).remove(s) } @@ -1020,7 +1020,7 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i s.npreleased = 0 // Coalesce with earlier, later spans. - if before := spanOf(s.base() - 1); before != nil && before.state == _MSpanFree { + if before := spanOf(s.base() - 1); before != nil && before.state == mSpanFree { // Now adjust s. s.startAddr = before.startAddr s.npages += before.npages @@ -1035,12 +1035,12 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i } else { h.freeList(before.npages).remove(before) } - before.state = _MSpanDead + before.state = mSpanDead h.spanalloc.free(unsafe.Pointer(before)) } // Now check to see if next (greater addresses) span is free and can be coalesced. - if after := spanOf(s.base() + s.npages*pageSize); after != nil && after.state == _MSpanFree { + if after := spanOf(s.base() + s.npages*pageSize); after != nil && after.state == mSpanFree { s.npages += after.npages s.npreleased += after.npreleased s.needzero |= after.needzero @@ -1050,7 +1050,7 @@ func (h *mheap) freeSpanLocked(s *mspan, acctinuse, acctidle bool, unusedsince i } else { h.freeList(after.npages).remove(after) } - after.state = _MSpanDead + after.state = mSpanDead h.spanalloc.free(unsafe.Pointer(after)) } @@ -1187,7 +1187,7 @@ func (span *mspan) init(base uintptr, npages uintptr) { span.spanclass = 0 span.incache = false span.elemsize = 0 - span.state = _MSpanDead + span.state = mSpanDead span.unusedsince = 0 span.npreleased = 0 span.speciallock.key = 0 diff --git a/src/runtime/mstats.go b/src/runtime/mstats.go index f67d05414d..1bd6566052 100644 --- a/src/runtime/mstats.go +++ b/src/runtime/mstats.go @@ -38,7 +38,7 @@ type mstats struct { heap_alloc uint64 // bytes allocated and not yet freed (same as alloc above) heap_sys uint64 // virtual address space obtained from system for GC'd heap heap_idle uint64 // bytes in idle spans - heap_inuse uint64 // bytes in _MSpanInUse spans + heap_inuse uint64 // bytes in mSpanInUse spans heap_released uint64 // bytes released to the os heap_objects uint64 // total number of allocated objects diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index d9c6f6d22a..1ed6b39303 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -484,7 +484,7 @@ func TestGdbConst(t *testing.T) { "-ex", "print main.aConstant", "-ex", "print main.largeConstant", "-ex", "print main.minusOne", - "-ex", "print 'runtime._MSpanInUse'", + "-ex", "print 'runtime.mSpanInUse'", "-ex", "print 'runtime._PageSize'", filepath.Join(dir, "a.exe"), } diff --git a/src/runtime/stack.go b/src/runtime/stack.go index c7bfc0434b..d56b864c5e 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -211,7 +211,7 @@ func stackpoolalloc(order uint8) gclinkptr { // Adds stack x to the free pool. Must be called with stackpoolmu held. func stackpoolfree(x gclinkptr, order uint8) { s := spanOfUnchecked(uintptr(x)) - if s.state != _MSpanManual { + if s.state != mSpanManual { throw("freeing stack not in a stack span") } if s.manualFreeList.ptr() == nil { @@ -459,7 +459,7 @@ func stackfree(stk stack) { } } else { s := spanOfUnchecked(uintptr(v)) - if s.state != _MSpanManual { + if s.state != mSpanManual { println(hex(s.base()), v) throw("bad span state") } -- cgit v1.3-5-g9baa From e35a41261b19589f40d32bd66274c23ab4b9b32e Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 25 Sep 2018 14:32:44 -0700 Subject: reflect: use correct write barrier operations for method funcs Fix the code to use write barriers on heap memory, and no write barriers on stack memory. These errors were discoverd as part of fixing #27695. They may have something to do with that issue, but hard to be sure. The core cause is different, so this fix is a separate CL. Update #27695 Change-Id: Ib005f6b3308de340be83c3d07d049d5e316b1e3c Reviewed-on: https://go-review.googlesource.com/137438 Reviewed-by: Austin Clements --- src/reflect/type.go | 2 ++ src/reflect/value.go | 39 ++++++++++++++++++++++++++------------- src/runtime/mbarrier.go | 13 +++++++++++++ 3 files changed, 41 insertions(+), 13 deletions(-) (limited to 'src/runtime') diff --git a/src/reflect/type.go b/src/reflect/type.go index 58cfc0e884..6b0ce431a6 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -3066,6 +3066,8 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin // space no matter how big they actually are. if ifaceIndir(rcvr) || rcvr.pointers() { ptrmap.append(1) + } else { + ptrmap.append(0) } offset += ptrSize } diff --git a/src/reflect/value.go b/src/reflect/value.go index 1c3e590377..854a5b153e 100644 --- a/src/reflect/value.go +++ b/src/reflect/value.go @@ -453,15 +453,14 @@ func (v Value) call(op string, in []Value) []Value { var ret []Value if nout == 0 { - // This is untyped because the frame is really a - // stack, even though it's a heap object. - memclrNoHeapPointers(args, frametype.size) + typedmemclr(frametype, args) framePool.Put(args) } else { // Zero the now unused input area of args, // because the Values returned by this function contain pointers to the args object, // and will thus keep the args object alive indefinitely. - memclrNoHeapPointers(args, retOffset) + typedmemclrpartial(frametype, args, 0, retOffset) + // Wrap Values around return values in args. ret = make([]Value, nout) off = retOffset @@ -472,6 +471,10 @@ func (v Value) call(op string, in []Value) []Value { if tv.Size() != 0 { fl := flagIndir | flag(tv.Kind()) ret[i] = Value{tv.common(), add(args, off, "tv.Size() != 0"), fl} + // Note: this does introduce false sharing between results - + // if any result is live, they are all live. + // (And the space for the args is live as well, but as we've + // cleared that space it isn't as big a deal.) } else { // For zero-sized return value, args+off may point to the next object. // In this case, return the zero value instead. @@ -660,6 +663,8 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { } // Call. + // Call copies the arguments from args to the stack, calls fn, + // and then copies the results back into args. call(frametype, fn, args, uint32(frametype.size), uint32(retOffset)) // Copy return values. On amd64p32, the beginning of return values @@ -673,16 +678,14 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { if runtime.GOARCH == "amd64p32" { callerRetOffset = align(argSize-ptrSize, 8) } - typedmemmovepartial(frametype, - add(frame, callerRetOffset, "frametype.size > retOffset"), + // This copies to the stack. Write barriers are not needed. + memmove(add(frame, callerRetOffset, "frametype.size > retOffset"), add(args, retOffset, "frametype.size > retOffset"), - retOffset, frametype.size-retOffset) } - // This is untyped because the frame is really a stack, even - // though it's a heap object. - memclrNoHeapPointers(args, frametype.size) + // Put the args scratch space back in the pool. + typedmemclr(frametype, args) framePool.Put(args) // See the comment in callReflect. @@ -2641,6 +2644,10 @@ func call(argtype *rtype, fn, arg unsafe.Pointer, n uint32, retoffset uint32) func ifaceE2I(t *rtype, src interface{}, dst unsafe.Pointer) +// memmove copies size bytes to dst from src. No write barriers are used. +//go:noescape +func memmove(dst, src unsafe.Pointer, size uintptr) + // typedmemmove copies a value of type t to dst from src. //go:noescape func typedmemmove(t *rtype, dst, src unsafe.Pointer) @@ -2650,14 +2657,20 @@ func typedmemmove(t *rtype, dst, src unsafe.Pointer) //go:noescape func typedmemmovepartial(t *rtype, dst, src unsafe.Pointer, off, size uintptr) +// typedmemclr zeros the value at ptr of type t. +//go:noescape +func typedmemclr(t *rtype, ptr unsafe.Pointer) + +// typedmemclrpartial is like typedmemclr but assumes that +// dst points off bytes into the value and only clears size bytes. +//go:noescape +func typedmemclrpartial(t *rtype, ptr unsafe.Pointer, off, size uintptr) + // typedslicecopy copies a slice of elemType values from src to dst, // returning the number of elements copied. //go:noescape func typedslicecopy(elemType *rtype, dst, src sliceHeader) int -//go:noescape -func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) - // Dummy annotation marking that the value x escapes, // for use in cases where the reflect code is so clever that // the compiler cannot follow. diff --git a/src/runtime/mbarrier.go b/src/runtime/mbarrier.go index 5142f4327a..6da8cf2ccb 100644 --- a/src/runtime/mbarrier.go +++ b/src/runtime/mbarrier.go @@ -318,6 +318,19 @@ func typedmemclr(typ *_type, ptr unsafe.Pointer) { memclrNoHeapPointers(ptr, typ.size) } +//go:linkname reflect_typedmemclr reflect.typedmemclr +func reflect_typedmemclr(typ *_type, ptr unsafe.Pointer) { + typedmemclr(typ, ptr) +} + +//go:linkname reflect_typedmemclrpartial reflect.typedmemclrpartial +func reflect_typedmemclrpartial(typ *_type, ptr unsafe.Pointer, off, size uintptr) { + if typ.kind&kindNoPointers == 0 { + bulkBarrierPreWrite(uintptr(ptr), 0, size) + } + memclrNoHeapPointers(ptr, size) +} + // memclrHasPointers clears n bytes of typed memory starting at ptr. // The caller must ensure that the type of the object at ptr has // pointers, usually by checking typ.kind&kindNoPointers. However, ptr -- cgit v1.3-5-g9baa From da0d1a44bac379f5acedb1933f85400de08f4ac6 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Wed, 26 Sep 2018 21:10:21 +0000 Subject: all: use strings.ReplaceAll and bytes.ReplaceAll where applicable I omitted vendor directories and anything necessary for bootstrapping. (Tested by bootstrapping with Go 1.4) Updates #27864 Change-Id: I7d9b68d0372d3a34dee22966cca323513ece7e8a Reviewed-on: https://go-review.googlesource.com/137856 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/cmd/fix/main.go | 2 +- src/cmd/fix/typecheck.go | 4 ++-- src/cmd/go/go_test.go | 10 +++++----- src/cmd/go/internal/envcmd/env.go | 2 +- src/cmd/go/internal/get/vcs.go | 2 +- src/cmd/go/internal/modconv/convert_test.go | 2 +- src/cmd/go/internal/modfetch/codehost/codehost.go | 2 +- src/cmd/go/internal/modfetch/coderepo_test.go | 12 ++++++------ src/cmd/go/internal/modfetch/proxy.go | 2 +- src/cmd/go/internal/modload/import_test.go | 2 +- src/cmd/go/internal/modload/query_test.go | 2 +- src/cmd/go/internal/search/search.go | 4 ++-- src/cmd/go/internal/work/build.go | 4 ++-- src/cmd/go/internal/work/exec.go | 10 +++++----- src/cmd/go/proxy_test.go | 4 ++-- src/cmd/go/script_test.go | 12 ++++++------ src/cmd/go/testdata/addmod.go | 2 +- src/cmd/go/vendor_test.go | 2 +- src/cmd/gofmt/gofmt_test.go | 2 +- src/cmd/internal/goobj/read.go | 2 +- src/cmd/trace/trace.go | 2 +- src/cmd/vet/main.go | 2 +- src/cmd/vet/vet_test.go | 2 +- src/crypto/rsa/pss_test.go | 2 +- src/crypto/x509/root_darwin_arm_gen.go | 6 +++--- src/encoding/base32/base32_test.go | 10 +++++----- src/encoding/base64/base64_test.go | 8 ++++---- src/flag/flag.go | 2 +- src/go/constant/value_test.go | 2 +- src/go/doc/doc_test.go | 6 +++--- src/go/printer/example_test.go | 4 ++-- src/html/template/js.go | 2 +- src/html/template/url.go | 2 +- src/mime/multipart/formdata_test.go | 8 ++++---- src/mime/multipart/multipart_test.go | 6 +++--- src/net/http/cgi/child.go | 2 +- src/net/http/httputil/dump_test.go | 2 +- src/net/http/readrequest_test.go | 2 +- src/net/http/request_test.go | 6 +++--- src/net/http/serve_test.go | 2 +- src/net/lookup_windows_test.go | 2 +- src/net/mail/message_test.go | 4 ++-- src/net/net_windows_test.go | 2 +- src/net/url/example_test.go | 2 +- src/net/url/url_test.go | 6 +++--- src/os/path_windows_test.go | 6 +++--- src/path/filepath/path.go | 4 ++-- src/path/filepath/path_test.go | 2 +- src/path/filepath/path_windows.go | 2 +- src/path/filepath/path_windows_test.go | 2 +- src/runtime/pprof/internal/profile/profile.go | 2 +- src/runtime/pprof/pprof_test.go | 2 +- src/runtime/runtime-gdb_test.go | 2 +- src/strings/example_test.go | 2 +- src/testing/sub_test.go | 4 ++-- src/text/template/exec.go | 2 +- 56 files changed, 104 insertions(+), 104 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/fix/main.go b/src/cmd/fix/main.go index f06abae171..f54a5e0d96 100644 --- a/src/cmd/fix/main.go +++ b/src/cmd/fix/main.go @@ -52,7 +52,7 @@ func usage() { fmt.Fprintf(os.Stderr, "\n%s\n", f.name) } desc := strings.TrimSpace(f.desc) - desc = strings.Replace(desc, "\n", "\n\t", -1) + desc = strings.ReplaceAll(desc, "\n", "\n\t") fmt.Fprintf(os.Stderr, "\t%s\n", desc) } os.Exit(2) diff --git a/src/cmd/fix/typecheck.go b/src/cmd/fix/typecheck.go index eafb626c74..66e0cdcec0 100644 --- a/src/cmd/fix/typecheck.go +++ b/src/cmd/fix/typecheck.go @@ -193,12 +193,12 @@ func typecheck(cfg *TypeConfig, f *ast.File) (typeof map[interface{}]string, ass var params, results []string for _, p := range fn.Type.Params.List { t := gofmt(p.Type) - t = strings.Replace(t, "_Ctype_", "C.", -1) + t = strings.ReplaceAll(t, "_Ctype_", "C.") params = append(params, t) } for _, r := range fn.Type.Results.List { t := gofmt(r.Type) - t = strings.Replace(t, "_Ctype_", "C.", -1) + t = strings.ReplaceAll(t, "_Ctype_", "C.") results = append(results, t) } cfg.External["C."+fn.Name.Name[7:]] = joinFunc(params, results) diff --git a/src/cmd/go/go_test.go b/src/cmd/go/go_test.go index 962b52fd3d..139ee73ae0 100644 --- a/src/cmd/go/go_test.go +++ b/src/cmd/go/go_test.go @@ -1090,7 +1090,7 @@ func testMove(t *testing.T, vcs, url, base, config string) { path := tg.path(filepath.Join("src", config)) data, err := ioutil.ReadFile(path) tg.must(err) - data = bytes.Replace(data, []byte(base), []byte(base+"XXX"), -1) + data = bytes.ReplaceAll(data, []byte(base), []byte(base+"XXX")) tg.must(ioutil.WriteFile(path, data, 0644)) } if vcs == "git" { @@ -2360,14 +2360,14 @@ func TestShadowingLogic(t *testing.T) { // The math in root1 is not "math" because the standard math is. tg.run("list", "-f", "({{.ImportPath}}) ({{.ConflictDir}})", "./testdata/shadow/root1/src/math") - pwdForwardSlash := strings.Replace(pwd, string(os.PathSeparator), "/", -1) + pwdForwardSlash := strings.ReplaceAll(pwd, string(os.PathSeparator), "/") if !strings.HasPrefix(pwdForwardSlash, "/") { pwdForwardSlash = "/" + pwdForwardSlash } // The output will have makeImportValid applies, but we only // bother to deal with characters we might reasonably see. for _, r := range " :" { - pwdForwardSlash = strings.Replace(pwdForwardSlash, string(r), "_", -1) + pwdForwardSlash = strings.ReplaceAll(pwdForwardSlash, string(r), "_") } want := "(_" + pwdForwardSlash + "/testdata/shadow/root1/src/math) (" + filepath.Join(runtime.GOROOT(), "src", "math") + ")" if strings.TrimSpace(tg.getStdout()) != want { @@ -2557,7 +2557,7 @@ func TestCoverageErrorLine(t *testing.T) { // It's OK that stderr2 drops the character position in the error, // because of the //line directive (see golang.org/issue/22662). - stderr = strings.Replace(stderr, "p.go:4:2:", "p.go:4:", -1) + stderr = strings.ReplaceAll(stderr, "p.go:4:2:", "p.go:4:") if stderr != stderr2 { t.Logf("test -cover changed error messages:\nbefore:\n%s\n\nafter:\n%s", stderr, stderr2) t.Skip("golang.org/issue/22660") @@ -6171,7 +6171,7 @@ func TestCDAndGOPATHAreDifferent(t *testing.T) { testCDAndGOPATHAreDifferent(tg, cd, gopath) if runtime.GOOS == "windows" { - testCDAndGOPATHAreDifferent(tg, cd, strings.Replace(gopath, `\`, `/`, -1)) + testCDAndGOPATHAreDifferent(tg, cd, strings.ReplaceAll(gopath, `\`, `/`)) testCDAndGOPATHAreDifferent(tg, cd, strings.ToUpper(gopath)) testCDAndGOPATHAreDifferent(tg, cd, strings.ToLower(gopath)) } diff --git a/src/cmd/go/internal/envcmd/env.go b/src/cmd/go/internal/envcmd/env.go index afadbade38..85a42e0519 100644 --- a/src/cmd/go/internal/envcmd/env.go +++ b/src/cmd/go/internal/envcmd/env.go @@ -203,7 +203,7 @@ func runEnv(cmd *base.Command, args []string) { fmt.Printf("%s=\"%s\"\n", e.Name, e.Value) case "plan9": if strings.IndexByte(e.Value, '\x00') < 0 { - fmt.Printf("%s='%s'\n", e.Name, strings.Replace(e.Value, "'", "''", -1)) + fmt.Printf("%s='%s'\n", e.Name, strings.ReplaceAll(e.Value, "'", "''")) } else { v := strings.Split(e.Value, "\x00") fmt.Printf("%s=(", e.Name) diff --git a/src/cmd/go/internal/get/vcs.go b/src/cmd/go/internal/get/vcs.go index 0f7b623ec3..173934b84e 100644 --- a/src/cmd/go/internal/get/vcs.go +++ b/src/cmd/go/internal/get/vcs.go @@ -964,7 +964,7 @@ func matchGoImport(imports []metaImport, importPath string) (metaImport, error) // expand rewrites s to replace {k} with match[k] for each key k in match. func expand(match map[string]string, s string) string { for k, v := range match { - s = strings.Replace(s, "{"+k+"}", v, -1) + s = strings.ReplaceAll(s, "{"+k+"}", v) } return s } diff --git a/src/cmd/go/internal/modconv/convert_test.go b/src/cmd/go/internal/modconv/convert_test.go index ad27abb8ef..4d55d73f21 100644 --- a/src/cmd/go/internal/modconv/convert_test.go +++ b/src/cmd/go/internal/modconv/convert_test.go @@ -146,7 +146,7 @@ func TestConvertLegacyConfig(t *testing.T) { } for _, tt := range tests { - t.Run(strings.Replace(tt.path, "/", "_", -1)+"_"+tt.vers, func(t *testing.T) { + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"_"+tt.vers, func(t *testing.T) { f, err := modfile.Parse("golden", []byte(tt.gomod), nil) if err != nil { t.Fatal(err) diff --git a/src/cmd/go/internal/modfetch/codehost/codehost.go b/src/cmd/go/internal/modfetch/codehost/codehost.go index 4103ddc717..4205cd26bd 100644 --- a/src/cmd/go/internal/modfetch/codehost/codehost.go +++ b/src/cmd/go/internal/modfetch/codehost/codehost.go @@ -185,7 +185,7 @@ func (e *RunError) Error() string { text := e.Cmd + ": " + e.Err.Error() stderr := bytes.TrimRight(e.Stderr, "\n") if len(stderr) > 0 { - text += ":\n\t" + strings.Replace(string(stderr), "\n", "\n\t", -1) + text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t") } return text } diff --git a/src/cmd/go/internal/modfetch/coderepo_test.go b/src/cmd/go/internal/modfetch/coderepo_test.go index 79b82786cb..0b62b9ee76 100644 --- a/src/cmd/go/internal/modfetch/coderepo_test.go +++ b/src/cmd/go/internal/modfetch/coderepo_test.go @@ -423,7 +423,7 @@ func TestCodeRepo(t *testing.T) { } } } - t.Run(strings.Replace(tt.path, "/", "_", -1)+"/"+tt.rev, f) + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.rev, f) if strings.HasPrefix(tt.path, vgotest1git) { for _, alt := range altVgotests { // Note: Communicating with f through tt; should be cleaned up. @@ -442,7 +442,7 @@ func TestCodeRepo(t *testing.T) { tt.rev = remap(tt.rev, m) tt.gomoderr = remap(tt.gomoderr, m) tt.ziperr = remap(tt.ziperr, m) - t.Run(strings.Replace(tt.path, "/", "_", -1)+"/"+tt.rev, f) + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.rev, f) tt = old } } @@ -473,9 +473,9 @@ func remap(name string, m map[string]string) string { } } for k, v := range m { - name = strings.Replace(name, k, v, -1) + name = strings.ReplaceAll(name, k, v) if codehost.AllHex(k) { - name = strings.Replace(name, k[:12], v[:12], -1) + name = strings.ReplaceAll(name, k[:12], v[:12]) } } return name @@ -522,7 +522,7 @@ func TestCodeRepoVersions(t *testing.T) { } defer os.RemoveAll(tmpdir) for _, tt := range codeRepoVersionsTests { - t.Run(strings.Replace(tt.path, "/", "_", -1), func(t *testing.T) { + t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { repo, err := Lookup(tt.path) if err != nil { t.Fatalf("Lookup(%q): %v", tt.path, err) @@ -570,7 +570,7 @@ func TestLatest(t *testing.T) { } defer os.RemoveAll(tmpdir) for _, tt := range latestTests { - name := strings.Replace(tt.path, "/", "_", -1) + name := strings.ReplaceAll(tt.path, "/", "_") t.Run(name, func(t *testing.T) { repo, err := Lookup(tt.path) if err != nil { diff --git a/src/cmd/go/internal/modfetch/proxy.go b/src/cmd/go/internal/modfetch/proxy.go index 5f856b80d2..7c78502f31 100644 --- a/src/cmd/go/internal/modfetch/proxy.go +++ b/src/cmd/go/internal/modfetch/proxy.go @@ -248,5 +248,5 @@ func (p *proxyRepo) Zip(version string, tmpdir string) (tmpfile string, err erro // That is, it escapes things like ? and # (which really shouldn't appear anyway). // It does not escape / to %2F: our REST API is designed so that / can be left as is. func pathEscape(s string) string { - return strings.Replace(url.PathEscape(s), "%2F", "/", -1) + return strings.ReplaceAll(url.PathEscape(s), "%2F", "/") } diff --git a/src/cmd/go/internal/modload/import_test.go b/src/cmd/go/internal/modload/import_test.go index 3f4ddab436..9422a3d960 100644 --- a/src/cmd/go/internal/modload/import_test.go +++ b/src/cmd/go/internal/modload/import_test.go @@ -45,7 +45,7 @@ func TestImport(t *testing.T) { testenv.MustHaveExternalNetwork(t) for _, tt := range importTests { - t.Run(strings.Replace(tt.path, "/", "_", -1), func(t *testing.T) { + t.Run(strings.ReplaceAll(tt.path, "/", "_"), func(t *testing.T) { // Note that there is no build list, so Import should always fail. m, dir, err := Import(tt.path) if err == nil { diff --git a/src/cmd/go/internal/modload/query_test.go b/src/cmd/go/internal/modload/query_test.go index 7f3ffabef7..9b07383217 100644 --- a/src/cmd/go/internal/modload/query_test.go +++ b/src/cmd/go/internal/modload/query_test.go @@ -132,7 +132,7 @@ func TestQuery(t *testing.T) { ok, _ := path.Match(allow, m.Version) return ok } - t.Run(strings.Replace(tt.path, "/", "_", -1)+"/"+tt.query+"/"+allow, func(t *testing.T) { + t.Run(strings.ReplaceAll(tt.path, "/", "_")+"/"+tt.query+"/"+allow, func(t *testing.T) { info, err := Query(tt.path, tt.query, allowed) if tt.err != "" { if err != nil && err.Error() == tt.err { diff --git a/src/cmd/go/internal/search/search.go b/src/cmd/go/internal/search/search.go index 60ae73696b..0ca60e7349 100644 --- a/src/cmd/go/internal/search/search.go +++ b/src/cmd/go/internal/search/search.go @@ -275,7 +275,7 @@ func MatchPattern(pattern string) func(name string) bool { case strings.HasSuffix(re, `/\.\.\.`): re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` } - re = strings.Replace(re, `\.\.\.`, `[^`+vendorChar+`]*`, -1) + re = strings.ReplaceAll(re, `\.\.\.`, `[^`+vendorChar+`]*`) reg := regexp.MustCompile(`^` + re + `$`) @@ -353,7 +353,7 @@ func CleanPatterns(patterns []string) []string { // as a courtesy to Windows developers, rewrite \ to / // in command-line arguments. Handles .\... and so on. if filepath.Separator == '\\' { - a = strings.Replace(a, `\`, `/`, -1) + a = strings.ReplaceAll(a, `\`, `/`) } // Put argument in canonical form, but preserve leading ./. diff --git a/src/cmd/go/internal/work/build.go b/src/cmd/go/internal/work/build.go index dd482b677d..145b87513a 100644 --- a/src/cmd/go/internal/work/build.go +++ b/src/cmd/go/internal/work/build.go @@ -398,10 +398,10 @@ func libname(args []string, pkgs []*load.Package) (string, error) { arg = bp.ImportPath } } - appendName(strings.Replace(arg, "/", "-", -1)) + appendName(strings.ReplaceAll(arg, "/", "-")) } else { for _, pkg := range pkgs { - appendName(strings.Replace(pkg.ImportPath, "/", "-", -1)) + appendName(strings.ReplaceAll(pkg.ImportPath, "/", "-")) } } } else if haveNonMeta { // have both meta package and a non-meta one diff --git a/src/cmd/go/internal/work/exec.go b/src/cmd/go/internal/work/exec.go index 01414a3d57..158f5f3b17 100644 --- a/src/cmd/go/internal/work/exec.go +++ b/src/cmd/go/internal/work/exec.go @@ -1705,14 +1705,14 @@ func (b *Builder) fmtcmd(dir string, format string, args ...interface{}) string if dir[len(dir)-1] == filepath.Separator { dot += string(filepath.Separator) } - cmd = strings.Replace(" "+cmd, " "+dir, dot, -1)[1:] + cmd = strings.ReplaceAll(" "+cmd, " "+dir, dot)[1:] if b.scriptDir != dir { b.scriptDir = dir cmd = "cd " + dir + "\n" + cmd } } if b.WorkDir != "" { - cmd = strings.Replace(cmd, b.WorkDir, "$WORK", -1) + cmd = strings.ReplaceAll(cmd, b.WorkDir, "$WORK") } return cmd } @@ -1754,10 +1754,10 @@ func (b *Builder) showOutput(a *Action, dir, desc, out string) { prefix := "# " + desc suffix := "\n" + out if reldir := base.ShortPath(dir); reldir != dir { - suffix = strings.Replace(suffix, " "+dir, " "+reldir, -1) - suffix = strings.Replace(suffix, "\n"+dir, "\n"+reldir, -1) + suffix = strings.ReplaceAll(suffix, " "+dir, " "+reldir) + suffix = strings.ReplaceAll(suffix, "\n"+dir, "\n"+reldir) } - suffix = strings.Replace(suffix, " "+b.WorkDir, " $WORK", -1) + suffix = strings.ReplaceAll(suffix, " "+b.WorkDir, " $WORK") if a != nil && a.output != nil { a.output = append(a.output, prefix...) diff --git a/src/cmd/go/proxy_test.go b/src/cmd/go/proxy_test.go index 212e5aa08f..97fc4b0e80 100644 --- a/src/cmd/go/proxy_test.go +++ b/src/cmd/go/proxy_test.go @@ -78,7 +78,7 @@ func readModList() { if i < 0 { continue } - encPath := strings.Replace(name[:i], "_", "/", -1) + encPath := strings.ReplaceAll(name[:i], "_", "/") path, err := module.DecodePath(encPath) if err != nil { fmt.Fprintf(os.Stderr, "go proxy_test: %v\n", err) @@ -256,7 +256,7 @@ func readArchive(path, vers string) *txtar.Archive { return nil } - prefix := strings.Replace(enc, "/", "_", -1) + prefix := strings.ReplaceAll(enc, "/", "_") name := filepath.Join(cmdGoDir, "testdata/mod", prefix+"_"+encVers+".txt") a := archiveCache.Do(name, func() interface{} { a, err := txtar.ParseFile(name) diff --git a/src/cmd/go/script_test.go b/src/cmd/go/script_test.go index 7c083a87b9..31c6ede2a5 100644 --- a/src/cmd/go/script_test.go +++ b/src/cmd/go/script_test.go @@ -329,7 +329,7 @@ func (ts *testScript) cmdAddcrlf(neg bool, args []string) { file = ts.mkabs(file) data, err := ioutil.ReadFile(file) ts.check(err) - ts.check(ioutil.WriteFile(file, bytes.Replace(data, []byte("\n"), []byte("\r\n"), -1), 0666)) + ts.check(ioutil.WriteFile(file, bytes.ReplaceAll(data, []byte("\n"), []byte("\r\n")), 0666)) } } @@ -630,7 +630,7 @@ func scriptMatch(ts *testScript, neg bool, args []string, text, name string) { } // Matching against workdir would be misleading. - text = strings.Replace(text, ts.workdir, "$WORK", -1) + text = strings.ReplaceAll(text, ts.workdir, "$WORK") if neg { if re.MatchString(text) { @@ -691,7 +691,7 @@ func (ts *testScript) cmdSymlink(neg bool, args []string) { // abbrev abbreviates the actual work directory in the string s to the literal string "$WORK". func (ts *testScript) abbrev(s string) string { - s = strings.Replace(s, ts.workdir, "$WORK", -1) + s = strings.ReplaceAll(s, ts.workdir, "$WORK") if *testWork { // Expose actual $WORK value in environment dump on first line of work script, // so that the user can find out what directory -testwork left behind. @@ -885,17 +885,17 @@ var diffTests = []struct { func TestDiff(t *testing.T) { for _, tt := range diffTests { // Turn spaces into \n. - text1 := strings.Replace(tt.text1, " ", "\n", -1) + text1 := strings.ReplaceAll(tt.text1, " ", "\n") if text1 != "" { text1 += "\n" } - text2 := strings.Replace(tt.text2, " ", "\n", -1) + text2 := strings.ReplaceAll(tt.text2, " ", "\n") if text2 != "" { text2 += "\n" } out := diff(text1, text2) // Cut final \n, cut spaces, turn remaining \n into spaces. - out = strings.Replace(strings.Replace(strings.TrimSuffix(out, "\n"), " ", "", -1), "\n", " ", -1) + out = strings.ReplaceAll(strings.ReplaceAll(strings.TrimSuffix(out, "\n"), " ", ""), "\n", " ") if out != tt.diff { t.Errorf("diff(%q, %q) = %q, want %q", text1, text2, out, tt.diff) } diff --git a/src/cmd/go/testdata/addmod.go b/src/cmd/go/testdata/addmod.go index 19850af0f3..8bb6056a54 100644 --- a/src/cmd/go/testdata/addmod.go +++ b/src/cmd/go/testdata/addmod.go @@ -142,7 +142,7 @@ func main() { } data := txtar.Format(a) - target := filepath.Join("mod", strings.Replace(path, "/", "_", -1)+"_"+vers+".txt") + target := filepath.Join("mod", strings.ReplaceAll(path, "/", "_")+"_"+vers+".txt") if err := ioutil.WriteFile(target, data, 0666); err != nil { log.Printf("%s: %v", arg, err) exitCode = 1 diff --git a/src/cmd/go/vendor_test.go b/src/cmd/go/vendor_test.go index 22aa643b00..c302d7e9b5 100644 --- a/src/cmd/go/vendor_test.go +++ b/src/cmd/go/vendor_test.go @@ -37,7 +37,7 @@ func TestVendorImports(t *testing.T) { vend/x/vendor/p/p [notfound] vend/x/vendor/r [] ` - want = strings.Replace(want+"\t", "\n\t\t", "\n", -1) + want = strings.ReplaceAll(want+"\t", "\n\t\t", "\n") want = strings.TrimPrefix(want, "\n") have := tg.stdout.String() diff --git a/src/cmd/gofmt/gofmt_test.go b/src/cmd/gofmt/gofmt_test.go index 16b653b646..3008365cd2 100644 --- a/src/cmd/gofmt/gofmt_test.go +++ b/src/cmd/gofmt/gofmt_test.go @@ -200,7 +200,7 @@ func TestDiff(t *testing.T) { } if runtime.GOOS == "windows" { - b = bytes.Replace(b, []byte{'\r', '\n'}, []byte{'\n'}, -1) + b = bytes.ReplaceAll(b, []byte{'\r', '\n'}, []byte{'\n'}) } bs := bytes.SplitN(b, []byte{'\n'}, 3) diff --git a/src/cmd/internal/goobj/read.go b/src/cmd/internal/goobj/read.go index e39180cad6..2d618eefa5 100644 --- a/src/cmd/internal/goobj/read.go +++ b/src/cmd/internal/goobj/read.go @@ -293,7 +293,7 @@ func (r *objReader) readRef() { // In a symbol name in an object file, "". denotes the // prefix for the package in which the object file has been found. // Expand it. - name = strings.Replace(name, `"".`, r.pkgprefix, -1) + name = strings.ReplaceAll(name, `"".`, r.pkgprefix) // An individual object file only records version 0 (extern) or 1 (static). // To make static symbols unique across all files being read, we diff --git a/src/cmd/trace/trace.go b/src/cmd/trace/trace.go index 676b9ffa5a..07fc4333eb 100644 --- a/src/cmd/trace/trace.go +++ b/src/cmd/trace/trace.go @@ -38,7 +38,7 @@ func httpTrace(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - html := strings.Replace(templTrace, "{{PARAMS}}", r.Form.Encode(), -1) + html := strings.ReplaceAll(templTrace, "{{PARAMS}}", r.Form.Encode()) w.Write([]byte(html)) } diff --git a/src/cmd/vet/main.go b/src/cmd/vet/main.go index 646adf4d76..6e885121c8 100644 --- a/src/cmd/vet/main.go +++ b/src/cmd/vet/main.go @@ -273,7 +273,7 @@ func main() { // Accept space-separated tags because that matches // the go command's other subcommands. // Accept commas because go tool vet traditionally has. - tagList = strings.Fields(strings.Replace(*tags, ",", " ", -1)) + tagList = strings.Fields(strings.ReplaceAll(*tags, ",", " ")) initPrintFlags() initUnusedFlags() diff --git a/src/cmd/vet/vet_test.go b/src/cmd/vet/vet_test.go index 90665d77bc..df84d6cc98 100644 --- a/src/cmd/vet/vet_test.go +++ b/src/cmd/vet/vet_test.go @@ -243,7 +243,7 @@ func errorCheck(outStr string, wantAuto bool, fullshort ...string) (err error) { for i := range out { for j := 0; j < len(fullshort); j += 2 { full, short := fullshort[j], fullshort[j+1] - out[i] = strings.Replace(out[i], full, short, -1) + out[i] = strings.ReplaceAll(out[i], full, short) } } diff --git a/src/crypto/rsa/pss_test.go b/src/crypto/rsa/pss_test.go index cae24e58c6..dfa8d8bb5a 100644 --- a/src/crypto/rsa/pss_test.go +++ b/src/crypto/rsa/pss_test.go @@ -103,7 +103,7 @@ func TestPSSGolden(t *testing.T) { switch { case len(line) == 0: if len(partialValue) > 0 { - values <- strings.Replace(partialValue, " ", "", -1) + values <- strings.ReplaceAll(partialValue, " ", "") partialValue = "" lastWasValue = true } diff --git a/src/crypto/x509/root_darwin_arm_gen.go b/src/crypto/x509/root_darwin_arm_gen.go index b5580d6f02..bc44124a78 100644 --- a/src/crypto/x509/root_darwin_arm_gen.go +++ b/src/crypto/x509/root_darwin_arm_gen.go @@ -154,9 +154,9 @@ func fetchCertIDs() ([]certID, error) { values := regexp.MustCompile("(?s)(.*?)").FindAllStringSubmatch(row, -1) name := values[cols["Certificate name"]][1] fingerprint := values[cols["Fingerprint (SHA-256)"]][1] - fingerprint = strings.Replace(fingerprint, "
", "", -1) - fingerprint = strings.Replace(fingerprint, "\n", "", -1) - fingerprint = strings.Replace(fingerprint, " ", "", -1) + fingerprint = strings.ReplaceAll(fingerprint, "
", "") + fingerprint = strings.ReplaceAll(fingerprint, "\n", "") + fingerprint = strings.ReplaceAll(fingerprint, " ", "") fingerprint = strings.ToLower(fingerprint) ids = append(ids, certID{ diff --git a/src/encoding/base32/base32_test.go b/src/encoding/base32/base32_test.go index c5506ed4de..b74054ba40 100644 --- a/src/encoding/base32/base32_test.go +++ b/src/encoding/base32/base32_test.go @@ -425,7 +425,7 @@ IZJAOZSWY2LUEBSXG43FEBRWS3DMOVWSAZDPNRXXEZJAMV2SAZTVM5UWC5BANZ2WY3DBBJYGC4TJMF NZ2CYIDTOVXHIIDJNYFGG5LMOBQSA4LVNEQG6ZTGNFRWSYJAMRSXGZLSOVXHIIDNN5WGY2LUEBQW42 LNEBUWIIDFON2CA3DBMJXXE5LNFY== ====` - encodedShort := strings.Replace(encoded, "\n", "", -1) + encodedShort := strings.ReplaceAll(encoded, "\n", "") dec := NewDecoder(StdEncoding, strings.NewReader(encoded)) res1, err := ioutil.ReadAll(dec) @@ -465,7 +465,7 @@ func TestWithCustomPadding(t *testing.T) { for _, testcase := range pairs { defaultPadding := StdEncoding.EncodeToString([]byte(testcase.decoded)) customPadding := StdEncoding.WithPadding('@').EncodeToString([]byte(testcase.decoded)) - expected := strings.Replace(defaultPadding, "=", "@", -1) + expected := strings.ReplaceAll(defaultPadding, "=", "@") if expected != customPadding { t.Errorf("Expected custom %s, got %s", expected, customPadding) @@ -675,7 +675,7 @@ func TestWithoutPaddingClose(t *testing.T) { expected := testpair.encoded if encoding.padChar == NoPadding { - expected = strings.Replace(expected, "=", "", -1) + expected = strings.ReplaceAll(expected, "=", "") } res := buf.String() @@ -697,7 +697,7 @@ func TestDecodeReadAll(t *testing.T) { for encIndex, encoding := range encodings { encoded := pair.encoded if encoding.padChar == NoPadding { - encoded = strings.Replace(encoded, "=", "", -1) + encoded = strings.ReplaceAll(encoded, "=", "") } decReader, err := ioutil.ReadAll(NewDecoder(encoding, strings.NewReader(encoded))) @@ -723,7 +723,7 @@ func TestDecodeSmallBuffer(t *testing.T) { for encIndex, encoding := range encodings { encoded := pair.encoded if encoding.padChar == NoPadding { - encoded = strings.Replace(encoded, "=", "", -1) + encoded = strings.ReplaceAll(encoded, "=", "") } decoder := NewDecoder(encoding, strings.NewReader(encoded)) diff --git a/src/encoding/base64/base64_test.go b/src/encoding/base64/base64_test.go index f019654f5b..f7f312ca39 100644 --- a/src/encoding/base64/base64_test.go +++ b/src/encoding/base64/base64_test.go @@ -53,8 +53,8 @@ func stdRef(ref string) string { // Convert a reference string to URL-encoding func urlRef(ref string) string { - ref = strings.Replace(ref, "+", "-", -1) - ref = strings.Replace(ref, "/", "_", -1) + ref = strings.ReplaceAll(ref, "+", "-") + ref = strings.ReplaceAll(ref, "/", "_") return ref } @@ -72,7 +72,7 @@ func rawURLRef(ref string) string { var funnyEncoding = NewEncoding(encodeStd).WithPadding(rune('@')) func funnyRef(ref string) string { - return strings.Replace(ref, "=", "@", -1) + return strings.ReplaceAll(ref, "=", "@") } type encodingTest struct { @@ -418,7 +418,7 @@ j+mSARB/17pKVXYWHXjsj7yIex0PadzXMO1zT5KHoNA3HT8ietoGhgjsfA+CSnvvqh/jJtqsrwOv 2b6NGNzXfTYexzJ+nU7/ALkf4P8Awv6P9KvTQQ4AgyDqCF85Pho3CTB7eHwXoH+LT65uZbX9X+o2 bqbPb06551Y4 ` - encodedShort := strings.Replace(encoded, "\n", "", -1) + encodedShort := strings.ReplaceAll(encoded, "\n", "") dec := NewDecoder(StdEncoding, strings.NewReader(encoded)) res1, err := ioutil.ReadAll(dec) diff --git a/src/flag/flag.go b/src/flag/flag.go index 2cd7829c1a..ae84e1f775 100644 --- a/src/flag/flag.go +++ b/src/flag/flag.go @@ -472,7 +472,7 @@ func (f *FlagSet) PrintDefaults() { // for both 4- and 8-space tab stops. s += "\n \t" } - s += strings.Replace(usage, "\n", "\n \t", -1) + s += strings.ReplaceAll(usage, "\n", "\n \t") if !isZeroValue(flag, flag.DefValue) { if _, ok := flag.Value.(*stringValue); ok { diff --git a/src/go/constant/value_test.go b/src/go/constant/value_test.go index e6fca76e18..68b87eaa55 100644 --- a/src/go/constant/value_test.go +++ b/src/go/constant/value_test.go @@ -296,7 +296,7 @@ func val(lit string) Value { switch first, last := lit[0], lit[len(lit)-1]; { case first == '"' || first == '`': tok = token.STRING - lit = strings.Replace(lit, "_", " ", -1) + lit = strings.ReplaceAll(lit, "_", " ") case first == '\'': tok = token.CHAR case last == 'i': diff --git a/src/go/doc/doc_test.go b/src/go/doc/doc_test.go index 902a79f63f..0b2d2b63cc 100644 --- a/src/go/doc/doc_test.go +++ b/src/go/doc/doc_test.go @@ -40,7 +40,7 @@ func readTemplate(filename string) *template.Template { func nodeFmt(node interface{}, fset *token.FileSet) string { var buf bytes.Buffer printer.Fprint(&buf, fset, node) - return strings.Replace(strings.TrimSpace(buf.String()), "\n", "\n\t", -1) + return strings.ReplaceAll(strings.TrimSpace(buf.String()), "\n", "\n\t") } func synopsisFmt(s string) string { @@ -53,7 +53,7 @@ func synopsisFmt(s string) string { } s = strings.TrimSpace(s) + " ..." } - return "// " + strings.Replace(s, "\n", " ", -1) + return "// " + strings.ReplaceAll(s, "\n", " ") } func indentFmt(indent, s string) string { @@ -62,7 +62,7 @@ func indentFmt(indent, s string) string { end = "\n" s = s[:len(s)-1] } - return indent + strings.Replace(s, "\n", "\n"+indent, -1) + end + return indent + strings.ReplaceAll(s, "\n", "\n"+indent) + end } func isGoFile(fi os.FileInfo) bool { diff --git a/src/go/printer/example_test.go b/src/go/printer/example_test.go index e570040ba1..30816931a8 100644 --- a/src/go/printer/example_test.go +++ b/src/go/printer/example_test.go @@ -48,7 +48,7 @@ func ExampleFprint() { // and trim leading and trailing white space. s := buf.String() s = s[1 : len(s)-1] - s = strings.TrimSpace(strings.Replace(s, "\n\t", "\n", -1)) + s = strings.TrimSpace(strings.ReplaceAll(s, "\n\t", "\n")) // Print the cleaned-up body text to stdout. fmt.Println(s) @@ -61,7 +61,7 @@ func ExampleFprint() { // // s := buf.String() // s = s[1 : len(s)-1] - // s = strings.TrimSpace(strings.Replace(s, "\n\t", "\n", -1)) + // s = strings.TrimSpace(strings.ReplaceAll(s, "\n\t", "\n")) // // fmt.Println(s) } diff --git a/src/html/template/js.go b/src/html/template/js.go index 33a18b4186..2291f47c33 100644 --- a/src/html/template/js.go +++ b/src/html/template/js.go @@ -172,7 +172,7 @@ func jsValEscaper(args ...interface{}) string { // turning into // x//* error marshaling y: // second line of error message */null - return fmt.Sprintf(" /* %s */null ", strings.Replace(err.Error(), "*/", "* /", -1)) + return fmt.Sprintf(" /* %s */null ", strings.ReplaceAll(err.Error(), "*/", "* /")) } // TODO: maybe post-process output to prevent it from containing diff --git a/src/html/template/url.go b/src/html/template/url.go index f0516300de..8a4f727e50 100644 --- a/src/html/template/url.go +++ b/src/html/template/url.go @@ -156,7 +156,7 @@ func srcsetFilterAndEscaper(args ...interface{}) string { s = b.String() } // Additionally, commas separate one source from another. - return strings.Replace(s, ",", "%2c", -1) + return strings.ReplaceAll(s, ",", "%2c") } var b bytes.Buffer diff --git a/src/mime/multipart/formdata_test.go b/src/mime/multipart/formdata_test.go index 2d6a830cb6..105a82c417 100644 --- a/src/mime/multipart/formdata_test.go +++ b/src/mime/multipart/formdata_test.go @@ -13,7 +13,7 @@ import ( ) func TestReadForm(t *testing.T) { - b := strings.NewReader(strings.Replace(message, "\n", "\r\n", -1)) + b := strings.NewReader(strings.ReplaceAll(message, "\n", "\r\n")) r := NewReader(b, boundary) f, err := r.ReadForm(25) if err != nil { @@ -39,7 +39,7 @@ func TestReadForm(t *testing.T) { } func TestReadFormWithNamelessFile(t *testing.T) { - b := strings.NewReader(strings.Replace(messageWithFileWithoutName, "\n", "\r\n", -1)) + b := strings.NewReader(strings.ReplaceAll(messageWithFileWithoutName, "\n", "\r\n")) r := NewReader(b, boundary) f, err := r.ReadForm(25) if err != nil { @@ -54,7 +54,7 @@ func TestReadFormWithNamelessFile(t *testing.T) { func TestReadFormWithTextContentType(t *testing.T) { // From https://github.com/golang/go/issues/24041 - b := strings.NewReader(strings.Replace(messageWithTextContentType, "\n", "\r\n", -1)) + b := strings.NewReader(strings.ReplaceAll(messageWithTextContentType, "\n", "\r\n")) r := NewReader(b, boundary) f, err := r.ReadForm(25) if err != nil { @@ -184,7 +184,7 @@ Content-Disposition: form-data; name="largetext" --MyBoundary-- ` - testBody := strings.Replace(message, "\n", "\r\n", -1) + testBody := strings.ReplaceAll(message, "\n", "\r\n") testCases := []struct { name string maxMemory int64 diff --git a/src/mime/multipart/multipart_test.go b/src/mime/multipart/multipart_test.go index abe1cc8e77..7bf606765c 100644 --- a/src/mime/multipart/multipart_test.go +++ b/src/mime/multipart/multipart_test.go @@ -105,7 +105,7 @@ never read data useless trailer ` - testBody = strings.Replace(testBody, "\n", sep, -1) + testBody = strings.ReplaceAll(testBody, "\n", sep) return strings.Replace(testBody, "[longline]", longLine, 1) } @@ -151,7 +151,7 @@ func testMultipart(t *testing.T, r io.Reader, onlyNewlines bool) { adjustNewlines := func(s string) string { if onlyNewlines { - return strings.Replace(s, "\r\n", "\n", -1) + return strings.ReplaceAll(s, "\r\n", "\n") } return s } @@ -299,7 +299,7 @@ foo-bar: baz Oh no, premature EOF! ` - body := strings.Replace(testBody, "\n", "\r\n", -1) + body := strings.ReplaceAll(testBody, "\n", "\r\n") bodyReader := strings.NewReader(body) r := NewReader(bodyReader, "MyBoundary") diff --git a/src/net/http/cgi/child.go b/src/net/http/cgi/child.go index da12ac3498..10325c2eb5 100644 --- a/src/net/http/cgi/child.go +++ b/src/net/http/cgi/child.go @@ -86,7 +86,7 @@ func RequestFromMap(params map[string]string) (*http.Request, error) { if !strings.HasPrefix(k, "HTTP_") || k == "HTTP_HOST" { continue } - r.Header.Add(strings.Replace(k[5:], "_", "-", -1), v) + r.Header.Add(strings.ReplaceAll(k[5:], "_", "-"), v) } // TODO: cookies. parsing them isn't exported, though. diff --git a/src/net/http/httputil/dump_test.go b/src/net/http/httputil/dump_test.go index 5703a7fb86..63312dd885 100644 --- a/src/net/http/httputil/dump_test.go +++ b/src/net/http/httputil/dump_test.go @@ -370,7 +370,7 @@ func TestDumpResponse(t *testing.T) { } got := string(gotb) got = strings.TrimSpace(got) - got = strings.Replace(got, "\r", "", -1) + got = strings.ReplaceAll(got, "\r", "") if got != tt.want { t.Errorf("%d.\nDumpResponse got:\n%s\n\nWant:\n%s\n", i, got, tt.want) diff --git a/src/net/http/readrequest_test.go b/src/net/http/readrequest_test.go index 18eed345a8..517a8189e1 100644 --- a/src/net/http/readrequest_test.go +++ b/src/net/http/readrequest_test.go @@ -438,7 +438,7 @@ func TestReadRequest(t *testing.T) { // reqBytes treats req as a request (with \n delimiters) and returns it with \r\n delimiters, // ending in \r\n\r\n func reqBytes(req string) []byte { - return []byte(strings.Replace(strings.TrimSpace(req), "\n", "\r\n", -1) + "\r\n\r\n") + return []byte(strings.ReplaceAll(strings.TrimSpace(req), "\n", "\r\n") + "\r\n\r\n") } var badRequestTests = []struct { diff --git a/src/net/http/request_test.go b/src/net/http/request_test.go index 7a83ae5b1c..e8005571df 100644 --- a/src/net/http/request_test.go +++ b/src/net/http/request_test.go @@ -878,7 +878,7 @@ func testMissingFile(t *testing.T, req *Request) { } func newTestMultipartRequest(t *testing.T) *Request { - b := strings.NewReader(strings.Replace(message, "\n", "\r\n", -1)) + b := strings.NewReader(strings.ReplaceAll(message, "\n", "\r\n")) req, err := NewRequest("POST", "/", b) if err != nil { t.Fatal("NewRequest:", err) @@ -970,8 +970,8 @@ Content-Disposition: form-data; name="textb" ` func benchmarkReadRequest(b *testing.B, request string) { - request = request + "\n" // final \n - request = strings.Replace(request, "\n", "\r\n", -1) // expand \n to \r\n + request = request + "\n" // final \n + request = strings.ReplaceAll(request, "\n", "\r\n") // expand \n to \r\n b.SetBytes(int64(len(request))) r := bufio.NewReader(&infiniteReader{buf: []byte(request)}) b.ReportAllocs() diff --git a/src/net/http/serve_test.go b/src/net/http/serve_test.go index 8dae95678d..b12fcf4f9e 100644 --- a/src/net/http/serve_test.go +++ b/src/net/http/serve_test.go @@ -130,7 +130,7 @@ func (c *testConn) Close() error { // reqBytes treats req as a request (with \n delimiters) and returns it with \r\n delimiters, // ending in \r\n\r\n func reqBytes(req string) []byte { - return []byte(strings.Replace(strings.TrimSpace(req), "\n", "\r\n", -1) + "\r\n\r\n") + return []byte(strings.ReplaceAll(strings.TrimSpace(req), "\n", "\r\n") + "\r\n\r\n") } type handlerTest struct { diff --git a/src/net/lookup_windows_test.go b/src/net/lookup_windows_test.go index cebb2d0558..d3748f28c3 100644 --- a/src/net/lookup_windows_test.go +++ b/src/net/lookup_windows_test.go @@ -150,7 +150,7 @@ func nslookup(qtype, name string) (string, error) { if err := cmd.Run(); err != nil { return "", err } - r := strings.Replace(out.String(), "\r\n", "\n", -1) + r := strings.ReplaceAll(out.String(), "\r\n", "\n") // nslookup stderr output contains also debug information such as // "Non-authoritative answer" and it doesn't return the correct errcode if strings.Contains(err.String(), "can't find") { diff --git a/src/net/mail/message_test.go b/src/net/mail/message_test.go index b19da52c42..14ac9192a4 100644 --- a/src/net/mail/message_test.go +++ b/src/net/mail/message_test.go @@ -668,9 +668,9 @@ func TestAddressParser(t *testing.T) { switch charset { case "iso-8859-15": - in = bytes.Replace(in, []byte("\xf6"), []byte("ö"), -1) + in = bytes.ReplaceAll(in, []byte("\xf6"), []byte("ö")) case "windows-1252": - in = bytes.Replace(in, []byte("\xe9"), []byte("é"), -1) + in = bytes.ReplaceAll(in, []byte("\xe9"), []byte("é")) } return bytes.NewReader(in), nil diff --git a/src/net/net_windows_test.go b/src/net/net_windows_test.go index 8dfd312980..8aa719f433 100644 --- a/src/net/net_windows_test.go +++ b/src/net/net_windows_test.go @@ -571,7 +571,7 @@ func TestInterfaceHardwareAddrWithGetmac(t *testing.T) { // skip these return } - addr = strings.Replace(addr, "-", ":", -1) + addr = strings.ReplaceAll(addr, "-", ":") cname := getValue("Connection Name") want[cname] = addr group = make(map[string]string) diff --git a/src/net/url/example_test.go b/src/net/url/example_test.go index d8eb6dcd20..ad67f5328a 100644 --- a/src/net/url/example_test.go +++ b/src/net/url/example_test.go @@ -219,5 +219,5 @@ func toJSON(m interface{}) string { if err != nil { log.Fatal(err) } - return strings.Replace(string(js), ",", ", ", -1) + return strings.ReplaceAll(string(js), ",", ", ") } diff --git a/src/net/url/url_test.go b/src/net/url/url_test.go index 5d3f91248f..7c4ada245a 100644 --- a/src/net/url/url_test.go +++ b/src/net/url/url_test.go @@ -848,18 +848,18 @@ func TestUnescape(t *testing.T) { in := tt.in out := tt.out if strings.Contains(tt.in, "+") { - in = strings.Replace(tt.in, "+", "%20", -1) + in = strings.ReplaceAll(tt.in, "+", "%20") actual, err := PathUnescape(in) if actual != tt.out || (err != nil) != (tt.err != nil) { t.Errorf("PathUnescape(%q) = %q, %s; want %q, %s", in, actual, err, tt.out, tt.err) } if tt.err == nil { - s, err := QueryUnescape(strings.Replace(tt.in, "+", "XXX", -1)) + s, err := QueryUnescape(strings.ReplaceAll(tt.in, "+", "XXX")) if err != nil { continue } in = tt.in - out = strings.Replace(s, "XXX", "+", -1) + out = strings.ReplaceAll(s, "XXX", "+") } } diff --git a/src/os/path_windows_test.go b/src/os/path_windows_test.go index 00a3e63bf3..f1745ad132 100644 --- a/src/os/path_windows_test.go +++ b/src/os/path_windows_test.go @@ -38,10 +38,10 @@ func TestFixLongPath(t *testing.T) { {`\\?\c:\long\foo.txt`, `\\?\c:\long\foo.txt`}, {`\\?\c:\long/foo.txt`, `\\?\c:\long/foo.txt`}, } { - in := strings.Replace(test.in, "long", veryLong, -1) - want := strings.Replace(test.want, "long", veryLong, -1) + in := strings.ReplaceAll(test.in, "long", veryLong) + want := strings.ReplaceAll(test.want, "long", veryLong) if got := os.FixLongPath(in); got != want { - got = strings.Replace(got, veryLong, "long", -1) + got = strings.ReplaceAll(got, veryLong, "long") t.Errorf("fixLongPath(%q) = %q; want %q", test.in, got, test.want) } } diff --git a/src/path/filepath/path.go b/src/path/filepath/path.go index 1508137a33..aba1717e7d 100644 --- a/src/path/filepath/path.go +++ b/src/path/filepath/path.go @@ -166,7 +166,7 @@ func ToSlash(path string) string { if Separator == '/' { return path } - return strings.Replace(path, string(Separator), "/", -1) + return strings.ReplaceAll(path, string(Separator), "/") } // FromSlash returns the result of replacing each slash ('/') character @@ -176,7 +176,7 @@ func FromSlash(path string) string { if Separator == '/' { return path } - return strings.Replace(path, "/", string(Separator), -1) + return strings.ReplaceAll(path, "/", string(Separator)) } // SplitList splits a list of paths joined by the OS-specific ListSeparator, diff --git a/src/path/filepath/path_test.go b/src/path/filepath/path_test.go index a221a3d4fa..e1b5ad1d40 100644 --- a/src/path/filepath/path_test.go +++ b/src/path/filepath/path_test.go @@ -1062,7 +1062,7 @@ func TestAbs(t *testing.T) { } for _, path := range absTests { - path = strings.Replace(path, "$", root, -1) + path = strings.ReplaceAll(path, "$", root) info, err := os.Stat(path) if err != nil { t.Errorf("%s: %s", path, err) diff --git a/src/path/filepath/path_windows.go b/src/path/filepath/path_windows.go index 519b6ebc32..6a144d9e0b 100644 --- a/src/path/filepath/path_windows.go +++ b/src/path/filepath/path_windows.go @@ -100,7 +100,7 @@ func splitList(path string) []string { // Remove quotes. for i, s := range list { - list[i] = strings.Replace(s, `"`, ``, -1) + list[i] = strings.ReplaceAll(s, `"`, ``) } return list diff --git a/src/path/filepath/path_windows_test.go b/src/path/filepath/path_windows_test.go index e36a3c9b64..63eab18116 100644 --- a/src/path/filepath/path_windows_test.go +++ b/src/path/filepath/path_windows_test.go @@ -431,7 +431,7 @@ func TestToNorm(t *testing.T) { t.Fatal(err) } - err = os.MkdirAll(strings.Replace(testPath, "{{tmp}}", ctmp, -1), 0777) + err = os.MkdirAll(strings.ReplaceAll(testPath, "{{tmp}}", ctmp), 0777) if err != nil { t.Fatal(err) } diff --git a/src/runtime/pprof/internal/profile/profile.go b/src/runtime/pprof/internal/profile/profile.go index 64c3e3f054..863bd403a4 100644 --- a/src/runtime/pprof/internal/profile/profile.go +++ b/src/runtime/pprof/internal/profile/profile.go @@ -200,7 +200,7 @@ var libRx = regexp.MustCompile(`([.]so$|[.]so[._][0-9]+)`) // first. func (p *Profile) setMain() { for i := 0; i < len(p.Mapping); i++ { - file := strings.TrimSpace(strings.Replace(p.Mapping[i].File, "(deleted)", "", -1)) + file := strings.TrimSpace(strings.ReplaceAll(p.Mapping[i].File, "(deleted)", "")) if len(file) == 0 { continue } diff --git a/src/runtime/pprof/pprof_test.go b/src/runtime/pprof/pprof_test.go index 126ba50054..593924183f 100644 --- a/src/runtime/pprof/pprof_test.go +++ b/src/runtime/pprof/pprof_test.go @@ -602,7 +602,7 @@ func TestBlockProfile(t *testing.T) { } for _, test := range tests { - if !regexp.MustCompile(strings.Replace(test.re, "\t", "\t+", -1)).MatchString(prof) { + if !regexp.MustCompile(strings.ReplaceAll(test.re, "\t", "\t+")).MatchString(prof) { t.Errorf("Bad %v entry, expect:\n%v\ngot:\n%v", test.name, test.re, prof) } } diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 1ed6b39303..26f507159b 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -490,7 +490,7 @@ func TestGdbConst(t *testing.T) { } got, _ := exec.Command("gdb", args...).CombinedOutput() - sgot := strings.Replace(string(got), "\r\n", "\n", -1) + sgot := strings.ReplaceAll(string(got), "\r\n", "\n") t.Logf("output %q", sgot) diff --git a/src/strings/example_test.go b/src/strings/example_test.go index 607e4a0a70..103ef51f29 100644 --- a/src/strings/example_test.go +++ b/src/strings/example_test.go @@ -199,7 +199,7 @@ func ExampleRepeat() { func ExampleReplace() { fmt.Println(strings.Replace("oink oink oink", "k", "ky", 2)) - fmt.Println(strings.Replace("oink oink oink", "oink", "moo", -1)) + fmt.Println(strings.ReplaceAll("oink oink oink", "oink", "moo")) // Output: // oinky oinky oink // moo moo moo diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 9af3909b35..29803c06e2 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -594,8 +594,8 @@ func TestBRun(t *T) { func makeRegexp(s string) string { s = regexp.QuoteMeta(s) - s = strings.Replace(s, ":NNN:", `:\d\d\d:`, -1) - s = strings.Replace(s, "N\\.NNs", `\d*\.\d*s`, -1) + s = strings.ReplaceAll(s, ":NNN:", `:\d\d\d:`) + s = strings.ReplaceAll(s, "N\\.NNs", `\d*\.\d*s`) return s } diff --git a/src/text/template/exec.go b/src/text/template/exec.go index 214f72d51b..1d04c2982f 100644 --- a/src/text/template/exec.go +++ b/src/text/template/exec.go @@ -102,7 +102,7 @@ func (s *state) at(node parse.Node) { // doublePercent returns the string with %'s replaced by %%, if necessary, // so it can be used safely inside a Printf format string. func doublePercent(str string) string { - return strings.Replace(str, "%", "%%", -1) + return strings.ReplaceAll(str, "%", "%%") } // TODO: It would be nice if ExecError was more broken down, but -- cgit v1.3-5-g9baa From 06afc8b152bc01e1b4f5dce074bae531dd29a9b9 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 19 Sep 2018 16:42:13 -0400 Subject: runtime: simplify the control flow in sweepone Ending a loop with a break is confusing. Rewrite the loop so the default behavior is to loop and then do the "post-loop" work outside of the loop. Change-Id: Ie49b4132541dfb5124c31a8163f2c883aa4abc75 Reviewed-on: https://go-review.googlesource.com/138155 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgcsweep.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index ecfdee59f4..5cdede002a 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -88,10 +88,11 @@ func sweepone() uintptr { } atomic.Xadd(&mheap_.sweepers, +1) - npages := ^uintptr(0) + // Find a span to sweep. + var s *mspan sg := mheap_.sweepgen for { - s := mheap_.sweepSpans[1-sg/2%2].pop() + s = mheap_.sweepSpans[1-sg/2%2].pop() if s == nil { atomic.Store(&mheap_.sweepdone, 1) break @@ -106,9 +107,14 @@ func sweepone() uintptr { } continue } - if s.sweepgen != sg-2 || !atomic.Cas(&s.sweepgen, sg-2, sg-1) { - continue + if s.sweepgen == sg-2 && atomic.Cas(&s.sweepgen, sg-2, sg-1) { + break } + } + + // Sweep the span we found. + npages := ^uintptr(0) + if s != nil { npages = s.npages if !s.sweep(false) { // Span is still in-use, so this returned no @@ -116,7 +122,6 @@ func sweepone() uintptr { // move to the swept in-use list. npages = 0 } - break } // Decrement the number of active sweepers and if this is the -- cgit v1.3-5-g9baa From bf8e6b70276ddfd3e070964542020fe782c1eec8 Mon Sep 17 00:00:00 2001 From: Clément Chigot Date: Thu, 27 Sep 2018 15:26:18 +0200 Subject: go/build, runtime/internal/sys: add GOOS=aix This is the first commit of a series that will add AIX as an operating system target for ppc64 architecture. Updates #25893 Change-Id: I865b67a9c98277c11c1a56107be404ac5253277d Reviewed-on: https://go-review.googlesource.com/138115 Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/go/build/syslist.go | 2 +- src/runtime/internal/sys/zgoos_aix.go | 21 +++++++++++++++++++++ src/runtime/internal/sys/zgoos_android.go | 1 + src/runtime/internal/sys/zgoos_darwin.go | 1 + src/runtime/internal/sys/zgoos_dragonfly.go | 1 + src/runtime/internal/sys/zgoos_freebsd.go | 1 + src/runtime/internal/sys/zgoos_linux.go | 1 + src/runtime/internal/sys/zgoos_nacl.go | 1 + src/runtime/internal/sys/zgoos_netbsd.go | 1 + src/runtime/internal/sys/zgoos_openbsd.go | 1 + src/runtime/internal/sys/zgoos_plan9.go | 1 + src/runtime/internal/sys/zgoos_solaris.go | 1 + src/runtime/internal/sys/zgoos_windows.go | 1 + src/runtime/internal/sys/zgoos_zos.go | 1 + 14 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 src/runtime/internal/sys/zgoos_aix.go (limited to 'src/runtime') diff --git a/src/go/build/syslist.go b/src/go/build/syslist.go index d7938fad54..597212b6d0 100644 --- a/src/go/build/syslist.go +++ b/src/go/build/syslist.go @@ -4,5 +4,5 @@ package build -const goosList = "android darwin dragonfly freebsd js linux nacl netbsd openbsd plan9 solaris windows zos " +const goosList = "aix android darwin dragonfly freebsd js linux nacl netbsd openbsd plan9 solaris windows zos " const goarchList = "386 amd64 amd64p32 arm armbe arm64 arm64be ppc64 ppc64le mips mipsle mips64 mips64le mips64p32 mips64p32le ppc riscv riscv64 s390 s390x sparc sparc64 wasm " diff --git a/src/runtime/internal/sys/zgoos_aix.go b/src/runtime/internal/sys/zgoos_aix.go new file mode 100644 index 0000000000..9ce5b3434f --- /dev/null +++ b/src/runtime/internal/sys/zgoos_aix.go @@ -0,0 +1,21 @@ +// Code generated by gengoos.go using 'go generate'. DO NOT EDIT. + +// +build aix + +package sys + +const GOOS = `aix` + +const GoosAndroid = 0 +const GoosAix = 1 +const GoosDarwin = 0 +const GoosDragonfly = 0 +const GoosFreebsd = 0 +const GoosLinux = 0 +const GoosNacl = 0 +const GoosNetbsd = 0 +const GoosOpenbsd = 0 +const GoosPlan9 = 0 +const GoosSolaris = 0 +const GoosWindows = 0 +const GoosZos = 0 diff --git a/src/runtime/internal/sys/zgoos_android.go b/src/runtime/internal/sys/zgoos_android.go index bfdc37792e..36a5768ab6 100644 --- a/src/runtime/internal/sys/zgoos_android.go +++ b/src/runtime/internal/sys/zgoos_android.go @@ -7,6 +7,7 @@ package sys const GOOS = `android` const GoosAndroid = 1 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_darwin.go b/src/runtime/internal/sys/zgoos_darwin.go index 1c4667f6de..10c0e88e9a 100644 --- a/src/runtime/internal/sys/zgoos_darwin.go +++ b/src/runtime/internal/sys/zgoos_darwin.go @@ -7,6 +7,7 @@ package sys const GOOS = `darwin` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 1 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_dragonfly.go b/src/runtime/internal/sys/zgoos_dragonfly.go index 728bf6abe8..5cb47cb84e 100644 --- a/src/runtime/internal/sys/zgoos_dragonfly.go +++ b/src/runtime/internal/sys/zgoos_dragonfly.go @@ -7,6 +7,7 @@ package sys const GOOS = `dragonfly` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 1 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_freebsd.go b/src/runtime/internal/sys/zgoos_freebsd.go index a8d659169b..470406ce5f 100644 --- a/src/runtime/internal/sys/zgoos_freebsd.go +++ b/src/runtime/internal/sys/zgoos_freebsd.go @@ -7,6 +7,7 @@ package sys const GOOS = `freebsd` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 1 diff --git a/src/runtime/internal/sys/zgoos_linux.go b/src/runtime/internal/sys/zgoos_linux.go index 289400c612..76235b748c 100644 --- a/src/runtime/internal/sys/zgoos_linux.go +++ b/src/runtime/internal/sys/zgoos_linux.go @@ -8,6 +8,7 @@ package sys const GOOS = `linux` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_nacl.go b/src/runtime/internal/sys/zgoos_nacl.go index 3fedb0a2c3..6d28b59667 100644 --- a/src/runtime/internal/sys/zgoos_nacl.go +++ b/src/runtime/internal/sys/zgoos_nacl.go @@ -7,6 +7,7 @@ package sys const GOOS = `nacl` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_netbsd.go b/src/runtime/internal/sys/zgoos_netbsd.go index 3346e3711c..ef8d938ddb 100644 --- a/src/runtime/internal/sys/zgoos_netbsd.go +++ b/src/runtime/internal/sys/zgoos_netbsd.go @@ -7,6 +7,7 @@ package sys const GOOS = `netbsd` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_openbsd.go b/src/runtime/internal/sys/zgoos_openbsd.go index 13c0323249..2e43847396 100644 --- a/src/runtime/internal/sys/zgoos_openbsd.go +++ b/src/runtime/internal/sys/zgoos_openbsd.go @@ -7,6 +7,7 @@ package sys const GOOS = `openbsd` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_plan9.go b/src/runtime/internal/sys/zgoos_plan9.go index 6b2e977b5e..ed598dcaac 100644 --- a/src/runtime/internal/sys/zgoos_plan9.go +++ b/src/runtime/internal/sys/zgoos_plan9.go @@ -7,6 +7,7 @@ package sys const GOOS = `plan9` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_solaris.go b/src/runtime/internal/sys/zgoos_solaris.go index cbf70f079a..fe690df6c2 100644 --- a/src/runtime/internal/sys/zgoos_solaris.go +++ b/src/runtime/internal/sys/zgoos_solaris.go @@ -7,6 +7,7 @@ package sys const GOOS = `solaris` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_windows.go b/src/runtime/internal/sys/zgoos_windows.go index 70839ca793..ea7c43bdf4 100644 --- a/src/runtime/internal/sys/zgoos_windows.go +++ b/src/runtime/internal/sys/zgoos_windows.go @@ -7,6 +7,7 @@ package sys const GOOS = `windows` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 diff --git a/src/runtime/internal/sys/zgoos_zos.go b/src/runtime/internal/sys/zgoos_zos.go index ecf449f703..d4027cf876 100644 --- a/src/runtime/internal/sys/zgoos_zos.go +++ b/src/runtime/internal/sys/zgoos_zos.go @@ -7,6 +7,7 @@ package sys const GOOS = `zos` const GoosAndroid = 0 +const GoosAix = 0 const GoosDarwin = 0 const GoosDragonfly = 0 const GoosFreebsd = 0 -- cgit v1.3-5-g9baa From 2d23ece135076698ea4724b02f07d71d1f2145fb Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sat, 22 Sep 2018 15:59:01 -0400 Subject: runtime: remove redundant locking in mcache.refill mcache.refill acquires g.m.locks, which is pointless because the caller itself absolutely must have done so already to prevent ownership of mcache from shifting. Also, mcache.refill's documentation is generally a bit out-of-date, so this cleans this up. Change-Id: Idc8de666fcaf3c3d96006bd23a8f307539587d6c Reviewed-on: https://go-review.googlesource.com/138195 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mcache.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index d0b007f915..8486f69569 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -101,12 +101,12 @@ func freemcache(c *mcache) { }) } -// Gets a span that has a free object in it and assigns it -// to be the cached span for the given sizeclass. Returns this span. +// refill acquires a new span of span class spc for c. This span will +// have at least one free object. The current span in c must be full. +// +// Must run in a non-preemptible context since otherwise the owner of +// c could change. func (c *mcache) refill(spc spanClass) { - _g_ := getg() - - _g_.m.locks++ // Return the current cached span to the central lists. s := c.alloc[spc] @@ -129,7 +129,6 @@ func (c *mcache) refill(spc spanClass) { } c.alloc[spc] = s - _g_.m.locks-- } func (c *mcache) releaseAll() { -- cgit v1.3-5-g9baa From 01e6cfc2a0b2f9e363c6e305b14f3393d06b13b8 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sun, 23 Sep 2018 19:12:15 -0400 Subject: runtime: don't call mcache.refill on systemstack mcache.refill doesn't need to run on the system stack; it just needs to be non-preemptible. Its only caller, mcache.nextFree, also needs to be non-preemptible, so we can remove the unnecessary systemstack switch. Change-Id: Iba5b3f4444855f1dc134485ba588efff3b54c426 Reviewed-on: https://go-review.googlesource.com/138196 Run-TryBot: Austin Clements Reviewed-by: Rick Hudson TryBot-Result: Gobot Gobot --- src/runtime/malloc.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index dd88d353dd..5755c9e263 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -733,6 +733,9 @@ func nextFreeFast(s *mspan) gclinkptr { // weight allocation. If it is a heavy weight allocation the caller must // determine whether a new GC cycle needs to be started or if the GC is active // whether this goroutine needs to assist the GC. +// +// Must run in a non-preemptible context since otherwise the owner of +// c could change. func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bool) { s = c.alloc[spc] shouldhelpgc = false @@ -743,9 +746,7 @@ func (c *mcache) nextFree(spc spanClass) (v gclinkptr, s *mspan, shouldhelpgc bo println("runtime: s.allocCount=", s.allocCount, "s.nelems=", s.nelems) throw("s.allocCount != s.nelems && freeIndex == s.nelems") } - systemstack(func() { - c.refill(spc) - }) + c.refill(spc) shouldhelpgc = true s = c.alloc[spc] -- cgit v1.3-5-g9baa From ef5037398385ff52b17af325a0ad82017bd65820 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 25 Sep 2018 15:54:11 -0700 Subject: reflect: ensure correct scanning of return values During a call to a reflect-generated function or method (via makeFuncStub or methodValueCall), when should we scan the return values? When we're starting a reflect call, the space on the stack for the return values is not initialized yet, as it contains whatever junk was on the stack of the caller at the time. The return space must not be scanned during a GC. When we're finishing a reflect call, the return values are initialized, and must be scanned during a GC to make sure that any pointers in the return values are found and their referents retained. When the GC stack walk comes across a reflect call in progress on the stack, it needs to know whether to scan the results or not. It doesn't know the progress of the reflect call, so it can't decide by itself. The reflect package needs to tell it. This CL adds another slot in the frame of makeFuncStub and methodValueCall so we can put a boolean in there which tells the runtime whether to scan the results or not. This CL also adds the args length to reflectMethodValue so the runtime can restrict its scanning to only the args section (not the results) if the reflect package says the results aren't ready yet. Do a delicate dance in the reflect package to set the "results are valid" bit. We need to make sure we set the bit only after we've copied the results back to the stack. But we must set the bit before we drop reflect's copy of the results. Otherwise, we might have a state where (temporarily) no one has a live copy of the results. That's the state we were observing in issue #27695 before this CL. The bitmap used by the runtime currently contains only the args. (Actually, it contains all the bits, but the size is set so we use only the args portion.) This is safe for early in a reflect call, but unsafe late in a reflect call. The test issue27695.go demonstrates this unsafety. We change the bitmap to always include both args and results, and decide at runtime which portion to use. issue27695.go only has a test for method calls. Function calls were ok because there wasn't a safepoint between when reflect dropped its copy of the return values and when the caller is resumed. This may change when we introduce safepoints everywhere. This truncate-to-only-the-args was part of CL 9888 (in 2015). That part of the CL fixed the problem demonstrated in issue27695b.go but introduced the problem demonstrated in issue27695.go. TODO, in another CL: simplify FuncLayout and its test. stack return value is now identical to frametype.ptrdata + frametype.gcdata. Fixes #27695 Change-Id: I2d49b34e34a82c6328b34f02610587a291b25c5f Reviewed-on: https://go-review.googlesource.com/137440 Run-TryBot: Keith Randall TryBot-Result: Gobot Gobot Reviewed-by: Austin Clements --- src/cmd/compile/internal/types/type.go | 2 +- src/reflect/all_test.go | 4 +-- src/reflect/asm_386.s | 10 ++++-- src/reflect/asm_amd64.s | 10 ++++-- src/reflect/asm_amd64p32.s | 10 ++++-- src/reflect/asm_arm.s | 12 +++++-- src/reflect/asm_arm64.s | 10 ++++-- src/reflect/asm_mips64x.s | 10 ++++-- src/reflect/asm_mipsx.s | 10 ++++-- src/reflect/asm_ppc64x.s | 10 ++++-- src/reflect/asm_s390x.s | 10 ++++-- src/reflect/asm_wasm.s | 10 ++++-- src/reflect/export_test.go | 4 +-- src/reflect/makefunc.go | 27 +++++++------- src/reflect/type.go | 14 +++----- src/reflect/value.go | 61 +++++++++++++++++++++++--------- src/runtime/traceback.go | 13 +++++-- test/fixedbugs/issue27695.go | 62 ++++++++++++++++++++++++++++++++ test/fixedbugs/issue27695b.go | 64 ++++++++++++++++++++++++++++++++++ 19 files changed, 288 insertions(+), 65 deletions(-) create mode 100644 test/fixedbugs/issue27695.go create mode 100644 test/fixedbugs/issue27695b.go (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index d367cd1944..25f8f826e6 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -817,7 +817,7 @@ func (t *Type) ChanArgs() *Type { return t.Extra.(ChanArgs).T } -// FuncArgs returns the channel type for TFUNCARGS type t. +// FuncArgs returns the func type for TFUNCARGS type t. func (t *Type) FuncArgs() *Type { t.wantEtype(TFUNCARGS) return t.Extra.(FuncArgs).T diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index c616b37008..5b8bbad383 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -5844,7 +5844,7 @@ func clobber() { type funcLayoutTest struct { rcvr, t Type size, argsize, retOffset uintptr - stack []byte // pointer bitmap: 1 is pointer, 0 is scalar (or uninitialized) + stack []byte // pointer bitmap: 1 is pointer, 0 is scalar gc []byte } @@ -5866,7 +5866,7 @@ func init() { 6 * PtrSize, 4 * PtrSize, 4 * PtrSize, - []byte{1, 0, 1}, + []byte{1, 0, 1, 0, 1}, []byte{1, 0, 1, 0, 1}, }) diff --git a/src/reflect/asm_386.s b/src/reflect/asm_386.s index d827360006..e79beb6dc9 100644 --- a/src/reflect/asm_386.s +++ b/src/reflect/asm_386.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callMethod(SB) RET diff --git a/src/reflect/asm_amd64.s b/src/reflect/asm_amd64.s index 1272c489de..fb28ab87f1 100644 --- a/src/reflect/asm_amd64.s +++ b/src/reflect/asm_amd64.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX MOVQ CX, 8(SP) + MOVB $0, 24(SP) + LEAQ 24(SP), AX + MOVQ AX, 16(SP) CALL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVQ DX, 0(SP) LEAQ argframe+0(FP), CX MOVQ CX, 8(SP) + MOVB $0, 24(SP) + LEAQ 24(SP), AX + MOVQ AX, 16(SP) CALL ·callMethod(SB) RET diff --git a/src/reflect/asm_amd64p32.s b/src/reflect/asm_amd64p32.s index d827360006..e79beb6dc9 100644 --- a/src/reflect/asm_amd64p32.s +++ b/src/reflect/asm_amd64p32.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVL DX, 0(SP) LEAL argframe+0(FP), CX MOVL CX, 4(SP) + MOVB $0, 12(SP) + LEAL 12(SP), AX + MOVL AX, 8(SP) CALL ·callMethod(SB) RET diff --git a/src/reflect/asm_arm.s b/src/reflect/asm_arm.s index b721ed28c6..cd50d33918 100644 --- a/src/reflect/asm_arm.s +++ b/src/reflect/asm_arm.s @@ -9,11 +9,15 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW R7, 4(R13) MOVW $argframe+0(FP), R1 MOVW R1, 8(R13) + MOVW $0, R1 + MOVB R1, 16(R13) + ADD $16, R13, R1 + MOVW R1, 12(R13) BL ·callReflect(SB) RET @@ -21,10 +25,14 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No argsize here, gc generates argsize info at call site. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW R7, 4(R13) MOVW $argframe+0(FP), R1 MOVW R1, 8(R13) + MOVW $0, R1 + MOVB R1, 16(R13) + ADD $16, R13, R1 + MOVW R1, 12(R13) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_arm64.s b/src/reflect/asm_arm64.s index d1563709f2..28bb86c2a4 100644 --- a/src/reflect/asm_arm64.s +++ b/src/reflect/asm_arm64.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$24 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$40 NO_LOCAL_POINTERS MOVD R26, 8(RSP) MOVD $argframe+0(FP), R3 MOVD R3, 16(RSP) + MOVB $0, 32(RSP) + ADD $32, RSP, R3 + MOVD R3, 24(RSP) BL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$24 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$24 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$40 NO_LOCAL_POINTERS MOVD R26, 8(RSP) MOVD $argframe+0(FP), R3 MOVD R3, 16(RSP) + MOVB $0, 32(RSP) + ADD $32, RSP, R3 + MOVD R3, 24(RSP) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_mips64x.s b/src/reflect/asm_mips64x.s index 98afb52f6a..6f76685567 100644 --- a/src/reflect/asm_mips64x.s +++ b/src/reflect/asm_mips64x.s @@ -13,11 +13,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVV REGCTXT, 8(R29) MOVV $argframe+0(FP), R1 MOVV R1, 16(R29) + MOVB R0, 32(R29) + ADDV $32, R29, R1 + MOVV R1, 24(R29) JAL ·callReflect(SB) RET @@ -25,10 +28,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVV REGCTXT, 8(R29) MOVV $argframe+0(FP), R1 MOVV R1, 16(R29) + MOVB R0, 32(R29) + ADDV $32, R29, R1 + MOVV R1, 24(R29) JAL ·callMethod(SB) RET diff --git a/src/reflect/asm_mipsx.s b/src/reflect/asm_mipsx.s index b6df4e636e..5a5c53ef9f 100644 --- a/src/reflect/asm_mipsx.s +++ b/src/reflect/asm_mipsx.s @@ -13,11 +13,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW REGCTXT, 4(R29) MOVW $argframe+0(FP), R1 MOVW R1, 8(R29) + MOVB R0, 16(R29) + ADD $16, R29, R1 + MOVW R1, 12(R29) JAL ·callReflect(SB) RET @@ -25,10 +28,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$8 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$8 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 NO_LOCAL_POINTERS MOVW REGCTXT, 4(R29) MOVW $argframe+0(FP), R1 MOVW R1, 8(R29) + MOVB R0, 16(R29) + ADD $16, R29, R1 + MOVW R1, 12(R29) JAL ·callMethod(SB) RET diff --git a/src/reflect/asm_ppc64x.s b/src/reflect/asm_ppc64x.s index 42f57743e6..4609f6bb75 100644 --- a/src/reflect/asm_ppc64x.s +++ b/src/reflect/asm_ppc64x.s @@ -12,11 +12,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R11, FIXED_FRAME+0(R1) MOVD $argframe+0(FP), R3 MOVD R3, FIXED_FRAME+8(R1) + MOVB R0, FIXED_FRAME+24(R1) + ADD $FIXED_FRAME+24, R1, R3 + MOVD R3, FIXED_FRAME+16(R1) BL ·callReflect(SB) RET @@ -24,10 +27,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R11, FIXED_FRAME+0(R1) MOVD $argframe+0(FP), R3 MOVD R3, FIXED_FRAME+8(R1) + MOVB R0, FIXED_FRAME+24(R1) + ADD $FIXED_FRAME+24, R1, R3 + MOVD R3, FIXED_FRAME+16(R1) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_s390x.s b/src/reflect/asm_s390x.s index e6b86cfaa9..2ab5481c9b 100644 --- a/src/reflect/asm_s390x.s +++ b/src/reflect/asm_s390x.s @@ -9,11 +9,14 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here, runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R12, 8(R15) MOVD $argframe+0(FP), R3 MOVD R3, 16(R15) + MOVB R0, 32(R15) + ADD $32, R15, R3 + MOVD R3, 24(R15) BL ·callReflect(SB) RET @@ -21,10 +24,13 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD R12, 8(R15) MOVD $argframe+0(FP), R3 MOVD R3, 16(R15) + MOVB R0, 32(R15) + ADD $32, R15, R3 + MOVD R3, 24(R15) BL ·callMethod(SB) RET diff --git a/src/reflect/asm_wasm.s b/src/reflect/asm_wasm.s index 0f9b5aa130..627e295769 100644 --- a/src/reflect/asm_wasm.s +++ b/src/reflect/asm_wasm.s @@ -9,7 +9,7 @@ // See the comment on the declaration of makeFuncStub in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD CTXT, 0(SP) @@ -21,6 +21,9 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 I64Add I64Store $8 + MOVB $0, 24(SP) + MOVD $24(SP), 16(SP) + CALL ·callReflect(SB) RET @@ -28,7 +31,7 @@ TEXT ·makeFuncStub(SB),(NOSPLIT|WRAPPER),$16 // See the comment on the declaration of methodValueCall in makefunc.go // for more details. // No arg size here; runtime pulls arg map out of the func value. -TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 +TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$32 NO_LOCAL_POINTERS MOVD CTXT, 0(SP) @@ -40,5 +43,8 @@ TEXT ·methodValueCall(SB),(NOSPLIT|WRAPPER),$16 I64Add I64Store $8 + MOVB $0, 24(SP) + MOVD $24(SP), 16(SP) + CALL ·callMethod(SB) RET diff --git a/src/reflect/export_test.go b/src/reflect/export_test.go index 14a6981fde..3c47d6712f 100644 --- a/src/reflect/export_test.go +++ b/src/reflect/export_test.go @@ -25,9 +25,9 @@ func FuncLayout(t Type, rcvr Type) (frametype Type, argSize, retOffset uintptr, var ft *rtype var s *bitVector if rcvr != nil { - ft, argSize, retOffset, s, _ = funcLayout(t.(*rtype), rcvr.(*rtype)) + ft, argSize, retOffset, s, _ = funcLayout((*funcType)(unsafe.Pointer(t.(*rtype))), rcvr.(*rtype)) } else { - ft, argSize, retOffset, s, _ = funcLayout(t.(*rtype), nil) + ft, argSize, retOffset, s, _ = funcLayout((*funcType)(unsafe.Pointer(t.(*rtype))), nil) } frametype = ft for i := uint32(0); i < s.n; i++ { diff --git a/src/reflect/makefunc.go b/src/reflect/makefunc.go index 885966db6f..67dc4859b9 100644 --- a/src/reflect/makefunc.go +++ b/src/reflect/makefunc.go @@ -12,14 +12,15 @@ import ( // makeFuncImpl is the closure value implementing the function // returned by MakeFunc. -// The first two words of this type must be kept in sync with +// The first three words of this type must be kept in sync with // methodValue and runtime.reflectMethodValue. // Any changes should be reflected in all three. type makeFuncImpl struct { - code uintptr - stack *bitVector - typ *funcType - fn func([]Value) []Value + code uintptr + stack *bitVector // ptrmap for both args and results + argLen uintptr // just args + ftyp *funcType + fn func([]Value) []Value } // MakeFunc returns a new function of the given Type @@ -59,9 +60,9 @@ func MakeFunc(typ Type, fn func(args []Value) (results []Value)) Value { code := **(**uintptr)(unsafe.Pointer(&dummy)) // makeFuncImpl contains a stack map for use by the runtime - _, _, _, stack, _ := funcLayout(t, nil) + _, argLen, _, stack, _ := funcLayout(ftyp, nil) - impl := &makeFuncImpl{code: code, stack: stack, typ: ftyp, fn: fn} + impl := &makeFuncImpl{code: code, stack: stack, argLen: argLen, ftyp: ftyp, fn: fn} return Value{t, unsafe.Pointer(impl), flag(Func)} } @@ -73,12 +74,13 @@ func MakeFunc(typ Type, fn func(args []Value) (results []Value)) Value { // word in the passed-in argument frame. func makeFuncStub() -// The first two words of this type must be kept in sync with +// The first 3 words of this type must be kept in sync with // makeFuncImpl and runtime.reflectMethodValue. // Any changes should be reflected in all three. type methodValue struct { fn uintptr - stack *bitVector + stack *bitVector // ptrmap for both args and results + argLen uintptr // just args method int rcvr Value } @@ -101,7 +103,7 @@ func makeMethodValue(op string, v Value) Value { rcvr := Value{v.typ, v.ptr, fl} // v.Type returns the actual type of the method value. - funcType := v.Type().(*rtype) + ftyp := (*funcType)(unsafe.Pointer(v.Type().(*rtype))) // Indirect Go func value (dummy) to obtain // actual code address. (A Go func value is a pointer @@ -110,11 +112,12 @@ func makeMethodValue(op string, v Value) Value { code := **(**uintptr)(unsafe.Pointer(&dummy)) // methodValue contains a stack map for use by the runtime - _, _, _, stack, _ := funcLayout(funcType, nil) + _, argLen, _, stack, _ := funcLayout(ftyp, nil) fv := &methodValue{ fn: code, stack: stack, + argLen: argLen, method: int(v.flag) >> flagMethodShift, rcvr: rcvr, } @@ -124,7 +127,7 @@ func makeMethodValue(op string, v Value) Value { // but we want Interface() and other operations to fail early. methodReceiver(op, fv.rcvr, fv.method) - return Value{funcType, unsafe.Pointer(fv), v.flag&flagRO | flag(Func)} + return Value{&ftyp.rtype, unsafe.Pointer(fv), v.flag&flagRO | flag(Func)} } // methodValueCall is an assembly function that is the code half of diff --git a/src/reflect/type.go b/src/reflect/type.go index 6b0ce431a6..d8971d620e 100644 --- a/src/reflect/type.go +++ b/src/reflect/type.go @@ -3022,8 +3022,8 @@ func toType(t *rtype) Type { } type layoutKey struct { - t *rtype // function signature - rcvr *rtype // receiver type, or nil if none + ftyp *funcType // function signature + rcvr *rtype // receiver type, or nil if none } type layoutType struct { @@ -3042,7 +3042,7 @@ var layoutCache sync.Map // map[layoutKey]layoutType // The returned type exists only for GC, so we only fill out GC relevant info. // Currently, that's just size and the GC program. We also fill in // the name for possible debugging use. -func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uintptr, stk *bitVector, framePool *sync.Pool) { +func funcLayout(t *funcType, rcvr *rtype) (frametype *rtype, argSize, retOffset uintptr, stk *bitVector, framePool *sync.Pool) { if t.Kind() != Func { panic("reflect: funcLayout of non-func type") } @@ -3055,8 +3055,6 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin return lt.t, lt.argSize, lt.retOffset, lt.stack, lt.framePool } - tt := (*funcType)(unsafe.Pointer(t)) - // compute gc program & stack bitmap for arguments ptrmap := new(bitVector) var offset uintptr @@ -3071,19 +3069,18 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin } offset += ptrSize } - for _, arg := range tt.in() { + for _, arg := range t.in() { offset += -offset & uintptr(arg.align-1) addTypeBits(ptrmap, offset, arg) offset += arg.size } - argN := ptrmap.n argSize = offset if runtime.GOARCH == "amd64p32" { offset += -offset & (8 - 1) } offset += -offset & (ptrSize - 1) retOffset = offset - for _, res := range tt.out() { + for _, res := range t.out() { offset += -offset & uintptr(res.align-1) addTypeBits(ptrmap, offset, res) offset += res.size @@ -3104,7 +3101,6 @@ func funcLayout(t *rtype, rcvr *rtype) (frametype *rtype, argSize, retOffset uin } else { x.kind |= kindNoPointers } - ptrmap.n = argN var s string if rcvr != nil { diff --git a/src/reflect/value.go b/src/reflect/value.go index 854a5b153e..6d2862dbab 100644 --- a/src/reflect/value.go +++ b/src/reflect/value.go @@ -325,7 +325,7 @@ var callGC bool // for testing; see TestCallMethodJump func (v Value) call(op string, in []Value) []Value { // Get function pointer, type. - t := v.typ + t := (*funcType)(unsafe.Pointer(v.typ)) var ( fn unsafe.Pointer rcvr Value @@ -499,8 +499,13 @@ func (v Value) call(op string, in []Value) []Value { // NOTE: This function must be marked as a "wrapper" in the generated code, // so that the linker can make it work correctly for panic and recover. // The gc compilers know to do that for the name "reflect.callReflect". -func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { - ftyp := ctxt.typ +// +// ctxt is the "closure" generated by MakeFunc. +// frame is a pointer to the arguments to that closure on the stack. +// retValid points to a boolean which should be set when the results +// section of frame is set. +func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer, retValid *bool) { + ftyp := ctxt.ftyp f := ctxt.fn // Copy argument frame into Values. @@ -565,6 +570,16 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { } } + // Announce that the return values are valid. + // After this point the runtime can depend on the return values being valid. + *retValid = true + + // We have to make sure that the out slice lives at least until + // the runtime knows the return values are valid. Otherwise, the + // return values might not be scanned by anyone during a GC. + // (out would be dead, and the return slots not yet alive.) + runtime.KeepAlive(out) + // runtime.getArgInfo expects to be able to find ctxt on the // stack when it finds our caller, makeFuncStub. Make sure it // doesn't get garbage collected. @@ -578,7 +593,7 @@ func callReflect(ctxt *makeFuncImpl, frame unsafe.Pointer) { // The return value rcvrtype gives the method's actual receiver type. // The return value t gives the method type signature (without the receiver). // The return value fn is a pointer to the method code. -func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn unsafe.Pointer) { +func methodReceiver(op string, v Value, methodIndex int) (rcvrtype *rtype, t *funcType, fn unsafe.Pointer) { i := methodIndex if v.typ.Kind() == Interface { tt := (*interfaceType)(unsafe.Pointer(v.typ)) @@ -595,7 +610,7 @@ func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn } rcvrtype = iface.itab.typ fn = unsafe.Pointer(&iface.itab.fun[i]) - t = tt.typeOff(m.typ) + t = (*funcType)(unsafe.Pointer(tt.typeOff(m.typ))) } else { rcvrtype = v.typ ms := v.typ.exportedMethods() @@ -608,7 +623,7 @@ func methodReceiver(op string, v Value, methodIndex int) (rcvrtype, t *rtype, fn } ifn := v.typ.textOff(m.ifn) fn = unsafe.Pointer(&ifn) - t = v.typ.typeOff(m.mtyp) + t = (*funcType)(unsafe.Pointer(v.typ.typeOff(m.mtyp))) } return } @@ -647,25 +662,31 @@ func align(x, n uintptr) uintptr { // NOTE: This function must be marked as a "wrapper" in the generated code, // so that the linker can make it work correctly for panic and recover. // The gc compilers know to do that for the name "reflect.callMethod". -func callMethod(ctxt *methodValue, frame unsafe.Pointer) { +// +// ctxt is the "closure" generated by makeVethodValue. +// frame is a pointer to the arguments to that closure on the stack. +// retValid points to a boolean which should be set when the results +// section of frame is set. +func callMethod(ctxt *methodValue, frame unsafe.Pointer, retValid *bool) { rcvr := ctxt.rcvr rcvrtype, t, fn := methodReceiver("call", rcvr, ctxt.method) frametype, argSize, retOffset, _, framePool := funcLayout(t, rcvrtype) // Make a new frame that is one word bigger so we can store the receiver. - args := framePool.Get().(unsafe.Pointer) + // This space is used for both arguments and return values. + scratch := framePool.Get().(unsafe.Pointer) // Copy in receiver and rest of args. // Avoid constructing out-of-bounds pointers if there are no args. - storeRcvr(rcvr, args) + storeRcvr(rcvr, scratch) if argSize-ptrSize > 0 { - typedmemmovepartial(frametype, add(args, ptrSize, "argSize > ptrSize"), frame, ptrSize, argSize-ptrSize) + typedmemmovepartial(frametype, add(scratch, ptrSize, "argSize > ptrSize"), frame, ptrSize, argSize-ptrSize) } // Call. - // Call copies the arguments from args to the stack, calls fn, - // and then copies the results back into args. - call(frametype, fn, args, uint32(frametype.size), uint32(retOffset)) + // Call copies the arguments from scratch to the stack, calls fn, + // and then copies the results back into scratch. + call(frametype, fn, scratch, uint32(frametype.size), uint32(retOffset)) // Copy return values. On amd64p32, the beginning of return values // is 64-bit aligned, so the caller's frame layout (which doesn't have @@ -680,13 +701,19 @@ func callMethod(ctxt *methodValue, frame unsafe.Pointer) { } // This copies to the stack. Write barriers are not needed. memmove(add(frame, callerRetOffset, "frametype.size > retOffset"), - add(args, retOffset, "frametype.size > retOffset"), + add(scratch, retOffset, "frametype.size > retOffset"), frametype.size-retOffset) } - // Put the args scratch space back in the pool. - typedmemclr(frametype, args) - framePool.Put(args) + // Tell the runtime it can now depend on the return values + // being properly initialized. + *retValid = true + + // Clear the scratch space and put it back in the pool. + // This must happen after the statement above, so that the return + // values will always be scanned by someone. + typedmemclr(frametype, scratch) + framePool.Put(scratch) // See the comment in callReflect. runtime.KeepAlive(ctxt) diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 4c2010493a..8e104ae89e 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -571,8 +571,9 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // reflectMethodValue is a partial duplicate of reflect.makeFuncImpl // and reflect.methodValue. type reflectMethodValue struct { - fn uintptr - stack *bitvector // args bitmap + fn uintptr + stack *bitvector // ptrmap for both args and results + argLen uintptr // just args } // getArgInfoFast returns the argument frame information for a call to f. @@ -601,6 +602,7 @@ func getArgInfo(frame *stkframe, f funcInfo, needArgMap bool, ctxt *funcval) (ar // These take a *reflect.methodValue as their // context register. var mv *reflectMethodValue + var retValid bool if ctxt != nil { // This is not an actual call, but a // deferred call. The function value @@ -614,6 +616,10 @@ func getArgInfo(frame *stkframe, f funcInfo, needArgMap bool, ctxt *funcval) (ar // 0(SP). arg0 := frame.sp + sys.MinFrameSize mv = *(**reflectMethodValue)(unsafe.Pointer(arg0)) + // Figure out whether the return values are valid. + // Reflect will update this value after it copies + // in the return values. + retValid = *(*bool)(unsafe.Pointer(arg0 + 3*sys.PtrSize)) } if mv.fn != f.entry { print("runtime: confused by ", funcname(f), "\n") @@ -621,6 +627,9 @@ func getArgInfo(frame *stkframe, f funcInfo, needArgMap bool, ctxt *funcval) (ar } bv := mv.stack arglen = uintptr(bv.n * sys.PtrSize) + if !retValid { + arglen = uintptr(mv.argLen) &^ (sys.PtrSize - 1) + } argmap = bv } } diff --git a/test/fixedbugs/issue27695.go b/test/fixedbugs/issue27695.go new file mode 100644 index 0000000000..8bd4939e7e --- /dev/null +++ b/test/fixedbugs/issue27695.go @@ -0,0 +1,62 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure return values are always scanned, when +// calling methods (+functions, TODO) with reflect. + +package main + +import ( + "reflect" + "runtime/debug" + "sync" +) + +func main() { + debug.SetGCPercent(1) // run GC frequently + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 2000; i++ { + _test() + } + }() + } + wg.Wait() +} + +type Stt struct { + Data interface{} +} + +type My struct { + b byte +} + +func (this *My) Run(rawData []byte) (Stt, error) { + var data string = "hello" + stt := Stt{ + Data: data, + } + return stt, nil +} + +func _test() (interface{}, error) { + f := reflect.ValueOf(&My{}).MethodByName("Run") + if method, ok := f.Interface().(func([]byte) (Stt, error)); ok { + s, e := method(nil) + // The bug in issue27695 happens here, during the return + // from the above call (at the end of reflect.callMethod + // when preparing to return). The result value that + // is assigned to s was not being scanned if GC happens + // to occur there. + i := interface{}(s) + return i, e + } + return nil, nil +} diff --git a/test/fixedbugs/issue27695b.go b/test/fixedbugs/issue27695b.go new file mode 100644 index 0000000000..d80acfb8b4 --- /dev/null +++ b/test/fixedbugs/issue27695b.go @@ -0,0 +1,64 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Make sure return values aren't scanned until they +// are initialized, when calling functions and methods +// via reflect. + +package main + +import ( + "reflect" + "runtime" + "unsafe" +) + +var badPtr uintptr + +var sink []byte + +func init() { + // Allocate large enough to use largeAlloc. + b := make([]byte, 1<<16-1) + sink = b // force heap allocation + // Any space between the object and the end of page is invalid to point to. + badPtr = uintptr(unsafe.Pointer(&b[len(b)-1])) + 1 +} + +func f(d func() *byte) *byte { + // Initialize callee args section with a bad pointer. + g(badPtr) + + // Then call a function which returns a pointer. + // That return slot starts out holding a bad pointer. + return d() +} + +//go:noinline +func g(x uintptr) { +} + +type T struct { +} + +func (t *T) Foo() *byte { + runtime.GC() + return nil +} + +func main() { + // Functions + d := reflect.MakeFunc(reflect.TypeOf(func() *byte { return nil }), + func(args []reflect.Value) []reflect.Value { + runtime.GC() + return []reflect.Value{reflect.ValueOf((*byte)(nil))} + }).Interface().(func() *byte) + f(d) + + // Methods + e := reflect.ValueOf(&T{}).Method(0).Interface().(func() *byte) + f(e) +} -- cgit v1.3-5-g9baa From a9c69e0a657cbbdd4df3a77bd202b7f03a104094 Mon Sep 17 00:00:00 2001 From: Shulhan Date: Thu, 13 Sep 2018 01:46:39 +0700 Subject: runtime: fix runtime gdb test with gdb v8.2 Previously, some of output from gdb matched with literal string, while gdb v8.2 print the address of variable (e.g. map key and value) in output. This commit fix the regex in testing the output. Fixes #27608 Change-Id: Ic3fe8280b9f93fda2799116804822616caa66beb Reviewed-on: https://go-review.googlesource.com/135055 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: David Chase --- src/runtime/runtime-gdb_test.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/runtime-gdb_test.go b/src/runtime/runtime-gdb_test.go index 26f507159b..0c24d3dce6 100644 --- a/src/runtime/runtime-gdb_test.go +++ b/src/runtime/runtime-gdb_test.go @@ -242,14 +242,14 @@ func testGdbPython(t *testing.T, cgo bool) { t.Fatalf("info goroutines failed: %s", bl) } - printMapvarRe1 := regexp.MustCompile(`\Q = map[string]string = {["abc"] = "def", ["ghi"] = "jkl"}\E$`) - printMapvarRe2 := regexp.MustCompile(`\Q = map[string]string = {["ghi"] = "jkl", ["abc"] = "def"}\E$`) + printMapvarRe1 := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def", \[(0x[0-9a-f]+\s+)?"ghi"\] = (0x[0-9a-f]+\s+)?"jkl"}$`) + printMapvarRe2 := regexp.MustCompile(`^\$[0-9]+ = map\[string\]string = {\[(0x[0-9a-f]+\s+)?"ghi"\] = (0x[0-9a-f]+\s+)?"jkl", \[(0x[0-9a-f]+\s+)?"abc"\] = (0x[0-9a-f]+\s+)?"def"}$`) if bl := blocks["print mapvar"]; !printMapvarRe1.MatchString(bl) && !printMapvarRe2.MatchString(bl) { t.Fatalf("print mapvar failed: %s", bl) } - strVarRe := regexp.MustCompile(`\Q = "abc"\E$`) + strVarRe := regexp.MustCompile(`^\$[0-9]+ = (0x[0-9a-f]+\s+)?"abc"$`) if bl := blocks["print strvar"]; !strVarRe.MatchString(bl) { t.Fatalf("print strvar failed: %s", bl) } @@ -263,8 +263,11 @@ func testGdbPython(t *testing.T, cgo bool) { // aggregates from their fields and reverted their printing // back to its original form. - infoLocalsRe := regexp.MustCompile(`slicevar *= *\[\]string *= *{"def"}`) - if bl := blocks["info locals"]; !infoLocalsRe.MatchString(bl) { + infoLocalsRe1 := regexp.MustCompile(`slicevar *= *\[\]string *= *{"def"}`) + // Format output from gdb v8.2 + infoLocalsRe2 := regexp.MustCompile(`^slicevar = .*\nmapvar = .*\nstrvar = 0x[0-9a-f]+ "abc"`) + if bl := blocks["info locals"]; !infoLocalsRe1.MatchString(bl) && + !infoLocalsRe2.MatchString(bl) { t.Fatalf("info locals failed: %s", bl) } @@ -425,11 +428,11 @@ func TestGdbAutotmpTypes(t *testing.T) { // Check that the backtrace matches the source code. types := []string{ - "struct []main.astruct;", - "struct bucket;", - "struct hash;", - "struct main.astruct;", - "typedef struct hash * map[string]main.astruct;", + "[]main.astruct;", + "bucket;", + "hash;", + "main.astruct;", + "hash * map[string]main.astruct;", } for _, name := range types { if !strings.Contains(sgot, name) { -- cgit v1.3-5-g9baa From 9c634ea889fdfa41aec9183bd5693c155374ba76 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 3 Aug 2018 14:56:55 -0400 Subject: runtime: flush write barrier buffer to create work Currently, if the gcWork runs out of work, we'll fall out of the GC worker, even though flushing the write barrier buffer could produce more work. While this is not a correctness issue, it can lead to premature mark 2 or mark termination. Fix this by flushing the write barrier buffer if the local gcWork runs out of work and then checking the local gcWork again. This reduces the number of premature mark terminations during all.bash by about a factor of 10. Updates #26903. This is preparation for eliminating mark 2. Change-Id: I48577220b90c86bfd28d498e8603bc379a8cd617 Reviewed-on: https://go-review.googlesource.com/c/134315 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgcmark.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index f713841cfa..bf69172f6a 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -882,6 +882,13 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) { b = gcw.tryGetFast() if b == 0 { b = gcw.tryGet() + if b == 0 { + // Flush the write barrier + // buffer; this may create + // more work. + wbBufFlush(nil, 0) + b = gcw.tryGet() + } } } if b == 0 { @@ -963,6 +970,12 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 { b := gcw.tryGetFast() if b == 0 { b = gcw.tryGet() + if b == 0 { + // Flush the write barrier buffer; + // this may create more work. + wbBufFlush(nil, 0) + b = gcw.tryGet() + } } if b == 0 { -- cgit v1.3-5-g9baa From edc2d17086a6d21039f45506459cf1b9f40eb95c Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Wed, 15 Aug 2018 16:19:21 -0400 Subject: runtime: remove GODEBUG=gctrace=2 mode It turns out if you set GODEBUG=gctrace=2, it enables an obscure debugging mode that, in addition to printing gctrace statistics, also does a second STW GC following each regular GC. This debugging mode has long since lost its value (you could maybe use it to analyze floating garbage, except that we don't print the gctrace line on the second GC), and it interferes substantially with the operation of the GC by messing up the statistics used to schedule GCs. It's also a source of mark termination GC work when we're in concurrent GC mode, so it's going to interfere with eliminating mark 2. And it's going to get in the way of unifying STW and concurrent GC. This CL removes this debugging mode. Updates #26903. This is preparation for eliminating mark 2 and unifying STW GC and concurrent GC. Change-Id: Ib5bce05d8c4d5b6559c89a65165d49532165df07 Reviewed-on: https://go-review.googlesource.com/c/134316 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/extern.go | 3 +-- src/runtime/mgc.go | 17 ----------------- 2 files changed, 1 insertion(+), 19 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 1773c8fe7e..3be1eca09c 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -61,8 +61,7 @@ It is a comma-separated list of name=val pairs setting these named variables: gctrace: setting gctrace=1 causes the garbage collector to emit a single line to standard error at each collection, summarizing the amount of memory collected and the - length of the pause. Setting gctrace=2 emits the same summary but also - repeats each collection. The format of this line is subject to change. + length of the pause. The format of this line is subject to change. Currently, it is: gc # @#s #%: #+#+# ms clock, #+#/#/#+# ms cpu, #->#-># MB, # MB goal, # P where the fields are as follows: diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index c95b5ed37f..f975011191 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1558,23 +1558,6 @@ func gcMarkTermination(nextTriggerRatio float64) { // marking is complete so we can turn the write barrier off setGCPhase(_GCoff) gcSweep(work.mode) - - if debug.gctrace > 1 { - startTime = nanotime() - // The g stacks have been scanned so - // they have gcscanvalid==true and gcworkdone==true. - // Reset these so that all stacks will be rescanned. - gcResetMarkState() - finishsweep_m() - - // Still in STW but gcphase is _GCoff, reset to _GCmarktermination - // At this point all objects will be found during the gcMark which - // does a complete STW mark and object scan. - setGCPhase(_GCmarktermination) - gcMark(startTime) - setGCPhase(_GCoff) // marking is done, turn off wb. - gcSweep(work.mode) - } }) _g_.m.traceback = 0 -- cgit v1.3-5-g9baa From a2a2901b252c089e737a1ddf8117f3f6698cb958 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 3 Aug 2018 15:04:14 -0400 Subject: runtime: track whether any buffer has been flushed from gcWork Nothing currently consumes the flag, but we'll use it in the distributed termination detection algorithm. Updates #26903. This is preparation for eliminating mark 2. Change-Id: I5e149a05b1c878fe1009150da21f8bd8ae2b9b6a Reviewed-on: https://go-review.googlesource.com/c/134317 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgcwork.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index 99771e2e57..27e73d6c4a 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -83,6 +83,12 @@ type gcWork struct { // Scan work performed on this gcWork. This is aggregated into // gcController by dispose and may also be flushed by callers. scanWork int64 + + // flushedWork indicates that a non-empty work buffer was + // flushed to the global work list since the last gcMarkDone + // termination check. Specifically, this indicates that this + // gcWork may have communicated work to another gcWork. + flushedWork bool } // Most of the methods of gcWork are go:nowritebarrierrec because the @@ -116,6 +122,7 @@ func (w *gcWork) put(obj uintptr) { wbuf = w.wbuf1 if wbuf.nobj == len(wbuf.obj) { putfull(wbuf) + w.flushedWork = true wbuf = getempty() w.wbuf1 = wbuf flushed = true @@ -169,6 +176,7 @@ func (w *gcWork) putBatch(obj []uintptr) { for len(obj) > 0 { for wbuf.nobj == len(wbuf.obj) { putfull(wbuf) + w.flushedWork = true w.wbuf1, w.wbuf2 = w.wbuf2, getempty() wbuf = w.wbuf1 flushed = true @@ -275,6 +283,7 @@ func (w *gcWork) dispose() { putempty(wbuf) } else { putfull(wbuf) + w.flushedWork = true } w.wbuf1 = nil @@ -283,6 +292,7 @@ func (w *gcWork) dispose() { putempty(wbuf) } else { putfull(wbuf) + w.flushedWork = true } w.wbuf2 = nil } @@ -309,9 +319,11 @@ func (w *gcWork) balance() { } if wbuf := w.wbuf2; wbuf.nobj != 0 { putfull(wbuf) + w.flushedWork = true w.wbuf2 = getempty() } else if wbuf := w.wbuf1; wbuf.nobj > 4 { w.wbuf1 = handoff(wbuf) + w.flushedWork = true // handoff did putfull } else { return } -- cgit v1.3-5-g9baa From 9108ae775145a4ecd086f04eb5c45155d8636298 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 3 Aug 2018 17:13:09 -0400 Subject: runtime: eliminate mark 2 and fix mark termination race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The mark 2 phase was originally introduced as a way to reduce the chance of entering STW mark termination while there was still marking work to do. It works by flushing and disabling all local work caches so that all enqueued work becomes immediately globally visible. However, mark 2 is not only slow–disabling caches makes marking and the write barrier both much more expensive–but also imperfect. There is still a rare but possible race (~once per all.bash) that can cause GC to enter mark termination while there is still marking work. This race is detailed at https://github.com/golang/proposal/blob/master/design/17503-eliminate-rescan.md#appendix-mark-completion-race The effect of this is that mark termination must still cope with the possibility that there may be work remaining after a concurrent mark phase. Dealing with this increases STW pause time and increases the complexity of mark termination. Furthermore, a similar but far more likely race can cause early transition from mark 1 to mark 2. This is unfortunate because it causes performance instability because of the cost of mark 2. This CL fixes this by replacing mark 2 with a distributed termination detection algorithm. This algorithm is correct, so it eliminates the mark termination race, and doesn't require disabling local caches. It ensures that there are no grey objects upon entering mark termination. With this change, we're one step closer to eliminating marking from mark termination entirely (it's still used by STW GC and checkmarks mode). This CL does not eliminate the gcBlackenPromptly global flag, though it is always set to false now. It will be removed in a cleanup CL. This led to only minor variations in the go1 benchmarks (https://perf.golang.org/search?q=upload:20180909.1) and compilebench benchmarks (https://perf.golang.org/search?q=upload:20180910.2). This significantly improves performance of the garbage benchmark, with no impact on STW times: name old time/op new time/op delta Garbage/benchmem-MB=64-12 2.21ms ± 1% 2.05ms ± 1% -7.38% (p=0.000 n=18+19) Garbage/benchmem-MB=1024-12 2.30ms ±16% 2.20ms ± 7% -4.51% (p=0.001 n=20+20) name old STW-ns/GC new STW-ns/GC delta Garbage/benchmem-MB=64-12 138k ±44% 141k ±23% ~ (p=0.309 n=19+20) Garbage/benchmem-MB=1024-12 159k ±25% 178k ±98% ~ (p=0.798 n=16+18) name old STW-ns/op new STW-ns/op delta Garbage/benchmem-MB=64-12 4.42k ±44% 4.24k ±23% ~ (p=0.531 n=19+20) Garbage/benchmem-MB=1024-12 591 ±24% 636 ±111% ~ (p=0.309 n=16+18) (https://perf.golang.org/search?q=upload:20180910.1) Updates #26903. Updates #17503. Change-Id: Icbd1e12b7a12a76f423c9bf033b13cb363e4cd19 Reviewed-on: https://go-review.googlesource.com/c/134318 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/gc_test.go | 4 +- src/runtime/mgc.go | 279 +++++++++++++++++++++++++------------------------ src/runtime/mwbbuf.go | 5 + 3 files changed, 148 insertions(+), 140 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/gc_test.go b/src/runtime/gc_test.go index 1f7715c672..51e8ea4d31 100644 --- a/src/runtime/gc_test.go +++ b/src/runtime/gc_test.go @@ -574,8 +574,8 @@ func BenchmarkWriteBarrier(b *testing.B) { n := &node{mkTree(level - 1), mkTree(level - 1)} if level == 10 { // Seed GC with enough early pointers so it - // doesn't accidentally switch to mark 2 when - // it only has the top of the tree. + // doesn't start termination barriers when it + // only has the top of the tree. wbRoots = append(wbRoots, n) } return n diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index f975011191..2fd849c196 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -28,8 +28,7 @@ // b. Sweep any unswept spans. There will only be unswept spans if // this GC cycle was forced before the expected time. // -// 2. GC performs the "mark 1" sub-phase. In this sub-phase, Ps are -// allowed to locally cache parts of the work queue. +// 2. GC performs the mark phase. // // a. Prepare for the mark phase by setting gcphase to _GCmark // (from _GCoff), enabling the write barrier, enabling mutator @@ -54,28 +53,21 @@ // object to black and shading all pointers found in the object // (which in turn may add those pointers to the work queue). // -// 3. Once the global work queue is empty (but local work queue caches -// may still contain work), GC performs the "mark 2" sub-phase. +// e. Because GC work is spread across local caches, GC uses a +// distributed termination algorithm to detect when there are no +// more root marking jobs or grey objects (see gcMarkDone). At this +// point, GC transitions to mark termination. // -// a. GC stops all workers, disables local work queue caches, -// flushes each P's local work queue cache to the global work queue -// cache, and reenables workers. -// -// b. GC again drains the work queue, as in 2d above. -// -// 4. Once the work queue is empty, GC performs mark termination. +// 3. GC performs mark termination. // // a. Stop the world. // // b. Set gcphase to _GCmarktermination, and disable workers and // assists. // -// c. Drain any remaining work from the work queue (typically there -// will be none). +// c. Perform housekeeping like flushing mcaches. // -// d. Perform other housekeeping like flushing mcaches. -// -// 5. GC performs the sweep phase. +// 4. GC performs the sweep phase. // // a. Prepare for the sweep phase by setting gcphase to _GCoff, // setting up sweep state and disabling the write barrier. @@ -86,7 +78,7 @@ // c. GC does concurrent sweeping in the background and in response // to allocation. See description below. // -// 6. When sufficient allocation has taken place, replay the sequence +// 5. When sufficient allocation has taken place, replay the sequence // starting with 1 above. See discussion of GC rate below. // Concurrent sweep. @@ -996,8 +988,7 @@ var work struct { // startSema protects the transition from "off" to mark or // mark termination. startSema uint32 - // markDoneSema protects transitions from mark 1 to mark 2 and - // from mark 2 to mark termination. + // markDoneSema protects transitions from mark to mark termination. markDoneSema uint32 bgMarkReady note // signal background mark worker has started @@ -1385,128 +1376,121 @@ func gcStart(mode gcMode, trigger gcTrigger) { semrelease(&work.startSema) } -// gcMarkDone transitions the GC from mark 1 to mark 2 and from mark 2 -// to mark termination. +// gcMarkDoneFlushed counts the number of P's with flushed work. +// +// Ideally this would be a captured local in gcMarkDone, but forEachP +// escapes its callback closure, so it can't capture anything. // -// This should be called when all mark work has been drained. In mark -// 1, this includes all root marking jobs, global work buffers, and -// active work buffers in assists and background workers; however, -// work may still be cached in per-P work buffers. In mark 2, per-P -// caches are disabled. +// This is protected by markDoneSema. +var gcMarkDoneFlushed uint32 + +// gcMarkDone transitions the GC from mark to mark termination if all +// reachable objects have been marked (that is, there are no grey +// objects and can be no more in the future). Otherwise, it flushes +// all local work to the global queues where it can be discovered by +// other workers. +// +// This should be called when all local mark work has been drained and +// there are no remaining workers. Specifically, when +// +// work.nwait == work.nproc && !gcMarkWorkAvailable(p) // // The calling context must be preemptible. // -// Note that it is explicitly okay to have write barriers in this -// function because completion of concurrent mark is best-effort -// anyway. Any work created by write barriers here will be cleaned up -// by mark termination. +// Flushing local work is important because idle Ps may have local +// work queued. This is the only way to make that work visible and +// drive GC to completion. +// +// It is explicitly okay to have write barriers in this function. If +// it does transition to mark termination, then all reachable objects +// have been marked, so the write barrier cannot shade any more +// objects. func gcMarkDone() { -top: + // Ensure only one thread is running the ragged barrier at a + // time. semacquire(&work.markDoneSema) +top: // Re-check transition condition under transition lock. + // + // It's critical that this checks the global work queues are + // empty before performing the ragged barrier. Otherwise, + // there could be global work that a P could take after the P + // has passed the ragged barrier. if !(gcphase == _GCmark && work.nwait == work.nproc && !gcMarkWorkAvailable(nil)) { semrelease(&work.markDoneSema) return } - // Disallow starting new workers so that any remaining workers - // in the current mark phase will drain out. - // - // TODO(austin): Should dedicated workers keep an eye on this - // and exit gcDrain promptly? - atomic.Xaddint64(&gcController.dedicatedMarkWorkersNeeded, -0xffffffff) - prevFractionalGoal := gcController.fractionalUtilizationGoal - gcController.fractionalUtilizationGoal = 0 - - if !gcBlackenPromptly { - // Transition from mark 1 to mark 2. - // - // The global work list is empty, but there can still be work - // sitting in the per-P work caches. - // Flush and disable work caches. - - // Disallow caching workbufs and indicate that we're in mark 2. - gcBlackenPromptly = true - - // Prevent completion of mark 2 until we've flushed - // cached workbufs. - atomic.Xadd(&work.nwait, -1) - - // GC is set up for mark 2. Let Gs blocked on the - // transition lock go while we flush caches. - semrelease(&work.markDoneSema) - - systemstack(func() { - // Flush all currently cached workbufs and - // ensure all Ps see gcBlackenPromptly. This - // also blocks until any remaining mark 1 - // workers have exited their loop so we can - // start new mark 2 workers. - forEachP(func(_p_ *p) { - wbBufFlush1(_p_) - _p_.gcw.dispose() - }) + // Flush all local buffers and collect flushedWork flags. + gcMarkDoneFlushed = 0 + systemstack(func() { + forEachP(func(_p_ *p) { + // Flush the write barrier buffer, since this may add + // work to the gcWork. + wbBufFlush1(_p_) + // Flush the gcWork, since this may create global work + // and set the flushedWork flag. + // + // TODO(austin): Break up these workbufs to + // better distribute work. + _p_.gcw.dispose() + // Collect the flushedWork flag. + if _p_.gcw.flushedWork { + atomic.Xadd(&gcMarkDoneFlushed, 1) + _p_.gcw.flushedWork = false + } }) + }) - // Check that roots are marked. We should be able to - // do this before the forEachP, but based on issue - // #16083 there may be a (harmless) race where we can - // enter mark 2 while some workers are still scanning - // stacks. The forEachP ensures these scans are done. - // - // TODO(austin): Figure out the race and fix this - // properly. - gcMarkRootCheck() - - // Now we can start up mark 2 workers. - atomic.Xaddint64(&gcController.dedicatedMarkWorkersNeeded, 0xffffffff) - gcController.fractionalUtilizationGoal = prevFractionalGoal + if gcMarkDoneFlushed != 0 { + // More grey objects were discovered since the + // previous termination check, so there may be more + // work to do. Keep going. It's possible the + // transition condition became true again during the + // ragged barrier, so re-check it. + goto top + } - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait == work.nproc && !gcMarkWorkAvailable(nil) { - // This loop will make progress because - // gcBlackenPromptly is now true, so it won't - // take this same "if" branch. - goto top - } - } else { - // Transition to mark termination. - now := nanotime() - work.tMarkTerm = now - work.pauseStart = now - getg().m.preemptoff = "gcing" - if trace.enabled { - traceGCSTWStart(0) - } - systemstack(stopTheWorldWithSema) - // The gcphase is _GCmark, it will transition to _GCmarktermination - // below. The important thing is that the wb remains active until - // all marking is complete. This includes writes made by the GC. + // There was no global work, no local work, and no Ps + // communicated work since we took markDoneSema. Therefore + // there are no grey objects and no more objects can be + // shaded. Transition to mark termination. + now := nanotime() + work.tMarkTerm = now + work.pauseStart = now + getg().m.preemptoff = "gcing" + if trace.enabled { + traceGCSTWStart(0) + } + systemstack(stopTheWorldWithSema) + // The gcphase is _GCmark, it will transition to _GCmarktermination + // below. The important thing is that the wb remains active until + // all marking is complete. This includes writes made by the GC. - // Record that one root marking pass has completed. - work.markrootDone = true + // Record that one root marking pass has completed. + work.markrootDone = true - // Disable assists and background workers. We must do - // this before waking blocked assists. - atomic.Store(&gcBlackenEnabled, 0) + // Disable assists and background workers. We must do + // this before waking blocked assists. + atomic.Store(&gcBlackenEnabled, 0) - // Wake all blocked assists. These will run when we - // start the world again. - gcWakeAllAssists() + // Wake all blocked assists. These will run when we + // start the world again. + gcWakeAllAssists() - // Likewise, release the transition lock. Blocked - // workers and assists will run when we start the - // world again. - semrelease(&work.markDoneSema) + // Likewise, release the transition lock. Blocked + // workers and assists will run when we start the + // world again. + semrelease(&work.markDoneSema) - // endCycle depends on all gcWork cache stats being - // flushed. This is ensured by mark 2. - nextTriggerRatio := gcController.endCycle() + // endCycle depends on all gcWork cache stats being flushed. + // The termination algorithm above ensured that up to + // allocations since the ragged barrier. + nextTriggerRatio := gcController.endCycle() - // Perform mark termination. This will restart the world. - gcMarkTermination(nextTriggerRatio) - } + // Perform mark termination. This will restart the world. + gcMarkTermination(nextTriggerRatio) } func gcMarkTermination(nextTriggerRatio float64) { @@ -1940,23 +1924,23 @@ func gcMark(start_time int64) { if work.full == 0 && work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots == 0 { // There's no work on the work queue and no root jobs // that can produce work, so don't bother entering the - // getfull() barrier. - // - // This will be the situation the vast majority of the - // time after concurrent mark. However, we still need - // a fallback for STW GC and because there are some - // known races that occasionally leave work around for - // mark termination. - // - // We're still hedging our bets here: if we do - // accidentally produce some work, we'll still process - // it, just not necessarily in parallel. + // getfull() barrier. There will be flushCacheRoots + // work, but that doesn't gray anything. // - // TODO(austin): Fix the races and and remove - // work draining from mark termination so we don't - // need the fallback path. + // This should always be the situation after + // concurrent mark. work.helperDrainBlock = false } else { + // There's marking work to do. This is the case during + // STW GC and in checkmark mode. Instruct GC workers + // to block in getfull until all GC workers are in getfull. + // + // TODO(austin): Move STW and checkmark marking out of + // mark termination and eliminate this code path. + if !useCheckmark && debug.gcstoptheworld == 0 && debug.gcrescanstacks == 0 { + print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") + panic("non-empty mark queue after concurrent mark") + } work.helperDrainBlock = true } @@ -1991,16 +1975,35 @@ func gcMark(start_time int64) { // Record that at least one root marking pass has completed. work.markrootDone = true - // Double-check that all gcWork caches are empty. This should - // be ensured by mark 2 before we enter mark termination. + // Clear out buffers and double-check that all gcWork caches + // are empty. This should be ensured by gcMarkDone before we + // enter mark termination. + // + // TODO: We could clear out buffers just before mark if this + // has a non-negligible impact on STW time. for _, p := range allp { + // The write barrier may have buffered pointers since + // the gcMarkDone barrier. However, since the barrier + // ensured all reachable objects were marked, all of + // these must be pointers to black objects. Hence we + // can just discard the write barrier buffer. + if debug.gccheckmark > 0 { + // For debugging, flush the buffer and make + // sure it really was all marked. + wbBufFlush1(p) + } else { + p.wbBuf.reset() + } + gcw := &p.gcw if !gcw.empty() { throw("P has cached GC work at end of mark termination") } - if gcw.scanWork != 0 || gcw.bytesMarked != 0 { - throw("P has unflushed stats at end of mark termination") - } + // There may still be cached empty buffers, which we + // need to flush since we're going to free them. Also, + // there may be non-zero stats because we allocated + // black after the gcMarkDone barrier. + gcw.dispose() } cachestats() diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go index 4df16d55b8..657c2fd2ba 100644 --- a/src/runtime/mwbbuf.go +++ b/src/runtime/mwbbuf.go @@ -107,6 +107,11 @@ func (b *wbBuf) discard() { b.next = uintptr(unsafe.Pointer(&b.buf[0])) } +// empty returns true if b contains no pointers. +func (b *wbBuf) empty() bool { + return b.next == uintptr(unsafe.Pointer(&b.buf[0])) +} + // putFast adds old and new to the write barrier buffer and returns // false if a flush is necessary. Callers should use this as: // -- cgit v1.3-5-g9baa From d398dbdfc38838b63e39cd8ee2c9d30a09b5989f Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 9 Aug 2018 15:25:10 -0400 Subject: runtime: eliminate gcBlackenPromptly mode Now that there is no mark 2 phase, gcBlackenPromptly is no longer used. Updates #26903. This is a follow-up to eliminating mark 2. Change-Id: Ib9c534f21b36b8416fcf3cab667f186167b827f8 Reviewed-on: https://go-review.googlesource.com/c/134319 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 26 -------------------------- src/runtime/mgcmark.go | 20 +++----------------- src/runtime/mgcwork.go | 3 --- src/runtime/mheap.go | 3 --- src/runtime/mwbbuf.go | 4 ++-- src/runtime/stack.go | 3 --- 6 files changed, 5 insertions(+), 54 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 2fd849c196..69fd8df662 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -253,21 +253,6 @@ var writeBarrier struct { // gcphase == _GCmark. var gcBlackenEnabled uint32 -// gcBlackenPromptly indicates that optimizations that may -// hide work from the global work queue should be disabled. -// -// If gcBlackenPromptly is true, per-P gcWork caches should -// be flushed immediately and new objects should be allocated black. -// -// There is a tension between allocating objects white and -// allocating them black. If white and the objects die before being -// marked they can be collected during this GC cycle. On the other -// hand allocating them black will reduce _GCmarktermination latency -// since more work is done in the mark phase. This tension is resolved -// by allocating white until the mark phase is approaching its end and -// then allocating black for the remainder of the mark phase. -var gcBlackenPromptly bool - const ( _GCoff = iota // GC not running; sweeping in background, write barrier disabled _GCmark // GC marking roots and workbufs: allocate black, write barrier ENABLED @@ -1497,7 +1482,6 @@ func gcMarkTermination(nextTriggerRatio float64) { // World is stopped. // Start marktermination which includes enabling the write barrier. atomic.Store(&gcBlackenEnabled, 0) - gcBlackenPromptly = false setGCPhase(_GCmarktermination) work.heap1 = memstats.heap_live @@ -1828,16 +1812,6 @@ func gcBgMarkWorker(_p_ *p) { casgstatus(gp, _Gwaiting, _Grunning) }) - // If we are nearing the end of mark, dispose - // of the cache promptly. We must do this - // before signaling that we're no longer - // working so that other workers can't observe - // no workers and no work while we have this - // cached, and before we compute done. - if gcBlackenPromptly { - _p_.gcw.dispose() - } - // Account for time. duration := nanotime() - startTime switch _p_.gcMarkWorkerMode { diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index bf69172f6a..6c641e3fbf 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -556,11 +556,6 @@ func gcAssistAlloc1(gp *g, scanWork int64) { // will be more cache friendly. gcw := &getg().m.p.ptr().gcw workDone := gcDrainN(gcw, scanWork) - // If we are near the end of the mark phase - // dispose of the gcw. - if gcBlackenPromptly { - gcw.dispose() - } casgstatus(gp, _Gwaiting, _Grunning) @@ -577,8 +572,7 @@ func gcAssistAlloc1(gp *g, scanWork int64) { incnwait := atomic.Xadd(&work.nwait, +1) if incnwait > work.nproc { println("runtime: work.nwait=", incnwait, - "work.nproc=", work.nproc, - "gcBlackenPromptly=", gcBlackenPromptly) + "work.nproc=", work.nproc) throw("work.nwait > work.nproc") } @@ -1155,7 +1149,7 @@ func shade(b uintptr) { if obj, span, objIndex := findObject(b, 0, 0); obj != 0 { gcw := &getg().m.p.ptr().gcw greyobject(obj, 0, 0, span, gcw, objIndex) - if gcphase == _GCmarktermination || gcBlackenPromptly { + if gcphase == _GCmarktermination { // Ps aren't allowed to cache work during mark // termination. gcw.dispose() @@ -1289,18 +1283,13 @@ func gcDumpObject(label string, obj, off uintptr) { //go:nowritebarrier //go:nosplit func gcmarknewobject(obj, size, scanSize uintptr) { - if useCheckmark && !gcBlackenPromptly { // The world should be stopped so this should not happen. + if useCheckmark { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } markBitsForAddr(obj).setMarked() gcw := &getg().m.p.ptr().gcw gcw.bytesMarked += uint64(size) gcw.scanWork += int64(scanSize) - if gcBlackenPromptly { - // There shouldn't be anything in the work queue, but - // we still need to flush stats. - gcw.dispose() - } } // gcMarkTinyAllocs greys all active tiny alloc blocks. @@ -1315,9 +1304,6 @@ func gcMarkTinyAllocs() { _, span, objIndex := findObject(c.tiny, 0, 0) gcw := &p.gcw greyobject(c.tiny, 0, 0, span, gcw, objIndex) - if gcBlackenPromptly { - gcw.dispose() - } } } diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index 27e73d6c4a..3a99260e53 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -47,9 +47,6 @@ func init() { // (preemption must be disabled) // gcw := &getg().m.p.ptr().gcw // .. call gcw.put() to produce and gcw.get() to consume .. -// if gcBlackenPromptly { -// gcw.dispose() -// } // // It's important that any use of gcWork during the mark phase prevent // the garbage collector from transitioning to mark termination since diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 65d6b0c7d4..2dd66f7c2b 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -1438,9 +1438,6 @@ func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *p // Mark the finalizer itself, since the // special isn't part of the GC'd heap. scanblock(uintptr(unsafe.Pointer(&s.fn)), sys.PtrSize, &oneptrmask[0], gcw) - if gcBlackenPromptly { - gcw.dispose() - } releasem(mp) } return true diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go index 657c2fd2ba..335b10f5b7 100644 --- a/src/runtime/mwbbuf.go +++ b/src/runtime/mwbbuf.go @@ -79,7 +79,7 @@ const ( func (b *wbBuf) reset() { start := uintptr(unsafe.Pointer(&b.buf[0])) b.next = start - if gcBlackenPromptly || writeBarrier.cgo { + if writeBarrier.cgo { // Effectively disable the buffer by forcing a flush // on every barrier. b.end = uintptr(unsafe.Pointer(&b.buf[wbBufEntryPointers])) @@ -275,7 +275,7 @@ func wbBufFlush1(_p_ *p) { // Enqueue the greyed objects. gcw.putBatch(ptrs[:pos]) - if gcphase == _GCmarktermination || gcBlackenPromptly { + if gcphase == _GCmarktermination { // Ps aren't allowed to cache work during mark // termination. gcw.dispose() diff --git a/src/runtime/stack.go b/src/runtime/stack.go index d56b864c5e..fd9aafd15b 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -981,9 +981,6 @@ func newstack() { // system stack. gcw := &gp.m.p.ptr().gcw scanstack(gp, gcw) - if gcBlackenPromptly { - gcw.dispose() - } gp.gcscandone = true } gp.preemptscan = false -- cgit v1.3-5-g9baa From e25ef35254cc4372256d0c7f4521df3cf3d092bf Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 14 Aug 2018 17:08:27 -0400 Subject: runtime: don't disable GC work caching during mark termination Currently, we disable GC work caching during mark termination. This is no longer necessary with the new mark completion detection because 1. There's no way for any of the GC mark termination helpers to have any real work queued and, 2. Mark termination has to explicitly flush every P's buffers anyway in order to flush Ps that didn't run a GC mark termination helper. Hence, remove the code that disposes gcWork buffers during mark termination. Updates #26903. This is a follow-up to eliminating mark 2. Change-Id: I81f002ee25d5c10f42afd39767774636519007f9 Reviewed-on: https://go-review.googlesource.com/c/134320 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 2 -- src/runtime/mgcmark.go | 5 ----- src/runtime/mwbbuf.go | 5 ----- 3 files changed, 12 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 69fd8df662..83d9a49a46 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1931,7 +1931,6 @@ func gcMark(start_time int64) { } else { gcDrain(gcw, gcDrainNoBlock) } - gcw.dispose() if debug.gccheckmark > 0 { // This is expensive when there's a large number of @@ -2130,7 +2129,6 @@ func gchelper() { } else { gcDrain(gcw, gcDrainNoBlock) } - gcw.dispose() } nproc := atomic.Load(&work.nproc) // work.nproc can change right after we increment work.ndone diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 6c641e3fbf..b86b2d012e 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1149,11 +1149,6 @@ func shade(b uintptr) { if obj, span, objIndex := findObject(b, 0, 0); obj != 0 { gcw := &getg().m.p.ptr().gcw greyobject(obj, 0, 0, span, gcw, objIndex) - if gcphase == _GCmarktermination { - // Ps aren't allowed to cache work during mark - // termination. - gcw.dispose() - } } } diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go index 335b10f5b7..f35f7286ac 100644 --- a/src/runtime/mwbbuf.go +++ b/src/runtime/mwbbuf.go @@ -275,9 +275,4 @@ func wbBufFlush1(_p_ *p) { // Enqueue the greyed objects. gcw.putBatch(ptrs[:pos]) - if gcphase == _GCmarktermination { - // Ps aren't allowed to cache work during mark - // termination. - gcw.dispose() - } } -- cgit v1.3-5-g9baa From 918ed88e475d39e40baf9cd6539b618e05d12e5e Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 13 Aug 2018 16:14:19 -0400 Subject: runtime: remove gcStart's mode argument This argument is always gcBackgroundMode since only debug.gcstoptheworld can trigger a STW GC at this point. Remove the unnecessary argument. Updates #26903. This is preparation for unifying STW GC and concurrent GC. Change-Id: Icb4ba8f10f80c2b69cf51a21e04fa2c761b71c94 Reviewed-on: https://go-review.googlesource.com/c/134775 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/malloc.go | 2 +- src/runtime/mgc.go | 21 ++++++++++----------- src/runtime/proc.go | 2 +- 3 files changed, 12 insertions(+), 13 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 5755c9e263..c3fe1169dc 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -992,7 +992,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if shouldhelpgc { if t := (gcTrigger{kind: gcTriggerHeap}); t.test() { - gcStart(gcBackgroundMode, t) + gcStart(t) } } diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 83d9a49a46..25bb210475 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1063,7 +1063,7 @@ func GC() { // We're now in sweep N or later. Trigger GC cycle N+1, which // will first finish sweep N if necessary and then enter sweep // termination N+1. - gcStart(gcBackgroundMode, gcTrigger{kind: gcTriggerCycle, n: n + 1}) + gcStart(gcTrigger{kind: gcTriggerCycle, n: n + 1}) // Wait for mark termination N+1 to complete. gcWaitOnMark(n + 1) @@ -1201,13 +1201,13 @@ func (t gcTrigger) test() bool { return true } -// gcStart transitions the GC from _GCoff to _GCmark (if -// !mode.stwMark) or _GCmarktermination (if mode.stwMark) by -// performing sweep termination and GC initialization. +// gcStart starts the GC. It transitions from _GCoff to _GCmark (if +// debug.gcstoptheworld == 0) or performs all of GC (if +// debug.gcstoptheworld != 0). // // This may return without performing this transition in some cases, // such as when called on a system stack or with locks held. -func gcStart(mode gcMode, trigger gcTrigger) { +func gcStart(trigger gcTrigger) { // Since this is called from malloc and malloc is called in // the guts of a number of libraries that might be holding // locks, don't attempt to start GC in non-preemptible or @@ -1250,12 +1250,11 @@ func gcStart(mode gcMode, trigger gcTrigger) { // We do this after re-checking the transition condition so // that multiple goroutines that detect the heap trigger don't // start multiple STW GCs. - if mode == gcBackgroundMode { - if debug.gcstoptheworld == 1 { - mode = gcForceMode - } else if debug.gcstoptheworld == 2 { - mode = gcForceBlockMode - } + mode := gcBackgroundMode + if debug.gcstoptheworld == 1 { + mode = gcForceMode + } else if debug.gcstoptheworld == 2 { + mode = gcForceBlockMode } // Ok, we're doing it! Stop everybody else diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 73b4a1d9d6..ec73b4d918 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -254,7 +254,7 @@ func forcegchelper() { println("GC forced") } // Time-triggered, fully concurrent. - gcStart(gcBackgroundMode, gcTrigger{kind: gcTriggerTime, now: nanotime()}) + gcStart(gcTrigger{kind: gcTriggerTime, now: nanotime()}) } } -- cgit v1.3-5-g9baa From ecc365960ba7ff72f650b32190f40a1f1b6ff992 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 14 Aug 2018 17:04:04 -0400 Subject: runtime: avoid using STW GC mechanism for checkmarks mode Currently, checkmarks mode uses the full STW GC infrastructure to perform mark checking. We're about to remove that infrastructure and, furthermore, since checkmarks is about doing the simplest thing possible to check concurrent GC, it's valuable for it to be simpler. Hence, this CL makes checkmarks even simpler by making it non-parallel and divorcing it from the STW GC infrastructure (including the gchelper mechanism). Updates #26903. This is preparation for unifying STW GC and concurrent GC. Change-Id: Iad21158123e025e3f97d7986d577315e994bd43e Reviewed-on: https://go-review.googlesource.com/c/134776 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 25bb210475..9ae5eb7a62 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1513,12 +1513,16 @@ func gcMarkTermination(nextTriggerRatio float64) { systemstack(func() { work.heap2 = work.bytesMarked if debug.gccheckmark > 0 { - // Run a full stop-the-world mark using checkmark bits, - // to check that we didn't forget to mark anything during - // the concurrent mark process. + // Run a full non-parallel, stop-the-world + // mark using checkmark bits, to check that we + // didn't forget to mark anything during the + // concurrent mark process. gcResetMarkState() initCheckmarks() - gcMark(startTime) + gcw := &getg().m.p.ptr().gcw + gcDrain(gcw, gcDrainNoBlock) + wbBufFlush1(getg().m.p.ptr()) + gcw.dispose() clearCheckmarks() } @@ -1905,12 +1909,12 @@ func gcMark(start_time int64) { work.helperDrainBlock = false } else { // There's marking work to do. This is the case during - // STW GC and in checkmark mode. Instruct GC workers + // STW GC. Instruct GC workers // to block in getfull until all GC workers are in getfull. // - // TODO(austin): Move STW and checkmark marking out of + // TODO(austin): Move STW marking out of // mark termination and eliminate this code path. - if !useCheckmark && debug.gcstoptheworld == 0 && debug.gcrescanstacks == 0 { + if debug.gcstoptheworld == 0 && debug.gcrescanstacks == 0 { print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") panic("non-empty mark queue after concurrent mark") } -- cgit v1.3-5-g9baa From 198440cc3d3453d349fbc7894a5d91dd7b16e6a0 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 16 Aug 2018 11:47:36 -0400 Subject: runtime: remove GODEBUG=gcrescanstacks=1 mode Currently, setting GODEBUG=gcrescanstacks=1 enables a debugging mode where the garbage collector re-scans goroutine stacks during mark termination. This was introduced in Go 1.8 to debug the hybrid write barrier, but I don't think we ever used it. Now it's one of the last sources of mark work during mark termination. This CL removes it. Updates #26903. This is preparation for unifying STW GC and concurrent GC. Updates #17503. Change-Id: I6ae04d3738aa9c448e6e206e21857a33ecd12acf Reviewed-on: https://go-review.googlesource.com/c/134777 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/extern.go | 5 ----- src/runtime/mgc.go | 2 +- src/runtime/mgcmark.go | 22 ++++------------------ src/runtime/runtime1.go | 2 -- 4 files changed, 5 insertions(+), 26 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/extern.go b/src/runtime/extern.go index 3be1eca09c..640688e004 100644 --- a/src/runtime/extern.go +++ b/src/runtime/extern.go @@ -50,11 +50,6 @@ It is a comma-separated list of name=val pairs setting these named variables: gcshrinkstackoff: setting gcshrinkstackoff=1 disables moving goroutines onto smaller stacks. In this mode, a goroutine's stack can only grow. - gcrescanstacks: setting gcrescanstacks=1 enables stack - re-scanning during the STW mark termination phase. This is - helpful for debugging if objects are being prematurely - garbage collected. - gcstoptheworld: setting gcstoptheworld=1 disables concurrent garbage collection, making every garbage collection a stop-the-world event. Setting gcstoptheworld=2 also disables concurrent sweeping after the garbage collection finishes. diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 9ae5eb7a62..b685415872 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1914,7 +1914,7 @@ func gcMark(start_time int64) { // // TODO(austin): Move STW marking out of // mark termination and eliminate this code path. - if debug.gcstoptheworld == 0 && debug.gcrescanstacks == 0 { + if debug.gcstoptheworld == 0 { print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") panic("non-empty mark queue after concurrent mark") } diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index b86b2d012e..07b8f791d4 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -116,11 +116,6 @@ func gcMarkRootPrepare() { // contain pointers to unmarked objects, so on the // second markroot, there's no need to scan stacks. work.nStackRoots = 0 - - if debug.gcrescanstacks > 0 { - // Scan stacks anyway for debugging. - work.nStackRoots = int(atomic.Loaduintptr(&allglen)) - } } work.markrootNext = 0 @@ -138,19 +133,10 @@ func gcMarkRootCheck() { lock(&allglock) // Check that stacks have been scanned. var gp *g - if gcphase == _GCmarktermination && debug.gcrescanstacks > 0 { - for i := 0; i < len(allgs); i++ { - gp = allgs[i] - if !(gp.gcscandone && gp.gcscanvalid) && readgstatus(gp) != _Gdead { - goto fail - } - } - } else { - for i := 0; i < work.nStackRoots; i++ { - gp = allgs[i] - if !gp.gcscandone { - goto fail - } + for i := 0; i < work.nStackRoots; i++ { + gp = allgs[i] + if !gp.gcscandone { + goto fail } } unlock(&allglock) diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go index 85a9ba2521..8b8f4dcb1e 100644 --- a/src/runtime/runtime1.go +++ b/src/runtime/runtime1.go @@ -305,7 +305,6 @@ var debug struct { gccheckmark int32 gcpacertrace int32 gcshrinkstackoff int32 - gcrescanstacks int32 gcstoptheworld int32 gctrace int32 invalidptr int32 @@ -323,7 +322,6 @@ var dbgvars = []dbgVar{ {"gccheckmark", &debug.gccheckmark}, {"gcpacertrace", &debug.gcpacertrace}, {"gcshrinkstackoff", &debug.gcshrinkstackoff}, - {"gcrescanstacks", &debug.gcrescanstacks}, {"gcstoptheworld", &debug.gcstoptheworld}, {"gctrace", &debug.gctrace}, {"invalidptr", &debug.invalidptr}, -- cgit v1.3-5-g9baa From 29b21ec4c371061a99dfaac356e54b3c62c5853f Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 13 Aug 2018 16:08:03 -0400 Subject: runtime: add a more stable isSystemGoroutine mode Currently, isSystemGoroutine varies on whether it considers the finalizer goroutine a user goroutine or a system goroutine. For the next CL, we're going to want to always consider the finalier goroutine a user goroutine, so add a flag that indicates that. Updates #26903. This is preparation for unifying STW GC and concurrent GC. Change-Id: Iafc92e519c13d9f8d879332cb5f0d12164104c33 Reviewed-on: https://go-review.googlesource.com/c/134778 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/heapdump.go | 2 +- src/runtime/mprof.go | 2 +- src/runtime/proc.go | 6 +++--- src/runtime/traceback.go | 13 +++++++++++-- 4 files changed, 16 insertions(+), 7 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/heapdump.go b/src/runtime/heapdump.go index e2c6f18714..eadbcaeee1 100644 --- a/src/runtime/heapdump.go +++ b/src/runtime/heapdump.go @@ -346,7 +346,7 @@ func dumpgoroutine(gp *g) { dumpint(uint64(gp.goid)) dumpint(uint64(gp.gopc)) dumpint(uint64(readgstatus(gp))) - dumpbool(isSystemGoroutine(gp)) + dumpbool(isSystemGoroutine(gp, false)) dumpbool(false) // isbackground dumpint(uint64(gp.waitsince)) dumpstr(gp.waitreason.String()) diff --git a/src/runtime/mprof.go b/src/runtime/mprof.go index 43e4810d97..2bd41b650f 100644 --- a/src/runtime/mprof.go +++ b/src/runtime/mprof.go @@ -723,7 +723,7 @@ func GoroutineProfile(p []StackRecord) (n int, ok bool) { isOK := func(gp1 *g) bool { // Checking isSystemGoroutine here makes GoroutineProfile // consistent with both NumGoroutine and Stack. - return gp1 != gp && readgstatus(gp1) != _Gdead && !isSystemGoroutine(gp1) + return gp1 != gp && readgstatus(gp1) != _Gdead && !isSystemGoroutine(gp1, false) } stopTheWorld("profile") diff --git a/src/runtime/proc.go b/src/runtime/proc.go index ec73b4d918..c477910c9e 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2730,7 +2730,7 @@ func goexit0(gp *g) { _g_ := getg() casgstatus(gp, _Grunning, _Gdead) - if isSystemGoroutine(gp) { + if isSystemGoroutine(gp, false) { atomic.Xadd(&sched.ngsys, -1) } gp.m = nil @@ -3381,7 +3381,7 @@ func newproc1(fn *funcval, argp *uint8, narg int32, callergp *g, callerpc uintpt if _g_.m.curg != nil { newg.labels = _g_.m.curg.labels } - if isSystemGoroutine(newg) { + if isSystemGoroutine(newg, false) { atomic.Xadd(&sched.ngsys, +1) } newg.gcscanvalid = false @@ -4244,7 +4244,7 @@ func checkdead() { lock(&allglock) for i := 0; i < len(allgs); i++ { gp := allgs[i] - if isSystemGoroutine(gp) { + if isSystemGoroutine(gp, false) { continue } s := readgstatus(gp) diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 8e104ae89e..69d5764c8f 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -945,7 +945,7 @@ func tracebackothers(me *g) { lock(&allglock) for _, gp := range allgs { - if gp == me || gp == g.m.curg || readgstatus(gp) == _Gdead || isSystemGoroutine(gp) && level < 2 { + if gp == me || gp == g.m.curg || readgstatus(gp) == _Gdead || isSystemGoroutine(gp, false) && level < 2 { continue } print("\n") @@ -1031,7 +1031,11 @@ func topofstack(f funcInfo, g0 bool) bool { // in stack dumps and deadlock detector. This is any goroutine that // starts at a runtime.* entry point, except for runtime.main and // sometimes runtime.runfinq. -func isSystemGoroutine(gp *g) bool { +// +// If fixed is true, any goroutine that can vary between user and +// system (that is, the finalizer goroutine) is considered a user +// goroutine. +func isSystemGoroutine(gp *g, fixed bool) bool { // Keep this in sync with cmd/trace/trace.go:isSystemGoroutine. f := findfunc(gp.startpc) if !f.valid() { @@ -1043,6 +1047,11 @@ func isSystemGoroutine(gp *g) bool { if f.funcID == funcID_runfinq { // We include the finalizer goroutine if it's calling // back into user code. + if fixed { + // This goroutine can vary. In fixed mode, + // always consider it a user goroutine. + return false + } return !fingRunning } return hasPrefix(funcname(f), "runtime.") -- cgit v1.3-5-g9baa From 6e9fb11b3a314b78f9c2cdb35e3d71a5cce4e06b Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 11 Sep 2018 11:28:24 -0400 Subject: runtime: support disabling goroutine scheduling by class This adds support for disabling the scheduling of user goroutines while allowing system goroutines like the garbage collector to continue running. User goroutines pass through the usual state transitions, but if we attempt to actually schedule one, it will get put on a deferred scheduling list. Updates #26903. This is preparation for unifying STW GC and concurrent GC. Updates #25578. This same mechanism can form the basis for disabling all but a single user goroutine for the purposes of debugger function call injection. Change-Id: Ib72a808e00c25613fe6982f5528160d3de3dbbc6 Reviewed-on: https://go-review.googlesource.com/c/134779 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/proc.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++- src/runtime/runtime2.go | 12 ++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) (limited to 'src/runtime') diff --git a/src/runtime/proc.go b/src/runtime/proc.go index c477910c9e..0a7321254c 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -2615,6 +2615,23 @@ top: resetspinning() } + if sched.disable.user && !schedEnabled(gp) { + // Scheduling of this goroutine is disabled. Put it on + // the list of pending runnable goroutines for when we + // re-enable user scheduling and look again. + lock(&sched.lock) + if schedEnabled(gp) { + // Something re-enabled scheduling while we + // were acquiring the lock. + unlock(&sched.lock) + } else { + sched.disable.runnable.pushBack(gp) + sched.disable.n++ + unlock(&sched.lock) + goto top + } + } + if gp.lockedm != 0 { // Hands off own p to the locked m, // then blocks waiting for a new p. @@ -3033,6 +3050,12 @@ func exitsyscall() { _g_.stackguard0 = _g_.stack.lo + _StackGuard } _g_.throwsplit = false + + if sched.disable.user && !schedEnabled(_g_) { + // Scheduling of this goroutine is disabled. + Gosched() + } + return } @@ -3168,7 +3191,10 @@ func exitsyscall0(gp *g) { casgstatus(gp, _Gsyscall, _Grunnable) dropg() lock(&sched.lock) - _p_ := pidleget() + var _p_ *p + if schedEnabled(_g_) { + _p_ = pidleget() + } if _p_ == nil { globrunqput(gp) } else if atomic.Load(&sched.sysmonwait) != 0 { @@ -4625,6 +4651,40 @@ func schedtrace(detailed bool) { unlock(&sched.lock) } +// schedEnableUser enables or disables the scheduling of user +// goroutines. +// +// This does not stop already running user goroutines, so the caller +// should first stop the world when disabling user goroutines. +func schedEnableUser(enable bool) { + lock(&sched.lock) + if sched.disable.user == !enable { + unlock(&sched.lock) + return + } + sched.disable.user = !enable + if enable { + n := sched.disable.n + sched.disable.n = 0 + globrunqputbatch(&sched.disable.runnable, n) + unlock(&sched.lock) + for ; n != 0 && sched.npidle != 0; n-- { + startm(nil, false) + } + } else { + unlock(&sched.lock) + } +} + +// schedEnabled returns whether gp should be scheduled. It returns +// false is scheduling of gp is disabled. +func schedEnabled(gp *g) bool { + if sched.disable.user { + return isSystemGoroutine(gp, true) + } + return true +} + // Put mp on midle list. // Sched must be locked. // May run during STW, so write barriers are not allowed. diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 259bb376ae..fbca3d3ba6 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -580,6 +580,18 @@ type schedt struct { runq gQueue runqsize int32 + // disable controls selective disabling of the scheduler. + // + // Use schedEnableUser to control this. + // + // disable is protected by sched.lock. + disable struct { + // user disables scheduling of user goroutines. + user bool + runnable gQueue // pending runnable Gs + n int32 // length of runnable + } + // Global cache of dead G's. gFree struct { lock mutex -- cgit v1.3-5-g9baa From 1678b2c580584fa1ea8de3c14df0d8d77b6f7387 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 13 Aug 2018 16:30:54 -0400 Subject: runtime: implement STW GC in terms of concurrent GC Currently, STW GC works very differently from concurrent GC. The largest differences in that in concurrent GC, all marking work is done by background mark workers during the mark phase, while in STW GC, all marking work is done by gchelper during the mark termination phase. This is a consequence of the evolution of Go's GC from a STW GC by incrementally moving work from STW mark termination into concurrent mark. However, at this point, the STW code paths exist only as a debugging mode. Having separate code paths for this increases the maintenance burden and complexity of the garbage collector. At the same time, these code paths aren't tested nearly as well, making it far more likely that they will bit-rot. This CL reverses the relationship between STW GC, by re-implementing STW GC in terms of concurrent GC. This builds on the new scheduled support for disabling user goroutine scheduling. During sweep termination, it disables user scheduling, so when the GC starts the world again for concurrent mark, it's really only "concurrent" with itself. There are several code paths that were specific to STW GC that are now vestigial. We'll remove these in the follow-up CLs. Updates #26903. Change-Id: Ia3883d2fcf7ab1d89bdc9c8ee54bf9bffb32c096 Reviewed-on: https://go-review.googlesource.com/c/134780 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 126 ++++++++++++++++++++++++++++------------------------- 1 file changed, 67 insertions(+), 59 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index b685415872..0bdff3d657 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -455,6 +455,12 @@ func (c *gcControllerState) startCycle() { c.fractionalUtilizationGoal = 0 } + // In STW mode, we just want dedicated workers. + if debug.gcstoptheworld > 0 { + c.dedicatedMarkWorkersNeeded = int64(gomaxprocs) + c.fractionalUtilizationGoal = 0 + } + // Clear per-P state for _, p := range allp { p.gcAssistTime = 0 @@ -1264,9 +1270,7 @@ func gcStart(trigger gcTrigger) { traceGCStart() } - if mode == gcBackgroundMode { - gcBgMarkStartWorkers() - } + gcBgMarkStartWorkers() gcResetMarkState() @@ -1296,65 +1300,65 @@ func gcStart(trigger gcTrigger) { clearpools() work.cycles++ - if mode == gcBackgroundMode { // Do as much work concurrently as possible - gcController.startCycle() - work.heapGoal = memstats.next_gc - // Enter concurrent mark phase and enable - // write barriers. - // - // Because the world is stopped, all Ps will - // observe that write barriers are enabled by - // the time we start the world and begin - // scanning. - // - // Write barriers must be enabled before assists are - // enabled because they must be enabled before - // any non-leaf heap objects are marked. Since - // allocations are blocked until assists can - // happen, we want enable assists as early as - // possible. - setGCPhase(_GCmark) - - gcBgMarkPrepare() // Must happen before assist enable. - gcMarkRootPrepare() - - // Mark all active tinyalloc blocks. Since we're - // allocating from these, they need to be black like - // other allocations. The alternative is to blacken - // the tiny block on every allocation from it, which - // would slow down the tiny allocator. - gcMarkTinyAllocs() - - // At this point all Ps have enabled the write - // barrier, thus maintaining the no white to - // black invariant. Enable mutator assists to - // put back-pressure on fast allocating - // mutators. - atomic.Store(&gcBlackenEnabled, 1) - - // Assists and workers can start the moment we start - // the world. - gcController.markStartTime = now - - // Concurrent mark. - systemstack(func() { - now = startTheWorldWithSema(trace.enabled) - }) + gcController.startCycle() + work.heapGoal = memstats.next_gc + + // In STW mode, disable scheduling of user Gs. This may also + // disable scheduling of this goroutine, so it may block as + // soon as we start the world again. + if mode != gcBackgroundMode { + schedEnableUser(false) + } + + // Enter concurrent mark phase and enable + // write barriers. + // + // Because the world is stopped, all Ps will + // observe that write barriers are enabled by + // the time we start the world and begin + // scanning. + // + // Write barriers must be enabled before assists are + // enabled because they must be enabled before + // any non-leaf heap objects are marked. Since + // allocations are blocked until assists can + // happen, we want enable assists as early as + // possible. + setGCPhase(_GCmark) + + gcBgMarkPrepare() // Must happen before assist enable. + gcMarkRootPrepare() + + // Mark all active tinyalloc blocks. Since we're + // allocating from these, they need to be black like + // other allocations. The alternative is to blacken + // the tiny block on every allocation from it, which + // would slow down the tiny allocator. + gcMarkTinyAllocs() + + // At this point all Ps have enabled the write + // barrier, thus maintaining the no white to + // black invariant. Enable mutator assists to + // put back-pressure on fast allocating + // mutators. + atomic.Store(&gcBlackenEnabled, 1) + + // Assists and workers can start the moment we start + // the world. + gcController.markStartTime = now + + // Concurrent mark. + systemstack(func() { + now = startTheWorldWithSema(trace.enabled) work.pauseNS += now - work.pauseStart work.tMark = now - } else { - if trace.enabled { - // Switch to mark termination STW. - traceGCSTWDone() - traceGCSTWStart(0) - } - t := nanotime() - work.tMark, work.tMarkTerm = t, t - work.heapGoal = work.heap0 - - // Perform mark termination. This will restart the world. - gcMarkTermination(memstats.triggerRatio) + }) + // In STW mode, we could block the instant systemstack + // returns, so don't do anything important here. Make sure we + // block rather than returning to user code. + if mode != gcBackgroundMode { + Gosched() } semrelease(&work.startSema) @@ -1468,6 +1472,10 @@ top: // world again. semrelease(&work.markDoneSema) + // In STW mode, re-enable user goroutines. These will be + // queued to run after we start the world. + schedEnableUser(true) + // endCycle depends on all gcWork cache stats being flushed. // The termination algorithm above ensured that up to // allocations since the ragged barrier. -- cgit v1.3-5-g9baa From 143b13ae82d81020dfa6db40818bef5a1f701c3f Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 16 Aug 2018 12:17:32 -0400 Subject: runtime: clean up remaining mark work check Now that STW GC marking is unified with concurrent marking, there should never be mark work remaining in mark termination. Hence, we can make that check unconditional. Updates #26903. This is a follow-up to unifying STW GC and concurrent GC. Change-Id: I43a21df5577635ab379c397a7405ada68d331e03 Reviewed-on: https://go-review.googlesource.com/c/134781 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 0bdff3d657..b390d031ce 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1905,28 +1905,12 @@ func gcMark(start_time int64) { work.nwait = 0 work.ndone = 0 work.nproc = uint32(gcprocs()) + work.helperDrainBlock = false - if work.full == 0 && work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots == 0 { - // There's no work on the work queue and no root jobs - // that can produce work, so don't bother entering the - // getfull() barrier. There will be flushCacheRoots - // work, but that doesn't gray anything. - // - // This should always be the situation after - // concurrent mark. - work.helperDrainBlock = false - } else { - // There's marking work to do. This is the case during - // STW GC. Instruct GC workers - // to block in getfull until all GC workers are in getfull. - // - // TODO(austin): Move STW marking out of - // mark termination and eliminate this code path. - if debug.gcstoptheworld == 0 { - print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") - panic("non-empty mark queue after concurrent mark") - } - work.helperDrainBlock = true + // Check that there's no marking work remaining. + if work.full != 0 || work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots != 0 { + print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") + panic("non-empty mark queue after concurrent mark") } if work.nproc > 1 { -- cgit v1.3-5-g9baa From 457c8f4fe9e4d45f97d0a3f3c4a80789c6616fd6 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 16 Aug 2018 12:25:38 -0400 Subject: runtime: eliminate blocking GC work drains Now work.helperDrainBlock is always false, so we can remove it and code paths that only ran when it was true. That means we no longer use the gcDrainBlock mode of gcDrain, so we can eliminate that. That means we no longer use gcWork.get, so we can eliminate that. That means we no longer use getfull, so we can eliminate that. Updates #26903. This is a follow-up to unifying STW GC and concurrent GC. Change-Id: I8dbcf8ce24861df0a6149e0b7c5cd0eadb5c13f6 Reviewed-on: https://go-review.googlesource.com/c/134782 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 25 +++----------- src/runtime/mgcmark.go | 44 ++++++++----------------- src/runtime/mgcwork.go | 88 +------------------------------------------------- 3 files changed, 18 insertions(+), 139 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index b390d031ce..d52c8cd791 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -944,14 +944,6 @@ var work struct { ndone uint32 alldone note - // helperDrainBlock indicates that GC mark termination helpers - // should pass gcDrainBlock to gcDrain to block in the - // getfull() barrier. Otherwise, they should pass gcDrainNoBlock. - // - // TODO: This is a temporary fallback to work around races - // that cause early mark termination. - helperDrainBlock bool - // Number of roots of various root types. Set by gcMarkRootPrepare. nFlushCacheRoots int nDataRoots, nBSSRoots, nSpanRoots, nStackRoots int @@ -1528,7 +1520,7 @@ func gcMarkTermination(nextTriggerRatio float64) { gcResetMarkState() initCheckmarks() gcw := &getg().m.p.ptr().gcw - gcDrain(gcw, gcDrainNoBlock) + gcDrain(gcw, 0) wbBufFlush1(getg().m.p.ptr()) gcw.dispose() clearCheckmarks() @@ -1814,7 +1806,7 @@ func gcBgMarkWorker(_p_ *p) { } // Go back to draining, this time // without preemption. - gcDrain(&_p_.gcw, gcDrainNoBlock|gcDrainFlushBgCredit) + gcDrain(&_p_.gcw, gcDrainFlushBgCredit) case gcMarkWorkerFractionalMode: gcDrain(&_p_.gcw, gcDrainFractional|gcDrainUntilPreempt|gcDrainFlushBgCredit) case gcMarkWorkerIdleMode: @@ -1905,7 +1897,6 @@ func gcMark(start_time int64) { work.nwait = 0 work.ndone = 0 work.nproc = uint32(gcprocs()) - work.helperDrainBlock = false // Check that there's no marking work remaining. if work.full != 0 || work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots != 0 { @@ -1921,11 +1912,7 @@ func gcMark(start_time int64) { gchelperstart() gcw := &getg().m.p.ptr().gcw - if work.helperDrainBlock { - gcDrain(gcw, gcDrainBlock) - } else { - gcDrain(gcw, gcDrainNoBlock) - } + gcDrain(gcw, 0) if debug.gccheckmark > 0 { // This is expensive when there's a large number of @@ -2119,11 +2106,7 @@ func gchelper() { // Parallel mark over GC roots and heap if gcphase == _GCmarktermination { gcw := &_g_.m.p.ptr().gcw - if work.helperDrainBlock { - gcDrain(gcw, gcDrainBlock) // blocks in getfull - } else { - gcDrain(gcw, gcDrainNoBlock) - } + gcDrain(gcw, 0) } nproc := atomic.Load(&work.nproc) // work.nproc can change right after we increment work.ndone diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 07b8f791d4..cdbe988a1e 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -771,34 +771,26 @@ type gcDrainFlags int const ( gcDrainUntilPreempt gcDrainFlags = 1 << iota - gcDrainNoBlock gcDrainFlushBgCredit gcDrainIdle gcDrainFractional - - // gcDrainBlock means neither gcDrainUntilPreempt or - // gcDrainNoBlock. It is the default, but callers should use - // the constant for documentation purposes. - gcDrainBlock gcDrainFlags = 0 ) // gcDrain scans roots and objects in work buffers, blackening grey -// objects until all roots and work buffers have been drained. +// objects until it is unable to get more work. It may return before +// GC is done; it's the caller's responsibility to balance work from +// other Ps. // // If flags&gcDrainUntilPreempt != 0, gcDrain returns when g.preempt -// is set. This implies gcDrainNoBlock. +// is set. // // If flags&gcDrainIdle != 0, gcDrain returns when there is other work -// to do. This implies gcDrainNoBlock. +// to do. // // If flags&gcDrainFractional != 0, gcDrain self-preempts when // pollFractionalWorkerExit() returns true. This implies // gcDrainNoBlock. // -// If flags&gcDrainNoBlock != 0, gcDrain returns as soon as it is -// unable to get more work. Otherwise, it will block until all -// blocking calls are blocked in gcDrain. -// // If flags&gcDrainFlushBgCredit != 0, gcDrain flushes scan work // credit to gcController.bgScanCredit every gcCreditSlack units of // scan work. @@ -811,7 +803,6 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) { gp := getg().m.curg preemptible := flags&gcDrainUntilPreempt != 0 - blocking := flags&(gcDrainUntilPreempt|gcDrainIdle|gcDrainFractional|gcDrainNoBlock) == 0 flushBgCredit := flags&gcDrainFlushBgCredit != 0 idle := flags&gcDrainIdle != 0 @@ -855,24 +846,19 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) { gcw.balance() } - var b uintptr - if blocking { - b = gcw.get() - } else { - b = gcw.tryGetFast() + b := gcw.tryGetFast() + if b == 0 { + b = gcw.tryGet() if b == 0 { + // Flush the write barrier + // buffer; this may create + // more work. + wbBufFlush(nil, 0) b = gcw.tryGet() - if b == 0 { - // Flush the write barrier - // buffer; this may create - // more work. - wbBufFlush(nil, 0) - b = gcw.tryGet() - } } } if b == 0 { - // work barrier reached or tryGet failed. + // Unable to get work. break } scanobject(b, gcw) @@ -898,10 +884,6 @@ func gcDrain(gcw *gcWork, flags gcDrainFlags) { } } - // In blocking mode, write barriers are not allowed after this - // point because we must preserve the condition that the work - // buffers are empty. - done: // Flush remaining scan work credit. if gcw.scanWork > 0 { diff --git a/src/runtime/mgcwork.go b/src/runtime/mgcwork.go index 3a99260e53..c32c5eddd7 100644 --- a/src/runtime/mgcwork.go +++ b/src/runtime/mgcwork.go @@ -46,7 +46,7 @@ func init() { // // (preemption must be disabled) // gcw := &getg().m.p.ptr().gcw -// .. call gcw.put() to produce and gcw.get() to consume .. +// .. call gcw.put() to produce and gcw.tryGet() to consume .. // // It's important that any use of gcWork during the mark phase prevent // the garbage collector from transitioning to mark termination since @@ -236,37 +236,6 @@ func (w *gcWork) tryGetFast() uintptr { return wbuf.obj[wbuf.nobj] } -// get dequeues a pointer for the garbage collector to trace, blocking -// if necessary to ensure all pointers from all queues and caches have -// been retrieved. get returns 0 if there are no pointers remaining. -//go:nowritebarrierrec -func (w *gcWork) get() uintptr { - wbuf := w.wbuf1 - if wbuf == nil { - w.init() - wbuf = w.wbuf1 - // wbuf is empty at this point. - } - if wbuf.nobj == 0 { - w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1 - wbuf = w.wbuf1 - if wbuf.nobj == 0 { - owbuf := wbuf - wbuf = getfull() - if wbuf == nil { - return 0 - } - putempty(owbuf) - w.wbuf1 = wbuf - } - } - - // TODO: This might be a good place to add prefetch code - - wbuf.nobj-- - return wbuf.obj[wbuf.nobj] -} - // dispose returns any cached pointers to the global queue. // The buffers are being put on the full queue so that the // write barriers will not simply reacquire them before the @@ -449,61 +418,6 @@ func trygetfull() *workbuf { return b } -// Get a full work buffer off the work.full list. -// If nothing is available wait until all the other gc helpers have -// finished and then return nil. -// getfull acts as a barrier for work.nproc helpers. As long as one -// gchelper is actively marking objects it -// may create a workbuffer that the other helpers can work on. -// The for loop either exits when a work buffer is found -// or when _all_ of the work.nproc GC helpers are in the loop -// looking for work and thus not capable of creating new work. -// This is in fact the termination condition for the STW mark -// phase. -//go:nowritebarrier -func getfull() *workbuf { - b := (*workbuf)(work.full.pop()) - if b != nil { - b.checknonempty() - return b - } - - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait > work.nproc { - println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - for i := 0; ; i++ { - if work.full != 0 { - decnwait := atomic.Xadd(&work.nwait, -1) - if decnwait == work.nproc { - println("runtime: work.nwait=", decnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - b = (*workbuf)(work.full.pop()) - if b != nil { - b.checknonempty() - return b - } - incnwait := atomic.Xadd(&work.nwait, +1) - if incnwait > work.nproc { - println("runtime: work.nwait=", incnwait, "work.nproc=", work.nproc) - throw("work.nwait > work.nproc") - } - } - if work.nwait == work.nproc && work.markrootNext >= work.markrootJobs { - return nil - } - if i < 10 { - procyield(20) - } else if i < 20 { - osyield() - } else { - usleep(100) - } - } -} - //go:nowritebarrier func handoff(b *workbuf) *workbuf { // Make new buffer with half of b's pointers. -- cgit v1.3-5-g9baa From 873bd47dfb34ba4416d4df30180905250b91f137 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 23 Aug 2018 13:14:19 -0400 Subject: runtime: flush mcaches lazily Currently, all mcaches are flushed during STW mark termination as a root marking job. This is currently necessary because all spans must be out of these caches before sweeping begins to avoid races with allocation and to ensure the spans are in the state expected by sweeping. We do it as a root marking job because mcache flushing is somewhat expensive and O(GOMAXPROCS) and this parallelizes the work across the Ps. However, it's also the last remaining root marking job performed during mark termination. This CL moves mcache flushing out of mark termination and performs it lazily. We keep track of the last sweepgen at which each mcache was flushed and as each P is woken from STW, it observes that its mcache is out-of-date and flushes it. The introduces a complication for spans cached in stale mcaches. These may now be observed by background or proportional sweeping or when attempting to add a finalizer, but aren't in a stable state. For example, they are likely to be on the wrong mcentral list. To fix this, this CL extends the sweepgen protocol to also capture whether a span is cached and, if so, whether or not its cache is stale. This protocol blocks asynchronous sweeping from touching cached spans and makes it the responsibility of mcache flushing to sweep the flushed spans. This eliminates the last mark termination root marking job, which means we can now eliminate that entire infrastructure. Updates #26903. This implements lazy mcache flushing. Change-Id: Iadda7aabe540b2026cffc5195da7be37d5b4125e Reviewed-on: https://go-review.googlesource.com/c/134783 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mcache.go | 46 ++++++++++++++++++++++++++++++++++++++--- src/runtime/mcentral.go | 55 +++++++++++++++++++++++++++++++++++-------------- src/runtime/mgc.go | 18 ++++++++++++++++ src/runtime/mgcmark.go | 9 +++----- src/runtime/mgcsweep.go | 9 ++++++-- src/runtime/mheap.go | 4 ++-- src/runtime/proc.go | 5 +++++ 7 files changed, 118 insertions(+), 28 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mcache.go b/src/runtime/mcache.go index 8486f69569..e20e92cdf4 100644 --- a/src/runtime/mcache.go +++ b/src/runtime/mcache.go @@ -4,7 +4,10 @@ package runtime -import "unsafe" +import ( + "runtime/internal/atomic" + "unsafe" +) // Per-thread (in Go, per-P) cache for small objects. // No locking needed because it is per-thread (per-P). @@ -42,6 +45,12 @@ type mcache struct { local_largefree uintptr // bytes freed for large objects (>maxsmallsize) local_nlargefree uintptr // number of frees for large objects (>maxsmallsize) local_nsmallfree [_NumSizeClasses]uintptr // number of frees for small objects (<=maxsmallsize) + + // flushGen indicates the sweepgen during which this mcache + // was last flushed. If flushGen != mheap_.sweepgen, the spans + // in this mcache are stale and need to the flushed so they + // can be swept. This is done in acquirep. + flushGen uint32 } // A gclink is a node in a linked list of blocks, like mlink, @@ -76,6 +85,7 @@ var emptymspan mspan func allocmcache() *mcache { lock(&mheap_.lock) c := (*mcache)(mheap_.cachealloc.alloc()) + c.flushGen = mheap_.sweepgen unlock(&mheap_.lock) for i := range c.alloc { c.alloc[i] = &emptymspan @@ -113,9 +123,12 @@ func (c *mcache) refill(spc spanClass) { if uintptr(s.allocCount) != s.nelems { throw("refill of span with free space remaining") } - if s != &emptymspan { - s.incache = false + // Mark this span as no longer cached. + if s.sweepgen != mheap_.sweepgen+3 { + throw("bad sweepgen in refill") + } + atomic.Store(&s.sweepgen, mheap_.sweepgen) } // Get a new cached span from the central lists. @@ -128,6 +141,10 @@ func (c *mcache) refill(spc spanClass) { throw("span has no free space") } + // Indicate that this span is cached and prevent asynchronous + // sweeping in the next sweep phase. + s.sweepgen = mheap_.sweepgen + 3 + c.alloc[spc] = s } @@ -143,3 +160,26 @@ func (c *mcache) releaseAll() { c.tiny = 0 c.tinyoffset = 0 } + +// prepareForSweep flushes c if the system has entered a new sweep phase +// since c was populated. This must happen between the sweep phase +// starting and the first allocation from c. +func (c *mcache) prepareForSweep() { + // Alternatively, instead of making sure we do this on every P + // between starting the world and allocating on that P, we + // could leave allocate-black on, allow allocation to continue + // as usual, use a ragged barrier at the beginning of sweep to + // ensure all cached spans are swept, and then disable + // allocate-black. However, with this approach it's difficult + // to avoid spilling mark bits into the *next* GC cycle. + sg := mheap_.sweepgen + if c.flushGen == sg { + return + } else if c.flushGen != sg-2 { + println("bad flushGen", c.flushGen, "in prepareForSweep; sweepgen", sg) + throw("bad flushGen") + } + c.releaseAll() + stackcache_clear(c) + atomic.Store(&c.flushGen, mheap_.sweepgen) // Synchronizes with gcStart +} diff --git a/src/runtime/mcentral.go b/src/runtime/mcentral.go index c1e0b472bc..9ca8e5d222 100644 --- a/src/runtime/mcentral.go +++ b/src/runtime/mcentral.go @@ -135,7 +135,6 @@ havespan: // heap_live changed. gcController.revise() } - s.incache = true freeByteBase := s.freeindex &^ (64 - 1) whichByte := freeByteBase / 8 // Init alloc bits cache. @@ -150,28 +149,54 @@ havespan: // Return span from an MCache. func (c *mcentral) uncacheSpan(s *mspan) { - lock(&c.lock) - - s.incache = false - if s.allocCount == 0 { throw("uncaching span but s.allocCount == 0") } cap := int32((s.npages << _PageShift) / s.elemsize) n := cap - int32(s.allocCount) + + // cacheSpan updated alloc assuming all objects on s were + // going to be allocated. Adjust for any that weren't. We must + // do this before potentially sweeping the span. if n > 0 { + atomic.Xadd64(&c.nmalloc, -int64(n)) + } + + sg := mheap_.sweepgen + stale := s.sweepgen == sg+1 + if stale { + // Span was cached before sweep began. It's our + // responsibility to sweep it. + // + // Set sweepgen to indicate it's not cached but needs + // sweeping. sweep will set s.sweepgen to indicate s + // is swept. + s.sweepgen = sg - 1 + s.sweep(true) + // sweep may have freed objects, so recompute n. + n = cap - int32(s.allocCount) + } else { + // Indicate that s is no longer cached. + s.sweepgen = sg + } + + if n > 0 { + lock(&c.lock) c.empty.remove(s) c.nonempty.insert(s) - // mCentral_CacheSpan conservatively counted - // unallocated slots in heap_live. Undo this. - atomic.Xadd64(&memstats.heap_live, -int64(n)*int64(s.elemsize)) - // cacheSpan updated alloc assuming all objects on s - // were going to be allocated. Adjust for any that - // weren't. - atomic.Xadd64(&c.nmalloc, -int64(n)) + if !stale { + // mCentral_CacheSpan conservatively counted + // unallocated slots in heap_live. Undo this. + // + // If this span was cached before sweep, then + // heap_live was totally recomputed since + // caching this span, so we don't do this for + // stale spans. + atomic.Xadd64(&memstats.heap_live, -int64(n)*int64(s.elemsize)) + } + unlock(&c.lock) } - unlock(&c.lock) } // freeSpan updates c and s after sweeping s. @@ -183,13 +208,13 @@ func (c *mcentral) uncacheSpan(s *mspan) { // If preserve=true, it does not move s (the caller // must take care of it). func (c *mcentral) freeSpan(s *mspan, preserve bool, wasempty bool) bool { - if s.incache { + if sg := mheap_.sweepgen; s.sweepgen == sg+1 || s.sweepgen == sg+3 { throw("freeSpan given cached span") } s.needzero = 1 if preserve { - // preserve is set only when called from MCentral_CacheSpan above, + // preserve is set only when called from (un)cacheSpan above, // the span must be in the empty list. if !s.inList() { throw("can't preserve unlinked span") diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index d52c8cd791..83980e6020 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1262,6 +1262,14 @@ func gcStart(trigger gcTrigger) { traceGCStart() } + // Check that all Ps have finished deferred mcache flushes. + for _, p := range allp { + if fg := atomic.Load(&p.mcache.flushGen); fg != mheap_.sweepgen { + println("runtime: p", p.id, "flushGen", fg, "!= sweepgen", mheap_.sweepgen) + throw("p mcache not flushed") + } + } + gcBgMarkStartWorkers() gcResetMarkState() @@ -1606,6 +1614,16 @@ func gcMarkTermination(nextTriggerRatio float64) { // Free stack spans. This must be done between GC cycles. systemstack(freeStackSpans) + // Ensure all mcaches are flushed. Each P will flush its own + // mcache before allocating, but idle Ps may not. Since this + // is necessary to sweep all spans, we need to ensure all + // mcaches are flushed before we start the next GC cycle. + systemstack(func() { + forEachP(func(_p_ *p) { + _p_.mcache.prepareForSweep() + }) + }) + // Print gctrace before dropping worldsema. As soon as we drop // worldsema another cycle could start and smash the stats // we're trying to print. diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index cdbe988a1e..78a597f007 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -52,11 +52,7 @@ const ( // //go:nowritebarrier func gcMarkRootPrepare() { - if gcphase == _GCmarktermination { - work.nFlushCacheRoots = int(gomaxprocs) - } else { - work.nFlushCacheRoots = 0 - } + work.nFlushCacheRoots = 0 // Compute how many data and BSS root blocks there are. nBlocks := func(bytes uintptr) int { @@ -344,7 +340,8 @@ func markrootSpans(gcw *gcWork, shard int) { if s.state != mSpanInUse { continue } - if !useCheckmark && s.sweepgen != sg { + // Check that this span was swept (it may be cached or uncached). + if !useCheckmark && !(s.sweepgen == sg || s.sweepgen == sg+3) { // sweepgen was updated (+2) during non-checkmark GC pass print("sweep ", s.sweepgen, " ", sg, "\n") throw("gc: unswept span") diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index 5cdede002a..00950aede2 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -161,7 +161,8 @@ func (s *mspan) ensureSwept() { } sg := mheap_.sweepgen - if atomic.Load(&s.sweepgen) == sg { + spangen := atomic.Load(&s.sweepgen) + if spangen == sg || spangen == sg+3 { return } // The caller must be sure that the span is a mSpanInUse span. @@ -170,7 +171,11 @@ func (s *mspan) ensureSwept() { return } // unfortunate condition, and we don't have efficient means to wait - for atomic.Load(&s.sweepgen) != sg { + for { + spangen := atomic.Load(&s.sweepgen) + if spangen == sg || spangen == sg+3 { + break + } osyield() } } diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index 2dd66f7c2b..e29af677a2 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -317,6 +317,8 @@ type mspan struct { // if sweepgen == h->sweepgen - 2, the span needs sweeping // if sweepgen == h->sweepgen - 1, the span is currently being swept // if sweepgen == h->sweepgen, the span is swept and ready to use + // if sweepgen == h->sweepgen + 1, the span was cached before sweep began and is still cached, and needs sweeping + // if sweepgen == h->sweepgen + 3, the span was swept and then cached and is still cached // h->sweepgen is incremented by 2 after every GC sweepgen uint32 @@ -324,7 +326,6 @@ type mspan struct { baseMask uint16 // if non-0, elemsize is a power of 2, & this will get object allocation base allocCount uint16 // number of allocated objects spanclass spanClass // size class and noscan (uint8) - incache bool // being used by an mcache state mSpanState // mspaninuse etc needzero uint8 // needs to be zeroed before allocation divShift uint8 // for divide by elemsize - divMagic.shift @@ -1185,7 +1186,6 @@ func (span *mspan) init(base uintptr, npages uintptr) { span.npages = npages span.allocCount = 0 span.spanclass = 0 - span.incache = false span.elemsize = 0 span.state = mSpanDead span.unusedsince = 0 diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 0a7321254c..910918f4b4 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -4119,6 +4119,7 @@ func procresize(nprocs int32) *p { if _g_.m.p != 0 && _g_.m.p.ptr().id < nprocs { // continue to use the current P _g_.m.p.ptr().status = _Prunning + _g_.m.p.ptr().mcache.prepareForSweep() } else { // release the current P and acquire allp[0] if _g_.m.p != 0 { @@ -4169,6 +4170,10 @@ func acquirep(_p_ *p) { _g_ := getg() _g_.m.mcache = _p_.mcache + // Perform deferred mcache flush before this P can allocate + // from a potentially stale mcache. + _p_.mcache.prepareForSweep() + if trace.enabled { traceProcStart() } -- cgit v1.3-5-g9baa From 550dfc8ae1651eb954274045e31f8ef2e95f6c6c Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Thu, 16 Aug 2018 12:32:46 -0400 Subject: runtime: eliminate work.markrootdone and second root marking pass Before STW and concurrent GC were unified, there could be either one or two root marking passes per GC cycle. There were several tasks we had to make sure happened once and only once (whether that was at the beginning of concurrent mark for concurrent GC or during mark termination for STW GC). We kept track of this in work.markrootdone. Now that STW and concurrent GC both use the concurrent marking code and we've eliminated all work done by the second root marking pass, we only ever need a single root marking pass. Hence, we can eliminate work.markrootdone and all of the code that's conditional on it. Updates #26903. Change-Id: I654a0f5e21b9322279525560a31e64b8d33b790f Reviewed-on: https://go-review.googlesource.com/c/134784 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 26 +++---------- src/runtime/mgcmark.go | 102 +++++++++++++++++-------------------------------- 2 files changed, 41 insertions(+), 87 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 83980e6020..253b2df9e4 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -948,14 +948,6 @@ var work struct { nFlushCacheRoots int nDataRoots, nBSSRoots, nSpanRoots, nStackRoots int - // markrootDone indicates that roots have been marked at least - // once during the current GC cycle. This is checked by root - // marking operations that have to happen only during the - // first root marking pass, whether that's during the - // concurrent mark phase in current GC or mark termination in - // STW GC. - markrootDone bool - // Each type of GC state transition is protected by a lock. // Since multiple threads can simultaneously detect the state // transition condition, any thread that detects a transition @@ -1456,9 +1448,6 @@ top: // below. The important thing is that the wb remains active until // all marking is complete. This includes writes made by the GC. - // Record that one root marking pass has completed. - work.markrootDone = true - // Disable assists and background workers. We must do // this before waking blocked assists. atomic.Store(&gcBlackenEnabled, 0) @@ -1909,19 +1898,20 @@ func gcMark(start_time int64) { } work.tstart = start_time - // Queue root marking jobs. - gcMarkRootPrepare() - work.nwait = 0 work.ndone = 0 work.nproc = uint32(gcprocs()) // Check that there's no marking work remaining. - if work.full != 0 || work.nDataRoots+work.nBSSRoots+work.nSpanRoots+work.nStackRoots != 0 { - print("runtime: full=", hex(work.full), " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") + if work.full != 0 || work.markrootNext < work.markrootJobs { + print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") panic("non-empty mark queue after concurrent mark") } + // Clear root marking queue. + work.markrootNext = 0 + work.markrootJobs = 0 + if work.nproc > 1 { noteclear(&work.alldone) helpgc(int32(work.nproc)) @@ -1945,9 +1935,6 @@ func gcMark(start_time int64) { notesleep(&work.alldone) } - // Record that at least one root marking pass has completed. - work.markrootDone = true - // Clear out buffers and double-check that all gcWork caches // are empty. This should be ensured by gcMarkDone before we // enter mark termination. @@ -2061,7 +2048,6 @@ func gcResetMarkState() { work.bytesMarked = 0 work.initialHeapLive = atomic.Load64(&memstats.heap_live) - work.markrootDone = false } // Hooks for other packages diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 78a597f007..0f220dd1b9 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -62,57 +62,41 @@ func gcMarkRootPrepare() { work.nDataRoots = 0 work.nBSSRoots = 0 - // Only scan globals once per cycle; preferably concurrently. - if !work.markrootDone { - for _, datap := range activeModules() { - nDataRoots := nBlocks(datap.edata - datap.data) - if nDataRoots > work.nDataRoots { - work.nDataRoots = nDataRoots - } + // Scan globals. + for _, datap := range activeModules() { + nDataRoots := nBlocks(datap.edata - datap.data) + if nDataRoots > work.nDataRoots { + work.nDataRoots = nDataRoots } + } - for _, datap := range activeModules() { - nBSSRoots := nBlocks(datap.ebss - datap.bss) - if nBSSRoots > work.nBSSRoots { - work.nBSSRoots = nBSSRoots - } + for _, datap := range activeModules() { + nBSSRoots := nBlocks(datap.ebss - datap.bss) + if nBSSRoots > work.nBSSRoots { + work.nBSSRoots = nBSSRoots } } - if !work.markrootDone { - // On the first markroot, we need to scan span roots. - // In concurrent GC, this happens during concurrent - // mark and we depend on addfinalizer to ensure the - // above invariants for objects that get finalizers - // after concurrent mark. In STW GC, this will happen - // during mark termination. - // - // We're only interested in scanning the in-use spans, - // which will all be swept at this point. More spans - // may be added to this list during concurrent GC, but - // we only care about spans that were allocated before - // this mark phase. - work.nSpanRoots = mheap_.sweepSpans[mheap_.sweepgen/2%2].numBlocks() - - // On the first markroot, we need to scan all Gs. Gs - // may be created after this point, but it's okay that - // we ignore them because they begin life without any - // roots, so there's nothing to scan, and any roots - // they create during the concurrent phase will be - // scanned during mark termination. During mark - // termination, allglen isn't changing, so we'll scan - // all Gs. - work.nStackRoots = int(atomic.Loaduintptr(&allglen)) - } else { - // We've already scanned span roots and kept the scan - // up-to-date during concurrent mark. - work.nSpanRoots = 0 - - // The hybrid barrier ensures that stacks can't - // contain pointers to unmarked objects, so on the - // second markroot, there's no need to scan stacks. - work.nStackRoots = 0 - } + // Scan span roots for finalizer specials. + // + // We depend on addfinalizer to mark objects that get + // finalizers after root marking. + // + // We're only interested in scanning the in-use spans, + // which will all be swept at this point. More spans + // may be added to this list during concurrent GC, but + // we only care about spans that were allocated before + // this mark phase. + work.nSpanRoots = mheap_.sweepSpans[mheap_.sweepgen/2%2].numBlocks() + + // Scan stacks. + // + // Gs may be created after this point, but it's okay that we + // ignore them because they begin life without any roots, so + // there's nothing to scan, and any roots they create during + // the concurrent phase will be scanned during mark + // termination. + work.nStackRoots = int(atomic.Loaduintptr(&allglen)) work.markrootNext = 0 work.markrootJobs = uint32(fixedRootCount + work.nFlushCacheRoots + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots) @@ -183,24 +167,15 @@ func markroot(gcw *gcWork, i uint32) { } case i == fixedRootFinalizers: - // Only do this once per GC cycle since we don't call - // queuefinalizer during marking. - if work.markrootDone { - break - } for fb := allfin; fb != nil; fb = fb.alllink { cnt := uintptr(atomic.Load(&fb.cnt)) scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw) } case i == fixedRootFreeGStacks: - // Only do this once per GC cycle; preferably - // concurrently. - if !work.markrootDone { - // Switch to the system stack so we can call - // stackfree. - systemstack(markrootFreeGStacks) - } + // Switch to the system stack so we can call + // stackfree. + systemstack(markrootFreeGStacks) case baseSpans <= i && i < baseStacks: // mark MSpan.specials @@ -324,10 +299,6 @@ func markrootSpans(gcw *gcWork, shard int) { // TODO(austin): There are several ideas for making this more // efficient in issue #11485. - if work.markrootDone { - throw("markrootSpans during second markroot") - } - sg := mheap_.sweepgen spans := mheap_.sweepSpans[mheap_.sweepgen/2%2].block(shard) // Note that work.spans may not include spans that were @@ -719,11 +690,8 @@ func scanstack(gp *g, gcw *gcWork) { throw("can't scan gchelper stack") } - // Shrink the stack if not much of it is being used. During - // concurrent GC, we can do this during concurrent mark. - if !work.markrootDone { - shrinkstack(gp) - } + // Shrink the stack if not much of it is being used. + shrinkstack(gp) // Scan the saved context register. This is effectively a live // register that gets moved back and forth between the -- cgit v1.3-5-g9baa From 0906d648aa173f65aaafd6296d43247b426f05f3 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sun, 26 Aug 2018 21:33:26 -0400 Subject: runtime: eliminate gchelper mechanism Now that we do no mark work during mark termination, we no longer need the gchelper mechanism. Updates #26903. Updates #17503. Change-Id: Ie94e5c0f918cfa047e88cae1028fece106955c1b Reviewed-on: https://go-review.googlesource.com/c/134785 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Rick Hudson --- src/runtime/mgc.go | 68 +++---------------------------------- src/runtime/mgcmark.go | 4 --- src/runtime/proc.go | 89 ++----------------------------------------------- src/runtime/runtime2.go | 1 - src/runtime/stack.go | 4 +-- 5 files changed, 9 insertions(+), 157 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 253b2df9e4..9dfee5a4dc 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -938,11 +938,10 @@ var work struct { markrootNext uint32 // next markroot job markrootJobs uint32 // number of markroot jobs - nproc uint32 - tstart int64 - nwait uint32 - ndone uint32 - alldone note + nproc uint32 + tstart int64 + nwait uint32 + ndone uint32 // Number of roots of various root types. Set by gcMarkRootPrepare. nFlushCacheRoots int @@ -1898,30 +1897,12 @@ func gcMark(start_time int64) { } work.tstart = start_time - work.nwait = 0 - work.ndone = 0 - work.nproc = uint32(gcprocs()) - // Check that there's no marking work remaining. if work.full != 0 || work.markrootNext < work.markrootJobs { print("runtime: full=", hex(work.full), " next=", work.markrootNext, " jobs=", work.markrootJobs, " nDataRoots=", work.nDataRoots, " nBSSRoots=", work.nBSSRoots, " nSpanRoots=", work.nSpanRoots, " nStackRoots=", work.nStackRoots, "\n") panic("non-empty mark queue after concurrent mark") } - // Clear root marking queue. - work.markrootNext = 0 - work.markrootJobs = 0 - - if work.nproc > 1 { - noteclear(&work.alldone) - helpgc(int32(work.nproc)) - } - - gchelperstart() - - gcw := &getg().m.p.ptr().gcw - gcDrain(gcw, 0) - if debug.gccheckmark > 0 { // This is expensive when there's a large number of // Gs, so only do it if checkmark is also enabled. @@ -1931,10 +1912,6 @@ func gcMark(start_time int64) { throw("work.full != 0") } - if work.nproc > 1 { - notesleep(&work.alldone) - } - // Clear out buffers and double-check that all gcWork caches // are empty. This should be ensured by gcMarkDone before we // enter mark termination. @@ -2094,43 +2071,6 @@ func clearpools() { unlock(&sched.deferlock) } -// gchelper runs mark termination tasks on Ps other than the P -// coordinating mark termination. -// -// The caller is responsible for ensuring that this has a P to run on, -// even though it's running during STW. Because of this, it's allowed -// to have write barriers. -// -//go:yeswritebarrierrec -func gchelper() { - _g_ := getg() - _g_.m.traceback = 2 - gchelperstart() - - // Parallel mark over GC roots and heap - if gcphase == _GCmarktermination { - gcw := &_g_.m.p.ptr().gcw - gcDrain(gcw, 0) - } - - nproc := atomic.Load(&work.nproc) // work.nproc can change right after we increment work.ndone - if atomic.Xadd(&work.ndone, +1) == nproc-1 { - notewakeup(&work.alldone) - } - _g_.m.traceback = 0 -} - -func gchelperstart() { - _g_ := getg() - - if _g_.m.helpgc < 0 || _g_.m.helpgc >= _MaxGcproc { - throw("gchelperstart: bad m->helpgc") - } - if _g_ != _g_.m.g0 { - throw("gchelper not running on g0 stack") - } -} - // Timing // itoaDiv formats val/(10**dec) into buf. diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 0f220dd1b9..34e9776d27 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -685,10 +685,6 @@ func scanstack(gp *g, gcw *gcWork) { if gp == getg() { throw("can't scan our own stack") } - mp := gp.m - if mp != nil && mp.helpgc != 0 { - throw("can't scan gchelper stack") - } // Shrink the stack if not much of it is being used. shrinkstack(gp) diff --git a/src/runtime/proc.go b/src/runtime/proc.go index 910918f4b4..db6f908e8c 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -663,59 +663,6 @@ func ready(gp *g, traceskip int, next bool) { } } -func gcprocs() int32 { - // Figure out how many CPUs to use during GC. - // Limited by gomaxprocs, number of actual CPUs, and MaxGcproc. - lock(&sched.lock) - n := gomaxprocs - if n > ncpu { - n = ncpu - } - if n > _MaxGcproc { - n = _MaxGcproc - } - if n > sched.nmidle+1 { // one M is currently running - n = sched.nmidle + 1 - } - unlock(&sched.lock) - return n -} - -func needaddgcproc() bool { - lock(&sched.lock) - n := gomaxprocs - if n > ncpu { - n = ncpu - } - if n > _MaxGcproc { - n = _MaxGcproc - } - n -= sched.nmidle + 1 // one M is currently running - unlock(&sched.lock) - return n > 0 -} - -func helpgc(nproc int32) { - _g_ := getg() - lock(&sched.lock) - pos := 0 - for n := int32(1); n < nproc; n++ { // one M is currently running - if allp[pos].mcache == _g_.m.mcache { - pos++ - } - mp := mget() - if mp == nil { - throw("gcprocs inconsistency") - } - mp.helpgc = n - mp.p.set(allp[pos]) - mp.mcache = allp[pos].mcache - pos++ - notewakeup(&mp.park) - } - unlock(&sched.lock) -} - // freezeStopWait is a large value that freezetheworld sets // sched.stopwait to in order to request that all Gs permanently stop. const freezeStopWait = 0x7fffffff @@ -1132,11 +1079,6 @@ func stopTheWorldWithSema() { } } -func mhelpgc() { - _g_ := getg() - _g_.m.helpgc = -1 -} - func startTheWorldWithSema(emitTraceEvent bool) int64 { _g_ := getg() @@ -1145,7 +1087,6 @@ func startTheWorldWithSema(emitTraceEvent bool) int64 { list := netpoll(false) // non-blocking injectglist(&list) } - add := needaddgcproc() lock(&sched.lock) procs := gomaxprocs @@ -1175,7 +1116,6 @@ func startTheWorldWithSema(emitTraceEvent bool) int64 { } else { // Start M to run P. Do not start another M below. newm(nil, p) - add = false } } @@ -1192,16 +1132,6 @@ func startTheWorldWithSema(emitTraceEvent bool) int64 { wakep() } - if add { - // If GC could have used another helper proc, start one now, - // in the hope that it will be available next time. - // It would have been even better to start it before the collection, - // but doing so requires allocating memory, so it's tricky to - // coordinate. This lazy approach works out in practice: - // we don't mind if the first couple gc rounds don't have quite - // the maximum number of procs. - newm(mhelpgc, nil) - } _g_.m.locks-- if _g_.m.locks == 0 && _g_.preempt { // restore the preemption request in case we've cleared it in newstack _g_.stackguard0 = stackPreempt @@ -1276,10 +1206,7 @@ func mstart1() { fn() } - if _g_.m.helpgc != 0 { - _g_.m.helpgc = 0 - stopm() - } else if _g_.m != &m0 { + if _g_.m != &m0 { acquirep(_g_.m.nextp.ptr()) _g_.m.nextp = 0 } @@ -2003,21 +1930,11 @@ func stopm() { throw("stopm spinning") } -retry: lock(&sched.lock) mput(_g_.m) unlock(&sched.lock) notesleep(&_g_.m.park) noteclear(&_g_.m.park) - if _g_.m.helpgc != 0 { - // helpgc() set _g_.m.p and _g_.m.mcache, so we have a P. - gchelper() - // Undo the effects of helpgc(). - _g_.m.helpgc = 0 - _g_.m.mcache = nil - _g_.m.p = 0 - goto retry - } acquirep(_g_.m.nextp.ptr()) _g_.m.nextp = 0 } @@ -3857,7 +3774,7 @@ func sigprof(pc, sp, lr uintptr, gp *g, mp *m) { pc = funcPC(_ExternalCode) + sys.PCQuantum } stk[0] = pc - if mp.preemptoff != "" || mp.helpgc != 0 { + if mp.preemptoff != "" { stk[1] = funcPC(_GC) + sys.PCQuantum } else { stk[1] = funcPC(_System) + sys.PCQuantum @@ -4634,7 +4551,7 @@ func schedtrace(detailed bool) { if lockedg != nil { id3 = lockedg.goid } - print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " helpgc=", mp.helpgc, " spinning=", mp.spinning, " blocked=", mp.blocked, " lockedg=", id3, "\n") + print(" M", mp.id, ": p=", id1, " curg=", id2, " mallocing=", mp.mallocing, " throwing=", mp.throwing, " preemptoff=", mp.preemptoff, ""+" locks=", mp.locks, " dying=", mp.dying, " spinning=", mp.spinning, " blocked=", mp.blocked, " lockedg=", id3, "\n") } lock(&allglock) diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index fbca3d3ba6..2f009abdbb 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -424,7 +424,6 @@ type m struct { locks int32 dying int32 profilehz int32 - helpgc int32 spinning bool // m is out of work and is actively looking for work blocked bool // m is blocked on a note inwb bool // m is executing a write barrier diff --git a/src/runtime/stack.go b/src/runtime/stack.go index fd9aafd15b..582e94e9d0 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -350,7 +350,7 @@ func stackalloc(n uint32) stack { } var x gclinkptr c := thisg.m.mcache - if stackNoCache != 0 || c == nil || thisg.m.preemptoff != "" || thisg.m.helpgc != 0 { + if stackNoCache != 0 || c == nil || thisg.m.preemptoff != "" { // c == nil can happen in the guts of exitsyscall or // procresize. Just get a stack from the global pool. // Also don't touch stackcache during gc @@ -445,7 +445,7 @@ func stackfree(stk stack) { } x := gclinkptr(v) c := gp.m.mcache - if stackNoCache != 0 || c == nil || gp.m.preemptoff != "" || gp.m.helpgc != 0 { + if stackNoCache != 0 || c == nil || gp.m.preemptoff != "" { lock(&stackpoolmu) stackpoolfree(x, order) unlock(&stackpoolmu) -- cgit v1.3-5-g9baa From b8ac64a581b1e8e033cd7c9919d7dcf8fb16d17b Mon Sep 17 00:00:00 2001 From: Zhou Peng Date: Thu, 17 May 2018 13:45:53 +0000 Subject: all: this big patch remove whitespace from assembly files Don't worry, this patch just remove trailing whitespace from assembly files, and does not touch any logical changes. Change-Id: Ia724ac0b1abf8bc1e41454bdc79289ef317c165d Reviewed-on: https://go-review.googlesource.com/c/113595 Reviewed-by: Brad Fitzpatrick Run-TryBot: Brad Fitzpatrick TryBot-Result: Gobot Gobot --- src/cmd/asm/internal/asm/testdata/386.s | 2 +- src/cmd/asm/internal/asm/testdata/ppc64.s | 2 +- src/crypto/md5/md5block_386.s | 2 +- src/crypto/md5/md5block_amd64.s | 4 +- src/crypto/md5/md5block_amd64p32.s | 4 +- src/crypto/sha1/sha1block_386.s | 8 +-- src/crypto/sha1/sha1block_amd64.s | 10 ++-- src/crypto/sha1/sha1block_amd64p32.s | 10 ++-- src/crypto/sha1/sha1block_arm.s | 10 ++-- src/internal/bytealg/compare_amd64.s | 2 +- src/internal/bytealg/compare_amd64p32.s | 2 +- src/internal/bytealg/equal_amd64.s | 2 +- src/internal/bytealg/equal_amd64p32.s | 2 +- src/internal/bytealg/equal_arm.s | 2 +- src/math/big/arith_386.s | 2 +- src/math/big/arith_amd64.s | 6 +-- src/math/big/arith_arm.s | 4 +- src/math/big/arith_s390x.s | 14 +++--- src/math/expm1_386.s | 4 +- src/math/sin_386.s | 2 +- src/math/sqrt_386.s | 2 +- src/math/sqrt_arm.s | 2 +- src/runtime/asm_386.s | 84 +++++++++++++++---------------- src/runtime/asm_amd64.s | 26 +++++----- src/runtime/asm_amd64p32.s | 4 +- src/runtime/asm_arm.s | 8 +-- src/runtime/cgo/asm_amd64.s | 6 +-- src/runtime/cgo/asm_arm.s | 2 +- src/runtime/cgo/signal_darwin_arm64.s | 2 +- src/runtime/memmove_amd64p32.s | 2 +- src/runtime/memmove_arm.s | 2 +- src/runtime/memmove_plan9_386.s | 2 +- src/runtime/memmove_plan9_amd64.s | 2 +- src/runtime/rt0_linux_ppc64le.s | 2 +- src/runtime/rt0_nacl_amd64p32.s | 2 +- src/runtime/sys_dragonfly_amd64.s | 2 +- src/runtime/sys_freebsd_386.s | 4 +- src/runtime/sys_linux_amd64.s | 2 +- src/runtime/sys_nacl_386.s | 8 +-- src/runtime/sys_nacl_amd64p32.s | 4 +- src/runtime/sys_netbsd_amd64.s | 2 +- src/runtime/sys_openbsd_386.s | 2 +- src/runtime/sys_plan9_386.s | 2 +- src/runtime/sys_plan9_amd64.s | 2 +- src/runtime/sys_solaris_amd64.s | 8 +-- src/runtime/sys_windows_386.s | 8 +-- src/runtime/sys_windows_amd64.s | 2 +- src/runtime/vlop_arm.s | 2 +- src/syscall/asm_nacl_386.s | 2 +- src/syscall/asm_nacl_amd64p32.s | 2 +- src/syscall/asm_nacl_arm.s | 2 +- src/syscall/asm_plan9_386.s | 20 ++++---- src/syscall/asm_plan9_amd64.s | 20 ++++---- 53 files changed, 168 insertions(+), 168 deletions(-) (limited to 'src/runtime') diff --git a/src/cmd/asm/internal/asm/testdata/386.s b/src/cmd/asm/internal/asm/testdata/386.s index 90a66167a1..d524a4c8c1 100644 --- a/src/cmd/asm/internal/asm/testdata/386.s +++ b/src/cmd/asm/internal/asm/testdata/386.s @@ -70,7 +70,7 @@ label: // LTYPEM spec6 { outcode(int($1), &$2); } MOVL AX, BX MOVL $4, BX - + // LTYPEI spec7 { outcode(int($1), &$2); } IMULL AX IMULL $4, CX diff --git a/src/cmd/asm/internal/asm/testdata/ppc64.s b/src/cmd/asm/internal/asm/testdata/ppc64.s index ba00b8f6e2..9e8929dac4 100644 --- a/src/cmd/asm/internal/asm/testdata/ppc64.s +++ b/src/cmd/asm/internal/asm/testdata/ppc64.s @@ -550,7 +550,7 @@ label1: // ftsqrt BF, FRB FTSQRT F2,$7 -// FCFID +// FCFID // FCFIDS FCFID F2,F3 diff --git a/src/crypto/md5/md5block_386.s b/src/crypto/md5/md5block_386.s index 8e426d148f..30d4209a62 100644 --- a/src/crypto/md5/md5block_386.s +++ b/src/crypto/md5/md5block_386.s @@ -122,7 +122,7 @@ loop: ROUND2(DX,AX,BX,CX, 7,0xfcefa3f8, 9); ROUND2(CX,DX,AX,BX,12,0x676f02d9,14); ROUND2(BX,CX,DX,AX, 0,0x8d2a4c8a,20); - + MOVL (5*4)(SI), DI MOVL CX, BP diff --git a/src/crypto/md5/md5block_amd64.s b/src/crypto/md5/md5block_amd64.s index a3ae7d97b2..90d932b146 100644 --- a/src/crypto/md5/md5block_amd64.s +++ b/src/crypto/md5/md5block_amd64.s @@ -99,7 +99,7 @@ loop: ROUND2(DX,AX,BX,CX, 7,0xfcefa3f8, 9); ROUND2(CX,DX,AX,BX,12,0x676f02d9,14); ROUND2(BX,CX,DX,AX, 0,0x8d2a4c8a,20); - + MOVL (5*4)(SI), R8 MOVL CX, R9 @@ -144,7 +144,7 @@ loop: ROLL $shift, a; \ XORL c, R9; \ ADDL b, a - + ROUND4(AX,BX,CX,DX, 7,0xf4292244, 6); ROUND4(DX,AX,BX,CX,14,0x432aff97,10); ROUND4(CX,DX,AX,BX, 5,0xab9423a7,15); diff --git a/src/crypto/md5/md5block_amd64p32.s b/src/crypto/md5/md5block_amd64p32.s index ee05f8613b..7295942dc8 100644 --- a/src/crypto/md5/md5block_amd64p32.s +++ b/src/crypto/md5/md5block_amd64p32.s @@ -103,7 +103,7 @@ loop: ROUND2(DX,AX,BX,CX, 7,0xfcefa3f8, 9); ROUND2(CX,DX,AX,BX,12,0x676f02d9,14); ROUND2(BX,CX,DX,AX, 0,0x8d2a4c8a,20); - + MOVL (5*4)(SI), R8 MOVL CX, R9 @@ -148,7 +148,7 @@ loop: ROLL $shift, a; \ XORL c, R9; \ ADDL b, a - + ROUND4(AX,BX,CX,DX, 7,0xf4292244, 6); ROUND4(DX,AX,BX,CX,14,0x432aff97,10); ROUND4(CX,DX,AX,BX, 5,0xab9423a7,15); diff --git a/src/crypto/sha1/sha1block_386.s b/src/crypto/sha1/sha1block_386.s index 0a7e45a79c..34d023d424 100644 --- a/src/crypto/sha1/sha1block_386.s +++ b/src/crypto/sha1/sha1block_386.s @@ -105,7 +105,7 @@ TEXT ·block(SB),NOSPLIT,$92-16 MOVL p_len+8(FP), DX SHRL $6, DX SHLL $6, DX - + LEAL (SI)(DX*1), DI MOVL (0*4)(BP), AX MOVL (1*4)(BP), BX @@ -148,7 +148,7 @@ loop: ROUND1x(DX, BP, AX, BX, CX, 17) ROUND1x(CX, DX, BP, AX, BX, 18) ROUND1x(BX, CX, DX, BP, AX, 19) - + ROUND2(AX, BX, CX, DX, BP, 20) ROUND2(BP, AX, BX, CX, DX, 21) ROUND2(DX, BP, AX, BX, CX, 22) @@ -169,7 +169,7 @@ loop: ROUND2(DX, BP, AX, BX, CX, 37) ROUND2(CX, DX, BP, AX, BX, 38) ROUND2(BX, CX, DX, BP, AX, 39) - + ROUND3(AX, BX, CX, DX, BP, 40) ROUND3(BP, AX, BX, CX, DX, 41) ROUND3(DX, BP, AX, BX, CX, 42) @@ -190,7 +190,7 @@ loop: ROUND3(DX, BP, AX, BX, CX, 57) ROUND3(CX, DX, BP, AX, BX, 58) ROUND3(BX, CX, DX, BP, AX, 59) - + ROUND4(AX, BX, CX, DX, BP, 60) ROUND4(BP, AX, BX, CX, DX, 61) ROUND4(DX, BP, AX, BX, CX, 62) diff --git a/src/crypto/sha1/sha1block_amd64.s b/src/crypto/sha1/sha1block_amd64.s index 135f113898..42f03fb268 100644 --- a/src/crypto/sha1/sha1block_amd64.s +++ b/src/crypto/sha1/sha1block_amd64.s @@ -60,7 +60,7 @@ MOVL b, R9; \ ANDL c, R9; \ ORL R8, R9 - + #define FUNC4 FUNC2 #define MIX(a, b, c, d, e, const) \ @@ -102,7 +102,7 @@ TEXT ·blockAMD64(SB),NOSPLIT,$64-32 MOVQ p_len+16(FP), DX SHRQ $6, DX SHLQ $6, DX - + LEAQ (SI)(DX*1), DI MOVL (0*4)(BP), AX MOVL (1*4)(BP), BX @@ -141,7 +141,7 @@ loop: ROUND1x(DX, BP, AX, BX, CX, 17) ROUND1x(CX, DX, BP, AX, BX, 18) ROUND1x(BX, CX, DX, BP, AX, 19) - + ROUND2(AX, BX, CX, DX, BP, 20) ROUND2(BP, AX, BX, CX, DX, 21) ROUND2(DX, BP, AX, BX, CX, 22) @@ -162,7 +162,7 @@ loop: ROUND2(DX, BP, AX, BX, CX, 37) ROUND2(CX, DX, BP, AX, BX, 38) ROUND2(BX, CX, DX, BP, AX, 39) - + ROUND3(AX, BX, CX, DX, BP, 40) ROUND3(BP, AX, BX, CX, DX, 41) ROUND3(DX, BP, AX, BX, CX, 42) @@ -183,7 +183,7 @@ loop: ROUND3(DX, BP, AX, BX, CX, 57) ROUND3(CX, DX, BP, AX, BX, 58) ROUND3(BX, CX, DX, BP, AX, 59) - + ROUND4(AX, BX, CX, DX, BP, 60) ROUND4(BP, AX, BX, CX, DX, 61) ROUND4(DX, BP, AX, BX, CX, 62) diff --git a/src/crypto/sha1/sha1block_amd64p32.s b/src/crypto/sha1/sha1block_amd64p32.s index e5404e8997..32058ba01d 100644 --- a/src/crypto/sha1/sha1block_amd64p32.s +++ b/src/crypto/sha1/sha1block_amd64p32.s @@ -55,7 +55,7 @@ MOVL b, R9; \ ANDL c, R9; \ ORL R8, R9 - + #define FUNC4 FUNC2 #define MIX(a, b, c, d, e, const) \ @@ -97,7 +97,7 @@ TEXT ·block(SB),NOSPLIT,$64-16 MOVL p_len+8(FP), DX SHRQ $6, DX SHLQ $6, DX - + LEAQ (SI)(DX*1), DI MOVL (0*4)(R14), AX MOVL (1*4)(R14), BX @@ -131,7 +131,7 @@ loop: ROUND1x(DX, BP, AX, BX, CX, 17) ROUND1x(CX, DX, BP, AX, BX, 18) ROUND1x(BX, CX, DX, BP, AX, 19) - + ROUND2(AX, BX, CX, DX, BP, 20) ROUND2(BP, AX, BX, CX, DX, 21) ROUND2(DX, BP, AX, BX, CX, 22) @@ -152,7 +152,7 @@ loop: ROUND2(DX, BP, AX, BX, CX, 37) ROUND2(CX, DX, BP, AX, BX, 38) ROUND2(BX, CX, DX, BP, AX, 39) - + ROUND3(AX, BX, CX, DX, BP, 40) ROUND3(BP, AX, BX, CX, DX, 41) ROUND3(DX, BP, AX, BX, CX, 42) @@ -173,7 +173,7 @@ loop: ROUND3(DX, BP, AX, BX, CX, 57) ROUND3(CX, DX, BP, AX, BX, 58) ROUND3(BX, CX, DX, BP, AX, 59) - + ROUND4(AX, BX, CX, DX, BP, 60) ROUND4(BP, AX, BX, CX, DX, 61) ROUND4(DX, BP, AX, BX, CX, 62) diff --git a/src/crypto/sha1/sha1block_arm.s b/src/crypto/sha1/sha1block_arm.s index 055edc9f3f..2236533ab4 100644 --- a/src/crypto/sha1/sha1block_arm.s +++ b/src/crypto/sha1/sha1block_arm.s @@ -63,10 +63,10 @@ ORR Rt1<<24, Rt0, Rt0 ; \ MOVW.P Rt0, 4(Rw) ; \ ADD Rt0, Re, Re - + // tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf] // w[i&0xf] = tmp<<1 | tmp>>(32-1) - // e += w[i&0xf] + // e += w[i&0xf] #define SHUFFLE(Re) \ MOVW (-16*4)(Rw), Rt0 ; \ MOVW (-14*4)(Rw), Rt1 ; \ @@ -167,7 +167,7 @@ loop1: ROUND1(Ra, Rb, Rc, Rd, Re) ROUND1x(Rd, Re, Ra, Rb, Rc) ROUND1x(Rc, Rd, Re, Ra, Rb) ROUND1x(Rb, Rc, Rd, Re, Ra) - + MOVW $0x6ED9EBA1, Rconst MOVW $4, Rctr loop2: ROUND2(Ra, Rb, Rc, Rd, Re) @@ -177,7 +177,7 @@ loop2: ROUND2(Ra, Rb, Rc, Rd, Re) ROUND2(Rb, Rc, Rd, Re, Ra) SUB.S $1, Rctr BNE loop2 - + MOVW $0x8F1BBCDC, Rconst MOVW $4, Rctr loop3: ROUND3(Ra, Rb, Rc, Rd, Re) @@ -187,7 +187,7 @@ loop3: ROUND3(Ra, Rb, Rc, Rd, Re) ROUND3(Rb, Rc, Rd, Re, Ra) SUB.S $1, Rctr BNE loop3 - + MOVW $0xCA62C1D6, Rconst MOVW $4, Rctr loop4: ROUND4(Ra, Rb, Rc, Rd, Re) diff --git a/src/internal/bytealg/compare_amd64.s b/src/internal/bytealg/compare_amd64.s index 05bef4aad9..25effbc56f 100644 --- a/src/internal/bytealg/compare_amd64.s +++ b/src/internal/bytealg/compare_amd64.s @@ -63,7 +63,7 @@ loop: ADDQ $16, DI SUBQ $16, R8 JMP loop - + diff64: ADDQ $48, SI ADDQ $48, DI diff --git a/src/internal/bytealg/compare_amd64p32.s b/src/internal/bytealg/compare_amd64p32.s index 85ba6fa9ac..4687fd8a04 100644 --- a/src/internal/bytealg/compare_amd64p32.s +++ b/src/internal/bytealg/compare_amd64p32.s @@ -62,7 +62,7 @@ loop: ADDQ $16, DI SUBQ $16, R8 JMP loop - + // AX = bit mask of differences diff16: BSFQ AX, BX // index of first byte that differs diff --git a/src/internal/bytealg/equal_amd64.s b/src/internal/bytealg/equal_amd64.s index 5263d3040d..b695d9cf42 100644 --- a/src/internal/bytealg/equal_amd64.s +++ b/src/internal/bytealg/equal_amd64.s @@ -79,7 +79,7 @@ TEXT memeqbody<>(SB),NOSPLIT,$0-0 JB bigloop CMPB internal∕cpu·X86+const_offsetX86HasAVX2(SB), $1 JE hugeloop_avx2 - + // 64 bytes at a time using xmm registers hugeloop: CMPQ BX, $64 diff --git a/src/internal/bytealg/equal_amd64p32.s b/src/internal/bytealg/equal_amd64p32.s index c841f98b2f..9be4274c11 100644 --- a/src/internal/bytealg/equal_amd64p32.s +++ b/src/internal/bytealg/equal_amd64p32.s @@ -80,7 +80,7 @@ TEXT memeqbody<>(SB),NOSPLIT,$0-0 CMPQ BX, $8 JB small - + // 64 bytes at a time using xmm registers hugeloop: CMPQ BX, $64 diff --git a/src/internal/bytealg/equal_arm.s b/src/internal/bytealg/equal_arm.s index 6b0d7deed9..e8a92b3cf2 100644 --- a/src/internal/bytealg/equal_arm.s +++ b/src/internal/bytealg/equal_arm.s @@ -9,7 +9,7 @@ TEXT ·Equal(SB),NOSPLIT,$0-25 MOVW a_len+4(FP), R1 MOVW b_len+16(FP), R3 - + CMP R1, R3 // unequal lengths are not equal B.NE notequal diff --git a/src/math/big/arith_386.s b/src/math/big/arith_386.s index 6c080f074a..864fbc554e 100644 --- a/src/math/big/arith_386.s +++ b/src/math/big/arith_386.s @@ -183,7 +183,7 @@ L9: MOVL AX, DX // w = w1 SHRL CX, DX:AX // w>>s | w1<<ŝ MOVL DX, (DI)(BX*4) // z[i] = w>>s | w1<<ŝ ADDL $1, BX // i++ - + E9: CMPL BX, BP JL L9 // i < n-1 diff --git a/src/math/big/arith_amd64.s b/src/math/big/arith_amd64.s index 1b950a4a25..e9c8887523 100644 --- a/src/math/big/arith_amd64.s +++ b/src/math/big/arith_amd64.s @@ -324,10 +324,10 @@ TEXT ·mulAddVWW(SB),NOSPLIT,$0 MOVQ r+56(FP), CX // c = r MOVQ z_len+8(FP), R11 MOVQ $0, BX // i = 0 - + CMPQ R11, $4 JL E5 - + U5: // i+4 <= n // regular loop body unrolled 4x MOVQ (0*8)(R8)(BX*8), AX @@ -355,7 +355,7 @@ U5: // i+4 <= n MOVQ AX, (3*8)(R10)(BX*8) MOVQ DX, CX ADDQ $4, BX // i += 4 - + LEAQ 4(BX), DX CMPQ DX, R11 JLE U5 diff --git a/src/math/big/arith_arm.s b/src/math/big/arith_arm.s index ba65fd2b1f..33aa36f709 100644 --- a/src/math/big/arith_arm.s +++ b/src/math/big/arith_arm.s @@ -123,7 +123,7 @@ TEXT ·shlVU(SB),NOSPLIT,$0 MOVW z_len+4(FP), R5 TEQ $0, R5 BEQ X7 - + MOVW z+0(FP), R1 MOVW x+12(FP), R2 ADD R5<<2, R2, R2 @@ -135,7 +135,7 @@ TEXT ·shlVU(SB),NOSPLIT,$0 MOVW $32, R4 SUB R3, R4 MOVW $0, R7 - + MOVW.W -4(R2), R6 MOVW R6<>R4, R6 diff --git a/src/math/big/arith_s390x.s b/src/math/big/arith_s390x.s index 4520d161d7..9156d9debe 100644 --- a/src/math/big/arith_s390x.s +++ b/src/math/big/arith_s390x.s @@ -54,7 +54,7 @@ TEXT ·divWW(SB),NOSPLIT,$0 TEXT ·addVV(SB),NOSPLIT,$0 MOVD addvectorfacility+0x00(SB),R1 BR (R1) - + TEXT ·addVV_check(SB),NOSPLIT, $0 MOVB ·hasVX(SB), R1 CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported @@ -89,7 +89,7 @@ TEXT ·addVV_vec(SB),NOSPLIT,$0 BLT v1 SUB $12, R3 // n -= 16 BLT A1 // if n < 0 goto A1 - + MOVD R8, R5 MOVD R9, R6 MOVD R2, R7 @@ -291,7 +291,7 @@ E1n: NEG R4, R4 TEXT ·subVV(SB),NOSPLIT,$0 MOVD subvectorfacility+0x00(SB),R1 BR (R1) - + TEXT ·subVV_check(SB),NOSPLIT,$0 MOVB ·hasVX(SB), R1 CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported @@ -321,7 +321,7 @@ TEXT ·subVV_vec(SB),NOSPLIT,$0 MOVD $0, R4 // c = 0 MOVD $0, R0 // make sure it's zero MOVD $0, R10 // i = 0 - + // s/JL/JMP/ below to disable the unrolled loop SUB $4, R3 // n -= 4 BLT v1 // if n < 0 goto v1 @@ -413,7 +413,7 @@ UU1: VLM 0(R5), V1, V4 // 64-bytes into V1..V8 A1: ADD $12, R3 // n += 16 BLT v1 // if n < 0 goto v1 - + U1: // n >= 0 // regular loop body unrolled 4x MOVD 0(R8)(R10*1), R5 @@ -532,7 +532,7 @@ E1: NEG R4, R4 TEXT ·addVW(SB),NOSPLIT,$0 MOVD addwvectorfacility+0x00(SB),R1 BR (R1) - + TEXT ·addVW_check(SB),NOSPLIT,$0 MOVB ·hasVX(SB), R1 CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported @@ -742,7 +742,7 @@ E4: MOVD R4, c+56(FP) // return c TEXT ·subVW(SB),NOSPLIT,$0 MOVD subwvectorfacility+0x00(SB),R1 BR (R1) - + TEXT ·subVW_check(SB),NOSPLIT,$0 MOVB ·hasVX(SB), R1 CMPBEQ R1, $1, vectorimpl // vectorfacility = 1, vector supported diff --git a/src/math/expm1_386.s b/src/math/expm1_386.s index c1392cd52b..d020296ca7 100644 --- a/src/math/expm1_386.s +++ b/src/math/expm1_386.s @@ -8,7 +8,7 @@ TEXT ·Expm1(SB),NOSPLIT,$0 FLDLN2 // F0=log(2) = 1/log2(e) ~ 0.693147 FMOVD x+0(FP), F0 // F0=x, F1=1/log2(e) - FABS // F0=|x|, F1=1/log2(e) + FABS // F0=|x|, F1=1/log2(e) FUCOMPP F0, F1 // compare F0 to F1 FSTSW AX SAHF @@ -36,7 +36,7 @@ use_exp: FSCALE // F0=e**x, F1=int(x*log2(e)) FMOVDP F0, F1 // F0=e**x FLD1 // F0=1, F1=e**x - FSUBDP F0, F1 // F0=e**x-1 + FSUBDP F0, F1 // F0=e**x-1 FMOVDP F0, ret+8(FP) RET not_finite: diff --git a/src/math/sin_386.s b/src/math/sin_386.s index 9d605a1e38..45d12e00c8 100644 --- a/src/math/sin_386.s +++ b/src/math/sin_386.s @@ -24,7 +24,7 @@ TEXT ·Cos(SB),NOSPLIT,$0 FCOS // F0=cos(reduced_x) FMOVDP F0, ret+8(FP) RET - + // func Sin(x float64) float64 TEXT ·Sin(SB),NOSPLIT,$0 FMOVD x+0(FP), F0 // F0=x diff --git a/src/math/sqrt_386.s b/src/math/sqrt_386.s index 402d152785..5a5c33a79a 100644 --- a/src/math/sqrt_386.s +++ b/src/math/sqrt_386.s @@ -4,7 +4,7 @@ #include "textflag.h" -// func Sqrt(x float64) float64 +// func Sqrt(x float64) float64 TEXT ·Sqrt(SB),NOSPLIT,$0 FMOVD x+0(FP),F0 FSQRT diff --git a/src/math/sqrt_arm.s b/src/math/sqrt_arm.s index deb6712553..ffc7d1026d 100644 --- a/src/math/sqrt_arm.s +++ b/src/math/sqrt_arm.s @@ -4,7 +4,7 @@ #include "textflag.h" -// func Sqrt(x float64) float64 +// func Sqrt(x float64) float64 TEXT ·Sqrt(SB),NOSPLIT,$0 MOVB runtime·goarm(SB), R11 CMP $5, R11 diff --git a/src/runtime/asm_386.s b/src/runtime/asm_386.s index 7761415ecd..5bc2063bed 100644 --- a/src/runtime/asm_386.s +++ b/src/runtime/asm_386.s @@ -107,7 +107,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0 MOVL BX, g_stackguard1(BP) MOVL BX, (g_stack+stack_lo)(BP) MOVL SP, (g_stack+stack_hi)(BP) - + // find out information about the processor we're on #ifdef GOOS_nacl // NaCl doesn't like PUSHFL/POPFL JMP has_cpuid @@ -827,7 +827,7 @@ havem: MOVL (g_sched+gobuf_sp)(SI), SP MOVL 0(SP), AX MOVL AX, (g_sched+gobuf_sp)(SI) - + // If the m on entry was nil, we called needm above to borrow an m // for the duration of the call. Since the call is over, return it with dropm. CMPL DX, $0 @@ -942,7 +942,7 @@ TEXT runtime·aeshashbody(SB),NOSPLIT,$0-0 CMPL BX, $64 JBE aes33to64 JMP aes65plus - + aes0to15: TESTL BX, BX JE aes0 @@ -957,7 +957,7 @@ aes0to15: ADDL BX, BX PAND masks<>(SB)(BX*8), X1 -final1: +final1: AESENC X0, X1 // scramble input, xor in seed AESENC X1, X1 // scramble combo 2 times AESENC X1, X1 @@ -987,7 +987,7 @@ aes17to32: // make second starting seed PXOR runtime·aeskeysched+16(SB), X1 AESENC X1, X1 - + // load data to be hashed MOVOU (AX), X2 MOVOU -16(AX)(BX*1), X3 @@ -1015,22 +1015,22 @@ aes33to64: AESENC X1, X1 AESENC X2, X2 AESENC X3, X3 - + MOVOU (AX), X4 MOVOU 16(AX), X5 MOVOU -32(AX)(BX*1), X6 MOVOU -16(AX)(BX*1), X7 - + AESENC X0, X4 AESENC X1, X5 AESENC X2, X6 AESENC X3, X7 - + AESENC X4, X4 AESENC X5, X5 AESENC X6, X6 AESENC X7, X7 - + AESENC X4, X4 AESENC X5, X5 AESENC X6, X6 @@ -1052,7 +1052,7 @@ aes65plus: AESENC X1, X1 AESENC X2, X2 AESENC X3, X3 - + // start with last (possibly overlapping) block MOVOU -64(AX)(BX*1), X4 MOVOU -48(AX)(BX*1), X5 @@ -1068,7 +1068,7 @@ aes65plus: // compute number of remaining 64-byte blocks DECL BX SHRL $6, BX - + aesloop: // scramble state, xor in a block MOVOU (AX), X0 @@ -1095,7 +1095,7 @@ aesloop: AESENC X5, X5 AESENC X6, X6 AESENC X7, X7 - + AESENC X4, X4 AESENC X5, X5 AESENC X6, X6 @@ -1132,77 +1132,77 @@ DATA masks<>+0x00(SB)/4, $0x00000000 DATA masks<>+0x04(SB)/4, $0x00000000 DATA masks<>+0x08(SB)/4, $0x00000000 DATA masks<>+0x0c(SB)/4, $0x00000000 - + DATA masks<>+0x10(SB)/4, $0x000000ff DATA masks<>+0x14(SB)/4, $0x00000000 DATA masks<>+0x18(SB)/4, $0x00000000 DATA masks<>+0x1c(SB)/4, $0x00000000 - + DATA masks<>+0x20(SB)/4, $0x0000ffff DATA masks<>+0x24(SB)/4, $0x00000000 DATA masks<>+0x28(SB)/4, $0x00000000 DATA masks<>+0x2c(SB)/4, $0x00000000 - + DATA masks<>+0x30(SB)/4, $0x00ffffff DATA masks<>+0x34(SB)/4, $0x00000000 DATA masks<>+0x38(SB)/4, $0x00000000 DATA masks<>+0x3c(SB)/4, $0x00000000 - + DATA masks<>+0x40(SB)/4, $0xffffffff DATA masks<>+0x44(SB)/4, $0x00000000 DATA masks<>+0x48(SB)/4, $0x00000000 DATA masks<>+0x4c(SB)/4, $0x00000000 - + DATA masks<>+0x50(SB)/4, $0xffffffff DATA masks<>+0x54(SB)/4, $0x000000ff DATA masks<>+0x58(SB)/4, $0x00000000 DATA masks<>+0x5c(SB)/4, $0x00000000 - + DATA masks<>+0x60(SB)/4, $0xffffffff DATA masks<>+0x64(SB)/4, $0x0000ffff DATA masks<>+0x68(SB)/4, $0x00000000 DATA masks<>+0x6c(SB)/4, $0x00000000 - + DATA masks<>+0x70(SB)/4, $0xffffffff DATA masks<>+0x74(SB)/4, $0x00ffffff DATA masks<>+0x78(SB)/4, $0x00000000 DATA masks<>+0x7c(SB)/4, $0x00000000 - + DATA masks<>+0x80(SB)/4, $0xffffffff DATA masks<>+0x84(SB)/4, $0xffffffff DATA masks<>+0x88(SB)/4, $0x00000000 DATA masks<>+0x8c(SB)/4, $0x00000000 - + DATA masks<>+0x90(SB)/4, $0xffffffff DATA masks<>+0x94(SB)/4, $0xffffffff DATA masks<>+0x98(SB)/4, $0x000000ff DATA masks<>+0x9c(SB)/4, $0x00000000 - + DATA masks<>+0xa0(SB)/4, $0xffffffff DATA masks<>+0xa4(SB)/4, $0xffffffff DATA masks<>+0xa8(SB)/4, $0x0000ffff DATA masks<>+0xac(SB)/4, $0x00000000 - + DATA masks<>+0xb0(SB)/4, $0xffffffff DATA masks<>+0xb4(SB)/4, $0xffffffff DATA masks<>+0xb8(SB)/4, $0x00ffffff DATA masks<>+0xbc(SB)/4, $0x00000000 - + DATA masks<>+0xc0(SB)/4, $0xffffffff DATA masks<>+0xc4(SB)/4, $0xffffffff DATA masks<>+0xc8(SB)/4, $0xffffffff DATA masks<>+0xcc(SB)/4, $0x00000000 - + DATA masks<>+0xd0(SB)/4, $0xffffffff DATA masks<>+0xd4(SB)/4, $0xffffffff DATA masks<>+0xd8(SB)/4, $0xffffffff DATA masks<>+0xdc(SB)/4, $0x000000ff - + DATA masks<>+0xe0(SB)/4, $0xffffffff DATA masks<>+0xe4(SB)/4, $0xffffffff DATA masks<>+0xe8(SB)/4, $0xffffffff DATA masks<>+0xec(SB)/4, $0x0000ffff - + DATA masks<>+0xf0(SB)/4, $0xffffffff DATA masks<>+0xf4(SB)/4, $0xffffffff DATA masks<>+0xf8(SB)/4, $0xffffffff @@ -1217,77 +1217,77 @@ DATA shifts<>+0x00(SB)/4, $0x00000000 DATA shifts<>+0x04(SB)/4, $0x00000000 DATA shifts<>+0x08(SB)/4, $0x00000000 DATA shifts<>+0x0c(SB)/4, $0x00000000 - + DATA shifts<>+0x10(SB)/4, $0xffffff0f DATA shifts<>+0x14(SB)/4, $0xffffffff DATA shifts<>+0x18(SB)/4, $0xffffffff DATA shifts<>+0x1c(SB)/4, $0xffffffff - + DATA shifts<>+0x20(SB)/4, $0xffff0f0e DATA shifts<>+0x24(SB)/4, $0xffffffff DATA shifts<>+0x28(SB)/4, $0xffffffff DATA shifts<>+0x2c(SB)/4, $0xffffffff - + DATA shifts<>+0x30(SB)/4, $0xff0f0e0d DATA shifts<>+0x34(SB)/4, $0xffffffff DATA shifts<>+0x38(SB)/4, $0xffffffff DATA shifts<>+0x3c(SB)/4, $0xffffffff - + DATA shifts<>+0x40(SB)/4, $0x0f0e0d0c DATA shifts<>+0x44(SB)/4, $0xffffffff DATA shifts<>+0x48(SB)/4, $0xffffffff DATA shifts<>+0x4c(SB)/4, $0xffffffff - + DATA shifts<>+0x50(SB)/4, $0x0e0d0c0b DATA shifts<>+0x54(SB)/4, $0xffffff0f DATA shifts<>+0x58(SB)/4, $0xffffffff DATA shifts<>+0x5c(SB)/4, $0xffffffff - + DATA shifts<>+0x60(SB)/4, $0x0d0c0b0a DATA shifts<>+0x64(SB)/4, $0xffff0f0e DATA shifts<>+0x68(SB)/4, $0xffffffff DATA shifts<>+0x6c(SB)/4, $0xffffffff - + DATA shifts<>+0x70(SB)/4, $0x0c0b0a09 DATA shifts<>+0x74(SB)/4, $0xff0f0e0d DATA shifts<>+0x78(SB)/4, $0xffffffff DATA shifts<>+0x7c(SB)/4, $0xffffffff - + DATA shifts<>+0x80(SB)/4, $0x0b0a0908 DATA shifts<>+0x84(SB)/4, $0x0f0e0d0c DATA shifts<>+0x88(SB)/4, $0xffffffff DATA shifts<>+0x8c(SB)/4, $0xffffffff - + DATA shifts<>+0x90(SB)/4, $0x0a090807 DATA shifts<>+0x94(SB)/4, $0x0e0d0c0b DATA shifts<>+0x98(SB)/4, $0xffffff0f DATA shifts<>+0x9c(SB)/4, $0xffffffff - + DATA shifts<>+0xa0(SB)/4, $0x09080706 DATA shifts<>+0xa4(SB)/4, $0x0d0c0b0a DATA shifts<>+0xa8(SB)/4, $0xffff0f0e DATA shifts<>+0xac(SB)/4, $0xffffffff - + DATA shifts<>+0xb0(SB)/4, $0x08070605 DATA shifts<>+0xb4(SB)/4, $0x0c0b0a09 DATA shifts<>+0xb8(SB)/4, $0xff0f0e0d DATA shifts<>+0xbc(SB)/4, $0xffffffff - + DATA shifts<>+0xc0(SB)/4, $0x07060504 DATA shifts<>+0xc4(SB)/4, $0x0b0a0908 DATA shifts<>+0xc8(SB)/4, $0x0f0e0d0c DATA shifts<>+0xcc(SB)/4, $0xffffffff - + DATA shifts<>+0xd0(SB)/4, $0x06050403 DATA shifts<>+0xd4(SB)/4, $0x0a090807 DATA shifts<>+0xd8(SB)/4, $0x0e0d0c0b DATA shifts<>+0xdc(SB)/4, $0xffffff0f - + DATA shifts<>+0xe0(SB)/4, $0x05040302 DATA shifts<>+0xe4(SB)/4, $0x09080706 DATA shifts<>+0xe8(SB)/4, $0x0d0c0b0a DATA shifts<>+0xec(SB)/4, $0xffff0f0e - + DATA shifts<>+0xf0(SB)/4, $0x04030201 DATA shifts<>+0xf4(SB)/4, $0x08070605 DATA shifts<>+0xf8(SB)/4, $0x0c0b0a09 diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 2a15910aea..ab891154c8 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -92,7 +92,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0 ANDQ $~15, SP MOVQ AX, 16(SP) MOVQ BX, 24(SP) - + // create istack out of the given (operating system) stack. // _cgo_init may update stackguard. MOVQ $runtime·g0(SB), DI @@ -273,7 +273,7 @@ TEXT runtime·gogo(SB), NOSPLIT, $16-8 // to keep running g. TEXT runtime·mcall(SB), NOSPLIT, $0-8 MOVQ fn+0(FP), DI - + get_tls(CX) MOVQ g(CX), AX // save state in g->sched MOVQ 0(SP), BX // caller's PC @@ -617,7 +617,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20 MOVQ m_gsignal(R8), SI CMPQ SI, DI JEQ nosave - + // Switch to system stack. MOVQ m_g0(R8), SI CALL gosave<>(SB) @@ -717,7 +717,7 @@ needm: get_tls(CX) MOVQ g(CX), BX MOVQ g_m(BX), BX - + // Set m->sched.sp = SP, so that if a panic happens // during the function we are about to execute, it will // have a valid SP to run on the g0 stack. @@ -801,7 +801,7 @@ havem: MOVQ (g_sched+gobuf_sp)(SI), SP MOVQ 0(SP), AX MOVQ AX, (g_sched+gobuf_sp)(SI) - + // If the m on entry was nil, we called needm above to borrow an m // for the duration of the call. Since the call is over, return it with dropm. CMPQ R8, $0 @@ -953,7 +953,7 @@ aes17to32: // make second starting seed PXOR runtime·aeskeysched+16(SB), X1 AESENC X1, X1 - + // load data to be hashed MOVOU (AX), X2 MOVOU -16(AX)(CX*1), X3 @@ -985,7 +985,7 @@ aes33to64: AESENC X1, X1 AESENC X2, X2 AESENC X3, X3 - + MOVOU (AX), X4 MOVOU 16(AX), X5 MOVOU -32(AX)(CX*1), X6 @@ -995,17 +995,17 @@ aes33to64: PXOR X1, X5 PXOR X2, X6 PXOR X3, X7 - + AESENC X4, X4 AESENC X5, X5 AESENC X6, X6 AESENC X7, X7 - + AESENC X4, X4 AESENC X5, X5 AESENC X6, X6 AESENC X7, X7 - + AESENC X4, X4 AESENC X5, X5 AESENC X6, X6 @@ -1121,7 +1121,7 @@ aes129plus: AESENC X5, X5 AESENC X6, X6 AESENC X7, X7 - + // start with last (possibly overlapping) block MOVOU -128(AX)(CX*1), X8 MOVOU -112(AX)(CX*1), X9 @@ -1141,11 +1141,11 @@ aes129plus: PXOR X5, X13 PXOR X6, X14 PXOR X7, X15 - + // compute number of remaining 128-byte blocks DECQ CX SHRQ $7, CX - + aesloop: // scramble state AESENC X8, X8 diff --git a/src/runtime/asm_amd64p32.s b/src/runtime/asm_amd64p32.s index 49958d0c88..7f194d2403 100644 --- a/src/runtime/asm_amd64p32.s +++ b/src/runtime/asm_amd64p32.s @@ -18,7 +18,7 @@ TEXT runtime·rt0_go(SB),NOSPLIT,$0 MOVL AX, 16(SP) MOVL BX, 24(SP) - + // create istack out of the given (operating system) stack. MOVL $runtime·g0(SB), DI LEAL (-64*1024+104)(SP), BX @@ -150,7 +150,7 @@ TEXT runtime·gogo(SB), NOSPLIT, $8-4 // to keep running g. TEXT runtime·mcall(SB), NOSPLIT, $0-4 MOVL fn+0(FP), DI - + get_tls(CX) MOVL g(CX), AX // save state in g->sched MOVL 0(SP), BX // caller's PC diff --git a/src/runtime/asm_arm.s b/src/runtime/asm_arm.s index ace2b6def7..174dc46389 100644 --- a/src/runtime/asm_arm.s +++ b/src/runtime/asm_arm.s @@ -514,7 +514,7 @@ TEXT callRet<>(SB), NOSPLIT, $16-0 MOVW R1, 12(R13) MOVW R2, 16(R13) BL runtime·reflectcallmove(SB) - RET + RET CALLFN(·call16, 16) CALLFN(·call32, 32) @@ -673,7 +673,7 @@ TEXT runtime·cgocallback(SB),NOSPLIT,$16-16 // See cgocall.go for more details. TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16 NO_LOCAL_POINTERS - + // Load m and g from thread-local storage. MOVB runtime·iscgo(SB), R0 CMP $0, R0 @@ -855,12 +855,12 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$8 // callee-save in the gcc calling convention, so save them here. MOVW R11, saveR11-4(SP) MOVW g, saveG-8(SP) - + BL runtime·load_g(SB) MOVW g_m(g), R0 MOVW m_curg(R0), R0 MOVW (g_stack+stack_hi)(R0), R0 - + MOVW saveG-8(SP), g MOVW saveR11-4(SP), R11 RET diff --git a/src/runtime/cgo/asm_amd64.s b/src/runtime/cgo/asm_amd64.s index 0e33fc4796..06c538b9bc 100644 --- a/src/runtime/cgo/asm_amd64.s +++ b/src/runtime/cgo/asm_amd64.s @@ -36,9 +36,9 @@ TEXT crosscall2(SB),NOSPLIT,$0x110-0 /* also need to save xmm6 - xmm15 */ MOVQ DX, 0x0(SP) /* arg */ MOVQ R8, 0x8(SP) /* argsize (includes padding) */ MOVQ R9, 0x10(SP) /* ctxt */ - + CALL CX /* fn */ - + MOVQ 0x48(SP), DI MOVQ 0x50(SP), SI MOVUPS 0x60(SP), X6 @@ -64,5 +64,5 @@ TEXT crosscall2(SB),NOSPLIT,$0x110-0 /* also need to save xmm6 - xmm15 */ MOVQ 0x30(SP), R13 MOVQ 0x38(SP), R14 MOVQ 0x40(SP), R15 - + RET diff --git a/src/runtime/cgo/asm_arm.s b/src/runtime/cgo/asm_arm.s index 36dab286ae..60132c14a8 100644 --- a/src/runtime/cgo/asm_arm.s +++ b/src/runtime/cgo/asm_arm.s @@ -8,7 +8,7 @@ // func crosscall2(fn func(a unsafe.Pointer, n int32, ctxt uintptr), a unsafe.Pointer, n int32, ctxt uintptr) // Saves C callee-saved registers and calls fn with three arguments. TEXT crosscall2(SB),NOSPLIT|NOFRAME,$0 - /* + /* * We still need to save all callee save register as before, and then * push 3 args for fn (R1, R2, R3). * Also note that at procedure entry in gc world, 4(R13) will be the diff --git a/src/runtime/cgo/signal_darwin_arm64.s b/src/runtime/cgo/signal_darwin_arm64.s index 60443b64c8..1ae00d13f3 100644 --- a/src/runtime/cgo/signal_darwin_arm64.s +++ b/src/runtime/cgo/signal_darwin_arm64.s @@ -37,7 +37,7 @@ ongothread: // Build a 32-byte stack frame for us for this call. // Saved LR (none available) is at the bottom, - // then the PC argument for setsigsegv, + // then the PC argument for setsigsegv, // then a copy of the LR for us to restore. MOVD.W $0, -32(RSP) MOVD R1, 8(RSP) diff --git a/src/runtime/memmove_amd64p32.s b/src/runtime/memmove_amd64p32.s index 8e9fdd14c5..114077311c 100644 --- a/src/runtime/memmove_amd64p32.s +++ b/src/runtime/memmove_amd64p32.s @@ -34,7 +34,7 @@ back: ADDL BX, DI ADDL BX, SI STD - + MOVL BX, CX SHRL $2, CX ANDL $3, BX diff --git a/src/runtime/memmove_arm.s b/src/runtime/memmove_arm.s index 324b21bf7a..8352fb7860 100644 --- a/src/runtime/memmove_arm.s +++ b/src/runtime/memmove_arm.s @@ -138,7 +138,7 @@ _f32loop: CMP TMP, TS BHS _f4tail - MOVM.IA.W (FROM), [R1-R8] + MOVM.IA.W (FROM), [R1-R8] MOVM.IA.W [R1-R8], (TS) B _f32loop diff --git a/src/runtime/memmove_plan9_386.s b/src/runtime/memmove_plan9_386.s index 7ff01940a2..65dec93f6b 100644 --- a/src/runtime/memmove_plan9_386.s +++ b/src/runtime/memmove_plan9_386.s @@ -56,7 +56,7 @@ tail: /* * forward copy loop */ -forward: +forward: MOVL BX, CX SHRL $2, CX ANDL $3, BX diff --git a/src/runtime/memmove_plan9_amd64.s b/src/runtime/memmove_plan9_amd64.s index f18b59f3d2..b729c7c0e7 100644 --- a/src/runtime/memmove_plan9_amd64.s +++ b/src/runtime/memmove_plan9_amd64.s @@ -73,7 +73,7 @@ back: ADDQ BX, CX CMPQ CX, DI JLS forward - + /* * whole thing backwards has * adjusted addresses diff --git a/src/runtime/rt0_linux_ppc64le.s b/src/runtime/rt0_linux_ppc64le.s index 73b9ae392d..54ea9d58f7 100644 --- a/src/runtime/rt0_linux_ppc64le.s +++ b/src/runtime/rt0_linux_ppc64le.s @@ -12,7 +12,7 @@ TEXT _rt0_ppc64le_linux_lib(SB),NOSPLIT,$-8 MOVW CR, R0 // Save CR in caller's frame MOVD R0, 8(R1) MOVDU R1, -320(R1) // Allocate frame. - + // Preserve callee-save registers. MOVD R14, 24(R1) MOVD R15, 32(R1) diff --git a/src/runtime/rt0_nacl_amd64p32.s b/src/runtime/rt0_nacl_amd64p32.s index 54e4b1de89..38583c58b2 100644 --- a/src/runtime/rt0_nacl_amd64p32.s +++ b/src/runtime/rt0_nacl_amd64p32.s @@ -11,7 +11,7 @@ // 8(DI) - argc // 12(DI) - argv, then 0, then envv, then 0, then auxv // NaCl entry here is almost the same, except that there -// is no saved caller PC, so 0(FP) is -8(FP) and so on. +// is no saved caller PC, so 0(FP) is -8(FP) and so on. TEXT _rt0_amd64p32_nacl(SB),NOSPLIT,$16 MOVL DI, 0(SP) CALL runtime·nacl_sysinfo(SB) diff --git a/src/runtime/sys_dragonfly_amd64.s b/src/runtime/sys_dragonfly_amd64.s index b18e967651..b771850aaf 100644 --- a/src/runtime/sys_dragonfly_amd64.s +++ b/src/runtime/sys_dragonfly_amd64.s @@ -9,7 +9,7 @@ #include "go_asm.h" #include "go_tls.h" #include "textflag.h" - + TEXT runtime·sys_umtx_sleep(SB),NOSPLIT,$0 MOVQ addr+0(FP), DI // arg 1 - ptr MOVL val+8(FP), SI // arg 2 - value diff --git a/src/runtime/sys_freebsd_386.s b/src/runtime/sys_freebsd_386.s index 754689ba05..bc309ba453 100644 --- a/src/runtime/sys_freebsd_386.s +++ b/src/runtime/sys_freebsd_386.s @@ -9,7 +9,7 @@ #include "go_asm.h" #include "go_tls.h" #include "textflag.h" - + TEXT runtime·sys_umtx_op(SB),NOSPLIT,$-4 MOVL $454, AX INT $0x80 @@ -39,7 +39,7 @@ TEXT runtime·thr_start(SB),NOSPLIT,$0 POPAL get_tls(CX) MOVL BX, g(CX) - + MOVL AX, g_m(BX) CALL runtime·stackcheck(SB) // smashes AX CALL runtime·mstart(SB) diff --git a/src/runtime/sys_linux_amd64.s b/src/runtime/sys_linux_amd64.s index 7e846371e5..b709f77060 100644 --- a/src/runtime/sys_linux_amd64.s +++ b/src/runtime/sys_linux_amd64.s @@ -608,7 +608,7 @@ TEXT runtime·settls(SB),NOSPLIT,$32 // Same as in sys_darwin_386.s:/ugliness, different constant. // DI currently holds m->tls, which must be fs:0x1d0. // See cgo/gcc_android_amd64.c for the derivation of the constant. - SUBQ $0x1d0, DI // In android, the tls base + SUBQ $0x1d0, DI // In android, the tls base #else ADDQ $8, DI // ELF wants to use -8(FS) #endif diff --git a/src/runtime/sys_nacl_386.s b/src/runtime/sys_nacl_386.s index cdc8ff1a02..24eaeb238c 100644 --- a/src/runtime/sys_nacl_386.s +++ b/src/runtime/sys_nacl_386.s @@ -266,7 +266,7 @@ TEXT runtime·nacl_clock_gettime(SB),NOSPLIT,$8 NACL_SYSCALL(SYS_clock_gettime) MOVL AX, ret+8(FP) RET - + TEXT runtime·nanotime(SB),NOSPLIT,$20 MOVL $0, 0(SP) // real time clock LEAL 8(SP), AX @@ -308,12 +308,12 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$0 // save g MOVL DI, 20(SP) - + // g = m->gsignal MOVL g_m(DI), BX MOVL m_gsignal(BX), BX MOVL BX, g(CX) - + // copy arguments for sighandler MOVL $11, 0(SP) // signal MOVL $0, 4(SP) // siginfo @@ -356,7 +356,7 @@ ret: // Today those registers are just PC and SP, but in case additional registers // are relevant in the future (for example DX is the Go func context register) // we restore as many registers as possible. - // + // // We smash BP, because that's what the linker smashes during RET. // LEAL ctxt+4(FP), BP diff --git a/src/runtime/sys_nacl_amd64p32.s b/src/runtime/sys_nacl_amd64p32.s index 4c4d509576..b4a108346d 100644 --- a/src/runtime/sys_nacl_amd64p32.s +++ b/src/runtime/sys_nacl_amd64p32.s @@ -334,13 +334,13 @@ TEXT runtime·sigtramp(SB),NOSPLIT,$80 // check that g exists get_tls(CX) MOVL g(CX), DI - + CMPL DI, $0 JEQ nog // save g MOVL DI, 20(SP) - + // g = m->gsignal MOVL g_m(DI), BX MOVL m_gsignal(BX), BX diff --git a/src/runtime/sys_netbsd_amd64.s b/src/runtime/sys_netbsd_amd64.s index 5523659196..531c227a7b 100644 --- a/src/runtime/sys_netbsd_amd64.s +++ b/src/runtime/sys_netbsd_amd64.s @@ -23,7 +23,7 @@ TEXT runtime·lwp_create(SB),NOSPLIT,$0 RET TEXT runtime·lwp_tramp(SB),NOSPLIT,$0 - + // Set FS to point at m->tls. LEAQ m_tls(R8), DI CALL runtime·settls(SB) diff --git a/src/runtime/sys_openbsd_386.s b/src/runtime/sys_openbsd_386.s index 8e34ab497a..d555edb71f 100644 --- a/src/runtime/sys_openbsd_386.s +++ b/src/runtime/sys_openbsd_386.s @@ -295,7 +295,7 @@ TEXT runtime·tfork(SB),NOSPLIT,$12 CALL runtime·settls(SB) POPL AX POPAL - + // Now segment is established. Initialize m, g. get_tls(AX) MOVL DX, g(AX) diff --git a/src/runtime/sys_plan9_386.s b/src/runtime/sys_plan9_386.s index 47dcb8db04..a7fb9fe6f7 100644 --- a/src/runtime/sys_plan9_386.s +++ b/src/runtime/sys_plan9_386.s @@ -126,7 +126,7 @@ TEXT runtime·noted(SB),NOSPLIT,$0 INT $64 MOVL AX, ret+4(FP) RET - + TEXT runtime·plan9_semrelease(SB),NOSPLIT,$0 MOVL $38, AX INT $64 diff --git a/src/runtime/sys_plan9_amd64.s b/src/runtime/sys_plan9_amd64.s index 8077d6d324..4ef4aab376 100644 --- a/src/runtime/sys_plan9_amd64.s +++ b/src/runtime/sys_plan9_amd64.s @@ -123,7 +123,7 @@ TEXT runtime·noted(SB),NOSPLIT,$0 SYSCALL MOVL AX, ret+8(FP) RET - + TEXT runtime·plan9_semrelease(SB),NOSPLIT,$0 MOVQ $38, BP SYSCALL diff --git a/src/runtime/sys_solaris_amd64.s b/src/runtime/sys_solaris_amd64.s index 2b6dabab99..930fc88997 100644 --- a/src/runtime/sys_solaris_amd64.s +++ b/src/runtime/sys_solaris_amd64.s @@ -63,9 +63,9 @@ TEXT runtime·pipe1(SB),NOSPLIT,$0 // Call a library function with SysV calling conventions. // The called function can take a maximum of 6 INTEGER class arguments, -// see +// see // Michael Matz, Jan Hubicka, Andreas Jaeger, and Mark Mitchell -// System V Application Binary Interface +// System V Application Binary Interface // AMD64 Architecture Processor Supplement // section 3.2.3. // @@ -119,7 +119,7 @@ skipargs: MOVL 0(AX), AX MOVQ AX, libcall_err(DI) -skiperrno2: +skiperrno2: RET // uint32 tstart_sysvicall(M *newm); @@ -186,7 +186,7 @@ allgood: // Save m->libcall and m->scratch. We need to do this because we // might get interrupted by a signal in runtime·asmcgocall. - // save m->libcall + // save m->libcall MOVQ g_m(R10), BP LEAQ m_libcall(BP), R11 MOVQ libcall_fn(R11), R10 diff --git a/src/runtime/sys_windows_386.s b/src/runtime/sys_windows_386.s index 3c091adcb1..babd91c936 100644 --- a/src/runtime/sys_windows_386.s +++ b/src/runtime/sys_windows_386.s @@ -494,13 +494,13 @@ wall: MOVL (_SYSTEM_TIME+time_hi2), DX CMPL CX, DX JNE wall - + // w = DX:AX // convert to Unix epoch (but still 100ns units) #define delta 116444736000000000 SUBL $(delta & 0xFFFFFFFF), AX SBBL $(delta >> 32), DX - + // nano/100 = DX:AX // split into two decimal halves by div 1e9. // (decimal point is two spots over from correct place, @@ -509,7 +509,7 @@ wall: DIVL CX MOVL AX, DI MOVL DX, SI - + // DI = nano/100/1e9 = nano/1e11 = sec/100, DX = SI = nano/100%1e9 // split DX into seconds and nanoseconds by div 1e7 magic multiply. MOVL DX, AX @@ -520,7 +520,7 @@ wall: IMULL $10000000, DX MOVL SI, CX SUBL DX, CX - + // DI = sec/100 (still) // BX = (nano/100%1e9)/1e7 = (nano/1e9)%100 = sec%100 // CX = (nano/100%1e9)%1e7 = (nano%1e9)/100 = nsec/100 diff --git a/src/runtime/sys_windows_amd64.s b/src/runtime/sys_windows_amd64.s index c9127ac2d2..ec49caa43e 100644 --- a/src/runtime/sys_windows_amd64.s +++ b/src/runtime/sys_windows_amd64.s @@ -89,7 +89,7 @@ TEXT runtime·badsignal2(SB),NOSPLIT|NOFRAME,$48 MOVQ $0, 32(SP) // overlapped MOVQ runtime·_WriteFile(SB), AX CALL AX - + RET // faster get/set last error diff --git a/src/runtime/vlop_arm.s b/src/runtime/vlop_arm.s index 729653488f..41d285874d 100644 --- a/src/runtime/vlop_arm.s +++ b/src/runtime/vlop_arm.s @@ -30,7 +30,7 @@ // func runtime·udiv(n, d uint32) (q, r uint32) // compiler knowns the register usage of this function -// Reference: +// Reference: // Sloss, Andrew et. al; ARM System Developer's Guide: Designing and Optimizing System Software // Morgan Kaufmann; 1 edition (April 8, 2004), ISBN 978-1558608740 #define Rq R0 // input d, output q diff --git a/src/syscall/asm_nacl_386.s b/src/syscall/asm_nacl_386.s index 9d1e541c7c..b989171120 100644 --- a/src/syscall/asm_nacl_386.s +++ b/src/syscall/asm_nacl_386.s @@ -42,4 +42,4 @@ ok: MOVL DX, r2+20(FP) MOVL $0, err+24(FP) CALL runtime·exitsyscall(SB) - RET + RET diff --git a/src/syscall/asm_nacl_amd64p32.s b/src/syscall/asm_nacl_amd64p32.s index b8c097b539..816f7dccfb 100644 --- a/src/syscall/asm_nacl_amd64p32.s +++ b/src/syscall/asm_nacl_amd64p32.s @@ -39,4 +39,4 @@ ok: MOVL DX, r2+20(FP) MOVL $0, err+24(FP) CALL runtime·exitsyscall(SB) - RET + RET diff --git a/src/syscall/asm_nacl_arm.s b/src/syscall/asm_nacl_arm.s index 3e7df1aadf..6092afd9e6 100644 --- a/src/syscall/asm_nacl_arm.s +++ b/src/syscall/asm_nacl_arm.s @@ -41,4 +41,4 @@ ok: MOVW $0, R2 MOVW R2, err+24(FP) BL runtime·exitsyscall(SB) - RET + RET diff --git a/src/syscall/asm_plan9_386.s b/src/syscall/asm_plan9_386.s index 65ae6c77fb..7a2c2daaaa 100644 --- a/src/syscall/asm_plan9_386.s +++ b/src/syscall/asm_plan9_386.s @@ -45,11 +45,11 @@ TEXT ·Syscall(SB),NOSPLIT,$148-32 CALL runtime·gostring(SB) LEAL str-144(SP), SI JMP copyresult3 - + ok3: CALL runtime·exitsyscall(SB) LEAL ·emptystring(SB), SI - + copyresult3: LEAL err+24(FP), DI @@ -78,7 +78,7 @@ TEXT ·Syscall6(SB),NOSPLIT,$148-44 MOVL $0, r2+32(FP) CMPL AX, $-1 JNE ok4 - + LEAL errbuf-128(SP), AX MOVL AX, sysargs-144(SP) MOVL $128, sysargs1-140(SP) @@ -90,11 +90,11 @@ TEXT ·Syscall6(SB),NOSPLIT,$148-44 CALL runtime·gostring(SB) LEAL str-144(SP), SI JMP copyresult4 - + ok4: CALL runtime·exitsyscall(SB) LEAL ·emptystring(SB), SI - + copyresult4: LEAL err+36(FP), DI @@ -144,7 +144,7 @@ TEXT ·seek(SB),NOSPLIT,$24-36 NO_LOCAL_POINTERS LEAL newoffset+20(FP), AX MOVL AX, placeholder+0(FP) - + // copy args down LEAL placeholder+0(FP), SI LEAL sysargs-20(SP), DI @@ -156,19 +156,19 @@ TEXT ·seek(SB),NOSPLIT,$24-36 MOVSL MOVL $SYS_SEEK, AX // syscall entry INT $64 - + CMPL AX, $-1 JNE ok6 MOVL AX, newoffset_lo+20(FP) MOVL AX, newoffset_hi+24(FP) - + CALL syscall·errstr(SB) MOVL SP, SI JMP copyresult6 - + ok6: LEAL ·emptystring(SB), SI - + copyresult6: LEAL err+28(FP), DI diff --git a/src/syscall/asm_plan9_amd64.s b/src/syscall/asm_plan9_amd64.s index bba4012e5c..d5c9f6c63f 100644 --- a/src/syscall/asm_plan9_amd64.s +++ b/src/syscall/asm_plan9_amd64.s @@ -44,11 +44,11 @@ TEXT ·Syscall(SB),NOSPLIT,$168-64 CALL runtime·gostring(SB) LEAQ str-160(SP), SI JMP copyresult3 - + ok3: CALL runtime·exitsyscall(SB) LEAQ ·emptystring(SB), SI - + copyresult3: LEAQ err+48(FP), DI @@ -77,7 +77,7 @@ TEXT ·Syscall6(SB),NOSPLIT,$168-88 MOVQ $0, r2+64(FP) CMPL AX, $-1 JNE ok4 - + LEAQ errbuf-128(SP), AX MOVQ AX, sysargs-160(SP) MOVQ $128, sysargs1-152(SP) @@ -89,11 +89,11 @@ TEXT ·Syscall6(SB),NOSPLIT,$168-88 CALL runtime·gostring(SB) LEAQ str-160(SP), SI JMP copyresult4 - + ok4: CALL runtime·exitsyscall(SB) LEAQ ·emptystring(SB), SI - + copyresult4: LEAQ err+72(FP), DI @@ -143,7 +143,7 @@ TEXT ·seek(SB),NOSPLIT,$48-56 NO_LOCAL_POINTERS LEAQ newoffset+32(FP), AX MOVQ AX, placeholder+0(FP) - + // copy args down LEAQ placeholder+0(FP), SI LEAQ sysargs-40(SP), DI @@ -155,18 +155,18 @@ TEXT ·seek(SB),NOSPLIT,$48-56 MOVSQ MOVQ $SYS_SEEK, BP // syscall entry SYSCALL - + CMPL AX, $-1 JNE ok6 MOVQ AX, newoffset+32(FP) - + CALL syscall·errstr(SB) MOVQ SP, SI JMP copyresult6 - + ok6: LEAQ ·emptystring(SB), SI - + copyresult6: LEAQ err+40(FP), DI -- cgit v1.3-5-g9baa From cbafcc55e80d5b444e659a892b739c04a27980d3 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Sat, 1 Sep 2018 20:16:39 -0700 Subject: cmd/compile,runtime: implement stack objects Rework how the compiler+runtime handles stack-allocated variables whose address is taken. Direct references to such variables work as before. References through pointers, however, use a new mechanism. The new mechanism is more precise than the old "ambiguously live" mechanism. It computes liveness at runtime based on the actual references among objects on the stack. Each function records all of its address-taken objects in a FUNCDATA. These are called "stack objects". The runtime then uses that information while scanning a stack to find all of the stack objects on a stack. It then does a mark phase on the stack objects, using all the pointers found on the stack (and ancillary structures, like defer records) as the root set. Only stack objects which are found to be live during this mark phase will be scanned and thus retain any heap objects they point to. A subsequent CL will remove all the "ambiguously live" logic from the compiler, so that the stack object tracing will be required. For this CL, the stack tracing is all redundant with the current ambiguously live logic. Update #22350 Change-Id: Ide19f1f71a5b6ec8c4d54f8f66f0e9a98344772f Reviewed-on: https://go-review.googlesource.com/c/134155 Reviewed-by: Austin Clements --- src/cmd/compile/internal/gc/obj.go | 5 +- src/cmd/compile/internal/gc/pgen.go | 20 +++ src/cmd/compile/internal/gc/ssa.go | 48 +++++- src/cmd/internal/obj/link.go | 7 +- src/cmd/internal/objabi/funcdata.go | 1 + src/reflect/all_test.go | 3 +- src/runtime/funcdata.h | 1 + src/runtime/mbitmap.go | 21 ++- src/runtime/mgcmark.go | 127 ++++++++++++-- src/runtime/mgcstack.go | 330 ++++++++++++++++++++++++++++++++++++ src/runtime/mheap.go | 2 +- src/runtime/stack.go | 68 +++++++- src/runtime/symtab.go | 1 + 13 files changed, 607 insertions(+), 27 deletions(-) create mode 100644 src/runtime/mgcstack.go (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/obj.go b/src/cmd/compile/internal/gc/obj.go index fb749d171f..19862c03aa 100644 --- a/src/cmd/compile/internal/gc/obj.go +++ b/src/cmd/compile/internal/gc/obj.go @@ -281,7 +281,7 @@ func dumpglobls() { funcsyms = nil } -// addGCLocals adds gcargs and gclocals symbols to Ctxt.Data. +// addGCLocals adds gcargs, gclocals, gcregs, and stack object symbols to Ctxt.Data. // It takes care not to add any duplicates. // Though the object file format handles duplicates efficiently, // storing only a single copy of the data, @@ -299,6 +299,9 @@ func addGCLocals() { Ctxt.Data = append(Ctxt.Data, gcsym) seen[gcsym.Name] = true } + if x := s.Func.StackObjects; x != nil { + ggloblsym(x, int32(len(x.P)), obj.RODATA|obj.LOCAL) + } } } diff --git a/src/cmd/compile/internal/gc/pgen.go b/src/cmd/compile/internal/gc/pgen.go index 563eb9e966..e6bbf04400 100644 --- a/src/cmd/compile/internal/gc/pgen.go +++ b/src/cmd/compile/internal/gc/pgen.go @@ -233,6 +233,26 @@ func compile(fn *Node) { // Set up the function's LSym early to avoid data races with the assemblers. fn.Func.initLSym() + // Make sure type syms are declared for all types that might + // be types of stack objects. We need to do this here + // because symbols must be allocated before the parallel + // phase of the compiler. + if fn.Func.lsym != nil { // not func _(){} + for _, n := range fn.Func.Dcl { + switch n.Class() { + case PPARAM, PPARAMOUT, PAUTO: + if livenessShouldTrack(n) && n.Addrtaken() { + dtypesym(n.Type) + // Also make sure we allocate a linker symbol + // for the stack object data, for the same reason. + if fn.Func.lsym.Func.StackObjects == nil { + fn.Func.lsym.Func.StackObjects = lookup(fmt.Sprintf("%s.stkobj", fn.funcname())).Linksym() + } + } + } + } + } + if compilenow() { compileSSA(fn, 0) } else { diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index eee3a71ba3..663042754e 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -16,6 +16,7 @@ import ( "cmd/compile/internal/ssa" "cmd/compile/internal/types" "cmd/internal/obj" + "cmd/internal/objabi" "cmd/internal/src" "cmd/internal/sys" ) @@ -4933,6 +4934,51 @@ func (s *SSAGenState) DebugFriendlySetPosFrom(v *ssa.Value) { } } +// byXoffset implements sort.Interface for []*Node using Xoffset as the ordering. +type byXoffset []*Node + +func (s byXoffset) Len() int { return len(s) } +func (s byXoffset) Less(i, j int) bool { return s[i].Xoffset < s[j].Xoffset } +func (s byXoffset) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +func emitStackObjects(e *ssafn, pp *Progs) { + var vars []*Node + for _, n := range e.curfn.Func.Dcl { + if livenessShouldTrack(n) && n.Addrtaken() { + vars = append(vars, n) + } + } + if len(vars) == 0 { + return + } + + // Sort variables from lowest to highest address. + sort.Sort(byXoffset(vars)) + + // Populate the stack object data. + // Format must match runtime/stack.go:stackObjectRecord. + x := e.curfn.Func.lsym.Func.StackObjects + off := 0 + off = duintptr(x, off, uint64(len(vars))) + for _, v := range vars { + // Note: arguments and return values have non-negative Xoffset, + // in which case the offset is relative to argp. + // Locals have a negative Xoffset, in which case the offset is relative to varp. + off = duintptr(x, off, uint64(v.Xoffset)) + if !typesym(v.Type).Siggen() { + Fatalf("stack object's type symbol not generated for type %s", v.Type) + } + off = dsymptr(x, off, dtypesym(v.Type), 0) + } + + // Emit a funcdata pointing at the stack object data. + p := pp.Prog(obj.AFUNCDATA) + Addrconst(&p.From, objabi.FUNCDATA_StackObjects) + p.To.Type = obj.TYPE_MEM + p.To.Name = obj.NAME_EXTERN + p.To.Sym = x +} + // genssa appends entries to pp for each instruction in f. func genssa(f *ssa.Func, pp *Progs) { var s SSAGenState @@ -4940,6 +4986,7 @@ func genssa(f *ssa.Func, pp *Progs) { e := f.Frontend().(*ssafn) s.livenessMap = liveness(e, f) + emitStackObjects(e, pp) // Remember where each block starts. s.bstart = make([]*obj.Prog, f.NumBlocks()) @@ -5054,7 +5101,6 @@ func genssa(f *ssa.Func, pp *Progs) { } } } - // Emit control flow instructions for block var next *ssa.Block if i < len(f.Blocks)-1 && Debug['N'] == 0 { diff --git a/src/cmd/internal/obj/link.go b/src/cmd/internal/obj/link.go index 132f7836ef..354bda5e48 100644 --- a/src/cmd/internal/obj/link.go +++ b/src/cmd/internal/obj/link.go @@ -402,9 +402,10 @@ type FuncInfo struct { dwarfAbsFnSym *LSym dwarfIsStmtSym *LSym - GCArgs LSym - GCLocals LSym - GCRegs LSym + GCArgs LSym + GCLocals LSym + GCRegs LSym + StackObjects *LSym } // Attribute is a set of symbol attributes. diff --git a/src/cmd/internal/objabi/funcdata.go b/src/cmd/internal/objabi/funcdata.go index a7827125bf..231d11b185 100644 --- a/src/cmd/internal/objabi/funcdata.go +++ b/src/cmd/internal/objabi/funcdata.go @@ -18,6 +18,7 @@ const ( FUNCDATA_LocalsPointerMaps = 1 FUNCDATA_InlTree = 2 FUNCDATA_RegPointerMaps = 3 + FUNCDATA_StackObjects = 4 // ArgsSizeUnknown is set in Func.argsize to mark all functions // whose argument size is unknown (C vararg functions, and diff --git a/src/reflect/all_test.go b/src/reflect/all_test.go index 5b8bbad383..c463b61c57 100644 --- a/src/reflect/all_test.go +++ b/src/reflect/all_test.go @@ -5988,7 +5988,8 @@ func TestFuncLayout(t *testing.T) { func verifyGCBits(t *testing.T, typ Type, bits []byte) { heapBits := GCBits(New(typ).Interface()) if !bytes.Equal(heapBits, bits) { - t.Errorf("heapBits incorrect for %v\nhave %v\nwant %v", typ, heapBits, bits) + _, _, line, _ := runtime.Caller(1) + t.Errorf("line %d: heapBits incorrect for %v\nhave %v\nwant %v", line, typ, heapBits, bits) } } diff --git a/src/runtime/funcdata.h b/src/runtime/funcdata.h index e6e0306e65..1ee67c8683 100644 --- a/src/runtime/funcdata.h +++ b/src/runtime/funcdata.h @@ -16,6 +16,7 @@ #define FUNCDATA_LocalsPointerMaps 1 #define FUNCDATA_InlTree 2 #define FUNCDATA_RegPointerMaps 3 +#define FUNCDATA_StackObjects 4 // Pseudo-assembly statements. diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 553d83f7f2..5301e692e0 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -1911,6 +1911,20 @@ Run: return totalBits } +// materializeGCProg allocates space for the (1-bit) pointer bitmask +// for an object of size ptrdata. Then it fills that space with the +// pointer bitmask specified by the program prog. +// The bitmask starts at s.startAddr. +// The result must be deallocated with dematerializeGCProg. +func materializeGCProg(ptrdata uintptr, prog *byte) *mspan { + s := mheap_.allocManual((ptrdata/(8*sys.PtrSize)+pageSize-1)/pageSize, &memstats.gc_sys) + runGCProg(addb(prog, 4), nil, (*byte)(unsafe.Pointer(s.startAddr)), 1) + return s +} +func dematerializeGCProg(s *mspan) { + mheap_.freeManual(s, &memstats.gc_sys) +} + func dumpGCProg(p *byte) { nptr := 0 for { @@ -2037,7 +2051,12 @@ func getgcmask(ep interface{}) (mask []byte) { _g_ := getg() gentraceback(_g_.m.curg.sched.pc, _g_.m.curg.sched.sp, 0, _g_.m.curg, 0, nil, 1000, getgcmaskcb, noescape(unsafe.Pointer(&frame)), 0) if frame.fn.valid() { - locals, _ := getStackMap(&frame, nil, false) + // TODO: once stack objects are enabled (and their pointers + // are no longer described by the stack pointermap directly), + // tests using this will probably need fixing. We might need + // to loop through the stackobjects and if we're inside one, + // use the pointermap from that object. + locals, _, _ := getStackMap(&frame, nil, false) if locals.n == 0 { return } diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index 34e9776d27..d4dcfb6cb9 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -169,7 +169,7 @@ func markroot(gcw *gcWork, i uint32) { case i == fixedRootFinalizers: for fb := allfin; fb != nil; fb = fb.alllink { cnt := uintptr(atomic.Load(&fb.cnt)) - scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw) + scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw, nil) } case i == fixedRootFreeGStacks: @@ -248,7 +248,7 @@ func markrootBlock(b0, n0 uintptr, ptrmask0 *uint8, gcw *gcWork, shard int) { } // Scan this shard. - scanblock(b, n, ptrmask, gcw) + scanblock(b, n, ptrmask, gcw, nil) } // markrootFreeGStacks frees stacks of dead Gs. @@ -349,7 +349,7 @@ func markrootSpans(gcw *gcWork, shard int) { scanobject(p, gcw) // The special itself is a root. - scanblock(uintptr(unsafe.Pointer(&spf.fn)), sys.PtrSize, &oneptrmask[0], gcw) + scanblock(uintptr(unsafe.Pointer(&spf.fn)), sys.PtrSize, &oneptrmask[0], gcw, nil) } unlock(&s.speciallock) @@ -689,42 +689,136 @@ func scanstack(gp *g, gcw *gcWork) { // Shrink the stack if not much of it is being used. shrinkstack(gp) + var state stackScanState + state.stack = gp.stack + + if stackTraceDebug { + println("stack trace goroutine", gp.goid) + } + // Scan the saved context register. This is effectively a live // register that gets moved back and forth between the // register and sched.ctxt without a write barrier. if gp.sched.ctxt != nil { - scanblock(uintptr(unsafe.Pointer(&gp.sched.ctxt)), sys.PtrSize, &oneptrmask[0], gcw) + scanblock(uintptr(unsafe.Pointer(&gp.sched.ctxt)), sys.PtrSize, &oneptrmask[0], gcw, &state) } - // Scan the stack. - var cache pcvalueCache + // Scan the stack. Accumulate a list of stack objects. scanframe := func(frame *stkframe, unused unsafe.Pointer) bool { - scanframeworker(frame, &cache, gcw) + scanframeworker(frame, &state, gcw) return true } gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, scanframe, nil, 0) tracebackdefers(gp, scanframe, nil) + + // Find and scan all reachable stack objects. + state.buildIndex() + for { + p := state.getPtr() + if p == 0 { + break + } + obj := state.findObject(p) + if obj == nil { + continue + } + t := obj.typ + if t == nil { + // We've already scanned this object. + continue + } + obj.setType(nil) // Don't scan it again. + if stackTraceDebug { + println(" live stkobj at", hex(state.stack.lo+uintptr(obj.off)), "of type", t.string()) + } + gcdata := t.gcdata + var s *mspan + if t.kind&kindGCProg != 0 { + // This path is pretty unlikely, an object large enough + // to have a GC program allocated on the stack. + // We need some space to unpack the program into a straight + // bitmask, which we allocate/free here. + // TODO: it would be nice if there were a way to run a GC + // program without having to store all its bits. We'd have + // to change from a Lempel-Ziv style program to something else. + // Or we can forbid putting objects on stacks if they require + // a gc program (see issue 27447). + s = materializeGCProg(t.ptrdata, gcdata) + gcdata = (*byte)(unsafe.Pointer(s.startAddr)) + } + + scanblock(state.stack.lo+uintptr(obj.off), t.ptrdata, gcdata, gcw, &state) + + if s != nil { + dematerializeGCProg(s) + } + } + + // Deallocate object buffers. + // (Pointer buffers were all deallocated in the loop above.) + for state.head != nil { + x := state.head + state.head = x.next + if stackTraceDebug { + for _, obj := range x.obj[:x.nobj] { + if obj.typ == nil { // reachable + continue + } + println(" dead stkobj at", hex(gp.stack.lo+uintptr(obj.off)), "of type", obj.typ.string()) + // Note: not necessarily really dead - only reachable-from-ptr dead. + } + } + x.nobj = 0 + putempty((*workbuf)(unsafe.Pointer(x))) + } + if state.buf != nil || state.freeBuf != nil { + throw("remaining pointer buffers") + } + gp.gcscanvalid = true } // Scan a stack frame: local variables and function arguments/results. //go:nowritebarrier -func scanframeworker(frame *stkframe, cache *pcvalueCache, gcw *gcWork) { +func scanframeworker(frame *stkframe, state *stackScanState, gcw *gcWork) { if _DebugGC > 1 && frame.continpc != 0 { print("scanframe ", funcname(frame.fn), "\n") } - locals, args := getStackMap(frame, cache, false) + locals, args, objs := getStackMap(frame, &state.cache, false) // Scan local variables if stack frame has been allocated. if locals.n > 0 { size := uintptr(locals.n) * sys.PtrSize - scanblock(frame.varp-size, size, locals.bytedata, gcw) + scanblock(frame.varp-size, size, locals.bytedata, gcw, state) } // Scan arguments. if args.n > 0 { - scanblock(frame.argp, uintptr(args.n)*sys.PtrSize, args.bytedata, gcw) + scanblock(frame.argp, uintptr(args.n)*sys.PtrSize, args.bytedata, gcw, state) + } + + // Add all stack objects to the stack object list. + if frame.varp != 0 { + // varp is 0 for defers, where there are no locals. + // In that case, there can't be a pointer to its args, either. + // (And all args would be scanned above anyway.) + for _, obj := range objs { + off := obj.off + base := frame.varp // locals base pointer + if off >= 0 { + base = frame.argp // arguments and return values base pointer + } + ptr := base + uintptr(off) + if ptr < frame.sp { + // object hasn't been allocated in the frame yet. + continue + } + if stackTraceDebug { + println("stkobj at", hex(ptr), "of type", obj.typ.string()) + } + state.addObject(ptr, obj.typ) + } } } @@ -939,8 +1033,9 @@ func gcDrainN(gcw *gcWork, scanWork int64) int64 { // This is used to scan non-heap roots, so it does not update // gcw.bytesMarked or gcw.scanWork. // +// If stk != nil, possible stack pointers are also reported to stk.putPtr. //go:nowritebarrier -func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) { +func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) { // Use local copies of original parameters, so that a stack trace // due to one of the throws below shows the original block // base and extent. @@ -957,10 +1052,12 @@ func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork) { for j := 0; j < 8 && i < n; j++ { if bits&1 != 0 { // Same work as in scanobject; see comments there. - obj := *(*uintptr)(unsafe.Pointer(b + i)) - if obj != 0 { - if obj, span, objIndex := findObject(obj, b, i); obj != 0 { + p := *(*uintptr)(unsafe.Pointer(b + i)) + if p != 0 { + if obj, span, objIndex := findObject(p, b, i); obj != 0 { greyobject(obj, b, i, span, gcw, objIndex) + } else if stk != nil && p >= stk.stack.lo && p < stk.stack.hi { + stk.putPtr(p) } } } diff --git a/src/runtime/mgcstack.go b/src/runtime/mgcstack.go new file mode 100644 index 0000000000..86e60d4381 --- /dev/null +++ b/src/runtime/mgcstack.go @@ -0,0 +1,330 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Garbage collector: stack objects and stack tracing +// See the design doc at https://docs.google.com/document/d/1un-Jn47yByHL7I0aVIP_uVCMxjdM5mpelJhiKlIqxkE/edit?usp=sharing +// Also see issue 22350. + +// Stack tracing solves the problem of determining which parts of the +// stack are live and should be scanned. It runs as part of scanning +// a single goroutine stack. +// +// Normally determining which parts of the stack are live is easy to +// do statically, as user code has explicit references (reads and +// writes) to stack variables. The compiler can do a simple dataflow +// analysis to determine liveness of stack variables at every point in +// the code. See cmd/compile/internal/gc/plive.go for that analysis. +// +// However, when we take the address of a stack variable, determining +// whether that variable is still live is less clear. We can still +// look for static accesses, but accesses through a pointer to the +// variable are difficult in general to track statically. That pointer +// can be passed among functions on the stack, conditionally retained, +// etc. +// +// Instead, we will track pointers to stack variables dynamically. +// All pointers to stack-allocated variables will themselves be on the +// stack somewhere (or in associated locations, like defer records), so +// we can find them all efficiently. +// +// Stack tracing is organized as a mini garbage collection tracing +// pass. The objects in this garbage collection are all the variables +// on the stack whose address is taken, and which themselves contain a +// pointer. We call these variables "stack objects". +// +// We begin by determining all the stack objects on the stack and all +// the statically live pointers that may point into the stack. We then +// process each pointer to see if it points to a stack object. If it +// does, we scan that stack object. It may contain pointers into the +// heap, in which case those pointers are passed to the main garbage +// collection. It may also contain pointers into the stack, in which +// case we add them to our set of stack pointers. +// +// Once we're done processing all the pointers (including the ones we +// added during processing), we've found all the stack objects that +// are live. Any dead stack objects are not scanned and their contents +// will not keep heap objects live. Unlike the main garbage +// collection, we can't sweep the dead stack objects; they live on in +// a moribund state until the stack frame that contains them is +// popped. +// +// A stack can look like this: +// +// +----------+ +// | foo() | +// | +------+ | +// | | A | | <---\ +// | +------+ | | +// | | | +// | +------+ | | +// | | B | | | +// | +------+ | | +// | | | +// +----------+ | +// | bar() | | +// | +------+ | | +// | | C | | <-\ | +// | +----|-+ | | | +// | | | | | +// | +----v-+ | | | +// | | D ---------/ +// | +------+ | | +// | | | +// +----------+ | +// | baz() | | +// | +------+ | | +// | | E -------/ +// | +------+ | +// | ^ | +// | F: --/ | +// | | +// +----------+ +// +// foo() calls bar() calls baz(). Each has a frame on the stack. +// foo() has stack objects A and B. +// bar() has stack objects C and D, with C pointing to D and D pointing to A. +// baz() has a stack object E pointing to C, and a local variable F pointing to E. +// +// Starting from the pointer in local variable F, we will eventually +// scan all of E, C, D, and A (in that order). B is never scanned +// because there is no live pointer to it. If B is also statically +// dead (meaning that foo() never accesses B again after it calls +// bar()), then B's pointers into the heap are not considered live. + +package runtime + +import ( + "runtime/internal/sys" + "unsafe" +) + +const stackTraceDebug = false + +// Buffer for pointers found during stack tracing. +// Must be smaller than or equal to workbuf. +// +//go:notinheap +type stackWorkBuf struct { + stackWorkBufHdr + obj [(_WorkbufSize - unsafe.Sizeof(stackWorkBufHdr{})) / sys.PtrSize]uintptr +} + +// Header declaration must come after the buf declaration above, because of issue #14620. +// +//go:notinheap +type stackWorkBufHdr struct { + workbufhdr + next *stackWorkBuf // linked list of workbufs + // Note: we could theoretically repurpose lfnode.next as this next pointer. + // It would save 1 word, but that probably isn't worth busting open + // the lfnode API. +} + +// Buffer for stack objects found on a goroutine stack. +// Must be smaller than or equal to workbuf. +// +//go:notinheap +type stackObjectBuf struct { + stackObjectBufHdr + obj [(_WorkbufSize - unsafe.Sizeof(stackObjectBufHdr{})) / unsafe.Sizeof(stackObject{})]stackObject +} + +//go:notinheap +type stackObjectBufHdr struct { + workbufhdr + next *stackObjectBuf +} + +func init() { + if unsafe.Sizeof(stackWorkBuf{}) > unsafe.Sizeof(workbuf{}) { + panic("stackWorkBuf too big") + } + if unsafe.Sizeof(stackObjectBuf{}) > unsafe.Sizeof(workbuf{}) { + panic("stackObjectBuf too big") + } +} + +// A stackObject represents a variable on the stack that has had +// its address taken. +// +//go:notinheap +type stackObject struct { + off uint32 // offset above stack.lo + size uint32 // size of object + typ *_type // type info (for ptr/nonptr bits). nil if object has been scanned. + left *stackObject // objects with lower addresses + right *stackObject // objects with higher addresses +} + +// obj.typ = typ, but with no write barrier. +//go:nowritebarrier +func (obj *stackObject) setType(typ *_type) { + // Types of stack objects are always in read-only memory, not the heap. + // So not using a write barrier is ok. + *(*uintptr)(unsafe.Pointer(&obj.typ)) = uintptr(unsafe.Pointer(typ)) +} + +// A stackScanState keeps track of the state used during the GC walk +// of a goroutine. +// +//go:notinheap +type stackScanState struct { + cache pcvalueCache + + // stack limits + stack stack + + // buf contains the set of possible pointers to stack objects. + // Organized as a LIFO linked list of buffers. + // All buffers except possibly the head buffer are full. + buf *stackWorkBuf + freeBuf *stackWorkBuf // keep around one free buffer for allocation hysteresis + + // list of stack objects + // Objects are in increasing address order. + head *stackObjectBuf + tail *stackObjectBuf + nobjs int + + // root of binary tree for fast object lookup by address + // Initialized by buildIndex. + root *stackObject +} + +// Add p as a potential pointer to a stack object. +// p must be a stack address. +func (s *stackScanState) putPtr(p uintptr) { + if p < s.stack.lo || p >= s.stack.hi { + throw("address not a stack address") + } + buf := s.buf + if buf == nil { + // Initial setup. + buf = (*stackWorkBuf)(unsafe.Pointer(getempty())) + buf.nobj = 0 + buf.next = nil + s.buf = buf + } else if buf.nobj == len(buf.obj) { + if s.freeBuf != nil { + buf = s.freeBuf + s.freeBuf = nil + } else { + buf = (*stackWorkBuf)(unsafe.Pointer(getempty())) + } + buf.nobj = 0 + buf.next = s.buf + s.buf = buf + } + buf.obj[buf.nobj] = p + buf.nobj++ +} + +// Remove and return a potential pointer to a stack object. +// Returns 0 if there are no more pointers available. +func (s *stackScanState) getPtr() uintptr { + buf := s.buf + if buf == nil { + // Never had any data. + return 0 + } + if buf.nobj == 0 { + if s.freeBuf != nil { + // Free old freeBuf. + putempty((*workbuf)(unsafe.Pointer(s.freeBuf))) + } + // Move buf to the freeBuf. + s.freeBuf = buf + buf = buf.next + s.buf = buf + if buf == nil { + // No more data. + putempty((*workbuf)(unsafe.Pointer(s.freeBuf))) + s.freeBuf = nil + return 0 + } + } + buf.nobj-- + return buf.obj[buf.nobj] +} + +// addObject adds a stack object at addr of type typ to the set of stack objects. +func (s *stackScanState) addObject(addr uintptr, typ *_type) { + x := s.tail + if x == nil { + // initial setup + x = (*stackObjectBuf)(unsafe.Pointer(getempty())) + x.next = nil + s.head = x + s.tail = x + } + if x.nobj > 0 && uint32(addr-s.stack.lo) < x.obj[x.nobj-1].off+x.obj[x.nobj-1].size { + throw("objects added out of order or overlapping") + } + if x.nobj == len(x.obj) { + // full buffer - allocate a new buffer, add to end of linked list + y := (*stackObjectBuf)(unsafe.Pointer(getempty())) + y.next = nil + x.next = y + s.tail = y + x = y + } + obj := &x.obj[x.nobj] + x.nobj++ + obj.off = uint32(addr - s.stack.lo) + obj.size = uint32(typ.size) + obj.setType(typ) + // obj.left and obj.right will be initalized by buildIndex before use. + s.nobjs++ +} + +// buildIndex initializes s.root to a binary search tree. +// It should be called after all addObject calls but before +// any call of findObject. +func (s *stackScanState) buildIndex() { + s.root, _, _ = binarySearchTree(s.head, 0, s.nobjs) +} + +// Build a binary search tree with the n objects in the list +// x.obj[idx], x.obj[idx+1], ..., x.next.obj[0], ... +// Returns the root of that tree, and the buf+idx of the nth object after x.obj[idx]. +// (The first object that was not included in the binary search tree.) +// If n == 0, returns nil, x. +func binarySearchTree(x *stackObjectBuf, idx int, n int) (root *stackObject, restBuf *stackObjectBuf, restIdx int) { + if n == 0 { + return nil, x, idx + } + var left, right *stackObject + left, x, idx = binarySearchTree(x, idx, n/2) + root = &x.obj[idx] + idx++ + if idx == len(x.obj) { + x = x.next + idx = 0 + } + right, x, idx = binarySearchTree(x, idx, n-n/2-1) + root.left = left + root.right = right + return root, x, idx +} + +// findObject returns the stack object containing address a, if any. +// Must have called buildIndex previously. +func (s *stackScanState) findObject(a uintptr) *stackObject { + off := uint32(a - s.stack.lo) + obj := s.root + for { + if obj == nil { + return nil + } + if off < obj.off { + obj = obj.left + continue + } + if off >= obj.off+obj.size { + obj = obj.right + continue + } + return obj + } +} diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index e29af677a2..7a11bdc058 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -1437,7 +1437,7 @@ func addfinalizer(p unsafe.Pointer, f *funcval, nret uintptr, fint *_type, ot *p scanobject(base, gcw) // Mark the finalizer itself, since the // special isn't part of the GC'd heap. - scanblock(uintptr(unsafe.Pointer(&s.fn)), sys.PtrSize, &oneptrmask[0], gcw) + scanblock(uintptr(unsafe.Pointer(&s.fn)), sys.PtrSize, &oneptrmask[0], gcw, nil) releasem(mp) } return true diff --git a/src/runtime/stack.go b/src/runtime/stack.go index 582e94e9d0..b815aa859e 100644 --- a/src/runtime/stack.go +++ b/src/runtime/stack.go @@ -625,7 +625,7 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { return true } - locals, args := getStackMap(frame, &adjinfo.cache, true) + locals, args, objs := getStackMap(frame, &adjinfo.cache, true) // Adjust local variables if stack frame has been allocated. if locals.n > 0 { @@ -663,6 +663,42 @@ func adjustframe(frame *stkframe, arg unsafe.Pointer) bool { } adjustpointers(unsafe.Pointer(frame.argp), &args, adjinfo, funcInfo{}) } + + // Adjust pointers in all stack objects (whether they are live or not). + // See comments in mgcmark.go:scanframeworker. + if frame.varp != 0 { + for _, obj := range objs { + off := obj.off + base := frame.varp // locals base pointer + if off >= 0 { + base = frame.argp // arguments and return values base pointer + } + p := base + uintptr(off) + if p < frame.sp { + // Object hasn't been allocated in the frame yet. + // (Happens when the stack bounds check fails and + // we call into morestack.) + continue + } + t := obj.typ + gcdata := t.gcdata + var s *mspan + if t.kind&kindGCProg != 0 { + // See comments in mgcmark.go:scanstack + s = materializeGCProg(t.ptrdata, gcdata) + gcdata = (*byte)(unsafe.Pointer(s.startAddr)) + } + for i := uintptr(0); i < t.ptrdata; i += sys.PtrSize { + if *addb(gcdata, i/(8*sys.PtrSize))>>(i/sys.PtrSize&7)&1 != 0 { + adjustpointer(adjinfo, unsafe.Pointer(p+i)) + } + } + if s != nil { + dematerializeGCProg(s) + } + } + } + return true } @@ -1136,9 +1172,9 @@ func freeStackSpans() { unlock(&stackLarge.lock) } -// getStackMap returns the locals and arguments live pointer maps for -// frame. -func getStackMap(frame *stkframe, cache *pcvalueCache, debug bool) (locals, args bitvector) { +// getStackMap returns the locals and arguments live pointer maps, and +// stack object list for frame. +func getStackMap(frame *stkframe, cache *pcvalueCache, debug bool) (locals, args bitvector, objs []stackObjectRecord) { targetpc := frame.continpc if targetpc == 0 { // Frame is dead. Return empty bitvectors. @@ -1235,9 +1271,33 @@ func getStackMap(frame *stkframe, cache *pcvalueCache, debug bool) (locals, args } } } + + // stack objects. + p := funcdata(f, _FUNCDATA_StackObjects) + if p != nil { + n := *(*uintptr)(p) + p = add(p, sys.PtrSize) + *(*slice)(unsafe.Pointer(&objs)) = slice{array: noescape(p), len: int(n), cap: int(n)} + // Note: the noescape above is needed to keep + // getStackMap from from "leaking param content: + // frame". That leak propagates up to getgcmask, then + // GCMask, then verifyGCInfo, which converts the stack + // gcinfo tests into heap gcinfo tests :( + } + return } +// A stackObjectRecord is generated by the compiler for each stack object in a stack frame. +// This record must match the generator code in cmd/compile/internal/gc/ssa.go:emitStackObjects. +type stackObjectRecord struct { + // offset in frame + // if negative, offset from varp + // if non-negative, offset from argp + off int + typ *_type +} + //go:nosplit func morestackc() { throw("attempt to execute system stack code on user stack") diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index d90ab86ffa..452e9d06ae 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -348,6 +348,7 @@ const ( _FUNCDATA_LocalsPointerMaps = 1 _FUNCDATA_InlTree = 2 _FUNCDATA_RegPointerMaps = 3 + _FUNCDATA_StackObjects = 4 _ArgsSizeUnknown = -0x80000000 ) -- cgit v1.3-5-g9baa From 9a8372f8bd5a39d2476bfa9247407b51f9193b9e Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Fri, 7 Sep 2018 14:55:09 -0700 Subject: cmd/compile,runtime: remove ambiguously live logic The previous CL introduced stack objects. This CL removes the old ambiguously live liveness analysis. After this CL we're relying on stack objects exclusively. Update a bunch of liveness tests to reflect the new world. Fixes #22350 Change-Id: I739b26e015882231011ce6bc1a7f426049e59f31 Reviewed-on: https://go-review.googlesource.com/c/134156 Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang --- src/cmd/compile/internal/gc/op_string.go | 4 +- src/cmd/compile/internal/gc/plive.go | 235 +++++-------------------- src/cmd/compile/internal/gc/sinit.go | 6 + src/cmd/compile/internal/gc/ssa.go | 32 ++-- src/cmd/compile/internal/gc/syntax.go | 1 + src/cmd/compile/internal/ssa/gen/genericOps.go | 5 +- src/runtime/chan.go | 5 + src/runtime/gcinfo_test.go | 48 ++++- src/runtime/mbitmap.go | 9 +- test/fixedbugs/issue20250.go | 8 +- test/live.go | 175 +++++++++--------- test/live2.go | 15 +- test/live_syscall.go | 8 +- test/stackobj.go | 57 ++++++ test/stackobj3.go | 93 ++++++++++ test/uintptrescapes2.go | 8 +- 16 files changed, 375 insertions(+), 334 deletions(-) create mode 100644 test/stackobj.go create mode 100644 test/stackobj3.go (limited to 'src/runtime') diff --git a/src/cmd/compile/internal/gc/op_string.go b/src/cmd/compile/internal/gc/op_string.go index 857234e45d..90b95d769f 100644 --- a/src/cmd/compile/internal/gc/op_string.go +++ b/src/cmd/compile/internal/gc/op_string.go @@ -4,9 +4,9 @@ package gc import "strconv" -const _Op_name = "XXXNAMENONAMETYPEPACKLITERALADDSUBORXORADDSTRADDRANDANDAPPENDARRAYBYTESTRARRAYBYTESTRTMPARRAYRUNESTRSTRARRAYBYTESTRARRAYBYTETMPSTRARRAYRUNEASAS2AS2FUNCAS2RECVAS2MAPRAS2DOTTYPEASOPCALLCALLFUNCCALLMETHCALLINTERCALLPARTCAPCLOSECLOSURECMPIFACECMPSTRCOMPLITMAPLITSTRUCTLITARRAYLITSLICELITPTRLITCONVCONVIFACECONVNOPCOPYDCLDCLFUNCDCLFIELDDCLCONSTDCLTYPEDELETEDOTDOTPTRDOTMETHDOTINTERXDOTDOTTYPEDOTTYPE2EQNELTLEGEGTINDINDEXINDEXMAPKEYSTRUCTKEYLENMAKEMAKECHANMAKEMAPMAKESLICEMULDIVMODLSHRSHANDANDNOTNEWNOTCOMPLUSMINUSORORPANICPRINTPRINTNPARENSENDSLICESLICEARRSLICESTRSLICE3SLICE3ARRRECOVERRECVRUNESTRSELRECVSELRECV2IOTAREALIMAGCOMPLEXALIGNOFOFFSETOFSIZEOFBLOCKBREAKCASEXCASECONTINUEDEFEREMPTYFALLFORFORUNTILGOTOIFLABELPROCRANGERETURNSELECTSWITCHTYPESWTCHANTMAPTSTRUCTTINTERTFUNCTARRAYDDDDDDARGINLCALLEFACEITABIDATASPTRCLOSUREVARCFUNCCHECKNILVARKILLVARLIVEINDREGSPRETJMPGETGEND" +const _Op_name = "XXXNAMENONAMETYPEPACKLITERALADDSUBORXORADDSTRADDRANDANDAPPENDARRAYBYTESTRARRAYBYTESTRTMPARRAYRUNESTRSTRARRAYBYTESTRARRAYBYTETMPSTRARRAYRUNEASAS2AS2FUNCAS2RECVAS2MAPRAS2DOTTYPEASOPCALLCALLFUNCCALLMETHCALLINTERCALLPARTCAPCLOSECLOSURECMPIFACECMPSTRCOMPLITMAPLITSTRUCTLITARRAYLITSLICELITPTRLITCONVCONVIFACECONVNOPCOPYDCLDCLFUNCDCLFIELDDCLCONSTDCLTYPEDELETEDOTDOTPTRDOTMETHDOTINTERXDOTDOTTYPEDOTTYPE2EQNELTLEGEGTINDINDEXINDEXMAPKEYSTRUCTKEYLENMAKEMAKECHANMAKEMAPMAKESLICEMULDIVMODLSHRSHANDANDNOTNEWNOTCOMPLUSMINUSORORPANICPRINTPRINTNPARENSENDSLICESLICEARRSLICESTRSLICE3SLICE3ARRRECOVERRECVRUNESTRSELRECVSELRECV2IOTAREALIMAGCOMPLEXALIGNOFOFFSETOFSIZEOFBLOCKBREAKCASEXCASECONTINUEDEFEREMPTYFALLFORFORUNTILGOTOIFLABELPROCRANGERETURNSELECTSWITCHTYPESWTCHANTMAPTSTRUCTTINTERTFUNCTARRAYDDDDDDARGINLCALLEFACEITABIDATASPTRCLOSUREVARCFUNCCHECKNILVARDEFVARKILLVARLIVEINDREGSPRETJMPGETGEND" -var _Op_index = [...]uint16{0, 3, 7, 13, 17, 21, 28, 31, 34, 36, 39, 45, 49, 55, 61, 73, 88, 100, 112, 127, 139, 141, 144, 151, 158, 165, 175, 179, 183, 191, 199, 208, 216, 219, 224, 231, 239, 245, 252, 258, 267, 275, 283, 289, 293, 302, 309, 313, 316, 323, 331, 339, 346, 352, 355, 361, 368, 376, 380, 387, 395, 397, 399, 401, 403, 405, 407, 410, 415, 423, 426, 435, 438, 442, 450, 457, 466, 469, 472, 475, 478, 481, 484, 490, 493, 496, 499, 503, 508, 512, 517, 522, 528, 533, 537, 542, 550, 558, 564, 573, 580, 584, 591, 598, 606, 610, 614, 618, 625, 632, 640, 646, 651, 656, 660, 665, 673, 678, 683, 687, 690, 698, 702, 704, 709, 713, 718, 724, 730, 736, 742, 747, 751, 758, 764, 769, 775, 778, 784, 791, 796, 800, 805, 809, 819, 824, 832, 839, 846, 854, 860, 864, 867} +var _Op_index = [...]uint16{0, 3, 7, 13, 17, 21, 28, 31, 34, 36, 39, 45, 49, 55, 61, 73, 88, 100, 112, 127, 139, 141, 144, 151, 158, 165, 175, 179, 183, 191, 199, 208, 216, 219, 224, 231, 239, 245, 252, 258, 267, 275, 283, 289, 293, 302, 309, 313, 316, 323, 331, 339, 346, 352, 355, 361, 368, 376, 380, 387, 395, 397, 399, 401, 403, 405, 407, 410, 415, 423, 426, 435, 438, 442, 450, 457, 466, 469, 472, 475, 478, 481, 484, 490, 493, 496, 499, 503, 508, 512, 517, 522, 528, 533, 537, 542, 550, 558, 564, 573, 580, 584, 591, 598, 606, 610, 614, 618, 625, 632, 640, 646, 651, 656, 660, 665, 673, 678, 683, 687, 690, 698, 702, 704, 709, 713, 718, 724, 730, 736, 742, 747, 751, 758, 764, 769, 775, 778, 784, 791, 796, 800, 805, 809, 819, 824, 832, 838, 845, 852, 860, 866, 870, 873} func (i Op) String() string { if i >= Op(len(_Op_index)-1) { diff --git a/src/cmd/compile/internal/gc/plive.go b/src/cmd/compile/internal/gc/plive.go index e070a5cd1a..563b425db7 100644 --- a/src/cmd/compile/internal/gc/plive.go +++ b/src/cmd/compile/internal/gc/plive.go @@ -78,6 +78,10 @@ import ( // that its argument is certainly dead, for use when the liveness analysis // would not otherwise be able to deduce that fact. +// TODO: get rid of OpVarKill here. It's useful for stack frame allocation +// so the compiler can allocate two temps to the same location. Here it's now +// useless, since the implementation of stack objects. + // BlockEffects summarizes the liveness effects on an SSA block. type BlockEffects struct { // Computed during Liveness.prologue using only the content of @@ -85,23 +89,15 @@ type BlockEffects struct { // // uevar: upward exposed variables (used before set in block) // varkill: killed variables (set in block) - // avarinit: addrtaken variables set or used (proof of initialization) - uevar varRegVec - varkill varRegVec - avarinit bvec + uevar varRegVec + varkill varRegVec // Computed during Liveness.solve using control flow information: // // livein: variables live at block entry // liveout: variables live at block exit - // avarinitany: addrtaken variables possibly initialized at block exit - // (initialized in block or at exit from any predecessor block) - // avarinitall: addrtaken variables certainly initialized at block exit - // (initialized in block or at exit from all predecessor blocks) - livein varRegVec - liveout varRegVec - avarinitany bvec - avarinitall bvec + livein varRegVec + liveout varRegVec } // A collection of global state used by liveness analysis. @@ -186,10 +182,9 @@ func (idx LivenessIndex) Valid() bool { } type progeffectscache struct { - textavarinit []int32 - retuevar []int32 - tailuevar []int32 - initialized bool + retuevar []int32 + tailuevar []int32 + initialized bool } // varRegVec contains liveness bitmaps for variables and registers. @@ -264,24 +259,13 @@ func (lv *Liveness) initcache() { // all the parameters for correctness, and similarly it must not // read the out arguments - they won't be set until the new // function runs. - lv.cache.tailuevar = append(lv.cache.tailuevar, int32(i)) - if node.Addrtaken() { - lv.cache.textavarinit = append(lv.cache.textavarinit, int32(i)) - } - case PPARAMOUT: - // If the result had its address taken, it is being tracked - // by the avarinit code, which does not use uevar. - // If we added it to uevar too, we'd not see any kill - // and decide that the variable was live entry, which it is not. - // So only use uevar in the non-addrtaken case. - // The p.to.type == obj.TYPE_NONE limits the bvset to - // non-tail-call return instructions; see note below for details. - if !node.Addrtaken() { - lv.cache.retuevar = append(lv.cache.retuevar, int32(i)) - } + // All results are live at every return point. + // Note that this point is after escaping return values + // are copied back to the stack using their PAUTOHEAP references. + lv.cache.retuevar = append(lv.cache.retuevar, int32(i)) } } } @@ -291,21 +275,13 @@ func (lv *Liveness) initcache() { // // The possible flags are: // uevar - used by the instruction -// varkill - killed by the instruction -// for variables without address taken, means variable was set -// for variables with address taken, means variable was marked dead -// avarinit - initialized or referred to by the instruction, -// only for variables with address taken but not escaping to heap -// -// The avarinit output serves as a signal that the data has been -// initialized, because any use of a variable must come after its -// initialization. +// varkill - killed by the instruction (set) +// A kill happens after the use (for an instruction that updates a value, for example). type liveEffect int const ( uevar liveEffect = 1 << iota varkill - avarinit ) // valueEffects returns the index of a variable in lv.vars and the @@ -329,27 +305,15 @@ func (lv *Liveness) valueEffects(v *ssa.Value) (int32, liveEffect) { } var effect liveEffect - if n.Addrtaken() { - if v.Op != ssa.OpVarKill { - effect |= avarinit - } - if v.Op == ssa.OpVarDef || v.Op == ssa.OpVarKill { - effect |= varkill - } - } else { - // Read is a read, obviously. - // Addr by itself is also implicitly a read. - // - // Addr|Write means that the address is being taken - // but only so that the instruction can write to the value. - // It is not a read. - - if e&ssa.SymRead != 0 || e&(ssa.SymAddr|ssa.SymWrite) == ssa.SymAddr { - effect |= uevar - } - if e&ssa.SymWrite != 0 && (!isfat(n.Type) || v.Op == ssa.OpVarDef) { - effect |= varkill - } + // Read is a read, obviously. + // + // Addr is a read also, as any subseqent holder of the pointer must be able + // to see all the values (including initialization) written so far. + if e&(ssa.SymRead|ssa.SymAddr) != 0 { + effect |= uevar + } + if e&ssa.SymWrite != 0 && (!isfat(n.Type) || v.Op == ssa.OpVarDef) { + effect |= varkill } if effect == 0 { @@ -545,9 +509,6 @@ func newliveness(fn *Node, f *ssa.Func, vars []*Node, idx map[*Node]int32, stkpt be.varkill = varRegVec{vars: bulk.next()} be.livein = varRegVec{vars: bulk.next()} be.liveout = varRegVec{vars: bulk.next()} - be.avarinit = bulk.next() - be.avarinitany = bulk.next() - be.avarinitall = bulk.next() } lv.livenessMap.reset(lv.f.NumValues()) @@ -869,19 +830,6 @@ func (lv *Liveness) prologue() { } be.uevar.regs |= regUevar } - - // Walk the block instructions forward to update avarinit bits. - // avarinit describes the effect at the end of the block, not the beginning. - for _, val := range b.Values { - pos, e := lv.valueEffects(val) - // No need for regEffects because registers never appear in avarinit. - if e&varkill != 0 { - be.avarinit.Unset(pos) - } - if e&avarinit != 0 { - be.avarinit.Set(pos) - } - } } } @@ -892,51 +840,10 @@ func (lv *Liveness) solve() { nvars := int32(len(lv.vars)) newlivein := varRegVec{vars: bvalloc(nvars)} newliveout := varRegVec{vars: bvalloc(nvars)} - any := bvalloc(nvars) - all := bvalloc(nvars) - // Push avarinitall, avarinitany forward. - // avarinitall says the addressed var is initialized along all paths reaching the block exit. - // avarinitany says the addressed var is initialized along some path reaching the block exit. - for _, b := range lv.f.Blocks { - be := lv.blockEffects(b) - if b == lv.f.Entry { - be.avarinitall.Copy(be.avarinit) - } else { - be.avarinitall.Clear() - be.avarinitall.Not() - } - be.avarinitany.Copy(be.avarinit) - } - - // Walk blocks in the general direction of propagation (RPO - // for avarinit{any,all}, and PO for live{in,out}). This - // improves convergence. + // Walk blocks in postorder ordering. This improves convergence. po := lv.f.Postorder() - for change := true; change; { - change = false - for i := len(po) - 1; i >= 0; i-- { - b := po[i] - be := lv.blockEffects(b) - lv.avarinitanyall(b, any, all) - - any.AndNot(any, be.varkill.vars) - all.AndNot(all, be.varkill.vars) - any.Or(any, be.avarinit) - all.Or(all, be.avarinit) - if !any.Eq(be.avarinitany) { - change = true - be.avarinitany.Copy(any) - } - - if !all.Eq(be.avarinitall) { - change = true - be.avarinitall.Copy(all) - } - } - } - // Iterate through the blocks in reverse round-robin fashion. A work // queue might be slightly faster. As is, the number of iterations is // so low that it hardly seems to be worth the complexity. @@ -957,7 +864,7 @@ func (lv *Liveness) solve() { newliveout.vars.Set(pos) } case ssa.BlockExit: - // nothing to do + // panic exit - nothing to do default: // A variable is live on output from this block // if it is live on input to some successor. @@ -975,7 +882,7 @@ func (lv *Liveness) solve() { } // A variable is live on input to this block - // if it is live on output from this block and + // if it is used by this block, or live on output from this block and // not set by the code in this block. // // in[b] = uevar[b] \cup (out[b] \setminus varkill[b]) @@ -990,8 +897,6 @@ func (lv *Liveness) solve() { func (lv *Liveness) epilogue() { nvars := int32(len(lv.vars)) liveout := varRegVec{vars: bvalloc(nvars)} - any := bvalloc(nvars) - all := bvalloc(nvars) livedefer := bvalloc(nvars) // always-live variables // If there is a defer (that could recover), then all output @@ -1017,6 +922,9 @@ func (lv *Liveness) epilogue() { livedefer.Set(int32(i)) } if n.IsOutputParamHeapAddr() { + // This variable will be overwritten early in the function + // prologue (from the result of a mallocgc) but we need to + // zero it in case that malloc causes a stack scan. n.Name.SetNeedzero(true) livedefer.Set(int32(i)) } @@ -1033,9 +941,6 @@ func (lv *Liveness) epilogue() { { // Reserve an entry for function entry. live := bvalloc(nvars) - for _, pos := range lv.cache.textavarinit { - live.Set(pos) - } lv.livevars = append(lv.livevars, varRegVec{vars: live}) } @@ -1043,53 +948,14 @@ func (lv *Liveness) epilogue() { be := lv.blockEffects(b) firstBitmapIndex := len(lv.livevars) - // Compute avarinitany and avarinitall for entry to block. - // This duplicates information known during Liveness.solve - // but avoids storing two more vectors for each block. - lv.avarinitanyall(b, any, all) - // Walk forward through the basic block instructions and // allocate liveness maps for those instructions that need them. - // Seed the maps with information about the addrtaken variables. for _, v := range b.Values { - pos, e := lv.valueEffects(v) - // No need for regEffects because registers never appear in avarinit. - if e&varkill != 0 { - any.Unset(pos) - all.Unset(pos) - } - if e&avarinit != 0 { - any.Set(pos) - all.Set(pos) - } - if !lv.issafepoint(v) { continue } - // Annotate ambiguously live variables so that they can - // be zeroed at function entry and at VARKILL points. - // liveout is dead here and used as a temporary. - liveout.vars.AndNot(any, all) - if !liveout.vars.IsEmpty() { - for pos := int32(0); pos < liveout.vars.n; pos++ { - if !liveout.vars.Get(pos) { - continue - } - all.Set(pos) // silence future warnings in this block - n := lv.vars[pos] - if !n.Name.Needzero() { - n.Name.SetNeedzero(true) - if debuglive >= 1 { - Warnl(v.Pos, "%v: %L is ambiguously live", lv.fn.Func.Nname, n) - } - } - } - } - - // Live stuff first. live := bvalloc(nvars) - live.Copy(any) lv.livevars = append(lv.livevars, varRegVec{vars: live}) } @@ -1128,6 +994,17 @@ func (lv *Liveness) epilogue() { Fatalf("bad index for entry point: %v", index) } + // Check to make sure only input variables are live. + for i, n := range lv.vars { + if !liveout.vars.Get(int32(i)) { + continue + } + if n.Class() == PPARAM { + continue // ok + } + Fatalf("bad live variable at entry of %v: %L", lv.fn.Func.Nname, n) + } + // Record live variables. live := &lv.livevars[index] live.Or(*live, liveout) @@ -1330,28 +1207,6 @@ func clobberPtr(b *ssa.Block, v *Node, offset int64) { b.NewValue0IA(src.NoXPos, ssa.OpClobber, types.TypeVoid, offset, v) } -func (lv *Liveness) avarinitanyall(b *ssa.Block, any, all bvec) { - if len(b.Preds) == 0 { - any.Clear() - all.Clear() - for _, pos := range lv.cache.textavarinit { - any.Set(pos) - all.Set(pos) - } - return - } - - be := lv.blockEffects(b.Preds[0].Block()) - any.Copy(be.avarinitany) - all.Copy(be.avarinitall) - - for _, pred := range b.Preds[1:] { - be := lv.blockEffects(pred.Block()) - any.Or(any, be.avarinitany) - all.And(all, be.avarinitall) - } -} - // Compact coalesces identical bitmaps from lv.livevars into the sets // lv.stackMapSet and lv.regMaps. // @@ -1559,7 +1414,6 @@ func (lv *Liveness) printDebug() { printed = false printed = lv.printeffect(printed, "uevar", pos, effect&uevar != 0, regUevar) printed = lv.printeffect(printed, "varkill", pos, effect&varkill != 0, regKill) - printed = lv.printeffect(printed, "avarinit", pos, effect&avarinit != 0, 0) if printed { fmt.Printf("\n") } @@ -1596,9 +1450,6 @@ func (lv *Liveness) printDebug() { printed = false printed = lv.printbvec(printed, "varkill", be.varkill) printed = lv.printbvec(printed, "liveout", be.liveout) - printed = lv.printbvec(printed, "avarinit", varRegVec{vars: be.avarinit}) - printed = lv.printbvec(printed, "avarinitany", varRegVec{vars: be.avarinitany}) - printed = lv.printbvec(printed, "avarinitall", varRegVec{vars: be.avarinitall}) if printed { fmt.Printf("\n") } diff --git a/src/cmd/compile/internal/gc/sinit.go b/src/cmd/compile/internal/gc/sinit.go index f76b02828f..9d1114fa43 100644 --- a/src/cmd/compile/internal/gc/sinit.go +++ b/src/cmd/compile/internal/gc/sinit.go @@ -839,6 +839,10 @@ func slicelit(ctxt initContext, n *Node, var_ *Node, init *Nodes) { a = nod(OAS, x, nil) a = typecheck(a, Etop) init.Append(a) // zero new temp + } else { + // Declare that we're about to initialize all of x. + // (Which happens at the *vauto = vstat below.) + init.Append(nod(OVARDEF, x, nil)) } a = nod(OADDR, x, nil) @@ -849,6 +853,8 @@ func slicelit(ctxt initContext, n *Node, var_ *Node, init *Nodes) { a = typecheck(a, Etop) init.Append(a) // zero new temp a = a.Left + } else { + init.Append(nod(OVARDEF, a, nil)) } a = nod(OADDR, a, nil) diff --git a/src/cmd/compile/internal/gc/ssa.go b/src/cmd/compile/internal/gc/ssa.go index 663042754e..138ce08fec 100644 --- a/src/cmd/compile/internal/gc/ssa.go +++ b/src/cmd/compile/internal/gc/ssa.go @@ -755,8 +755,8 @@ func (s *state) stmtList(l Nodes) { // stmt converts the statement n to SSA and adds it to s. func (s *state) stmt(n *Node) { - if !(n.Op == OVARKILL || n.Op == OVARLIVE) { - // OVARKILL and OVARLIVE are invisible to the programmer, so we don't use their line numbers to avoid confusion in debugging. + if !(n.Op == OVARKILL || n.Op == OVARLIVE || n.Op == OVARDEF) { + // OVARKILL, OVARLIVE, and OVARDEF are invisible to the programmer, so we don't use their line numbers to avoid confusion in debugging. s.pushLine(n.Pos) defer s.popLine() } @@ -1170,6 +1170,10 @@ func (s *state) stmt(n *Node) { } s.startBlock(bEnd) + case OVARDEF: + if !s.canSSA(n.Left) { + s.vars[&memVar] = s.newValue1Apos(ssa.OpVarDef, types.TypeMem, n.Left, s.mem(), false) + } case OVARKILL: // Insert a varkill op to record that a variable is no longer live. // We only care about liveness info at call sites, so putting the @@ -4977,6 +4981,12 @@ func emitStackObjects(e *ssafn, pp *Progs) { p.To.Type = obj.TYPE_MEM p.To.Name = obj.NAME_EXTERN p.To.Sym = x + + if debuglive != 0 { + for _, v := range vars { + Warnl(v.Pos, "stack object %v %s", v, v.Type.String()) + } + } } // genssa appends entries to pp for each instruction in f. @@ -5056,24 +5066,8 @@ func genssa(f *ssa.Func, pp *Progs) { case ssa.OpGetG: // nothing to do when there's a g register, // and checkLower complains if there's not - case ssa.OpVarDef, ssa.OpVarLive, ssa.OpKeepAlive: + case ssa.OpVarDef, ssa.OpVarLive, ssa.OpKeepAlive, ssa.OpVarKill: // nothing to do; already used by liveness - case ssa.OpVarKill: - // Zero variable if it is ambiguously live. - // After the VARKILL anything this variable references - // might be collected. If it were to become live again later, - // the GC will see references to already-collected objects. - // See issue 20029. - n := v.Aux.(*Node) - if n.Name.Needzero() { - if n.Class() != PAUTO { - v.Fatalf("zero of variable which isn't PAUTO %v", n) - } - if n.Type.Size()%int64(Widthptr) != 0 { - v.Fatalf("zero of variable not a multiple of ptr size %v", n) - } - thearch.ZeroAuto(s.pp, n) - } case ssa.OpPhi: CheckLoweredPhi(v) case ssa.OpConvert: diff --git a/src/cmd/compile/internal/gc/syntax.go b/src/cmd/compile/internal/gc/syntax.go index df23b83f29..eb2ab6b916 100644 --- a/src/cmd/compile/internal/gc/syntax.go +++ b/src/cmd/compile/internal/gc/syntax.go @@ -739,6 +739,7 @@ const ( OCLOSUREVAR // variable reference at beginning of closure function OCFUNC // reference to c function pointer (not go func value) OCHECKNIL // emit code to ensure pointer/interface not nil + OVARDEF // variable is about to be fully initialized OVARKILL // variable is dead OVARLIVE // variable is alive OINDREGSP // offset plus indirect of REGSP, such as 8(SP). diff --git a/src/cmd/compile/internal/ssa/gen/genericOps.go b/src/cmd/compile/internal/ssa/gen/genericOps.go index 7292012d26..ee9c6fa0f6 100644 --- a/src/cmd/compile/internal/ssa/gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/gen/genericOps.go @@ -470,8 +470,9 @@ var genericOps = []opData{ {name: "VarDef", argLength: 1, aux: "Sym", typ: "Mem", symEffect: "None", zeroWidth: true}, // aux is a *gc.Node of a variable that is about to be initialized. arg0=mem, returns mem {name: "VarKill", argLength: 1, aux: "Sym", symEffect: "None"}, // aux is a *gc.Node of a variable that is known to be dead. arg0=mem, returns mem - {name: "VarLive", argLength: 1, aux: "Sym", symEffect: "Read", zeroWidth: true}, // aux is a *gc.Node of a variable that must be kept live. arg0=mem, returns mem - {name: "KeepAlive", argLength: 2, typ: "Mem", zeroWidth: true}, // arg[0] is a value that must be kept alive until this mark. arg[1]=mem, returns mem + // TODO: what's the difference betweeen VarLive and KeepAlive? + {name: "VarLive", argLength: 1, aux: "Sym", symEffect: "Read", zeroWidth: true}, // aux is a *gc.Node of a variable that must be kept live. arg0=mem, returns mem + {name: "KeepAlive", argLength: 2, typ: "Mem", zeroWidth: true}, // arg[0] is a value that must be kept alive until this mark. arg[1]=mem, returns mem // Ops for breaking 64-bit operations on 32-bit architectures {name: "Int64Make", argLength: 2, typ: "UInt64"}, // arg0=hi, arg1=lo diff --git a/src/runtime/chan.go b/src/runtime/chan.go index a4ee51ca39..5cf0b86f58 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -232,6 +232,11 @@ func chansend(c *hchan, ep unsafe.Pointer, block bool, callerpc uintptr) bool { gp.param = nil c.sendq.enqueue(mysg) goparkunlock(&c.lock, waitReasonChanSend, traceEvGoBlockSend, 3) + // Ensure the value being sent is kept alive until the + // receiver copies it out. The sudog has a pointer to the + // stack object, but sudogs aren't considered as roots of the + // stack tracer. + KeepAlive(ep) // someone woke us up. if mysg != gp.waiting { diff --git a/src/runtime/gcinfo_test.go b/src/runtime/gcinfo_test.go index 7dd1a5607c..0741f6361c 100644 --- a/src/runtime/gcinfo_test.go +++ b/src/runtime/gcinfo_test.go @@ -35,14 +35,46 @@ func TestGCInfo(t *testing.T) { verifyGCInfo(t, "data eface", &dataEface, infoEface) verifyGCInfo(t, "data iface", &dataIface, infoIface) - verifyGCInfo(t, "stack Ptr", new(Ptr), infoPtr) - verifyGCInfo(t, "stack ScalarPtr", new(ScalarPtr), infoScalarPtr) - verifyGCInfo(t, "stack PtrScalar", new(PtrScalar), infoPtrScalar) - verifyGCInfo(t, "stack BigStruct", new(BigStruct), infoBigStruct()) - verifyGCInfo(t, "stack string", new(string), infoString) - verifyGCInfo(t, "stack slice", new([]string), infoSlice) - verifyGCInfo(t, "stack eface", new(interface{}), infoEface) - verifyGCInfo(t, "stack iface", new(Iface), infoIface) + { + var x Ptr + verifyGCInfo(t, "stack Ptr", &x, infoPtr) + runtime.KeepAlive(x) + } + { + var x ScalarPtr + verifyGCInfo(t, "stack ScalarPtr", &x, infoScalarPtr) + runtime.KeepAlive(x) + } + { + var x PtrScalar + verifyGCInfo(t, "stack PtrScalar", &x, infoPtrScalar) + runtime.KeepAlive(x) + } + { + var x BigStruct + verifyGCInfo(t, "stack BigStruct", &x, infoBigStruct()) + runtime.KeepAlive(x) + } + { + var x string + verifyGCInfo(t, "stack string", &x, infoString) + runtime.KeepAlive(x) + } + { + var x []string + verifyGCInfo(t, "stack slice", &x, infoSlice) + runtime.KeepAlive(x) + } + { + var x interface{} + verifyGCInfo(t, "stack eface", &x, infoEface) + runtime.KeepAlive(x) + } + { + var x Iface + verifyGCInfo(t, "stack iface", &x, infoIface) + runtime.KeepAlive(x) + } for i := 0; i < 10; i++ { verifyGCInfo(t, "heap Ptr", escape(new(Ptr)), trimDead(padDead(infoPtr))) diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go index 5301e692e0..87fa027b4e 100644 --- a/src/runtime/mbitmap.go +++ b/src/runtime/mbitmap.go @@ -1994,7 +1994,9 @@ func reflect_gcbits(x interface{}) []byte { return ret } -// Returns GC type info for object p for testing. +// Returns GC type info for the pointer stored in ep for testing. +// If ep points to the stack, only static live information will be returned +// (i.e. not for objects which are only dynamically live stack objects). func getgcmask(ep interface{}) (mask []byte) { e := *efaceOf(&ep) p := e.data @@ -2051,11 +2053,6 @@ func getgcmask(ep interface{}) (mask []byte) { _g_ := getg() gentraceback(_g_.m.curg.sched.pc, _g_.m.curg.sched.sp, 0, _g_.m.curg, 0, nil, 1000, getgcmaskcb, noescape(unsafe.Pointer(&frame)), 0) if frame.fn.valid() { - // TODO: once stack objects are enabled (and their pointers - // are no longer described by the stack pointermap directly), - // tests using this will probably need fixing. We might need - // to loop through the stackobjects and if we're inside one, - // use the pointermap from that object. locals, _, _ := getStackMap(&frame, nil, false) if locals.n == 0 { return diff --git a/test/fixedbugs/issue20250.go b/test/fixedbugs/issue20250.go index 6fc861a8dc..47879385d2 100644 --- a/test/fixedbugs/issue20250.go +++ b/test/fixedbugs/issue20250.go @@ -15,10 +15,10 @@ type T struct { } func f(a T) { // ERROR "live at entry to f: a" - var e interface{} - func() { // ERROR "live at entry to f.func1: a &e" - e = a.s // ERROR "live at call to convT2Estring: a &e" - }() // ERROR "live at call to f.func1: e$" + var e interface{} // ERROR "stack object e interface \{\}$" + func() { // ERROR "live at entry to f.func1: a &e" + e = a.s // ERROR "live at call to convT2Estring: &e" "stack object a T$" + }() // Before the fix, both a and e were live at the previous line. _ = e } diff --git a/test/live.go b/test/live.go index 13bdc4aae1..679562d9bf 100644 --- a/test/live.go +++ b/test/live.go @@ -32,9 +32,9 @@ func printbytepointer(*byte) func printint(int) func f1() { - var x *int - printpointer(&x) // ERROR "live at call to printpointer: x$" + var x *int // ERROR "stack object x \*int$" printpointer(&x) // ERROR "live at call to printpointer: x$" + printpointer(&x) } func f2(b bool) { @@ -42,9 +42,9 @@ func f2(b bool) { printint(0) // nothing live here return } - var x *int - printpointer(&x) // ERROR "live at call to printpointer: x$" + var x *int // ERROR "stack object x \*int$" printpointer(&x) // ERROR "live at call to printpointer: x$" + printpointer(&x) } func f3(b1, b2 bool) { @@ -60,15 +60,15 @@ func f3(b1, b2 bool) { } if b2 { - var x *int - printpointer(&x) // ERROR "live at call to printpointer: x$" + var x *int // ERROR "stack object x \*int$" printpointer(&x) // ERROR "live at call to printpointer: x$" + printpointer(&x) } else { - var y *int - printpointer(&y) // ERROR "live at call to printpointer: y$" + var y *int // ERROR "stack object y \*int$" printpointer(&y) // ERROR "live at call to printpointer: y$" + printpointer(&y) } - printint(0) // ERROR "f3: x \(type \*int\) is ambiguously live$" "f3: y \(type \*int\) is ambiguously live$" "live at call to printint: x y$" + printint(0) // nothing is live here } // The old algorithm treated x as live on all code that @@ -83,7 +83,7 @@ func f4(b1, b2 bool) { // x not live here return } var z **int - x := new(int) + x := new(int) // ERROR "stack object x \*int$" *x = 42 z = &x printint(**z) // ERROR "live at call to printint: x$" @@ -99,15 +99,15 @@ func f4(b1, b2 bool) { // x not live here func f5(b1 bool) { var z **int if b1 { - x := new(int) + x := new(int) // ERROR "stack object x \*int$" *x = 42 z = &x } else { - y := new(int) + y := new(int) // ERROR "stack object y \*int$" *y = 54 z = &y } - printint(**z) // ERROR "f5: x \(type \*int\) is ambiguously live$" "f5: y \(type \*int\) is ambiguously live$" "live at call to printint: x y$" + printint(**z) // nothing live here } // confusion about the _ result used to cause spurious "live at entry to f6: _". @@ -119,7 +119,7 @@ func f6() (_, y string) { // confusion about addressed results used to cause "live at entry to f7: x". -func f7() (x string) { +func f7() (x string) { // ERROR "stack object x string" _ = &x x = "hello" return @@ -141,7 +141,7 @@ var i9 interface{} func f9() bool { g8() x := i9 - y := interface{}(str()) // ERROR "live at call to convT2Estring: .autotmp_[0-9]+ x.data$" "live at call to str: x.data$" + y := interface{}(str()) // ERROR "live at call to convT2Estring: x.data$" "live at call to str: x.data$" "stack object .autotmp_[0-9]+ string$" i9 = y // make y escape so the line above has to call convT2E return x != y } @@ -163,7 +163,7 @@ var b bool // this used to have a spurious "live at entry to f11a: ~r0" func f11a() *int { - select { // ERROR "live at call to selectgo: .autotmp_[0-9]+$" + select { // ERROR "stack object .autotmp_[0-9]+ \[2\]struct" case <-c: return nil case <-c: @@ -178,7 +178,7 @@ func f11b() *int { // get to the bottom of the function. // This used to have a spurious "live at call to printint: p". printint(1) // nothing live here! - select { // ERROR "live at call to selectgo: .autotmp_[0-9]+$" + select { // ERROR "stack object .autotmp_[0-9]+ \[2\]struct" case <-c: return nil case <-c: @@ -198,7 +198,7 @@ func f11c() *int { // Unlike previous, the cases in this select fall through, // so we can get to the println, so p is not dead. printint(1) // ERROR "live at call to printint: p$" - select { // ERROR "live at call to selectgo: .autotmp_[0-9]+ p$" + select { // ERROR "live at call to selectgo: p$" "stack object .autotmp_[0-9]+ \[2\]struct" case <-c: case <-c: } @@ -233,8 +233,8 @@ func h13(string, string) string // more incorrectly placed VARDEF. func f14() { - x := g14() - printstringpointer(&x) // ERROR "live at call to printstringpointer: x$" + x := g14() // ERROR "stack object x string$" + printstringpointer(&x) } func g14() string @@ -254,10 +254,10 @@ func iface() interface{} func f16() { if b { - delete(mi, iface()) // ERROR "live at call to mapdelete: .autotmp_[0-9]+$" + delete(mi, iface()) // ERROR "stack object .autotmp_[0-9]+ interface \{\}$" } - delete(mi, iface()) // ERROR "live at call to mapdelete: .autotmp_[0-9]+$" - delete(mi, iface()) // ERROR "live at call to mapdelete: .autotmp_[0-9]+$" + delete(mi, iface()) // ERROR "stack object .autotmp_[0-9]+ interface \{\}$" + delete(mi, iface()) // ERROR "stack object .autotmp_[0-9]+ interface \{\}$" } var m2s map[string]*byte @@ -300,10 +300,10 @@ func f18() { // temporary introduced by orderexpr. var z *byte if b { - z = m2[g18()] // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" + z = m2[g18()] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } - z = m2[g18()] // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" - z = m2[g18()] // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" + z = m2[g18()] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z = m2[g18()] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" printbytepointer(z) } @@ -317,30 +317,30 @@ func f19() { var z *byte if b { - z = <-ch // ERROR "live at call to chanrecv1: .autotmp_[0-9]+$" + z = <-ch // ERROR "stack object .autotmp_[0-9]+ \*byte$" } - z = <-ch // ERROR "live at call to chanrecv1: .autotmp_[0-9]+$" - z = <-ch // ERROR "live at call to chanrecv1: .autotmp_[0-9]+$" + z = <-ch // ERROR "stack object .autotmp_[0-9]+ \*byte$" + z = <-ch // ERROR "stack object .autotmp_[0-9]+ \*byte$" "live at call to chanrecv1: .autotmp_[0-9]+$" printbytepointer(z) } func f20() { // src temporary for channel send if b { - ch <- byteptr() // ERROR "live at call to chansend1: .autotmp_[0-9]+$" + ch <- byteptr() // ERROR "stack object .autotmp_[0-9]+ \*byte$" } - ch <- byteptr() // ERROR "live at call to chansend1: .autotmp_[0-9]+$" - ch <- byteptr() // ERROR "live at call to chansend1: .autotmp_[0-9]+$" + ch <- byteptr() // ERROR "stack object .autotmp_[0-9]+ \*byte$" + ch <- byteptr() // ERROR "stack object .autotmp_[0-9]+ \*byte$" } func f21() { // key temporary for mapaccess using array literal key. var z *byte if b { - z = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" + z = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } - z = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" - z = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" + z = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" printbytepointer(z) } @@ -349,10 +349,10 @@ func f23() { var z *byte var ok bool if b { - z, ok = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess2: .autotmp_[0-9]+$" + z, ok = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } - z, ok = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess2: .autotmp_[0-9]+$" - z, ok = m2[[2]string{"x", "y"}] // ERROR "live at call to mapaccess2: .autotmp_[0-9]+$" + z, ok = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + z, ok = m2[[2]string{"x", "y"}] // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" printbytepointer(z) print(ok) } @@ -361,10 +361,10 @@ func f24() { // key temporary for map access using array literal key. // value temporary too. if b { - m2[[2]string{"x", "y"}] = nil // ERROR "live at call to mapassign: .autotmp_[0-9]+$" + m2[[2]string{"x", "y"}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } - m2[[2]string{"x", "y"}] = nil // ERROR "live at call to mapassign: .autotmp_[0-9]+$" - m2[[2]string{"x", "y"}] = nil // ERROR "live at call to mapassign: .autotmp_[0-9]+$" + m2[[2]string{"x", "y"}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" + m2[[2]string{"x", "y"}] = nil // ERROR "stack object .autotmp_[0-9]+ \[2\]string$" } // defer should not cause spurious ambiguously live variables @@ -387,10 +387,10 @@ func g25() func f26(b bool) { if b { - print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "live at call to print26: .autotmp_[0-9]+$" + print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "stack object .autotmp_[0-9]+ \[3\]interface \{\}$" } - print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "live at call to print26: .autotmp_[0-9]+$" - print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "live at call to print26: .autotmp_[0-9]+$" + print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "stack object .autotmp_[0-9]+ \[3\]interface \{\}$" + print26((*int)(nil), (*int)(nil), (*int)(nil)) // ERROR "stack object .autotmp_[0-9]+ \[3\]interface \{\}$" printnl() } @@ -402,10 +402,10 @@ func print26(...interface{}) func f27(b bool) { x := 0 if b { - call27(func() { x++ }) // ERROR "live at call to call27: .autotmp_[0-9]+$" + call27(func() { x++ }) // ERROR "stack object .autotmp_[0-9]+ struct \{" } - call27(func() { x++ }) // ERROR "live at call to call27: .autotmp_[0-9]+$" - call27(func() { x++ }) // ERROR "live at call to call27: .autotmp_[0-9]+$" + call27(func() { x++ }) // ERROR "stack object .autotmp_[0-9]+ struct \{" + call27(func() { x++ }) // ERROR "stack object .autotmp_[0-9]+ struct \{" printnl() } @@ -414,11 +414,11 @@ func f27(b bool) { func f27defer(b bool) { x := 0 if b { - defer call27(func() { x++ }) // ERROR "live at call to deferproc: .autotmp_[0-9]+$" "live at call to deferreturn: .autotmp_[0-9]+$" + defer call27(func() { x++ }) // ERROR "stack object .autotmp_[0-9]+ struct \{" } - defer call27(func() { x++ }) // ERROR "f27defer: .autotmp_[0-9]+ \(type struct { F uintptr; x \*int }\) is ambiguously live$" "live at call to deferproc: .autotmp_[0-9]+ .autotmp_[0-9]+$" "live at call to deferreturn: .autotmp_[0-9]+ .autotmp_[0-9]+$" - printnl() // ERROR "live at call to printnl: .autotmp_[0-9]+ .autotmp_[0-9]+$" -} // ERROR "live at call to deferreturn: .autotmp_[0-9]+ .autotmp_[0-9]+$" + defer call27(func() { x++ }) // ERROR "stack object .autotmp_[0-9]+ struct \{" + printnl() +} // and newproc (go) escapes to the heap @@ -440,24 +440,24 @@ var s1, s2, s3, s4, s5, s6, s7, s8, s9, s10 string func f28(b bool) { if b { - printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: .autotmp_[0-9]+$" "live at call to printstring: .autotmp_[0-9]+$" + printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "stack object .autotmp_[0-9]+ \[10\]string$" } - printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: .autotmp_[0-9]+$" "live at call to printstring: .autotmp_[0-9]+$" - printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "live at call to concatstrings: .autotmp_[0-9]+$" "live at call to printstring: .autotmp_[0-9]+$" + printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "stack object .autotmp_[0-9]+ \[10\]string$" + printstring(s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10) // ERROR "stack object .autotmp_[0-9]+ \[10\]string$" } // map iterator should die on end of range loop func f29(b bool) { if b { - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ map.iter\[string\]int$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ map.iter\[string\]int$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } - for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" + for k := range m { // ERROR "live at call to mapiterinit: .autotmp_[0-9]+$" "live at call to mapiternext: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ map.iter\[string\]int$" printstring(k) // ERROR "live at call to printstring: .autotmp_[0-9]+$" } } @@ -473,20 +473,19 @@ type pstruct struct { } func f30(b bool) { - // two live temps during printintpointer(p): - // in the copy of p.intp and + // live temp during printintpointer(p): // the internal iterator pointer if a pointer to pstruct in pstructarr // can not be easily computed by strength reduction. if b { - for _, p := range pstructarr { - printintpointer(p.intp) // ERROR "live at call to printintpointer: .autotmp_[0-9]+ .autotmp_[0-9]+$" + for _, p := range pstructarr { // ERROR "stack object .autotmp_[0-9]+ \[10\]pstruct$" + printintpointer(p.intp) // ERROR "live at call to printintpointer: .autotmp_[0-9]+$" } } - for _, p := range pstructarr { - printintpointer(p.intp) // ERROR "live at call to printintpointer: .autotmp_[0-9]+ .autotmp_[0-9]+$" + for _, p := range pstructarr { // ERROR "stack object .autotmp_[0-9]+ \[10\]pstruct$" + printintpointer(p.intp) // ERROR "live at call to printintpointer: .autotmp_[0-9]+$" } - for _, p := range pstructarr { - printintpointer(p.intp) // ERROR "live at call to printintpointer: .autotmp_[0-9]+ .autotmp_[0-9]+$" + for _, p := range pstructarr { // ERROR "stack object .autotmp_[0-9]+ \[10\]pstruct$" + printintpointer(p.intp) // ERROR "live at call to printintpointer: .autotmp_[0-9]+$" } } @@ -494,13 +493,13 @@ func f30(b bool) { func f31(b1, b2, b3 bool) { if b1 { - g31(str()) // ERROR "live at call to convT2Estring: .autotmp_[0-9]+$" "live at call to g31: .autotmp_[0-9]+$" + g31(str()) // ERROR "stack object .autotmp_[0-9]+ string$" } if b2 { - h31(str()) // ERROR "live at call to convT2Estring: .autotmp_[0-9]+ .autotmp_[0-9]+$" "live at call to h31: .autotmp_[0-9]+$" "live at call to newobject: .autotmp_[0-9]+$" + h31(str()) // ERROR "live at call to convT2Estring: .autotmp_[0-9]+$" "live at call to newobject: .autotmp_[0-9]+$" "stack object .autotmp_[0-9]+ string$" } if b3 { - panic(str()) // ERROR "live at call to convT2Estring: .autotmp_[0-9]+$" "live at call to gopanic: .autotmp_[0-9]+$" + panic(str()) // ERROR "stack object .autotmp_[0-9]+ string$" } print(b3) } @@ -520,10 +519,10 @@ var t32 T32 func f32(b bool) { if b { - call32(t32.Inc) // ERROR "live at call to call32: .autotmp_[0-9]+$" + call32(t32.Inc) // ERROR "stack object .autotmp_[0-9]+ struct \{" } - call32(t32.Inc) // ERROR "live at call to call32: .autotmp_[0-9]+$" - call32(t32.Inc) // ERROR "live at call to call32: .autotmp_[0-9]+$" + call32(t32.Inc) // ERROR "stack object .autotmp_[0-9]+ struct \{" + call32(t32.Inc) // ERROR "stack object .autotmp_[0-9]+ struct \{" } //go:noescape @@ -535,7 +534,7 @@ func call32(func()) var m33 map[interface{}]int func f33() { - if m33[byteptr()] == 0 { // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" + if m33[byteptr()] == 0 { // ERROR "stack object .autotmp_[0-9]+ interface \{\}$" printnl() return } else { @@ -545,7 +544,7 @@ func f33() { } func f34() { - if m33[byteptr()] == 0 { // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" + if m33[byteptr()] == 0 { // ERROR "stack object .autotmp_[0-9]+ interface \{\}$" printnl() return } @@ -553,7 +552,8 @@ func f34() { } func f35() { - if m33[byteptr()] == 0 && m33[byteptr()] == 0 { // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" "f35: .autotmp_[0-9]+ \(type interface \{\}\) is ambiguously live$" + if m33[byteptr()] == 0 && // ERROR "stack object .autotmp_[0-9]+ interface \{\}" + m33[byteptr()] == 0 { // ERROR "stack object .autotmp_[0-9]+ interface \{\}" printnl() return } @@ -561,7 +561,8 @@ func f35() { } func f36() { - if m33[byteptr()] == 0 || m33[byteptr()] == 0 { // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" "f36: .autotmp_[0-9]+ \(type interface \{\}\) is ambiguously live$" + if m33[byteptr()] == 0 || // ERROR "stack object .autotmp_[0-9]+ interface \{\}" + m33[byteptr()] == 0 { // ERROR "stack object .autotmp_[0-9]+ interface \{\}" printnl() return } @@ -569,7 +570,9 @@ func f36() { } func f37() { - if (m33[byteptr()] == 0 || m33[byteptr()] == 0) && m33[byteptr()] == 0 { // ERROR "live at call to mapaccess1: .autotmp_[0-9]+$" "f37: .autotmp_[0-9]+ \(type interface \{\}\) is ambiguously live$" + if (m33[byteptr()] == 0 || // ERROR "stack object .autotmp_[0-9]+ interface \{\}" + m33[byteptr()] == 0) && // ERROR "stack object .autotmp_[0-9]+ interface \{\}" + m33[byteptr()] == 0 { // ERROR "stack object .autotmp_[0-9]+ interface \{\}" printnl() return } @@ -589,14 +592,14 @@ func f38(b bool) { // we care that the println lines have no live variables // and therefore no output. if b { - select { // ERROR "live at call to selectgo:( .autotmp_[0-9]+)+$" + select { // ERROR "live at call to selectgo:( .autotmp_[0-9]+)+$" "stack object .autotmp_[0-9]+ \[4\]struct \{" case <-fc38(): printnl() - case fc38() <- *fi38(1): // ERROR "live at call to fc38:( .autotmp_[0-9]+)+$" "live at call to fi38:( .autotmp_[0-9]+)+$" + case fc38() <- *fi38(1): // ERROR "live at call to fc38:( .autotmp_[0-9]+)+$" "live at call to fi38:( .autotmp_[0-9]+)+$" "stack object .autotmp_[0-9]+ string$" printnl() - case *fi38(2) = <-fc38(): // ERROR "live at call to fc38:( .autotmp_[0-9]+)+$" "live at call to fi38:( .autotmp_[0-9]+)+$" + case *fi38(2) = <-fc38(): // ERROR "live at call to fc38:( .autotmp_[0-9]+)+$" "live at call to fi38:( .autotmp_[0-9]+)+$" "stack object .autotmp_[0-9]+ string$" printnl() - case *fi38(3), *fb38() = <-fc38(): // ERROR "live at call to fb38:( .autotmp_[0-9]+)+$" "live at call to fc38:( .autotmp_[0-9]+)+$" "live at call to fi38:( .autotmp_[0-9]+)+$" + case *fi38(3), *fb38() = <-fc38(): // ERROR "stack object .autotmp_[0-9]+ string$" "live at call to fc38:( .autotmp_[0-9]+)+$" "live at call to fi38:( .autotmp_[0-9]+)+$" printnl() } printnl() @@ -655,15 +658,17 @@ func bad40() { } func good40() { - ret := T40{} - ret.m = make(map[int]int) // ERROR "live at call to fastrand: .autotmp_[0-9]+ ret$" + ret := T40{} // ERROR "stack object ret T40$" + ret.m = make(map[int]int) // ERROR "live at call to fastrand: .autotmp_[0-9]+ ret$" "stack object .autotmp_[0-9]+ map.hdr\[int\]int$" t := &ret - printnl() // ERROR "live at call to printnl: .autotmp_[0-9]+ ret$" - useT40(t) // ERROR "live at call to useT40: .autotmp_[0-9]+ ret$" + printnl() // ERROR "live at call to printnl: ret$" + // Note: ret is live at the printnl because the compiler moves &ret + // from before the printnl to after. + useT40(t) } func ddd1(x, y *int) { // ERROR "live at entry to ddd1: x y$" - ddd2(x, y) // ERROR "live at call to ddd2: .autotmp_[0-9]+$" + ddd2(x, y) // ERROR "stack object .autotmp_[0-9]+ \[2\]\*int$" printnl() // Note: no .?autotmp live at printnl. See issue 16996. } diff --git a/test/live2.go b/test/live2.go index cc1b0b7acf..cea312f075 100644 --- a/test/live2.go +++ b/test/live2.go @@ -10,7 +10,6 @@ package main // issue 8142: lost 'addrtaken' bit on inlined variables. -// no inlining in this test, so just checking that non-inlined works. func printnl() @@ -28,15 +27,15 @@ func newT40() *T40 { } func bad40() { - t := newT40() // ERROR "live at call to makemap: .autotmp_[0-9]+ ret$" - printnl() // ERROR "live at call to printnl: .autotmp_[0-9]+ ret$" - useT40(t) // ERROR "live at call to useT40: .autotmp_[0-9]+ ret$" + t := newT40() // ERROR "live at call to makemap: ret$" "stack object ret T40$" "stack object .autotmp_[0-9]+ map.hdr\[int\]int$" + printnl() // ERROR "live at call to printnl: ret$" + useT40(t) } func good40() { - ret := T40{} - ret.m = make(map[int]int, 42) // ERROR "live at call to makemap: .autotmp_[0-9]+ ret$" + ret := T40{} // ERROR "stack object ret T40$" + ret.m = make(map[int]int, 42) // ERROR "live at call to makemap: ret$" "stack object .autotmp_[0-9]+ map.hdr\[int\]int$" t := &ret - printnl() // ERROR "live at call to printnl: .autotmp_[0-9]+ ret$" - useT40(t) // ERROR "live at call to useT40: .autotmp_[0-9]+ ret$" + printnl() // ERROR "live at call to printnl: ret$" + useT40(t) } diff --git a/test/live_syscall.go b/test/live_syscall.go index 65a161c028..b7b85bcabf 100644 --- a/test/live_syscall.go +++ b/test/live_syscall.go @@ -19,22 +19,22 @@ func f(uintptr) // ERROR "f assuming arg#1 is unsafe uintptr" func g() { var t int - f(uintptr(unsafe.Pointer(&t))) // ERROR "live at call to f: .?autotmp" "g &t does not escape" + f(uintptr(unsafe.Pointer(&t))) // ERROR "live at call to f: .?autotmp" "g &t does not escape" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func h() { var v int - syscall.Syscall(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to Syscall: .?autotmp" "h &v does not escape" + syscall.Syscall(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to Syscall: .?autotmp" "h &v does not escape" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func i() { var t int p := unsafe.Pointer(&t) // ERROR "i &t does not escape" - f(uintptr(p)) // ERROR "live at call to f: .?autotmp" + f(uintptr(p)) // ERROR "live at call to f: .?autotmp" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func j() { var v int p := unsafe.Pointer(&v) // ERROR "j &v does not escape" - syscall.Syscall(0, 1, uintptr(p), 2) // ERROR "live at call to Syscall: .?autotmp" + syscall.Syscall(0, 1, uintptr(p), 2) // ERROR "live at call to Syscall: .?autotmp" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } diff --git a/test/stackobj.go b/test/stackobj.go new file mode 100644 index 0000000000..b6af4bd8f3 --- /dev/null +++ b/test/stackobj.go @@ -0,0 +1,57 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "runtime" +) + +type HeapObj [8]int64 + +type StkObj struct { + h *HeapObj +} + +var n int +var c int = -1 + +func gc() { + // encourage heap object to be collected, and have its finalizer run. + runtime.GC() + runtime.GC() + runtime.GC() + n++ +} + +func main() { + f() + gc() // prior to stack objects, heap object is not collected until here + if c < 0 { + panic("heap object never collected") + } + if c != 1 { + panic(fmt.Sprintf("expected collection at phase 1, got phase %d", c)) + } +} + +func f() { + var s StkObj + s.h = new(HeapObj) + runtime.SetFinalizer(s.h, func(h *HeapObj) { + // Remember at what phase the heap object was collected. + c = n + }) + g(&s) + gc() +} + +func g(s *StkObj) { + gc() // heap object is still live here + runtime.KeepAlive(s) + gc() // heap object should be collected here +} diff --git a/test/stackobj3.go b/test/stackobj3.go new file mode 100644 index 0000000000..8bb8fb3270 --- /dev/null +++ b/test/stackobj3.go @@ -0,0 +1,93 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This test makes sure that ambiguously live arguments work correctly. + +package main + +import ( + "runtime" +) + +type HeapObj [8]int64 + +type StkObj struct { + h *HeapObj +} + +var n int +var c int = -1 + +func gc() { + // encourage heap object to be collected, and have its finalizer run. + runtime.GC() + runtime.GC() + runtime.GC() + n++ +} + +var null StkObj + +var sink *HeapObj + +//go:noinline +func use(p *StkObj) { +} + +//go:noinline +func f(s StkObj, b bool) { + var p *StkObj + if b { + p = &s + } else { + p = &null + } + // use is required here to prevent the conditional + // code above from being executed after the first gc() call. + use(p) + // If b==false, h should be collected here. + gc() // 0 + sink = p.h + gc() // 1 + sink = nil + // If b==true, h should be collected here. + gc() // 2 +} + +func fTrue() { + var s StkObj + s.h = new(HeapObj) + c = -1 + n = 0 + runtime.SetFinalizer(s.h, func(h *HeapObj) { + // Remember at what phase the heap object was collected. + c = n + }) + f(s, true) + if c != 2 { + panic("bad liveness") + } +} + +func fFalse() { + var s StkObj + s.h = new(HeapObj) + c = -1 + n = 0 + runtime.SetFinalizer(s.h, func(h *HeapObj) { + // Remember at what phase the heap object was collected. + c = n + }) + f(s, false) + if c != 0 { + panic("bad liveness") + } +} + +func main() { + fTrue() + fFalse() +} diff --git a/test/uintptrescapes2.go b/test/uintptrescapes2.go index c94bc148c8..e7b5d721f5 100644 --- a/test/uintptrescapes2.go +++ b/test/uintptrescapes2.go @@ -30,14 +30,14 @@ func F4(...uintptr) {} // ERROR "escaping ...uintptr" func G() { var t int // ERROR "moved to heap" - F1(uintptr(unsafe.Pointer(&t))) // ERROR "live at call to F1: .?autotmp" "&t escapes to heap" + F1(uintptr(unsafe.Pointer(&t))) // ERROR "live at call to F1: .?autotmp" "&t escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" var t2 int // ERROR "moved to heap" - F3(uintptr(unsafe.Pointer(&t2))) // ERROR "live at call to F3: .?autotmp" "&t2 escapes to heap" + F3(uintptr(unsafe.Pointer(&t2))) // ERROR "live at call to F3: .?autotmp" "&t2 escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } func H() { var v int // ERROR "moved to heap" - F2(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to F2: .?autotmp" "escapes to heap" + F2(0, 1, uintptr(unsafe.Pointer(&v)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to F2: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" var v2 int // ERROR "moved to heap" - F4(0, 1, uintptr(unsafe.Pointer(&v2)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to F4: .?autotmp" "escapes to heap" + F4(0, 1, uintptr(unsafe.Pointer(&v2)), 2) // ERROR "live at call to newobject: .?autotmp" "live at call to F4: .?autotmp" "escapes to heap" "stack object .autotmp_[0-9]+ unsafe.Pointer$" } -- cgit v1.3-5-g9baa From 9dac0a8132d7db5225b27bdd8faeb3158e624159 Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 11 Sep 2018 15:14:28 -0700 Subject: runtime: on a signal, set traceback address to a deferreturn call When a function triggers a signal (like a segfault which translates to a nil pointer exception) during execution, a sigpanic handler is just below it on the stack. The function itself did not stop at a safepoint, so we have to figure out what safepoint we should use to scan its stack frame. Previously we used the site of the most recent defer to get the live variables at the signal site. That answer is not quite correct, as explained in #27518. Instead, use the site of a deferreturn call. It has all the right variables marked as live (no args, all the return values, except those that escape to the heap, in which case the corresponding PAUTOHEAP variables will be live instead). This CL requires stack objects, so that all the local variables and args referenced by the deferred closures keep the right variables alive. Fixes #27518 Change-Id: Id45d8a8666759986c203181090b962e2981e48ca Reviewed-on: https://go-review.googlesource.com/c/134637 Reviewed-by: Austin Clements Reviewed-by: Cherry Zhang --- src/cmd/internal/objabi/funcid.go | 2 +- src/cmd/link/internal/ld/pcln.go | 97 +++++++++++++++++++++++---------------- src/runtime/runtime2.go | 8 ++-- src/runtime/symtab.go | 4 +- src/runtime/traceback.go | 8 ++-- test/fixedbugs/issue27518a.go | 45 ++++++++++++++++++ test/fixedbugs/issue27518b.go | 72 +++++++++++++++++++++++++++++ 7 files changed, 188 insertions(+), 48 deletions(-) create mode 100644 test/fixedbugs/issue27518a.go create mode 100644 test/fixedbugs/issue27518b.go (limited to 'src/runtime') diff --git a/src/cmd/internal/objabi/funcid.go b/src/cmd/internal/objabi/funcid.go index 15a63ab8b3..92799107da 100644 --- a/src/cmd/internal/objabi/funcid.go +++ b/src/cmd/internal/objabi/funcid.go @@ -9,7 +9,7 @@ package objabi // Note that in some situations involving plugins, there may be multiple // copies of a particular special runtime function. // Note: this list must match the list in runtime/symtab.go. -type FuncID uint32 +type FuncID uint8 const ( FuncID_normal FuncID = iota // not a special function diff --git a/src/cmd/link/internal/ld/pcln.go b/src/cmd/link/internal/ld/pcln.go index 7b7f7068e7..24398fcc87 100644 --- a/src/cmd/link/internal/ld/pcln.go +++ b/src/cmd/link/internal/ld/pcln.go @@ -312,45 +312,19 @@ func (ctxt *Link) pclntab() { } off = int32(ftab.SetUint32(ctxt.Arch, int64(off), args)) - // funcID uint32 - funcID := objabi.FuncID_normal - switch s.Name { - case "runtime.main": - funcID = objabi.FuncID_runtime_main - case "runtime.goexit": - funcID = objabi.FuncID_goexit - case "runtime.jmpdefer": - funcID = objabi.FuncID_jmpdefer - case "runtime.mcall": - funcID = objabi.FuncID_mcall - case "runtime.morestack": - funcID = objabi.FuncID_morestack - case "runtime.mstart": - funcID = objabi.FuncID_mstart - case "runtime.rt0_go": - funcID = objabi.FuncID_rt0_go - case "runtime.asmcgocall": - funcID = objabi.FuncID_asmcgocall - case "runtime.sigpanic": - funcID = objabi.FuncID_sigpanic - case "runtime.runfinq": - funcID = objabi.FuncID_runfinq - case "runtime.gcBgMarkWorker": - funcID = objabi.FuncID_gcBgMarkWorker - case "runtime.systemstack_switch": - funcID = objabi.FuncID_systemstack_switch - case "runtime.systemstack": - funcID = objabi.FuncID_systemstack - case "runtime.cgocallback_gofunc": - funcID = objabi.FuncID_cgocallback_gofunc - case "runtime.gogo": - funcID = objabi.FuncID_gogo - case "runtime.externalthreadhandler": - funcID = objabi.FuncID_externalthreadhandler - case "runtime.debugCallV1": - funcID = objabi.FuncID_debugCallV1 + // deferreturn + deferreturn := uint32(0) + for _, r := range s.R { + if r.Sym != nil && r.Sym.Name == "runtime.deferreturn" && r.Add == 0 { + // Note: the relocation target is in the call instruction, but + // is not necessarily the whole instruction (for instance, on + // x86 the relocation applies to bytes [1:5] of the 5 byte call + // instruction). + deferreturn = uint32(r.Off) + break // only need one + } } - off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(funcID))) + off = int32(ftab.SetUint32(ctxt.Arch, int64(off), deferreturn)) if pcln != &pclntabZpcln { renumberfiles(ctxt, pcln.File, &pcln.Pcfile) @@ -396,7 +370,52 @@ func (ctxt *Link) pclntab() { off = addpctab(ctxt, ftab, off, &pcln.Pcfile) off = addpctab(ctxt, ftab, off, &pcln.Pcline) off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(len(pcln.Pcdata)))) - off = int32(ftab.SetUint32(ctxt.Arch, int64(off), uint32(len(pcln.Funcdata)))) + + // funcID uint8 + funcID := objabi.FuncID_normal + switch s.Name { + case "runtime.main": + funcID = objabi.FuncID_runtime_main + case "runtime.goexit": + funcID = objabi.FuncID_goexit + case "runtime.jmpdefer": + funcID = objabi.FuncID_jmpdefer + case "runtime.mcall": + funcID = objabi.FuncID_mcall + case "runtime.morestack": + funcID = objabi.FuncID_morestack + case "runtime.mstart": + funcID = objabi.FuncID_mstart + case "runtime.rt0_go": + funcID = objabi.FuncID_rt0_go + case "runtime.asmcgocall": + funcID = objabi.FuncID_asmcgocall + case "runtime.sigpanic": + funcID = objabi.FuncID_sigpanic + case "runtime.runfinq": + funcID = objabi.FuncID_runfinq + case "runtime.gcBgMarkWorker": + funcID = objabi.FuncID_gcBgMarkWorker + case "runtime.systemstack_switch": + funcID = objabi.FuncID_systemstack_switch + case "runtime.systemstack": + funcID = objabi.FuncID_systemstack + case "runtime.cgocallback_gofunc": + funcID = objabi.FuncID_cgocallback_gofunc + case "runtime.gogo": + funcID = objabi.FuncID_gogo + case "runtime.externalthreadhandler": + funcID = objabi.FuncID_externalthreadhandler + case "runtime.debugCallV1": + funcID = objabi.FuncID_debugCallV1 + } + off = int32(ftab.SetUint8(ctxt.Arch, int64(off), uint8(funcID))) + + // unused + off += 2 + + // nfuncdata must be the final entry. + off = int32(ftab.SetUint8(ctxt.Arch, int64(off), uint8(len(pcln.Funcdata)))) for i := range pcln.Pcdata { off = addpctab(ctxt, ftab, off, &pcln.Pcdata[i]) } diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index 2f009abdbb..bbb66bb8fa 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -650,14 +650,16 @@ type _func struct { entry uintptr // start pc nameoff int32 // function name - args int32 // in/out args size - funcID funcID // set for certain special runtime functions + args int32 // in/out args size + deferreturn uint32 // offset of a deferreturn block from entry, if any. pcsp int32 pcfile int32 pcln int32 npcdata int32 - nfuncdata int32 + funcID funcID // set for certain special runtime functions + _ [2]int8 // unused + nfuncdata uint8 // must be last } // layout of Itab known to compilers diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go index 452e9d06ae..1dc7ab740e 100644 --- a/src/runtime/symtab.go +++ b/src/runtime/symtab.go @@ -357,7 +357,7 @@ const ( // Note that in some situations involving plugins, there may be multiple // copies of a particular special runtime function. // Note: this list must match the list in cmd/internal/objabi/funcid.go. -type funcID uint32 +type funcID uint8 const ( funcID_normal funcID = iota // not a special function @@ -856,7 +856,7 @@ func pcdatavalue(f funcInfo, table int32, targetpc uintptr, cache *pcvalueCache) return pcvalue(f, off, targetpc, cache, true) } -func funcdata(f funcInfo, i int32) unsafe.Pointer { +func funcdata(f funcInfo, i uint8) unsafe.Pointer { if i < 0 || i >= f.nfuncdata { return nil } diff --git a/src/runtime/traceback.go b/src/runtime/traceback.go index 69d5764c8f..d7265b2bb9 100644 --- a/src/runtime/traceback.go +++ b/src/runtime/traceback.go @@ -312,8 +312,7 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // the function either doesn't return at all (if it has no defers or if the // defers do not recover) or it returns from one of the calls to // deferproc a second time (if the corresponding deferred func recovers). - // It suffices to assume that the most recent deferproc is the one that - // returns; everything live at earlier deferprocs is still live at that one. + // In the latter case, use a deferreturn call site as the continuation pc. frame.continpc = frame.pc if waspanic { // We match up defers with frames using the SP. @@ -324,7 +323,10 @@ func gentraceback(pc0, sp0, lr0 uintptr, gp *g, skip int, pcbuf *uintptr, max in // can't push a defer, the defer can't belong // to that frame. if _defer != nil && _defer.sp == frame.sp && frame.sp != frame.fp { - frame.continpc = _defer.pc + frame.continpc = frame.fn.entry + uintptr(frame.fn.deferreturn) + 1 + // Note: the +1 is to offset the -1 that + // stack.go:getStackMap does to back up a return + // address make sure the pc is in the CALL instruction. } else { frame.continpc = 0 } diff --git a/test/fixedbugs/issue27518a.go b/test/fixedbugs/issue27518a.go new file mode 100644 index 0000000000..d6224df017 --- /dev/null +++ b/test/fixedbugs/issue27518a.go @@ -0,0 +1,45 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "runtime" +) + +var nilp *int +var forceHeap interface{} + +func main() { + // x is a pointer on the stack to heap-allocated memory. + x := new([32]*int) + forceHeap = x + forceHeap = nil + + // Push a defer to be run when we panic below. + defer func() { + // Ignore the panic. + recover() + // Force a stack walk. Go 1.11 will fail because x is now + // considered live again. + runtime.GC() + }() + // Make x live at the defer's PC. + runtime.KeepAlive(x) + + // x is no longer live. Garbage collect the [32]*int on the + // heap. + runtime.GC() + // At this point x's dead stack slot points to dead memory. + + // Trigger a sigpanic. Since this is an implicit panic, we + // don't have an explicit liveness map here. + // Traceback used to use the liveness map of the most recent defer, + // but in that liveness map, x will be live again even though + // it points to dead memory. The fix is to use the liveness + // map of a deferreturn call instead. + *nilp = 0 +} diff --git a/test/fixedbugs/issue27518b.go b/test/fixedbugs/issue27518b.go new file mode 100644 index 0000000000..ea72a30885 --- /dev/null +++ b/test/fixedbugs/issue27518b.go @@ -0,0 +1,72 @@ +// run + +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import "runtime" + +var finalized bool +var err string + +type HeapObj [8]int64 + +const filler int64 = 0x123456789abcdef0 + +func (h *HeapObj) init() { + for i := 0; i < len(*h); i++ { + h[i] = filler + } +} +func (h *HeapObj) check() { + for i := 0; i < len(*h); i++ { + if h[i] != filler { + err = "filler overwritten" + } + } +} + +type StackObj struct { + h *HeapObj +} + +func gc(shouldFinalize bool) { + runtime.GC() + runtime.GC() + runtime.GC() + if shouldFinalize != finalized { + err = "heap object finalized at the wrong time" + } +} + +func main() { + var s StackObj + s.h = new(HeapObj) + s.h.init() + runtime.SetFinalizer(s.h, func(h *HeapObj) { + finalized = true + }) + gc(false) + h := g(&s) + gc(false) + h.check() + gc(true) // finalize here, after return value's last use. (Go1.11 never runs the finalizer.) + if err != "" { + panic(err) + } +} + +func g(p *StackObj) (v *HeapObj) { + gc(false) + v = p.h // last use of the stack object. the only reference to the heap object is in the return slot. + gc(false) + defer func() { + gc(false) + recover() + gc(false) + }() + *(*int)(nil) = 0 + return +} -- cgit v1.3-5-g9baa From f6fea0f31de094f43a4e7f659342de02585964ac Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Sun, 26 Aug 2018 21:30:19 -0400 Subject: runtime: skip debug call injection tests under a debugger The debug call injection tests will freeze when run under a debugger because they depend on catching SIGTRAP, which is usually swallowed by a debugger. Change-Id: If6b86ca279b0489182990dd513444ca3062973f1 Reviewed-on: https://go-review.googlesource.com/c/139437 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Keith Randall --- src/runtime/debug_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'src/runtime') diff --git a/src/runtime/debug_test.go b/src/runtime/debug_test.go index a34f4c77f7..37dcafd145 100644 --- a/src/runtime/debug_test.go +++ b/src/runtime/debug_test.go @@ -17,6 +17,8 @@ package runtime_test import ( "fmt" + "io/ioutil" + "regexp" "runtime" "runtime/debug" "sync/atomic" @@ -25,6 +27,11 @@ import ( ) func startDebugCallWorker(t *testing.T) (g *runtime.G, after func()) { + // This can deadlock if run under a debugger because it + // depends on catching SIGTRAP, which is usually swallowed by + // a debugger. + skipUnderDebugger(t) + // This can deadlock if there aren't enough threads or if a GC // tries to interrupt an atomic loop (see issue #10958). ogomaxprocs := runtime.GOMAXPROCS(2) @@ -73,6 +80,28 @@ func debugCallTKill(tid int) error { return syscall.Tgkill(syscall.Getpid(), tid, syscall.SIGTRAP) } +// skipUnderDebugger skips the current test when running under a +// debugger (specifically if this process has a tracer). This is +// Linux-specific. +func skipUnderDebugger(t *testing.T) { + pid := syscall.Getpid() + status, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/status", pid)) + if err != nil { + t.Logf("couldn't get proc tracer: %s", err) + return + } + re := regexp.MustCompile(`TracerPid:\s+([0-9]+)`) + sub := re.FindSubmatch(status) + if sub == nil { + t.Logf("couldn't find proc tracer PID") + return + } + if string(sub[1]) == "0" { + return + } + t.Skip("test will deadlock under a debugger") +} + func TestDebugCall(t *testing.T) { g, after := startDebugCallWorker(t) defer after() @@ -160,6 +189,8 @@ func debugCallUnsafePointWorker(gpp **runtime.G, ready, stop *uint32) { } func TestDebugCallUnsafePoint(t *testing.T) { + skipUnderDebugger(t) + // This can deadlock if there aren't enough threads or if a GC // tries to interrupt an atomic loop (see issue #10958). defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2)) @@ -181,6 +212,8 @@ func TestDebugCallUnsafePoint(t *testing.T) { } func TestDebugCallPanic(t *testing.T) { + skipUnderDebugger(t) + // This can deadlock if there aren't enough threads. defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2)) -- cgit v1.3-5-g9baa From c216c3aa9fe34cd81e9d4bcc28c64257064eddc9 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Fri, 17 Aug 2018 15:42:19 -0400 Subject: runtime: fix race between unminit and Windows profile loop Currently, the Windows profile loop isn't robust against racing with unminit. For example, T1 is running profileloop1, T2 is another thread T1: thread := atomic.Loaduintptr(&T2.thread) T2: calls unminit, which does CloseHandle(T2.thread) T1: attempts to suspends T2 In this case the SuspendThread will fail, but currently we ignore this failure and forge ahead, which will cause further failures and probably bad profile data. Handle this race by defending against SuspendThread failing. If SuspendThread succeeds, then we know the thread is no longer going anywhere. Change-Id: I4726553239b17f05ca07a0cf7df49631e0cb550d Reviewed-on: https://go-review.googlesource.com/c/129685 Run-TryBot: Austin Clements TryBot-Result: Gobot Gobot Reviewed-by: Alex Brainman --- src/runtime/os_windows.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'src/runtime') diff --git a/src/runtime/os_windows.go b/src/runtime/os_windows.go index 409d537839..03dd95bf17 100644 --- a/src/runtime/os_windows.go +++ b/src/runtime/os_windows.go @@ -858,14 +858,14 @@ func profileloop() var profiletimer uintptr -func profilem(mp *m) { +func profilem(mp *m, thread uintptr) { var r *context rbuf := make([]byte, unsafe.Sizeof(*r)+15) // align Context to 16 bytes r = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&rbuf[15]))) &^ 15)) r.contextflags = _CONTEXT_CONTROL - stdcall2(_GetThreadContext, mp.thread, uintptr(unsafe.Pointer(r))) + stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(r))) var gp *g switch GOARCH { @@ -906,9 +906,16 @@ func profileloop1(param uintptr) uint32 { if thread == 0 || mp.profilehz == 0 || mp.blocked { continue } - stdcall1(_SuspendThread, thread) + // mp may exit between the load above and the + // SuspendThread, so be careful. + if int32(stdcall1(_SuspendThread, thread)) == -1 { + // The thread no longer exists. + continue + } if mp.profilehz != 0 && !mp.blocked { - profilem(mp) + // Pass the thread handle in case mp + // was in the process of shutting down. + profilem(mp, thread) } stdcall1(_ResumeThread, thread) } -- cgit v1.3-5-g9baa From 9f78aa77443389d61a3ef910a1c3d4628999d82c Mon Sep 17 00:00:00 2001 From: Clément Chigot Date: Fri, 28 Sep 2018 14:54:43 +0200 Subject: runtime: add AIX operating system This commit adds AIX operating system to runtime package for ppc64 architecture. Only new files and minor changes are in this commit. Others modifications in files like asm_ppc64.s will be in separate commits. Updates: #25893 Change-Id: I9c5e073f5f3debb43b004ad1167694a5afd31cfd Reviewed-on: https://go-review.googlesource.com/c/138716 Run-TryBot: Ian Lance Taylor TryBot-Result: Gobot Gobot Reviewed-by: Ian Lance Taylor --- src/runtime/crash_unix_test.go | 2 +- src/runtime/defs_aix.go | 170 ++++++++++++++ src/runtime/defs_aix_ppc64.go | 203 +++++++++++++++++ src/runtime/env_posix.go | 2 +- src/runtime/export_unix_test.go | 2 +- src/runtime/lock_sema.go | 2 +- src/runtime/mem_aix.go | 74 ++++++ src/runtime/mmap.go | 1 + src/runtime/netpoll.go | 6 +- src/runtime/netpoll_aix.go | 247 ++++++++++++++++++++ src/runtime/os2_aix.go | 479 +++++++++++++++++++++++++++++++++++++++ src/runtime/os_aix.go | 262 +++++++++++++++++++++ src/runtime/proc.go | 4 +- src/runtime/rt0_aix_ppc64.s | 40 ++++ src/runtime/runtime_unix_test.go | 2 +- src/runtime/signal_aix_ppc64.go | 85 +++++++ src/runtime/signal_ppc64x.go | 2 +- src/runtime/signal_sighandler.go | 2 +- src/runtime/signal_unix.go | 2 +- src/runtime/sigtab_aix.go | 264 +++++++++++++++++++++ src/runtime/stubs2.go | 1 + src/runtime/stubs3.go | 1 + src/runtime/sys_aix_ppc64.s | 201 ++++++++++++++++ src/runtime/timestub2.go | 1 + 24 files changed, 2042 insertions(+), 13 deletions(-) create mode 100644 src/runtime/defs_aix.go create mode 100644 src/runtime/defs_aix_ppc64.go create mode 100644 src/runtime/mem_aix.go create mode 100644 src/runtime/netpoll_aix.go create mode 100644 src/runtime/os2_aix.go create mode 100644 src/runtime/os_aix.go create mode 100644 src/runtime/rt0_aix_ppc64.s create mode 100644 src/runtime/signal_aix_ppc64.go create mode 100644 src/runtime/sigtab_aix.go create mode 100644 src/runtime/sys_aix_ppc64.s (limited to 'src/runtime') diff --git a/src/runtime/crash_unix_test.go b/src/runtime/crash_unix_test.go index af9e6430da..1384e00210 100644 --- a/src/runtime/crash_unix_test.go +++ b/src/runtime/crash_unix_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris package runtime_test diff --git a/src/runtime/defs_aix.go b/src/runtime/defs_aix.go new file mode 100644 index 0000000000..812c7fcfa2 --- /dev/null +++ b/src/runtime/defs_aix.go @@ -0,0 +1,170 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs +GOARCH=ppc64 go tool cgo -godefs defs_aix.go > defs_aix_ppc64_tmp.go + +This is only an helper to create defs_aix_ppc64.go +Go runtime functions require the "linux" name of fields (ss_sp, si_addr, etc) +However, AIX structures don't provide such names and must be modified. + +TODO(aix): create a script to automatise defs_aix creation. + +Modifications made: + - sigset replaced by a [4]uint64 array + - add sigset_all variable + - siginfo.si_addr uintptr instead of *byte + - add (*timeval) set_usec + - stackt.ss_sp uintptr instead of *byte + - stackt.ss_size uintptr instead of uint64 + - sigcontext.sc_jmpbuf context64 instead of jumbuf + - ucontext.__extctx is a uintptr because we don't need extctx struct + - ucontext.uc_mcontext: replace jumbuf structure by context64 structure + - sigaction.sa_handler represents union field as both are uintptr + - tstate.* replace *byte by uintptr + + +*/ + +package runtime + +/* + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +*/ +import "C" + +const ( + _EPERM = C.EPERM + _ENOENT = C.ENOENT + _EINTR = C.EINTR + _EAGAIN = C.EAGAIN + _ENOMEM = C.ENOMEM + _EACCES = C.EACCES + _EFAULT = C.EFAULT + _EINVAL = C.EINVAL + _ETIMEDOUT = C.ETIMEDOUT + + _PROT_NONE = C.PROT_NONE + _PROT_READ = C.PROT_READ + _PROT_WRITE = C.PROT_WRITE + _PROT_EXEC = C.PROT_EXEC + + _MAP_ANONYMOUS = C.MAP_ANONYMOUS + _MAP_PRIVATE = C.MAP_PRIVATE + _MAP_FIXED = C.MAP_FIXED + _MADV_DONTNEED = C.MADV_DONTNEED + + _SIGHUP = C.SIGHUP + _SIGINT = C.SIGINT + _SIGQUIT = C.SIGQUIT + _SIGILL = C.SIGILL + _SIGTRAP = C.SIGTRAP + _SIGABRT = C.SIGABRT + _SIGBUS = C.SIGBUS + _SIGFPE = C.SIGFPE + _SIGKILL = C.SIGKILL + _SIGUSR1 = C.SIGUSR1 + _SIGSEGV = C.SIGSEGV + _SIGUSR2 = C.SIGUSR2 + _SIGPIPE = C.SIGPIPE + _SIGALRM = C.SIGALRM + _SIGCHLD = C.SIGCHLD + _SIGCONT = C.SIGCONT + _SIGSTOP = C.SIGSTOP + _SIGTSTP = C.SIGTSTP + _SIGTTIN = C.SIGTTIN + _SIGTTOU = C.SIGTTOU + _SIGURG = C.SIGURG + _SIGXCPU = C.SIGXCPU + _SIGXFSZ = C.SIGXFSZ + _SIGVTALRM = C.SIGVTALRM + _SIGPROF = C.SIGPROF + _SIGWINCH = C.SIGWINCH + _SIGIO = C.SIGIO + _SIGPWR = C.SIGPWR + _SIGSYS = C.SIGSYS + _SIGTERM = C.SIGTERM + _SIGEMT = C.SIGEMT + _SIGWAITING = C.SIGWAITING + + _FPE_INTDIV = C.FPE_INTDIV + _FPE_INTOVF = C.FPE_INTOVF + _FPE_FLTDIV = C.FPE_FLTDIV + _FPE_FLTOVF = C.FPE_FLTOVF + _FPE_FLTUND = C.FPE_FLTUND + _FPE_FLTRES = C.FPE_FLTRES + _FPE_FLTINV = C.FPE_FLTINV + _FPE_FLTSUB = C.FPE_FLTSUB + + _BUS_ADRALN = C.BUS_ADRALN + _BUS_ADRERR = C.BUS_ADRERR + _BUS_OBJERR = C.BUS_OBJERR + + _SEGV_MAPERR = C.SEGV_MAPERR + _SEGV_ACCERR = C.SEGV_ACCERR + + _ITIMER_REAL = C.ITIMER_REAL + _ITIMER_VIRTUAL = C.ITIMER_VIRTUAL + _ITIMER_PROF = C.ITIMER_PROF + + _O_RDONLY = C.O_RDONLY + + _SS_DISABLE = C.SS_DISABLE + _SI_USER = C.SI_USER + _SIG_BLOCK = C.SIG_BLOCK + _SIG_UNBLOCK = C.SIG_UNBLOCK + _SIG_SETMASK = C.SIG_SETMASK + + _SA_SIGINFO = C.SA_SIGINFO + _SA_RESTART = C.SA_RESTART + _SA_ONSTACK = C.SA_ONSTACK + + _PTHREAD_CREATE_DETACHED = C.PTHREAD_CREATE_DETACHED + + __SC_PAGE_SIZE = C._SC_PAGE_SIZE + __SC_NPROCESSORS_ONLN = C._SC_NPROCESSORS_ONLN + + _F_SETFD = C.F_SETFD + _F_SETFL = C.F_SETFL + _F_GETFD = C.F_GETFD + _F_GETFL = C.F_GETFL + _FD_CLOEXEC = C.FD_CLOEXEC +) + +type sigset C.sigset_t +type siginfo C.siginfo_t +type timespec C.struct_timespec +type timestruc C.struct_timestruc_t +type timeval C.struct_timeval +type itimerval C.struct_itimerval + +type stackt C.stack_t +type sigcontext C.struct_sigcontext +type ucontext C.ucontext_t +type _Ctype_struct___extctx uint64 // ucontext use a pointer to this structure but it shouldn't be used +type jmpbuf C.struct___jmpbuf +type context64 C.struct___context64 +type sigactiont C.struct_sigaction +type tstate C.struct_tstate +type rusage C.struct_rusage + +type pthread C.pthread_t +type pthread_attr C.pthread_attr_t + +type semt C.sem_t diff --git a/src/runtime/defs_aix_ppc64.go b/src/runtime/defs_aix_ppc64.go new file mode 100644 index 0000000000..e7480d06ba --- /dev/null +++ b/src/runtime/defs_aix_ppc64.go @@ -0,0 +1,203 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix + +package runtime + +const ( + _EPERM = 0x1 + _ENOENT = 0x2 + _EINTR = 0x4 + _EAGAIN = 0xb + _ENOMEM = 0xc + _EACCES = 0xd + _EFAULT = 0xe + _EINVAL = 0x16 + _ETIMEDOUT = 0x4e + + _PROT_NONE = 0x0 + _PROT_READ = 0x1 + _PROT_WRITE = 0x2 + _PROT_EXEC = 0x4 + + _MAP_ANONYMOUS = 0x10 + _MAP_PRIVATE = 0x2 + _MAP_FIXED = 0x100 + _MADV_DONTNEED = 0x4 + + _SIGHUP = 0x1 + _SIGINT = 0x2 + _SIGQUIT = 0x3 + _SIGILL = 0x4 + _SIGTRAP = 0x5 + _SIGABRT = 0x6 + _SIGBUS = 0xa + _SIGFPE = 0x8 + _SIGKILL = 0x9 + _SIGUSR1 = 0x1e + _SIGSEGV = 0xb + _SIGUSR2 = 0x1f + _SIGPIPE = 0xd + _SIGALRM = 0xe + _SIGCHLD = 0x14 + _SIGCONT = 0x13 + _SIGSTOP = 0x11 + _SIGTSTP = 0x12 + _SIGTTIN = 0x15 + _SIGTTOU = 0x16 + _SIGURG = 0x10 + _SIGXCPU = 0x18 + _SIGXFSZ = 0x19 + _SIGVTALRM = 0x22 + _SIGPROF = 0x20 + _SIGWINCH = 0x1c + _SIGIO = 0x17 + _SIGPWR = 0x1d + _SIGSYS = 0xc + _SIGTERM = 0xf + _SIGEMT = 0x7 + _SIGWAITING = 0x27 + + _FPE_INTDIV = 0x14 + _FPE_INTOVF = 0x15 + _FPE_FLTDIV = 0x16 + _FPE_FLTOVF = 0x17 + _FPE_FLTUND = 0x18 + _FPE_FLTRES = 0x19 + _FPE_FLTINV = 0x1a + _FPE_FLTSUB = 0x1b + + _BUS_ADRALN = 0x1 + _BUS_ADRERR = 0x2 + _BUS_OBJERR = 0x3 + _ + _SEGV_MAPERR = 0x32 + _SEGV_ACCERR = 0x33 + + _ITIMER_REAL = 0x0 + _ITIMER_VIRTUAL = 0x1 + _ITIMER_PROF = 0x2 + + _O_RDONLY = 0x0 + + _SS_DISABLE = 0x2 + _SI_USER = 0x0 + _SIG_BLOCK = 0x0 + _SIG_UNBLOCK = 0x1 + _SIG_SETMASK = 0x2 + + _SA_SIGINFO = 0x100 + _SA_RESTART = 0x8 + _SA_ONSTACK = 0x1 + + _PTHREAD_CREATE_DETACHED = 0x1 + + __SC_PAGE_SIZE = 0x30 + __SC_NPROCESSORS_ONLN = 0x48 + + _F_SETFD = 0x2 + _F_SETFL = 0x4 + _F_GETFD = 0x1 + _F_GETFL = 0x3 + _FD_CLOEXEC = 0x1 +) + +type sigset [4]uint64 + +var sigset_all = sigset{^uint64(0), ^uint64(0), ^uint64(0), ^uint64(0)} + +type siginfo struct { + si_signo int32 + si_errno int32 + si_code int32 + si_pid int32 + si_uid uint32 + si_status int32 + si_addr uintptr + si_band int64 + si_value [2]int32 // [8]byte + __si_flags int32 + __pad [3]int32 +} + +type timespec struct { + tv_sec int64 + tv_nsec int64 +} +type timeval struct { + tv_sec int64 + tv_usec int32 + pad_cgo_0 [4]byte +} + +func (tv *timeval) set_usec(x int32) { + tv.tv_usec = x +} + +type itimerval struct { + it_interval timeval + it_value timeval +} + +type stackt struct { + ss_sp uintptr + ss_size uintptr + ss_flags int32 + __pad [4]int32 + pas_cgo_0 [4]byte +} + +type sigcontext struct { + sc_onstack int32 + pad_cgo_0 [4]byte + sc_mask sigset + sc_uerror int32 + sc_jmpbuf context64 +} + +type ucontext struct { + __sc_onstack int32 + pad_cgo_0 [4]byte + uc_sigmask sigset + __sc_error int32 + pad_cgo_1 [4]byte + uc_mcontext context64 + uc_link *ucontext + uc_stack stackt + __extctx uintptr // pointer to struct __extctx but we don't use it + __extctx_magic int32 + __pad int32 +} + +type context64 struct { + gpr [32]uint64 + msr uint64 + iar uint64 + lr uint64 + ctr uint64 + cr uint32 + xer uint32 + fpscr uint32 + fpscrx uint32 + except [1]uint64 + fpr [32]float64 + fpeu uint8 + fpinfo uint8 + fpscr24_31 uint8 + pad [1]uint8 + excp_type int32 +} + +type sigactiont struct { + sa_handler uintptr // a union of two pointer + sa_mask sigset + sa_flags int32 + pad_cgo_0 [4]byte +} + +type pthread uint32 +type pthread_attr *byte + +type semt int32 diff --git a/src/runtime/env_posix.go b/src/runtime/env_posix.go index 032e7122ce..a2daeb7f27 100644 --- a/src/runtime/env_posix.go +++ b/src/runtime/env_posix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package runtime diff --git a/src/runtime/export_unix_test.go b/src/runtime/export_unix_test.go index 54d577072e..eecdfb7eb2 100644 --- a/src/runtime/export_unix_test.go +++ b/src/runtime/export_unix_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris package runtime diff --git a/src/runtime/lock_sema.go b/src/runtime/lock_sema.go index 6e01d70f75..d21a055685 100644 --- a/src/runtime/lock_sema.go +++ b/src/runtime/lock_sema.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin nacl netbsd openbsd plan9 solaris windows +// +build aix darwin nacl netbsd openbsd plan9 solaris windows package runtime diff --git a/src/runtime/mem_aix.go b/src/runtime/mem_aix.go new file mode 100644 index 0000000000..f11f0aba52 --- /dev/null +++ b/src/runtime/mem_aix.go @@ -0,0 +1,74 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import ( + "unsafe" +) + +// Don't split the stack as this method may be invoked without a valid G, which +// prevents us from allocating more stack. +//go:nosplit +func sysAlloc(n uintptr, sysStat *uint64) unsafe.Pointer { + p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANONYMOUS|_MAP_PRIVATE, -1, 0) + if err != 0 { + if err == _EACCES { + print("runtime: mmap: access denied\n") + exit(2) + } + if err == _EAGAIN { + print("runtime: mmap: too much locked memory (check 'ulimit -l').\n") + exit(2) + } + //println("sysAlloc failed: ", err) + return nil + } + mSysStatInc(sysStat, n) + return p +} + +func sysUnused(v unsafe.Pointer, n uintptr) { + madvise(v, n, _MADV_DONTNEED) +} + +func sysUsed(v unsafe.Pointer, n uintptr) { +} + +// Don't split the stack as this function may be invoked without a valid G, +// which prevents us from allocating more stack. +//go:nosplit +func sysFree(v unsafe.Pointer, n uintptr, sysStat *uint64) { + mSysStatDec(sysStat, n) + munmap(v, n) + +} + +func sysFault(v unsafe.Pointer, n uintptr) { + mmap(v, n, _PROT_NONE, _MAP_ANONYMOUS|_MAP_PRIVATE|_MAP_FIXED, -1, 0) +} + +func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer { + p, err := mmap(v, n, _PROT_NONE, _MAP_ANONYMOUS|_MAP_PRIVATE, -1, 0) + if err != 0 { + return nil + } + return p +} + +func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) { + mSysStatInc(sysStat, n) + + // AIX does not allow mapping a range that is already mapped. + // So always unmap first even if it is already unmapped. + munmap(v, n) + p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANONYMOUS|_MAP_FIXED|_MAP_PRIVATE, -1, 0) + + if err == _ENOMEM { + throw("runtime: out of memory") + } + if p != v || err != 0 { + throw("runtime: cannot map pages in arena address space") + } +} diff --git a/src/runtime/mmap.go b/src/runtime/mmap.go index fe09e7029e..2868f3fd4e 100644 --- a/src/runtime/mmap.go +++ b/src/runtime/mmap.go @@ -10,6 +10,7 @@ // +build !linux !arm64 // +build !js // +build !darwin +// +build !aix package runtime diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index f9c422650a..da822a7308 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows +// +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris windows package runtime @@ -166,8 +166,8 @@ func poll_runtime_pollWait(pd *pollDesc, mode int) int { if err != 0 { return err } - // As for now only Solaris uses level-triggered IO. - if GOOS == "solaris" { + // As for now only Solaris and AIX use level-triggered IO. + if GOOS == "solaris" || GOOS == "aix" { netpollarm(pd, mode) } for !netpollblock(pd, int32(mode), false) { diff --git a/src/runtime/netpoll_aix.go b/src/runtime/netpoll_aix.go new file mode 100644 index 0000000000..1e886dae94 --- /dev/null +++ b/src/runtime/netpoll_aix.go @@ -0,0 +1,247 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +import "unsafe" + +// This is based on the former libgo/runtime/netpoll_select.c implementation +// except that it uses poll instead of select and is written in Go. +// It's also based on Solaris implementation for the arming mechanisms + +//go:cgo_import_dynamic libc_poll poll "libc.a/shr_64.o" +//go:linkname libc_poll libc_poll + +var libc_poll libFunc + +//go:nosplit +func poll(pfds *pollfd, npfds uintptr, timeout uintptr) (int32, int32) { + r, err := syscall3(&libc_poll, uintptr(unsafe.Pointer(pfds)), npfds, timeout) + return int32(r), int32(err) +} + +//go:nosplit +func fcntl(fd, cmd int32, arg uintptr) int32 { + r, _ := syscall3(&libc_fcntl, uintptr(fd), uintptr(cmd), arg) + return int32(r) +} + +// pollfd represents the poll structure for AIX operating system. +type pollfd struct { + fd int32 + events int16 + revents int16 +} + +const _POLLIN = 0x0001 +const _POLLOUT = 0x0002 +const _POLLHUP = 0x2000 +const _POLLERR = 0x4000 +const _O_NONBLOCK = 0x4 + +var ( + pfds []pollfd + pds []*pollDesc + mtxpoll mutex + mtxset mutex + rdwake int32 + wrwake int32 + pendingUpdates int32 +) + +const pollVerbose = false + +func netpollinit() { + var p [2]int32 + + // Create the pipe we use to wakeup poll. + if err := pipe(&p[0]); err < 0 { + throw("netpollinit: failed to create pipe") + } + rdwake = p[0] + wrwake = p[1] + + fl := uintptr(fcntl(rdwake, _F_GETFL, 0)) + fcntl(rdwake, _F_SETFL, fl|_O_NONBLOCK) + fcntl(rdwake, _F_SETFD, _FD_CLOEXEC) + + fl = uintptr(fcntl(wrwake, _F_GETFL, 0)) + fcntl(wrwake, _F_SETFL, fl|_O_NONBLOCK) + fcntl(wrwake, _F_SETFD, _FD_CLOEXEC) + + // Pre-allocate array of pollfd structures for poll. + if pollVerbose { + println("*** allocating") + } + pfds = make([]pollfd, 1, 128) + if pollVerbose { + println("*** allocating done", &pfds[0]) + } + + // Poll the read side of the pipe. + pfds[0].fd = rdwake + pfds[0].events = _POLLIN + + pds = make([]*pollDesc, 1, 128) + pds[0] = nil +} + +func netpolldescriptor() uintptr { + // Both fd must be returned + if rdwake > 0xFFFF || wrwake > 0xFFFF { + throw("netpolldescriptor: invalid fd number") + } + return uintptr(rdwake<<16 | wrwake) +} + +// netpollwakeup writes on wrwake to wakeup poll before any changes. +func netpollwakeup() { + if pendingUpdates == 0 { + pendingUpdates = 1 + if pollVerbose { + println("*** writing 1 byte") + } + b := [1]byte{0} + write(uintptr(wrwake), unsafe.Pointer(&b[0]), 1) + } +} + +func netpollopen(fd uintptr, pd *pollDesc) int32 { + if pollVerbose { + println("*** netpollopen", fd) + } + lock(&mtxpoll) + netpollwakeup() + + lock(&mtxset) + unlock(&mtxpoll) + + pd.user = uint32(len(pfds)) + pfds = append(pfds, pollfd{fd: int32(fd)}) + pds = append(pds, pd) + unlock(&mtxset) + return 0 +} + +func netpollclose(fd uintptr) int32 { + if pollVerbose { + println("*** netpollclose", fd) + } + lock(&mtxpoll) + netpollwakeup() + + lock(&mtxset) + unlock(&mtxpoll) + + for i := 0; i < len(pfds); i++ { + if pfds[i].fd == int32(fd) { + pfds[i] = pfds[len(pfds)-1] + pfds = pfds[:len(pfds)-1] + + pds[i] = pds[len(pds)-1] + pds[i].user = uint32(i) + pds = pds[:len(pds)-1] + break + } + } + unlock(&mtxset) + return 0 +} + +func netpollarm(pd *pollDesc, mode int) { + if pollVerbose { + println("*** netpollarm", pd.fd, mode) + } + lock(&mtxpoll) + netpollwakeup() + + lock(&mtxset) + unlock(&mtxpoll) + + switch mode { + case 'r': + pfds[pd.user].events |= _POLLIN + case 'w': + pfds[pd.user].events |= _POLLOUT + } + unlock(&mtxset) +} + +//go:nowritebarrierrec +func netpoll(block bool) gList { + timeout := ^uintptr(0) + if !block { + timeout = 0 + return gList{} + } + if pollVerbose { + println("*** netpoll", block) + } +retry: + lock(&mtxpoll) + lock(&mtxset) + pendingUpdates = 0 + unlock(&mtxpoll) + + if pollVerbose { + println("*** netpoll before poll") + } + n, e := poll(&pfds[0], uintptr(len(pfds)), timeout) + if pollVerbose { + println("*** netpoll after poll", n) + } + if n < 0 { + if e != _EINTR { + println("errno=", e, " len(pfds)=", len(pfds)) + throw("poll failed") + } + if pollVerbose { + println("*** poll failed") + } + unlock(&mtxset) + goto retry + } + // Check if some descriptors need to be changed + if n != 0 && pfds[0].revents&(_POLLIN|_POLLHUP|_POLLERR) != 0 { + var b [1]byte + for read(rdwake, unsafe.Pointer(&b[0]), 1) == 1 { + if pollVerbose { + println("*** read 1 byte from pipe") + } + } + // Do not look at the other fds in this case as the mode may have changed + // XXX only additions of flags are made, so maybe it is ok + unlock(&mtxset) + goto retry + } + var toRun gList + for i := 0; i < len(pfds) && n > 0; i++ { + pfd := &pfds[i] + + var mode int32 + if pfd.revents&(_POLLIN|_POLLHUP|_POLLERR) != 0 { + mode += 'r' + pfd.events &= ^_POLLIN + } + if pfd.revents&(_POLLOUT|_POLLHUP|_POLLERR) != 0 { + mode += 'w' + pfd.events &= ^_POLLOUT + } + if mode != 0 { + if pollVerbose { + println("*** netpollready i=", i, "revents=", pfd.revents, "events=", pfd.events, "pd=", pds[i]) + } + netpollready(&toRun, pds[i], mode) + n-- + } + } + unlock(&mtxset) + if block && toRun.empty() { + goto retry + } + if pollVerbose { + println("*** netpoll returning end") + } + return toRun +} diff --git a/src/runtime/os2_aix.go b/src/runtime/os2_aix.go new file mode 100644 index 0000000000..9e26ce23fc --- /dev/null +++ b/src/runtime/os2_aix.go @@ -0,0 +1,479 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file contains main runtime AIX syscalls. +// Pollset syscalls are in netpoll_aix.go. +// The implementation is based on Solaris and Windows. +// Each syscall is made by calling its libc symbol using asmcgocall and asmsyscall6 +// asssembly functions. + +package runtime + +import ( + "unsafe" +) + +// Symbols imported for __start function. + +//go:cgo_import_dynamic libc___n_pthreads __n_pthreads "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libc___mod_init __mod_init "libc.a/shr_64.o" +//go:linkname libc___n_pthreads libc___n_pthread +//go:linkname libc___mod_init libc___mod_init + +var ( + libc___n_pthread, + libc___mod_init libFunc +) + +// Syscalls + +//go:cgo_import_dynamic libc__Errno _Errno "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_close close "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_exit exit "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_getpid getpid "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_kill kill "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_madvise madvise "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_malloc malloc "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_mmap mmap "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_munmap munmap "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_open open "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_pipe pipe "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_raise raise "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_read read "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sched_yield sched_yield "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sem_init sem_init "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sem_post sem_post "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sem_timedwait sem_timedwait "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sem_wait sem_wait "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_setitimer setitimer "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sigaction sigaction "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_sysconf sysconf "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_usleep usleep "libc.a/shr_64.o" +//go:cgo_import_dynamic libc_write write "libc.a/shr_64.o" + +//go:cgo_import_dynamic libpthread___pth_init __pth_init "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_attr_destroy pthread_attr_destroy "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_attr_init pthread_attr_init "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_attr_getstacksize pthread_attr_getstacksize "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_attr_setstacksize pthread_attr_setstacksize "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_attr_setdetachstate pthread_attr_setdetachstate "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_attr_setstackaddr pthread_attr_setstackaddr "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_create pthread_create "libpthread.a/shr_xpg5_64.o" +//go:cgo_import_dynamic libpthread_sigthreadmask sigthreadmask "libpthread.a/shr_xpg5_64.o" + +//go:linkname libc__Errno libc__Errno +//go:linkname libc_clock_gettime libc_clock_gettime +//go:linkname libc_close libc_close +//go:linkname libc_exit libc_exit +//go:linkname libc_getpid libc_getpid +//go:linkname libc_kill libc_kill +//go:linkname libc_madvise libc_madvise +//go:linkname libc_malloc libc_malloc +//go:linkname libc_mmap libc_mmap +//go:linkname libc_munmap libc_munmap +//go:linkname libc_open libc_open +//go:linkname libc_pipe libc_pipe +//go:linkname libc_raise libc_raise +//go:linkname libc_read libc_read +//go:linkname libc_sched_yield libc_sched_yield +//go:linkname libc_sem_init libc_sem_init +//go:linkname libc_sem_post libc_sem_post +//go:linkname libc_sem_timedwait libc_sem_timedwait +//go:linkname libc_sem_wait libc_sem_wait +//go:linkname libc_setitimer libc_setitimer +//go:linkname libc_sigaction libc_sigaction +//go:linkname libc_sigaltstack libc_sigaltstack +//go:linkname libc_sysconf libc_sysconf +//go:linkname libc_usleep libc_usleep +//go:linkname libc_write libc_write + +//go:linkname libpthread___pth_init libpthread___pth_init +//go:linkname libpthread_attr_destroy libpthread_attr_destroy +//go:linkname libpthread_attr_init libpthread_attr_init +//go:linkname libpthread_attr_getstacksize libpthread_attr_getstacksize +//go:linkname libpthread_attr_setstacksize libpthread_attr_setstacksize +//go:linkname libpthread_attr_setdetachstate libpthread_attr_setdetachstate +//go:linkname libpthread_attr_setstackaddr libpthread_attr_setstackaddr +//go:linkname libpthread_create libpthread_create +//go:linkname libpthread_sigthreadmask libpthread_sigthreadmask + +var ( + //libc + libc__Errno, + libc_clock_gettime, + libc_close, + libc_exit, + libc_getpid, + libc_kill, + libc_madvise, + libc_malloc, + libc_mmap, + libc_munmap, + libc_open, + libc_pipe, + libc_raise, + libc_read, + libc_sched_yield, + libc_sem_init, + libc_sem_post, + libc_sem_timedwait, + libc_sem_wait, + libc_setitimer, + libc_sigaction, + libc_sigaltstack, + libc_sysconf, + libc_usleep, + libc_write, + //libpthread + libpthread___pth_init, + libpthread_attr_destroy, + libpthread_attr_init, + libpthread_attr_getstacksize, + libpthread_attr_setstacksize, + libpthread_attr_setdetachstate, + libpthread_attr_setstackaddr, + libpthread_create, + libpthread_sigthreadmask libFunc +) + +type libFunc uintptr + +// asmsyscall6 calls the libc symbol using a C convention. +// It's defined in sys_aix_ppc64.go. +var asmsyscall6 libFunc + +//go:nowritebarrier +//go:nosplit +func syscall0(fn *libFunc) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 0 + c.args = uintptr(noescape(unsafe.Pointer(&fn))) // it's unused but must be non-nil, otherwise crashes + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nowritebarrier +//go:nosplit +func syscall1(fn *libFunc, a0 uintptr) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 1 + c.args = uintptr(noescape(unsafe.Pointer(&a0))) + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nowritebarrier +//go:nosplit +func syscall2(fn *libFunc, a0, a1 uintptr) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 2 + c.args = uintptr(noescape(unsafe.Pointer(&a0))) + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nowritebarrier +//go:nosplit +func syscall3(fn *libFunc, a0, a1, a2 uintptr) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 3 + c.args = uintptr(noescape(unsafe.Pointer(&a0))) + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nowritebarrier +//go:nosplit +func syscall4(fn *libFunc, a0, a1, a2, a3 uintptr) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 4 + c.args = uintptr(noescape(unsafe.Pointer(&a0))) + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nowritebarrier +//go:nosplit +func syscall5(fn *libFunc, a0, a1, a2, a3, a4 uintptr) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 5 + c.args = uintptr(noescape(unsafe.Pointer(&a0))) + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nowritebarrier +//go:nosplit +func syscall6(fn *libFunc, a0, a1, a2, a3, a4, a5 uintptr) (r, err uintptr) { + c := &getg().m.libcall + c.fn = uintptr(unsafe.Pointer(fn)) + c.n = 6 + c.args = uintptr(noescape(unsafe.Pointer(&a0))) + + asmcgocall(unsafe.Pointer(&asmsyscall6), unsafe.Pointer(c)) + + return c.r1, c.err +} + +//go:nosplit +func exit(code int32) { + syscall1(&libc_exit, uintptr(code)) +} + +//go:nosplit +func write(fd uintptr, p unsafe.Pointer, n int32) int32 { + r, _ := syscall3(&libc_write, uintptr(fd), uintptr(p), uintptr(n)) + return int32(r) + +} + +//go:nosplit +func read(fd int32, p unsafe.Pointer, n int32) int32 { + r, _ := syscall3(&libc_read, uintptr(fd), uintptr(p), uintptr(n)) + return int32(r) +} + +//go:nosplit +func open(name *byte, mode, perm int32) int32 { + r, _ := syscall3(&libc_open, uintptr(unsafe.Pointer(name)), uintptr(mode), uintptr(perm)) + return int32(r) +} + +//go:nosplit +func closefd(fd int32) int32 { + r, _ := syscall1(&libc_close, uintptr(fd)) + return int32(r) +} + +//go:nosplit +func pipe(fd *int32) int32 { + r, _ := syscall1(&libc_pipe, uintptr(unsafe.Pointer(fd))) + return int32(r) +} + +// mmap calls the mmap system call. +// We only pass the lower 32 bits of file offset to the +// assembly routine; the higher bits (if required), should be provided +// by the assembly routine as 0. +// The err result is an OS error code such as ENOMEM. +//go:nosplit +func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (p unsafe.Pointer, err int) { + r, err0 := syscall6(&libc_mmap, uintptr(addr), uintptr(n), uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off)) + return unsafe.Pointer(r), int(err0) +} + +//go:nosplit +func munmap(addr unsafe.Pointer, n uintptr) { + r, err := syscall2(&libc_munmap, uintptr(addr), uintptr(n)) + if int32(r) == -1 { + println("syscall munmap failed: ", hex(err)) + throw("syscall munmap") + } +} + +//go:nosplit +func madvise(addr unsafe.Pointer, n uintptr, flags int32) { + r, err := syscall3(&libc_madvise, uintptr(addr), uintptr(n), uintptr(flags)) + if int32(r) == -1 { + println("syscall madvise failed: ", hex(err)) + throw("syscall madvise") + } +} + +//go:nosplit +func sigaction(sig uintptr, new, old *sigactiont) { + r, err := syscall3(&libc_sigaction, sig, uintptr(unsafe.Pointer(new)), uintptr(unsafe.Pointer(old))) + if int32(r) == -1 { + println("Sigaction failed for sig: ", sig, " with error:", hex(err)) + throw("syscall sigaction") + } +} + +//go:nosplit +func sigaltstack(new, old *stackt) { + r, err := syscall2(&libc_sigaltstack, uintptr(unsafe.Pointer(new)), uintptr(unsafe.Pointer(old))) + if int32(r) == -1 { + println("syscall sigaltstack failed: ", hex(err)) + throw("syscall sigaltstack") + } +} + +//go:nosplit +func usleep(us uint32) { + r, err := syscall1(&libc_usleep, uintptr(us)) + if int32(r) == -1 { + println("syscall usleep failed: ", hex(err)) + throw("syscall usleep") + } +} + +//go:nosplit +func clock_gettime(clockid int32, tp *timespec) int32 { + r, _ := syscall2(&libc_clock_gettime, uintptr(clockid), uintptr(unsafe.Pointer(tp))) + return int32(r) +} + +//go:nosplit +func setitimer(mode int32, new, old *itimerval) { + r, err := syscall3(&libc_setitimer, uintptr(mode), uintptr(unsafe.Pointer(new)), uintptr(unsafe.Pointer(old))) + if int32(r) == -1 { + println("syscall setitimer failed: ", hex(err)) + throw("syscall setitimer") + } +} + +//go:nosplit +func malloc(size uintptr) unsafe.Pointer { + r, _ := syscall1(&libc_malloc, size) + return unsafe.Pointer(r) +} + +//go:nosplit +func sem_init(sem *semt, pshared int32, value uint32) int32 { + r, _ := syscall3(&libc_sem_init, uintptr(unsafe.Pointer(sem)), uintptr(pshared), uintptr(value)) + return int32(r) +} + +//go:nosplit +func sem_wait(sem *semt) (int32, int32) { + r, err := syscall1(&libc_sem_wait, uintptr(unsafe.Pointer(sem))) + return int32(r), int32(err) +} + +//go:nosplit +func sem_post(sem *semt) int32 { + r, _ := syscall1(&libc_sem_post, uintptr(unsafe.Pointer(sem))) + return int32(r) +} + +//go:nosplit +func sem_timedwait(sem *semt, timeout *timespec) (int32, int32) { + r, err := syscall2(&libc_sem_timedwait, uintptr(unsafe.Pointer(sem)), uintptr(unsafe.Pointer(timeout))) + return int32(r), int32(err) +} + +//go:nosplit +func raise(sig uint32) { + r, err := syscall1(&libc_raise, uintptr(sig)) + if int32(r) == -1 { + println("syscall raise failed: ", hex(err)) + throw("syscall raise") + } +} + +//go:nosplit +func raiseproc(sig uint32) { + pid, err := syscall0(&libc_getpid) + if int32(pid) == -1 { + println("syscall getpid failed: ", hex(err)) + throw("syscall raiseproc") + } + + syscall2(&libc_kill, pid, uintptr(sig)) +} + +func osyield1() + +//go:nosplit +func osyield() { + _g_ := getg() + + // Check the validity of m because we might be called in cgo callback + // path early enough where there isn't a m available yet. + if _g_ != nil && _g_.m != nil { + r, err := syscall0(&libc_sched_yield) + if int32(r) == -1 { + println("syscall osyield failed: ", hex(err)) + throw("syscall osyield") + } + return + } + osyield1() +} + +//go:nosplit +func sysconf(name int32) uintptr { + r, _ := syscall1(&libc_sysconf, uintptr(name)) + if int32(r) == -1 { + throw("syscall sysconf") + } + return r + +} + +// pthread functions returns its error code in the main return value +// Therefore, err returns by syscall means nothing and must not be used + +//go:nosplit +func pthread_attr_destroy(attr *pthread_attr) int32 { + r, _ := syscall1(&libpthread_attr_destroy, uintptr(unsafe.Pointer(attr))) + return int32(r) +} + +//go:nosplit +func pthread_attr_init(attr *pthread_attr) int32 { + r, _ := syscall1(&libpthread_attr_init, uintptr(unsafe.Pointer(attr))) + return int32(r) +} + +//go:nosplit +func pthread_attr_setdetachstate(attr *pthread_attr, state int32) int32 { + r, _ := syscall2(&libpthread_attr_setdetachstate, uintptr(unsafe.Pointer(attr)), uintptr(state)) + return int32(r) +} + +//go:nosplit +func pthread_attr_setstackaddr(attr *pthread_attr, stk unsafe.Pointer) int32 { + r, _ := syscall2(&libpthread_attr_setstackaddr, uintptr(unsafe.Pointer(attr)), uintptr(stk)) + return int32(r) +} + +//go:nosplit +func pthread_attr_getstacksize(attr *pthread_attr, size *uint64) int32 { + r, _ := syscall2(&libpthread_attr_getstacksize, uintptr(unsafe.Pointer(attr)), uintptr(unsafe.Pointer(size))) + return int32(r) +} + +//go:nosplit +func pthread_attr_setstacksize(attr *pthread_attr, size uint64) int32 { + r, _ := syscall2(&libpthread_attr_setstacksize, uintptr(unsafe.Pointer(attr)), uintptr(size)) + return int32(r) +} + +//go:nosplit +func pthread_create(tid *pthread, attr *pthread_attr, fn *funcDescriptor, arg unsafe.Pointer) int32 { + r, _ := syscall4(&libpthread_create, uintptr(unsafe.Pointer(tid)), uintptr(unsafe.Pointer(attr)), uintptr(unsafe.Pointer(fn)), uintptr(arg)) + return int32(r) +} + +// On multi-thread program, sigprocmask must not be called. +// It's replaced by sigthreadmask. +//go:nosplit +func sigprocmask(how int32, new, old *sigset) { + r, err := syscall3(&libpthread_sigthreadmask, uintptr(how), uintptr(unsafe.Pointer(new)), uintptr(unsafe.Pointer(old))) + if int32(r) != 0 { + println("syscall sigthreadmask failed: ", hex(err)) + throw("syscall sigthreadmask") + } +} diff --git a/src/runtime/os_aix.go b/src/runtime/os_aix.go new file mode 100644 index 0000000000..31590f22d8 --- /dev/null +++ b/src/runtime/os_aix.go @@ -0,0 +1,262 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix + +package runtime + +import ( + "unsafe" +) + +const ( + threadStackSize = 0x100000 // size of a thread stack allocated by OS +) + +// funcDescriptor is a structure representing a function descriptor +// A variable with this type is always created in assembler +type funcDescriptor struct { + fn uintptr + toc uintptr + envPointer uintptr // unused in Golang +} + +type mOS struct { + waitsema uintptr // semaphore for parking on locks + perrno uintptr // pointer to tls errno +} + +//go:nosplit +func semacreate(mp *m) { + if mp.waitsema != 0 { + return + } + + var sem *semt + + // Call libc's malloc rather than malloc. This will + // allocate space on the C heap. We can't call mallocgc + // here because it could cause a deadlock. + sem = (*semt)(malloc(unsafe.Sizeof(*sem))) + if sem_init(sem, 0, 0) != 0 { + throw("sem_init") + } + mp.waitsema = uintptr(unsafe.Pointer(sem)) +} + +//go:nosplit +func semasleep(ns int64) int32 { + _m_ := getg().m + if ns >= 0 { + var ts timespec + + if clock_gettime(_CLOCK_REALTIME, &ts) != 0 { + throw("clock_gettime") + } + ts.tv_sec += ns / 1e9 + ts.tv_nsec += ns % 1e9 + if ts.tv_nsec >= 1e9 { + ts.tv_sec++ + ts.tv_nsec -= 1e9 + } + + if r, err := sem_timedwait((*semt)(unsafe.Pointer(_m_.waitsema)), &ts); r != 0 { + if err == _ETIMEDOUT || err == _EAGAIN || err == _EINTR { + return -1 + } + println("sem_timedwait err ", err, " ts.tv_sec ", ts.tv_sec, " ts.tv_nsec ", ts.tv_nsec, " ns ", ns, " id ", _m_.id) + throw("sem_timedwait") + } + return 0 + } + for { + r1, err := sem_wait((*semt)(unsafe.Pointer(_m_.waitsema))) + if r1 == 0 { + break + } + if err == _EINTR { + continue + } + throw("sem_wait") + } + return 0 +} + +//go:nosplit +func semawakeup(mp *m) { + if sem_post((*semt)(unsafe.Pointer(mp.waitsema))) != 0 { + throw("sem_post") + } +} + +func osinit() { + ncpu = int32(sysconf(__SC_NPROCESSORS_ONLN)) + physPageSize = sysconf(__SC_PAGE_SIZE) +} + +// Ms related functions +func mpreinit(mp *m) { + mp.gsignal = malg(32 * 1024) // AIX wants >= 8K + mp.gsignal.m = mp +} + +// errno address must be retrieved by calling _Errno libc function. +// This will return a pointer to errno +func miniterrno() { + mp := getg().m + r, _ := syscall0(&libc__Errno) + mp.perrno = r + +} + +func minit() { + miniterrno() + minitSignals() +} + +func unminit() { + unminitSignals() +} + +// tstart is a function descriptor to _tstart defined in assembly. +var tstart funcDescriptor + +func newosproc(mp *m) { + var ( + attr pthread_attr + oset sigset + tid pthread + ) + + if pthread_attr_init(&attr) != 0 { + throw("pthread_attr_init") + } + + if pthread_attr_setstacksize(&attr, threadStackSize) != 0 { + throw("pthread_attr_getstacksize") + } + + if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 { + throw("pthread_attr_setdetachstate") + } + + // Disable signals during create, so that the new thread starts + // with signals disabled. It will enable them in minit. + sigprocmask(_SIG_SETMASK, &sigset_all, &oset) + var ret int32 + for tries := 0; tries < 20; tries++ { + // pthread_create can fail with EAGAIN for no reasons + // but it will be ok if it retries. + ret = pthread_create(&tid, &attr, &tstart, unsafe.Pointer(mp)) + if ret != _EAGAIN { + break + } + usleep(uint32(tries+1) * 1000) // Milliseconds. + } + sigprocmask(_SIG_SETMASK, &oset, nil) + if ret != 0 { + print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n") + if ret == _EAGAIN { + println("runtime: may need to increase max user processes (ulimit -u)") + } + throw("newosproc") + } + +} + +func exitThread(wait *uint32) { + // We should never reach exitThread on AIX because we let + // libc clean up threads. + throw("exitThread") +} + +var urandom_dev = []byte("/dev/urandom\x00") + +//go:nosplit +func getRandomData(r []byte) { + fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0) + n := read(fd, unsafe.Pointer(&r[0]), int32(len(r))) + closefd(fd) + extendRandom(r, int(n)) +} + +func goenvs() { + goenvs_unix() +} + +/* SIGNAL */ + +const ( + _NSIG = 256 +) + +// sigtramp is a function descriptor to _sigtramp defined in assembly +var sigtramp funcDescriptor + +//go:nosplit +//go:nowritebarrierrec +func setsig(i uint32, fn uintptr) { + var sa sigactiont + sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTART + sa.sa_mask = sigset_all + if fn == funcPC(sighandler) { + fn = uintptr(unsafe.Pointer(&sigtramp)) + } + sa.sa_handler = fn + sigaction(uintptr(i), &sa, nil) + +} + +//go:nosplit +//go:nowritebarrierrec +func setsigstack(i uint32) { + throw("Not yet implemented\n") +} + +//go:nosplit +//go:nowritebarrierrec +func getsig(i uint32) uintptr { + var sa sigactiont + sigaction(uintptr(i), nil, &sa) + return sa.sa_handler +} + +// setSignaltstackSP sets the ss_sp field of a stackt. +//go:nosplit +func setSignalstackSP(s *stackt, sp uintptr) { + *(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp +} + +func (c *sigctxt) fixsigcode(sig uint32) { +} + +func sigaddset(mask *sigset, i int) { + (*mask)[(i-1)/64] |= 1 << ((uint32(i) - 1) & 63) +} + +func sigdelset(mask *sigset, i int) { + (*mask)[(i-1)/64] &^= 1 << ((uint32(i) - 1) & 63) +} + +const ( + _CLOCK_REALTIME = 9 + _CLOCK_MONOTONIC = 10 +) + +//go:nosplit +func nanotime() int64 { + tp := ×pec{} + if clock_gettime(_CLOCK_REALTIME, tp) != 0 { + throw("syscall clock_gettime failed") + } + return tp.tv_sec*1000000000 + tp.tv_nsec +} + +func walltime() (sec int64, nsec int32) { + ts := ×pec{} + if clock_gettime(_CLOCK_REALTIME, ts) != 0 { + throw("syscall clock_gettime failed") + } + return ts.tv_sec, int32(ts.tv_nsec) +} diff --git a/src/runtime/proc.go b/src/runtime/proc.go index db6f908e8c..acfdc8472e 100644 --- a/src/runtime/proc.go +++ b/src/runtime/proc.go @@ -1172,8 +1172,8 @@ func mstart() { mstart1() // Exit this thread. - if GOOS == "windows" || GOOS == "solaris" || GOOS == "plan9" || GOOS == "darwin" { - // Window, Solaris, Darwin and Plan 9 always system-allocate + if GOOS == "windows" || GOOS == "solaris" || GOOS == "plan9" || GOOS == "darwin" || GOOS == "aix" { + // Window, Solaris, Darwin, AIX and Plan 9 always system-allocate // the stack, but put it in _g_.stack before mstart, // so the logic above hasn't set osStack yet. osStack = true diff --git a/src/runtime/rt0_aix_ppc64.s b/src/runtime/rt0_aix_ppc64.s new file mode 100644 index 0000000000..0e3d582809 --- /dev/null +++ b/src/runtime/rt0_aix_ppc64.s @@ -0,0 +1,40 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +#include "textflag.h" + +// _rt0_ppc64_aix is a function descriptor of the entrypoint function +// __start. This name is needed by cmd/link. +DATA _rt0_ppc64_aix+0(SB)/8, $__start<>(SB) +DATA _rt0_ppc64_aix+8(SB)/8, $TOC(SB) +GLOBL _rt0_ppc64_aix(SB), NOPTR, $16 + + +// The starting function must return in the loader to +// initialise some librairies, especially libthread which +// creates the main thread and adds the TLS in R13 +// R19 contains a function descriptor to the loader function +// which needs to be called. +// This code is similar to the __start function in C +TEXT __start<>(SB),NOSPLIT,$-8 + XOR R0, R0 + MOVD $libc___n_pthreads(SB), R4 + MOVD 0(R4), R4 + MOVD $libc___mod_init(SB), R5 + MOVD 0(R5), R5 + MOVD 0(R19), R0 + MOVD R2, 40(R1) + MOVD 8(R19), R2 + MOVD R18, R3 + MOVD R0, CTR + BL (CTR) // Return to AIX loader + + // Launch rt0_go + MOVD 40(R1), R2 + MOVD R14, R3 // argc + MOVD R15, R4 // argv + MOVD $runtime·rt0_go(SB), R12 + MOVD R12, CTR + BR (CTR) + diff --git a/src/runtime/runtime_unix_test.go b/src/runtime/runtime_unix_test.go index e91216365e..b0cbbbe3e6 100644 --- a/src/runtime/runtime_unix_test.go +++ b/src/runtime/runtime_unix_test.go @@ -6,7 +6,7 @@ // We need a fast system call to provoke the race, // and Close(-1) is nearly universally fast. -// +build darwin dragonfly freebsd linux netbsd openbsd plan9 +// +build aix darwin dragonfly freebsd linux netbsd openbsd plan9 package runtime_test diff --git a/src/runtime/signal_aix_ppc64.go b/src/runtime/signal_aix_ppc64.go new file mode 100644 index 0000000000..c17563e2a5 --- /dev/null +++ b/src/runtime/signal_aix_ppc64.go @@ -0,0 +1,85 @@ +/// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix + +package runtime + +import ( + "runtime/internal/sys" + "unsafe" +) + +type sigctxt struct { + info *siginfo + ctxt unsafe.Pointer +} + +//go:nosplit +//go:nowritebarrierrec +func (c *sigctxt) regs() *context64 { return &(*ucontext)(c.ctxt).uc_mcontext } + +func (c *sigctxt) r0() uint64 { return c.regs().gpr[0] } +func (c *sigctxt) r1() uint64 { return c.regs().gpr[1] } +func (c *sigctxt) r2() uint64 { return c.regs().gpr[2] } +func (c *sigctxt) r3() uint64 { return c.regs().gpr[3] } +func (c *sigctxt) r4() uint64 { return c.regs().gpr[4] } +func (c *sigctxt) r5() uint64 { return c.regs().gpr[5] } +func (c *sigctxt) r6() uint64 { return c.regs().gpr[6] } +func (c *sigctxt) r7() uint64 { return c.regs().gpr[7] } +func (c *sigctxt) r8() uint64 { return c.regs().gpr[8] } +func (c *sigctxt) r9() uint64 { return c.regs().gpr[9] } +func (c *sigctxt) r10() uint64 { return c.regs().gpr[10] } +func (c *sigctxt) r11() uint64 { return c.regs().gpr[11] } +func (c *sigctxt) r12() uint64 { return c.regs().gpr[12] } +func (c *sigctxt) r13() uint64 { return c.regs().gpr[13] } +func (c *sigctxt) r14() uint64 { return c.regs().gpr[14] } +func (c *sigctxt) r15() uint64 { return c.regs().gpr[15] } +func (c *sigctxt) r16() uint64 { return c.regs().gpr[16] } +func (c *sigctxt) r17() uint64 { return c.regs().gpr[17] } +func (c *sigctxt) r18() uint64 { return c.regs().gpr[18] } +func (c *sigctxt) r19() uint64 { return c.regs().gpr[19] } +func (c *sigctxt) r20() uint64 { return c.regs().gpr[20] } +func (c *sigctxt) r21() uint64 { return c.regs().gpr[21] } +func (c *sigctxt) r22() uint64 { return c.regs().gpr[22] } +func (c *sigctxt) r23() uint64 { return c.regs().gpr[23] } +func (c *sigctxt) r24() uint64 { return c.regs().gpr[24] } +func (c *sigctxt) r25() uint64 { return c.regs().gpr[25] } +func (c *sigctxt) r26() uint64 { return c.regs().gpr[26] } +func (c *sigctxt) r27() uint64 { return c.regs().gpr[27] } +func (c *sigctxt) r28() uint64 { return c.regs().gpr[28] } +func (c *sigctxt) r29() uint64 { return c.regs().gpr[29] } +func (c *sigctxt) r30() uint64 { return c.regs().gpr[30] } +func (c *sigctxt) r31() uint64 { return c.regs().gpr[31] } +func (c *sigctxt) sp() uint64 { return c.regs().gpr[1] } + +//go:nosplit +//go:nowritebarrierrec +func (c *sigctxt) pc() uint64 { return c.regs().iar } + +func (c *sigctxt) ctr() uint64 { return c.regs().ctr } +func (c *sigctxt) link() uint64 { return c.regs().lr } +func (c *sigctxt) xer() uint32 { return c.regs().xer } +func (c *sigctxt) ccr() uint32 { return c.regs().cr } +func (c *sigctxt) fpscr() uint32 { return c.regs().fpscr } +func (c *sigctxt) fpscrx() uint32 { return c.regs().fpscrx } + +// TODO(aix): find trap equivalent +func (c *sigctxt) trap() uint32 { return 0x0 } + +func (c *sigctxt) sigcode() uint32 { return uint32(c.info.si_code) } +func (c *sigctxt) sigaddr() uint64 { return uint64(c.info.si_addr) } +func (c *sigctxt) fault() uintptr { return uintptr(c.sigaddr()) } + +func (c *sigctxt) set_r0(x uint64) { c.regs().gpr[0] = x } +func (c *sigctxt) set_r12(x uint64) { c.regs().gpr[12] = x } +func (c *sigctxt) set_r30(x uint64) { c.regs().gpr[30] = x } +func (c *sigctxt) set_pc(x uint64) { c.regs().iar = x } +func (c *sigctxt) set_sp(x uint64) { c.regs().gpr[1] = x } +func (c *sigctxt) set_link(x uint64) { c.regs().lr = x } + +func (c *sigctxt) set_sigcode(x uint32) { c.info.si_code = int32(x) } +func (c *sigctxt) set_sigaddr(x uint64) { + *(*uintptr)(add(unsafe.Pointer(c.info), 2*sys.PtrSize)) = uintptr(x) +} diff --git a/src/runtime/signal_ppc64x.go b/src/runtime/signal_ppc64x.go index 5a1a5cae60..cac1a23c9f 100644 --- a/src/runtime/signal_ppc64x.go +++ b/src/runtime/signal_ppc64x.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build linux +// +build aix linux // +build ppc64 ppc64le package runtime diff --git a/src/runtime/signal_sighandler.go b/src/runtime/signal_sighandler.go index 5a734f9050..6e71e41f52 100644 --- a/src/runtime/signal_sighandler.go +++ b/src/runtime/signal_sighandler.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux nacl netbsd openbsd solaris package runtime diff --git a/src/runtime/signal_unix.go b/src/runtime/signal_unix.go index 6cd9f8ddb6..12a938c8c9 100644 --- a/src/runtime/signal_unix.go +++ b/src/runtime/signal_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris package runtime diff --git a/src/runtime/sigtab_aix.go b/src/runtime/sigtab_aix.go new file mode 100644 index 0000000000..42e5606ab6 --- /dev/null +++ b/src/runtime/sigtab_aix.go @@ -0,0 +1,264 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package runtime + +var sigtable = [...]sigTabT{ + 0: {0, "SIGNONE: no trap"}, + _SIGHUP: {_SigNotify + _SigKill, "SIGHUP: terminal line hangup"}, + _SIGINT: {_SigNotify + _SigKill, "SIGINT: interrupt"}, + _SIGQUIT: {_SigNotify + _SigThrow, "SIGQUIT: quit"}, + _SIGILL: {_SigThrow + _SigUnblock, "SIGILL: illegal instruction"}, + _SIGTRAP: {_SigThrow + _SigUnblock, "SIGTRAP: trace trap"}, + _SIGABRT: {_SigNotify + _SigThrow, "SIGABRT: abort"}, + _SIGBUS: {_SigPanic + _SigUnblock, "SIGBUS: bus error"}, + _SIGFPE: {_SigPanic + _SigUnblock, "SIGFPE: floating-point exception"}, + _SIGKILL: {0, "SIGKILL: kill"}, + _SIGUSR1: {_SigNotify, "SIGUSR1: user-defined signal 1"}, + _SIGSEGV: {_SigPanic + _SigUnblock, "SIGSEGV: segmentation violation"}, + _SIGUSR2: {_SigNotify, "SIGUSR2: user-defined signal 2"}, + _SIGPIPE: {_SigNotify, "SIGPIPE: write to broken pipe"}, + _SIGALRM: {_SigNotify, "SIGALRM: alarm clock"}, + _SIGTERM: {_SigNotify + _SigKill, "SIGTERM: termination"}, + _SIGCHLD: {_SigNotify + _SigUnblock, "SIGCHLD: child status has changed"}, + _SIGCONT: {_SigNotify + _SigDefault, "SIGCONT: continue"}, + _SIGSTOP: {0, "SIGSTOP: stop"}, + _SIGTSTP: {_SigNotify + _SigDefault, "SIGTSTP: keyboard stop"}, + _SIGTTIN: {_SigNotify + _SigDefault, "SIGTTIN: background read from tty"}, + _SIGTTOU: {_SigNotify + _SigDefault, "SIGTTOU: background write to tty"}, + _SIGURG: {_SigNotify, "SIGURG: urgent condition on socket"}, + _SIGXCPU: {_SigNotify, "SIGXCPU: cpu limit exceeded"}, + _SIGXFSZ: {_SigNotify, "SIGXFSZ: file size limit exceeded"}, + _SIGVTALRM: {_SigNotify, "SIGVTALRM: virtual alarm clock"}, + _SIGPROF: {_SigNotify + _SigUnblock, "SIGPROF: profiling alarm clock"}, + _SIGWINCH: {_SigNotify, "SIGWINCH: window size change"}, + _SIGSYS: {_SigThrow, "SIGSYS: bad system call"}, + _SIGIO: {_SigNotify, "SIGIO: i/o now possible"}, + _SIGPWR: {_SigNotify, "SIGPWR: power failure restart"}, + _SIGEMT: {_SigThrow, "SIGEMT: emulate instruction executed"}, + _SIGWAITING: {0, "SIGWAITING: reserved signal no longer used by"}, + 26: {_SigNotify, "signal 26"}, + 27: {_SigNotify, "signal 27"}, + 33: {_SigNotify, "signal 33"}, + 35: {_SigNotify, "signal 35"}, + 36: {_SigNotify, "signal 36"}, + 37: {_SigNotify, "signal 37"}, + 38: {_SigNotify, "signal 38"}, + 40: {_SigNotify, "signal 40"}, + 41: {_SigNotify, "signal 41"}, + 42: {_SigNotify, "signal 42"}, + 43: {_SigNotify, "signal 43"}, + 44: {_SigNotify, "signal 44"}, + 45: {_SigNotify, "signal 45"}, + 46: {_SigNotify, "signal 46"}, + 47: {_SigNotify, "signal 47"}, + 48: {_SigNotify, "signal 48"}, + 49: {_SigNotify, "signal 49"}, + 50: {_SigNotify, "signal 50"}, + 51: {_SigNotify, "signal 51"}, + 52: {_SigNotify, "signal 52"}, + 53: {_SigNotify, "signal 53"}, + 54: {_SigNotify, "signal 54"}, + 55: {_SigNotify, "signal 55"}, + 56: {_SigNotify, "signal 56"}, + 57: {_SigNotify, "signal 57"}, + 58: {_SigNotify, "signal 58"}, + 59: {_SigNotify, "signal 59"}, + 60: {_SigNotify, "signal 60"}, + 61: {_SigNotify, "signal 61"}, + 62: {_SigNotify, "signal 62"}, + 63: {_SigNotify, "signal 63"}, + 64: {_SigNotify, "signal 64"}, + 65: {_SigNotify, "signal 65"}, + 66: {_SigNotify, "signal 66"}, + 67: {_SigNotify, "signal 67"}, + 68: {_SigNotify, "signal 68"}, + 69: {_SigNotify, "signal 69"}, + 70: {_SigNotify, "signal 70"}, + 71: {_SigNotify, "signal 71"}, + 72: {_SigNotify, "signal 72"}, + 73: {_SigNotify, "signal 73"}, + 74: {_SigNotify, "signal 74"}, + 75: {_SigNotify, "signal 75"}, + 76: {_SigNotify, "signal 76"}, + 77: {_SigNotify, "signal 77"}, + 78: {_SigNotify, "signal 78"}, + 79: {_SigNotify, "signal 79"}, + 80: {_SigNotify, "signal 80"}, + 81: {_SigNotify, "signal 81"}, + 82: {_SigNotify, "signal 82"}, + 83: {_SigNotify, "signal 83"}, + 84: {_SigNotify, "signal 84"}, + 85: {_SigNotify, "signal 85"}, + 86: {_SigNotify, "signal 86"}, + 87: {_SigNotify, "signal 87"}, + 88: {_SigNotify, "signal 88"}, + 89: {_SigNotify, "signal 89"}, + 90: {_SigNotify, "signal 90"}, + 91: {_SigNotify, "signal 91"}, + 92: {_SigNotify, "signal 92"}, + 93: {_SigNotify, "signal 93"}, + 94: {_SigNotify, "signal 94"}, + 95: {_SigNotify, "signal 95"}, + 96: {_SigNotify, "signal 96"}, + 97: {_SigNotify, "signal 97"}, + 98: {_SigNotify, "signal 98"}, + 99: {_SigNotify, "signal 99"}, + 100: {_SigNotify, "signal 100"}, + 101: {_SigNotify, "signal 101"}, + 102: {_SigNotify, "signal 102"}, + 103: {_SigNotify, "signal 103"}, + 104: {_SigNotify, "signal 104"}, + 105: {_SigNotify, "signal 105"}, + 106: {_SigNotify, "signal 106"}, + 107: {_SigNotify, "signal 107"}, + 108: {_SigNotify, "signal 108"}, + 109: {_SigNotify, "signal 109"}, + 110: {_SigNotify, "signal 110"}, + 111: {_SigNotify, "signal 111"}, + 112: {_SigNotify, "signal 112"}, + 113: {_SigNotify, "signal 113"}, + 114: {_SigNotify, "signal 114"}, + 115: {_SigNotify, "signal 115"}, + 116: {_SigNotify, "signal 116"}, + 117: {_SigNotify, "signal 117"}, + 118: {_SigNotify, "signal 118"}, + 119: {_SigNotify, "signal 119"}, + 120: {_SigNotify, "signal 120"}, + 121: {_SigNotify, "signal 121"}, + 122: {_SigNotify, "signal 122"}, + 123: {_SigNotify, "signal 123"}, + 124: {_SigNotify, "signal 124"}, + 125: {_SigNotify, "signal 125"}, + 126: {_SigNotify, "signal 126"}, + 127: {_SigNotify, "signal 127"}, + 128: {_SigNotify, "signal 128"}, + 129: {_SigNotify, "signal 129"}, + 130: {_SigNotify, "signal 130"}, + 131: {_SigNotify, "signal 131"}, + 132: {_SigNotify, "signal 132"}, + 133: {_SigNotify, "signal 133"}, + 134: {_SigNotify, "signal 134"}, + 135: {_SigNotify, "signal 135"}, + 136: {_SigNotify, "signal 136"}, + 137: {_SigNotify, "signal 137"}, + 138: {_SigNotify, "signal 138"}, + 139: {_SigNotify, "signal 139"}, + 140: {_SigNotify, "signal 140"}, + 141: {_SigNotify, "signal 141"}, + 142: {_SigNotify, "signal 142"}, + 143: {_SigNotify, "signal 143"}, + 144: {_SigNotify, "signal 144"}, + 145: {_SigNotify, "signal 145"}, + 146: {_SigNotify, "signal 146"}, + 147: {_SigNotify, "signal 147"}, + 148: {_SigNotify, "signal 148"}, + 149: {_SigNotify, "signal 149"}, + 150: {_SigNotify, "signal 150"}, + 151: {_SigNotify, "signal 151"}, + 152: {_SigNotify, "signal 152"}, + 153: {_SigNotify, "signal 153"}, + 154: {_SigNotify, "signal 154"}, + 155: {_SigNotify, "signal 155"}, + 156: {_SigNotify, "signal 156"}, + 157: {_SigNotify, "signal 157"}, + 158: {_SigNotify, "signal 158"}, + 159: {_SigNotify, "signal 159"}, + 160: {_SigNotify, "signal 160"}, + 161: {_SigNotify, "signal 161"}, + 162: {_SigNotify, "signal 162"}, + 163: {_SigNotify, "signal 163"}, + 164: {_SigNotify, "signal 164"}, + 165: {_SigNotify, "signal 165"}, + 166: {_SigNotify, "signal 166"}, + 167: {_SigNotify, "signal 167"}, + 168: {_SigNotify, "signal 168"}, + 169: {_SigNotify, "signal 169"}, + 170: {_SigNotify, "signal 170"}, + 171: {_SigNotify, "signal 171"}, + 172: {_SigNotify, "signal 172"}, + 173: {_SigNotify, "signal 173"}, + 174: {_SigNotify, "signal 174"}, + 175: {_SigNotify, "signal 175"}, + 176: {_SigNotify, "signal 176"}, + 177: {_SigNotify, "signal 177"}, + 178: {_SigNotify, "signal 178"}, + 179: {_SigNotify, "signal 179"}, + 180: {_SigNotify, "signal 180"}, + 181: {_SigNotify, "signal 181"}, + 182: {_SigNotify, "signal 182"}, + 183: {_SigNotify, "signal 183"}, + 184: {_SigNotify, "signal 184"}, + 185: {_SigNotify, "signal 185"}, + 186: {_SigNotify, "signal 186"}, + 187: {_SigNotify, "signal 187"}, + 188: {_SigNotify, "signal 188"}, + 189: {_SigNotify, "signal 189"}, + 190: {_SigNotify, "signal 190"}, + 191: {_SigNotify, "signal 191"}, + 192: {_SigNotify, "signal 192"}, + 193: {_SigNotify, "signal 193"}, + 194: {_SigNotify, "signal 194"}, + 195: {_SigNotify, "signal 195"}, + 196: {_SigNotify, "signal 196"}, + 197: {_SigNotify, "signal 197"}, + 198: {_SigNotify, "signal 198"}, + 199: {_SigNotify, "signal 199"}, + 200: {_SigNotify, "signal 200"}, + 201: {_SigNotify, "signal 201"}, + 202: {_SigNotify, "signal 202"}, + 203: {_SigNotify, "signal 203"}, + 204: {_SigNotify, "signal 204"}, + 205: {_SigNotify, "signal 205"}, + 206: {_SigNotify, "signal 206"}, + 207: {_SigNotify, "signal 207"}, + 208: {_SigNotify, "signal 208"}, + 209: {_SigNotify, "signal 209"}, + 210: {_SigNotify, "signal 210"}, + 211: {_SigNotify, "signal 211"}, + 212: {_SigNotify, "signal 212"}, + 213: {_SigNotify, "signal 213"}, + 214: {_SigNotify, "signal 214"}, + 215: {_SigNotify, "signal 215"}, + 216: {_SigNotify, "signal 216"}, + 217: {_SigNotify, "signal 217"}, + 218: {_SigNotify, "signal 218"}, + 219: {_SigNotify, "signal 219"}, + 220: {_SigNotify, "signal 220"}, + 221: {_SigNotify, "signal 221"}, + 222: {_SigNotify, "signal 222"}, + 223: {_SigNotify, "signal 223"}, + 224: {_SigNotify, "signal 224"}, + 225: {_SigNotify, "signal 225"}, + 226: {_SigNotify, "signal 226"}, + 227: {_SigNotify, "signal 227"}, + 228: {_SigNotify, "signal 228"}, + 229: {_SigNotify, "signal 229"}, + 230: {_SigNotify, "signal 230"}, + 231: {_SigNotify, "signal 231"}, + 232: {_SigNotify, "signal 232"}, + 233: {_SigNotify, "signal 233"}, + 234: {_SigNotify, "signal 234"}, + 235: {_SigNotify, "signal 235"}, + 236: {_SigNotify, "signal 236"}, + 237: {_SigNotify, "signal 237"}, + 238: {_SigNotify, "signal 238"}, + 239: {_SigNotify, "signal 239"}, + 240: {_SigNotify, "signal 240"}, + 241: {_SigNotify, "signal 241"}, + 242: {_SigNotify, "signal 242"}, + 243: {_SigNotify, "signal 243"}, + 244: {_SigNotify, "signal 244"}, + 245: {_SigNotify, "signal 245"}, + 246: {_SigNotify, "signal 246"}, + 247: {_SigNotify, "signal 247"}, + 248: {_SigNotify, "signal 248"}, + 249: {_SigNotify, "signal 249"}, + 250: {_SigNotify, "signal 250"}, + 251: {_SigNotify, "signal 251"}, + 252: {_SigNotify, "signal 252"}, + 253: {_SigNotify, "signal 253"}, + 254: {_SigNotify, "signal 254"}, + 255: {_SigNotify, "signal 255"}, +} diff --git a/src/runtime/stubs2.go b/src/runtime/stubs2.go index c14db74003..57134f7354 100644 --- a/src/runtime/stubs2.go +++ b/src/runtime/stubs2.go @@ -8,6 +8,7 @@ // +build !nacl // +build !js // +build !darwin +// +build !aix package runtime diff --git a/src/runtime/stubs3.go b/src/runtime/stubs3.go index 5c0786e411..a9ff689e79 100644 --- a/src/runtime/stubs3.go +++ b/src/runtime/stubs3.go @@ -8,6 +8,7 @@ // +build !nacl // +build !freebsd // +build !darwin +// +build !aix package runtime diff --git a/src/runtime/sys_aix_ppc64.s b/src/runtime/sys_aix_ppc64.s new file mode 100644 index 0000000000..38e60f99eb --- /dev/null +++ b/src/runtime/sys_aix_ppc64.s @@ -0,0 +1,201 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix +// +build ppc64 ppc64le + +// +// System calls and other sys.stuff for ppc64, Aix +// + +#include "go_asm.h" +#include "go_tls.h" +#include "textflag.h" +#include "asm_ppc64x.h" + +// This function calls a C function with the function descriptor in R12 +TEXT runtime·callCfunction(SB), NOSPLIT|NOFRAME,$0 + MOVD 0(R12), R12 + MOVD R2, 40(R1) + MOVD 0(R12), R0 + MOVD 8(R12), R2 + MOVD R0, CTR + BR (CTR) + + +// asmsyscall6 calls a library function with a function descriptor +// stored in libcall_fn and store the results in libcall struture +// Up to 6 arguments can be passed to this C function +// Called by runtime.asmcgocall +// It reserves a stack of 288 bytes for the C function. +// NOT USING GO CALLING CONVENTION +TEXT runtime·asmsyscall6(SB),NOSPLIT,$256 + MOVD R3, 48(R1) // Save libcall for later + MOVD libcall_fn(R3), R12 + MOVD libcall_args(R3), R9 + MOVD 0(R9), R3 + MOVD 8(R9), R4 + MOVD 16(R9), R5 + MOVD 24(R9), R6 + MOVD 32(R9), R7 + MOVD 40(R9), R8 + BL runtime·callCfunction(SB) + + // Restore R0 and TOC + XOR R0, R0 + MOVD 40(R1), R2 + + // Store result in libcall + MOVD 48(R1), R5 + MOVD R3, (libcall_r1)(R5) + MOVD $-1, R6 + CMP R6, R3 + BNE skiperrno + + // Save errno in libcall + BL runtime·load_g(SB) + MOVD g_m(g), R4 + MOVD (m_mOS + mOS_perrno)(R4), R9 + MOVW 0(R9), R9 + MOVD R9, (libcall_err)(R5) + RET +skiperrno: + // Reset errno if no error has been returned + MOVD R0, (libcall_err)(R5) + RET + + +TEXT runtime·sigfwd(SB),NOSPLIT,$0-32 + MOVW sig+8(FP), R3 + MOVD info+16(FP), R4 + MOVD ctx+24(FP), R5 + MOVD fn+0(FP), R12 + MOVD R12, CTR + BL (CTR) + RET + + +// runtime.sigtramp is a function descriptor to the real sigtramp. +DATA runtime·sigtramp+0(SB)/8, $runtime·_sigtramp(SB) +DATA runtime·sigtramp+8(SB)/8, $TOC(SB) +DATA runtime·sigtramp+16(SB)/8, $0 +GLOBL runtime·sigtramp(SB), NOPTR, $24 + +// This funcion must not have any frame as we want to control how +// every registers are used. +TEXT runtime·_sigtramp(SB),NOSPLIT|NOFRAME,$0 + MOVD LR, R0 + MOVD R0, 16(R1) + // initialize essential registers (just in case) + BL runtime·reginit(SB) + + // Note that we are executing on altsigstack here, so we have + // more stack available than NOSPLIT would have us believe. + // To defeat the linker, we make our own stack frame with + // more space. + SUB $128+FIXED_FRAME, R1 + + // Save registers + MOVD R31, 56(R1) + MOVD g, 64(R1) + MOVD R29, 72(R1) + + BL runtime·load_g(SB) + + // Save m->libcall. We need to do this because we + // might get interrupted by a signal in runtime·asmcgocall. + + // save m->libcall + MOVD g_m(g), R6 + MOVD (m_libcall+libcall_fn)(R6), R7 + MOVD R7, 80(R1) + MOVD (m_libcall+libcall_args)(R6), R7 + MOVD R7, 88(R1) + MOVD (m_libcall+libcall_n)(R6), R7 + MOVD R7, 96(R1) + MOVD (m_libcall+libcall_r1)(R6), R7 + MOVD R7, 104(R1) + MOVD (m_libcall+libcall_r2)(R6), R7 + MOVD R7, 112(R1) + + // save errno, it might be EINTR; stuff we do here might reset it. + MOVD (m_mOS+mOS_perrno)(R6), R8 + MOVD 0(R8), R8 + MOVD R8, 120(R1) + + MOVW R3, FIXED_FRAME+0(R1) + MOVD R4, FIXED_FRAME+8(R1) + MOVD R5, FIXED_FRAME+16(R1) + MOVD $runtime·sigtrampgo(SB), R12 + MOVD R12, CTR + BL (CTR) + + MOVD g_m(g), R6 + // restore libcall + MOVD 80(R1), R7 + MOVD R7, (m_libcall+libcall_fn)(R6) + MOVD 88(R1), R7 + MOVD R7, (m_libcall+libcall_args)(R6) + MOVD 96(R1), R7 + MOVD R7, (m_libcall+libcall_n)(R6) + MOVD 104(R1), R7 + MOVD R7, (m_libcall+libcall_r1)(R6) + MOVD 112(R1), R7 + MOVD R7, (m_libcall+libcall_r2)(R6) + + // restore errno + MOVD (m_mOS+mOS_perrno)(R6), R7 + MOVD 120(R1), R8 + MOVD R8, 0(R7) + + // restore registers + MOVD 56(R1),R31 + MOVD 64(R1),g + MOVD 72(R1),R29 + + // Don't use RET because we need to restore R31 ! + ADD $128+FIXED_FRAME, R1 + MOVD 16(R1), R0 + MOVD R0, LR + BR (LR) + +// runtime.tstart is a function descriptor to the real tstart. +DATA runtime·tstart+0(SB)/8, $runtime·_tstart(SB) +DATA runtime·tstart+8(SB)/8, $TOC(SB) +DATA runtime·tstart+16(SB)/8, $0 +GLOBL runtime·tstart(SB), NOPTR, $24 + +TEXT runtime·_tstart(SB),NOSPLIT,$0 + XOR R0, R0 // reset R0 + + // set g + MOVD m_g0(R3), g + BL runtime·save_g(SB) + MOVD R3, g_m(g) + + // Layout new m scheduler stack on os stack. + MOVD R1, R3 + MOVD R3, (g_stack+stack_hi)(g) + SUB $(const_threadStackSize), R3 // stack size + MOVD R3, (g_stack+stack_lo)(g) + ADD $const__StackGuard, R3 + MOVD R3, g_stackguard0(g) + MOVD R3, g_stackguard1(g) + + BL runtime·mstart(SB) + + MOVD R0, R3 + RET + +// Runs on OS stack, called from runtime·osyield. +TEXT runtime·osyield1(SB),NOSPLIT,$0 + MOVD $libc_sched_yield(SB), R12 + MOVD 0(R12), R12 + MOVD R2, 40(R1) + MOVD 0(R12), R0 + MOVD 8(R12), R2 + MOVD R0, CTR + BL (CTR) + MOVD 40(R1), R2 + RET diff --git a/src/runtime/timestub2.go b/src/runtime/timestub2.go index 9ddc6fed91..00c2c55f46 100644 --- a/src/runtime/timestub2.go +++ b/src/runtime/timestub2.go @@ -5,6 +5,7 @@ // +build !darwin // +build !windows // +build !freebsd +// +build !aix package runtime -- cgit v1.3-5-g9baa