From 21d82e6ac80fc2aea1eac9c8eec9afdd79cb5bdd Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Tue, 1 Nov 2022 16:46:43 -0700 Subject: cmd/compile: batch write barrier calls Have the write barrier call return a pointer to a buffer into which the generated code records pointers that need write barrier treatment. Change-Id: I7871764298e0aa1513de417010c8d46b296b199e Reviewed-on: https://go-review.googlesource.com/c/go/+/447781 Reviewed-by: Keith Randall Run-TryBot: Keith Randall TryBot-Bypass: Keith Randall Reviewed-by: Cherry Mui --- src/runtime/asm_amd64.s | 120 +++++++++++++++++++----------------------------- 1 file changed, 46 insertions(+), 74 deletions(-) (limited to 'src/runtime/asm_amd64.s') diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s index 69a363320d..8051b269d6 100644 --- a/src/runtime/asm_amd64.s +++ b/src/runtime/asm_amd64.s @@ -1621,15 +1621,25 @@ TEXT ·sigpanic0(SB),NOSPLIT,$0-0 #endif JMP ·sigpanic(SB) -// gcWriteBarrier performs a heap pointer write and informs the GC. +// gcWriteBarrier informs the GC about heap pointer writes. // -// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments: -// - DI is the destination of the write -// - AX is the value being written at DI +// gcWriteBarrier returns space in a write barrier buffer which +// should be filled in by the caller. +// gcWriteBarrier does NOT follow the Go ABI. It accepts the +// number of bytes of buffer needed in R11, and returns a pointer +// to the buffer space in R11. // It clobbers FLAGS. It does not clobber any general-purpose registers, // but may clobber others (e.g., SSE registers). -// Defined as ABIInternal since it does not use the stack-based Go ABI. -TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112 +// Typical use would be, when doing *(CX+88) = AX +// CMPL $0, runtime.writeBarrier(SB) +// JEQ dowrite +// CALL runtime.gcBatchBarrier2(SB) +// MOVQ AX, (R11) +// MOVQ 88(CX), DX +// MOVQ DX, 8(R11) +// dowrite: +// MOVQ AX, 88(CX) +TEXT gcWriteBarrier<>(SB),NOSPLIT,$112 // Save the registers clobbered by the fast path. This is slightly // faster than having the caller spill these. MOVQ R12, 96(SP) @@ -1640,24 +1650,17 @@ retry: MOVQ g_m(R14), R13 MOVQ m_p(R13), R13 // Get current buffer write position. - MOVQ (p_wbBuf+wbBuf_next)(R13), R12 - // Increment wbBuf.next position. - LEAQ 16(R12), R12 + MOVQ (p_wbBuf+wbBuf_next)(R13), R12 // original next position + ADDQ R11, R12 // new next position // Is the buffer full? CMPQ R12, (p_wbBuf+wbBuf_end)(R13) JA flush // Commit to the larger buffer. MOVQ R12, (p_wbBuf+wbBuf_next)(R13) - // Record the write. - MOVQ AX, -16(R12) // Record value - // Note: This turns bad pointer writes into bad - // pointer reads, which could be confusing. We could avoid - // reading from obviously bad pointers, which would - // take care of the vast majority of these. We could - // patch this up in the signal handler, or use XCHG to - // combine the read and the write. - MOVQ (DI), R13 - MOVQ R13, -8(R12) // Record *slot + // Make return value (the original next position) + SUBQ R11, R12 + MOVQ R12, R11 + // Restore registers. MOVQ 96(SP), R12 MOVQ 104(SP), R13 RET @@ -1708,61 +1711,30 @@ flush: MOVQ 88(SP), R15 JMP retry -// gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierCX(SB),NOSPLIT|NOFRAME,$0 - XCHGQ CX, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ CX, AX - RET - -// gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierDX(SB),NOSPLIT|NOFRAME,$0 - XCHGQ DX, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ DX, AX - RET - -// gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierBX(SB),NOSPLIT|NOFRAME,$0 - XCHGQ BX, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ BX, AX - RET - -// gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierBP(SB),NOSPLIT|NOFRAME,$0 - XCHGQ BP, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ BP, AX - RET - -// gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierSI(SB),NOSPLIT|NOFRAME,$0 - XCHGQ SI, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ SI, AX - RET - -// gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierR8(SB),NOSPLIT|NOFRAME,$0 - XCHGQ R8, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ R8, AX - RET - -// gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9. -// Defined as ABIInternal since it does not use the stable Go ABI. -TEXT runtime·gcWriteBarrierR9(SB),NOSPLIT|NOFRAME,$0 - XCHGQ R9, AX - CALL runtime·gcWriteBarrier(SB) - XCHGQ R9, AX - RET +TEXT runtime·gcWriteBarrier1(SB),NOSPLIT|NOFRAME,$0 + MOVL $8, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier2(SB),NOSPLIT|NOFRAME,$0 + MOVL $16, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier3(SB),NOSPLIT|NOFRAME,$0 + MOVL $24, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier4(SB),NOSPLIT|NOFRAME,$0 + MOVL $32, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier5(SB),NOSPLIT|NOFRAME,$0 + MOVL $40, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier6(SB),NOSPLIT|NOFRAME,$0 + MOVL $48, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier7(SB),NOSPLIT|NOFRAME,$0 + MOVL $56, R11 + JMP gcWriteBarrier<>(SB) +TEXT runtime·gcWriteBarrier8(SB),NOSPLIT|NOFRAME,$0 + MOVL $64, R11 + JMP gcWriteBarrier<>(SB) DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large" GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below -- cgit v1.3-6-g1900