internal/bytealg: optimise memequal on riscv64

Implement memequal using loops that process 32 bytes, 16 bytes, 4 bytes
or 1 byte depending on size and alignment. For comparisons that are less
than 32 bytes the overhead of checking and adjusting alignment usually
exceeds the overhead of reading and processing 4 bytes at a time.

Updates #50615

name                 old time/op    new time/op     delta
Equal/0-4              38.3ns _ 0%     43.1ns _ 0%    +12.54%  (p=0.000 n=3+3)
Equal/1-4              77.7ns _ 0%     90.3ns _ 0%    +16.27%  (p=0.000 n=3+3)
Equal/6-4               116ns _ 0%      121ns _ 0%     +3.85%  (p=0.002 n=3+3)
Equal/9-4               137ns _ 0%      126ns _ 0%     -7.98%  (p=0.000 n=3+3)
Equal/15-4              179ns _ 0%      170ns _ 0%     -4.77%  (p=0.001 n=3+3)
Equal/16-4              186ns _ 0%      159ns _ 0%    -14.65%  (p=0.000 n=3+3)
Equal/20-4              215ns _ 0%      178ns _ 0%    -17.18%  (p=0.000 n=3+3)
Equal/32-4              298ns _ 0%      101ns _ 0%    -66.22%  (p=0.000 n=3+3)
Equal/4K-4             28.9_s _ 0%      2.2_s _ 0%    -92.56%  (p=0.000 n=3+3)
Equal/4M-4             29.6ms _ 0%      2.2ms _ 0%    -92.72%  (p=0.000 n=3+3)
Equal/64M-4             758ms _75%       35ms _ 0%       ~     (p=0.127 n=3+3)
CompareBytesEqual-4     226ns _ 0%      131ns _ 0%    -41.76%  (p=0.000 n=3+3)

name                 old speed      new speed       delta
Equal/1-4            12.9MB/s _ 0%   11.1MB/s _ 0%    -13.98%  (p=0.000 n=3+3)
Equal/6-4            51.7MB/s _ 0%   49.8MB/s _ 0%     -3.72%  (p=0.002 n=3+3)
Equal/9-4            65.7MB/s _ 0%   71.4MB/s _ 0%     +8.67%  (p=0.000 n=3+3)
Equal/15-4           83.8MB/s _ 0%   88.0MB/s _ 0%     +5.02%  (p=0.001 n=3+3)
Equal/16-4           85.9MB/s _ 0%  100.6MB/s _ 0%    +17.19%  (p=0.000 n=3+3)
Equal/20-4           93.2MB/s _ 0%  112.6MB/s _ 0%    +20.74%  (p=0.000 n=3+3)
Equal/32-4            107MB/s _ 0%    317MB/s _ 0%   +195.97%  (p=0.000 n=3+3)
Equal/4K-4            142MB/s _ 0%   1902MB/s _ 0%  +1243.76%  (p=0.000 n=3+3)
Equal/4M-4            142MB/s _ 0%   1946MB/s _ 0%  +1274.22%  (p=0.000 n=3+3)
Equal/64M-4           111MB/s _55%   1941MB/s _ 0%  +1641.21%  (p=0.000 n=3+3)

Change-Id: I9af7e82de3c4c5af8813772ed139230900c03b92
Reviewed-on: https://go-review.googlesource.com/c/go/+/380075
Trust: Joel Sing <joel@sing.id.au>
Trust: mzh <mzh@golangcn.org>
Reviewed-by: mzh <mzh@golangcn.org>
Run-TryBot: Joel Sing <joel@sing.id.au>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Cherry Mui <cherryyz@google.com>

1 files changed, 110 insertions, 32 deletions

diff --git a/src/internal/bytealg/equal_riscv64.s b/src/internal/bytealg/equal_riscv64.s
index 22cb4fa97d..959a996f81 100644
--- a/src/internal/bytealg/equal_riscv64.s
+++ b/src/internal/bytealg/equal_riscv64.s
@@ -9,41 +9,119 @@
 
 // func memequal(a, b unsafe.Pointer, size uintptr) bool
 TEXT runtime·memequal(SB),NOSPLIT|NOFRAME,$0-25
-	MOV	a+0(FP), A1
-	MOV	b+8(FP), A2
-	BEQ	A1, A2, eq
-	MOV	size+16(FP), A3
-	ADD	A1, A3, A4
-loop:
-	BEQ	A1, A4, eq
+	MOV	a+0(FP), X5
+	MOV	b+8(FP), X6
+	MOV	size+16(FP), X7
+	MOV	$ret+24(FP), X19
+	JMP	memequal<>(SB)
 
-	MOVBU	(A1), A6
-	ADD	$1, A1
-	MOVBU	(A2), A7
-	ADD	$1, A2
-	BEQ	A6, A7, loop
+// func memequal_varlen(a, b unsafe.Pointer) bool
+TEXT runtime·memequal_varlen(SB),NOSPLIT|NOFRAME,$0-17
+	MOV	a+0(FP), X5
+	MOV	b+8(FP), X6
+	MOV	8(CTXT), X7    // compiler stores size at offset 8 in the closure
+	MOV	$ret+16(FP), X19
+	JMP	memequal<>(SB)
 
-	MOVB	ZERO, ret+24(FP)
-	RET
-eq:
-	MOV	$1, A1
-	MOVB	A1, ret+24(FP)
-	RET
+// On entry X5 and X6 contain pointers, X7 contains length.
+// X19 contains address for return value.
+TEXT memequal<>(SB),NOSPLIT|NOFRAME,$0
+	BEQ	X5, X6, eq
 
-// func memequal_varlen(a, b unsafe.Pointer) bool
-TEXT runtime·memequal_varlen(SB),NOSPLIT,$40-17
-	MOV	a+0(FP), A1
-	MOV	b+8(FP), A2
-	BEQ	A1, A2, eq
-	MOV	8(CTXT), A3    // compiler stores size at offset 8 in the closure
-	MOV	A1, 8(X2)
-	MOV	A2, 16(X2)
-	MOV	A3, 24(X2)
-	CALL	runtime·memequal(SB)
-	MOVBU	32(X2), A1
-	MOVB	A1, ret+16(FP)
+	MOV	$32, X8
+	BLT	X7, X8, loop4_check
+
+	// Check alignment - if alignment differs we have to do one byte at a time.
+	AND	$3, X5, X9
+	AND	$3, X6, X10
+	BNE	X9, X10, loop4_check
+	BEQZ	X9, loop32_check
+
+	// Check one byte at a time until we reach 8 byte alignment.
+	SUB	X9, X7, X7
+align:
+	ADD	$-1, X9
+	MOVBU	0(X5), X10
+	MOVBU	0(X6), X11
+	BNE	X10, X11, not_eq
+	ADD	$1, X5
+	ADD	$1, X6
+	BNEZ	X9, align
+
+loop32_check:
+	MOV	$32, X9
+	BLT	X7, X9, loop16_check
+loop32:
+	MOV	0(X5), X10
+	MOV	0(X6), X11
+	MOV	8(X5), X12
+	MOV	8(X6), X13
+	BNE	X10, X11, not_eq
+	BNE	X12, X13, not_eq
+	MOV	16(X5), X14
+	MOV	16(X6), X15
+	MOV	24(X5), X16
+	MOV	24(X6), X17
+	BNE	X14, X15, not_eq
+	BNE	X16, X17, not_eq
+	ADD	$32, X5
+	ADD	$32, X6
+	ADD	$-32, X7
+	BGE	X7, X9, loop32
+	BEQZ	X7, eq
+
+loop16_check:
+	MOV	$16, X8
+	BLT	X7, X8, loop4_check
+loop16:
+	MOV	0(X5), X10
+	MOV	0(X6), X11
+	MOV	8(X5), X12
+	MOV	8(X6), X13
+	BNE	X10, X11, not_eq
+	BNE	X12, X13, not_eq
+	ADD	$16, X5
+	ADD	$16, X6
+	ADD	$-16, X7
+	BGE	X7, X8, loop16
+	BEQZ	X7, eq
+
+loop4_check:
+	MOV	$4, X8
+	BLT	X7, X8, loop1
+loop4:
+	MOVBU	0(X5), X10
+	MOVBU	0(X6), X11
+	MOVBU	1(X5), X12
+	MOVBU	1(X6), X13
+	BNE	X10, X11, not_eq
+	BNE	X12, X13, not_eq
+	MOVBU	2(X5), X14
+	MOVBU	2(X6), X15
+	MOVBU	3(X5), X16
+	MOVBU	3(X6), X17
+	BNE	X14, X15, not_eq
+	BNE	X16, X17, not_eq
+	ADD	$4, X5
+	ADD	$4, X6
+	ADD	$-4, X7
+	BGE	X7, X8, loop4
+
+loop1:
+	BEQZ	X7, eq
+	MOVBU	0(X5), X10
+	MOVBU	0(X6), X11
+	BNE	X10, X11, not_eq
+	ADD	$1, X5
+	ADD	$1, X6
+	ADD	$-1, X7
+	JMP	loop1
+
+not_eq:
+	MOV	$0, X5
+	MOVB	X5, (X19)
 	RET
 eq:
-	MOV	$1, A1
-	MOVB	A1, ret+16(FP)
+	MOV	$1, X5
+	MOVB	X5, (X19)
 	RET