runtime, internal/runtime/maps: speed-up empty/zero map lookups

This lets the inliner do a better job optimizing the mapKeyError call.

goos: linux
goarch: amd64
pkg: runtime
cpu: AMD Ryzen 5 4600G with Radeon Graphics
                                 │ /tmp/before2 │             /tmp/after3             │
                                 │    sec/op    │   sec/op     vs base                │
MapAccessZero/Key=int64-12          1.875n ± 0%   1.875n ± 0%        ~ (p=0.506 n=25)
MapAccessZero/Key=int32-12          1.875n ± 0%   1.875n ± 0%        ~ (p=0.082 n=25)
MapAccessZero/Key=string-12         1.902n ± 1%   1.902n ± 1%        ~ (p=0.256 n=25)
MapAccessZero/Key=mediumType-12     2.816n ± 0%   1.958n ± 0%  -30.47% (p=0.000 n=25)
MapAccessZero/Key=bigType-12        2.815n ± 0%   1.935n ± 0%  -31.26% (p=0.000 n=25)
MapAccessEmpty/Key=int64-12         1.942n ± 0%   2.109n ± 0%   +8.60% (p=0.000 n=25)
MapAccessEmpty/Key=int32-12         2.110n ± 0%   1.940n ± 0%   -8.06% (p=0.000 n=25)
MapAccessEmpty/Key=string-12        2.024n ± 0%   2.109n ± 0%   +4.20% (p=0.000 n=25)
MapAccessEmpty/Key=mediumType-12    3.157n ± 0%   2.344n ± 0%  -25.75% (p=0.000 n=25)
MapAccessEmpty/Key=bigType-12       3.054n ± 0%   2.115n ± 0%  -30.75% (p=0.000 n=25)
geomean                             2.305n        2.011n       -12.75%

Change-Id: Iee83930884dc4c8a791a711aa189a1c93b68d536
Reviewed-on: https://go-review.googlesource.com/c/go/+/663495
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Pratt <mpratt@google.com>

5 files changed, 45 insertions, 76 deletions

diff --git a/src/runtime/alg.go b/src/runtime/alg.go
index 4626899aaf..df32bc7941 100644
--- a/src/runtime/alg.go
+++ b/src/runtime/alg.go
@@ -250,74 +250,6 @@ func typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr {
 	}
 }
 
-func mapKeyError(t *maptype, p unsafe.Pointer) error {
-	if !t.HashMightPanic() {
-		return nil
-	}
-	return mapKeyError2(t.Key, p)
-}
-
-func mapKeyError2(t *_type, p unsafe.Pointer) error {
-	if t.TFlag&abi.TFlagRegularMemory != 0 {
-		return nil
-	}
-	switch t.Kind_ & abi.KindMask {
-	case abi.Float32, abi.Float64, abi.Complex64, abi.Complex128, abi.String:
-		return nil
-	case abi.Interface:
-		i := (*interfacetype)(unsafe.Pointer(t))
-		var t *_type
-		var pdata *unsafe.Pointer
-		if len(i.Methods) == 0 {
-			a := (*eface)(p)
-			t = a._type
-			if t == nil {
-				return nil
-			}
-			pdata = &a.data
-		} else {
-			a := (*iface)(p)
-			if a.tab == nil {
-				return nil
-			}
-			t = a.tab.Type
-			pdata = &a.data
-		}
-
-		if t.Equal == nil {
-			return errorString("hash of unhashable type " + toRType(t).string())
-		}
-
-		if isDirectIface(t) {
-			return mapKeyError2(t, unsafe.Pointer(pdata))
-		} else {
-			return mapKeyError2(t, *pdata)
-		}
-	case abi.Array:
-		a := (*arraytype)(unsafe.Pointer(t))
-		for i := uintptr(0); i < a.Len; i++ {
-			if err := mapKeyError2(a.Elem, add(p, i*a.Elem.Size_)); err != nil {
-				return err
-			}
-		}
-		return nil
-	case abi.Struct:
-		s := (*structtype)(unsafe.Pointer(t))
-		for _, f := range s.Fields {
-			if f.Name.IsBlank() {
-				continue
-			}
-			if err := mapKeyError2(f.Typ, add(p, f.Offset)); err != nil {
-				return err
-			}
-		}
-		return nil
-	default:
-		// Should never happen, keep this case for robustness.
-		return errorString("hash of unhashable type " + toRType(t).string())
-	}
-}
-
 //go:linkname reflect_typehash reflect.typehash
 func reflect_typehash(t *_type, p unsafe.Pointer, h uintptr) uintptr {
 	return typehash(t, p, h)
diff --git a/src/runtime/map_benchmark_test.go b/src/runtime/map_benchmark_test.go
index bf195fa30d..a26b35b44d 100644
--- a/src/runtime/map_benchmark_test.go
+++ b/src/runtime/map_benchmark_test.go
@@ -1191,3 +1191,39 @@ func BenchmarkMapSmallAccessMiss(b *testing.B) {
 	b.Run("Key=string/Elem=string", smallBenchSizes(benchmarkMapAccessMiss[string, string]))
 	b.Run("Key=smallType/Elem=int32", smallBenchSizes(benchmarkMapAccessMiss[smallType, int32]))
 }
+
+func mapAccessZeroBenchmark[K comparable](b *testing.B) {
+	var m map[K]uint64
+	var key K
+	for i := 0; i < b.N; i++ {
+		sink = m[key]
+	}
+}
+
+func BenchmarkMapAccessZero(b *testing.B) {
+	b.Run("Key=int64", mapAccessZeroBenchmark[int64])
+	b.Run("Key=int32", mapAccessZeroBenchmark[int32])
+	b.Run("Key=string", mapAccessZeroBenchmark[string])
+	b.Run("Key=mediumType", mapAccessZeroBenchmark[mediumType])
+	b.Run("Key=bigType", mapAccessZeroBenchmark[bigType])
+}
+
+func mapAccessEmptyBenchmark[K mapBenchmarkKeyType](b *testing.B) {
+	m := make(map[K]uint64)
+	for i, v := range genValues[K](0, 1000) {
+		m[v] = uint64(i)
+	}
+	clear(m)
+	var key K
+	for i := 0; i < b.N; i++ {
+		sink = m[key]
+	}
+}
+
+func BenchmarkMapAccessEmpty(b *testing.B) {
+	b.Run("Key=int64", mapAccessEmptyBenchmark[int64])
+	b.Run("Key=int32", mapAccessEmptyBenchmark[int32])
+	b.Run("Key=string", mapAccessEmptyBenchmark[string])
+	b.Run("Key=mediumType", mapAccessEmptyBenchmark[mediumType])
+	b.Run("Key=bigType", mapAccessEmptyBenchmark[bigType])
+}
diff --git a/src/runtime/map_noswiss.go b/src/runtime/map_noswiss.go
index 327f0c81e8..7b3c98eb88 100644
--- a/src/runtime/map_noswiss.go
+++ b/src/runtime/map_noswiss.go
@@ -59,6 +59,7 @@ import (
 	"internal/abi"
 	"internal/goarch"
 	"internal/runtime/atomic"
+	"internal/runtime/maps"
 	"internal/runtime/math"
 	"internal/runtime/sys"
 	"unsafe"
@@ -426,7 +427,7 @@ func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
 		asanread(key, t.Key.Size_)
 	}
 	if h == nil || h.count == 0 {
-		if err := mapKeyError(t, key); err != nil {
+		if err := maps.OldMapKeyError(t, key); err != nil {
 			panic(err) // see issue 23734
 		}
 		return unsafe.Pointer(&zeroVal[0])
@@ -496,7 +497,7 @@ func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool)
 		asanread(key, t.Key.Size_)
 	}
 	if h == nil || h.count == 0 {
-		if err := mapKeyError(t, key); err != nil {
+		if err := maps.OldMapKeyError(t, key); err != nil {
 			panic(err) // see issue 23734
 		}
 		return unsafe.Pointer(&zeroVal[0]), false
@@ -757,7 +758,7 @@ func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
 		asanread(key, t.Key.Size_)
 	}
 	if h == nil || h.count == 0 {
-		if err := mapKeyError(t, key); err != nil {
+		if err := maps.OldMapKeyError(t, key); err != nil {
 			panic(err) // see issue 23734
 		}
 		return
diff --git a/src/runtime/map_swiss.go b/src/runtime/map_swiss.go
index a1e6ab6b9d..c2cf08fcaa 100644
--- a/src/runtime/map_swiss.go
+++ b/src/runtime/map_swiss.go
@@ -24,11 +24,6 @@ type maptype = abi.SwissMapType
 //go:linkname maps_errNilAssign internal/runtime/maps.errNilAssign
 var maps_errNilAssign error = plainError("assignment to entry in nil map")
 
-//go:linkname maps_mapKeyError internal/runtime/maps.mapKeyError
-func maps_mapKeyError(t *abi.SwissMapType, p unsafe.Pointer) error {
-	return mapKeyError(t, p)
-}
-
 func makemap64(t *abi.SwissMapType, hint int64, m *maps.Map) *maps.Map {
 	if int64(int(hint)) != hint {
 		hint = 0
diff --git a/src/runtime/type.go b/src/runtime/type.go
index 1edf9c9dd6..c11c866cd8 100644
--- a/src/runtime/type.go
+++ b/src/runtime/type.go
@@ -14,6 +14,11 @@ import (
 	"unsafe"
 )
 
+//go:linkname maps_typeString internal/runtime/maps.typeString
+func maps_typeString(typ *abi.Type) string {
+	return toRType(typ).string()
+}
+
 type nameOff = abi.NameOff
 type typeOff = abi.TypeOff
 type textOff = abi.TextOff