diff options
| author | Michael Pratt <mpratt@google.com> | 2024-04-19 13:52:31 -0400 |
|---|---|---|
| committer | Gopher Robot <gobot@golang.org> | 2024-08-02 16:41:53 +0000 |
| commit | 4f7dc282c4bdfba4e63b39bbe9846c1469dc7ee5 (patch) | |
| tree | b4fa4f7de70120c25c408b85056ac30245cfaef9 /src/cmd/compile | |
| parent | 057b703407fa833193cbdc1f37179561c6c9da90 (diff) | |
| download | go-4f7dc282c4bdfba4e63b39bbe9846c1469dc7ee5.tar.xz | |
all: split old and swiss map abi and compiler integration
The two map implementations are still identical, but now the compiler
targets the appropriate ABI depending on GOEXPERIMENT.
For #54766.
Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest,gotip-linux-amd64-longtest-swissmap
Change-Id: I8438f64f044ba9de30ddbf2b8ceb9b4edd2d5614
Reviewed-on: https://go-review.googlesource.com/c/go/+/580779
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: Keith Randall <khr@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Michael Pratt <mpratt@google.com>
Diffstat (limited to 'src/cmd/compile')
| -rw-r--r-- | src/cmd/compile/internal/reflectdata/map_noswiss.go | 303 | ||||
| -rw-r--r-- | src/cmd/compile/internal/reflectdata/map_swiss.go | 303 | ||||
| -rw-r--r-- | src/cmd/compile/internal/reflectdata/reflect.go | 291 | ||||
| -rw-r--r-- | src/cmd/compile/internal/rttype/rttype.go | 6 | ||||
| -rw-r--r-- | src/cmd/compile/internal/ssagen/ssa.go | 14 | ||||
| -rw-r--r-- | src/cmd/compile/internal/types/fmt.go | 2 | ||||
| -rw-r--r-- | src/cmd/compile/internal/types/sizeof_test.go | 2 | ||||
| -rw-r--r-- | src/cmd/compile/internal/types/type.go | 67 | ||||
| -rw-r--r-- | src/cmd/compile/internal/walk/builtin.go | 113 | ||||
| -rw-r--r-- | src/cmd/compile/internal/walk/order.go | 7 | ||||
| -rw-r--r-- | src/cmd/compile/internal/walk/walk.go | 38 |
11 files changed, 833 insertions, 313 deletions
diff --git a/src/cmd/compile/internal/reflectdata/map_noswiss.go b/src/cmd/compile/internal/reflectdata/map_noswiss.go new file mode 100644 index 0000000000..e2dc4ecace --- /dev/null +++ b/src/cmd/compile/internal/reflectdata/map_noswiss.go @@ -0,0 +1,303 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reflectdata + +import ( + "internal/abi" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/rttype" + "cmd/compile/internal/types" + "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/src" +) + +// OldMapBucketType makes the map bucket type given the type of the map. +func OldMapBucketType(t *types.Type) *types.Type { + // Builds a type representing a Bucket structure for + // the given map type. This type is not visible to users - + // we include only enough information to generate a correct GC + // program for it. + // Make sure this stays in sync with runtime/map.go. + // + // A "bucket" is a "struct" { + // tophash [abi.OldMapBucketCount]uint8 + // keys [abi.OldMapBucketCount]keyType + // elems [abi.OldMapBucketCount]elemType + // overflow *bucket + // } + if t.MapType().OldBucket != nil { + return t.MapType().OldBucket + } + + keytype := t.Key() + elemtype := t.Elem() + types.CalcSize(keytype) + types.CalcSize(elemtype) + if keytype.Size() > abi.OldMapMaxKeyBytes { + keytype = types.NewPtr(keytype) + } + if elemtype.Size() > abi.OldMapMaxElemBytes { + elemtype = types.NewPtr(elemtype) + } + + field := make([]*types.Field, 0, 5) + + // The first field is: uint8 topbits[BUCKETSIZE]. + arr := types.NewArray(types.Types[types.TUINT8], abi.OldMapBucketCount) + field = append(field, makefield("topbits", arr)) + + arr = types.NewArray(keytype, abi.OldMapBucketCount) + arr.SetNoalg(true) + keys := makefield("keys", arr) + field = append(field, keys) + + arr = types.NewArray(elemtype, abi.OldMapBucketCount) + arr.SetNoalg(true) + elems := makefield("elems", arr) + field = append(field, elems) + + // If keys and elems have no pointers, the map implementation + // can keep a list of overflow pointers on the side so that + // buckets can be marked as having no pointers. + // Arrange for the bucket to have no pointers by changing + // the type of the overflow field to uintptr in this case. + // See comment on hmap.overflow in runtime/map.go. + otyp := types.Types[types.TUNSAFEPTR] + if !elemtype.HasPointers() && !keytype.HasPointers() { + otyp = types.Types[types.TUINTPTR] + } + overflow := makefield("overflow", otyp) + field = append(field, overflow) + + // link up fields + bucket := types.NewStruct(field[:]) + bucket.SetNoalg(true) + types.CalcSize(bucket) + + // Check invariants that map code depends on. + if !types.IsComparable(t.Key()) { + base.Fatalf("unsupported map key type for %v", t) + } + if abi.OldMapBucketCount < 8 { + base.Fatalf("bucket size %d too small for proper alignment %d", abi.OldMapBucketCount, 8) + } + if uint8(keytype.Alignment()) > abi.OldMapBucketCount { + base.Fatalf("key align too big for %v", t) + } + if uint8(elemtype.Alignment()) > abi.OldMapBucketCount { + base.Fatalf("elem align %d too big for %v, BUCKETSIZE=%d", elemtype.Alignment(), t, abi.OldMapBucketCount) + } + if keytype.Size() > abi.OldMapMaxKeyBytes { + base.Fatalf("key size too large for %v", t) + } + if elemtype.Size() > abi.OldMapMaxElemBytes { + base.Fatalf("elem size too large for %v", t) + } + if t.Key().Size() > abi.OldMapMaxKeyBytes && !keytype.IsPtr() { + base.Fatalf("key indirect incorrect for %v", t) + } + if t.Elem().Size() > abi.OldMapMaxElemBytes && !elemtype.IsPtr() { + base.Fatalf("elem indirect incorrect for %v", t) + } + if keytype.Size()%keytype.Alignment() != 0 { + base.Fatalf("key size not a multiple of key align for %v", t) + } + if elemtype.Size()%elemtype.Alignment() != 0 { + base.Fatalf("elem size not a multiple of elem align for %v", t) + } + if uint8(bucket.Alignment())%uint8(keytype.Alignment()) != 0 { + base.Fatalf("bucket align not multiple of key align %v", t) + } + if uint8(bucket.Alignment())%uint8(elemtype.Alignment()) != 0 { + base.Fatalf("bucket align not multiple of elem align %v", t) + } + if keys.Offset%keytype.Alignment() != 0 { + base.Fatalf("bad alignment of keys in bmap for %v", t) + } + if elems.Offset%elemtype.Alignment() != 0 { + base.Fatalf("bad alignment of elems in bmap for %v", t) + } + + // Double-check that overflow field is final memory in struct, + // with no padding at end. + if overflow.Offset != bucket.Size()-int64(types.PtrSize) { + base.Fatalf("bad offset of overflow in bmap for %v, overflow.Offset=%d, bucket.Size()-int64(types.PtrSize)=%d", + t, overflow.Offset, bucket.Size()-int64(types.PtrSize)) + } + + t.MapType().OldBucket = bucket + + bucket.StructType().Map = t + return bucket +} + +var oldHmapType *types.Type + +// OldMapType returns a type interchangeable with runtime.hmap. +// Make sure this stays in sync with runtime/map.go. +func OldMapType() *types.Type { + if oldHmapType != nil { + return oldHmapType + } + + // build a struct: + // type hmap struct { + // count int + // flags uint8 + // B uint8 + // noverflow uint16 + // hash0 uint32 + // buckets unsafe.Pointer + // oldbuckets unsafe.Pointer + // nevacuate uintptr + // extra unsafe.Pointer // *mapextra + // } + // must match runtime/map.go:hmap. + fields := []*types.Field{ + makefield("count", types.Types[types.TINT]), + makefield("flags", types.Types[types.TUINT8]), + makefield("B", types.Types[types.TUINT8]), + makefield("noverflow", types.Types[types.TUINT16]), + makefield("hash0", types.Types[types.TUINT32]), // Used in walk.go for OMAKEMAP. + makefield("buckets", types.Types[types.TUNSAFEPTR]), // Used in walk.go for OMAKEMAP. + makefield("oldbuckets", types.Types[types.TUNSAFEPTR]), + makefield("nevacuate", types.Types[types.TUINTPTR]), + makefield("extra", types.Types[types.TUNSAFEPTR]), + } + + n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.Runtime.Lookup("hmap")) + hmap := types.NewNamed(n) + n.SetType(hmap) + n.SetTypecheck(1) + + hmap.SetUnderlying(types.NewStruct(fields)) + types.CalcSize(hmap) + + // The size of hmap should be 48 bytes on 64 bit + // and 28 bytes on 32 bit platforms. + if size := int64(8 + 5*types.PtrSize); hmap.Size() != size { + base.Fatalf("hmap size not correct: got %d, want %d", hmap.Size(), size) + } + + oldHmapType = hmap + return hmap +} + +var oldHiterType *types.Type + +// OldMapIterType returns a type interchangeable with runtime.hiter. +// Make sure this stays in sync with runtime/map.go. +func OldMapIterType() *types.Type { + if oldHiterType != nil { + return oldHiterType + } + + hmap := OldMapType() + + // build a struct: + // type hiter struct { + // key unsafe.Pointer // *Key + // elem unsafe.Pointer // *Elem + // t unsafe.Pointer // *OldMapType + // h *hmap + // buckets unsafe.Pointer + // bptr unsafe.Pointer // *bmap + // overflow unsafe.Pointer // *[]*bmap + // oldoverflow unsafe.Pointer // *[]*bmap + // startBucket uintptr + // offset uint8 + // wrapped bool + // B uint8 + // i uint8 + // bucket uintptr + // checkBucket uintptr + // } + // must match runtime/map.go:hiter. + fields := []*types.Field{ + makefield("key", types.Types[types.TUNSAFEPTR]), // Used in range.go for TMAP. + makefield("elem", types.Types[types.TUNSAFEPTR]), // Used in range.go for TMAP. + makefield("t", types.Types[types.TUNSAFEPTR]), + makefield("h", types.NewPtr(hmap)), + makefield("buckets", types.Types[types.TUNSAFEPTR]), + makefield("bptr", types.Types[types.TUNSAFEPTR]), + makefield("overflow", types.Types[types.TUNSAFEPTR]), + makefield("oldoverflow", types.Types[types.TUNSAFEPTR]), + makefield("startBucket", types.Types[types.TUINTPTR]), + makefield("offset", types.Types[types.TUINT8]), + makefield("wrapped", types.Types[types.TBOOL]), + makefield("B", types.Types[types.TUINT8]), + makefield("i", types.Types[types.TUINT8]), + makefield("bucket", types.Types[types.TUINTPTR]), + makefield("checkBucket", types.Types[types.TUINTPTR]), + } + + // build iterator struct holding the above fields + n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.Runtime.Lookup("hiter")) + hiter := types.NewNamed(n) + n.SetType(hiter) + n.SetTypecheck(1) + + hiter.SetUnderlying(types.NewStruct(fields)) + types.CalcSize(hiter) + if hiter.Size() != int64(12*types.PtrSize) { + base.Fatalf("hash_iter size not correct %d %d", hiter.Size(), 12*types.PtrSize) + } + + oldHiterType = hiter + return hiter +} + +func writeOldMapType(t *types.Type, lsym *obj.LSym, c rttype.Cursor) { + // internal/abi.OldMapType + s1 := writeType(t.Key()) + s2 := writeType(t.Elem()) + s3 := writeType(OldMapBucketType(t)) + hasher := genhash(t.Key()) + + c.Field("Key").WritePtr(s1) + c.Field("Elem").WritePtr(s2) + c.Field("Bucket").WritePtr(s3) + c.Field("Hasher").WritePtr(hasher) + var flags uint32 + // Note: flags must match maptype accessors in ../../../../runtime/type.go + // and maptype builder in ../../../../reflect/type.go:MapOf. + if t.Key().Size() > abi.OldMapMaxKeyBytes { + c.Field("KeySize").WriteUint8(uint8(types.PtrSize)) + flags |= 1 // indirect key + } else { + c.Field("KeySize").WriteUint8(uint8(t.Key().Size())) + } + + if t.Elem().Size() > abi.OldMapMaxElemBytes { + c.Field("ValueSize").WriteUint8(uint8(types.PtrSize)) + flags |= 2 // indirect value + } else { + c.Field("ValueSize").WriteUint8(uint8(t.Elem().Size())) + } + c.Field("BucketSize").WriteUint16(uint16(OldMapBucketType(t).Size())) + if types.IsReflexive(t.Key()) { + flags |= 4 // reflexive key + } + if needkeyupdate(t.Key()) { + flags |= 8 // need key update + } + if hashMightPanic(t.Key()) { + flags |= 16 // hash might panic + } + c.Field("Flags").WriteUint32(flags) + + if u := t.Underlying(); u != t { + // If t is a named map type, also keep the underlying map + // type live in the binary. This is important to make sure that + // a named map and that same map cast to its underlying type via + // reflection, use the same hash function. See issue 37716. + r := obj.Addrel(lsym) + r.Sym = writeType(u) + r.Type = objabi.R_KEEP + } +} diff --git a/src/cmd/compile/internal/reflectdata/map_swiss.go b/src/cmd/compile/internal/reflectdata/map_swiss.go new file mode 100644 index 0000000000..4fed93517e --- /dev/null +++ b/src/cmd/compile/internal/reflectdata/map_swiss.go @@ -0,0 +1,303 @@ +// Copyright 2024 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package reflectdata + +import ( + "internal/abi" + + "cmd/compile/internal/base" + "cmd/compile/internal/ir" + "cmd/compile/internal/rttype" + "cmd/compile/internal/types" + "cmd/internal/obj" + "cmd/internal/objabi" + "cmd/internal/src" +) + +// SwissMapBucketType makes the map bucket type given the type of the map. +func SwissMapBucketType(t *types.Type) *types.Type { + // Builds a type representing a Bucket structure for + // the given map type. This type is not visible to users - + // we include only enough information to generate a correct GC + // program for it. + // Make sure this stays in sync with runtime/map.go. + // + // A "bucket" is a "struct" { + // tophash [abi.SwissMapBucketCount]uint8 + // keys [abi.SwissMapBucketCount]keyType + // elems [abi.SwissMapBucketCount]elemType + // overflow *bucket + // } + if t.MapType().SwissBucket != nil { + return t.MapType().SwissBucket + } + + keytype := t.Key() + elemtype := t.Elem() + types.CalcSize(keytype) + types.CalcSize(elemtype) + if keytype.Size() > abi.SwissMapMaxKeyBytes { + keytype = types.NewPtr(keytype) + } + if elemtype.Size() > abi.SwissMapMaxElemBytes { + elemtype = types.NewPtr(elemtype) + } + + field := make([]*types.Field, 0, 5) + + // The first field is: uint8 topbits[BUCKETSIZE]. + arr := types.NewArray(types.Types[types.TUINT8], abi.SwissMapBucketCount) + field = append(field, makefield("topbits", arr)) + + arr = types.NewArray(keytype, abi.SwissMapBucketCount) + arr.SetNoalg(true) + keys := makefield("keys", arr) + field = append(field, keys) + + arr = types.NewArray(elemtype, abi.SwissMapBucketCount) + arr.SetNoalg(true) + elems := makefield("elems", arr) + field = append(field, elems) + + // If keys and elems have no pointers, the map implementation + // can keep a list of overflow pointers on the side so that + // buckets can be marked as having no pointers. + // Arrange for the bucket to have no pointers by changing + // the type of the overflow field to uintptr in this case. + // See comment on hmap.overflow in runtime/map.go. + otyp := types.Types[types.TUNSAFEPTR] + if !elemtype.HasPointers() && !keytype.HasPointers() { + otyp = types.Types[types.TUINTPTR] + } + overflow := makefield("overflow", otyp) + field = append(field, overflow) + + // link up fields + bucket := types.NewStruct(field[:]) + bucket.SetNoalg(true) + types.CalcSize(bucket) + + // Check invariants that map code depends on. + if !types.IsComparable(t.Key()) { + base.Fatalf("unsupported map key type for %v", t) + } + if abi.SwissMapBucketCount < 8 { + base.Fatalf("bucket size %d too small for proper alignment %d", abi.SwissMapBucketCount, 8) + } + if uint8(keytype.Alignment()) > abi.SwissMapBucketCount { + base.Fatalf("key align too big for %v", t) + } + if uint8(elemtype.Alignment()) > abi.SwissMapBucketCount { + base.Fatalf("elem align %d too big for %v, BUCKETSIZE=%d", elemtype.Alignment(), t, abi.SwissMapBucketCount) + } + if keytype.Size() > abi.SwissMapMaxKeyBytes { + base.Fatalf("key size too large for %v", t) + } + if elemtype.Size() > abi.SwissMapMaxElemBytes { + base.Fatalf("elem size too large for %v", t) + } + if t.Key().Size() > abi.SwissMapMaxKeyBytes && !keytype.IsPtr() { + base.Fatalf("key indirect incorrect for %v", t) + } + if t.Elem().Size() > abi.SwissMapMaxElemBytes && !elemtype.IsPtr() { + base.Fatalf("elem indirect incorrect for %v", t) + } + if keytype.Size()%keytype.Alignment() != 0 { + base.Fatalf("key size not a multiple of key align for %v", t) + } + if elemtype.Size()%elemtype.Alignment() != 0 { + base.Fatalf("elem size not a multiple of elem align for %v", t) + } + if uint8(bucket.Alignment())%uint8(keytype.Alignment()) != 0 { + base.Fatalf("bucket align not multiple of key align %v", t) + } + if uint8(bucket.Alignment())%uint8(elemtype.Alignment()) != 0 { + base.Fatalf("bucket align not multiple of elem align %v", t) + } + if keys.Offset%keytype.Alignment() != 0 { + base.Fatalf("bad alignment of keys in bmap for %v", t) + } + if elems.Offset%elemtype.Alignment() != 0 { + base.Fatalf("bad alignment of elems in bmap for %v", t) + } + + // Double-check that overflow field is final memory in struct, + // with no padding at end. + if overflow.Offset != bucket.Size()-int64(types.PtrSize) { + base.Fatalf("bad offset of overflow in bmap for %v, overflow.Offset=%d, bucket.Size()-int64(types.PtrSize)=%d", + t, overflow.Offset, bucket.Size()-int64(types.PtrSize)) + } + + t.MapType().SwissBucket = bucket + + bucket.StructType().Map = t + return bucket +} + +var swissHmapType *types.Type + +// SwissMapType returns a type interchangeable with runtime.hmap. +// Make sure this stays in sync with runtime/map.go. +func SwissMapType() *types.Type { + if swissHmapType != nil { + return swissHmapType + } + + // build a struct: + // type hmap struct { + // count int + // flags uint8 + // B uint8 + // noverflow uint16 + // hash0 uint32 + // buckets unsafe.Pointer + // oldbuckets unsafe.Pointer + // nevacuate uintptr + // extra unsafe.Pointer // *mapextra + // } + // must match runtime/map.go:hmap. + fields := []*types.Field{ + makefield("count", types.Types[types.TINT]), + makefield("flags", types.Types[types.TUINT8]), + makefield("B", types.Types[types.TUINT8]), + makefield("noverflow", types.Types[types.TUINT16]), + makefield("hash0", types.Types[types.TUINT32]), // Used in walk.go for OMAKEMAP. + makefield("buckets", types.Types[types.TUNSAFEPTR]), // Used in walk.go for OMAKEMAP. + makefield("oldbuckets", types.Types[types.TUNSAFEPTR]), + makefield("nevacuate", types.Types[types.TUINTPTR]), + makefield("extra", types.Types[types.TUNSAFEPTR]), + } + + n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.Runtime.Lookup("hmap")) + hmap := types.NewNamed(n) + n.SetType(hmap) + n.SetTypecheck(1) + + hmap.SetUnderlying(types.NewStruct(fields)) + types.CalcSize(hmap) + + // The size of hmap should be 48 bytes on 64 bit + // and 28 bytes on 32 bit platforms. + if size := int64(8 + 5*types.PtrSize); hmap.Size() != size { + base.Fatalf("hmap size not correct: got %d, want %d", hmap.Size(), size) + } + + swissHmapType = hmap + return hmap +} + +var swissHiterType *types.Type + +// SwissMapIterType returns a type interchangeable with runtime.hiter. +// Make sure this stays in sync with runtime/map.go. +func SwissMapIterType() *types.Type { + if swissHiterType != nil { + return swissHiterType + } + + hmap := SwissMapType() + + // build a struct: + // type hiter struct { + // key unsafe.Pointer // *Key + // elem unsafe.Pointer // *Elem + // t unsafe.Pointer // *SwissMapType + // h *hmap + // buckets unsafe.Pointer + // bptr unsafe.Pointer // *bmap + // overflow unsafe.Pointer // *[]*bmap + // oldoverflow unsafe.Pointer // *[]*bmap + // startBucket uintptr + // offset uint8 + // wrapped bool + // B uint8 + // i uint8 + // bucket uintptr + // checkBucket uintptr + // } + // must match runtime/map.go:hiter. + fields := []*types.Field{ + makefield("key", types.Types[types.TUNSAFEPTR]), // Used in range.go for TMAP. + makefield("elem", types.Types[types.TUNSAFEPTR]), // Used in range.go for TMAP. + makefield("t", types.Types[types.TUNSAFEPTR]), + makefield("h", types.NewPtr(hmap)), + makefield("buckets", types.Types[types.TUNSAFEPTR]), + makefield("bptr", types.Types[types.TUNSAFEPTR]), + makefield("overflow", types.Types[types.TUNSAFEPTR]), + makefield("oldoverflow", types.Types[types.TUNSAFEPTR]), + makefield("startBucket", types.Types[types.TUINTPTR]), + makefield("offset", types.Types[types.TUINT8]), + makefield("wrapped", types.Types[types.TBOOL]), + makefield("B", types.Types[types.TUINT8]), + makefield("i", types.Types[types.TUINT8]), + makefield("bucket", types.Types[types.TUINTPTR]), + makefield("checkBucket", types.Types[types.TUINTPTR]), + } + + // build iterator struct hswissing the above fields + n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.Runtime.Lookup("hiter")) + hiter := types.NewNamed(n) + n.SetType(hiter) + n.SetTypecheck(1) + + hiter.SetUnderlying(types.NewStruct(fields)) + types.CalcSize(hiter) + if hiter.Size() != int64(12*types.PtrSize) { + base.Fatalf("hash_iter size not correct %d %d", hiter.Size(), 12*types.PtrSize) + } + + swissHiterType = hiter + return hiter +} + +func writeSwissMapType(t *types.Type, lsym *obj.LSym, c rttype.Cursor) { + // internal/abi.SwissMapType + s1 := writeType(t.Key()) + s2 := writeType(t.Elem()) + s3 := writeType(SwissMapBucketType(t)) + hasher := genhash(t.Key()) + + c.Field("Key").WritePtr(s1) + c.Field("Elem").WritePtr(s2) + c.Field("Bucket").WritePtr(s3) + c.Field("Hasher").WritePtr(hasher) + var flags uint32 + // Note: flags must match maptype accessors in ../../../../runtime/type.go + // and maptype builder in ../../../../reflect/type.go:MapOf. + if t.Key().Size() > abi.SwissMapMaxKeyBytes { + c.Field("KeySize").WriteUint8(uint8(types.PtrSize)) + flags |= 1 // indirect key + } else { + c.Field("KeySize").WriteUint8(uint8(t.Key().Size())) + } + + if t.Elem().Size() > abi.SwissMapMaxElemBytes { + c.Field("ValueSize").WriteUint8(uint8(types.PtrSize)) + flags |= 2 // indirect value + } else { + c.Field("ValueSize").WriteUint8(uint8(t.Elem().Size())) + } + c.Field("BucketSize").WriteUint16(uint16(SwissMapBucketType(t).Size())) + if types.IsReflexive(t.Key()) { + flags |= 4 // reflexive key + } + if needkeyupdate(t.Key()) { + flags |= 8 // need key update + } + if hashMightPanic(t.Key()) { + flags |= 16 // hash might panic + } + c.Field("Flags").WriteUint32(flags) + + if u := t.Underlying(); u != t { + // If t is a named map type, also keep the underlying map + // type live in the binary. This is important to make sure that + // a named map and that same map cast to its underlying type via + // reflection, use the same hash function. See issue 37716. + r := obj.Addrel(lsym) + r.Sym = writeType(u) + r.Type = objabi.R_KEEP + } +} diff --git a/src/cmd/compile/internal/reflectdata/reflect.go b/src/cmd/compile/internal/reflectdata/reflect.go index 816ccc627f..06a3314c47 100644 --- a/src/cmd/compile/internal/reflectdata/reflect.go +++ b/src/cmd/compile/internal/reflectdata/reflect.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "fmt" "internal/abi" + "internal/buildcfg" "os" "sort" "strings" @@ -69,242 +70,6 @@ func makefield(name string, t *types.Type) *types.Field { return types.NewField(src.NoXPos, sym, t) } -// MapBucketType makes the map bucket type given the type of the map. -func MapBucketType(t *types.Type) *types.Type { - // Builds a type representing a Bucket structure for - // the given map type. This type is not visible to users - - // we include only enough information to generate a correct GC - // program for it. - // Make sure this stays in sync with runtime/map.go. - // - // A "bucket" is a "struct" { - // tophash [abi.MapBucketCount]uint8 - // keys [abi.MapBucketCount]keyType - // elems [abi.MapBucketCount]elemType - // overflow *bucket - // } - if t.MapType().Bucket != nil { - return t.MapType().Bucket - } - - keytype := t.Key() - elemtype := t.Elem() - types.CalcSize(keytype) - types.CalcSize(elemtype) - if keytype.Size() > abi.MapMaxKeyBytes { - keytype = types.NewPtr(keytype) - } - if elemtype.Size() > abi.MapMaxElemBytes { - elemtype = types.NewPtr(elemtype) - } - - field := make([]*types.Field, 0, 5) - - // The first field is: uint8 topbits[BUCKETSIZE]. - arr := types.NewArray(types.Types[types.TUINT8], abi.MapBucketCount) - field = append(field, makefield("topbits", arr)) - - arr = types.NewArray(keytype, abi.MapBucketCount) - arr.SetNoalg(true) - keys := makefield("keys", arr) - field = append(field, keys) - - arr = types.NewArray(elemtype, abi.MapBucketCount) - arr.SetNoalg(true) - elems := makefield("elems", arr) - field = append(field, elems) - - // If keys and elems have no pointers, the map implementation - // can keep a list of overflow pointers on the side so that - // buckets can be marked as having no pointers. - // Arrange for the bucket to have no pointers by changing - // the type of the overflow field to uintptr in this case. - // See comment on hmap.overflow in runtime/map.go. - otyp := types.Types[types.TUNSAFEPTR] - if !elemtype.HasPointers() && !keytype.HasPointers() { - otyp = types.Types[types.TUINTPTR] - } - overflow := makefield("overflow", otyp) - field = append(field, overflow) - - // link up fields - bucket := types.NewStruct(field[:]) - bucket.SetNoalg(true) - types.CalcSize(bucket) - - // Check invariants that map code depends on. - if !types.IsComparable(t.Key()) { - base.Fatalf("unsupported map key type for %v", t) - } - if abi.MapBucketCount < 8 { - base.Fatalf("bucket size %d too small for proper alignment %d", abi.MapBucketCount, 8) - } - if uint8(keytype.Alignment()) > abi.MapBucketCount { - base.Fatalf("key align too big for %v", t) - } - if uint8(elemtype.Alignment()) > abi.MapBucketCount { - base.Fatalf("elem align %d too big for %v, BUCKETSIZE=%d", elemtype.Alignment(), t, abi.MapBucketCount) - } - if keytype.Size() > abi.MapMaxKeyBytes { - base.Fatalf("key size too large for %v", t) - } - if elemtype.Size() > abi.MapMaxElemBytes { - base.Fatalf("elem size too large for %v", t) - } - if t.Key().Size() > abi.MapMaxKeyBytes && !keytype.IsPtr() { - base.Fatalf("key indirect incorrect for %v", t) - } - if t.Elem().Size() > abi.MapMaxElemBytes && !elemtype.IsPtr() { - base.Fatalf("elem indirect incorrect for %v", t) - } - if keytype.Size()%keytype.Alignment() != 0 { - base.Fatalf("key size not a multiple of key align for %v", t) - } - if elemtype.Size()%elemtype.Alignment() != 0 { - base.Fatalf("elem size not a multiple of elem align for %v", t) - } - if uint8(bucket.Alignment())%uint8(keytype.Alignment()) != 0 { - base.Fatalf("bucket align not multiple of key align %v", t) - } - if uint8(bucket.Alignment())%uint8(elemtype.Alignment()) != 0 { - base.Fatalf("bucket align not multiple of elem align %v", t) - } - if keys.Offset%keytype.Alignment() != 0 { - base.Fatalf("bad alignment of keys in bmap for %v", t) - } - if elems.Offset%elemtype.Alignment() != 0 { - base.Fatalf("bad alignment of elems in bmap for %v", t) - } - - // Double-check that overflow field is final memory in struct, - // with no padding at end. - if overflow.Offset != bucket.Size()-int64(types.PtrSize) { - base.Fatalf("bad offset of overflow in bmap for %v, overflow.Offset=%d, bucket.Size()-int64(types.PtrSize)=%d", - t, overflow.Offset, bucket.Size()-int64(types.PtrSize)) - } - - t.MapType().Bucket = bucket - - bucket.StructType().Map = t - return bucket -} - -var hmapType *types.Type - -// MapType returns a type interchangeable with runtime.hmap. -// Make sure this stays in sync with runtime/map.go. -func MapType() *types.Type { - if hmapType != nil { - return hmapType - } - - // build a struct: - // type hmap struct { - // count int - // flags uint8 - // B uint8 - // noverflow uint16 - // hash0 uint32 - // buckets unsafe.Pointer - // oldbuckets unsafe.Pointer - // nevacuate uintptr - // extra unsafe.Pointer // *mapextra - // } - // must match runtime/map.go:hmap. - fields := []*types.Field{ - makefield("count", types.Types[types.TINT]), - makefield("flags", types.Types[types.TUINT8]), - makefield("B", types.Types[types.TUINT8]), - makefield("noverflow", types.Types[types.TUINT16]), - makefield("hash0", types.Types[types.TUINT32]), // Used in walk.go for OMAKEMAP. - makefield("buckets", types.Types[types.TUNSAFEPTR]), // Used in walk.go for OMAKEMAP. - makefield("oldbuckets", types.Types[types.TUNSAFEPTR]), - makefield("nevacuate", types.Types[types.TUINTPTR]), - makefield("extra", types.Types[types.TUNSAFEPTR]), - } - - n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.Runtime.Lookup("hmap")) - hmap := types.NewNamed(n) - n.SetType(hmap) - n.SetTypecheck(1) - - hmap.SetUnderlying(types.NewStruct(fields)) - types.CalcSize(hmap) - - // The size of hmap should be 48 bytes on 64 bit - // and 28 bytes on 32 bit platforms. - if size := int64(8 + 5*types.PtrSize); hmap.Size() != size { - base.Fatalf("hmap size not correct: got %d, want %d", hmap.Size(), size) - } - - hmapType = hmap - return hmap -} - -var hiterType *types.Type - -// MapIterType returns a type interchangeable with runtime.hiter. -// Make sure this stays in sync with runtime/map.go. -func MapIterType() *types.Type { - if hiterType != nil { - return hiterType - } - - hmap := MapType() - - // build a struct: - // type hiter struct { - // key unsafe.Pointer // *Key - // elem unsafe.Pointer // *Elem - // t unsafe.Pointer // *MapType - // h *hmap - // buckets unsafe.Pointer - // bptr unsafe.Pointer // *bmap - // overflow unsafe.Pointer // *[]*bmap - // oldoverflow unsafe.Pointer // *[]*bmap - // startBucket uintptr - // offset uint8 - // wrapped bool - // B uint8 - // i uint8 - // bucket uintptr - // checkBucket uintptr - // } - // must match runtime/map.go:hiter. - fields := []*types.Field{ - makefield("key", types.Types[types.TUNSAFEPTR]), // Used in range.go for TMAP. - makefield("elem", types.Types[types.TUNSAFEPTR]), // Used in range.go for TMAP. - makefield("t", types.Types[types.TUNSAFEPTR]), - makefield("h", types.NewPtr(hmap)), - makefield("buckets", types.Types[types.TUNSAFEPTR]), - makefield("bptr", types.Types[types.TUNSAFEPTR]), - makefield("overflow", types.Types[types.TUNSAFEPTR]), - makefield("oldoverflow", types.Types[types.TUNSAFEPTR]), - makefield("startBucket", types.Types[types.TUINTPTR]), - makefield("offset", types.Types[types.TUINT8]), - makefield("wrapped", types.Types[types.TBOOL]), - makefield("B", types.Types[types.TUINT8]), - makefield("i", types.Types[types.TUINT8]), - makefield("bucket", types.Types[types.TUINTPTR]), - makefield("checkBucket", types.Types[types.TUINTPTR]), - } - - // build iterator struct holding the above fields - n := ir.NewDeclNameAt(src.NoXPos, ir.OTYPE, ir.Pkgs.Runtime.Lookup("hiter")) - hiter := types.NewNamed(n) - n.SetType(hiter) - n.SetTypecheck(1) - - hiter.SetUnderlying(types.NewStruct(fields)) - types.CalcSize(hiter) - if hiter.Size() != int64(12*types.PtrSize) { - base.Fatalf("hash_iter size not correct %d %d", hiter.Size(), 12*types.PtrSize) - } - - hiterType = hiter - return hiter -} - // methods returns the methods of the non-interface type t, sorted by name. // Generates stub functions as needed. func methods(t *types.Type) []*typeSig { @@ -1005,7 +770,11 @@ func writeType(t *types.Type) *obj.LSym { rt = rttype.InterfaceType dataAdd = len(imethods(t)) * int(rttype.IMethod.Size()) case types.TMAP: - rt = rttype.MapType + if buildcfg.Experiment.SwissMap { + rt = rttype.SwissMapType + } else { + rt = rttype.OldMapType + } case types.TPTR: rt = rttype.PtrType // TODO: use rttype.Type for Elem() is ANY? @@ -1105,52 +874,10 @@ func writeType(t *types.Type) *obj.LSym { } case types.TMAP: - // internal/abi.MapType - s1 := writeType(t.Key()) - s2 := writeType(t.Elem()) - s3 := writeType(MapBucketType(t)) - hasher := genhash(t.Key()) - - c.Field("Key").WritePtr(s1) - c.Field("Elem").WritePtr(s2) - c.Field("Bucket").WritePtr(s3) - c.Field("Hasher").WritePtr(hasher) - var flags uint32 - // Note: flags must match maptype accessors in ../../../../runtime/type.go - // and maptype builder in ../../../../reflect/type.go:MapOf. - if t.Key().Size() > abi.MapMaxKeyBytes { - c.Field("KeySize").WriteUint8(uint8(types.PtrSize)) - flags |= 1 // indirect key + if buildcfg.Experiment.SwissMap { + writeSwissMapType(t, lsym, c) } else { - c.Field("KeySize").WriteUint8(uint8(t.Key().Size())) - } - - if t.Elem().Size() > abi.MapMaxElemBytes { - c.Field("ValueSize").WriteUint8(uint8(types.PtrSize)) - flags |= 2 // indirect value - } else { - c.Field("ValueSize").WriteUint8(uint8(t.Elem().Size())) - } - c.Field("BucketSize").WriteUint16(uint16(MapBucketType(t).Size())) - if types.IsReflexive(t.Key()) { - flags |= 4 // reflexive key - } - if needkeyupdate(t.Key()) { - flags |= 8 // need key update - } - if hashMightPanic(t.Key()) { - flags |= 16 // hash might panic - } - c.Field("Flags").WriteUint32(flags) - - if u := t.Underlying(); u != t { - // If t is a named map type, also keep the underlying map - // type live in the binary. This is important to make sure that - // a named map and that same map cast to its underlying type via - // reflection, use the same hash function. See issue 37716. - r := obj.Addrel(lsym) - r.Sym = writeType(u) - r.Type = objabi.R_KEEP + writeOldMapType(t, lsym, c) } case types.TPTR: diff --git a/src/cmd/compile/internal/rttype/rttype.go b/src/cmd/compile/internal/rttype/rttype.go index b90e23dc5b..7a57f4a077 100644 --- a/src/cmd/compile/internal/rttype/rttype.go +++ b/src/cmd/compile/internal/rttype/rttype.go @@ -27,7 +27,8 @@ var ArrayType *types.Type var ChanType *types.Type var FuncType *types.Type var InterfaceType *types.Type -var MapType *types.Type +var OldMapType *types.Type +var SwissMapType *types.Type var PtrType *types.Type var SliceType *types.Type var StructType *types.Type @@ -54,7 +55,8 @@ func Init() { ChanType = fromReflect(reflect.TypeOf(abi.ChanType{})) FuncType = fromReflect(reflect.TypeOf(abi.FuncType{})) InterfaceType = fromReflect(reflect.TypeOf(abi.InterfaceType{})) - MapType = fromReflect(reflect.TypeOf(abi.MapType{})) + OldMapType = fromReflect(reflect.TypeOf(abi.OldMapType{})) + SwissMapType = fromReflect(reflect.TypeOf(abi.SwissMapType{})) PtrType = fromReflect(reflect.TypeOf(abi.PtrType{})) SliceType = fromReflect(reflect.TypeOf(abi.SliceType{})) StructType = fromReflect(reflect.TypeOf(abi.StructType{})) diff --git a/src/cmd/compile/internal/ssagen/ssa.go b/src/cmd/compile/internal/ssagen/ssa.go index 765f4c2e98..6919901f05 100644 --- a/src/cmd/compile/internal/ssagen/ssa.go +++ b/src/cmd/compile/internal/ssagen/ssa.go @@ -88,7 +88,11 @@ func InitConfig() { _ = types.NewPtr(types.Types[types.TINT16]) // *int16 _ = types.NewPtr(types.Types[types.TINT64]) // *int64 _ = types.NewPtr(types.ErrorType) // *error - _ = types.NewPtr(reflectdata.MapType()) // *runtime.hmap + if buildcfg.Experiment.SwissMap { + _ = types.NewPtr(reflectdata.SwissMapType()) // *runtime.hmap + } else { + _ = types.NewPtr(reflectdata.OldMapType()) // *runtime.hmap + } _ = types.NewPtr(deferstruct()) // *runtime._defer types.NewPtrCacheEnabled = false ssaConfig = ssa.NewConfig(base.Ctxt.Arch.Name, *types_, base.Ctxt, base.Flag.N == 0, Arch.SoftFloat) @@ -2939,7 +2943,13 @@ func (s *state) exprCheckPtr(n ir.Node, checkPtrOK bool) *ssa.Value { } // map <--> *hmap - if to.Kind() == types.TMAP && from == types.NewPtr(reflectdata.MapType()) { + var mt *types.Type + if buildcfg.Experiment.SwissMap { + mt = types.NewPtr(reflectdata.SwissMapType()) + } else { + mt = types.NewPtr(reflectdata.OldMapType()) + } + if to.Kind() == types.TMAP && from == mt { return v } diff --git a/src/cmd/compile/internal/types/fmt.go b/src/cmd/compile/internal/types/fmt.go index c9b9853f78..d6cc2483a6 100644 --- a/src/cmd/compile/internal/types/fmt.go +++ b/src/cmd/compile/internal/types/fmt.go @@ -474,7 +474,7 @@ func tconv2(b *bytes.Buffer, t *Type, verb rune, mode fmtMode, visited map[*Type // Format the bucket struct for map[x]y as map.bucket[x]y. // This avoids a recursive print that generates very long names. switch t { - case mt.Bucket: + case mt.OldBucket, mt.SwissBucket: b.WriteString("map.bucket[") default: base.Fatalf("unknown internal map type") diff --git a/src/cmd/compile/internal/types/sizeof_test.go b/src/cmd/compile/internal/types/sizeof_test.go index 7e3e7769d7..27845fbd2d 100644 --- a/src/cmd/compile/internal/types/sizeof_test.go +++ b/src/cmd/compile/internal/types/sizeof_test.go @@ -22,7 +22,7 @@ func TestSizeof(t *testing.T) { }{ {Sym{}, 32, 64}, {Type{}, 64, 104}, - {Map{}, 12, 24}, + {Map{}, 16, 32}, {Forward{}, 20, 32}, {Func{}, 32, 56}, {Struct{}, 12, 24}, diff --git a/src/cmd/compile/internal/types/type.go b/src/cmd/compile/internal/types/type.go index 693bd9385a..d8950ba894 100644 --- a/src/cmd/compile/internal/types/type.go +++ b/src/cmd/compile/internal/types/type.go @@ -10,6 +10,7 @@ import ( "cmd/internal/src" "fmt" "go/constant" + "internal/buildcfg" "internal/types/errors" "sync" ) @@ -313,7 +314,17 @@ type Map struct { Key *Type // Key type Elem *Type // Val (elem) type - Bucket *Type // internal struct type representing a hash bucket + // Note: It would be cleaner to completely split Map into OldMap and + // SwissMap, but 99% of the types map code doesn't care about the + // implementation at all, so it is tons of churn to split the type. + // Only code that looks at the bucket field can care about the + // implementation. + + // GOEXPERIMENT=noswissmap fields + OldBucket *Type // internal struct type representing a hash bucket + + // GOEXPERIMENT=swissmap fields + SwissBucket *Type // internal struct type representing a hash bucket } // MapType returns t's extra map-specific fields. @@ -1206,23 +1217,43 @@ func (t *Type) cmp(x *Type) Cmp { // by the general code after the switch. case TSTRUCT: - if t.StructType().Map == nil { - if x.StructType().Map != nil { - return CMPlt // nil < non-nil - } - // to the fallthrough - } else if x.StructType().Map == nil { - return CMPgt // nil > non-nil - } else if t.StructType().Map.MapType().Bucket == t { - // Both have non-nil Map - // Special case for Maps which include a recursive type where the recursion is not broken with a named type - if x.StructType().Map.MapType().Bucket != x { - return CMPlt // bucket maps are least - } - return t.StructType().Map.cmp(x.StructType().Map) - } else if x.StructType().Map.MapType().Bucket == x { - return CMPgt // bucket maps are least - } // If t != t.Map.Bucket, fall through to general case + if buildcfg.Experiment.SwissMap { + if t.StructType().Map == nil { + if x.StructType().Map != nil { + return CMPlt // nil < non-nil + } + // to the fallthrough + } else if x.StructType().Map == nil { + return CMPgt // nil > non-nil + } else if t.StructType().Map.MapType().SwissBucket == t { + // Both have non-nil Map + // Special case for Maps which include a recursive type where the recursion is not broken with a named type + if x.StructType().Map.MapType().SwissBucket != x { + return CMPlt // bucket maps are least + } + return t.StructType().Map.cmp(x.StructType().Map) + } else if x.StructType().Map.MapType().SwissBucket == x { + return CMPgt // bucket maps are least + } // If t != t.Map.SwissBucket, fall through to general case + } else { + if t.StructType().Map == nil { + if x.StructType().Map != nil { + return CMPlt // nil < non-nil + } + // to the fallthrough + } else if x.StructType().Map == nil { + return CMPgt // nil > non-nil + } else if t.StructType().Map.MapType().OldBucket == t { + // Both have non-nil Map + // Special case for Maps which include a recursive type where the recursion is not broken with a named type + if x.StructType().Map.MapType().OldBucket != x { + return CMPlt // bucket maps are least + } + return t.StructType().Map.cmp(x.StructType().Map) + } else if x.StructType().Map.MapType().OldBucket == x { + return CMPgt // bucket maps are least + } // If t != t.Map.OldBucket, fall through to general case + } tfs := t.Fields() xfs := x.Fields() diff --git a/src/cmd/compile/internal/walk/builtin.go b/src/cmd/compile/internal/walk/builtin.go index 02e64c12a1..c4147b2e2e 100644 --- a/src/cmd/compile/internal/walk/builtin.go +++ b/src/cmd/compile/internal/walk/builtin.go @@ -9,6 +9,7 @@ import ( "go/constant" "go/token" "internal/abi" + "internal/buildcfg" "strings" "cmd/compile/internal/base" @@ -311,8 +312,110 @@ func walkMakeChan(n *ir.MakeExpr, init *ir.Nodes) ir.Node { // walkMakeMap walks an OMAKEMAP node. func walkMakeMap(n *ir.MakeExpr, init *ir.Nodes) ir.Node { + if buildcfg.Experiment.SwissMap { + return walkMakeSwissMap(n, init) + } + return walkMakeOldMap(n, init) +} + +func walkMakeSwissMap(n *ir.MakeExpr, init *ir.Nodes) ir.Node { + t := n.Type() + hmapType := reflectdata.SwissMapType() + hint := n.Len + + // var h *hmap + var h ir.Node + if n.Esc() == ir.EscNone { + // Allocate hmap on stack. + + // var hv hmap + // h = &hv + h = stackTempAddr(init, hmapType) + + // Allocate one bucket pointed to by hmap.buckets on stack if hint + // is not larger than BUCKETSIZE. In case hint is larger than + // BUCKETSIZE runtime.makemap will allocate the buckets on the heap. + // Maximum key and elem size is 128 bytes, larger objects + // are stored with an indirection. So max bucket size is 2048+eps. + if !ir.IsConst(hint, constant.Int) || + constant.Compare(hint.Val(), token.LEQ, constant.MakeInt64(abi.SwissMapBucketCount)) { + + // In case hint is larger than BUCKETSIZE runtime.makemap + // will allocate the buckets on the heap, see #20184 + // + // if hint <= BUCKETSIZE { + // var bv bmap + // b = &bv + // h.buckets = b + // } + + nif := ir.NewIfStmt(base.Pos, ir.NewBinaryExpr(base.Pos, ir.OLE, hint, ir.NewInt(base.Pos, abi.SwissMapBucketCount)), nil, nil) + nif.Likely = true + + // var bv bmap + // b = &bv + b := stackTempAddr(&nif.Body, reflectdata.SwissMapBucketType(t)) + + // h.buckets = b + bsym := hmapType.Field(5).Sym // hmap.buckets see reflect.go:hmap + na := ir.NewAssignStmt(base.Pos, ir.NewSelectorExpr(base.Pos, ir.ODOT, h, bsym), typecheck.ConvNop(b, types.Types[types.TUNSAFEPTR])) + nif.Body.Append(na) + appendWalkStmt(init, nif) + } + } + + if ir.IsConst(hint, constant.Int) && constant.Compare(hint.Val(), token.LEQ, constant.MakeInt64(abi.SwissMapBucketCount)) { + // Handling make(map[any]any) and + // make(map[any]any, hint) where hint <= BUCKETSIZE + // special allows for faster map initialization and + // improves binary size by using calls with fewer arguments. + // For hint <= BUCKETSIZE overLoadFactor(hint, 0) is false + // and no buckets will be allocated by makemap. Therefore, + // no buckets need to be allocated in this code path. + if n.Esc() == ir.EscNone { + // Only need to initialize h.hash0 since + // hmap h has been allocated on the stack already. + // h.hash0 = rand32() + rand := mkcall("rand32", types.Types[types.TUINT32], init) + hashsym := hmapType.Field(4).Sym // hmap.hash0 see reflect.go:hmap + appendWalkStmt(init, ir.NewAssignStmt(base.Pos, ir.NewSelectorExpr(base.Pos, ir.ODOT, h, hashsym), rand)) + return typecheck.ConvNop(h, t) + } + // Call runtime.makehmap to allocate an + // hmap on the heap and initialize hmap's hash0 field. + fn := typecheck.LookupRuntime("makemap_small", t.Key(), t.Elem()) + return mkcall1(fn, n.Type(), init) + } + + if n.Esc() != ir.EscNone { + h = typecheck.NodNil() + } + // Map initialization with a variable or large hint is + // more complicated. We therefore generate a call to + // runtime.makemap to initialize hmap and allocate the + // map buckets. + + // When hint fits into int, use makemap instead of + // makemap64, which is faster and shorter on 32 bit platforms. + fnname := "makemap64" + argtype := types.Types[types.TINT64] + + // Type checking guarantees that TIDEAL hint is positive and fits in an int. + // See checkmake call in TMAP case of OMAKE case in OpSwitch in typecheck1 function. + // The case of hint overflow when converting TUINT or TUINTPTR to TINT + // will be handled by the negative range checks in makemap during runtime. + if hint.Type().IsKind(types.TIDEAL) || hint.Type().Size() <= types.Types[types.TUINT].Size() { + fnname = "makemap" + argtype = types.Types[types.TINT] + } + + fn := typecheck.LookupRuntime(fnname, hmapType, t.Key(), t.Elem()) + return mkcall1(fn, n.Type(), init, reflectdata.MakeMapRType(base.Pos, n), typecheck.Conv(hint, argtype), h) +} + +func walkMakeOldMap(n *ir.MakeExpr, init *ir.Nodes) ir.Node { t := n.Type() - hmapType := reflectdata.MapType() + hmapType := reflectdata.OldMapType() hint := n.Len // var h *hmap @@ -330,7 +433,7 @@ func walkMakeMap(n *ir.MakeExpr, init *ir.Nodes) ir.Node { // Maximum key and elem size is 128 bytes, larger objects // are stored with an indirection. So max bucket size is 2048+eps. if !ir.IsConst(hint, constant.Int) || - constant.Compare(hint.Val(), token.LEQ, constant.MakeInt64(abi.MapBucketCount)) { + constant.Compare(hint.Val(), token.LEQ, constant.MakeInt64(abi.OldMapBucketCount)) { // In case hint is larger than BUCKETSIZE runtime.makemap // will allocate the buckets on the heap, see #20184 @@ -341,12 +444,12 @@ func walkMakeMap(n *ir.MakeExpr, init *ir.Nodes) ir.Node { // h.buckets = b // } - nif := ir.NewIfStmt(base.Pos, ir.NewBinaryExpr(base.Pos, ir.OLE, hint, ir.NewInt(base.Pos, abi.MapBucketCount)), nil, nil) + nif := ir.NewIfStmt(base.Pos, ir.NewBinaryExpr(base.Pos, ir.OLE, hint, ir.NewInt(base.Pos, abi.OldMapBucketCount)), nil, nil) nif.Likely = true // var bv bmap // b = &bv - b := stackTempAddr(&nif.Body, reflectdata.MapBucketType(t)) + b := stackTempAddr(&nif.Body, reflectdata.OldMapBucketType(t)) // h.buckets = b bsym := hmapType.Field(5).Sym // hmap.buckets see reflect.go:hmap @@ -356,7 +459,7 @@ func walkMakeMap(n *ir.MakeExpr, init *ir.Nodes) ir.Node { } } - if ir.IsConst(hint, constant.Int) && constant.Compare(hint.Val(), token.LEQ, constant.MakeInt64(abi.MapBucketCount)) { + if ir.IsConst(hint, constant.Int) && constant.Compare(hint.Val(), token.LEQ, constant.MakeInt64(abi.OldMapBucketCount)) { // Handling make(map[any]any) and // make(map[any]any, hint) where hint <= BUCKETSIZE // special allows for faster map initialization and diff --git a/src/cmd/compile/internal/walk/order.go b/src/cmd/compile/internal/walk/order.go index de180a4a8d..896088901e 100644 --- a/src/cmd/compile/internal/walk/order.go +++ b/src/cmd/compile/internal/walk/order.go @@ -7,6 +7,7 @@ package walk import ( "fmt" "go/constant" + "internal/buildcfg" "cmd/compile/internal/base" "cmd/compile/internal/ir" @@ -926,7 +927,11 @@ func (o *orderState) stmt(n ir.Node) { // n.Prealloc is the temp for the iterator. // MapIterType contains pointers and needs to be zeroed. - n.Prealloc = o.newTemp(reflectdata.MapIterType(), true) + if buildcfg.Experiment.SwissMap { + n.Prealloc = o.newTemp(reflectdata.SwissMapIterType(), true) + } else { + n.Prealloc = o.newTemp(reflectdata.OldMapIterType(), true) + } } n.Key = o.exprInPlace(n.Key) n.Value = o.exprInPlace(n.Value) diff --git a/src/cmd/compile/internal/walk/walk.go b/src/cmd/compile/internal/walk/walk.go index 439f3ac71b..3c7af883c1 100644 --- a/src/cmd/compile/internal/walk/walk.go +++ b/src/cmd/compile/internal/walk/walk.go @@ -7,6 +7,7 @@ package walk import ( "fmt" "internal/abi" + "internal/buildcfg" "cmd/compile/internal/base" "cmd/compile/internal/ir" @@ -184,7 +185,42 @@ var mapassign = mkmapnames("mapassign", "ptr") var mapdelete = mkmapnames("mapdelete", "") func mapfast(t *types.Type) int { - if t.Elem().Size() > abi.MapMaxElemBytes { + if buildcfg.Experiment.SwissMap { + return mapfastSwiss(t) + } + return mapfastOld(t) +} + +func mapfastSwiss(t *types.Type) int { + if t.Elem().Size() > abi.SwissMapMaxElemBytes { + return mapslow + } + switch reflectdata.AlgType(t.Key()) { + case types.AMEM32: + if !t.Key().HasPointers() { + return mapfast32 + } + if types.PtrSize == 4 { + return mapfast32ptr + } + base.Fatalf("small pointer %v", t.Key()) + case types.AMEM64: + if !t.Key().HasPointers() { + return mapfast64 + } + if types.PtrSize == 8 { + return mapfast64ptr + } + // Two-word object, at least one of which is a pointer. + // Use the slow path. + case types.ASTRING: + return mapfaststr + } + return mapslow +} + +func mapfastOld(t *types.Type) int { + if t.Elem().Size() > abi.OldMapMaxElemBytes { return mapslow } switch reflectdata.AlgType(t.Key()) { |
