From 7f6418bb4e44b1b11d428d402dd6935a2a1ea335 Mon Sep 17 00:00:00 2001 From: cuishuang Date: Wed, 7 Jan 2026 13:46:43 +0800 Subject: all: fix misspellings in comments Change-Id: I121847e7f68c602dd8e9ecddfc41b547f8a86f10 Reviewed-on: https://go-review.googlesource.com/c/go/+/734361 LUCI-TryBot-Result: Go LUCI Reviewed-by: Robert Griesemer Reviewed-by: Junyang Shao --- src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go | 2 +- src/simd/archsimd/_gen/simdgen/gen_simdssa.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'src/simd') diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go b/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go index 3d99dd2a81..94b122ac39 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go @@ -181,7 +181,7 @@ func writeSIMDMachineOps(ops []Operation) *bytes.Buffer { } hasMerging = gOp.hasMaskedMerging(maskType, shapeOut) if hasMerging && !resultInArg0 { - // We have to copy the slice here becasue the sort will be visible from other + // We have to copy the slice here because the sort will be visible from other // aliases when no reslicing is happening. newIn := make([]Operand, len(op.In), len(op.In)+1) copy(newIn, op.In) diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdssa.go b/src/simd/archsimd/_gen/simdgen/gen_simdssa.go index 876ffabe3d..96d096688f 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdssa.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdssa.go @@ -133,7 +133,7 @@ func writeSIMDSSA(ops []Operation) *bytes.Buffer { if mem == NoMem && op.hasMaskedMerging(maskType, shapeOut) { regShapeMerging := regShape if shapeOut != OneVregOutAtIn { - // We have to copy the slice here becasue the sort will be visible from other + // We have to copy the slice here because the sort will be visible from other // aliases when no reslicing is happening. newIn := make([]Operand, len(op.In), len(op.In)+1) copy(newIn, op.In) -- cgit v1.3 From 9ef1692c93bf96328bcaf7a5c8a46094748da7f3 Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 13 Jan 2026 09:34:53 -0500 Subject: simd/archsimd/_gen/simdgen: feature implications This simplifies our handling of XED features, adds a table of which features imply which other features, and adds this information to the documentation of the CPU features APIs. As part of this we fix an issue around the "AVXAES" feature. AVXAES is defined as the combination of the AVX and AES CPUID flags. Several other features also work like this, but have hand-written logic in internal/cpu to compute logical feature flags from the underlying CPUID bits. For these, we expose a single feature check function from the SIMD API. AVXAES currently doesn't work like this: it requires the user to check both features. However, this forces the SIMD API to expose an "AES" feature check, which really has nothing to do with SIMD. To make this consistent, we introduce an AVXAES feature check function and use it in feature requirement docs. Unlike the others combo features, this is implemented in the simd package, but the difference is invisible to the user. Change-Id: I2985ebd361f0ecd45fd428903efe4c981a5ec65d Reviewed-on: https://go-review.googlesource.com/c/go/+/736100 LUCI-TryBot-Result: Go LUCI Reviewed-by: Cherry Mui Reviewed-by: Junyang Shao Reviewed-on: https://go-review.googlesource.com/c/go/+/736200 Reviewed-by: Austin Clements --- src/simd/archsimd/_gen/simdgen/gen_simdTypes.go | 97 ++++++++++++++++++- src/simd/archsimd/_gen/simdgen/xed.go | 121 +++++++++++++++--------- src/simd/archsimd/cpu.go | 38 ++++++-- src/simd/archsimd/ops_amd64.go | 12 +-- 4 files changed, 205 insertions(+), 63 deletions(-) (limited to 'src/simd') diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go index dd3a75eb44..4f1c70e211 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go @@ -189,6 +189,7 @@ type X86Features struct {} var X86 X86Features {{range .}} +{{$f := .}} {{- if eq .Feature "AVX512"}} // {{.Feature}} returns whether the CPU supports the AVX512F+CD+BW+DQ+VL features. // @@ -199,11 +200,19 @@ var X86 X86Features {{- else -}} // {{.Feature}} returns whether the CPU supports the {{.Feature}} feature. {{- end}} +{{- if ne .ImpliesAll ""}} +// +// If it returns true, then the CPU also supports {{.ImpliesAll}}. +{{- end}} // // {{.Feature}} is defined on all GOARCHes, but will only return true on // GOARCH {{.GoArch}}. -func (X86Features) {{.Feature}}() bool { - return cpu.X86.Has{{.Feature}} +func ({{.FeatureVar}}Features) {{.Feature}}() bool { +{{- if .Virtual}} + return {{range $i, $dep := .Implies}}{{if $i}} && {{end}}cpu.{{$f.FeatureVar}}.Has{{$dep}}{{end}} +{{- else}} + return cpu.{{.FeatureVar}}.Has{{.Feature}} +{{- end}} } {{end}} ` @@ -591,6 +600,65 @@ func writeSIMDTypes(typeMap simdTypeMap) *bytes.Buffer { return buffer } +type goarchFeatures struct { + // featureVar is the name of the exported feature-check variable for this + // architecture. + featureVar string + + // features records per-feature information. + features map[string]featureInfo +} + +type featureInfo struct { + // Implies is a list of other CPU features that are required for this + // feature. These are allowed to chain. + // + // For example, if the Frob feature lists "Baz", then if X.Frob() returns + // true, it must also be true that the CPU has feature Baz. + Implies []string + + // Virtual means this feature is not represented directly in internal/cpu, + // but is instead the logical AND of the features in Implies. + Virtual bool +} + +// goarchFeatureInfo maps from GOARCH to CPU feature to additional information +// about that feature. Not all features need to be in this map. +var goarchFeatureInfo = make(map[string]goarchFeatures) + +func registerFeatureInfo(goArch string, features goarchFeatures) { + goarchFeatureInfo[goArch] = features +} + +func featureImplies(goarch string, base string) string { + // Compute the transitive closure of base. + var list []string + var visit func(f string) + visit = func(f string) { + list = append(list, f) + for _, dep := range goarchFeatureInfo[goarch].features[f].Implies { + visit(dep) + } + } + visit(base) + // Drop base + list = list[1:] + // Put in "nice" order + slices.Reverse(list) + // Combine into a comment-ready form + switch len(list) { + case 0: + return "" + case 1: + return list[0] + case 2: + return list[0] + " and " + list[1] + default: + list[len(list)-1] = "and " + list[len(list)-1] + return strings.Join(list, ", ") + } +} + func writeSIMDFeatures(ops []Operation) *bytes.Buffer { // Gather all features type featureKey struct { @@ -606,13 +674,36 @@ func writeSIMDFeatures(ops []Operation) *bytes.Buffer { featureSet[featureKey{op.GoArch, feature}] = struct{}{} } } - features := slices.SortedFunc(maps.Keys(featureSet), func(a, b featureKey) int { + featureKeys := slices.SortedFunc(maps.Keys(featureSet), func(a, b featureKey) int { if c := cmp.Compare(a.GoArch, b.GoArch); c != 0 { return c } return compareNatural(a.Feature, b.Feature) }) + // TODO: internal/cpu doesn't enforce these at all. You can even do + // GODEBUG=cpu.avx=off and it will happily turn off AVX without turning off + // AVX2. We need to push these dependencies into it somehow. + type feature struct { + featureKey + FeatureVar string + Virtual bool + Implies []string + ImpliesAll string + } + var features []feature + for _, k := range featureKeys { + featureVar := goarchFeatureInfo[k.GoArch].featureVar + fi := goarchFeatureInfo[k.GoArch].features[k.Feature] + features = append(features, feature{ + featureKey: k, + FeatureVar: featureVar, + Virtual: fi.Virtual, + Implies: fi.Implies, + ImpliesAll: featureImplies(k.GoArch, k.Feature), + }) + } + // If we ever have the same feature name on more than one GOARCH, we'll have // to be more careful about this. t := templateOf(simdFeaturesTemplate, "features") diff --git a/src/simd/archsimd/_gen/simdgen/xed.go b/src/simd/archsimd/_gen/simdgen/xed.go index 4ba6738e7e..49bedc9b28 100644 --- a/src/simd/archsimd/_gen/simdgen/xed.go +++ b/src/simd/archsimd/_gen/simdgen/xed.go @@ -5,7 +5,6 @@ package main import ( - "cmp" "fmt" "log" "maps" @@ -210,16 +209,9 @@ func loadXED(xedPath string) []*unify.Value { } log.Printf("%d unhandled CPU features for %d instructions (use -v for details)", len(unknownFeatures), nInst) } else { - keys := slices.SortedFunc(maps.Keys(unknownFeatures), func(a, b cpuFeatureKey) int { - return cmp.Or(cmp.Compare(a.Extension, b.Extension), - cmp.Compare(a.ISASet, b.ISASet)) - }) + keys := slices.Sorted(maps.Keys(unknownFeatures)) for _, key := range keys { - if key.ISASet == "" || key.ISASet == key.Extension { - log.Printf("unhandled Extension %s", key.Extension) - } else { - log.Printf("unhandled Extension %s and ISASet %s", key.Extension, key.ISASet) - } + log.Printf("unhandled ISASet %s", key) log.Printf(" opcodes: %s", slices.Sorted(maps.Keys(unknownFeatures[key]))) } } @@ -763,16 +755,24 @@ func instToUVal1(inst *xeddata.Inst, ops []operand, feature string, variant inst // decodeCPUFeature returns the CPU feature name required by inst. These match // the names of the "Has*" feature checks in the simd package. func decodeCPUFeature(inst *xeddata.Inst) (string, bool) { - key := cpuFeatureKey{ - Extension: inst.Extension, - ISASet: isaSetStrip.ReplaceAllLiteralString(inst.ISASet, ""), + isaSet := inst.ISASet + if isaSet == "" { + // Older instructions don't have an ISA set. Use their "extension" + // instead. + isaSet = inst.Extension + } + // We require AVX512VL to use AVX512 at all, so strip off the vector length + // suffixes. + if strings.HasPrefix(isaSet, "AVX512") { + isaSet = isaSetVL.ReplaceAllLiteralString(isaSet, "") } - feat, ok := cpuFeatureMap[key] + + feat, ok := cpuFeatureMap[isaSet] if !ok { - imap := unknownFeatures[key] + imap := unknownFeatures[isaSet] if imap == nil { imap = make(map[string]struct{}) - unknownFeatures[key] = imap + unknownFeatures[isaSet] = imap } imap[inst.Opcode()] = struct{}{} return "", false @@ -783,45 +783,74 @@ func decodeCPUFeature(inst *xeddata.Inst) (string, bool) { return feat, true } -var isaSetStrip = regexp.MustCompile("_(128N?|256N?|512)$") - -type cpuFeatureKey struct { - Extension, ISASet string -} - -// cpuFeatureMap maps from XED's "EXTENSION" and "ISA_SET" to a CPU feature name -// that can be used in the SIMD API. -var cpuFeatureMap = map[cpuFeatureKey]string{ - {"SHA", "SHA"}: "SHA", +var isaSetVL = regexp.MustCompile("_(128N?|256N?|512)$") - {"AVX", ""}: "AVX", - {"AVX_VNNI", "AVX_VNNI"}: "AVXVNNI", - {"AVX2", ""}: "AVX2", - {"AVXAES", ""}: "AVX, AES", +// cpuFeatureMap maps from XED's "ISA_SET" (or "EXTENSION") to a CPU feature +// name to expose in the SIMD feature check API. +// +// See XED's datafiles/*/cpuid.xed.txt for how ISA set names map to CPUID flags. +var cpuFeatureMap = map[string]string{ + "AVX": "AVX", + "AVX_VNNI": "AVXVNNI", + "AVX2": "AVX2", + "AVXAES": "AVXAES", + "SHA": "SHA", // AVX-512 foundational features. We combine all of these into one "AVX512" feature. - {"AVX512EVEX", "AVX512F"}: "AVX512", - {"AVX512EVEX", "AVX512CD"}: "AVX512", - {"AVX512EVEX", "AVX512BW"}: "AVX512", - {"AVX512EVEX", "AVX512DQ"}: "AVX512", - // AVX512VL doesn't appear explicitly in the ISASet. I guess it's implied by - // the vector length suffix. + "AVX512F": "AVX512", + "AVX512BW": "AVX512", + "AVX512CD": "AVX512", + "AVX512DQ": "AVX512", + // AVX512VL doesn't appear as its own ISASet; instead, the CPUID flag is + // required by the *_128 and *_256 ISASets. We fold it into "AVX512" anyway. // AVX-512 extension features - {"AVX512EVEX", "AVX512_BITALG"}: "AVX512BITALG", - {"AVX512EVEX", "AVX512_GFNI"}: "AVX512GFNI", - {"AVX512EVEX", "AVX512_VBMI2"}: "AVX512VBMI2", - {"AVX512EVEX", "AVX512_VBMI"}: "AVX512VBMI", - {"AVX512EVEX", "AVX512_VNNI"}: "AVX512VNNI", - {"AVX512EVEX", "AVX512_VPOPCNTDQ"}: "AVX512VPOPCNTDQ", - {"AVX512EVEX", "AVX512_VAES"}: "AVX512VAES", - {"AVX512EVEX", "AVX512_VPCLMULQDQ"}: "AVX512VPCLMULQDQ", + "AVX512_BITALG": "AVX512BITALG", + "AVX512_GFNI": "AVX512GFNI", + "AVX512_VBMI": "AVX512VBMI", + "AVX512_VBMI2": "AVX512VBMI2", + "AVX512_VNNI": "AVX512VNNI", + "AVX512_VPOPCNTDQ": "AVX512VPOPCNTDQ", + "AVX512_VAES": "AVX512VAES", + "AVX512_VPCLMULQDQ": "AVX512VPCLMULQDQ", // AVX 10.2 (not yet supported) - {"AVX512EVEX", "AVX10_2_RC"}: "ignore", + "AVX10_2_RC": "ignore", +} + +func init() { + // TODO: In general, Intel doesn't make any guarantees about what flags are + // set, so this means our feature checks need to ensure these, just to be + // sure. + var features = map[string]featureInfo{ + "AVX2": {Implies: []string{"AVX"}}, + "AVX512": {Implies: []string{"AVX2"}}, + + "AVXAES": {Virtual: true, Implies: []string{"AVX", "AES"}}, + + // AVX-512 subfeatures. + "AVX512BITALG": {Implies: []string{"AVX512"}}, + "AVX512GFNI": {Implies: []string{"AVX512"}}, + "AVX512VBMI": {Implies: []string{"AVX512"}}, + "AVX512VBMI2": {Implies: []string{"AVX512"}}, + "AVX512VNNI": {Implies: []string{"AVX512"}}, + "AVX512VPOPCNTDQ": {Implies: []string{"AVX512"}}, + "AVX512VAES": {Implies: []string{"AVX512"}}, + + // AVX-VNNI and AVX-IFMA are "backports" of the AVX512-VNNI/IFMA + // instructions to VEX encoding, limited to 256 bit vectors. They're + // intended for lower end CPUs that want to support VNNI/IFMA without + // supporting AVX-512. As such, they're built on AVX2's VEX encoding. + "AVXVNNI": {Implies: []string{"AVX2"}}, + "AVXIFMA": {Implies: []string{"AVX2"}}, + } + registerFeatureInfo("amd64", goarchFeatures{ + featureVar: "X86", + features: features, + }) } -var unknownFeatures = map[cpuFeatureKey]map[string]struct{}{} +var unknownFeatures = map[string]map[string]struct{}{} // hasOptionalMask returns whether there is an optional mask operand in ops. func hasOptionalMask(ops []operand) bool { diff --git a/src/simd/archsimd/cpu.go b/src/simd/archsimd/cpu.go index d0c0ff5426..1cd765268f 100644 --- a/src/simd/archsimd/cpu.go +++ b/src/simd/archsimd/cpu.go @@ -10,14 +10,6 @@ type X86Features struct{} var X86 X86Features -// AES returns whether the CPU supports the AES feature. -// -// AES is defined on all GOARCHes, but will only return true on -// GOARCH amd64. -func (X86Features) AES() bool { - return cpu.X86.HasAES -} - // AVX returns whether the CPU supports the AVX feature. // // AVX is defined on all GOARCHes, but will only return true on @@ -28,6 +20,8 @@ func (X86Features) AVX() bool { // AVX2 returns whether the CPU supports the AVX2 feature. // +// If it returns true, then the CPU also supports AVX. +// // AVX2 is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX2() bool { @@ -41,6 +35,8 @@ func (X86Features) AVX2() bool { // Nearly every CPU that has shipped with any support for AVX-512 has // supported all five of these features. // +// If it returns true, then the CPU also supports AVX and AVX2. +// // AVX512 is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512() bool { @@ -49,6 +45,8 @@ func (X86Features) AVX512() bool { // AVX512BITALG returns whether the CPU supports the AVX512BITALG feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512BITALG is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512BITALG() bool { @@ -57,6 +55,8 @@ func (X86Features) AVX512BITALG() bool { // AVX512GFNI returns whether the CPU supports the AVX512GFNI feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512GFNI is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512GFNI() bool { @@ -65,6 +65,8 @@ func (X86Features) AVX512GFNI() bool { // AVX512VAES returns whether the CPU supports the AVX512VAES feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512VAES is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512VAES() bool { @@ -73,6 +75,8 @@ func (X86Features) AVX512VAES() bool { // AVX512VBMI returns whether the CPU supports the AVX512VBMI feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512VBMI is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512VBMI() bool { @@ -81,6 +85,8 @@ func (X86Features) AVX512VBMI() bool { // AVX512VBMI2 returns whether the CPU supports the AVX512VBMI2 feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512VBMI2 is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512VBMI2() bool { @@ -89,6 +95,8 @@ func (X86Features) AVX512VBMI2() bool { // AVX512VNNI returns whether the CPU supports the AVX512VNNI feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512VNNI is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512VNNI() bool { @@ -105,14 +113,28 @@ func (X86Features) AVX512VPCLMULQDQ() bool { // AVX512VPOPCNTDQ returns whether the CPU supports the AVX512VPOPCNTDQ feature. // +// If it returns true, then the CPU also supports AVX, AVX2, and AVX512. +// // AVX512VPOPCNTDQ is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVX512VPOPCNTDQ() bool { return cpu.X86.HasAVX512VPOPCNTDQ } +// AVXAES returns whether the CPU supports the AVXAES feature. +// +// If it returns true, then the CPU also supports AES and AVX. +// +// AVXAES is defined on all GOARCHes, but will only return true on +// GOARCH amd64. +func (X86Features) AVXAES() bool { + return cpu.X86.HasAVX && cpu.X86.HasAES +} + // AVXVNNI returns whether the CPU supports the AVXVNNI feature. // +// If it returns true, then the CPU also supports AVX and AVX2. +// // AVXVNNI is defined on all GOARCHes, but will only return true on // GOARCH amd64. func (X86Features) AVXVNNI() bool { diff --git a/src/simd/archsimd/ops_amd64.go b/src/simd/archsimd/ops_amd64.go index bb162c4ff9..d65c7dde0c 100644 --- a/src/simd/archsimd/ops_amd64.go +++ b/src/simd/archsimd/ops_amd64.go @@ -11,7 +11,7 @@ package archsimd // y is the chunk of dw array in use. // result = AddRoundKey(InvShiftRows(InvSubBytes(x)), y) // -// Asm: VAESDECLAST, CPU Feature: AVX, AES +// Asm: VAESDECLAST, CPU Feature: AVXAES func (x Uint8x16) AESDecryptLastRound(y Uint32x4) Uint8x16 // AESDecryptLastRound performs a series of operations in AES cipher algorithm defined in FIPS 197. @@ -37,7 +37,7 @@ func (x Uint8x64) AESDecryptLastRound(y Uint32x16) Uint8x64 // y is the chunk of dw array in use. // result = AddRoundKey(InvMixColumns(InvShiftRows(InvSubBytes(x))), y) // -// Asm: VAESDEC, CPU Feature: AVX, AES +// Asm: VAESDEC, CPU Feature: AVXAES func (x Uint8x16) AESDecryptOneRound(y Uint32x4) Uint8x16 // AESDecryptOneRound performs a series of operations in AES cipher algorithm defined in FIPS 197. @@ -63,7 +63,7 @@ func (x Uint8x64) AESDecryptOneRound(y Uint32x16) Uint8x64 // y is the chunk of w array in use. // result = AddRoundKey((ShiftRows(SubBytes(x))), y) // -// Asm: VAESENCLAST, CPU Feature: AVX, AES +// Asm: VAESENCLAST, CPU Feature: AVXAES func (x Uint8x16) AESEncryptLastRound(y Uint32x4) Uint8x16 // AESEncryptLastRound performs a series of operations in AES cipher algorithm defined in FIPS 197. @@ -89,7 +89,7 @@ func (x Uint8x64) AESEncryptLastRound(y Uint32x16) Uint8x64 // y is the chunk of w array in use. // result = AddRoundKey(MixColumns(ShiftRows(SubBytes(x))), y) // -// Asm: VAESENC, CPU Feature: AVX, AES +// Asm: VAESENC, CPU Feature: AVXAES func (x Uint8x16) AESEncryptOneRound(y Uint32x4) Uint8x16 // AESEncryptOneRound performs a series of operations in AES cipher algorithm defined in FIPS 197. @@ -114,7 +114,7 @@ func (x Uint8x64) AESEncryptOneRound(y Uint32x16) Uint8x64 // x is the chunk of w array in use. // result = InvMixColumns(x) // -// Asm: VAESIMC, CPU Feature: AVX, AES +// Asm: VAESIMC, CPU Feature: AVXAES func (x Uint32x4) AESInvMixColumns() Uint32x4 /* AESRoundKeyGenAssist */ @@ -129,7 +129,7 @@ func (x Uint32x4) AESInvMixColumns() Uint32x4 // // rconVal results in better performance when it's a constant, a non-constant value will be translated into a jump table. // -// Asm: VAESKEYGENASSIST, CPU Feature: AVX, AES +// Asm: VAESKEYGENASSIST, CPU Feature: AVXAES func (x Uint32x4) AESRoundKeyGenAssist(rconVal uint8) Uint32x4 /* Abs */ -- cgit v1.3 From e2429619605951b137e25f6a51fbc39d9f0f1e9b Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Tue, 13 Jan 2026 14:18:14 -0500 Subject: simd/archsimd: 128- and 256-bit FMA operations do not require AVX-512 Currently, all FMA operations are marked as requiring AVX512, even on smaller vector widths. This is happening because the narrower FMA operations are marked as extension "FMA" in the XED. Since this extension doesn't start with "AVX", we filter them out very early in the XED process. However, this is just a quirk of naming: the FMA feature depends on the AVX feature, so it is part of AVX, even if it doesn't say so on the tin. Fix this by accepting the FMA extension and adding FMA to the table of CPU features. We also tweak internal/cpu slightly do it correctly enforces that the logical FMA feature depends on both the FMA and AVX CPUID flags. This actually *deletes* a lot of generated code because we no longer need the AVX-512 encoding of these 128- and 256-bit operations. Change-Id: I744a18d0be888f536ac034fe88b110347622be7e Reviewed-on: https://go-review.googlesource.com/c/go/+/736160 Auto-Submit: Austin Clements Reviewed-by: Junyang Shao LUCI-TryBot-Result: Go LUCI Reviewed-on: https://go-review.googlesource.com/c/go/+/736201 Reviewed-by: Austin Clements --- src/cmd/compile/internal/amd64/simdssa.go | 12 - src/cmd/compile/internal/ssa/_gen/simdAMD64.rules | 12 - src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go | 36 +-- src/cmd/compile/internal/ssa/opGen.go | 324 +++---------------- src/cmd/compile/internal/ssa/rewriteAMD64.go | 372 ---------------------- src/internal/cpu/cpu_x86.go | 14 +- src/simd/archsimd/_gen/simdgen/xed.go | 4 +- src/simd/archsimd/cpu.go | 10 + src/simd/archsimd/ops_amd64.go | 24 +- 9 files changed, 93 insertions(+), 715 deletions(-) (limited to 'src/simd') diff --git a/src/cmd/compile/internal/amd64/simdssa.go b/src/cmd/compile/internal/amd64/simdssa.go index a028cbe86d..ea33808a1c 100644 --- a/src/cmd/compile/internal/amd64/simdssa.go +++ b/src/cmd/compile/internal/amd64/simdssa.go @@ -1959,23 +1959,11 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssa.OpAMD64VPERMI2Q256load, ssa.OpAMD64VPERMI2PD512load, ssa.OpAMD64VPERMI2Q512load, - ssa.OpAMD64VFMADD213PS128load, - ssa.OpAMD64VFMADD213PS256load, ssa.OpAMD64VFMADD213PS512load, - ssa.OpAMD64VFMADD213PD128load, - ssa.OpAMD64VFMADD213PD256load, ssa.OpAMD64VFMADD213PD512load, - ssa.OpAMD64VFMADDSUB213PS128load, - ssa.OpAMD64VFMADDSUB213PS256load, ssa.OpAMD64VFMADDSUB213PS512load, - ssa.OpAMD64VFMADDSUB213PD128load, - ssa.OpAMD64VFMADDSUB213PD256load, ssa.OpAMD64VFMADDSUB213PD512load, - ssa.OpAMD64VFMSUBADD213PS128load, - ssa.OpAMD64VFMSUBADD213PS256load, ssa.OpAMD64VFMSUBADD213PS512load, - ssa.OpAMD64VFMSUBADD213PD128load, - ssa.OpAMD64VFMSUBADD213PD256load, ssa.OpAMD64VFMSUBADD213PD512load, ssa.OpAMD64VPSHLDVD128load, ssa.OpAMD64VPSHLDVD256load, diff --git a/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules b/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules index 799461610d..63b37f99cb 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules @@ -2771,11 +2771,7 @@ (VPMULLQ128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPMULLQ128load {sym} [off] x ptr mem) (VPMULLQ256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPMULLQ256load {sym} [off] x ptr mem) (VPMULLQ512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VPMULLQ512load {sym} [off] x ptr mem) -(VFMADD213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PS128load {sym} [off] x y ptr mem) -(VFMADD213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PS256load {sym} [off] x y ptr mem) (VFMADD213PS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PS512load {sym} [off] x y ptr mem) -(VFMADD213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PD128load {sym} [off] x y ptr mem) -(VFMADD213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PD256load {sym} [off] x y ptr mem) (VFMADD213PD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PD512load {sym} [off] x y ptr mem) (VFMADD213PSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PSMasked128load {sym} [off] x y ptr mask mem) (VFMADD213PSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PSMasked256load {sym} [off] x y ptr mask mem) @@ -2783,11 +2779,7 @@ (VFMADD213PDMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PDMasked128load {sym} [off] x y ptr mask mem) (VFMADD213PDMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PDMasked256load {sym} [off] x y ptr mask mem) (VFMADD213PDMasked512 x y l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADD213PDMasked512load {sym} [off] x y ptr mask mem) -(VFMADDSUB213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PS128load {sym} [off] x y ptr mem) -(VFMADDSUB213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PS256load {sym} [off] x y ptr mem) (VFMADDSUB213PS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PS512load {sym} [off] x y ptr mem) -(VFMADDSUB213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PD128load {sym} [off] x y ptr mem) -(VFMADDSUB213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PD256load {sym} [off] x y ptr mem) (VFMADDSUB213PD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PD512load {sym} [off] x y ptr mem) (VFMADDSUB213PSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PSMasked128load {sym} [off] x y ptr mask mem) (VFMADDSUB213PSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMADDSUB213PSMasked256load {sym} [off] x y ptr mask mem) @@ -2807,11 +2799,7 @@ (VPMULLQMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPMULLQMasked128load {sym} [off] x ptr mask mem) (VPMULLQMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPMULLQMasked256load {sym} [off] x ptr mask mem) (VPMULLQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VPMULLQMasked512load {sym} [off] x ptr mask mem) -(VFMSUBADD213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PS128load {sym} [off] x y ptr mem) -(VFMSUBADD213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PS256load {sym} [off] x y ptr mem) (VFMSUBADD213PS512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PS512load {sym} [off] x y ptr mem) -(VFMSUBADD213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PD128load {sym} [off] x y ptr mem) -(VFMSUBADD213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PD256load {sym} [off] x y ptr mem) (VFMSUBADD213PD512 x y l:(VMOVDQUload512 {sym} [off] ptr mem)) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PD512load {sym} [off] x y ptr mem) (VFMSUBADD213PSMasked128 x y l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PSMasked128load {sym} [off] x y ptr mask mem) (VFMSUBADD213PSMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && canMergeLoad(v, l) && clobber(l) => (VFMSUBADD213PSMasked256load {sym} [off] x y ptr mask mem) diff --git a/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go b/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go index 648e372fb4..339ec7fce3 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go +++ b/src/cmd/compile/internal/ssa/_gen/simdAMD64ops.go @@ -172,38 +172,38 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf {name: "VEXPANDPSMasked128", argLength: 2, reg: wkw, asm: "VEXPANDPS", commutative: false, typ: "Vec128", resultInArg0: false}, {name: "VEXPANDPSMasked256", argLength: 2, reg: wkw, asm: "VEXPANDPS", commutative: false, typ: "Vec256", resultInArg0: false}, {name: "VEXPANDPSMasked512", argLength: 2, reg: wkw, asm: "VEXPANDPS", commutative: false, typ: "Vec512", resultInArg0: false}, - {name: "VFMADD213PD128", argLength: 3, reg: w31, asm: "VFMADD213PD", commutative: false, typ: "Vec128", resultInArg0: true}, - {name: "VFMADD213PD256", argLength: 3, reg: w31, asm: "VFMADD213PD", commutative: false, typ: "Vec256", resultInArg0: true}, + {name: "VFMADD213PD128", argLength: 3, reg: v31, asm: "VFMADD213PD", commutative: false, typ: "Vec128", resultInArg0: true}, + {name: "VFMADD213PD256", argLength: 3, reg: v31, asm: "VFMADD213PD", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADD213PD512", argLength: 3, reg: w31, asm: "VFMADD213PD", commutative: false, typ: "Vec512", resultInArg0: true}, {name: "VFMADD213PDMasked128", argLength: 4, reg: w3kw, asm: "VFMADD213PD", commutative: false, typ: "Vec128", resultInArg0: true}, {name: "VFMADD213PDMasked256", argLength: 4, reg: w3kw, asm: "VFMADD213PD", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADD213PDMasked512", argLength: 4, reg: w3kw, asm: "VFMADD213PD", commutative: false, typ: "Vec512", resultInArg0: true}, - {name: "VFMADD213PS128", argLength: 3, reg: w31, asm: "VFMADD213PS", commutative: false, typ: "Vec128", resultInArg0: true}, - {name: "VFMADD213PS256", argLength: 3, reg: w31, asm: "VFMADD213PS", commutative: false, typ: "Vec256", resultInArg0: true}, + {name: "VFMADD213PS128", argLength: 3, reg: v31, asm: "VFMADD213PS", commutative: false, typ: "Vec128", resultInArg0: true}, + {name: "VFMADD213PS256", argLength: 3, reg: v31, asm: "VFMADD213PS", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADD213PS512", argLength: 3, reg: w31, asm: "VFMADD213PS", commutative: false, typ: "Vec512", resultInArg0: true}, {name: "VFMADD213PSMasked128", argLength: 4, reg: w3kw, asm: "VFMADD213PS", commutative: false, typ: "Vec128", resultInArg0: true}, {name: "VFMADD213PSMasked256", argLength: 4, reg: w3kw, asm: "VFMADD213PS", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADD213PSMasked512", argLength: 4, reg: w3kw, asm: "VFMADD213PS", commutative: false, typ: "Vec512", resultInArg0: true}, - {name: "VFMADDSUB213PD128", argLength: 3, reg: w31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", resultInArg0: true}, - {name: "VFMADDSUB213PD256", argLength: 3, reg: w31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", resultInArg0: true}, + {name: "VFMADDSUB213PD128", argLength: 3, reg: v31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", resultInArg0: true}, + {name: "VFMADDSUB213PD256", argLength: 3, reg: v31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADDSUB213PD512", argLength: 3, reg: w31, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", resultInArg0: true}, {name: "VFMADDSUB213PDMasked128", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", resultInArg0: true}, {name: "VFMADDSUB213PDMasked256", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADDSUB213PDMasked512", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", resultInArg0: true}, - {name: "VFMADDSUB213PS128", argLength: 3, reg: w31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", resultInArg0: true}, - {name: "VFMADDSUB213PS256", argLength: 3, reg: w31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", resultInArg0: true}, + {name: "VFMADDSUB213PS128", argLength: 3, reg: v31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", resultInArg0: true}, + {name: "VFMADDSUB213PS256", argLength: 3, reg: v31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADDSUB213PS512", argLength: 3, reg: w31, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", resultInArg0: true}, {name: "VFMADDSUB213PSMasked128", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", resultInArg0: true}, {name: "VFMADDSUB213PSMasked256", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMADDSUB213PSMasked512", argLength: 4, reg: w3kw, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", resultInArg0: true}, - {name: "VFMSUBADD213PD128", argLength: 3, reg: w31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", resultInArg0: true}, - {name: "VFMSUBADD213PD256", argLength: 3, reg: w31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", resultInArg0: true}, + {name: "VFMSUBADD213PD128", argLength: 3, reg: v31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", resultInArg0: true}, + {name: "VFMSUBADD213PD256", argLength: 3, reg: v31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMSUBADD213PD512", argLength: 3, reg: w31, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", resultInArg0: true}, {name: "VFMSUBADD213PDMasked128", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", resultInArg0: true}, {name: "VFMSUBADD213PDMasked256", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMSUBADD213PDMasked512", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", resultInArg0: true}, - {name: "VFMSUBADD213PS128", argLength: 3, reg: w31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", resultInArg0: true}, - {name: "VFMSUBADD213PS256", argLength: 3, reg: w31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", resultInArg0: true}, + {name: "VFMSUBADD213PS128", argLength: 3, reg: v31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", resultInArg0: true}, + {name: "VFMSUBADD213PS256", argLength: 3, reg: v31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", resultInArg0: true}, {name: "VFMSUBADD213PS512", argLength: 3, reg: w31, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec512", resultInArg0: true}, {name: "VFMSUBADD213PSMasked128", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", resultInArg0: true}, {name: "VFMSUBADD213PSMasked256", argLength: 4, reg: w3kw, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", resultInArg0: true}, @@ -1594,38 +1594,26 @@ func simdAMD64Ops(v11, v21, v2k, vkv, v2kv, v2kk, v31, v3kv, vgpv, vgp, vfpv, vf {name: "VDIVPSMasked128load", argLength: 4, reg: w2kwload, asm: "VDIVPS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: false}, {name: "VDIVPSMasked256load", argLength: 4, reg: w2kwload, asm: "VDIVPS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: false}, {name: "VDIVPSMasked512load", argLength: 4, reg: w2kwload, asm: "VDIVPS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: false}, - {name: "VFMADD213PD128load", argLength: 4, reg: w31load, asm: "VFMADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADD213PD256load", argLength: 4, reg: w31load, asm: "VFMADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PD512load", argLength: 4, reg: w31load, asm: "VFMADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PDMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PDMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PDMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADD213PS128load", argLength: 4, reg: w31load, asm: "VFMADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADD213PS256load", argLength: 4, reg: w31load, asm: "VFMADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PS512load", argLength: 4, reg: w31load, asm: "VFMADD213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PSMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PSMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADD213PSMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADD213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADDSUB213PD128load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADDSUB213PD256load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PD512load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PDMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PDMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PDMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADDSUB213PS128load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMADDSUB213PS256load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PS512load", argLength: 4, reg: w31load, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PSMasked128load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PSMasked256load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMADDSUB213PSMasked512load", argLength: 5, reg: w3kwload, asm: "VFMADDSUB213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMSUBADD213PD128load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMSUBADD213PD256load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PD512load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PDMasked128load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PDMasked256load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PDMasked512load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PD", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMSUBADD213PS128load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, - {name: "VFMSUBADD213PS256load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PS512load", argLength: 4, reg: w31load, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec512", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PSMasked128load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec128", aux: "SymOff", symEffect: "Read", resultInArg0: true}, {name: "VFMSUBADD213PSMasked256load", argLength: 5, reg: w3kwload, asm: "VFMSUBADD213PS", commutative: false, typ: "Vec256", aux: "SymOff", symEffect: "Read", resultInArg0: true}, diff --git a/src/cmd/compile/internal/ssa/opGen.go b/src/cmd/compile/internal/ssa/opGen.go index 9e5fdb1fc1..a84fc161e9 100644 --- a/src/cmd/compile/internal/ssa/opGen.go +++ b/src/cmd/compile/internal/ssa/opGen.go @@ -2841,38 +2841,26 @@ const ( OpAMD64VDIVPSMasked128load OpAMD64VDIVPSMasked256load OpAMD64VDIVPSMasked512load - OpAMD64VFMADD213PD128load - OpAMD64VFMADD213PD256load OpAMD64VFMADD213PD512load OpAMD64VFMADD213PDMasked128load OpAMD64VFMADD213PDMasked256load OpAMD64VFMADD213PDMasked512load - OpAMD64VFMADD213PS128load - OpAMD64VFMADD213PS256load OpAMD64VFMADD213PS512load OpAMD64VFMADD213PSMasked128load OpAMD64VFMADD213PSMasked256load OpAMD64VFMADD213PSMasked512load - OpAMD64VFMADDSUB213PD128load - OpAMD64VFMADDSUB213PD256load OpAMD64VFMADDSUB213PD512load OpAMD64VFMADDSUB213PDMasked128load OpAMD64VFMADDSUB213PDMasked256load OpAMD64VFMADDSUB213PDMasked512load - OpAMD64VFMADDSUB213PS128load - OpAMD64VFMADDSUB213PS256load OpAMD64VFMADDSUB213PS512load OpAMD64VFMADDSUB213PSMasked128load OpAMD64VFMADDSUB213PSMasked256load OpAMD64VFMADDSUB213PSMasked512load - OpAMD64VFMSUBADD213PD128load - OpAMD64VFMSUBADD213PD256load OpAMD64VFMSUBADD213PD512load OpAMD64VFMSUBADD213PDMasked128load OpAMD64VFMSUBADD213PDMasked256load OpAMD64VFMSUBADD213PDMasked512load - OpAMD64VFMSUBADD213PS128load - OpAMD64VFMSUBADD213PS256load OpAMD64VFMSUBADD213PS512load OpAMD64VFMSUBADD213PSMasked128load OpAMD64VFMSUBADD213PSMasked256load @@ -23179,12 +23167,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADD213PD, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23195,12 +23183,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADD213PD, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23278,12 +23266,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADD213PS, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23294,12 +23282,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADD213PS, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23377,12 +23365,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADDSUB213PD, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23393,12 +23381,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADDSUB213PD, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23476,12 +23464,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADDSUB213PS, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23492,12 +23480,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMADDSUB213PS, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23575,12 +23563,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMSUBADD213PD, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23591,12 +23579,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMSUBADD213PD, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23674,12 +23662,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMSUBADD213PS, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -23690,12 +23678,12 @@ var opcodeTable = [...]opInfo{ asm: x86.AVFMSUBADD213PS, reg: regInfo{ inputs: []inputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {2, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 + {1, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 + {2, 4294901760}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 }, outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 + {0, 2147418112}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 }, }, }, @@ -44193,42 +44181,6 @@ var opcodeTable = [...]opInfo{ }, }, }, - { - name: "VFMADD213PD128load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADD213PD, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, - { - name: "VFMADD213PD256load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADD213PD, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, { name: "VFMADD213PD512load", auxType: auxSymOff, @@ -44304,42 +44256,6 @@ var opcodeTable = [...]opInfo{ }, }, }, - { - name: "VFMADD213PS128load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADD213PS, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, - { - name: "VFMADD213PS256load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADD213PS, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, { name: "VFMADD213PS512load", auxType: auxSymOff, @@ -44415,42 +44331,6 @@ var opcodeTable = [...]opInfo{ }, }, }, - { - name: "VFMADDSUB213PD128load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADDSUB213PD, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, - { - name: "VFMADDSUB213PD256load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADDSUB213PD, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, { name: "VFMADDSUB213PD512load", auxType: auxSymOff, @@ -44526,42 +44406,6 @@ var opcodeTable = [...]opInfo{ }, }, }, - { - name: "VFMADDSUB213PS128load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADDSUB213PS, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, - { - name: "VFMADDSUB213PS256load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMADDSUB213PS, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, { name: "VFMADDSUB213PS512load", auxType: auxSymOff, @@ -44637,42 +44481,6 @@ var opcodeTable = [...]opInfo{ }, }, }, - { - name: "VFMSUBADD213PD128load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMSUBADD213PD, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, - { - name: "VFMSUBADD213PD256load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMSUBADD213PD, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, { name: "VFMSUBADD213PD512load", auxType: auxSymOff, @@ -44748,42 +44556,6 @@ var opcodeTable = [...]opInfo{ }, }, }, - { - name: "VFMSUBADD213PS128load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMSUBADD213PS, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, - { - name: "VFMSUBADD213PS256load", - auxType: auxSymOff, - argLen: 4, - resultInArg0: true, - symEffect: SymRead, - asm: x86.AVFMSUBADD213PS, - reg: regInfo{ - inputs: []inputInfo{ - {2, 72057594037977087}, // AX CX DX BX SP BP SI DI R8 R9 R10 R11 R12 R13 R15 SB - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - {1, 281474976645120}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - outputs: []outputInfo{ - {0, 281472829161472}, // X0 X1 X2 X3 X4 X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X27 X28 X29 X30 X31 - }, - }, - }, { name: "VFMSUBADD213PS512load", auxType: auxSymOff, diff --git a/src/cmd/compile/internal/ssa/rewriteAMD64.go b/src/cmd/compile/internal/ssa/rewriteAMD64.go index fe0005bb05..b1b1c84046 100644 --- a/src/cmd/compile/internal/ssa/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssa/rewriteAMD64.go @@ -782,10 +782,6 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64VDIVPSMasked256(v) case OpAMD64VDIVPSMasked512: return rewriteValueAMD64_OpAMD64VDIVPSMasked512(v) - case OpAMD64VFMADD213PD128: - return rewriteValueAMD64_OpAMD64VFMADD213PD128(v) - case OpAMD64VFMADD213PD256: - return rewriteValueAMD64_OpAMD64VFMADD213PD256(v) case OpAMD64VFMADD213PD512: return rewriteValueAMD64_OpAMD64VFMADD213PD512(v) case OpAMD64VFMADD213PDMasked128: @@ -794,10 +790,6 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64VFMADD213PDMasked256(v) case OpAMD64VFMADD213PDMasked512: return rewriteValueAMD64_OpAMD64VFMADD213PDMasked512(v) - case OpAMD64VFMADD213PS128: - return rewriteValueAMD64_OpAMD64VFMADD213PS128(v) - case OpAMD64VFMADD213PS256: - return rewriteValueAMD64_OpAMD64VFMADD213PS256(v) case OpAMD64VFMADD213PS512: return rewriteValueAMD64_OpAMD64VFMADD213PS512(v) case OpAMD64VFMADD213PSMasked128: @@ -806,10 +798,6 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64VFMADD213PSMasked256(v) case OpAMD64VFMADD213PSMasked512: return rewriteValueAMD64_OpAMD64VFMADD213PSMasked512(v) - case OpAMD64VFMADDSUB213PD128: - return rewriteValueAMD64_OpAMD64VFMADDSUB213PD128(v) - case OpAMD64VFMADDSUB213PD256: - return rewriteValueAMD64_OpAMD64VFMADDSUB213PD256(v) case OpAMD64VFMADDSUB213PD512: return rewriteValueAMD64_OpAMD64VFMADDSUB213PD512(v) case OpAMD64VFMADDSUB213PDMasked128: @@ -818,10 +806,6 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64VFMADDSUB213PDMasked256(v) case OpAMD64VFMADDSUB213PDMasked512: return rewriteValueAMD64_OpAMD64VFMADDSUB213PDMasked512(v) - case OpAMD64VFMADDSUB213PS128: - return rewriteValueAMD64_OpAMD64VFMADDSUB213PS128(v) - case OpAMD64VFMADDSUB213PS256: - return rewriteValueAMD64_OpAMD64VFMADDSUB213PS256(v) case OpAMD64VFMADDSUB213PS512: return rewriteValueAMD64_OpAMD64VFMADDSUB213PS512(v) case OpAMD64VFMADDSUB213PSMasked128: @@ -830,10 +814,6 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64VFMADDSUB213PSMasked256(v) case OpAMD64VFMADDSUB213PSMasked512: return rewriteValueAMD64_OpAMD64VFMADDSUB213PSMasked512(v) - case OpAMD64VFMSUBADD213PD128: - return rewriteValueAMD64_OpAMD64VFMSUBADD213PD128(v) - case OpAMD64VFMSUBADD213PD256: - return rewriteValueAMD64_OpAMD64VFMSUBADD213PD256(v) case OpAMD64VFMSUBADD213PD512: return rewriteValueAMD64_OpAMD64VFMSUBADD213PD512(v) case OpAMD64VFMSUBADD213PDMasked128: @@ -842,10 +822,6 @@ func rewriteValueAMD64(v *Value) bool { return rewriteValueAMD64_OpAMD64VFMSUBADD213PDMasked256(v) case OpAMD64VFMSUBADD213PDMasked512: return rewriteValueAMD64_OpAMD64VFMSUBADD213PDMasked512(v) - case OpAMD64VFMSUBADD213PS128: - return rewriteValueAMD64_OpAMD64VFMSUBADD213PS128(v) - case OpAMD64VFMSUBADD213PS256: - return rewriteValueAMD64_OpAMD64VFMSUBADD213PS256(v) case OpAMD64VFMSUBADD213PS512: return rewriteValueAMD64_OpAMD64VFMSUBADD213PS512(v) case OpAMD64VFMSUBADD213PSMasked128: @@ -31492,64 +31468,6 @@ func rewriteValueAMD64_OpAMD64VDIVPSMasked512(v *Value) bool { } return false } -func rewriteValueAMD64_OpAMD64VFMADD213PD128(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADD213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADD213PD128load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload128 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADD213PD128load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} -func rewriteValueAMD64_OpAMD64VFMADD213PD256(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADD213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADD213PD256load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload256 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADD213PD256load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} func rewriteValueAMD64_OpAMD64VFMADD213PD512(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] @@ -31672,64 +31590,6 @@ func rewriteValueAMD64_OpAMD64VFMADD213PDMasked512(v *Value) bool { } return false } -func rewriteValueAMD64_OpAMD64VFMADD213PS128(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADD213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADD213PS128load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload128 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADD213PS128load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} -func rewriteValueAMD64_OpAMD64VFMADD213PS256(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADD213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADD213PS256load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload256 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADD213PS256load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} func rewriteValueAMD64_OpAMD64VFMADD213PS512(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] @@ -31852,64 +31712,6 @@ func rewriteValueAMD64_OpAMD64VFMADD213PSMasked512(v *Value) bool { } return false } -func rewriteValueAMD64_OpAMD64VFMADDSUB213PD128(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADDSUB213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADDSUB213PD128load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload128 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADDSUB213PD128load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} -func rewriteValueAMD64_OpAMD64VFMADDSUB213PD256(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADDSUB213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADDSUB213PD256load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload256 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADDSUB213PD256load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} func rewriteValueAMD64_OpAMD64VFMADDSUB213PD512(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] @@ -32032,64 +31834,6 @@ func rewriteValueAMD64_OpAMD64VFMADDSUB213PDMasked512(v *Value) bool { } return false } -func rewriteValueAMD64_OpAMD64VFMADDSUB213PS128(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADDSUB213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADDSUB213PS128load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload128 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADDSUB213PS128load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} -func rewriteValueAMD64_OpAMD64VFMADDSUB213PS256(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMADDSUB213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMADDSUB213PS256load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload256 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMADDSUB213PS256load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} func rewriteValueAMD64_OpAMD64VFMADDSUB213PS512(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] @@ -32212,64 +31956,6 @@ func rewriteValueAMD64_OpAMD64VFMADDSUB213PSMasked512(v *Value) bool { } return false } -func rewriteValueAMD64_OpAMD64VFMSUBADD213PD128(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMSUBADD213PD128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMSUBADD213PD128load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload128 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMSUBADD213PD128load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} -func rewriteValueAMD64_OpAMD64VFMSUBADD213PD256(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMSUBADD213PD256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMSUBADD213PD256load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload256 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMSUBADD213PD256load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} func rewriteValueAMD64_OpAMD64VFMSUBADD213PD512(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] @@ -32392,64 +32078,6 @@ func rewriteValueAMD64_OpAMD64VFMSUBADD213PDMasked512(v *Value) bool { } return false } -func rewriteValueAMD64_OpAMD64VFMSUBADD213PS128(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMSUBADD213PS128 x y l:(VMOVDQUload128 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMSUBADD213PS128load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload128 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMSUBADD213PS128load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} -func rewriteValueAMD64_OpAMD64VFMSUBADD213PS256(v *Value) bool { - v_2 := v.Args[2] - v_1 := v.Args[1] - v_0 := v.Args[0] - // match: (VFMSUBADD213PS256 x y l:(VMOVDQUload256 {sym} [off] ptr mem)) - // cond: canMergeLoad(v, l) && clobber(l) - // result: (VFMSUBADD213PS256load {sym} [off] x y ptr mem) - for { - x := v_0 - y := v_1 - l := v_2 - if l.Op != OpAMD64VMOVDQUload256 { - break - } - off := auxIntToInt32(l.AuxInt) - sym := auxToSym(l.Aux) - mem := l.Args[1] - ptr := l.Args[0] - if !(canMergeLoad(v, l) && clobber(l)) { - break - } - v.reset(OpAMD64VFMSUBADD213PS256load) - v.AuxInt = int32ToAuxInt(off) - v.Aux = symToAux(sym) - v.AddArg4(x, y, ptr, mem) - return true - } - return false -} func rewriteValueAMD64_OpAMD64VFMSUBADD213PS512(v *Value) bool { v_2 := v.Args[2] v_1 := v.Args[1] diff --git a/src/internal/cpu/cpu_x86.go b/src/internal/cpu/cpu_x86.go index 711fb045c3..0e97651e3d 100644 --- a/src/internal/cpu/cpu_x86.go +++ b/src/internal/cpu/cpu_x86.go @@ -136,12 +136,6 @@ func doinit() { // e.g. setting the xsavedisable boot option on Windows 10. X86.HasOSXSAVE = isSet(ecx1, cpuid_OSXSAVE) - // The FMA instruction set extension only has VEX prefixed instructions. - // VEX prefixed instructions require OSXSAVE to be enabled. - // See Intel 64 and IA-32 Architecture Software Developer’s Manual Volume 2 - // Section 2.4 "AVX and SSE Instruction Exception Specification" - X86.HasFMA = isSet(ecx1, cpuid_FMA) && X86.HasOSXSAVE - osSupportsAVX := false osSupportsAVX512 := false // For XGETBV, OSXSAVE bit is required and sufficient. @@ -159,6 +153,14 @@ func doinit() { X86.HasAVX = isSet(ecx1, cpuid_AVX) && osSupportsAVX + // The FMA instruction set extension requires both the FMA and AVX flags. + // + // Furthermore, the FMA instructions are all VEX prefixed instructions. + // VEX prefixed instructions require OSXSAVE to be enabled. + // See Intel 64 and IA-32 Architecture Software Developer’s Manual Volume 2 + // Section 2.4 "AVX and SSE Instruction Exception Specification" + X86.HasFMA = isSet(ecx1, cpuid_FMA) && X86.HasAVX && X86.HasOSXSAVE + if maxID < 7 { osInit() return diff --git a/src/simd/archsimd/_gen/simdgen/xed.go b/src/simd/archsimd/_gen/simdgen/xed.go index 49bedc9b28..5d6fac64d0 100644 --- a/src/simd/archsimd/_gen/simdgen/xed.go +++ b/src/simd/archsimd/_gen/simdgen/xed.go @@ -77,7 +77,7 @@ func loadXED(xedPath string) []*unify.Value { switch { case inst.RealOpcode == "N": return // Skip unstable instructions - case !(strings.HasPrefix(inst.Extension, "AVX") || strings.HasPrefix(inst.Extension, "SHA")): + case !(strings.HasPrefix(inst.Extension, "AVX") || strings.HasPrefix(inst.Extension, "SHA") || inst.Extension == "FMA"): // We're only interested in AVX and SHA instructions. return } @@ -795,6 +795,7 @@ var cpuFeatureMap = map[string]string{ "AVX2": "AVX2", "AVXAES": "AVXAES", "SHA": "SHA", + "FMA": "FMA", // AVX-512 foundational features. We combine all of these into one "AVX512" feature. "AVX512F": "AVX512", @@ -827,6 +828,7 @@ func init() { "AVX512": {Implies: []string{"AVX2"}}, "AVXAES": {Virtual: true, Implies: []string{"AVX", "AES"}}, + "FMA": {Implies: []string{"AVX"}}, // AVX-512 subfeatures. "AVX512BITALG": {Implies: []string{"AVX512"}}, diff --git a/src/simd/archsimd/cpu.go b/src/simd/archsimd/cpu.go index 1cd765268f..8069ee7f26 100644 --- a/src/simd/archsimd/cpu.go +++ b/src/simd/archsimd/cpu.go @@ -141,6 +141,16 @@ func (X86Features) AVXVNNI() bool { return cpu.X86.HasAVXVNNI } +// FMA returns whether the CPU supports the FMA feature. +// +// If it returns true, then the CPU also supports AVX. +// +// FMA is defined on all GOARCHes, but will only return true on +// GOARCH amd64. +func (X86Features) FMA() bool { + return cpu.X86.HasFMA +} + // SHA returns whether the CPU supports the SHA feature. // // SHA is defined on all GOARCHes, but will only return true on diff --git a/src/simd/archsimd/ops_amd64.go b/src/simd/archsimd/ops_amd64.go index d65c7dde0c..ec50cc72c5 100644 --- a/src/simd/archsimd/ops_amd64.go +++ b/src/simd/archsimd/ops_amd64.go @@ -4088,12 +4088,12 @@ func (x Uint64x8) Mul(y Uint64x8) Uint64x8 // MulAdd performs a fused (x * y) + z. // -// Asm: VFMADD213PS, CPU Feature: AVX512 +// Asm: VFMADD213PS, CPU Feature: FMA func (x Float32x4) MulAdd(y Float32x4, z Float32x4) Float32x4 // MulAdd performs a fused (x * y) + z. // -// Asm: VFMADD213PS, CPU Feature: AVX512 +// Asm: VFMADD213PS, CPU Feature: FMA func (x Float32x8) MulAdd(y Float32x8, z Float32x8) Float32x8 // MulAdd performs a fused (x * y) + z. @@ -4103,12 +4103,12 @@ func (x Float32x16) MulAdd(y Float32x16, z Float32x16) Float32x16 // MulAdd performs a fused (x * y) + z. // -// Asm: VFMADD213PD, CPU Feature: AVX512 +// Asm: VFMADD213PD, CPU Feature: FMA func (x Float64x2) MulAdd(y Float64x2, z Float64x2) Float64x2 // MulAdd performs a fused (x * y) + z. // -// Asm: VFMADD213PD, CPU Feature: AVX512 +// Asm: VFMADD213PD, CPU Feature: FMA func (x Float64x4) MulAdd(y Float64x4, z Float64x4) Float64x4 // MulAdd performs a fused (x * y) + z. @@ -4120,12 +4120,12 @@ func (x Float64x8) MulAdd(y Float64x8, z Float64x8) Float64x8 // MulAddSub performs a fused (x * y) - z for odd-indexed elements, and (x * y) + z for even-indexed elements. // -// Asm: VFMADDSUB213PS, CPU Feature: AVX512 +// Asm: VFMADDSUB213PS, CPU Feature: FMA func (x Float32x4) MulAddSub(y Float32x4, z Float32x4) Float32x4 // MulAddSub performs a fused (x * y) - z for odd-indexed elements, and (x * y) + z for even-indexed elements. // -// Asm: VFMADDSUB213PS, CPU Feature: AVX512 +// Asm: VFMADDSUB213PS, CPU Feature: FMA func (x Float32x8) MulAddSub(y Float32x8, z Float32x8) Float32x8 // MulAddSub performs a fused (x * y) - z for odd-indexed elements, and (x * y) + z for even-indexed elements. @@ -4135,12 +4135,12 @@ func (x Float32x16) MulAddSub(y Float32x16, z Float32x16) Float32x16 // MulAddSub performs a fused (x * y) - z for odd-indexed elements, and (x * y) + z for even-indexed elements. // -// Asm: VFMADDSUB213PD, CPU Feature: AVX512 +// Asm: VFMADDSUB213PD, CPU Feature: FMA func (x Float64x2) MulAddSub(y Float64x2, z Float64x2) Float64x2 // MulAddSub performs a fused (x * y) - z for odd-indexed elements, and (x * y) + z for even-indexed elements. // -// Asm: VFMADDSUB213PD, CPU Feature: AVX512 +// Asm: VFMADDSUB213PD, CPU Feature: FMA func (x Float64x4) MulAddSub(y Float64x4, z Float64x4) Float64x4 // MulAddSub performs a fused (x * y) - z for odd-indexed elements, and (x * y) + z for even-indexed elements. @@ -4210,12 +4210,12 @@ func (x Uint16x32) MulHigh(y Uint16x32) Uint16x32 // MulSubAdd performs a fused (x * y) + z for odd-indexed elements, and (x * y) - z for even-indexed elements. // -// Asm: VFMSUBADD213PS, CPU Feature: AVX512 +// Asm: VFMSUBADD213PS, CPU Feature: FMA func (x Float32x4) MulSubAdd(y Float32x4, z Float32x4) Float32x4 // MulSubAdd performs a fused (x * y) + z for odd-indexed elements, and (x * y) - z for even-indexed elements. // -// Asm: VFMSUBADD213PS, CPU Feature: AVX512 +// Asm: VFMSUBADD213PS, CPU Feature: FMA func (x Float32x8) MulSubAdd(y Float32x8, z Float32x8) Float32x8 // MulSubAdd performs a fused (x * y) + z for odd-indexed elements, and (x * y) - z for even-indexed elements. @@ -4225,12 +4225,12 @@ func (x Float32x16) MulSubAdd(y Float32x16, z Float32x16) Float32x16 // MulSubAdd performs a fused (x * y) + z for odd-indexed elements, and (x * y) - z for even-indexed elements. // -// Asm: VFMSUBADD213PD, CPU Feature: AVX512 +// Asm: VFMSUBADD213PD, CPU Feature: FMA func (x Float64x2) MulSubAdd(y Float64x2, z Float64x2) Float64x2 // MulSubAdd performs a fused (x * y) + z for odd-indexed elements, and (x * y) - z for even-indexed elements. // -// Asm: VFMSUBADD213PD, CPU Feature: AVX512 +// Asm: VFMSUBADD213PD, CPU Feature: FMA func (x Float64x4) MulSubAdd(y Float64x4, z Float64x4) Float64x4 // MulSubAdd performs a fused (x * y) + z for odd-indexed elements, and (x * y) - z for even-indexed elements. -- cgit v1.3