diff --git a/arch/arm64_ext.go b/arch/arm64_ext.go index e2917bd..147d99b 100644 --- a/arch/arm64_ext.go +++ b/arch/arm64_ext.go @@ -690,6 +690,24 @@ const ( // shifted data shares one narrower arrangement, which feeds the size // field. Operands: Zm.D, Zn.T, Zd.T. ExtFormShiftWide + // ExtFormConvert is the SVE2 BFloat16 convert with a governing + // predicate, ZBFCVT Z13.S, P1.M, Z22.H: a widening convert whose + // source is locked to .S and destination to .H. Operands: Zn.S, + // Pg/M or Pg/Z (the row's PgQual names which), Zd.H. Fields: Zn in + // the first-source slot, Pg narrow, Zd in the destination slot. + ExtFormConvert + // ExtFormMulPred is the SVE2 BFloat16 multiply-accumulate with an + // independent destination, ZBFMLA Z23.H, Z13.H, P1.M, Z22.H. Operands: + // Zm.H, Zn.H, Pg/M, Zda.H. Fields: Zm at bits 16..20, Zn in the + // first-source slot, Pg narrow, Zda in the destination slot. + ExtFormMulPred + // ExtFormLongMul is the SVE2 BFloat16 lengthened multiply, + // ZBFMLALB Z7.H, Z6.H, Z23.S: two halfword vectors multiply into a + // word accumulator, the wide source locked to .S. Operands: Za.H, + // Zn.H, Zm.S (the row's Arr names the halfword pair). Fields: Za at + // bits 16..20, Zn in the first-source slot, Zm in the destination + // slot. + ExtFormLongMul // ExtFormThreeSource is the SVE2 three-source bitwise form, ZBCAX // Z23.D, Z13.D, Z21.D, Z21.D: two sources combine with the destination // under the class's own operation, locked to quadwords. Operands: @@ -707,7 +725,7 @@ func (f ExtForm) Arity() int { ExtFormZUnary, ExtFormZGov, ExtFormVectorsZm, ExtFormZImmediateSource: return 3 case ExtFormPredicateLogical, ExtFormPredicateSelect, ExtFormPredicateLogicalDest, - ExtFormZPredicatedSource, ExtFormZSelect, ExtFormThreeSource: + ExtFormZPredicatedSource, ExtFormZSelect, ExtFormThreeSource, ExtFormMulPred: return 4 case ExtFormImmediate, ExtFormSignedImmediate, ExtFormPredicateFFRRead, ExtFormPredicateMove, ExtFormPredicateUnpack, ExtFormPredicateTest, @@ -733,7 +751,8 @@ func (f ExtForm) Arity() int { return 2 case ExtFormReduceReadBack: return 4 - case ExtFormSveLoad, ExtFormSveStore, ExtFormNarrow, ExtFormQuadReduce, ExtFormShiftWide: + case ExtFormSveLoad, ExtFormSveStore, ExtFormNarrow, ExtFormQuadReduce, ExtFormShiftWide, + ExtFormConvert, ExtFormLongMul: return 3 case ExtFormNone: return 0 @@ -804,10 +823,12 @@ func (f ExtForm) Kinds() []ExtOperandKind { return []ExtOperandKind{ExtSveMem, ExtPReg, ExtZReg} case ExtFormSveStore: return []ExtOperandKind{ExtZReg, ExtPReg, ExtSveMem} - case ExtFormNarrow, ExtFormShiftWide: + case ExtFormNarrow, ExtFormShiftWide, ExtFormLongMul: return []ExtOperandKind{ExtZReg, ExtZReg, ExtZReg} - case ExtFormThreeSource: - return []ExtOperandKind{ExtZReg, ExtZReg, ExtZReg, ExtZReg} + case ExtFormConvert: + return []ExtOperandKind{ExtZReg, ExtPReg, ExtZReg} + case ExtFormThreeSource, ExtFormMulPred: + return []ExtOperandKind{ExtZReg, ExtZReg, ExtPReg, ExtZReg} case ExtFormQuadReduce: return []ExtOperandKind{ExtZReg, ExtPReg, ExtVReg} case ExtFormNone: @@ -940,6 +961,12 @@ func (f ExtForm) String() string { return "wide shift" case ExtFormThreeSource: return "three-source bitwise" + case ExtFormConvert: + return "widening convert" + case ExtFormMulPred: + return "multiply-accumulate" + case ExtFormLongMul: + return "lengthened multiply" case ExtFormNone: return "no operands" default: @@ -1200,6 +1227,12 @@ func (in ExtInstr) Encode(ops []ExtOperand) ([]byte, error) { return in.encodeShiftWide(ops) case ExtFormThreeSource: return in.encodeThreeSource(ops) + case ExtFormConvert: + return in.encodeConvert(ops) + case ExtFormMulPred: + return in.encodeMulPred(ops) + case ExtFormLongMul: + return in.encodeLongMul(ops) case ExtFormNone: if len(ops) != 0 { return nil, fmt.Errorf("%s: the %s form takes no operands, got %d", @@ -2725,6 +2758,110 @@ func (in ExtInstr) encodeThreeSource(ops []ExtOperand) ([]byte, error) { return extWordLE(word), nil } +// encodeConvert fills the SVE2 BFloat16 convert with a governing +// predicate: Zn.S, Pg{M or Z}, Zd.H, the source locked to .S and the +// destination to .H, the qualifier the row's PgQual names. +func (in ExtInstr) encodeConvert(ops []ExtOperand) ([]byte, error) { + if err := in.zVector(ops[0], 1); err != nil { + return nil, err + } + if ops[0].Arr != ExtArrS { + return nil, fmt.Errorf("%s: operand 1 carries arrangement %s, want .S", in.Name, ops[0].Arr) + } + pg := ops[1] + if err := in.predicateOperand(pg, 2, 0, 7, ExtArrNone); err != nil { + return nil, err + } + pg.Arr = ExtArrNone + want := in.PgQual + if want != ExtQualMerging && want != ExtQualZeroing { + want = ExtQualMerging + } + if pg.Qual != want { + return nil, fmt.Errorf("%s: operand 2 wants the %s qualifier, got %q", in.Name, want.String(), pg.Qual) + } + if err := in.zVector(ops[2], 3); err != nil { + return nil, err + } + if ops[2].Arr != ExtArrH { + return nil, fmt.Errorf("%s: operand 3 carries arrangement %s, want .H", in.Name, ops[2].Arr) + } + word := in.Word + word = extSet(word, extFieldPn, uint32(ops[0].Reg)) + word = extSet(word, extFieldPgN, uint32(pg.Reg)) + word = extSet(word, extFieldRd, uint32(ops[2].Reg)) + return extWordLE(word), nil +} + +// encodeMulPred fills the SVE2 BFloat16 multiply-accumulate with an +// independent destination: Zm.H, Zn.H, Pg/M, Zda.H. +func (in ExtInstr) encodeMulPred(ops []ExtOperand) ([]byte, error) { + zm, zn, pg, zda := ops[0], ops[1], ops[2], ops[3] + if err := in.zVector(zm, 1); err != nil { + return nil, err + } + if zm.Arr != ExtArrH { + return nil, fmt.Errorf("%s: operand 1 carries arrangement %s, want .H", in.Name, zm.Arr) + } + if err := in.zVector(zn, 2); err != nil { + return nil, err + } + if zn.Arr != ExtArrH { + return nil, fmt.Errorf("%s: operand 2 carries arrangement %s, want .H", in.Name, zn.Arr) + } + if err := in.predicateOperand(pg, 3, 0, 7, ExtArrNone); err != nil { + return nil, err + } + if pg.Qual != ExtQualMerging { + return nil, fmt.Errorf("%s: operand 3 wants the merging qualifier, got %q", in.Name, pg.Qual) + } + if err := in.zVector(zda, 4); err != nil { + return nil, err + } + if zda.Arr != ExtArrH { + return nil, fmt.Errorf("%s: operand 4 carries arrangement %s, want .H", in.Name, zda.Arr) + } + word := in.Word + word = extSet(word, extFieldRm, uint32(zm.Reg)) + word = extSet(word, extFieldRn, uint32(zn.Reg)) + word = extSet(word, extFieldPgN, uint32(pg.Reg)) + word = extSet(word, extFieldRd, uint32(zda.Reg)) + return extWordLE(word), nil +} + +// encodeLongMul fills the SVE2 BFloat16 lengthened multiply: Za.H, Zn.H, +// Zm.S, the two halfword vectors under the row's Arr and the wide source +// one step above it. +func (in ExtInstr) encodeLongMul(ops []ExtOperand) ([]byte, error) { + za, zn, zm := ops[0], ops[1], ops[2] + if err := in.zVector(za, 1); err != nil { + return nil, err + } + if in.Arr != ExtArrNone && za.Arr != in.Arr { + return nil, fmt.Errorf("%s: operand 1 carries arrangement %s, want %s", in.Name, za.Arr, in.Arr) + } + if err := in.zVector(zn, 2); err != nil { + return nil, err + } + if zn.Arr != za.Arr { + return nil, fmt.Errorf("%s: operands 1 and 2 carry arrangements %s and %s, they must match", in.Name, za.Arr, zn.Arr) + } + if zm.Kind != ExtZReg { + return nil, fmt.Errorf("%s: operand 3 wants a scalable vector register, got %s", in.Name, zm.Kind) + } + if zm.Reg < 0 || zm.Reg > 31 { + return nil, fmt.Errorf("%s: operand 3 is Z%d, outside Z0-Z31", in.Name, zm.Reg) + } + if zm.Arr != za.Arr+1 { + return nil, fmt.Errorf("%s: operand 3 carries arrangement %s, want %s", in.Name, zm.Arr, za.Arr+1) + } + word := in.Word + word = extSet(word, extFieldRm, uint32(za.Reg)) + word = extSet(word, extFieldRn, uint32(zn.Reg)) + word = extSet(word, extFieldRd, uint32(zm.Reg)) + return extWordLE(word), nil +} + // sharedSize returns the one element-size encoding the given arrangements // agree on, or an error when any operand is bare, they disagree, or the // arrangement has no size field. @@ -3492,6 +3629,45 @@ var arm64Extensions = []ExtInstr{ {Name: "ZBFSUB", Summary: "Subtract BFloat16 halfwords, unpredicated", Word: 0x65000400, Form: ExtFormVectorsZm, Arr: ExtArrH, Feature: ExtFeatureSVE2, Ref: "ARM DDI 0487J SVE2: ZBFSUB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFCVT", Summary: "Convert BFloat16 halfwords to words from a source, merging", + Word: 0x658aa000, Form: ExtFormConvert, PgQual: ExtQualMerging, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFCVT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFCVT", Summary: "Convert BFloat16 halfwords to words from a source, zeroing", + Word: 0x649ac000, Form: ExtFormConvert, PgQual: ExtQualZeroing, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFCVT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFCVTNT", Summary: "Convert BFloat16 halfwords to words from a source, merging, no rounding", + Word: 0x648aa000, Form: ExtFormConvert, PgQual: ExtQualMerging, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFCVTNT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFCVTNT", Summary: "Convert BFloat16 halfwords to words from a source, zeroing, no rounding", + Word: 0x6482a000, Form: ExtFormConvert, PgQual: ExtQualZeroing, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFCVTNT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFDOT", Summary: "Dot product of BFloat16 pairs into words", + Word: 0x64608000, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFDOT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMLA", Summary: "Multiply-accumulate BFloat16 halfwords, merging", + Word: 0x65200000, Form: ExtFormMulPred, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMLA; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMLALB", Summary: "Lengthened multiply-accumulate of BFloat16 halfwords, bottom", + Word: 0x64e08000, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMLALB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMLALT", Summary: "Lengthened multiply-accumulate of BFloat16 halfwords, top", + Word: 0x64e08400, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMLALT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMLS", Summary: "Multiply-subtract BFloat16 halfwords, merging", + Word: 0x65202000, Form: ExtFormMulPred, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMLS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMLSLB", Summary: "Lengthened multiply-subtract of BFloat16 halfwords, bottom", + Word: 0x64e0a000, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMLSLB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMLSLT", Summary: "Lengthened multiply-subtract of BFloat16 halfwords, top", + Word: 0x64e0a400, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMLSLT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFMMLA", Summary: "Matrix multiply-accumulate of BFloat16 halfwords", + Word: 0x6460e400, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFMMLA; inst_gen.go + arm64sveenc.s"}, + {Name: "ZBFSCALE", Summary: "Scale BFloat16 halfwords by a power of two, merging", + Word: 0x65098000, Form: ExtFormZPredicatedSource, Arr: ExtArrH, PgQual: ExtQualMerging, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ZBFSCALE; inst_gen.go + arm64sveenc.s"}, // --- the SVE multiple-structure loads and stores -------------------------- // // The two-, three- and four-register list loads and stores over the diff --git a/arch/arm64_ext_test.go b/arch/arm64_ext_test.go index 3918809..ebcf62d 100644 --- a/arch/arm64_ext_test.go +++ b/arch/arm64_ext_test.go @@ -1551,6 +1551,28 @@ func TestArm64ExtStage4Golden(t *testing.T) { {"ZBFSUB Z7.H, Z6.H, Z23.H", "ZBFSUB", ExtFormVectorsZm, ExtQualNone, ExtArrH, []ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrH)}, 0x650704d7}, + // The rest of the BFloat16 wall. + {"ZBFCVT Z13.S, P1.M, Z22.H", "ZBFCVT", ExtFormConvert, ExtQualMerging, ExtArrNone, + []ExtOperand{ExtVector(13, ExtArrS), ExtPredicate(1, ExtQualMerging), ExtVector(22, ExtArrH)}, + 0x658aa5b6}, + {"ZBFCVT Z13.S, P1.Z, Z22.H", "ZBFCVT", ExtFormConvert, ExtQualZeroing, ExtArrNone, + []ExtOperand{ExtVector(13, ExtArrS), ExtPredicate(1, ExtQualZeroing), ExtVector(22, ExtArrH)}, + 0x649ac5b6}, + {"ZBFDOT Z7.H, Z6.H, Z23.S", "ZBFDOT", ExtFormLongMul, ExtQualNone, ExtArrH, + []ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrS)}, + 0x646780d7}, + {"ZBFMLA Z23.H, Z13.H, P1.M, Z22.H", "ZBFMLA", ExtFormMulPred, ExtQualNone, ExtArrNone, + []ExtOperand{ExtVector(23, ExtArrH), ExtVector(13, ExtArrH), ExtPredicate(1, ExtQualMerging), ExtVector(22, ExtArrH)}, + 0x653705b6}, + {"ZBFMLALB Z7.H, Z6.H, Z23.S", "ZBFMLALB", ExtFormLongMul, ExtQualNone, ExtArrH, + []ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrS)}, + 0x64e780d7}, + {"ZBFMMLA Z7.H, Z6.H, Z23.S", "ZBFMMLA", ExtFormLongMul, ExtQualNone, ExtArrH, + []ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrS)}, + 0x6467e4d7}, + {"ZBFSCALE Z23.H, Z13.H, P1.M, Z13.H", "ZBFSCALE", ExtFormZPredicatedSource, ExtQualMerging, ExtArrH, + []ExtOperand{ExtVector(23, ExtArrH), ExtVector(13, ExtArrH), ExtPredicate(1, ExtQualMerging), ExtVector(13, ExtArrH)}, + 0x650986ed}, // The multiple-structure loads and stores. {"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B]", "ZLD2B", ExtFormSveLoad, ExtQualNone, ExtArrB, []ExtOperand{ExtSveMemPair(6, 0, 14, false), ExtPredicate(4, ExtQualZeroing), ExtOperand{Kind: ExtZReg, Reg: 13, Arr: ExtArrB, List: 2}}, diff --git a/asm/arm64_ext_asm_test.go b/asm/arm64_ext_asm_test.go index a83e50a..2dd18f3 100644 --- a/asm/arm64_ext_asm_test.go +++ b/asm/arm64_ext_asm_test.go @@ -822,6 +822,19 @@ func TestArm64AssembleExtensionStage4Golden(t *testing.T) { {"ZBFMUL Z7.H, Z6.H, Z23.H", 0x650708d7}, {"ZBFSUB Z23.H, Z13.H, P1.M, Z13.H", 0x650186ed}, {"ZBFSUB Z7.H, Z6.H, Z23.H", 0x650704d7}, + // The rest of the BFloat16 wall. + {"ZBFCVT Z13.S, P1.M, Z22.H", 0x658aa5b6}, + {"ZBFCVT Z13.S, P1.Z, Z22.H", 0x649ac5b6}, + {"ZBFCVTNT Z13.S, P1.M, Z22.H", 0x648aa5b6}, + {"ZBFDOT Z7.H, Z6.H, Z23.S", 0x646780d7}, + {"ZBFMLA Z23.H, Z13.H, P1.M, Z22.H", 0x653705b6}, + {"ZBFMLALB Z7.H, Z6.H, Z23.S", 0x64e780d7}, + {"ZBFMLALT Z7.H, Z6.H, Z23.S", 0x64e784d7}, + {"ZBFMLS Z23.H, Z13.H, P1.M, Z22.H", 0x653725b6}, + {"ZBFMLSLB Z7.H, Z6.H, Z23.S", 0x64e7a0d7}, + {"ZBFMLSLT Z7.H, Z6.H, Z23.S", 0x64e7a4d7}, + {"ZBFMMLA Z7.H, Z6.H, Z23.S", 0x6467e4d7}, + {"ZBFSCALE Z23.H, Z13.H, P1.M, Z13.H", 0x650986ed}, // The multiple-structure loads and stores. {"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B]", 0xa426d1cd}, {"ZLD2H (R6<<1)(R14), P4.Z, [Z13.H, Z14.H]", 0xa4a6d1cd}, diff --git a/asm/extension_test.go b/asm/extension_test.go index 4d7241f..bda6c39 100644 --- a/asm/extension_test.go +++ b/asm/extension_test.go @@ -327,6 +327,17 @@ func TestExtensionNamesARM64(t *testing.T) { "ZBFMINNM", "ZBFMUL", "ZBFSUB", + "ZBFCVT", + "ZBFCVTNT", + "ZBFDOT", + "ZBFMLA", + "ZBFMLALB", + "ZBFMLALT", + "ZBFMLS", + "ZBFMLSLB", + "ZBFMLSLT", + "ZBFMMLA", + "ZBFSCALE", "ZLD2B", "ZLD2D", "ZLD2H", @@ -469,7 +480,7 @@ func TestExtensionNamesARM64(t *testing.T) { if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("ExtensionNames(ARM64) = %v, want %v", got, want) } - if n := len(arch.Extensions(arch.ARM64)); n != 470 { - t.Errorf("the family registers %d instructions, want 470", n) + if n := len(arch.Extensions(arch.ARM64)); n != 483 { + t.Errorf("the family registers %d instructions, want 483", n) } }