feat(arch): the rest of the SVE2 BFloat16 wall

Assisted-by: GLM 5.3 Flash

arch/arm64_ext.go

@@ -690,6 +690,24 @@ const (
 	// shifted data shares one narrower arrangement, which feeds the size
 	// field.  Operands: Zm.D, Zn.T, Zd.T.
 	ExtFormShiftWide
+	// ExtFormConvert is the SVE2 BFloat16 convert with a governing
+	// predicate, ZBFCVT Z13.S, P1.M, Z22.H: a widening convert whose
+	// source is locked to .S and destination to .H.  Operands: Zn.S,
+	// Pg/M or Pg/Z (the row's PgQual names which), Zd.H.  Fields: Zn in
+	// the first-source slot, Pg narrow, Zd in the destination slot.
+	ExtFormConvert
+	// ExtFormMulPred is the SVE2 BFloat16 multiply-accumulate with an
+	// independent destination, ZBFMLA Z23.H, Z13.H, P1.M, Z22.H.  Operands:
+	// Zm.H, Zn.H, Pg/M, Zda.H.  Fields: Zm at bits 16..20, Zn in the
+	// first-source slot, Pg narrow, Zda in the destination slot.
+	ExtFormMulPred
+	// ExtFormLongMul is the SVE2 BFloat16 lengthened multiply,
+	// ZBFMLALB Z7.H, Z6.H, Z23.S: two halfword vectors multiply into a
+	// word accumulator, the wide source locked to .S.  Operands: Za.H,
+	// Zn.H, Zm.S (the row's Arr names the halfword pair).  Fields: Za at
+	// bits 16..20, Zn in the first-source slot, Zm in the destination
+	// slot.
+	ExtFormLongMul
 	// ExtFormThreeSource is the SVE2 three-source bitwise form, ZBCAX
 	// Z23.D, Z13.D, Z21.D, Z21.D: two sources combine with the destination
 	// under the class's own operation, locked to quadwords.  Operands:
@@ -707,7 +725,7 @@ func (f ExtForm) Arity() int {
 		ExtFormZUnary, ExtFormZGov, ExtFormVectorsZm, ExtFormZImmediateSource:
 		return 3
 	case ExtFormPredicateLogical, ExtFormPredicateSelect, ExtFormPredicateLogicalDest,
-		ExtFormZPredicatedSource, ExtFormZSelect, ExtFormThreeSource:
+		ExtFormZPredicatedSource, ExtFormZSelect, ExtFormThreeSource, ExtFormMulPred:
 		return 4
 	case ExtFormImmediate, ExtFormSignedImmediate, ExtFormPredicateFFRRead,
 		ExtFormPredicateMove, ExtFormPredicateUnpack, ExtFormPredicateTest,
@@ -733,7 +751,8 @@ func (f ExtForm) Arity() int {
 		return 2
 	case ExtFormReduceReadBack:
 		return 4
-	case ExtFormSveLoad, ExtFormSveStore, ExtFormNarrow, ExtFormQuadReduce, ExtFormShiftWide:
+	case ExtFormSveLoad, ExtFormSveStore, ExtFormNarrow, ExtFormQuadReduce, ExtFormShiftWide,
+		ExtFormConvert, ExtFormLongMul:
 		return 3
 	case ExtFormNone:
 		return 0
@@ -804,10 +823,12 @@ func (f ExtForm) Kinds() []ExtOperandKind {
 		return []ExtOperandKind{ExtSveMem, ExtPReg, ExtZReg}
 	case ExtFormSveStore:
 		return []ExtOperandKind{ExtZReg, ExtPReg, ExtSveMem}
-	case ExtFormNarrow, ExtFormShiftWide:
+	case ExtFormNarrow, ExtFormShiftWide, ExtFormLongMul:
 		return []ExtOperandKind{ExtZReg, ExtZReg, ExtZReg}
-	case ExtFormThreeSource:
-		return []ExtOperandKind{ExtZReg, ExtZReg, ExtZReg, ExtZReg}
+	case ExtFormConvert:
+		return []ExtOperandKind{ExtZReg, ExtPReg, ExtZReg}
+	case ExtFormThreeSource, ExtFormMulPred:
+		return []ExtOperandKind{ExtZReg, ExtZReg, ExtPReg, ExtZReg}
 	case ExtFormQuadReduce:
 		return []ExtOperandKind{ExtZReg, ExtPReg, ExtVReg}
 	case ExtFormNone:
@@ -940,6 +961,12 @@ func (f ExtForm) String() string {
 		return "wide shift"
 	case ExtFormThreeSource:
 		return "three-source bitwise"
+	case ExtFormConvert:
+		return "widening convert"
+	case ExtFormMulPred:
+		return "multiply-accumulate"
+	case ExtFormLongMul:
+		return "lengthened multiply"
 	case ExtFormNone:
 		return "no operands"
 	default:
@@ -1200,6 +1227,12 @@ func (in ExtInstr) Encode(ops []ExtOperand) ([]byte, error) {
 		return in.encodeShiftWide(ops)
 	case ExtFormThreeSource:
 		return in.encodeThreeSource(ops)
+	case ExtFormConvert:
+		return in.encodeConvert(ops)
+	case ExtFormMulPred:
+		return in.encodeMulPred(ops)
+	case ExtFormLongMul:
+		return in.encodeLongMul(ops)
 	case ExtFormNone:
 		if len(ops) != 0 {
 			return nil, fmt.Errorf("%s: the %s form takes no operands, got %d",
@@ -2725,6 +2758,110 @@ func (in ExtInstr) encodeThreeSource(ops []ExtOperand) ([]byte, error) {
 	return extWordLE(word), nil
 }
 
+// encodeConvert fills the SVE2 BFloat16 convert with a governing
+// predicate: Zn.S, Pg{M or Z}, Zd.H, the source locked to .S and the
+// destination to .H, the qualifier the row's PgQual names.
+func (in ExtInstr) encodeConvert(ops []ExtOperand) ([]byte, error) {
+	if err := in.zVector(ops[0], 1); err != nil {
+		return nil, err
+	}
+	if ops[0].Arr != ExtArrS {
+		return nil, fmt.Errorf("%s: operand 1 carries arrangement %s, want .S", in.Name, ops[0].Arr)
+	}
+	pg := ops[1]
+	if err := in.predicateOperand(pg, 2, 0, 7, ExtArrNone); err != nil {
+		return nil, err
+	}
+	pg.Arr = ExtArrNone
+	want := in.PgQual
+	if want != ExtQualMerging && want != ExtQualZeroing {
+		want = ExtQualMerging
+	}
+	if pg.Qual != want {
+		return nil, fmt.Errorf("%s: operand 2 wants the %s qualifier, got %q", in.Name, want.String(), pg.Qual)
+	}
+	if err := in.zVector(ops[2], 3); err != nil {
+		return nil, err
+	}
+	if ops[2].Arr != ExtArrH {
+		return nil, fmt.Errorf("%s: operand 3 carries arrangement %s, want .H", in.Name, ops[2].Arr)
+	}
+	word := in.Word
+	word = extSet(word, extFieldPn, uint32(ops[0].Reg))
+	word = extSet(word, extFieldPgN, uint32(pg.Reg))
+	word = extSet(word, extFieldRd, uint32(ops[2].Reg))
+	return extWordLE(word), nil
+}
+
+// encodeMulPred fills the SVE2 BFloat16 multiply-accumulate with an
+// independent destination: Zm.H, Zn.H, Pg/M, Zda.H.
+func (in ExtInstr) encodeMulPred(ops []ExtOperand) ([]byte, error) {
+	zm, zn, pg, zda := ops[0], ops[1], ops[2], ops[3]
+	if err := in.zVector(zm, 1); err != nil {
+		return nil, err
+	}
+	if zm.Arr != ExtArrH {
+		return nil, fmt.Errorf("%s: operand 1 carries arrangement %s, want .H", in.Name, zm.Arr)
+	}
+	if err := in.zVector(zn, 2); err != nil {
+		return nil, err
+	}
+	if zn.Arr != ExtArrH {
+		return nil, fmt.Errorf("%s: operand 2 carries arrangement %s, want .H", in.Name, zn.Arr)
+	}
+	if err := in.predicateOperand(pg, 3, 0, 7, ExtArrNone); err != nil {
+		return nil, err
+	}
+	if pg.Qual != ExtQualMerging {
+		return nil, fmt.Errorf("%s: operand 3 wants the merging qualifier, got %q", in.Name, pg.Qual)
+	}
+	if err := in.zVector(zda, 4); err != nil {
+		return nil, err
+	}
+	if zda.Arr != ExtArrH {
+		return nil, fmt.Errorf("%s: operand 4 carries arrangement %s, want .H", in.Name, zda.Arr)
+	}
+	word := in.Word
+	word = extSet(word, extFieldRm, uint32(zm.Reg))
+	word = extSet(word, extFieldRn, uint32(zn.Reg))
+	word = extSet(word, extFieldPgN, uint32(pg.Reg))
+	word = extSet(word, extFieldRd, uint32(zda.Reg))
+	return extWordLE(word), nil
+}
+
+// encodeLongMul fills the SVE2 BFloat16 lengthened multiply: Za.H, Zn.H,
+// Zm.S, the two halfword vectors under the row's Arr and the wide source
+// one step above it.
+func (in ExtInstr) encodeLongMul(ops []ExtOperand) ([]byte, error) {
+	za, zn, zm := ops[0], ops[1], ops[2]
+	if err := in.zVector(za, 1); err != nil {
+		return nil, err
+	}
+	if in.Arr != ExtArrNone && za.Arr != in.Arr {
+		return nil, fmt.Errorf("%s: operand 1 carries arrangement %s, want %s", in.Name, za.Arr, in.Arr)
+	}
+	if err := in.zVector(zn, 2); err != nil {
+		return nil, err
+	}
+	if zn.Arr != za.Arr {
+		return nil, fmt.Errorf("%s: operands 1 and 2 carry arrangements %s and %s, they must match", in.Name, za.Arr, zn.Arr)
+	}
+	if zm.Kind != ExtZReg {
+		return nil, fmt.Errorf("%s: operand 3 wants a scalable vector register, got %s", in.Name, zm.Kind)
+	}
+	if zm.Reg < 0 || zm.Reg > 31 {
+		return nil, fmt.Errorf("%s: operand 3 is Z%d, outside Z0-Z31", in.Name, zm.Reg)
+	}
+	if zm.Arr != za.Arr+1 {
+		return nil, fmt.Errorf("%s: operand 3 carries arrangement %s, want %s", in.Name, zm.Arr, za.Arr+1)
+	}
+	word := in.Word
+	word = extSet(word, extFieldRm, uint32(za.Reg))
+	word = extSet(word, extFieldRn, uint32(zn.Reg))
+	word = extSet(word, extFieldRd, uint32(zm.Reg))
+	return extWordLE(word), nil
+}
+
 // sharedSize returns the one element-size encoding the given arrangements
 // agree on, or an error when any operand is bare, they disagree, or the
 // arrangement has no size field.
@@ -3492,6 +3629,45 @@ var arm64Extensions = []ExtInstr{
 	{Name: "ZBFSUB", Summary: "Subtract BFloat16 halfwords, unpredicated",
 		Word: 0x65000400, Form: ExtFormVectorsZm, Arr: ExtArrH, Feature: ExtFeatureSVE2,
 		Ref: "ARM DDI 0487J SVE2: ZBFSUB; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFCVT", Summary: "Convert BFloat16 halfwords to words from a source, merging",
+		Word: 0x658aa000, Form: ExtFormConvert, PgQual: ExtQualMerging, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFCVT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFCVT", Summary: "Convert BFloat16 halfwords to words from a source, zeroing",
+		Word: 0x649ac000, Form: ExtFormConvert, PgQual: ExtQualZeroing, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFCVT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFCVTNT", Summary: "Convert BFloat16 halfwords to words from a source, merging, no rounding",
+		Word: 0x648aa000, Form: ExtFormConvert, PgQual: ExtQualMerging, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFCVTNT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFCVTNT", Summary: "Convert BFloat16 halfwords to words from a source, zeroing, no rounding",
+		Word: 0x6482a000, Form: ExtFormConvert, PgQual: ExtQualZeroing, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFCVTNT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFDOT", Summary: "Dot product of BFloat16 pairs into words",
+		Word: 0x64608000, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFDOT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMLA", Summary: "Multiply-accumulate BFloat16 halfwords, merging",
+		Word: 0x65200000, Form: ExtFormMulPred, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMLA; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMLALB", Summary: "Lengthened multiply-accumulate of BFloat16 halfwords, bottom",
+		Word: 0x64e08000, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMLALB; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMLALT", Summary: "Lengthened multiply-accumulate of BFloat16 halfwords, top",
+		Word: 0x64e08400, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMLALT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMLS", Summary: "Multiply-subtract BFloat16 halfwords, merging",
+		Word: 0x65202000, Form: ExtFormMulPred, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMLS; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMLSLB", Summary: "Lengthened multiply-subtract of BFloat16 halfwords, bottom",
+		Word: 0x64e0a000, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMLSLB; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMLSLT", Summary: "Lengthened multiply-subtract of BFloat16 halfwords, top",
+		Word: 0x64e0a400, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMLSLT; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFMMLA", Summary: "Matrix multiply-accumulate of BFloat16 halfwords",
+		Word: 0x6460e400, Form: ExtFormLongMul, Arr: ExtArrH, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFMMLA; inst_gen.go + arm64sveenc.s"},
+	{Name: "ZBFSCALE", Summary: "Scale BFloat16 halfwords by a power of two, merging",
+		Word: 0x65098000, Form: ExtFormZPredicatedSource, Arr: ExtArrH, PgQual: ExtQualMerging, Feature: ExtFeatureSVE2,
+		Ref: "ARM DDI 0487J SVE2: ZBFSCALE; inst_gen.go + arm64sveenc.s"},
 	// --- the SVE multiple-structure loads and stores --------------------------
 	//
 	// The two-, three- and four-register list loads and stores over the

arch/arm64_ext_test.go

@@ -1551,6 +1551,28 @@ func TestArm64ExtStage4Golden(t *testing.T) {
 		{"ZBFSUB Z7.H, Z6.H, Z23.H", "ZBFSUB", ExtFormVectorsZm, ExtQualNone, ExtArrH,
 			[]ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrH)},
 			0x650704d7},
+		// The rest of the BFloat16 wall.
+		{"ZBFCVT Z13.S, P1.M, Z22.H", "ZBFCVT", ExtFormConvert, ExtQualMerging, ExtArrNone,
+			[]ExtOperand{ExtVector(13, ExtArrS), ExtPredicate(1, ExtQualMerging), ExtVector(22, ExtArrH)},
+			0x658aa5b6},
+		{"ZBFCVT Z13.S, P1.Z, Z22.H", "ZBFCVT", ExtFormConvert, ExtQualZeroing, ExtArrNone,
+			[]ExtOperand{ExtVector(13, ExtArrS), ExtPredicate(1, ExtQualZeroing), ExtVector(22, ExtArrH)},
+			0x649ac5b6},
+		{"ZBFDOT Z7.H, Z6.H, Z23.S", "ZBFDOT", ExtFormLongMul, ExtQualNone, ExtArrH,
+			[]ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrS)},
+			0x646780d7},
+		{"ZBFMLA Z23.H, Z13.H, P1.M, Z22.H", "ZBFMLA", ExtFormMulPred, ExtQualNone, ExtArrNone,
+			[]ExtOperand{ExtVector(23, ExtArrH), ExtVector(13, ExtArrH), ExtPredicate(1, ExtQualMerging), ExtVector(22, ExtArrH)},
+			0x653705b6},
+		{"ZBFMLALB Z7.H, Z6.H, Z23.S", "ZBFMLALB", ExtFormLongMul, ExtQualNone, ExtArrH,
+			[]ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrS)},
+			0x64e780d7},
+		{"ZBFMMLA Z7.H, Z6.H, Z23.S", "ZBFMMLA", ExtFormLongMul, ExtQualNone, ExtArrH,
+			[]ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrS)},
+			0x6467e4d7},
+		{"ZBFSCALE Z23.H, Z13.H, P1.M, Z13.H", "ZBFSCALE", ExtFormZPredicatedSource, ExtQualMerging, ExtArrH,
+			[]ExtOperand{ExtVector(23, ExtArrH), ExtVector(13, ExtArrH), ExtPredicate(1, ExtQualMerging), ExtVector(13, ExtArrH)},
+			0x650986ed},
 		// The multiple-structure loads and stores.
 		{"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B]", "ZLD2B", ExtFormSveLoad, ExtQualNone, ExtArrB,
 			[]ExtOperand{ExtSveMemPair(6, 0, 14, false), ExtPredicate(4, ExtQualZeroing), ExtOperand{Kind: ExtZReg, Reg: 13, Arr: ExtArrB, List: 2}},

asm/arm64_ext_asm_test.go

@@ -822,6 +822,19 @@ func TestArm64AssembleExtensionStage4Golden(t *testing.T) {
 		{"ZBFMUL Z7.H, Z6.H, Z23.H", 0x650708d7},
 		{"ZBFSUB Z23.H, Z13.H, P1.M, Z13.H", 0x650186ed},
 		{"ZBFSUB Z7.H, Z6.H, Z23.H", 0x650704d7},
+		// The rest of the BFloat16 wall.
+		{"ZBFCVT Z13.S, P1.M, Z22.H", 0x658aa5b6},
+		{"ZBFCVT Z13.S, P1.Z, Z22.H", 0x649ac5b6},
+		{"ZBFCVTNT Z13.S, P1.M, Z22.H", 0x648aa5b6},
+		{"ZBFDOT Z7.H, Z6.H, Z23.S", 0x646780d7},
+		{"ZBFMLA Z23.H, Z13.H, P1.M, Z22.H", 0x653705b6},
+		{"ZBFMLALB Z7.H, Z6.H, Z23.S", 0x64e780d7},
+		{"ZBFMLALT Z7.H, Z6.H, Z23.S", 0x64e784d7},
+		{"ZBFMLS Z23.H, Z13.H, P1.M, Z22.H", 0x653725b6},
+		{"ZBFMLSLB Z7.H, Z6.H, Z23.S", 0x64e7a0d7},
+		{"ZBFMLSLT Z7.H, Z6.H, Z23.S", 0x64e7a4d7},
+		{"ZBFMMLA Z7.H, Z6.H, Z23.S", 0x6467e4d7},
+		{"ZBFSCALE Z23.H, Z13.H, P1.M, Z13.H", 0x650986ed},
 		// The multiple-structure loads and stores.
 		{"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B]", 0xa426d1cd},
 		{"ZLD2H (R6<<1)(R14), P4.Z, [Z13.H, Z14.H]", 0xa4a6d1cd},

asm/extension_test.go

@@ -327,6 +327,17 @@ func TestExtensionNamesARM64(t *testing.T) {
 		"ZBFMINNM",
 		"ZBFMUL",
 		"ZBFSUB",
+		"ZBFCVT",
+		"ZBFCVTNT",
+		"ZBFDOT",
+		"ZBFMLA",
+		"ZBFMLALB",
+		"ZBFMLALT",
+		"ZBFMLS",
+		"ZBFMLSLB",
+		"ZBFMLSLT",
+		"ZBFMMLA",
+		"ZBFSCALE",
 		"ZLD2B",
 		"ZLD2D",
 		"ZLD2H",
@@ -469,7 +480,7 @@ func TestExtensionNamesARM64(t *testing.T) {
 	if strings.Join(got, ",") != strings.Join(want, ",") {
 		t.Errorf("ExtensionNames(ARM64) = %v, want %v", got, want)
 	}
-	if n := len(arch.Extensions(arch.ARM64)); n != 470 {
-		t.Errorf("the family registers %d instructions, want 470", n)
+	if n := len(arch.Extensions(arch.ARM64)); n != 483 {
+		t.Errorf("the family registers %d instructions, want 483", n)
 	}
 }