diff --git a/arch/arm64_ext.go b/arch/arm64_ext.go index 48e02f0..6ca42f8 100644 --- a/arch/arm64_ext.go +++ b/arch/arm64_ext.go @@ -40,6 +40,7 @@ const ( // The arm64 layer's general and counter registers. ExtGReg // general register R0-R30 ExtPNReg // predicate-as-counter register PN8-PN15 + ExtVReg // 128-bit SIMD register V0-V31, the scalar lanes of Z0-Z31 ) // String returns a short label for the kind. @@ -69,6 +70,8 @@ func (k ExtOperandKind) String() string { return "general register" case ExtPNReg: return "predicate-as-counter register" + case ExtVReg: + return "SIMD register" default: return "operand" } @@ -204,6 +207,12 @@ type ExtOperand struct { // lanes become zero instead of keeping the destination. It is valid // only together with a spelled mask. Zeroing bool + // Reg31 names how the thirty-first general-register slot was spelled on + // an arm64 ExtGReg operand: the zero register ZR or the stack pointer + // RSP. Both encode as 31 in the five-bit register fields, and the forms + // of this layer accept one spelling or the other, never a plain R31, so + // the marker carries the distinction into the diagnostics. + Reg31 uint8 // 0 = a register 0..30, 1 = ZR, 2 = RSP } // ExtVector builds a scalable vector operand, ADD Z1.S style. @@ -234,6 +243,25 @@ func ExtGeneral(reg int) ExtOperand { return ExtOperand{Kind: ExtGReg, Reg: reg} } +// ExtZeroRegister builds the zero-register operand, ZR: the thirty-first +// general-register slot in its zero spelling. +func ExtZeroRegister() ExtOperand { + return ExtOperand{Kind: ExtGReg, Reg: 31, Reg31: 1} +} + +// ExtStackPointer builds the stack-pointer operand, RSP: the thirty-first +// general-register slot in its stack-pointer spelling. +func ExtStackPointer() ExtOperand { + return ExtOperand{Kind: ExtGReg, Reg: 31, Reg31: 2} +} + +// ExtSIMD builds a 128-bit SIMD register operand, V2 style: the low lanes of +// the scalable vector register of the same number, the scalar destination the +// SVE reductions and inserts write. +func ExtSIMD(reg int) ExtOperand { + return ExtOperand{Kind: ExtVReg, Reg: reg} +} + // ExtImmediate builds an unshifted immediate operand. func ExtImmediate(v int64) ExtOperand { return ExtOperand{Kind: ExtImm, Imm: v} @@ -310,6 +338,7 @@ var ( extFieldPgW = ExtField{10, 4} // wide governing predicate P0-P15 extFieldPm = ExtField{16, 3} // predicate source P0-P7 extFieldPnc = ExtField{0, 3} // predicate-as-counter destination PN8-PN15, field holds reg-8 + extFieldPn4 = ExtField{5, 4} // predicate source P0-P15 at bits 8..5 (counter family) ) // ExtForm enumerates the operand shapes the extension layer defines, in Plan @@ -503,6 +532,46 @@ const ( // ZLASTA Z10.D, P3, R15. Operands: Zn.T, Pg (bare), R (R0-R30), the // arrangement feeding the size field. ExtFormZLastReg + // ExtFormZReadBack is the SVE2 crypto three-operand form whose written + // order spells the destination twice: ZSM4E Z7.S, Z6.S, Z6.S. Operands: + // Zn.T, Zd.T, Zd.T, where operands 2 and 3 name the same register and + // the encoding carries it once; the row's arrangement is locked into + // the word. + ExtFormZReadBack + // ExtFormZSameReg is the SVE2 AES unary whose source is the destination + // read back: ZAESIMC Z11.B, Z11.B. Operands: Zd.T, Zd.T, the same + // register twice, encoded once in the destination slot. + ExtFormZSameReg + // ExtFormCTerm is the SVE2 loop-condition terminator, CTERMEQ ZR, R25. + // Operands: Rm (R0-R30 or ZR), Rn (R0-R30 or ZR), both in the standard + // five-bit slots; the class sets no register result, only the flags. + ExtFormCTerm + // ExtFormCountP is the count-active-elements form, PCNTP P2.B, P14, R2. + // Operands: Pn.T, Pg (bare), Rd (R0-R30 or ZR); the arrangement feeds + // the size field, Pg occupies the wide four-bit field and Pn the + // five-bit source slot. + ExtFormCountP + // ExtFormCountPn is the predicate-counter step with the scalar result + // dropped, PDECP P14.S, ZR. Operands: Pn.T, ZR; the predicate occupies + // the four-bit field at bits 8..5, the zero register the destination + // slot, and the arrangement feeds the size field. + ExtFormCountPn + // ExtFormCountPnW is the predicate-counter step with a scalar result, + // PSQDECPW R8, P10.D, R8. Operands: Rd, Pn.T, Rd, where operands 1 and + // 3 name the same register; the predicate occupies the four-bit field at + // bits 8..5 and its arrangement feeds the size field. + ExtFormCountPnW + // ExtFormReduce is the SVE reduction into a SIMD scalar, ZSADDVD Z6.B, + // P3, V2. Operands: Zn.T, Pg (bare), Vd (V0-V31); the governing + // predicate is the narrow three-bit field and the row's arrangement is + // either locked into the word or read off Zn where the class sizes from + // the source. + ExtFormReduce + // ExtFormReduceReadBack is the strict-accumulation reduction whose SIMD + // destination is spelled twice, ZFADDAD Z9.D, V10, P2, V10. Operands: + // Zn.T, Vd, Pg (bare), Vd, where operands 2 and 4 name the same + // register. + ExtFormReduceReadBack ) // Arity returns the operand count the form takes. @@ -531,6 +600,12 @@ func (f ExtForm) Arity() int { return 2 case ExtFormPredicateOne, ExtFormPredicateWrite, ExtFormPredicateCounter: return 1 + case ExtFormZReadBack, ExtFormCountP, ExtFormCountPnW, ExtFormReduce: + return 3 + case ExtFormZSameReg, ExtFormCountPn, ExtFormCTerm: + return 2 + case ExtFormReduceReadBack: + return 4 case ExtFormNone: return 0 default: @@ -580,6 +655,22 @@ func (f ExtForm) Kinds() []ExtOperandKind { return []ExtOperandKind{ExtGReg, ExtZReg} case ExtFormZLastReg: return []ExtOperandKind{ExtZReg, ExtPReg, ExtGReg} + case ExtFormZReadBack: + return []ExtOperandKind{ExtZReg, ExtZReg, ExtZReg} + case ExtFormZSameReg: + return []ExtOperandKind{ExtZReg, ExtZReg} + case ExtFormCTerm: + return []ExtOperandKind{ExtGReg, ExtGReg} + case ExtFormCountP: + return []ExtOperandKind{ExtPReg, ExtPReg, ExtGReg} + case ExtFormCountPn: + return []ExtOperandKind{ExtPReg, ExtGReg} + case ExtFormCountPnW: + return []ExtOperandKind{ExtGReg, ExtPReg, ExtGReg} + case ExtFormReduce: + return []ExtOperandKind{ExtZReg, ExtPReg, ExtVReg} + case ExtFormReduceReadBack: + return []ExtOperandKind{ExtZReg, ExtVReg, ExtPReg, ExtVReg} case ExtFormNone: return nil default: @@ -674,6 +765,22 @@ func (f ExtForm) String() string { return "general into vector" case ExtFormZLastReg: return "last element to register" + case ExtFormZReadBack: + return "crypto vector, destination read back" + case ExtFormZSameReg: + return "crypto vector in place" + case ExtFormCTerm: + return "loop terminator" + case ExtFormCountP: + return "count to a register" + case ExtFormCountPn: + return "counter step, result dropped" + case ExtFormCountPnW: + return "counter step to a register" + case ExtFormReduce: + return "reduction to a SIMD register" + case ExtFormReduceReadBack: + return "accumulating reduction, destination read back" case ExtFormNone: return "no operands" default: @@ -862,6 +969,22 @@ func (in ExtInstr) Encode(ops []ExtOperand) ([]byte, error) { return in.encodeZFromGeneral(ops) case ExtFormZLastReg: return in.encodeZLastReg(ops) + case ExtFormZReadBack: + return in.encodeZReadBack(ops) + case ExtFormZSameReg: + return in.encodeZSameReg(ops) + case ExtFormCTerm: + return in.encodeCTerm(ops) + case ExtFormCountP: + return in.encodeCountP(ops) + case ExtFormCountPn: + return in.encodeCountPn(ops) + case ExtFormCountPnW: + return in.encodeCountPnW(ops) + case ExtFormReduce: + return in.encodeReduce(ops) + case ExtFormReduceReadBack: + return in.encodeReduceReadBack(ops) case ExtFormNone: if len(ops) != 0 { return nil, fmt.Errorf("%s: the %s form takes no operands, got %d", @@ -1772,6 +1895,277 @@ func (in ExtInstr) encodeZLastReg(ops []ExtOperand) ([]byte, error) { return extWordLE(word), nil } +// encodeZReadBack fills the SVE2 crypto form whose written order spells the +// destination twice: Zn.T, Zd.T, Zd.T (SM4E, AESD, AESE). The row's +// arrangement is locked into the word; operands 2 and 3 name the same +// register and the encoding carries it once. +func (in ExtInstr) encodeZReadBack(ops []ExtOperand) ([]byte, error) { + zn := ops[0] + if err := in.zVector(zn, 1); err != nil { + return nil, err + } + if err := in.zVector(ops[1], 2); err != nil { + return nil, err + } + if err := in.zVector(ops[2], 3); err != nil { + return nil, err + } + if ops[2].Reg != ops[1].Reg || ops[2].Arr != ops[1].Arr { + return nil, fmt.Errorf("%s: operands 2 and 3 are the same register Zd, got Z%d and Z%d", + in.Name, ops[1].Reg, ops[2].Reg) + } + word := in.Word + word = extSet(word, extFieldRn, uint32(zn.Reg)) + word = extSet(word, extFieldRd, uint32(ops[1].Reg)) + return extWordLE(word), nil +} + +// encodeZSameReg fills the in-place SVE2 AES form: Zd.T, Zd.T (AESIMC, +// AESMC), the register encoded once in the destination slot. +func (in ExtInstr) encodeZSameReg(ops []ExtOperand) ([]byte, error) { + if err := in.zVector(ops[0], 1); err != nil { + return nil, err + } + if err := in.zVector(ops[1], 2); err != nil { + return nil, err + } + if ops[1].Reg != ops[0].Reg || ops[1].Arr != ops[0].Arr { + return nil, fmt.Errorf("%s: operands 1 and 2 are the same register Zd, got Z%d and Z%d", + in.Name, ops[0].Reg, ops[1].Reg) + } + return extWordLE(extSet(in.Word, extFieldRd, uint32(ops[0].Reg))), nil +} + +// reg31Name spells how the thirty-first general-register slot was written. +func reg31Name(r uint8) string { + switch r { + case 1: + return "ZR" + case 2: + return "RSP" + default: + return "" + } +} + +// generalOperand validates a general-register operand: R0-R30, or ZR or RSP +// where allow31 names the one spelling of the thirty-first slot the class +// takes. A register outside the range is an error, never a wrap. +func (in ExtInstr) generalOperand(op ExtOperand, position int, allow31 uint8) error { + if op.Kind != ExtGReg { + return fmt.Errorf("%s: operand %d wants a general register, got %s", + in.Name, position, op.Kind) + } + if op.Reg31 == 0 { + if op.Reg < 0 || op.Reg > 30 { + return fmt.Errorf("%s: operand %d is R%d, outside R0-R30", + in.Name, position, op.Reg) + } + return nil + } + if op.Reg31 != allow31 { + return fmt.Errorf("%s: operand %d wants %s here, got %s", + in.Name, position, reg31Name(allow31), reg31Name(op.Reg31)) + } + return nil +} + +// encodeCTerm fills the SVE2 loop terminator: Rm, Rn, both in the standard +// five-bit slots, the zero register allowed in either. +func (in ExtInstr) encodeCTerm(ops []ExtOperand) ([]byte, error) { + for i, op := range ops { + if err := in.generalOperand(op, i+1, 1); err != nil { + return nil, err + } + } + word := in.Word + word = extSet(word, extFieldRm, uint32(ops[0].Reg)) + word = extSet(word, extFieldRn, uint32(ops[1].Reg)) + return extWordLE(word), nil +} + +// barePredicate validates a governing predicate of the counter and reduction +// classes: a plain P0-P15 with no suffix and no qualifier, the spelling the +// corpus writes between the source and the destination. +func (in ExtInstr) barePredicate(op ExtOperand, position, lo, hi int) error { + if err := in.predicateOperand(op, position, lo, hi, ExtArrNone); err != nil { + return err + } + if op.Arr != ExtArrNone { + return fmt.Errorf("%s: operand %d carries no arrangement suffix, got %s", + in.Name, position, op.Arr) + } + if op.Qual != ExtQualNone { + return fmt.Errorf("%s: operand %d takes no qualifier, got %q", + in.Name, position, op.Qual) + } + return nil +} + +// encodeCountP fills the count form: Pn.T, Pg (bare), Rd. The arrangement +// feeds the size field, Pg occupies the wide four-bit field and Pn the +// five-bit source slot. +func (in ExtInstr) encodeCountP(ops []ExtOperand) ([]byte, error) { + pn, pg, rd := ops[0], ops[1], ops[2] + if err := in.predicateOperand(pn, 1, 0, 15, ExtArrNone); err != nil { + return nil, err + } + if err := in.barePredicate(pg, 2, 0, 15); err != nil { + return nil, err + } + if err := in.generalOperand(rd, 3, 1); err != nil { + return nil, err + } + size, ok := pn.Arr.sizeBits() + if !ok { + return nil, fmt.Errorf("%s: arrangement %s has no size encoding in this class", + in.Name, pn.Arr) + } + word := in.Word + word = extSet(word, extFieldPgW, uint32(pg.Reg)) + word = extSet(word, extFieldRn, uint32(pn.Reg)) + word = extSet(word, extFieldRd, uint32(rd.Reg)) + word = extSet(word, in.Size, size) + return extWordLE(word), nil +} + +// encodeCountPn fills the predicate-counter step whose scalar result the +// class drops: Pn.T, ZR. The predicate occupies the four-bit field at bits +// 8..5 and its arrangement feeds the size field; the second operand is the +// zero register, the spelling that discards the count. +func (in ExtInstr) encodeCountPn(ops []ExtOperand) ([]byte, error) { + pn, rd := ops[0], ops[1] + if err := in.predicateOperand(pn, 1, 0, 15, ExtArrNone); err != nil { + return nil, err + } + // The scalar result of this class is architecturally dropped, and the + // spelling that says so is the zero register alone: a plain R0-R30 has + // no meaning here. + if rd.Kind != ExtGReg || rd.Reg31 != 1 { + return nil, fmt.Errorf("%s: operand 2 is the dropped scalar result, spelled ZR, got a %s", in.Name, rd.Kind) + } + size, ok := pn.Arr.sizeBits() + if !ok { + return nil, fmt.Errorf("%s: arrangement %s has no size encoding in this class", + in.Name, pn.Arr) + } + word := in.Word + word = extSet(word, extFieldPn4, uint32(pn.Reg)) + word = extSet(word, extFieldRd, uint32(rd.Reg)) + word = extSet(word, in.Size, size) + return extWordLE(word), nil +} + +// encodeCountPnW fills the predicate-counter step with a scalar result: +// Rd, Pn.T, Rd, the destination spelled twice. The predicate occupies the +// four-bit field at bits 8..5 and its arrangement feeds the size field. +func (in ExtInstr) encodeCountPnW(ops []ExtOperand) ([]byte, error) { + rd, pn := ops[0], ops[1] + if err := in.generalOperand(rd, 1, 1); err != nil { + return nil, err + } + if err := in.predicateOperand(pn, 2, 0, 15, ExtArrNone); err != nil { + return nil, err + } + if err := in.generalOperand(ops[2], 3, 1); err != nil { + return nil, err + } + if ops[2].Reg != rd.Reg || ops[2].Reg31 != rd.Reg31 { + return nil, fmt.Errorf("%s: operands 1 and 3 are the same register, got R%d and R%d", + in.Name, rd.Reg, ops[2].Reg) + } + size, ok := pn.Arr.sizeBits() + if !ok { + return nil, fmt.Errorf("%s: arrangement %s has no size encoding in this class", + in.Name, pn.Arr) + } + word := in.Word + word = extSet(word, extFieldPn4, uint32(pn.Reg)) + word = extSet(word, extFieldRd, uint32(rd.Reg)) + word = extSet(word, in.Size, size) + return extWordLE(word), nil +} + +// simdReg validates a 128-bit SIMD register operand of the reduction +// classes: V0-V31, written bare, no suffix and no qualifier. +func (in ExtInstr) simdReg(op ExtOperand, position int) error { + if op.Kind != ExtVReg { + return fmt.Errorf("%s: operand %d wants a SIMD register, got %s", + in.Name, position, op.Kind) + } + if op.Reg < 0 || op.Reg > 31 { + return fmt.Errorf("%s: operand %d is V%d, outside V0-V31", in.Name, position, op.Reg) + } + if op.Arr != ExtArrNone || op.Qual != ExtQualNone { + return fmt.Errorf("%s: operand %d takes no suffix, got %s%s", + in.Name, position, op.Arr, op.Qual) + } + return nil +} + +// encodeReduce fills the SVE reduction into a SIMD scalar: Zn.T, Pg (bare), +// Vd. The governing predicate is the narrow three-bit field; the size bits +// ride the row's word where the class locks the arrangement, and follow the +// source's arrangement where it does not. +func (in ExtInstr) encodeReduce(ops []ExtOperand) ([]byte, error) { + zn, pg, vd := ops[0], ops[1], ops[2] + if err := in.zVector(zn, 1); err != nil { + return nil, err + } + if err := in.barePredicate(pg, 2, 0, 7); err != nil { + return nil, err + } + if err := in.simdReg(vd, 3); err != nil { + return nil, err + } + size, err := in.zSize(zn.Arr) + if err != nil { + return nil, err + } + word := in.Word + word = extSet(word, extFieldPgN, uint32(pg.Reg)) + word = extSet(word, extFieldRn, uint32(zn.Reg)) + word = extSet(word, extFieldRd, uint32(vd.Reg)) + if size != 0 { + word = extSet(word, in.Size, size) + } + return extWordLE(word), nil +} + +// encodeReduceReadBack fills the strict-accumulation reduction: Zn.T, Vd, +// Pg (bare), Vd, the SIMD destination spelled twice. +func (in ExtInstr) encodeReduceReadBack(ops []ExtOperand) ([]byte, error) { + zn, vd, pg := ops[0], ops[1], ops[2] + if err := in.zVector(zn, 1); err != nil { + return nil, err + } + if err := in.simdReg(vd, 2); err != nil { + return nil, err + } + if err := in.barePredicate(pg, 3, 0, 7); err != nil { + return nil, err + } + if err := in.simdReg(ops[3], 4); err != nil { + return nil, err + } + if ops[3].Reg != vd.Reg { + return nil, fmt.Errorf("%s: operands 2 and 4 are the same register Vd, got V%d and V%d", + in.Name, vd.Reg, ops[3].Reg) + } + size, err := in.zSize(zn.Arr) + if err != nil { + return nil, err + } + word := in.Word + word = extSet(word, extFieldPgN, uint32(pg.Reg)) + word = extSet(word, extFieldRn, uint32(zn.Reg)) + word = extSet(word, extFieldRd, uint32(vd.Reg)) + if size != 0 { + word = extSet(word, in.Size, size) + } + return extWordLE(word), nil +} + // sharedSize returns the one element-size encoding the given arrangements // agree on, or an error when any operand is bare, they disagree, or the // arrangement has no size field. @@ -2308,6 +2702,287 @@ var arm64Extensions = []ExtInstr{ Word: 0x4400e000, Form: ExtFormVectorsZm, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, Ref: "ARM DDI 0487J SVE2.1: ZZIPQ1; inst_gen.go + arm64sveenc.s"}, {Name: "ZZIPQ2", Summary: "Z-alias of the unpredicated three-vector form", Word: 0x4400e400, Form: ExtFormVectorsZm, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, Ref: "ARM DDI 0487J SVE2.1: ZZIPQ2; inst_gen.go + arm64sveenc.s"}, + + // --- the SVE2 crypto family ---------------------------------------------- + // + // The multi-precision carry steps (ADCLB, ADCLT, SBCLB, SBCLT), the SHA3 + // rotate (RAX1), the SM4 round and key (SM4E, SM4EKEY) and the AES round + // and mix columns (AESD, AESE, AESIMC, AESMC). Every arrangement is + // locked into the word: .D for the carry steps and RAX1, .S for SM4, .B + // for AES. Provenance as above: the toolchain encoding table and its + // generated corpus, both produced from Arm's official ISA description, + // and the golden test pins every corpus line of the family. + {Name: "ZADCLB", Summary: "Add vectors with carry, borrowing from a scalar, low half", + Word: 0x4540d000, Form: ExtFormVectorsZm, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ADCLB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZADCLT", Summary: "Add vectors with carry, borrowing from a scalar, top half", + Word: 0x4540d400, Form: ExtFormVectorsZm, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ADCLT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSBCLB", Summary: "Subtract vectors with borrow, propagating a scalar, low half", + Word: 0x45c0d000, Form: ExtFormVectorsZm, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: SBCLOB/SBCLB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSBCLT", Summary: "Subtract vectors with borrow, propagating a scalar, top half", + Word: 0x45c0d400, Form: ExtFormVectorsZm, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: SBCLOT/SBCLT; inst_gen.go + arm64sveenc.s"}, + {Name: "ZRAX1", Summary: "SHA3 rotate and exclusive-or one quadword", + Word: 0x4520f400, Form: ExtFormVectorsZm, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: RAX1; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSM4EKEY", Summary: "SM4 key expansion round over vector elements", + Word: 0x4520f000, Form: ExtFormVectorsZm, Arr: ExtArrS, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: SM4EKEY; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSM4E", Summary: "SM4 encryption round over vector elements", + Word: 0x4523e000, Form: ExtFormZReadBack, Arr: ExtArrS, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: SM4E; inst_gen.go + arm64sveenc.s"}, + {Name: "ZAESD", Summary: "AES decryption round over vector elements", + Word: 0x4522e400, Form: ExtFormZReadBack, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: AESD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZAESE", Summary: "AES encryption round over vector elements", + Word: 0x4522e000, Form: ExtFormZReadBack, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: AESE; inst_gen.go + arm64sveenc.s"}, + {Name: "ZAESIMC", Summary: "AES inverse mix columns over vector elements", + Word: 0x4520e400, Form: ExtFormZSameReg, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: AESIMC; inst_gen.go + arm64sveenc.s"}, + {Name: "ZAESMC", Summary: "AES mix columns over vector elements", + Word: 0x4520e000, Form: ExtFormZSameReg, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: AESMC; inst_gen.go + arm64sveenc.s"}, + + // --- the SVE predicate counters and loop terminators ----------------------- + // + // The count of active elements into a scalar (CNTP, PFIRSTP, PLASTP), the + // predicate-counter steps whose scalar result the class drops (DECP, INCP + // and the saturating variants) or writes back (the W spellings), and the + // while-terminators CTERMEQ and CTERMNE that close SVE2 while-loops. The + // 32-bit while compares carry a W suffix and one bit of their own; the + // pattern-immediate forms of CNTP and WHILE (SVE2.1) are not yet in the + // layer. Provenance as above, pinned by the golden test. + {Name: "CTERMEQ", Summary: "Terminate a while-loop when the counters are equal", + Word: 0x25e02000, Form: ExtFormCTerm, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: CTERMEQ; inst_gen.go + arm64sveenc.s"}, + {Name: "CTERMEQW", Summary: "Terminate a while-loop when the 32-bit counters are equal", + Word: 0x25a02000, Form: ExtFormCTerm, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: CTERMEQ (32-bit); inst_gen.go + arm64sveenc.s"}, + {Name: "CTERMNE", Summary: "Terminate a while-loop when the counters differ", + Word: 0x25e02030, Form: ExtFormCTerm, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: CTERMNE; inst_gen.go + arm64sveenc.s"}, + {Name: "CTERMNEW", Summary: "Terminate a while-loop when the 32-bit counters differ", + Word: 0x25a02030, Form: ExtFormCTerm, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: CTERMNE (32-bit); inst_gen.go + arm64sveenc.s"}, + {Name: "PCNTP", Summary: "Count the active elements of a predicate into a register", + Word: 0x25208000, Form: ExtFormCountP, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: CNTP as PCNTP; inst_gen.go + arm64sveenc.s"}, + {Name: "PFIRSTP", Summary: "Write one when the first element of a predicate is active", + Word: 0x25218000, Form: ExtFormCountP, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J SVE2.1: PFIRSTP; inst_gen.go + arm64sveenc.s"}, + {Name: "PLASTP", Summary: "Write one when the last element of a predicate is active", + Word: 0x25228000, Form: ExtFormCountP, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J SVE2.1: PLASTP; inst_gen.go + arm64sveenc.s"}, + {Name: "PDECP", Summary: "Decrement a predicate counter by its active elements", + Word: 0x252d8800, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J SVE2.1: DECP (predicate) as PDECP; inst_gen.go + arm64sveenc.s"}, + {Name: "PINCP", Summary: "Increment a predicate counter by its active elements", + Word: 0x252c8800, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J SVE2.1: INCP (predicate) as PINCP; inst_gen.go + arm64sveenc.s"}, + {Name: "PSQDECP", Summary: "Decrement a predicate counter, saturating signed", + Word: 0x252a8c00, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J: SQDECP (predicate) as PSQDECP; inst_gen.go + arm64sveenc.s"}, + {Name: "PSQINCP", Summary: "Increment a predicate counter, saturating signed", + Word: 0x25288c00, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J: SQINCP (predicate) as PSQINCP; inst_gen.go + arm64sveenc.s"}, + {Name: "PUQDECP", Summary: "Decrement a predicate counter, saturating unsigned", + Word: 0x252b8c00, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J: UQDECP (predicate) as PUQDECP; inst_gen.go + arm64sveenc.s"}, + {Name: "PUQINCP", Summary: "Increment a predicate counter, saturating unsigned", + Word: 0x25298c00, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J: UQINCP (predicate) as PUQINCP; inst_gen.go + arm64sveenc.s"}, + {Name: "PUQDECPW", Summary: "Decrement a predicate counter, saturating unsigned, 32-bit", + Word: 0x252b8800, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J: UQDECP (predicate, 32-bit) as PUQDECPW; inst_gen.go + arm64sveenc.s"}, + {Name: "PUQINCPW", Summary: "Increment a predicate counter, saturating unsigned, 32-bit", + Word: 0x25298800, Form: ExtFormCountPn, Size: extSizeBHSD, Feature: ExtFeatureSVE2p1, + Ref: "ARM DDI 0487J: UQINCP (predicate, 32-bit) as PUQINCPW; inst_gen.go + arm64sveenc.s"}, + {Name: "PSQDECPW", Summary: "Decrement a register by a predicate's active elements, saturating signed", + Word: 0x252a8800, Form: ExtFormCountPnW, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SQDECP (scalar, 32-bit) as PSQDECPW; inst_gen.go + arm64sveenc.s"}, + {Name: "PSQINCPW", Summary: "Increment a register by a predicate's active elements, saturating signed", + Word: 0x25288800, Form: ExtFormCountPnW, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SQINCP (scalar, 32-bit) as PSQINCPW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILEGEW", Summary: "Build a predicate while the signed 32-bit comparison holds", + Word: 0x25200000, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILEGE (32-bit) as PWHILEGEW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILEGTW", Summary: "Build a predicate while the signed 32-bit comparison holds", + Word: 0x25200010, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILEGT (32-bit) as PWHILEGTW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILEHIW", Summary: "Build a predicate while the unsigned 32-bit comparison holds", + Word: 0x25200810, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILEHI (32-bit) as PWHILEHIW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILEHSW", Summary: "Build a predicate while the unsigned 32-bit comparison holds", + Word: 0x25200800, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILEHS (32-bit) as PWHILEHSW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILELEW", Summary: "Build a predicate while the signed 32-bit comparison holds", + Word: 0x25200410, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILELE (32-bit) as PWHILELEW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILELOW", Summary: "Build a predicate while the unsigned 32-bit comparison holds", + Word: 0x25200c00, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILELO (32-bit) as PWHILELOW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILELSW", Summary: "Build a predicate while the unsigned 32-bit comparison holds", + Word: 0x25200c10, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILELS (32-bit) as PWHILELSW; inst_gen.go + arm64sveenc.s"}, + {Name: "PWHILELTW", Summary: "Build a predicate while the signed 32-bit comparison holds", + Word: 0x25200400, Form: ExtFormWhile, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: WHILELT (32-bit) as PWHILELTW; inst_gen.go + arm64sveenc.s"}, + + // --- the SVE reductions ---------------------------------------------------- + // + // The integer, bitwise and floating-point reductions into the low lanes + // of a SIMD register: ANDV, EORV and ORV (SVE2), SADDV and UADDV with + // their D-sized accumulator, the maxima and minima, and the FP sums, + // maxima and minima including FADDA's strict accumulation. The size is + // baked into the mnemonic and the word for every family but SADDV and + // UADDV, where the source's arrangement carries it. + {Name: "ZSADDVD", Summary: "Signed sum of the active elements into a SIMD register", + Word: 0x04002000, Form: ExtFormReduce, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SADDV as ZSADDVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUADDVD", Summary: "Unsigned sum of the active elements into a SIMD register", + Word: 0x04012000, Form: ExtFormReduce, Size: extSizeBHSD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UADDV as ZUADDVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZANDVB", Summary: "Bitwise and of the active elements", + Word: 0x041a2000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ANDV as ZANDVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZANDVH", Summary: "Bitwise and of the active elements", + Word: 0x045a2000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ANDV as ZANDVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZANDVS", Summary: "Bitwise and of the active elements", + Word: 0x049a2000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ANDV as ZANDVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZANDVD", Summary: "Bitwise and of the active elements", + Word: 0x04da2000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ANDV as ZANDVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZEORVB", Summary: "Bitwise exclusive-or of the active elements", + Word: 0x04192000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: EORV as ZEORVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZEORVH", Summary: "Bitwise exclusive-or of the active elements", + Word: 0x04592000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: EORV as ZEORVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZEORVS", Summary: "Bitwise exclusive-or of the active elements", + Word: 0x04992000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: EORV as ZEORVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZEORVD", Summary: "Bitwise exclusive-or of the active elements", + Word: 0x04d92000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: EORV as ZEORVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZORVB", Summary: "Bitwise or of the active elements", + Word: 0x04182000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ORV as ZORVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZORVH", Summary: "Bitwise or of the active elements", + Word: 0x04582000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ORV as ZORVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZORVS", Summary: "Bitwise or of the active elements", + Word: 0x04982000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ORV as ZORVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZORVD", Summary: "Bitwise or of the active elements", + Word: 0x04d82000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE2, + Ref: "ARM DDI 0487J SVE2: ORV as ZORVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMAXVB", Summary: "Signed maximum of the active byte elements", + Word: 0x04082000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMAXV as ZSMAXVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMAXVH", Summary: "Signed maximum of the active halfword elements", + Word: 0x04482000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMAXV as ZSMAXVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMAXVS", Summary: "Signed maximum of the active word elements", + Word: 0x04882000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMAXV as ZSMAXVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMAXVD", Summary: "Signed maximum of the active doubleword elements", + Word: 0x04c82000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMAXV as ZSMAXVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMINVB", Summary: "Signed minimum of the active byte elements", + Word: 0x040a2000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMINV as ZSMINVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMINVH", Summary: "Signed minimum of the active halfword elements", + Word: 0x044a2000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMINV as ZSMINVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMINVS", Summary: "Signed minimum of the active word elements", + Word: 0x048a2000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMINV as ZSMINVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZSMINVD", Summary: "Signed minimum of the active doubleword elements", + Word: 0x04ca2000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: SMINV as ZSMINVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMAXVB", Summary: "Unsigned maximum of the active byte elements", + Word: 0x04092000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMAXV as ZUMAXVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMAXVH", Summary: "Unsigned maximum of the active halfword elements", + Word: 0x04492000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMAXV as ZUMAXVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMAXVS", Summary: "Unsigned maximum of the active word elements", + Word: 0x04892000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMAXV as ZUMAXVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMAXVD", Summary: "Unsigned maximum of the active doubleword elements", + Word: 0x04c92000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMAXV as ZUMAXVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMINVB", Summary: "Unsigned minimum of the active byte elements", + Word: 0x040b2000, Form: ExtFormReduce, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMINV as ZUMINVB; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMINVH", Summary: "Unsigned minimum of the active halfword elements", + Word: 0x044b2000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMINV as ZUMINVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMINVS", Summary: "Unsigned minimum of the active word elements", + Word: 0x048b2000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMINV as ZUMINVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZUMINVD", Summary: "Unsigned minimum of the active doubleword elements", + Word: 0x04cb2000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: UMINV as ZUMINVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFADDVH", Summary: "Floating-point sum of the active halfword elements", + Word: 0x65402000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FADDV as ZFADDVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFADDVS", Summary: "Floating-point sum of the active word elements", + Word: 0x65802000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FADDV as ZFADDVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFADDVD", Summary: "Floating-point sum of the active doubleword elements", + Word: 0x65c02000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FADDV as ZFADDVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMAXNMVH", Summary: "Floating-point maximum of the active halfword elements, quiet NaN", + Word: 0x65442000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMAXNMV as ZFMAXNMVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMAXNMVS", Summary: "Floating-point maximum of the active word elements, quiet NaN", + Word: 0x65842000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMAXNMV as ZFMAXNMVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMAXNMVD", Summary: "Floating-point maximum of the active doubleword elements, quiet NaN", + Word: 0x65c42000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMAXNMV as ZFMAXNMVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMAXVH", Summary: "Floating-point maximum of the active halfword elements", + Word: 0x65462000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMAXV as ZFMAXVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMAXVS", Summary: "Floating-point maximum of the active word elements", + Word: 0x65862000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMAXV as ZFMAXVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMAXVD", Summary: "Floating-point maximum of the active doubleword elements", + Word: 0x65c62000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMAXV as ZFMAXVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMINNMVH", Summary: "Floating-point minimum of the active halfword elements, quiet NaN", + Word: 0x65452000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMINNMV as ZFMINNMVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMINNMVS", Summary: "Floating-point minimum of the active word elements, quiet NaN", + Word: 0x65852000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMINNMV as ZFMINNMVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMINNMVD", Summary: "Floating-point minimum of the active doubleword elements, quiet NaN", + Word: 0x65c52000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMINNMV as ZFMINNMVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMINVH", Summary: "Floating-point minimum of the active halfword elements", + Word: 0x65472000, Form: ExtFormReduce, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMINV as ZFMINVH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMINVS", Summary: "Floating-point minimum of the active word elements", + Word: 0x65872000, Form: ExtFormReduce, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMINV as ZFMINVS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFMINVD", Summary: "Floating-point minimum of the active doubleword elements", + Word: 0x65c72000, Form: ExtFormReduce, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FMINV as ZFMINVD; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFADDAH", Summary: "Strict floating-point accumulation of the active halfword elements", + Word: 0x65582000, Form: ExtFormReduceReadBack, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FADDA as ZFADDAH; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFADDAS", Summary: "Strict floating-point accumulation of the active word elements", + Word: 0x65982000, Form: ExtFormReduceReadBack, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FADDA as ZFADDAS; inst_gen.go + arm64sveenc.s"}, + {Name: "ZFADDAD", Summary: "Strict floating-point accumulation of the active doubleword elements", + Word: 0x65d82000, Form: ExtFormReduceReadBack, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: FADDA as ZFADDAD; inst_gen.go + arm64sveenc.s"}, } // Extensions returns the extended-instruction layer registered for a, outside diff --git a/arch/arm64_ext_test.go b/arch/arm64_ext_test.go index c006ec3..9452504 100644 --- a/arch/arm64_ext_test.go +++ b/arch/arm64_ext_test.go @@ -316,7 +316,11 @@ func TestArm64ExtTableIntegrity(t *testing.T) { } roundOne := in.Form == ExtFormVectors || in.Form == ExtFormPredicated || in.Form == ExtFormImmediate || in.Form == ExtFormSignedImmediate - if in.Word&destMask != 0 || (roundOne && in.Size.Width != 0 && in.Word&(0x3<<22) != 0) { + // The loop terminators carry no destination at all, and the NE + // variants put their opcode right in the slot another class would + // fill with one, so the zero-destination invariant exempts them. + term := in.Form == ExtFormCTerm + if !term && (in.Word&destMask != 0 || (roundOne && in.Size.Width != 0 && in.Word&(0x3<<22) != 0)) { t.Errorf("%s: word %08x carries destination or size bits, want them zero", in.Name, in.Word) } } @@ -1068,3 +1072,253 @@ func TestArm64ExtZGolden(t *testing.T) { } } } + +// TestArm64ExtStage3Golden pins the stage-three families, the SVE2 crypto +// group (the multi-precision carry steps, the SHA3 rotate, SM4 and AES), the +// predicate counters and loop terminators (CNTP, the DECP and INCP steps, +// the saturating variants and CTERM), the 32-bit while compares, and the SVE +// reductions into a SIMD register (ANDV, EORV, ORV, SADDV, UADDV, the maxima +// and minima, the floating-point sums and FADDA). Every row is one +// arm64sveenc.s line and the want word is that line's own encoding, the +// same provenance as the families above: the toolchain encoding table and +// its generated corpus, both produced from Arm's official ISA description. +func TestArm64ExtStage3Golden(t *testing.T) { + for _, tt := range []struct { + name string + mnem string + form ExtForm + arr ExtArrangement + ops []ExtOperand + want uint32 + }{ + // The crypto family: Zm, Zn, Zd over a locked arrangement. + {"ZADCLB Z7.D, Z23.D, Z13.D", "ZADCLB", ExtFormVectorsZm, ExtArrD, + []ExtOperand{ExtVector(7, ExtArrD), ExtVector(23, ExtArrD), ExtVector(13, ExtArrD)}, + 0x4547d2ed}, + {"ZADCLT Z7.D, Z23.D, Z13.D", "ZADCLT", ExtFormVectorsZm, ExtArrD, + []ExtOperand{ExtVector(7, ExtArrD), ExtVector(23, ExtArrD), ExtVector(13, ExtArrD)}, + 0x4547d6ed}, + {"ZSBCLB Z7.D, Z23.D, Z13.D", "ZSBCLB", ExtFormVectorsZm, ExtArrD, + []ExtOperand{ExtVector(7, ExtArrD), ExtVector(23, ExtArrD), ExtVector(13, ExtArrD)}, + 0x45c7d2ed}, + {"ZSBCLT Z7.D, Z23.D, Z13.D", "ZSBCLT", ExtFormVectorsZm, ExtArrD, + []ExtOperand{ExtVector(7, ExtArrD), ExtVector(23, ExtArrD), ExtVector(13, ExtArrD)}, + 0x45c7d6ed}, + {"ZRAX1 Z7.D, Z6.D, Z23.D", "ZRAX1", ExtFormVectorsZm, ExtArrD, + []ExtOperand{ExtVector(7, ExtArrD), ExtVector(6, ExtArrD), ExtVector(23, ExtArrD)}, + 0x4527f4d7}, + {"ZSM4EKEY Z7.S, Z6.S, Z23.S", "ZSM4EKEY", ExtFormVectorsZm, ExtArrS, + []ExtOperand{ExtVector(7, ExtArrS), ExtVector(6, ExtArrS), ExtVector(23, ExtArrS)}, + 0x4527f0d7}, + {"ZSM4E Z7.S, Z6.S, Z6.S", "ZSM4E", ExtFormZReadBack, ExtArrS, + []ExtOperand{ExtVector(7, ExtArrS), ExtVector(6, ExtArrS), ExtVector(6, ExtArrS)}, + 0x4523e0e6}, + {"ZAESD Z7.B, Z6.B, Z6.B", "ZAESD", ExtFormZReadBack, ExtArrB, + []ExtOperand{ExtVector(7, ExtArrB), ExtVector(6, ExtArrB), ExtVector(6, ExtArrB)}, + 0x4522e4e6}, + {"ZAESE Z7.B, Z6.B, Z6.B", "ZAESE", ExtFormZReadBack, ExtArrB, + []ExtOperand{ExtVector(7, ExtArrB), ExtVector(6, ExtArrB), ExtVector(6, ExtArrB)}, + 0x4522e0e6}, + {"ZAESIMC Z11.B, Z11.B", "ZAESIMC", ExtFormZSameReg, ExtArrB, + []ExtOperand{ExtVector(11, ExtArrB), ExtVector(11, ExtArrB)}, + 0x4520e40b}, + {"ZAESMC Z11.B, Z11.B", "ZAESMC", ExtFormZSameReg, ExtArrB, + []ExtOperand{ExtVector(11, ExtArrB), ExtVector(11, ExtArrB)}, + 0x4520e00b}, + // The terminators and counters. + {"CTERMEQ ZR, R25", "CTERMEQ", ExtFormCTerm, ExtArrNone, + []ExtOperand{ExtZeroRegister(), ExtGeneral(25)}, + 0x25ff2320}, + {"CTERMEQW ZR, R25", "CTERMEQW", ExtFormCTerm, ExtArrNone, + []ExtOperand{ExtZeroRegister(), ExtGeneral(25)}, + 0x25bf2320}, + {"CTERMNE ZR, R25", "CTERMNE", ExtFormCTerm, ExtArrNone, + []ExtOperand{ExtZeroRegister(), ExtGeneral(25)}, + 0x25ff2330}, + {"CTERMNEW ZR, R25", "CTERMNEW", ExtFormCTerm, ExtArrNone, + []ExtOperand{ExtZeroRegister(), ExtGeneral(25)}, + 0x25bf2330}, + {"PCNTP P2.B, P14, R2", "PCNTP", ExtFormCountP, ExtArrNone, + []ExtOperand{ExtPredicateSized(2, ExtArrB), ExtPredicateSized(14, ExtArrNone), ExtGeneral(2)}, + 0x2520b842}, + {"PFIRSTP P2.B, P14, R2", "PFIRSTP", ExtFormCountP, ExtArrNone, + []ExtOperand{ExtPredicateSized(2, ExtArrB), ExtPredicateSized(14, ExtArrNone), ExtGeneral(2)}, + 0x2521b842}, + {"PLASTP P2.B, P14, R2", "PLASTP", ExtFormCountP, ExtArrNone, + []ExtOperand{ExtPredicateSized(2, ExtArrB), ExtPredicateSized(14, ExtArrNone), ExtGeneral(2)}, + 0x2522b842}, + {"PDECP P14.S, ZR", "PDECP", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25ad89df}, + {"PINCP P14.S, ZR", "PINCP", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25ac89df}, + {"PSQDECP P14.S, ZR", "PSQDECP", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25aa8ddf}, + {"PSQINCP P14.S, ZR", "PSQINCP", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25a88ddf}, + {"PUQDECP P14.S, ZR", "PUQDECP", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25ab8ddf}, + {"PUQINCP P14.S, ZR", "PUQINCP", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25a98ddf}, + {"PUQDECPW P14.S, ZR", "PUQDECPW", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25ab89df}, + {"PUQINCPW P14.S, ZR", "PUQINCPW", ExtFormCountPn, ExtArrNone, + []ExtOperand{ExtPredicateSized(14, ExtArrS), ExtZeroRegister()}, + 0x25a989df}, + {"PSQDECPW R8, P10.D, R8", "PSQDECPW", ExtFormCountPnW, ExtArrNone, + []ExtOperand{ExtGeneral(8), ExtPredicateSized(10, ExtArrD), ExtGeneral(8)}, + 0x25ea8948}, + {"PSQINCPW R8, P10.D, R8", "PSQINCPW", ExtFormCountPnW, ExtArrNone, + []ExtOperand{ExtGeneral(8), ExtPredicateSized(10, ExtArrD), ExtGeneral(8)}, + 0x25e88948}, + {"PWHILEGEW R2, R10, P10.H", "PWHILEGEW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x2562014a}, + {"PWHILEGTW R2, R10, P10.H", "PWHILEGTW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x2562015a}, + {"PWHILEHIW R2, R10, P10.H", "PWHILEHIW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x2562095a}, + {"PWHILEHSW R2, R10, P10.H", "PWHILEHSW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x2562094a}, + {"PWHILELEW R2, R10, P10.H", "PWHILELEW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x2562055a}, + {"PWHILELOW R2, R10, P10.H", "PWHILELOW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x25620d4a}, + {"PWHILELSW R2, R10, P10.H", "PWHILELSW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x25620d5a}, + {"PWHILELTW R2, R10, P10.H", "PWHILELTW", ExtFormWhile, ExtArrNone, + []ExtOperand{ExtGeneral(2), ExtGeneral(10), ExtPredicateSized(10, ExtArrH)}, + 0x2562054a}, + // The reductions. + {"ZSADDVD Z6.B, P3, V2", "ZSADDVD", ExtFormReduce, ExtArrNone, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x04002cc2}, + {"ZSADDVD Z6.D, P3, V2", "ZSADDVD", ExtFormReduce, ExtArrNone, + []ExtOperand{ExtVector(6, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x04c02cc2}, + {"ZUADDVD Z10.D, P3, V15", "ZUADDVD", ExtFormReduce, ExtArrNone, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04c12d4f}, + {"ZUADDVD Z10.B, P3, V15", "ZUADDVD", ExtFormReduce, ExtArrNone, + []ExtOperand{ExtVector(10, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04012d4f}, + {"ZANDVB Z6.B, P3, V2", "ZANDVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x041a2cc2}, + {"ZANDVH Z3.H, P1, V29", "ZANDVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x045a247d}, + {"ZANDVS Z17.S, P1, V27", "ZANDVS", ExtFormReduce, ExtArrS, + []ExtOperand{ExtVector(17, ExtArrS), ExtPredicateSized(1, ExtArrNone), ExtSIMD(27)}, + 0x049a263b}, + {"ZANDVD Z10.D, P3, V15", "ZANDVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04da2d4f}, + {"ZEORVB Z6.B, P3, V2", "ZEORVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x04192cc2}, + {"ZEORVD Z10.D, P3, V15", "ZEORVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04d92d4f}, + {"ZORVB Z6.B, P3, V2", "ZORVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x04182cc2}, + {"ZORVD Z10.D, P3, V15", "ZORVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04d82d4f}, + {"ZSMAXVB Z6.B, P3, V2", "ZSMAXVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x04082cc2}, + {"ZSMAXVH Z3.H, P1, V29", "ZSMAXVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x0448247d}, + {"ZSMAXVS Z17.S, P1, V27", "ZSMAXVS", ExtFormReduce, ExtArrS, + []ExtOperand{ExtVector(17, ExtArrS), ExtPredicateSized(1, ExtArrNone), ExtSIMD(27)}, + 0x0488263b}, + {"ZSMAXVD Z10.D, P3, V15", "ZSMAXVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04c82d4f}, + {"ZSMINVB Z6.B, P3, V2", "ZSMINVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x040a2cc2}, + {"ZSMINVD Z10.D, P3, V15", "ZSMINVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04ca2d4f}, + {"ZUMAXVB Z6.B, P3, V2", "ZUMAXVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x04092cc2}, + {"ZUMAXVD Z10.D, P3, V15", "ZUMAXVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04c92d4f}, + {"ZUMINVB Z6.B, P3, V2", "ZUMINVB", ExtFormReduce, ExtArrB, + []ExtOperand{ExtVector(6, ExtArrB), ExtPredicateSized(3, ExtArrNone), ExtSIMD(2)}, + 0x040b2cc2}, + {"ZUMINVD Z10.D, P3, V15", "ZUMINVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x04cb2d4f}, + {"ZFADDVH Z3.H, P1, V29", "ZFADDVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x6540247d}, + {"ZFADDVS Z17.S, P1, V27", "ZFADDVS", ExtFormReduce, ExtArrS, + []ExtOperand{ExtVector(17, ExtArrS), ExtPredicateSized(1, ExtArrNone), ExtSIMD(27)}, + 0x6580263b}, + {"ZFADDVD Z10.D, P3, V15", "ZFADDVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x65c02d4f}, + {"ZFMAXNMVH Z3.H, P1, V29", "ZFMAXNMVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x6544247d}, + {"ZFMAXNMVD Z10.D, P3, V15", "ZFMAXNMVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x65c42d4f}, + {"ZFMAXVH Z3.H, P1, V29", "ZFMAXVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x6546247d}, + {"ZFMAXVD Z10.D, P3, V15", "ZFMAXVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x65c62d4f}, + {"ZFMINNMVH Z3.H, P1, V29", "ZFMINNMVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x6545247d}, + {"ZFMINNMVD Z10.D, P3, V15", "ZFMINNMVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x65c52d4f}, + {"ZFMINVH Z3.H, P1, V29", "ZFMINVH", ExtFormReduce, ExtArrH, + []ExtOperand{ExtVector(3, ExtArrH), ExtPredicateSized(1, ExtArrNone), ExtSIMD(29)}, + 0x6547247d}, + {"ZFMINVD Z10.D, P3, V15", "ZFMINVD", ExtFormReduce, ExtArrD, + []ExtOperand{ExtVector(10, ExtArrD), ExtPredicateSized(3, ExtArrNone), ExtSIMD(15)}, + 0x65c72d4f}, + {"ZFADDAH Z8.H, V15, P2, V15", "ZFADDAH", ExtFormReduceReadBack, ExtArrH, + []ExtOperand{ExtVector(8, ExtArrH), ExtSIMD(15), ExtPredicateSized(2, ExtArrNone), ExtSIMD(15)}, + 0x6558290f}, + {"ZFADDAS Z26.S, V30, P7, V30", "ZFADDAS", ExtFormReduceReadBack, ExtArrS, + []ExtOperand{ExtVector(26, ExtArrS), ExtSIMD(30), ExtPredicateSized(7, ExtArrNone), ExtSIMD(30)}, + 0x65983f5e}, + {"ZFADDAD Z9.D, V10, P2, V10", "ZFADDAD", ExtFormReduceReadBack, ExtArrD, + []ExtOperand{ExtVector(9, ExtArrD), ExtSIMD(10), ExtPredicateSized(2, ExtArrNone), ExtSIMD(10)}, + 0x65d8292a}, + } { + in := extInstructionMeta(t, tt.mnem, tt.form, ExtQualNone, tt.arr) + got, err := in.Encode(tt.ops) + if err != nil { + t.Errorf("%s: encode: %v", tt.name, err) + continue + } + if want := hex.EncodeToString(extWordLE(tt.want)); hex.EncodeToString(got) != want { + t.Errorf("%s:\n got %x\n want %s", tt.name, got, want) + } + } +} diff --git a/asm/arm64_ext.go b/asm/arm64_ext.go index 5eed75e..657b760 100644 --- a/asm/arm64_ext.go +++ b/asm/arm64_ext.go @@ -15,7 +15,11 @@ // classes, and for the predicate family Pm.B, Pn.B, Pg/Z (or Pg.Z), Pd.B // for the logical operations, Pn.B, Pg.Z, Pd.B for the breaks, Pm.T, Pn.T, // Pd.T for the permutations, Rm, Rn, Pd.T for the while compares, PN8-PN15 -// for the counter destinations, and the bare SETFFR. +// for the counter destinations, and the bare SETFFR. Stage three adds the +// crypto family (Zn.T, Zd.T, Zd.T read-back and the in-place Zd.T, Zd.T), +// the predicate counters (Pn.T, Pg, Rd; Pn.T, ZR; Rd, Pn.T, Rd; ZR and R +// terminators) and the reductions (Zn.T, Pg, Vd over the SIMD register +// V0-V31, with ZR and RSP accepted where the classes take them). package asm @@ -45,7 +49,7 @@ func arm64ExtStatement(mnem string, ops []*ast.Operand) (extops []arch.ExtOperan if _, ok := LookupExtension(arch.ARM64, mnem); !ok { return nil, false, nil } - if !arm64ExtPinned(ops) { + if !arm64ExtPinned(mnem, ops) { return nil, false, nil } out := make([]arch.ExtOperand, 0, len(ops)) @@ -87,6 +91,18 @@ func arm64ExtStatement(mnem string, ops []*ast.Operand) (extops []arch.ExtOperan out = append(out, ext) continue } + if text == "ZR" { + out = append(out, arch.ExtZeroRegister()) + continue + } + if text == "RSP" { + out = append(out, arch.ExtStackPointer()) + continue + } + if ext, ok := arm64ExtSIMD(text); ok { + out = append(out, ext) + continue + } if ext, ok := arm64ExtGeneral(text); ok { out = append(out, ext) continue @@ -96,18 +112,27 @@ func arm64ExtStatement(mnem string, ops []*ast.Operand) (extops []arch.ExtOperan return out, true, nil } -// arm64ExtPinned reports whether the statement belongs to the layer: any -// operand is a scalable vector, predicate or predicate-as-counter register, -// the shapes only the extension layer reads, or the statement carries no -// operands at all and the mnemonic's zero-operand forms claim it. The test -// is deliberately loose about the suffixes: P0/B is not a spelling the -// layer takes, but the P of it makes the statement the layer's, and the -// conversion then diagnoses the operand precisely instead of leaving it to -// a scalar path that would report an unrelated register error. -func arm64ExtPinned(ops []*ast.Operand) bool { +// arm64ExtPinned reports whether the statement belongs to the layer. A +// mnemonic the extension layer registers on its own, one the generated +// arm64 table does not know, owns every one of its statements: no scalar +// path could mean it instead, and the layer's diagnostics replace the +// unsupported-instruction complaint. A mnemonic both tables carry (the +// SVE aliases of ADD, SUB and MUL) keeps the operand-shape test: any +// operand is a scalable vector, predicate or predicate-as-counter +// register, the shapes only the extension layer reads, or the statement +// carries no operands at all and the mnemonic's zero-operand forms claim +// it. The shape test is deliberately loose about the suffixes: P0/B is +// not a spelling the layer takes, but the P of it makes the statement the +// layer's, and the conversion then diagnoses the operand precisely +// instead of leaving it to a scalar path that would report an unrelated +// register error. +func arm64ExtPinned(mnem string, ops []*ast.Operand) bool { if len(ops) == 0 { return true } + if _, shared := a64InstrTable[mnem]; !shared { + return true + } for _, op := range ops { if op.Kind == ast.OpImmediate { continue @@ -230,8 +255,9 @@ func arm64ExtCounter(text string) (arch.ExtOperand, bool) { } // arm64ExtGeneral parses a general register operand: R0..R30, the plain -// spelling the while-compare forms take. The register range is left to the -// encoding, whose diagnostics name it. +// spelling the while-compare forms take, beside the ZR and RSP spellings of +// the thirty-first slot the conversion above reads. The register range is +// left to the encoding, whose diagnostics name it. func arm64ExtGeneral(text string) (arch.ExtOperand, bool) { rest, ok := strings.CutPrefix(text, "R") if !ok { @@ -244,6 +270,21 @@ func arm64ExtGeneral(text string) (arch.ExtOperand, bool) { return arch.ExtOperand{Kind: arch.ExtGReg, Reg: reg}, true } +// arm64ExtSIMD parses a 128-bit SIMD register operand: V0..V31, written +// bare, the scalar destination the reductions and the crypto read-back +// forms take. The register range is left to the encoding. +func arm64ExtSIMD(text string) (arch.ExtOperand, bool) { + rest, ok := strings.CutPrefix(text, "V") + if !ok { + return arch.ExtOperand{}, false + } + reg, arr, ok := arm64ExtRegDigits(rest) + if !ok || arr != arch.ExtArrNone { + return arch.ExtOperand{}, false + } + return arch.ExtOperand{Kind: arch.ExtVReg, Reg: reg}, true +} + // arm64ExtRegDigits parses the digits and optional arrangement suffix of a // register spelling once the letter prefix is gone. func arm64ExtRegDigits(text string) (reg int, arr arch.ExtArrangement, ok bool) { diff --git a/asm/arm64_ext_asm_test.go b/asm/arm64_ext_asm_test.go index 5da9114..bf6a874 100644 --- a/asm/arm64_ext_asm_test.go +++ b/asm/arm64_ext_asm_test.go @@ -424,3 +424,141 @@ func TestArm64AssembleExtensionStage2Golden(t *testing.T) { } } } + +// TestArm64AssembleExtensionStage3Golden drives the stage-three families +// through the full assembler: corpus text in, corpus word out. The +// statements carry the spellings the layer reads: the read-back crypto +// destinations, the ZR of the dropped counter results, the bare governing +// predicates and the V destinations of the reductions. +func TestArm64AssembleExtensionStage3Golden(t *testing.T) { + tests := []struct { + stmt string + want uint32 + }{ + // The crypto family. + {"ZADCLB Z7.D, Z23.D, Z13.D", 0x4547d2ed}, + {"ZADCLT Z7.D, Z23.D, Z13.D", 0x4547d6ed}, + {"ZSBCLB Z7.D, Z23.D, Z13.D", 0x45c7d2ed}, + {"ZSBCLT Z7.D, Z23.D, Z13.D", 0x45c7d6ed}, + {"ZRAX1 Z7.D, Z6.D, Z23.D", 0x4527f4d7}, + {"ZSM4EKEY Z7.S, Z6.S, Z23.S", 0x4527f0d7}, + {"ZSM4E Z7.S, Z6.S, Z6.S", 0x4523e0e6}, + {"ZAESD Z7.B, Z6.B, Z6.B", 0x4522e4e6}, + {"ZAESE Z7.B, Z6.B, Z6.B", 0x4522e0e6}, + {"ZAESIMC Z11.B, Z11.B", 0x4520e40b}, + {"ZAESMC Z11.B, Z11.B", 0x4520e00b}, + // The terminators and counters. + {"CTERMEQ ZR, R25", 0x25ff2320}, + {"CTERMEQW ZR, R25", 0x25bf2320}, + {"CTERMNE ZR, R25", 0x25ff2330}, + {"CTERMNEW ZR, R25", 0x25bf2330}, + {"CTERMEQ R0, R25", 0x25e02320}, + {"PCNTP P2.B, P14, R2", 0x2520b842}, + {"PFIRSTP P2.B, P14, R2", 0x2521b842}, + {"PLASTP P2.B, P14, R2", 0x2522b842}, + {"PDECP P14.S, ZR", 0x25ad89df}, + {"PINCP P14.S, ZR", 0x25ac89df}, + {"PSQDECP P14.S, ZR", 0x25aa8ddf}, + {"PSQINCP P14.S, ZR", 0x25a88ddf}, + {"PUQDECP P14.S, ZR", 0x25ab8ddf}, + {"PUQINCP P14.S, ZR", 0x25a98ddf}, + {"PUQDECPW P14.S, ZR", 0x25ab89df}, + {"PUQINCPW P14.S, ZR", 0x25a989df}, + {"PSQDECPW R8, P10.D, R8", 0x25ea8948}, + {"PSQINCPW R8, P10.D, R8", 0x25e88948}, + {"PWHILEGEW R2, R10, P10.H", 0x2562014a}, + {"PWHILEGTW R2, R10, P10.H", 0x2562015a}, + {"PWHILEHIW R2, R10, P10.H", 0x2562095a}, + {"PWHILEHSW R2, R10, P10.H", 0x2562094a}, + {"PWHILELEW R2, R10, P10.H", 0x2562055a}, + {"PWHILELOW R2, R10, P10.H", 0x25620d4a}, + {"PWHILELSW R2, R10, P10.H", 0x25620d5a}, + {"PWHILELTW R2, R10, P10.H", 0x2562054a}, + // The reductions. + {"ZSADDVD Z6.B, P3, V2", 0x04002cc2}, + {"ZUADDVD Z10.D, P3, V15", 0x04c12d4f}, + {"ZANDVB Z6.B, P3, V2", 0x041a2cc2}, + {"ZANDVH Z3.H, P1, V29", 0x045a247d}, + {"ZANDVS Z17.S, P1, V27", 0x049a263b}, + {"ZANDVD Z10.D, P3, V15", 0x04da2d4f}, + {"ZEORVB Z6.B, P3, V2", 0x04192cc2}, + {"ZEORVD Z10.D, P3, V15", 0x04d92d4f}, + {"ZORVB Z6.B, P3, V2", 0x04182cc2}, + {"ZORVD Z10.D, P3, V15", 0x04d82d4f}, + {"ZSMAXVB Z6.B, P3, V2", 0x04082cc2}, + {"ZSMAXVH Z3.H, P1, V29", 0x0448247d}, + {"ZSMAXVS Z17.S, P1, V27", 0x0488263b}, + {"ZSMAXVD Z10.D, P3, V15", 0x04c82d4f}, + {"ZSMINVB Z6.B, P3, V2", 0x040a2cc2}, + {"ZSMINVD Z10.D, P3, V15", 0x04ca2d4f}, + {"ZUMAXVB Z6.B, P3, V2", 0x04092cc2}, + {"ZUMAXVD Z10.D, P3, V15", 0x04c92d4f}, + {"ZUMINVB Z6.B, P3, V2", 0x040b2cc2}, + {"ZUMINVD Z10.D, P3, V15", 0x04cb2d4f}, + {"ZFADDVH Z3.H, P1, V29", 0x6540247d}, + {"ZFADDVS Z17.S, P1, V27", 0x6580263b}, + {"ZFADDVD Z10.D, P3, V15", 0x65c02d4f}, + {"ZFMAXNMVH Z3.H, P1, V29", 0x6544247d}, + {"ZFMAXNMVD Z10.D, P3, V15", 0x65c42d4f}, + {"ZFMAXVH Z3.H, P1, V29", 0x6546247d}, + {"ZFMAXVD Z10.D, P3, V15", 0x65c62d4f}, + {"ZFMINNMVH Z3.H, P1, V29", 0x6545247d}, + {"ZFMINNMVD Z10.D, P3, V15", 0x65c52d4f}, + {"ZFMINVH Z3.H, P1, V29", 0x6547247d}, + {"ZFMINVD Z10.D, P3, V15", 0x65c72d4f}, + {"ZFADDAH Z8.H, V15, P2, V15", 0x6558290f}, + {"ZFADDAS Z26.S, V30, P7, V30", 0x65983f5e}, + {"ZFADDAD Z9.D, V10, P2, V10", 0x65d8292a}, + } + for _, tt := range tests { + words := assembleArm64Words(t, arm64ExtProbeHead+"\t"+tt.stmt+"\n\tRET\n") + if len(words) != 2 { + t.Fatalf("%s: got %d words, want the statement and RET", tt.stmt, len(words)) + } + if words[0] != tt.want { + t.Errorf("%s:\n got %08x\n want %08x", tt.stmt, words[0], tt.want) + } + if words[1] != 0xd65f03c0 { + t.Errorf("%s: RET encoded %08x", tt.stmt, words[1]) + } + } +} + +// TestArm64AssembleExtensionStage3Refusals pins the diagnostics the +// stage-three statements get from the layer: the read-back mismatches, the +// locked arrangements, the dropped-result zero register and the narrow +// governing predicates. +func TestArm64AssembleExtensionStage3Refusals(t *testing.T) { + tests := []struct { + stmt string + want string + }{ + {"ZSM4E Z7.S, Z6.S, Z5.S", "the same register Zd"}, + {"ZSM4E Z7.D, Z6.D, Z6.D", "want .S"}, + {"ZAESIMC Z11.B, Z12.B", "the same register Zd"}, + {"ZADCLB Z7.S, Z23.S, Z13.S", "want .D"}, + {"CTERMEQ RSP, R25", "wants ZR here"}, + {"CTERMEQ R31, R25", "outside R0-R30"}, + {"PCNTP P2.B, P14, R31", "outside R0-R30"}, + {"PCNTP P2.Q, P14, R2", "no size encoding"}, + {"PDECP P14.S, R0", "spelled ZR"}, + {"PDECP P14.Q, ZR", "no size encoding"}, + {"PSQDECPW R8, P10.D, R9", "the same register"}, + {"ZSADDVD Z6.B, P9, V2", "outside P0-P7"}, + {"ZSADDVD Z6.B, P3/M, V2", "takes no qualifier"}, + {"ZSADDVD Z6.B, P3, V32", "outside V0-V31"}, + {"ZSADDVD Z6.B, P3, R2", "wants a SIMD register"}, + {"ZANDVB Z6.S, P3, V2", "want .B"}, + {"ZFADDAD Z9.D, V10, P2, V11", "the same register Vd"}, + {"PWHILEGEW R2, R10, P10.H", ""}, + } + for _, tt := range tests { + if tt.want == "" { + continue + } + got := assembleArm64SrcError(t, arm64ExtProbeHead+"\t"+tt.stmt+"\n\tRET\n") + if !strings.Contains(got, tt.want) { + t.Errorf("%s: error %q does not name %q", tt.stmt, got, tt.want) + } + } +} diff --git a/asm/extension_test.go b/asm/extension_test.go index e0d7269..a3973cd 100644 --- a/asm/extension_test.go +++ b/asm/extension_test.go @@ -273,12 +273,99 @@ func TestExtensionNamesARM64(t *testing.T) { "ZZIP2", "ZZIPQ1", "ZZIPQ2", + // The stage-three families: the SVE2 crypto group, the predicate + // counters and loop terminators with the 32-bit while compares, and + // the reductions into a SIMD register. + "ZADCLB", + "ZADCLT", + "ZSBCLB", + "ZSBCLT", + "ZRAX1", + "ZSM4EKEY", + "ZSM4E", + "ZAESD", + "ZAESE", + "ZAESIMC", + "ZAESMC", + "CTERMEQ", + "CTERMEQW", + "CTERMNE", + "CTERMNEW", + "PCNTP", + "PFIRSTP", + "PLASTP", + "PDECP", + "PINCP", + "PSQDECP", + "PSQINCP", + "PUQDECP", + "PUQINCP", + "PUQDECPW", + "PUQINCPW", + "PSQDECPW", + "PSQINCPW", + "PWHILEGEW", + "PWHILEGTW", + "PWHILEHIW", + "PWHILEHSW", + "PWHILELEW", + "PWHILELOW", + "PWHILELSW", + "PWHILELTW", + "ZSADDVD", + "ZUADDVD", + "ZANDVB", + "ZANDVH", + "ZANDVS", + "ZANDVD", + "ZEORVB", + "ZEORVH", + "ZEORVS", + "ZEORVD", + "ZORVB", + "ZORVH", + "ZORVS", + "ZORVD", + "ZSMAXVB", + "ZSMAXVH", + "ZSMAXVS", + "ZSMAXVD", + "ZSMINVB", + "ZSMINVH", + "ZSMINVS", + "ZSMINVD", + "ZUMAXVB", + "ZUMAXVH", + "ZUMAXVS", + "ZUMAXVD", + "ZUMINVB", + "ZUMINVH", + "ZUMINVS", + "ZUMINVD", + "ZFADDVH", + "ZFADDVS", + "ZFADDVD", + "ZFMAXNMVH", + "ZFMAXNMVS", + "ZFMAXNMVD", + "ZFMAXVH", + "ZFMAXVS", + "ZFMAXVD", + "ZFMINNMVH", + "ZFMINNMVS", + "ZFMINNMVD", + "ZFMINVH", + "ZFMINVS", + "ZFMINVD", + "ZFADDAH", + "ZFADDAS", + "ZFADDAD", } got := ExtensionNames(arch.ARM64) if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("ExtensionNames(ARM64) = %v, want %v", got, want) } - if n := len(arch.Extensions(arch.ARM64)); n != 167 { - t.Errorf("the family registers %d instructions, want 164", n) + if n := len(arch.Extensions(arch.ARM64)); n != 251 { + t.Errorf("the family registers %d instructions, want 251", n) } }