feat(asm): add EVEX masking, zeroing and the AVX-512 F/BW integer set
Assisted-by: Qwen 3.8 Max Preview
This commit is contained in:
+18
-5
@@ -51,9 +51,16 @@ func (e *enc) encode(mnem string, ops []Operand) error {
|
||||
|
||||
// VEX (AVX/AVX2) and EVEX (AVX-512) instructions: the trailing
|
||||
// B/W/L/Q/D is part of the mnemonic, not a size suffix, so dispatch
|
||||
// before splitSize.
|
||||
if isVex(upper) || isEvex(upper) || upper == "KMOVW" {
|
||||
return e.encodeVec(upper, ops)
|
||||
// before splitSize. A ".Z" suffix requests EVEX zeroing.
|
||||
base, zeroing, err := stripEvexSuffix(upper)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if isVex(base) || isEvex(base) || base == "KMOVW" {
|
||||
return e.encodeVec(base, ops, zeroing)
|
||||
}
|
||||
if zeroing {
|
||||
return fmt.Errorf("%s: the .Z suffix requires an EVEX instruction", mnem)
|
||||
}
|
||||
|
||||
// CMOVcc and SETcc carry the condition in the mnemonic (CMOVLGT, SETNE).
|
||||
@@ -121,14 +128,20 @@ func splitSize(upper string) (base string, size int) {
|
||||
// its own direction-dependent opcodes; KTESTW is always VEX; everything else
|
||||
// takes EVEX when an operand demands it (a ZMM or K register, or an
|
||||
// EVEX-only mnemonic) and VEX otherwise.
|
||||
func (e *enc) encodeVec(upper string, ops []Operand) error {
|
||||
func (e *enc) encodeVec(upper string, ops []Operand, zeroing bool) error {
|
||||
if upper == "KMOVW" {
|
||||
if zeroing {
|
||||
return fmt.Errorf("KMOVW takes no .Z suffix")
|
||||
}
|
||||
return e.encodeKmovw(ops)
|
||||
}
|
||||
if upper == "KTESTW" || !evexRequired(upper, ops) {
|
||||
if zeroing {
|
||||
return fmt.Errorf("%s: the .Z suffix requires an EVEX instruction", upper)
|
||||
}
|
||||
return e.encodeVex(upper, ops)
|
||||
}
|
||||
return e.encodeEvex(upper, ops)
|
||||
return e.encodeEvex(upper, ops, zeroing)
|
||||
}
|
||||
|
||||
// --- instruction components -------------------------------------------------
|
||||
|
||||
+186
-31
@@ -3,7 +3,10 @@
|
||||
|
||||
package asm
|
||||
|
||||
import "fmt"
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// This file implements EVEX (AVX-512) instruction encoding: the four-byte
|
||||
// EVEX prefix with 5-bit vector register fields (Z0–Z31, X/Y 16–31), the
|
||||
@@ -74,6 +77,48 @@ var evexTable = map[string]evexSpec{
|
||||
"VPMULLQ": {2, 0x40, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPERMD": {2, 0x36, 0, 1, -1, vexNDS3, [3]int{0, 32, 64}},
|
||||
|
||||
// EVEX.128/256/512 — the wider integer set (AVX-512 F/BW): byte/word
|
||||
// arithmetic, the bitwise ops with D/Q suffixes, min/max, averages and
|
||||
// variable shifts. All NDS form; W distinguishes element size.
|
||||
"VPADDB": {1, 0xFC, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPADDW": {1, 0xFD, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSUBB": {1, 0xF8, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSUBW": {1, 0xF9, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMULLW": {1, 0xD5, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPAVGB": {1, 0xE0, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPAVGW": {1, 0xE3, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINUB": {1, 0xDA, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXUB": {1, 0xDE, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINSW": {1, 0xEA, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXSW": {1, 0xEE, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPANDD": {1, 0xDB, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPANDQ": {1, 0xDB, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPANDND": {1, 0xDF, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPANDNQ": {1, 0xDF, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINSB": {2, 0x38, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXSB": {2, 0x3C, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINSQ": {2, 0x39, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXSQ": {2, 0x3D, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINUW": {2, 0x3A, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXUW": {2, 0x3E, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINSD": {2, 0x39, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXSD": {2, 0x3D, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINUD": {2, 0x3B, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXUD": {2, 0x3F, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMINUQ": {2, 0x3B, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPMAXUQ": {2, 0x3F, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSLLVD": {2, 0x47, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSLLVQ": {2, 0x47, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSRLVD": {2, 0x45, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSRLVQ": {2, 0x45, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSRAVD": {2, 0x46, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
"VPSRAVQ": {2, 0x46, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
|
||||
// EVEX forms of instructions that also exist in VEX (selected when a ZMM
|
||||
// or K register, or indices 16–31, demand EVEX).
|
||||
"VPSHUFD": {1, 0x70, 0, 1, -1, vexImmRM, [3]int{16, 32, 64}},
|
||||
"VPSHUFB": {2, 0x00, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}},
|
||||
|
||||
// EVEX.66.0F — immediate shift (VPSLLD /6).
|
||||
"VPSLLD": {1, 0x72, 0, 1, 6, vexShiftImm, [3]int{16, 32, 64}},
|
||||
|
||||
@@ -117,6 +162,13 @@ var evexMoveTable = map[string]evexMoveSpec{
|
||||
"VMOVDQU32": {1, 2, 0x6F, 0x7F, 0, [3]int{16, 32, 64}},
|
||||
// EVEX.128/256/512.F3.0F.W1 — unaligned qword move.
|
||||
"VMOVDQU64": {1, 2, 0x6F, 0x7F, 1, [3]int{16, 32, 64}},
|
||||
// EVEX.128/256/512.F2.0F.W0 — unaligned byte move (byte/word moves use the
|
||||
// F2 prefix, dword/qword moves F3; the element size only changes the tuple
|
||||
// semantics).
|
||||
"VMOVDQU8": {1, 3, 0x6F, 0x7F, 0, [3]int{16, 32, 64}},
|
||||
// EVEX.128/256/512.F2.0F.W1 — unaligned word move (shares the qword
|
||||
// encoding).
|
||||
"VMOVDQU16": {1, 3, 0x6F, 0x7F, 1, [3]int{16, 32, 64}},
|
||||
// EVEX.128/256/512.66.0F.W1 — unaligned packed double move.
|
||||
"VMOVUPD": {1, 1, 0x10, 0x11, 1, [3]int{16, 32, 64}},
|
||||
}
|
||||
@@ -151,13 +203,75 @@ func evexRequired(upper string, ops []Operand) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// encodeEvex encodes an EVEX instruction with operands in Plan 9 order.
|
||||
func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error {
|
||||
// stripEvexSuffix splits a ".Z" zeroing suffix off the mnemonic. It is the
|
||||
// only EVEX suffix supported; Go writes masking as an explicit K operand, not
|
||||
// a suffix.
|
||||
func stripEvexSuffix(mnem string) (base string, zeroing bool, err error) {
|
||||
i := strings.LastIndexByte(mnem, '.')
|
||||
if i < 0 {
|
||||
return mnem, false, nil
|
||||
}
|
||||
if mnem[i+1:] == "Z" {
|
||||
return mnem[:i], true, nil
|
||||
}
|
||||
return "", false, fmt.Errorf("unsupported EVEX suffix %q", mnem[i+1:])
|
||||
}
|
||||
|
||||
// splitMask extracts an explicit mask register (K1–K7) from the operand list,
|
||||
// returning the remaining operands and the mask index. K0 is not a usable
|
||||
// mask (aaa = 0 means "no mask"), matching the assembler.
|
||||
func splitMask(ops []Operand) ([]Operand, int, error) {
|
||||
var rest []Operand
|
||||
mask := 0
|
||||
for _, op := range ops {
|
||||
if r, ok := op.(Reg); ok && r.mask {
|
||||
if mask != 0 {
|
||||
return nil, 0, fmt.Errorf("at most one mask register operand")
|
||||
}
|
||||
if r.idx == 0 {
|
||||
return nil, 0, fmt.Errorf("K0 is not a usable mask register")
|
||||
}
|
||||
mask = r.idx
|
||||
continue
|
||||
}
|
||||
rest = append(rest, op)
|
||||
}
|
||||
return rest, mask, nil
|
||||
}
|
||||
|
||||
// encodeEvex encodes an EVEX instruction with operands in Plan 9 order. The
|
||||
// mask, when present, is an explicit K1–K7 operand anywhere among the
|
||||
// operands; zeroing comes from the .Z mnemonic suffix and requires a mask.
|
||||
func (e *enc) encodeEvex(mnemUpper string, ops []Operand, zeroing bool) error {
|
||||
// Mask-destination comparisons (VPCMPEQD …, K1): the last operand is the
|
||||
// destination K register, and any mask sits among the preceding operands.
|
||||
if spec, ok := evexTable[mnemUpper]; ok && spec.form == vexNDS3 && len(ops) > 0 {
|
||||
if dst, ok := ops[len(ops)-1].(Reg); ok && dst.mask {
|
||||
rest, mask, err := splitMask(ops[:len(ops)-1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if zeroing && mask == 0 {
|
||||
return fmt.Errorf("%s: zeroing (.Z) requires a mask register", mnemUpper)
|
||||
}
|
||||
return e.encodeEvexNDS3(spec, append(rest, dst), mask, zeroing)
|
||||
}
|
||||
}
|
||||
|
||||
rest, mask, err := splitMask(ops)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if zeroing && mask == 0 {
|
||||
return fmt.Errorf("%s: zeroing (.Z) requires a mask register", mnemUpper)
|
||||
}
|
||||
ops = rest
|
||||
|
||||
if bs, ok := evexBcastTable[mnemUpper]; ok {
|
||||
return e.encodeEvexBcast(bs, ops)
|
||||
return e.encodeEvexBcast(bs, ops, mask, zeroing)
|
||||
}
|
||||
if ms, ok := evexMoveTable[mnemUpper]; ok {
|
||||
return e.encodeEvexMove(mnemUpper, ms, ops)
|
||||
return e.encodeEvexMove(mnemUpper, ms, ops, mask, zeroing)
|
||||
}
|
||||
spec, ok := evexTable[mnemUpper]
|
||||
if !ok {
|
||||
@@ -165,17 +279,19 @@ func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error {
|
||||
}
|
||||
switch spec.form {
|
||||
case vexNDS3:
|
||||
return e.encodeEvexNDS3(spec, ops)
|
||||
return e.encodeEvexNDS3(spec, ops, mask, zeroing)
|
||||
case vexRM:
|
||||
return e.encodeEvexRM(spec, ops)
|
||||
return e.encodeEvexRM(spec, ops, mask, zeroing)
|
||||
case vexRMRev:
|
||||
return e.encodeEvexRMRev(spec, ops)
|
||||
return e.encodeEvexRMRev(spec, ops, mask, zeroing)
|
||||
case vexImmRM:
|
||||
return e.encodeEvexImmRM(spec, ops, mask, zeroing)
|
||||
case vexShiftImm:
|
||||
return e.encodeEvexShiftImm(spec, ops)
|
||||
return e.encodeEvexShiftImm(spec, ops, mask, zeroing)
|
||||
case vexNDS3Imm:
|
||||
return e.encodeEvexNDS3Imm(spec, ops)
|
||||
return e.encodeEvexNDS3Imm(spec, ops, mask, zeroing)
|
||||
case vexExtract:
|
||||
return e.encodeEvexExtract(spec, ops)
|
||||
return e.encodeEvexExtract(spec, ops, mask, zeroing)
|
||||
}
|
||||
return fmt.Errorf("unhandled EVEX form for %s", mnemUpper)
|
||||
}
|
||||
@@ -183,7 +299,7 @@ func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error {
|
||||
// encodeEvexNDS3 encodes the three-operand NDS form: OP src2, src1, dst. The
|
||||
// destination may be an opmask register (VPCMPEQD), in which case the vector
|
||||
// length comes from the sources.
|
||||
func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 3 {
|
||||
return fmt.Errorf("EVEX NDS instruction expects 3 operands, got %d", len(ops))
|
||||
}
|
||||
@@ -203,12 +319,12 @@ func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand) error {
|
||||
ll = r.vecLenBit()
|
||||
}
|
||||
}
|
||||
return e.emitEvexFields(spec, ll, dstReg.idx, vvvvReg.idx, src2)
|
||||
return e.emitEvexFields(spec, ll, dstReg.idx, vvvvReg.idx, src2, mask, zeroing)
|
||||
}
|
||||
|
||||
// encodeEvexRM encodes the two-operand form: OP src, dst (reg=dst, rm=src,
|
||||
// no vvvv), e.g. VCVTQQ2PD.
|
||||
func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 2 {
|
||||
return fmt.Errorf("EVEX two-operand instruction expects 2 operands, got %d", len(ops))
|
||||
}
|
||||
@@ -217,12 +333,42 @@ func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand) error {
|
||||
if !ok || !dstReg.isVec() {
|
||||
return fmt.Errorf("EVEX destination must be a vector register")
|
||||
}
|
||||
return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src)
|
||||
return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src, mask, zeroing)
|
||||
}
|
||||
|
||||
// encodeEvexImmRM encodes the immediate shuffle form: OP $imm, src, dst
|
||||
// (reg = dst, rm = src, imm8), e.g. VPSHUFD.
|
||||
func (e *enc) encodeEvexImmRM(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 3 {
|
||||
return fmt.Errorf("shuffle expects 3 operands ($imm, src, dst), got %d", len(ops))
|
||||
}
|
||||
imm, src, dst := ops[0], ops[1], ops[2]
|
||||
immVal, ok := imm.(Imm)
|
||||
if !ok {
|
||||
return fmt.Errorf("shuffle control must be an immediate")
|
||||
}
|
||||
dstReg, ok := dst.(Reg)
|
||||
if !ok || !dstReg.isVec() {
|
||||
return fmt.Errorf("shuffle destination must be a vector register")
|
||||
}
|
||||
ll := dstReg.vecLenBit()
|
||||
if r, ok := src.(Reg); ok && r.isVec() {
|
||||
ll = r.vecLenBit()
|
||||
}
|
||||
immByte, err := imm8(int64(immVal))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.emitEvexFields(spec, ll, dstReg.idx, -1, src, mask, zeroing); err != nil {
|
||||
return err
|
||||
}
|
||||
e.out = append(e.out, immByte)
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodeEvexShiftImm encodes an immediate shift: OP $imm, src, dst
|
||||
// (ModRM.reg = /digit, vvvv = dst, rm = src, imm8), e.g. VPSRAD $31, Z3, Z5.
|
||||
func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 3 {
|
||||
return fmt.Errorf("EVEX shift expects 3 operands ($imm, src, dst), got %d", len(ops))
|
||||
}
|
||||
@@ -243,7 +389,7 @@ func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.emitEvexFields(spec, dstReg.vecLenBit(), spec.opdigit, dstReg.idx, srcReg); err != nil {
|
||||
if err := e.emitEvexFields(spec, dstReg.vecLenBit(), spec.opdigit, dstReg.idx, srcReg, mask, zeroing); err != nil {
|
||||
return err
|
||||
}
|
||||
e.out = append(e.out, immByte)
|
||||
@@ -252,7 +398,7 @@ func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error {
|
||||
|
||||
// encodeEvexNDS3Imm encodes OP $imm, src2, src1, dst (reg=dst, vvvv=src1,
|
||||
// rm=src2, imm8), e.g. VALIGND.
|
||||
func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 4 {
|
||||
return fmt.Errorf("instruction expects 4 operands ($imm, src2, src1, dst), got %d", len(ops))
|
||||
}
|
||||
@@ -273,7 +419,7 @@ func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, vvvvReg.idx, src2); err != nil {
|
||||
if err := e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, vvvvReg.idx, src2, mask, zeroing); err != nil {
|
||||
return err
|
||||
}
|
||||
e.out = append(e.out, immByte)
|
||||
@@ -282,7 +428,7 @@ func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error {
|
||||
|
||||
// encodeEvexExtract encodes OP $imm, zsrc, ydst (reg=ZMM source, rm=YMM/memory
|
||||
// destination, imm8), e.g. VEXTRACTI64X4.
|
||||
func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 3 {
|
||||
return fmt.Errorf("extract expects 3 operands ($imm, zsrc, ydst), got %d", len(ops))
|
||||
}
|
||||
@@ -299,7 +445,7 @@ func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst); err != nil {
|
||||
if err := e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst, mask, zeroing); err != nil {
|
||||
return err
|
||||
}
|
||||
e.out = append(e.out, immByte)
|
||||
@@ -309,7 +455,7 @@ func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error {
|
||||
// encodeEvexMove encodes a two-operand EVEX move; a vector→vector move uses
|
||||
// the store-form opcode (reg = source, rm = destination), matching the Go
|
||||
// assembler.
|
||||
func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 2 {
|
||||
return fmt.Errorf("EVEX move expects 2 operands, got %d", len(ops))
|
||||
}
|
||||
@@ -338,7 +484,7 @@ func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand) error
|
||||
return fmt.Errorf("%s needs a vector register operand", mnem)
|
||||
}
|
||||
spec := evexSpec{mapSel: ms.mapSel, opcode: op, w: ms.w, pp: ms.pp, opdigit: -1, n: ms.n}
|
||||
return e.emitEvexFields(spec, reg.vecLenBit(), reg.idx, -1, rm)
|
||||
return e.emitEvexFields(spec, reg.vecLenBit(), reg.idx, -1, rm, mask, zeroing)
|
||||
}
|
||||
|
||||
// memOperand reports whether op is a memory reference (including a
|
||||
@@ -353,7 +499,7 @@ func memOperand(op Operand) bool {
|
||||
|
||||
// encodeEvexRMRev encodes the narrowing-store form: OP src, dst with the wide
|
||||
// source in the reg field and the narrow destination in r/m (VPMOVDW/QD).
|
||||
func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 2 {
|
||||
return fmt.Errorf("EVEX store instruction expects 2 operands, got %d", len(ops))
|
||||
}
|
||||
@@ -362,12 +508,12 @@ func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand) error {
|
||||
if !ok || !srcReg.isVec() {
|
||||
return fmt.Errorf("EVEX source must be a vector register")
|
||||
}
|
||||
return e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst)
|
||||
return e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst, mask, zeroing)
|
||||
}
|
||||
|
||||
// encodeEvexBcast encodes VPBROADCASTD/Q: OP src, dst with the GPR or memory
|
||||
// source broadcast to every lane of the vector destination.
|
||||
func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand) error {
|
||||
func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand, mask int, zeroing bool) error {
|
||||
if len(ops) != 2 {
|
||||
return fmt.Errorf("broadcast expects 2 operands, got %d", len(ops))
|
||||
}
|
||||
@@ -386,14 +532,15 @@ func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand) error {
|
||||
default:
|
||||
return fmt.Errorf("broadcast source must be a register or memory")
|
||||
}
|
||||
return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src)
|
||||
return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src, mask, zeroing)
|
||||
}
|
||||
|
||||
// emitEvexFields emits the EVEX prefix, opcode, ModR/M, SIB and displacement
|
||||
// (disp8×N compressed) for the given precomputed fields. regIdx is the
|
||||
// unextended reg-field register index, or a /digit (0–7); vvvvIdx is the
|
||||
// vvvv register index, or -1 when unused.
|
||||
func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand) error {
|
||||
// vvvv register index, or -1 when unused. mask (K1–K7, 0 = unmasked) and
|
||||
// zeroing fill the aaa and z bits of the P2 byte.
|
||||
func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand, mask int, zeroing bool) error {
|
||||
if ll > 2 {
|
||||
return fmt.Errorf("invalid vector length")
|
||||
}
|
||||
@@ -420,7 +567,8 @@ func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand)
|
||||
var sb *sbRef
|
||||
switch r := rm.(type) {
|
||||
case Reg:
|
||||
// ModRM.mod = 11: rm[3] extends via B̄, rm[4] via X̄.
|
||||
// ModRM.mod = 11: rm[3] extends via B̄, and rm[4] via X̄ (the EVEX
|
||||
// register-register quirk).
|
||||
modrm = 0xC0 | (regIdx&7)<<3 | (r.idx & 7)
|
||||
sib = -1
|
||||
if r.idx&8 != 0 {
|
||||
@@ -429,6 +577,9 @@ func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand)
|
||||
if r.idx&16 != 0 {
|
||||
xBar = 0
|
||||
}
|
||||
if r.idx&16 != 0 {
|
||||
xBar = 0
|
||||
}
|
||||
case Mem:
|
||||
var err error
|
||||
modrm, sib, disp, xBar, bBar, err = memComponentsEvex(regIdx&7, r, spec.n[ll])
|
||||
@@ -451,9 +602,13 @@ func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand)
|
||||
return fmt.Errorf("invalid EVEX r/m operand")
|
||||
}
|
||||
|
||||
z := 0
|
||||
if zeroing {
|
||||
z = 1
|
||||
}
|
||||
p0 := byte(rBar<<7 | xBar<<6 | bBar<<5 | rPrimeBar<<4 | spec.mapSel)
|
||||
p1 := byte(spec.w<<7 | vBar<<3 | 1<<2 | spec.pp)
|
||||
p2 := byte(ll<<5 | vPrimeBar<<3) // z = 0, b = 0, aaa = 0
|
||||
p2 := byte(z<<7 | ll<<5 | vPrimeBar<<3 | mask) // z, L'L, b=0, V', aaa
|
||||
e.out = append(e.out, 0x62, p0, p1, p2, spec.opcode, byte(modrm))
|
||||
if sib >= 0 {
|
||||
e.out = append(e.out, byte(sib))
|
||||
|
||||
+93
-1
@@ -65,6 +65,23 @@ func TestEvexGroundTruth(t *testing.T) {
|
||||
{"VMOVDQU64 (SI)(R15*4),Z3", "VMOVDQU64", []Operand{Idx(SI, vreg(t, "R15"), 4, 0, 64), vreg(t, "Z3")}, "62b1fe486f1cbe"},
|
||||
{"VMOVDQU64 Z0,4(SI)(AX*1)", "VMOVDQU64", []Operand{vreg(t, "Z0"), Idx(SI, AX, 1, 4, 64)}, "62f1fe487f840604000000"},
|
||||
{"VMOVDQU64 Z1,Z2", "VMOVDQU64", []Operand{vreg(t, "Z1"), vreg(t, "Z2")}, "62f1fe487fca"},
|
||||
// The wider AVX-512 F/BW integer set.
|
||||
{"VPADDB Z1,Z2,Z3", "VPADDB", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48fcd9"},
|
||||
{"VPSUBW Z1,Z2,Z3", "VPSUBW", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48f9d9"},
|
||||
{"VPANDQ Z1,Z2,Z3", "VPANDQ", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f1ed48dbd9"},
|
||||
{"VPANDND Z1,Z2,Z3", "VPANDND", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48dfd9"},
|
||||
{"VPMULLW Z1,Z2,Z3", "VPMULLW", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48d5d9"},
|
||||
{"VPMINUB Z1,Z2,Z3", "VPMINUB", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48dad9"},
|
||||
{"VPMAXUQ Z1,Z2,Z3", "VPMAXUQ", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f2ed483fd9"},
|
||||
{"VPAVGW Z1,Z2,Z3", "VPAVGW", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48e3d9"},
|
||||
{"VPSLLVQ Z3,Z1,Z2", "VPSLLVQ", []Operand{vreg(t, "Z3"), vreg(t, "Z1"), vreg(t, "Z2")}, "62f2f54847d3"},
|
||||
{"VPSRAVQ Z3,Z1,Z2", "VPSRAVQ", []Operand{vreg(t, "Z3"), vreg(t, "Z1"), vreg(t, "Z2")}, "62f2f54846d3"},
|
||||
{"VPSHUFD $0x1B,Z1,Z2", "VPSHUFD", []Operand{Imm(0x1B), vreg(t, "Z1"), vreg(t, "Z2")}, "62f17d4870d11b"},
|
||||
{"VPSHUFB Z1,Z2,Z3", "VPSHUFB", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f26d4800d9"},
|
||||
{"VMOVDQU8 Z1,Z2", "VMOVDQU8", []Operand{vreg(t, "Z1"), vreg(t, "Z2")}, "62f17f487fca"},
|
||||
{"VMOVDQU16 Z1,Z2", "VMOVDQU16", []Operand{vreg(t, "Z1"), vreg(t, "Z2")}, "62f1ff487fca"},
|
||||
// Indices 16–31: rm[4] rides in X̄ for register operands.
|
||||
{"VPSHUFD $1,X16,X17", "VPSHUFD", []Operand{Imm(1), vreg(t, "X16"), vreg(t, "X17")}, "62a17d0870c801"},
|
||||
{"VMOVUPD (DI),Z14", "VMOVUPD", []Operand{Ptr(DI, 0, 64), vreg(t, "Z14")}, "6271fd481037"},
|
||||
{"VMOVUPD 64(DI),Z14", "VMOVUPD", []Operand{Ptr(DI, 64, 64), vreg(t, "Z14")}, "6271fd48107701"},
|
||||
// Conversions and narrowing stores (reg = wide source).
|
||||
@@ -110,6 +127,81 @@ func TestEvexGroundTruth(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvexMasking checks the AVX-512 mask operand (K1–K7, placed freely among
|
||||
// the operands) and the .Z zeroing suffix, byte for byte against the Go
|
||||
// assembler.
|
||||
func TestEvexMasking(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
mnem string
|
||||
ops []Operand
|
||||
want string
|
||||
}{
|
||||
// Masked arithmetic: K anywhere among the operands; .Z sets the z bit.
|
||||
{"VPADDD.Z merging+zeroing", "VPADDD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K2"), vreg(t, "Z3")}, "62f16dcafed9"},
|
||||
{"VPADDD merging", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K1"), vreg(t, "Z3")}, "62f16d49fed9"},
|
||||
{"VADDPD.Z", "VADDPD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K2"), vreg(t, "Z3")}, "62f1edca58d9"},
|
||||
{"VPMINSD.Z", "VPMINSD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K5"), vreg(t, "Z3")}, "62f26dcd39d9"},
|
||||
{"VPMINSQ.Z", "VPMINSQ.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K5"), vreg(t, "Z3")}, "62f2edcd39d9"},
|
||||
// Masked immediate shift (K before the destination).
|
||||
{"VPSRAD.Z", "VPSRAD.Z", []Operand{Imm(1), vreg(t, "Z2"), vreg(t, "K1"), vreg(t, "Z3")}, "62f165c972e201"},
|
||||
{"VPSLLD merge", "VPSLLD", []Operand{Imm(4), vreg(t, "Z1"), vreg(t, "K2"), vreg(t, "Z3")}, "62f1654a72f104"},
|
||||
// Masked align.
|
||||
{"VALIGND", "VALIGND", []Operand{Imm(12), vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K3"), vreg(t, "Z4")}, "62f36d4b03e10c"},
|
||||
// Masked conversion and extract.
|
||||
{"VCVTQQ2PD.Z", "VCVTQQ2PD.Z", []Operand{vreg(t, "Z1"), vreg(t, "K2"), vreg(t, "Z3")}, "62f1fecae6d9"},
|
||||
{"VEXTRACTI64X4", "VEXTRACTI64X4", []Operand{Imm(1), vreg(t, "Z1"), vreg(t, "K2"), vreg(t, "Y3")}, "62f3fd4a3bcb01"},
|
||||
// Masked moves: K sits between the register and memory operands.
|
||||
{"VMOVDQU8 store", "VMOVDQU8", []Operand{vreg(t, "Z1"), vreg(t, "K3"), Ptr(SI, 0, 64)}, "62f17f4b7f0e"},
|
||||
{"VMOVDQU32 load", "VMOVDQU32", []Operand{Ptr(SI, 0, 64), vreg(t, "K4"), vreg(t, "Z1")}, "62f17e4c6f0e"},
|
||||
{"VMOVDQU32 store", "VMOVDQU32", []Operand{vreg(t, "Z1"), vreg(t, "K4"), Ptr(DI, 0, 64)}, "62f17e4c7f0f"},
|
||||
// Masked comparison with a K destination: dst K1, mask K2.
|
||||
{"VPCMPEQD k-dst+mask", "VPCMPEQD", []Operand{vreg(t, "Z0"), vreg(t, "Z3"), vreg(t, "K2"), vreg(t, "K1")}, "62f1654a76c8"},
|
||||
}
|
||||
for _, c := range cases {
|
||||
code, err := Encode(c.mnem, c.ops...)
|
||||
if err != nil {
|
||||
t.Errorf("%s: Encode: %v", c.name, err)
|
||||
continue
|
||||
}
|
||||
if got := hexCompact(code); got != c.want {
|
||||
t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
|
||||
continue
|
||||
}
|
||||
inst, err := x86asm.Decode(code, 64)
|
||||
if err != nil {
|
||||
t.Errorf("%s: Decode(%x): %v", c.name, code, err)
|
||||
continue
|
||||
}
|
||||
want := c.mnem
|
||||
if i := len(want) - 2; i > 0 && want[i:] == ".Z" {
|
||||
want = want[:i]
|
||||
}
|
||||
if inst.Op.String() != want {
|
||||
t.Errorf("%s: decoded as %s", c.name, inst.Op.String())
|
||||
}
|
||||
}
|
||||
|
||||
// Error cases.
|
||||
bad := []struct {
|
||||
name string
|
||||
mnem string
|
||||
ops []Operand
|
||||
}{
|
||||
{"zeroing without mask", "VPADDD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}},
|
||||
{"K0 mask", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K0"), vreg(t, "Z3")}},
|
||||
{"two masks", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "K1"), vreg(t, "K2"), vreg(t, "Z3")}},
|
||||
{".Z on VEX-only", "VPSHUFD.Z", []Operand{Imm(1), vreg(t, "X0"), vreg(t, "X1")}},
|
||||
{"unsupported suffix", "VPADDD.BCST", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}},
|
||||
{"KMOVW.Z", "KMOVW.Z", []Operand{vreg(t, "K1"), vreg(t, "K2")}},
|
||||
}
|
||||
for _, c := range bad {
|
||||
if _, err := Encode(c.mnem, c.ops...); err == nil {
|
||||
t.Errorf("%s: expected an error, got none", c.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEvexErrors checks the EVEX-specific error paths.
|
||||
func TestEvexErrors(t *testing.T) {
|
||||
cases := []struct {
|
||||
@@ -125,7 +217,7 @@ func TestEvexErrors(t *testing.T) {
|
||||
{"VPMOVDW src", "VPMOVDW", []Operand{AX, vreg(t, "Y0")}},
|
||||
{"align arity", "VALIGND", []Operand{Imm(1), vreg(t, "Z0"), vreg(t, "Z1")}},
|
||||
// VEX-only mnemonics reject registers only EVEX can encode.
|
||||
{"VPSHUFD X16", "VPSHUFD", []Operand{Imm(1), vreg(t, "X16"), vreg(t, "X17")}},
|
||||
{"VMOVMSKPS X16", "VMOVMSKPS", []Operand{vreg(t, "X16"), AX}},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if _, err := Encode(c.mnem, c.ops...); err == nil {
|
||||
|
||||
Reference in New Issue
Block a user