From 0f3146ff2c546690f4b5ee005189190dbbba9281 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Balv=C3=ADn?= Date: Tue, 14 Jul 2026 21:03:26 +0200 Subject: [PATCH] feat(asm): add EVEX masking, zeroing and the AVX-512 F/BW integer set Assisted-by: Qwen 3.8 Max Preview --- arch/arch.go | 10 ++ asm/encode.go | 23 ++++- asm/evex.go | 217 ++++++++++++++++++++++++++++++++++++------- asm/evex_test.go | 94 ++++++++++++++++++- cmd/gasm/main.go | 2 +- docs/ARCHITECTURE.md | 13 ++- justfile | 2 +- lint/lint.go | 24 ++++- lint/lint_test.go | 20 ++++ 9 files changed, 360 insertions(+), 45 deletions(-) diff --git a/arch/arch.go b/arch/arch.go index 8192595..b24cd6c 100644 --- a/arch/arch.go +++ b/arch/arch.go @@ -156,6 +156,16 @@ func (t *Table) Lookup(mnemonic string) (Instr, bool) { } } } + // amd64 EVEX instructions take a .Z zeroing suffix (masking is written as + // an explicit K operand rather than a suffix); strip it so the base + // instruction is still recognised. + if t.Arch == AMD64 { + if base, ok := strings.CutSuffix(key, ".Z"); ok { + if in, found := t.instrs[base]; found { + return in, true + } + } + } return Instr{}, false } diff --git a/asm/encode.go b/asm/encode.go index edc9f78..30a2f6d 100644 --- a/asm/encode.go +++ b/asm/encode.go @@ -51,9 +51,16 @@ func (e *enc) encode(mnem string, ops []Operand) error { // VEX (AVX/AVX2) and EVEX (AVX-512) instructions: the trailing // B/W/L/Q/D is part of the mnemonic, not a size suffix, so dispatch - // before splitSize. - if isVex(upper) || isEvex(upper) || upper == "KMOVW" { - return e.encodeVec(upper, ops) + // before splitSize. A ".Z" suffix requests EVEX zeroing. + base, zeroing, err := stripEvexSuffix(upper) + if err != nil { + return err + } + if isVex(base) || isEvex(base) || base == "KMOVW" { + return e.encodeVec(base, ops, zeroing) + } + if zeroing { + return fmt.Errorf("%s: the .Z suffix requires an EVEX instruction", mnem) } // CMOVcc and SETcc carry the condition in the mnemonic (CMOVLGT, SETNE). @@ -121,14 +128,20 @@ func splitSize(upper string) (base string, size int) { // its own direction-dependent opcodes; KTESTW is always VEX; everything else // takes EVEX when an operand demands it (a ZMM or K register, or an // EVEX-only mnemonic) and VEX otherwise. -func (e *enc) encodeVec(upper string, ops []Operand) error { +func (e *enc) encodeVec(upper string, ops []Operand, zeroing bool) error { if upper == "KMOVW" { + if zeroing { + return fmt.Errorf("KMOVW takes no .Z suffix") + } return e.encodeKmovw(ops) } if upper == "KTESTW" || !evexRequired(upper, ops) { + if zeroing { + return fmt.Errorf("%s: the .Z suffix requires an EVEX instruction", upper) + } return e.encodeVex(upper, ops) } - return e.encodeEvex(upper, ops) + return e.encodeEvex(upper, ops, zeroing) } // --- instruction components ------------------------------------------------- diff --git a/asm/evex.go b/asm/evex.go index 543219c..b0b099f 100644 --- a/asm/evex.go +++ b/asm/evex.go @@ -3,7 +3,10 @@ package asm -import "fmt" +import ( + "fmt" + "strings" +) // This file implements EVEX (AVX-512) instruction encoding: the four-byte // EVEX prefix with 5-bit vector register fields (Z0–Z31, X/Y 16–31), the @@ -74,6 +77,48 @@ var evexTable = map[string]evexSpec{ "VPMULLQ": {2, 0x40, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, "VPERMD": {2, 0x36, 0, 1, -1, vexNDS3, [3]int{0, 32, 64}}, + // EVEX.128/256/512 — the wider integer set (AVX-512 F/BW): byte/word + // arithmetic, the bitwise ops with D/Q suffixes, min/max, averages and + // variable shifts. All NDS form; W distinguishes element size. + "VPADDB": {1, 0xFC, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPADDW": {1, 0xFD, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSUBB": {1, 0xF8, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSUBW": {1, 0xF9, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMULLW": {1, 0xD5, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPAVGB": {1, 0xE0, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPAVGW": {1, 0xE3, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINUB": {1, 0xDA, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXUB": {1, 0xDE, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINSW": {1, 0xEA, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXSW": {1, 0xEE, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPANDD": {1, 0xDB, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPANDQ": {1, 0xDB, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPANDND": {1, 0xDF, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPANDNQ": {1, 0xDF, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINSB": {2, 0x38, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXSB": {2, 0x3C, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINSQ": {2, 0x39, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXSQ": {2, 0x3D, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINUW": {2, 0x3A, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXUW": {2, 0x3E, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINSD": {2, 0x39, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXSD": {2, 0x3D, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINUD": {2, 0x3B, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXUD": {2, 0x3F, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMINUQ": {2, 0x3B, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMAXUQ": {2, 0x3F, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSLLVD": {2, 0x47, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSLLVQ": {2, 0x47, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSRLVD": {2, 0x45, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSRLVQ": {2, 0x45, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSRAVD": {2, 0x46, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSRAVQ": {2, 0x46, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + + // EVEX forms of instructions that also exist in VEX (selected when a ZMM + // or K register, or indices 16–31, demand EVEX). + "VPSHUFD": {1, 0x70, 0, 1, -1, vexImmRM, [3]int{16, 32, 64}}, + "VPSHUFB": {2, 0x00, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + // EVEX.66.0F — immediate shift (VPSLLD /6). "VPSLLD": {1, 0x72, 0, 1, 6, vexShiftImm, [3]int{16, 32, 64}}, @@ -117,6 +162,13 @@ var evexMoveTable = map[string]evexMoveSpec{ "VMOVDQU32": {1, 2, 0x6F, 0x7F, 0, [3]int{16, 32, 64}}, // EVEX.128/256/512.F3.0F.W1 — unaligned qword move. "VMOVDQU64": {1, 2, 0x6F, 0x7F, 1, [3]int{16, 32, 64}}, + // EVEX.128/256/512.F2.0F.W0 — unaligned byte move (byte/word moves use the + // F2 prefix, dword/qword moves F3; the element size only changes the tuple + // semantics). + "VMOVDQU8": {1, 3, 0x6F, 0x7F, 0, [3]int{16, 32, 64}}, + // EVEX.128/256/512.F2.0F.W1 — unaligned word move (shares the qword + // encoding). + "VMOVDQU16": {1, 3, 0x6F, 0x7F, 1, [3]int{16, 32, 64}}, // EVEX.128/256/512.66.0F.W1 — unaligned packed double move. "VMOVUPD": {1, 1, 0x10, 0x11, 1, [3]int{16, 32, 64}}, } @@ -151,13 +203,75 @@ func evexRequired(upper string, ops []Operand) bool { return false } -// encodeEvex encodes an EVEX instruction with operands in Plan 9 order. -func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error { +// stripEvexSuffix splits a ".Z" zeroing suffix off the mnemonic. It is the +// only EVEX suffix supported; Go writes masking as an explicit K operand, not +// a suffix. +func stripEvexSuffix(mnem string) (base string, zeroing bool, err error) { + i := strings.LastIndexByte(mnem, '.') + if i < 0 { + return mnem, false, nil + } + if mnem[i+1:] == "Z" { + return mnem[:i], true, nil + } + return "", false, fmt.Errorf("unsupported EVEX suffix %q", mnem[i+1:]) +} + +// splitMask extracts an explicit mask register (K1–K7) from the operand list, +// returning the remaining operands and the mask index. K0 is not a usable +// mask (aaa = 0 means "no mask"), matching the assembler. +func splitMask(ops []Operand) ([]Operand, int, error) { + var rest []Operand + mask := 0 + for _, op := range ops { + if r, ok := op.(Reg); ok && r.mask { + if mask != 0 { + return nil, 0, fmt.Errorf("at most one mask register operand") + } + if r.idx == 0 { + return nil, 0, fmt.Errorf("K0 is not a usable mask register") + } + mask = r.idx + continue + } + rest = append(rest, op) + } + return rest, mask, nil +} + +// encodeEvex encodes an EVEX instruction with operands in Plan 9 order. The +// mask, when present, is an explicit K1–K7 operand anywhere among the +// operands; zeroing comes from the .Z mnemonic suffix and requires a mask. +func (e *enc) encodeEvex(mnemUpper string, ops []Operand, zeroing bool) error { + // Mask-destination comparisons (VPCMPEQD …, K1): the last operand is the + // destination K register, and any mask sits among the preceding operands. + if spec, ok := evexTable[mnemUpper]; ok && spec.form == vexNDS3 && len(ops) > 0 { + if dst, ok := ops[len(ops)-1].(Reg); ok && dst.mask { + rest, mask, err := splitMask(ops[:len(ops)-1]) + if err != nil { + return err + } + if zeroing && mask == 0 { + return fmt.Errorf("%s: zeroing (.Z) requires a mask register", mnemUpper) + } + return e.encodeEvexNDS3(spec, append(rest, dst), mask, zeroing) + } + } + + rest, mask, err := splitMask(ops) + if err != nil { + return err + } + if zeroing && mask == 0 { + return fmt.Errorf("%s: zeroing (.Z) requires a mask register", mnemUpper) + } + ops = rest + if bs, ok := evexBcastTable[mnemUpper]; ok { - return e.encodeEvexBcast(bs, ops) + return e.encodeEvexBcast(bs, ops, mask, zeroing) } if ms, ok := evexMoveTable[mnemUpper]; ok { - return e.encodeEvexMove(mnemUpper, ms, ops) + return e.encodeEvexMove(mnemUpper, ms, ops, mask, zeroing) } spec, ok := evexTable[mnemUpper] if !ok { @@ -165,17 +279,19 @@ func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error { } switch spec.form { case vexNDS3: - return e.encodeEvexNDS3(spec, ops) + return e.encodeEvexNDS3(spec, ops, mask, zeroing) case vexRM: - return e.encodeEvexRM(spec, ops) + return e.encodeEvexRM(spec, ops, mask, zeroing) case vexRMRev: - return e.encodeEvexRMRev(spec, ops) + return e.encodeEvexRMRev(spec, ops, mask, zeroing) + case vexImmRM: + return e.encodeEvexImmRM(spec, ops, mask, zeroing) case vexShiftImm: - return e.encodeEvexShiftImm(spec, ops) + return e.encodeEvexShiftImm(spec, ops, mask, zeroing) case vexNDS3Imm: - return e.encodeEvexNDS3Imm(spec, ops) + return e.encodeEvexNDS3Imm(spec, ops, mask, zeroing) case vexExtract: - return e.encodeEvexExtract(spec, ops) + return e.encodeEvexExtract(spec, ops, mask, zeroing) } return fmt.Errorf("unhandled EVEX form for %s", mnemUpper) } @@ -183,7 +299,7 @@ func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error { // encodeEvexNDS3 encodes the three-operand NDS form: OP src2, src1, dst. The // destination may be an opmask register (VPCMPEQD), in which case the vector // length comes from the sources. -func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand) error { +func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 3 { return fmt.Errorf("EVEX NDS instruction expects 3 operands, got %d", len(ops)) } @@ -203,12 +319,12 @@ func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand) error { ll = r.vecLenBit() } } - return e.emitEvexFields(spec, ll, dstReg.idx, vvvvReg.idx, src2) + return e.emitEvexFields(spec, ll, dstReg.idx, vvvvReg.idx, src2, mask, zeroing) } // encodeEvexRM encodes the two-operand form: OP src, dst (reg=dst, rm=src, // no vvvv), e.g. VCVTQQ2PD. -func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand) error { +func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 2 { return fmt.Errorf("EVEX two-operand instruction expects 2 operands, got %d", len(ops)) } @@ -217,12 +333,42 @@ func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand) error { if !ok || !dstReg.isVec() { return fmt.Errorf("EVEX destination must be a vector register") } - return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src) + return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src, mask, zeroing) +} + +// encodeEvexImmRM encodes the immediate shuffle form: OP $imm, src, dst +// (reg = dst, rm = src, imm8), e.g. VPSHUFD. +func (e *enc) encodeEvexImmRM(spec evexSpec, ops []Operand, mask int, zeroing bool) error { + if len(ops) != 3 { + return fmt.Errorf("shuffle expects 3 operands ($imm, src, dst), got %d", len(ops)) + } + imm, src, dst := ops[0], ops[1], ops[2] + immVal, ok := imm.(Imm) + if !ok { + return fmt.Errorf("shuffle control must be an immediate") + } + dstReg, ok := dst.(Reg) + if !ok || !dstReg.isVec() { + return fmt.Errorf("shuffle destination must be a vector register") + } + ll := dstReg.vecLenBit() + if r, ok := src.(Reg); ok && r.isVec() { + ll = r.vecLenBit() + } + immByte, err := imm8(int64(immVal)) + if err != nil { + return err + } + if err := e.emitEvexFields(spec, ll, dstReg.idx, -1, src, mask, zeroing); err != nil { + return err + } + e.out = append(e.out, immByte) + return nil } // encodeEvexShiftImm encodes an immediate shift: OP $imm, src, dst // (ModRM.reg = /digit, vvvv = dst, rm = src, imm8), e.g. VPSRAD $31, Z3, Z5. -func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error { +func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 3 { return fmt.Errorf("EVEX shift expects 3 operands ($imm, src, dst), got %d", len(ops)) } @@ -243,7 +389,7 @@ func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error { if err != nil { return err } - if err := e.emitEvexFields(spec, dstReg.vecLenBit(), spec.opdigit, dstReg.idx, srcReg); err != nil { + if err := e.emitEvexFields(spec, dstReg.vecLenBit(), spec.opdigit, dstReg.idx, srcReg, mask, zeroing); err != nil { return err } e.out = append(e.out, immByte) @@ -252,7 +398,7 @@ func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error { // encodeEvexNDS3Imm encodes OP $imm, src2, src1, dst (reg=dst, vvvv=src1, // rm=src2, imm8), e.g. VALIGND. -func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error { +func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 4 { return fmt.Errorf("instruction expects 4 operands ($imm, src2, src1, dst), got %d", len(ops)) } @@ -273,7 +419,7 @@ func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error { if err != nil { return err } - if err := e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, vvvvReg.idx, src2); err != nil { + if err := e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, vvvvReg.idx, src2, mask, zeroing); err != nil { return err } e.out = append(e.out, immByte) @@ -282,7 +428,7 @@ func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error { // encodeEvexExtract encodes OP $imm, zsrc, ydst (reg=ZMM source, rm=YMM/memory // destination, imm8), e.g. VEXTRACTI64X4. -func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error { +func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 3 { return fmt.Errorf("extract expects 3 operands ($imm, zsrc, ydst), got %d", len(ops)) } @@ -299,7 +445,7 @@ func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error { if err != nil { return err } - if err := e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst); err != nil { + if err := e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst, mask, zeroing); err != nil { return err } e.out = append(e.out, immByte) @@ -309,7 +455,7 @@ func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error { // encodeEvexMove encodes a two-operand EVEX move; a vector→vector move uses // the store-form opcode (reg = source, rm = destination), matching the Go // assembler. -func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand) error { +func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 2 { return fmt.Errorf("EVEX move expects 2 operands, got %d", len(ops)) } @@ -338,7 +484,7 @@ func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand) error return fmt.Errorf("%s needs a vector register operand", mnem) } spec := evexSpec{mapSel: ms.mapSel, opcode: op, w: ms.w, pp: ms.pp, opdigit: -1, n: ms.n} - return e.emitEvexFields(spec, reg.vecLenBit(), reg.idx, -1, rm) + return e.emitEvexFields(spec, reg.vecLenBit(), reg.idx, -1, rm, mask, zeroing) } // memOperand reports whether op is a memory reference (including a @@ -353,7 +499,7 @@ func memOperand(op Operand) bool { // encodeEvexRMRev encodes the narrowing-store form: OP src, dst with the wide // source in the reg field and the narrow destination in r/m (VPMOVDW/QD). -func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand) error { +func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 2 { return fmt.Errorf("EVEX store instruction expects 2 operands, got %d", len(ops)) } @@ -362,12 +508,12 @@ func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand) error { if !ok || !srcReg.isVec() { return fmt.Errorf("EVEX source must be a vector register") } - return e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst) + return e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst, mask, zeroing) } // encodeEvexBcast encodes VPBROADCASTD/Q: OP src, dst with the GPR or memory // source broadcast to every lane of the vector destination. -func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand) error { +func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand, mask int, zeroing bool) error { if len(ops) != 2 { return fmt.Errorf("broadcast expects 2 operands, got %d", len(ops)) } @@ -386,14 +532,15 @@ func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand) error { default: return fmt.Errorf("broadcast source must be a register or memory") } - return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src) + return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src, mask, zeroing) } // emitEvexFields emits the EVEX prefix, opcode, ModR/M, SIB and displacement // (disp8×N compressed) for the given precomputed fields. regIdx is the // unextended reg-field register index, or a /digit (0–7); vvvvIdx is the -// vvvv register index, or -1 when unused. -func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand) error { +// vvvv register index, or -1 when unused. mask (K1–K7, 0 = unmasked) and +// zeroing fill the aaa and z bits of the P2 byte. +func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand, mask int, zeroing bool) error { if ll > 2 { return fmt.Errorf("invalid vector length") } @@ -420,7 +567,8 @@ func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand) var sb *sbRef switch r := rm.(type) { case Reg: - // ModRM.mod = 11: rm[3] extends via B̄, rm[4] via X̄. + // ModRM.mod = 11: rm[3] extends via B̄, and rm[4] via X̄ (the EVEX + // register-register quirk). modrm = 0xC0 | (regIdx&7)<<3 | (r.idx & 7) sib = -1 if r.idx&8 != 0 { @@ -429,6 +577,9 @@ func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand) if r.idx&16 != 0 { xBar = 0 } + if r.idx&16 != 0 { + xBar = 0 + } case Mem: var err error modrm, sib, disp, xBar, bBar, err = memComponentsEvex(regIdx&7, r, spec.n[ll]) @@ -451,9 +602,13 @@ func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand) return fmt.Errorf("invalid EVEX r/m operand") } + z := 0 + if zeroing { + z = 1 + } p0 := byte(rBar<<7 | xBar<<6 | bBar<<5 | rPrimeBar<<4 | spec.mapSel) p1 := byte(spec.w<<7 | vBar<<3 | 1<<2 | spec.pp) - p2 := byte(ll<<5 | vPrimeBar<<3) // z = 0, b = 0, aaa = 0 + p2 := byte(z<<7 | ll<<5 | vPrimeBar<<3 | mask) // z, L'L, b=0, V', aaa e.out = append(e.out, 0x62, p0, p1, p2, spec.opcode, byte(modrm)) if sib >= 0 { e.out = append(e.out, byte(sib)) diff --git a/asm/evex_test.go b/asm/evex_test.go index 4dae01a..40cb84b 100644 --- a/asm/evex_test.go +++ b/asm/evex_test.go @@ -65,6 +65,23 @@ func TestEvexGroundTruth(t *testing.T) { {"VMOVDQU64 (SI)(R15*4),Z3", "VMOVDQU64", []Operand{Idx(SI, vreg(t, "R15"), 4, 0, 64), vreg(t, "Z3")}, "62b1fe486f1cbe"}, {"VMOVDQU64 Z0,4(SI)(AX*1)", "VMOVDQU64", []Operand{vreg(t, "Z0"), Idx(SI, AX, 1, 4, 64)}, "62f1fe487f840604000000"}, {"VMOVDQU64 Z1,Z2", "VMOVDQU64", []Operand{vreg(t, "Z1"), vreg(t, "Z2")}, "62f1fe487fca"}, + // The wider AVX-512 F/BW integer set. + {"VPADDB Z1,Z2,Z3", "VPADDB", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48fcd9"}, + {"VPSUBW Z1,Z2,Z3", "VPSUBW", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48f9d9"}, + {"VPANDQ Z1,Z2,Z3", "VPANDQ", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f1ed48dbd9"}, + {"VPANDND Z1,Z2,Z3", "VPANDND", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48dfd9"}, + {"VPMULLW Z1,Z2,Z3", "VPMULLW", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48d5d9"}, + {"VPMINUB Z1,Z2,Z3", "VPMINUB", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48dad9"}, + {"VPMAXUQ Z1,Z2,Z3", "VPMAXUQ", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f2ed483fd9"}, + {"VPAVGW Z1,Z2,Z3", "VPAVGW", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f16d48e3d9"}, + {"VPSLLVQ Z3,Z1,Z2", "VPSLLVQ", []Operand{vreg(t, "Z3"), vreg(t, "Z1"), vreg(t, "Z2")}, "62f2f54847d3"}, + {"VPSRAVQ Z3,Z1,Z2", "VPSRAVQ", []Operand{vreg(t, "Z3"), vreg(t, "Z1"), vreg(t, "Z2")}, "62f2f54846d3"}, + {"VPSHUFD $0x1B,Z1,Z2", "VPSHUFD", []Operand{Imm(0x1B), vreg(t, "Z1"), vreg(t, "Z2")}, "62f17d4870d11b"}, + {"VPSHUFB Z1,Z2,Z3", "VPSHUFB", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}, "62f26d4800d9"}, + {"VMOVDQU8 Z1,Z2", "VMOVDQU8", []Operand{vreg(t, "Z1"), vreg(t, "Z2")}, "62f17f487fca"}, + {"VMOVDQU16 Z1,Z2", "VMOVDQU16", []Operand{vreg(t, "Z1"), vreg(t, "Z2")}, "62f1ff487fca"}, + // Indices 16–31: rm[4] rides in X̄ for register operands. + {"VPSHUFD $1,X16,X17", "VPSHUFD", []Operand{Imm(1), vreg(t, "X16"), vreg(t, "X17")}, "62a17d0870c801"}, {"VMOVUPD (DI),Z14", "VMOVUPD", []Operand{Ptr(DI, 0, 64), vreg(t, "Z14")}, "6271fd481037"}, {"VMOVUPD 64(DI),Z14", "VMOVUPD", []Operand{Ptr(DI, 64, 64), vreg(t, "Z14")}, "6271fd48107701"}, // Conversions and narrowing stores (reg = wide source). @@ -110,6 +127,81 @@ func TestEvexGroundTruth(t *testing.T) { } } +// TestEvexMasking checks the AVX-512 mask operand (K1–K7, placed freely among +// the operands) and the .Z zeroing suffix, byte for byte against the Go +// assembler. +func TestEvexMasking(t *testing.T) { + cases := []struct { + name string + mnem string + ops []Operand + want string + }{ + // Masked arithmetic: K anywhere among the operands; .Z sets the z bit. + {"VPADDD.Z merging+zeroing", "VPADDD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K2"), vreg(t, "Z3")}, "62f16dcafed9"}, + {"VPADDD merging", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K1"), vreg(t, "Z3")}, "62f16d49fed9"}, + {"VADDPD.Z", "VADDPD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K2"), vreg(t, "Z3")}, "62f1edca58d9"}, + {"VPMINSD.Z", "VPMINSD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K5"), vreg(t, "Z3")}, "62f26dcd39d9"}, + {"VPMINSQ.Z", "VPMINSQ.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K5"), vreg(t, "Z3")}, "62f2edcd39d9"}, + // Masked immediate shift (K before the destination). + {"VPSRAD.Z", "VPSRAD.Z", []Operand{Imm(1), vreg(t, "Z2"), vreg(t, "K1"), vreg(t, "Z3")}, "62f165c972e201"}, + {"VPSLLD merge", "VPSLLD", []Operand{Imm(4), vreg(t, "Z1"), vreg(t, "K2"), vreg(t, "Z3")}, "62f1654a72f104"}, + // Masked align. + {"VALIGND", "VALIGND", []Operand{Imm(12), vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K3"), vreg(t, "Z4")}, "62f36d4b03e10c"}, + // Masked conversion and extract. + {"VCVTQQ2PD.Z", "VCVTQQ2PD.Z", []Operand{vreg(t, "Z1"), vreg(t, "K2"), vreg(t, "Z3")}, "62f1fecae6d9"}, + {"VEXTRACTI64X4", "VEXTRACTI64X4", []Operand{Imm(1), vreg(t, "Z1"), vreg(t, "K2"), vreg(t, "Y3")}, "62f3fd4a3bcb01"}, + // Masked moves: K sits between the register and memory operands. + {"VMOVDQU8 store", "VMOVDQU8", []Operand{vreg(t, "Z1"), vreg(t, "K3"), Ptr(SI, 0, 64)}, "62f17f4b7f0e"}, + {"VMOVDQU32 load", "VMOVDQU32", []Operand{Ptr(SI, 0, 64), vreg(t, "K4"), vreg(t, "Z1")}, "62f17e4c6f0e"}, + {"VMOVDQU32 store", "VMOVDQU32", []Operand{vreg(t, "Z1"), vreg(t, "K4"), Ptr(DI, 0, 64)}, "62f17e4c7f0f"}, + // Masked comparison with a K destination: dst K1, mask K2. + {"VPCMPEQD k-dst+mask", "VPCMPEQD", []Operand{vreg(t, "Z0"), vreg(t, "Z3"), vreg(t, "K2"), vreg(t, "K1")}, "62f1654a76c8"}, + } + for _, c := range cases { + code, err := Encode(c.mnem, c.ops...) + if err != nil { + t.Errorf("%s: Encode: %v", c.name, err) + continue + } + if got := hexCompact(code); got != c.want { + t.Errorf("%s: bytes %s, want %s", c.name, got, c.want) + continue + } + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Errorf("%s: Decode(%x): %v", c.name, code, err) + continue + } + want := c.mnem + if i := len(want) - 2; i > 0 && want[i:] == ".Z" { + want = want[:i] + } + if inst.Op.String() != want { + t.Errorf("%s: decoded as %s", c.name, inst.Op.String()) + } + } + + // Error cases. + bad := []struct { + name string + mnem string + ops []Operand + }{ + {"zeroing without mask", "VPADDD.Z", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}}, + {"K0 mask", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "K0"), vreg(t, "Z3")}}, + {"two masks", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "K1"), vreg(t, "K2"), vreg(t, "Z3")}}, + {".Z on VEX-only", "VPSHUFD.Z", []Operand{Imm(1), vreg(t, "X0"), vreg(t, "X1")}}, + {"unsupported suffix", "VPADDD.BCST", []Operand{vreg(t, "Z1"), vreg(t, "Z2"), vreg(t, "Z3")}}, + {"KMOVW.Z", "KMOVW.Z", []Operand{vreg(t, "K1"), vreg(t, "K2")}}, + } + for _, c := range bad { + if _, err := Encode(c.mnem, c.ops...); err == nil { + t.Errorf("%s: expected an error, got none", c.name) + } + } +} + // TestEvexErrors checks the EVEX-specific error paths. func TestEvexErrors(t *testing.T) { cases := []struct { @@ -125,7 +217,7 @@ func TestEvexErrors(t *testing.T) { {"VPMOVDW src", "VPMOVDW", []Operand{AX, vreg(t, "Y0")}}, {"align arity", "VALIGND", []Operand{Imm(1), vreg(t, "Z0"), vreg(t, "Z1")}}, // VEX-only mnemonics reject registers only EVEX can encode. - {"VPSHUFD X16", "VPSHUFD", []Operand{Imm(1), vreg(t, "X16"), vreg(t, "X17")}}, + {"VMOVMSKPS X16", "VMOVMSKPS", []Operand{vreg(t, "X16"), AX}}, } for _, c := range cases { if _, err := Encode(c.mnem, c.ops...); err == nil { diff --git a/cmd/gasm/main.go b/cmd/gasm/main.go index b7bba5f..86f24b5 100644 --- a/cmd/gasm/main.go +++ b/cmd/gasm/main.go @@ -28,7 +28,7 @@ import ( // version is the release version, stamped at build time via // -ldflags "-X main.version=…" (defaulting to the current release). -var version = "0.8.0" +var version = "0.9.0" func main() { if len(os.Args) < 2 { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b4748f0..1eb0b05 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -142,7 +142,7 @@ Two deeper analyses sit on top of the AST: has no System V style callee-saved registers (amd64 `BX`, `R12`–`R15` and the like are caller-saved or permanent scratch, and hand-written kernels may clobber them freely). The audited set is the frame pointer and the - goroutine pointer per architecture (amd64 `BP`/`R14`, arm64 `R18`/`R28`/ + the frame pointer, the goroutine pointer per architecture (amd64 `BP`/`R14`, arm64 `R18`/`R28`/ `R29`, riscv64 `X27`, loong64 `R22`); the goroutine pointer is reported only when the function can reach the runtime — it is not `NOSPLIT` or makes a call — since the ABI0 transition machinery restores it on those paths, and @@ -218,10 +218,13 @@ memory destination r/m), the direction-sensitive moves (`VMOVDQU`, `VMOVUPD`, `VFMADD231PD`) and the no-operand `VZEROUPPER` — together with `VPERMD` and the scalar families (`CMOVcc`, `SETcc`, `LZCNT`/`TZCNT`, the extending moves, `CVTSx2SD`, `IMUL3`) and the EVEX (AVX-512) prefix — the four-byte prefix with -5-bit register fields (Z0–Z31, X/Y 16–31), opmask registers as operands and -mask destinations, and the compressed disp8×N displacement, whose multiplier -follows the memory operand's size — covering every instruction the go-flac -AVX2 and AVX-512 kernels use. Every encoding is validated two ways: by +5-bit register fields (Z0–Z31, X/Y 16–31, with the mod=11 quirk that carries +rm[4] in X̄), opmask registers (K0–K7 as operands, mask destinations and +explicit merging/zeroing masks — written the way Go writes them, as a K +operand among the operands plus a `.Z` mnemonic suffix), and the compressed +disp8×N displacement, whose multiplier follows the memory operand's size — +covering every instruction the go-flac and go-lz4 AVX2/AVX-512 kernels use, +plus the common AVX-512 F/BW integer set. Every encoding is validated two ways: by round-trip decoding through `golang.org/x/arch`, and byte-for-byte against the machine code the real Go assembler emits — a comparison that holds for whole functions: all 27 functions of both kernels assemble to exactly the Go diff --git a/justfile b/justfile index e5f21b5..718190c 100644 --- a/justfile +++ b/justfile @@ -3,7 +3,7 @@ # gasm-devkit — developer tooling for Go's Plan 9 assembler (GAsm). -version := "0.8.0" +version := "0.9.0" default: @just --list diff --git a/lint/lint.go b/lint/lint.go index c3845c3..3e8fe44 100644 --- a/lint/lint.go +++ b/lint/lint.go @@ -242,7 +242,7 @@ func lintText(t *ast.Text, tab *arch.Table, archKnown bool, cfg Config, macros m } } - if archKnown && !cfg.Disable[CodeOperandCount] && !isMacroInvocation(mnem, macros) { + if archKnown && !cfg.Disable[CodeOperandCount] && !isMacroInvocation(mnem, macros) && !maskedEvex(mnem, st.Operands) { if in, ok := tab.Lookup(mnem); ok && in.MinOps >= 0 { n := len(st.Operands) if n < in.MinOps || n > in.MaxOps { @@ -438,6 +438,28 @@ func isMacroInvocation(mnem string, macros map[string]bool) bool { return strings.Contains(mnem, "_") || macros[mnem] } +// maskedEvex reports whether the instruction is a masked EVEX form: the +// mnemonic carries a .Z suffix, or the operand list contains an opmask +// register (K1–K7). Either way the operand count differs from the unmasked +// form, so count checks are skipped. +func maskedEvex(mnem string, ops []*ast.Operand) bool { + if strings.Contains(mnem, ".") { + return true + } + for _, op := range ops { + if op.Kind == ast.OpAddr && op.Addr.Sym != nil && op.Addr.Base == "" && + op.Addr.Index == "" && op.Addr.Sym.Pseudo == "" && isMaskReg(op.Addr.Sym.Name) { + return true + } + } + return false +} + +// isMaskReg reports whether name is an opmask register K0–K7. +func isMaskReg(name string) bool { + return len(name) == 2 && name[0] == 'K' && name[1] >= '0' && name[1] <= '7' +} + // isConditionalDirective reports whether a preprocessor directive (the text // after '#') is a conditional-compilation directive whose branches the parser // cannot resolve. diff --git a/lint/lint_test.go b/lint/lint_test.go index 1482d94..29f08d7 100644 --- a/lint/lint_test.go +++ b/lint/lint_test.go @@ -187,6 +187,26 @@ done: } } +// TestEvexMaskingRecognised checks that masked EVEX forms — the .Z suffix and +// an explicit K operand — are recognised and exempt from operand-count +// checks. +func TestEvexMaskingRecognised(t *testing.T) { + diags := lintSrc(t, ` +#include "textflag.h" +TEXT ·f(SB), NOSPLIT, $0 + VPADDD.Z Z1, Z2, K2, Z3 + VPMINSD Z1, Z2, K5, Z3 + VMOVDQU8 Z1, K3, (SI) + RET +`) + if codes(diags)[CodeUnknownInstr] != 0 { + t.Fatalf("masked EVEX must be recognised: %+v", diags) + } + if codes(diags)[CodeOperandCount] != 0 { + t.Fatalf("masked operand counts must not be flagged: %+v", diags) + } +} + func TestArm64AddressingSuffix(t *testing.T) { // .W (pre-index) and .P (post-index) suffixes must resolve to the base // instruction.