From 458cfb626ef4c6cdaf8913130fe0a027264d4114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Balv=C3=ADn?= Date: Fri, 10 Jul 2026 13:20:49 +0200 Subject: [PATCH] feat(asm): add EVEX/AVX-512 encoding and assemble the AVX-512 kernel byte-identically Assisted-by: Qwen 3.8 Max Preview --- asm/assemble.go | 16 +- asm/encode.go | 23 +- asm/evex.go | 556 +++++++++++++++++++++++++++++++++++++++++++ asm/evex_test.go | 221 +++++++++++++++++ asm/reg.go | 124 +++++----- asm/vex.go | 17 ++ cmd/gasm/main.go | 2 +- docs/ARCHITECTURE.md | 19 +- justfile | 2 +- 9 files changed, 903 insertions(+), 77 deletions(-) create mode 100644 asm/evex.go create mode 100644 asm/evex_test.go diff --git a/asm/assemble.go b/asm/assemble.go index d4595e9..0964cab 100644 --- a/asm/assemble.go +++ b/asm/assemble.go @@ -414,18 +414,18 @@ func operandFromAST(op *ast.Operand, size int, fi frameInfo, link *linkInfo) (Op off := fi.spAdjust + a.Sym.Offset return Mem{Base: spReg, Disp: off, HasBase: true, Size: size}, nil } - // SB (global symbol): a static (file-local, <>) symbol becomes a - // RIP-relative reference resolved by the file-level layout; anything - // external needs object-file emission. + // SB (global symbol): a symbol defined in the same file (GLOBL) is + // encoded RIP-relative and resolved by the file-level layout; + // anything not defined here needs object-file emission. if a.Sym != nil && a.Sym.Pseudo == "SB" { - if !a.Sym.Static { - return nil, fmt.Errorf("external symbol %q needs object-file emission", a.Sym.Name) - } if link == nil || link.symbols == nil { - return nil, fmt.Errorf("static symbol %q needs file-level assembly (AssembleFile)", a.Sym.Name) + return nil, fmt.Errorf("symbol %q needs file-level assembly (AssembleFile)", a.Sym.Name) } if !link.symbols[a.Sym.Name] { - return nil, fmt.Errorf("undefined symbol %q", a.Sym.Name) + if a.Sym.Static { + return nil, fmt.Errorf("undefined symbol %q", a.Sym.Name) + } + return nil, fmt.Errorf("external symbol %q needs object-file emission", a.Sym.Name) } return sbMem{size: size, name: a.Sym.Name, addend: a.Sym.Offset}, nil } diff --git a/asm/encode.go b/asm/encode.go index cf9651c..a83354d 100644 --- a/asm/encode.go +++ b/asm/encode.go @@ -49,10 +49,11 @@ func (e *enc) encode(mnem string, ops []Operand) error { return e.encodeJcc(cc, ops) } - // VEX (AVX/AVX2) instructions: the trailing B/W/L/Q/D is part of the - // mnemonic, not a size suffix, so dispatch before splitSize. - if isVex(upper) { - return e.encodeVex(upper, ops) + // VEX (AVX/AVX2) and EVEX (AVX-512) instructions: the trailing + // B/W/L/Q/D is part of the mnemonic, not a size suffix, so dispatch + // before splitSize. + if isVex(upper) || isEvex(upper) || upper == "KMOVW" { + return e.encodeVec(upper, ops) } // CMOVcc and SETcc carry the condition in the mnemonic (CMOVLGT, SETNE). @@ -114,6 +115,20 @@ func splitSize(upper string) (base string, size int) { return upper, 0 } +// encodeVec dispatches a VEX/EVEX mnemonic to the right encoding: KMOVW has +// its own direction-dependent opcodes; KTESTW is always VEX; everything else +// takes EVEX when an operand demands it (a ZMM or K register, or an +// EVEX-only mnemonic) and VEX otherwise. +func (e *enc) encodeVec(upper string, ops []Operand) error { + if upper == "KMOVW" { + return e.encodeKmovw(ops) + } + if upper == "KTESTW" || !evexRequired(upper, ops) { + return e.encodeVex(upper, ops) + } + return e.encodeEvex(upper, ops) +} + // --- instruction components ------------------------------------------------- type instr struct { diff --git a/asm/evex.go b/asm/evex.go new file mode 100644 index 0000000..516e8f4 --- /dev/null +++ b/asm/evex.go @@ -0,0 +1,556 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: BSD-3-Clause + +package asm + +import "fmt" + +// This file implements EVEX (AVX-512) instruction encoding: the four-byte +// EVEX prefix with 5-bit vector register fields (Z0–Z31, X/Y 16–31), the +// compressed disp8×N displacement, and the operand shapes the go-flac +// AVX-512 kernels use. Masking ({k}) and zeroing ({z}) are not supported — +// the kernels do not use them. K-register operands (mask destinations, +// KMOVW, KTESTW) are. + +// evexSpec describes one EVEX instruction's encoding parameters. The form +// field reuses the vexForm shapes, which carry over unchanged. +type evexSpec struct { + mapSel int // 1 = 0F, 2 = 0F38, 3 = 0F3A + opcode byte + w int + pp int // 0 = none, 1 = 66, 2 = F3, 3 = F2 + opdigit int // ModRM.reg /digit, or -1 when reg is a register + form vexForm // vexNDS3, vexRM, vexShiftImm, vexNDS3Imm, vexExtract + n [3]int // disp8×N multiplier per vector length (128/256/512) +} + +// evexTable maps an upper-case mnemonic to its EVEX encoding. Mnemonics +// that also have a VEX form (VPADDD, VMOVUPD, …) are dispatched here only +// when an operand demands EVEX (a ZMM or K register); EVEX-only mnemonics +// (VPXORD, VALIGND, …) always encode through this table. The N multipliers +// are taken from the Go assembler's opcode tables, which are authoritative +// for byte-for-byte agreement. +var evexTable = map[string]evexSpec{ + // EVEX.128/256/512.66.0F — integer arithmetic / logic, NDS form. + "VPADDD": {1, 0xFE, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPADDQ": {1, 0xD4, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSUBD": {1, 0xFA, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPSUBQ": {1, 0xFB, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPUNPCKLDQ": {1, 0x62, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPUNPCKHDQ": {1, 0x6A, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPXORD": {1, 0xEF, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPXORQ": {1, 0xEF, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPCMPEQD": {1, 0x76, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VFMADD231PD": {2, 0xB8, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + + // EVEX.128/256/512.66.0F.W1 — packed double arithmetic. + "VADDPD": {1, 0x58, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VMULPD": {1, 0x59, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + + // EVEX.512.66.0F3A — align (NDS + imm8). + "VALIGND": {3, 0x03, 0, 1, -1, vexNDS3Imm, [3]int{16, 32, 64}}, + + // EVEX.128/256/512.66.0F — immediate shift (VPSRAD /4). + "VPSRAD": {1, 0x72, 0, 1, 4, vexShiftImm, [3]int{16, 32, 64}}, + // EVEX.128/256/512.66.0F.W1 — variable shift with an XMM count (VPSRAQ; + // the W bit distinguishes it from VPSRAD's E2 form). + "VPSRAQ": {1, 0xE2, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + + // EVEX.128/256/512.F3.0F.W1 — signed qword to packed double (reg=dst, + // rm=src, no vvvv). + "VCVTQQ2PD": {1, 0xE6, 1, 2, -1, vexRM, [3]int{16, 32, 64}}, + // EVEX.128/256/512.66.0F38.W0 — sign-extend dwords to qwords; the memory + // operand is the narrow source, so disp8×N follows its size (8/16/32 for + // the xmm/ymm/zmm destination lengths). + "VPMOVSXDQ": {2, 0x25, 0, 1, -1, vexRM, [3]int{8, 16, 32}}, + + // EVEX.512.66.0F3A.W1 — lane extract (reg=ZMM source, rm=YMM/memory + // destination, imm8). + "VEXTRACTI64X4": {3, 0x3B, 1, 1, -1, vexExtract, [3]int{0, 0, 32}}, + "VEXTRACTF64X4": {3, 0x1B, 1, 1, -1, vexExtract, [3]int{0, 0, 32}}, + + // EVEX.66.0F38 — more integer NDS forms (W distinguishes D/Q). + "VPMULLD": {2, 0x40, 0, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPMULLQ": {2, 0x40, 1, 1, -1, vexNDS3, [3]int{16, 32, 64}}, + "VPERMD": {2, 0x36, 0, 1, -1, vexNDS3, [3]int{0, 32, 64}}, + + // EVEX.66.0F — immediate shift (VPSLLD /6). + "VPSLLD": {1, 0x72, 0, 1, 6, vexShiftImm, [3]int{16, 32, 64}}, + + // EVEX.F3.0F38.W0 — narrowing stores: reg = wide source, rm = narrow + // destination (VPMOVDW dword→word, VPMOVQD qword→dword). + "VPMOVDW": {2, 0x33, 0, 2, -1, vexRMRev, [3]int{8, 16, 32}}, + "VPMOVQD": {2, 0x35, 0, 2, -1, vexRMRev, [3]int{8, 16, 32}}, +} + +// evexBcastSpec describes an EVEX broadcast (VPBROADCASTD/Q): the opcode +// depends on the source kind — a GPR source uses opReg, a memory source uses +// opMem with a disp8×N of n. +type evexBcastSpec struct { + mapSel int + opReg byte + opMem byte + w int + n int +} + +var evexBcastTable = map[string]evexBcastSpec{ + // EVEX.128/256/512.66.0F38 — broadcast a dword/qword to all lanes. + "VPBROADCASTD": {2, 0x7C, 0x58, 0, 4}, + "VPBROADCASTQ": {2, 0x7C, 0x59, 1, 8}, +} + +// evexMoveSpec describes an EVEX move (load and store opcodes, like the VEX +// move table). +type evexMoveSpec struct { + mapSel int + pp int + load byte // r/m → vector + store byte // vector → r/m + w int + n [3]int +} + +// evexMoveTable maps an upper-case EVEX move mnemonic to its encoding. +var evexMoveTable = map[string]evexMoveSpec{ + // EVEX.128/256/512.F3.0F.W0 — unaligned integer move. + "VMOVDQU32": {1, 2, 0x6F, 0x7F, 0, [3]int{16, 32, 64}}, + // EVEX.128/256/512.66.0F.W1 — unaligned packed double move. + "VMOVUPD": {1, 1, 0x10, 0x11, 1, [3]int{16, 32, 64}}, +} + +// isEvex reports whether the mnemonic has an EVEX encoding we handle. +func isEvex(mnemUpper string) bool { + if _, ok := evexTable[mnemUpper]; ok { + return true + } + if _, ok := evexBcastTable[mnemUpper]; ok { + return true + } + _, ok := evexMoveTable[mnemUpper] + return ok +} + +// evexRequired reports whether the operands force the EVEX encoding of a +// mnemonic that also has a VEX form: ZMM and K registers do, and so do +// register indices 16–31, which only EVEX can represent (X16–Y31 exist +// solely under AVX-512). +func evexRequired(upper string, ops []Operand) bool { + _, inVex := vexTable[upper] + _, inVexMove := vexMoveTable[upper] + if !inVex && !inVexMove { + return true // EVEX-only mnemonic + } + for _, op := range ops { + if r, ok := op.(Reg); ok && (r.size == 64 || r.mask || (r.isVec() && r.idx >= 16)) { + return true + } + } + return false +} + +// encodeEvex encodes an EVEX instruction with operands in Plan 9 order. +func (e *enc) encodeEvex(mnemUpper string, ops []Operand) error { + if bs, ok := evexBcastTable[mnemUpper]; ok { + return e.encodeEvexBcast(bs, ops) + } + if ms, ok := evexMoveTable[mnemUpper]; ok { + return e.encodeEvexMove(mnemUpper, ms, ops) + } + spec, ok := evexTable[mnemUpper] + if !ok { + return fmt.Errorf("unsupported instruction %q for ZMM/K operands", mnemUpper) + } + switch spec.form { + case vexNDS3: + return e.encodeEvexNDS3(spec, ops) + case vexRM: + return e.encodeEvexRM(spec, ops) + case vexRMRev: + return e.encodeEvexRMRev(spec, ops) + case vexShiftImm: + return e.encodeEvexShiftImm(spec, ops) + case vexNDS3Imm: + return e.encodeEvexNDS3Imm(spec, ops) + case vexExtract: + return e.encodeEvexExtract(spec, ops) + } + return fmt.Errorf("unhandled EVEX form for %s", mnemUpper) +} + +// encodeEvexNDS3 encodes the three-operand NDS form: OP src2, src1, dst. The +// destination may be an opmask register (VPCMPEQD), in which case the vector +// length comes from the sources. +func (e *enc) encodeEvexNDS3(spec evexSpec, ops []Operand) error { + if len(ops) != 3 { + return fmt.Errorf("EVEX NDS instruction expects 3 operands, got %d", len(ops)) + } + src2, src1, dst := ops[0], ops[1], ops[2] + dstReg, ok := dst.(Reg) + if !ok || (!dstReg.isVec() && !dstReg.mask) { + return fmt.Errorf("EVEX destination must be a vector or mask register") + } + vvvvReg, ok := src1.(Reg) + if !ok || !vvvvReg.isVec() { + return fmt.Errorf("EVEX vvvv operand must be a vector register") + } + ll := dstReg.vecLenBit() + if dstReg.mask { + ll = vvvvReg.vecLenBit() + if r, ok := src2.(Reg); ok && r.isVec() { + ll = r.vecLenBit() + } + } + return e.emitEvexFields(spec, ll, dstReg.idx, vvvvReg.idx, src2) +} + +// encodeEvexRM encodes the two-operand form: OP src, dst (reg=dst, rm=src, +// no vvvv), e.g. VCVTQQ2PD. +func (e *enc) encodeEvexRM(spec evexSpec, ops []Operand) error { + if len(ops) != 2 { + return fmt.Errorf("EVEX two-operand instruction expects 2 operands, got %d", len(ops)) + } + src, dst := ops[0], ops[1] + dstReg, ok := dst.(Reg) + if !ok || !dstReg.isVec() { + return fmt.Errorf("EVEX destination must be a vector register") + } + return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src) +} + +// encodeEvexShiftImm encodes an immediate shift: OP $imm, src, dst +// (ModRM.reg = /digit, vvvv = dst, rm = src, imm8), e.g. VPSRAD $31, Z3, Z5. +func (e *enc) encodeEvexShiftImm(spec evexSpec, ops []Operand) error { + if len(ops) != 3 { + return fmt.Errorf("EVEX shift expects 3 operands ($imm, src, dst), got %d", len(ops)) + } + imm, src, dst := ops[0], ops[1], ops[2] + immVal, ok := imm.(Imm) + if !ok { + return fmt.Errorf("shift count must be an immediate") + } + srcReg, ok := src.(Reg) + if !ok || !srcReg.isVec() { + return fmt.Errorf("shift source must be a vector register") + } + dstReg, ok := dst.(Reg) + if !ok || !dstReg.isVec() { + return fmt.Errorf("shift destination must be a vector register") + } + immByte, err := imm8(int64(immVal)) + if err != nil { + return err + } + if err := e.emitEvexFields(spec, dstReg.vecLenBit(), spec.opdigit, dstReg.idx, srcReg); err != nil { + return err + } + e.out = append(e.out, immByte) + return nil +} + +// encodeEvexNDS3Imm encodes OP $imm, src2, src1, dst (reg=dst, vvvv=src1, +// rm=src2, imm8), e.g. VALIGND. +func (e *enc) encodeEvexNDS3Imm(spec evexSpec, ops []Operand) error { + if len(ops) != 4 { + return fmt.Errorf("instruction expects 4 operands ($imm, src2, src1, dst), got %d", len(ops)) + } + imm, src2, src1, dst := ops[0], ops[1], ops[2], ops[3] + immVal, ok := imm.(Imm) + if !ok { + return fmt.Errorf("shuffle control must be an immediate") + } + dstReg, ok := dst.(Reg) + if !ok || !dstReg.isVec() { + return fmt.Errorf("destination must be a vector register") + } + vvvvReg, ok := src1.(Reg) + if !ok || !vvvvReg.isVec() { + return fmt.Errorf("second source must be a vector register") + } + immByte, err := imm8(int64(immVal)) + if err != nil { + return err + } + if err := e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, vvvvReg.idx, src2); err != nil { + return err + } + e.out = append(e.out, immByte) + return nil +} + +// encodeEvexExtract encodes OP $imm, zsrc, ydst (reg=ZMM source, rm=YMM/memory +// destination, imm8), e.g. VEXTRACTI64X4. +func (e *enc) encodeEvexExtract(spec evexSpec, ops []Operand) error { + if len(ops) != 3 { + return fmt.Errorf("extract expects 3 operands ($imm, zsrc, ydst), got %d", len(ops)) + } + imm, src, dst := ops[0], ops[1], ops[2] + immVal, ok := imm.(Imm) + if !ok { + return fmt.Errorf("extract lane must be an immediate") + } + srcReg, ok := src.(Reg) + if !ok || !srcReg.isVec() { + return fmt.Errorf("extract source must be a vector register") + } + immByte, err := imm8(int64(immVal)) + if err != nil { + return err + } + if err := e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst); err != nil { + return err + } + e.out = append(e.out, immByte) + return nil +} + +// encodeEvexMove encodes a two-operand EVEX move; a vector→vector move uses +// the store-form opcode (reg = source, rm = destination), matching the Go +// assembler. +func (e *enc) encodeEvexMove(mnem string, ms evexMoveSpec, ops []Operand) error { + if len(ops) != 2 { + return fmt.Errorf("EVEX move expects 2 operands, got %d", len(ops)) + } + src, dst := ops[0], ops[1] + srcReg, srcIsVec := vecReg(src) + dstReg, dstIsVec := vecReg(dst) + + op := ms.store + var reg Reg + var rm Operand + switch { + case srcIsVec && dstIsVec: + reg, rm = srcReg, dst + case srcIsVec: + if !memOperand(dst) { + return fmt.Errorf("%s: invalid destination operand", mnem) + } + reg, rm = srcReg, dst + case dstIsVec: + if !memOperand(src) { + return fmt.Errorf("%s: invalid source operand", mnem) + } + op = ms.load + reg, rm = dstReg, src + default: + return fmt.Errorf("%s needs a vector register operand", mnem) + } + spec := evexSpec{mapSel: ms.mapSel, opcode: op, w: ms.w, pp: ms.pp, opdigit: -1, n: ms.n} + return e.emitEvexFields(spec, reg.vecLenBit(), reg.idx, -1, rm) +} + +// memOperand reports whether op is a memory reference (including a +// static-symbol reference). +func memOperand(op Operand) bool { + switch op.(type) { + case Mem, sbMem: + return true + } + return false +} + +// encodeEvexRMRev encodes the narrowing-store form: OP src, dst with the wide +// source in the reg field and the narrow destination in r/m (VPMOVDW/QD). +func (e *enc) encodeEvexRMRev(spec evexSpec, ops []Operand) error { + if len(ops) != 2 { + return fmt.Errorf("EVEX store instruction expects 2 operands, got %d", len(ops)) + } + src, dst := ops[0], ops[1] + srcReg, ok := src.(Reg) + if !ok || !srcReg.isVec() { + return fmt.Errorf("EVEX source must be a vector register") + } + return e.emitEvexFields(spec, srcReg.vecLenBit(), srcReg.idx, -1, dst) +} + +// encodeEvexBcast encodes VPBROADCASTD/Q: OP src, dst with the GPR or memory +// source broadcast to every lane of the vector destination. +func (e *enc) encodeEvexBcast(bs evexBcastSpec, ops []Operand) error { + if len(ops) != 2 { + return fmt.Errorf("broadcast expects 2 operands, got %d", len(ops)) + } + src, dst := ops[0], ops[1] + dstReg, ok := dst.(Reg) + if !ok || !dstReg.isVec() { + return fmt.Errorf("broadcast destination must be a vector register") + } + spec := evexSpec{mapSel: bs.mapSel, w: bs.w, pp: 1, opdigit: -1} + switch src.(type) { + case Mem, sbMem: + spec.opcode = bs.opMem + spec.n = [3]int{bs.n, bs.n, bs.n} + case Reg: + spec.opcode = bs.opReg + default: + return fmt.Errorf("broadcast source must be a register or memory") + } + return e.emitEvexFields(spec, dstReg.vecLenBit(), dstReg.idx, -1, src) +} + +// emitEvexFields emits the EVEX prefix, opcode, ModR/M, SIB and displacement +// (disp8×N compressed) for the given precomputed fields. regIdx is the +// unextended reg-field register index, or a /digit (0–7); vvvvIdx is the +// vvvv register index, or -1 when unused. +func (e *enc) emitEvexFields(spec evexSpec, ll, regIdx, vvvvIdx int, rm Operand) error { + if ll > 2 { + return fmt.Errorf("invalid vector length") + } + // reg-field extension bits (R̄, R'̄), inverted. + rBar, rPrimeBar := 1, 1 + if regIdx&8 != 0 { + rBar = 0 + } + if regIdx&16 != 0 { + rPrimeBar = 0 + } + // vvvv (inverted) and its extension bit V'̄. + vBar, vPrimeBar := 15, 1 + if vvvvIdx >= 0 { + vBar = 15 - (vvvvIdx & 15) + if vvvvIdx&16 != 0 { + vPrimeBar = 0 + } + } + + var modrm, sib int + var disp []byte + xBar, bBar := 1, 1 + var sb *sbRef + switch r := rm.(type) { + case Reg: + // ModRM.mod = 11: rm[3] extends via B̄, rm[4] via X̄. + modrm = 0xC0 | (regIdx&7)<<3 | (r.idx & 7) + sib = -1 + if r.idx&8 != 0 { + bBar = 0 + } + if r.idx&16 != 0 { + xBar = 0 + } + case Mem: + var err error + modrm, sib, disp, xBar, bBar, err = memComponentsEvex(regIdx&7, r, spec.n[ll]) + if err != nil { + return err + } + // An indexed memory operand carries index[4] in V'̄ (Go folds it + // together with vvvv[4] into the same bit). + if r.HasIndex && r.Index.idx&16 != 0 { + vPrimeBar = 0 + } + case sbMem: + // RIP-relative static-symbol reference; disp32 patched at link time + // (no disp8 scaling for RIP-relative addressing). + modrm = (regIdx&7)<<3 | 0x05 + sib = -1 + disp = le32(0) + sb = &sbRef{name: r.name, addend: r.addend} + default: + return fmt.Errorf("invalid EVEX r/m operand") + } + + p0 := byte(rBar<<7 | xBar<<6 | bBar<<5 | rPrimeBar<<4 | spec.mapSel) + p1 := byte(spec.w<<7 | vBar<<3 | 1<<2 | spec.pp) + p2 := byte(ll<<5 | vPrimeBar<<3) // z = 0, b = 0, aaa = 0 + e.out = append(e.out, 0x62, p0, p1, p2, spec.opcode, byte(modrm)) + if sib >= 0 { + e.out = append(e.out, byte(sib)) + } + if sb != nil { + e.patches = append(e.patches, encPatch{off: len(e.out), name: sb.name, addend: sb.addend}) + } + e.out = append(e.out, disp...) + return nil +} + +// memComponentsEvex computes the ModR/M byte (with the given reg field), the +// SIB byte (-1 if none), the displacement bytes and the (inverted sense) +// index/base extension bits for an EVEX memory operand. The displacement is +// compressed to disp8×N when it is a multiple of n and the quotient fits a +// signed byte; otherwise a full disp32 is used. +func memComponentsEvex(regField int, m Mem, n int) (modrm, sib int, disp []byte, xBar, bBar int, err error) { + sib = -1 + xBar, bBar = 1, 1 // inverted bits: 1 = no extension + if !m.HasBase && !m.HasIndex { + return regField<<3 | 0x05, -1, le32(m.Disp), 1, 1, nil // RIP-relative + } + + needSIB := m.HasIndex || (m.HasBase && m.Base.idx&7 == 4) + + var mod int + switch { + case !m.HasBase: + mod = 0 + disp = le32(m.Disp) + case m.Base.idx&7 == 5 && m.Disp == 0: + mod = 1 + disp = []byte{0} + case m.Disp == 0: + mod = 0 + case n > 0 && m.Disp%int64(n) == 0 && m.Disp/int64(n) >= -128 && m.Disp/int64(n) <= 127: + mod = 1 + disp = []byte{byte(int8(m.Disp / int64(n)))} + default: + mod = 2 + disp = le32(m.Disp) + } + + if needSIB { + idxField := 4 // 100 = no index + if m.HasIndex { + idxField = m.Index.idx & 7 + if m.Index.idx&8 != 0 { + xBar = 0 + } + } + baseField := 5 // 101 = no base (with mod=00 → disp32) + if m.HasBase { + baseField = m.Base.idx & 7 + if m.Base.idx&8 != 0 { + bBar = 0 + } + } + return mod<<6 | regField<<3 | 0x04, scaleBits(m.Scale)<<6 | idxField<<3 | baseField, disp, xBar, bBar, nil + } + + if m.Base.idx&8 != 0 { + bBar = 0 + } + return mod<<6 | regField<<3 | (m.Base.idx & 7), -1, disp, 1, bBar, nil +} + +// encodeKmovw encodes KMOVW, whose opcode depends on the operand direction: +// 90 (k/mem → K), 91 (K → mem), 92 (GPR → K), 93 (K → GPR); k → k uses 90. +func (e *enc) encodeKmovw(ops []Operand) error { + if len(ops) != 2 { + return fmt.Errorf("KMOVW expects 2 operands, got %d", len(ops)) + } + src, dst := ops[0], ops[1] + srcReg, srcIsReg := src.(Reg) + dstReg, dstIsReg := dst.(Reg) + srcK := srcIsReg && srcReg.mask + dstK := dstIsReg && dstReg.mask + spec := vexSpec{mapSel: 1, w: 0, pp: 0, opdigit: -1} + switch { + case srcK && dstK: + spec.opcode = 0x90 // k ← k: reg = dst, rm = src + return e.emitVexFields(spec, 0, dstReg.idx&7, 0, 15, src) + case srcK && dstIsReg: + spec.opcode = 0x93 // GPR ← k: reg = dst, rm = src + rBit := 0 + if dstReg.idx >= 8 { + rBit = 1 + } + return e.emitVexFields(spec, 0, dstReg.idx&7, rBit, 15, src) + case srcK: + if _, ok := dst.(Mem); !ok { + return fmt.Errorf("KMOVW: invalid destination operand") + } + spec.opcode = 0x91 // mem ← k: reg = src, rm = dst + return e.emitVexFields(spec, 0, srcReg.idx&7, 0, 15, dst) + case dstK: + spec.opcode = 0x92 // k ← GPR/mem: reg = dst, rm = src + return e.emitVexFields(spec, 0, dstReg.idx&7, 0, 15, src) + } + return fmt.Errorf("KMOVW requires a K register operand") +} diff --git a/asm/evex_test.go b/asm/evex_test.go new file mode 100644 index 0000000..22664a9 --- /dev/null +++ b/asm/evex_test.go @@ -0,0 +1,221 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: BSD-3-Clause + +package asm + +import ( + "os" + "strings" + "testing" + + "golang.org/x/arch/x86/x86asm" + + "sourcedock.dev/petrbalvin/gasm-devkit/parser" +) + +// TestEvexGroundTruth checks the EVEX (AVX-512) encodings byte for byte +// against machine code extracted from the Go toolchain's assembly of the +// same instructions, covering every operand shape the go-flac AVX-512 +// kernels use: NDS arithmetic, immediate and variable shifts, shuffles with +// an immediate, lane extracts, narrowing stores, broadcasts from a GPR or +// memory, mask destinations, mask moves, disp8×N compression and the 5-bit +// register fields (X/Y 16–31, Z 0–31). +func TestEvexGroundTruth(t *testing.T) { + cases := []struct { + name string + mnem string + ops []Operand + want string + }{ + // NDS integer arithmetic / logic. + {"VPXORD Z12,Z12,Z12", "VPXORD", []Operand{vreg(t, "Z12"), vreg(t, "Z12"), vreg(t, "Z12")}, "62511d48efe4"}, + {"VPXORQ Z8,Z9,Z10", "VPXORQ", []Operand{vreg(t, "Z8"), vreg(t, "Z9"), vreg(t, "Z10")}, "6251b548efd0"}, + {"VPADDD Z1,Z0,Z0", "VPADDD", []Operand{vreg(t, "Z1"), vreg(t, "Z0"), vreg(t, "Z0")}, "62f17d48fec1"}, + {"VPSUBQ Z8,Z11,Z11", "VPSUBQ", []Operand{vreg(t, "Z8"), vreg(t, "Z11"), vreg(t, "Z11")}, "6251a548fbd8"}, + {"VPUNPCKLDQ Z5,Z3,Z6", "VPUNPCKLDQ", []Operand{vreg(t, "Z5"), vreg(t, "Z3"), vreg(t, "Z6")}, "62f1654862f5"}, + {"VPUNPCKHDQ Z5,Z3,Z7", "VPUNPCKHDQ", []Operand{vreg(t, "Z5"), vreg(t, "Z3"), vreg(t, "Z7")}, "62f165486afd"}, + {"VPMULLQ Z9,Z10,Z10", "VPMULLQ", []Operand{vreg(t, "Z9"), vreg(t, "Z10"), vreg(t, "Z10")}, "6252ad4840d1"}, + {"VPMULLD Z13,Z11,Z2", "VPMULLD", []Operand{vreg(t, "Z13"), vreg(t, "Z11"), vreg(t, "Z2")}, "62d2254840d5"}, + {"VPERMD Z0,Z15,Z8", "VPERMD", []Operand{vreg(t, "Z0"), vreg(t, "Z15"), vreg(t, "Z8")}, "6272054836c0"}, + // Packed-double arithmetic (EVEX forms carry W=1). + {"VADDPD Z11,Z10,Z10", "VADDPD", []Operand{vreg(t, "Z11"), vreg(t, "Z10"), vreg(t, "Z10")}, "6251ad4858d3"}, + {"VMULPD Z13,Z12,Z12", "VMULPD", []Operand{vreg(t, "Z13"), vreg(t, "Z12"), vreg(t, "Z12")}, "62519d4859e5"}, + {"VFMADD231PD Z14,Z12,Z10", "VFMADD231PD", []Operand{vreg(t, "Z14"), vreg(t, "Z12"), vreg(t, "Z10")}, "62529d48b8d6"}, + // Align (NDS + imm8). + {"VALIGND $12,Z12,Z0,Z1", "VALIGND", []Operand{Imm(12), vreg(t, "Z12"), vreg(t, "Z0"), vreg(t, "Z1")}, "62d37d4803cc0c"}, + {"VALIGND $15,Z9,Z0,Z1", "VALIGND", []Operand{Imm(15), vreg(t, "Z9"), vreg(t, "Z0"), vreg(t, "Z1")}, "62d37d4803c90f"}, + // Shifts: immediate (/digit) and variable (XMM count). + {"VPSRAD $31,Z3,Z5", "VPSRAD", []Operand{Imm(31), vreg(t, "Z3"), vreg(t, "Z5")}, "62f1554872e31f"}, + {"VPSLLD $1,Z3,Z4", "VPSLLD", []Operand{Imm(1), vreg(t, "Z3"), vreg(t, "Z4")}, "62f15d4872f301"}, + {"VPSRAQ X31,Z8,Z8", "VPSRAQ", []Operand{vreg(t, "X31"), vreg(t, "Z8"), vreg(t, "Z8")}, "6211bd48e2c7"}, + // Mask destinations (the K register occupies the reg field). + {"VPCMPEQD Z0,Z3,K1", "VPCMPEQD", []Operand{vreg(t, "Z0"), vreg(t, "Z3"), vreg(t, "K1")}, "62f1654876c8"}, + {"VPCMPEQD Y30,Y11,K1", "VPCMPEQD", []Operand{vreg(t, "Y30"), vreg(t, "Y11"), vreg(t, "K1")}, "6291252876ce"}, + // Mask moves and test (VEX-encoded). + {"KMOVW K1,CX", "KMOVW", []Operand{vreg(t, "K1"), CX}, "c5f893c9"}, + {"KMOVW K1,R12", "KMOVW", []Operand{vreg(t, "K1"), vreg(t, "R12")}, "c57893e1"}, + {"KTESTW K1,K1", "KTESTW", []Operand{vreg(t, "K1"), vreg(t, "K1")}, "c5f899c9"}, + // Moves, incl. disp8×N (64 for a 512-bit operand). + {"VMOVDQU32 (SI)(R15*4),Z3", "VMOVDQU32", []Operand{Idx(SI, vreg(t, "R15"), 4, 0, 64), vreg(t, "Z3")}, "62b17e486f1cbe"}, + {"VMOVDQU32 4(SI)(AX*1),Z4", "VMOVDQU32", []Operand{Idx(SI, AX, 1, 4, 64), vreg(t, "Z4")}, "62f17e486fa40604000000"}, + {"VMOVDQU32 16(SI)(R15*4),Z4", "VMOVDQU32", []Operand{Idx(SI, vreg(t, "R15"), 4, 16, 64), vreg(t, "Z4")}, "62b17e486fa4be10000000"}, + {"VMOVDQU32 Z0,4(SI)(AX*1)", "VMOVDQU32", []Operand{vreg(t, "Z0"), Idx(SI, AX, 1, 4, 64)}, "62f17e487f840604000000"}, + {"VMOVDQU32 Z3,(DI)(R15*4)", "VMOVDQU32", []Operand{vreg(t, "Z3"), Idx(DI, vreg(t, "R15"), 4, 0, 64)}, "62b17e487f1cbf"}, + {"VMOVUPD (DI),Z14", "VMOVUPD", []Operand{Ptr(DI, 0, 64), vreg(t, "Z14")}, "6271fd481037"}, + {"VMOVUPD 64(DI),Z14", "VMOVUPD", []Operand{Ptr(DI, 64, 64), vreg(t, "Z14")}, "6271fd48107701"}, + // Conversions and narrowing stores (reg = wide source). + {"VCVTQQ2PD Z12,Z12", "VCVTQQ2PD", []Operand{vreg(t, "Z12"), vreg(t, "Z12")}, "6251fe48e6e4"}, + {"VCVTQQ2PD X13,X13", "VCVTQQ2PD", []Operand{vreg(t, "X13"), vreg(t, "X13")}, "6251fe08e6ed"}, + {"VPMOVSXDQ 32(SI),Z12", "VPMOVSXDQ", []Operand{Ptr(SI, 32, 32), vreg(t, "Z12")}, "62727d48256601"}, + {"VPMOVDW Z0,Y0", "VPMOVDW", []Operand{vreg(t, "Z0"), vreg(t, "Y0")}, "62f27e4833c0"}, + {"VPMOVQD Z11,Y11", "VPMOVQD", []Operand{vreg(t, "Z11"), vreg(t, "Y11")}, "62527e4835db"}, + // Lane extracts. + {"VEXTRACTI64X4 $1,Z8,Y9", "VEXTRACTI64X4", []Operand{Imm(1), vreg(t, "Z8"), vreg(t, "Y9")}, "6253fd483bc101"}, + {"VEXTRACTF64X4 $1,Z10,Y11", "VEXTRACTF64X4", []Operand{Imm(1), vreg(t, "Z10"), vreg(t, "Y11")}, "6253fd481bd301"}, + // Broadcasts: GPR source (0x7C) vs memory source (0x58/0x59, disp8×4/8). + {"VPBROADCASTD AX,Z15", "VPBROADCASTD", []Operand{AX, vreg(t, "Z15")}, "62727d487cf8"}, + {"VPBROADCASTD (SI),Z8", "VPBROADCASTD", []Operand{Ptr(SI, 0, 4), vreg(t, "Z8")}, "62727d485806"}, + {"VPBROADCASTD 4(SI),Z10", "VPBROADCASTD", []Operand{Ptr(SI, 4, 4), vreg(t, "Z10")}, "62727d48585601"}, + {"VPBROADCASTQ R8,X31", "VPBROADCASTQ", []Operand{vreg(t, "R8"), vreg(t, "X31")}, "6242fd087cf8"}, + {"VPBROADCASTQ AX,Z9", "VPBROADCASTQ", []Operand{AX, vreg(t, "Z9")}, "6272fd487cc8"}, + // Register indices 16–31 exist only in EVEX encodings. + {"VPBROADCASTD AX,Y30", "VPBROADCASTD", []Operand{AX, vreg(t, "Y30")}, "62627d287cf0"}, + } + for _, c := range cases { + want := strings.ReplaceAll(c.want, " ", "") + code, err := Encode(c.mnem, c.ops...) + if err != nil { + t.Errorf("%s: Encode: %v", c.name, err) + continue + } + if got := hexCompact(code); got != want { + t.Errorf("%s: bytes %s, want %s", c.name, got, want) + continue + } + inst, err := x86asm.Decode(code, 64) + if err != nil { + t.Errorf("%s: Decode(%x): %v", c.name, code, err) + continue + } + if inst.Len != len(code) { + t.Errorf("%s: Decode consumed %d of %d bytes", c.name, inst.Len, len(code)) + } + if inst.Op.String() != c.mnem { + t.Errorf("%s: decoded as %s", c.name, inst.Op.String()) + } + } +} + +// TestEvexErrors checks the EVEX-specific error paths. +func TestEvexErrors(t *testing.T) { + cases := []struct { + name string + mnem string + ops []Operand + }{ + {"NDS arity", "VPXORD", []Operand{vreg(t, "Z0"), vreg(t, "Z1")}}, + {"KMOVW arity", "KMOVW", []Operand{vreg(t, "K1")}}, + {"KMOVW no K", "KMOVW", []Operand{AX, CX}}, + {"VMOVUPD Z gpr", "VMOVUPD", []Operand{AX, vreg(t, "Z1")}}, + {"broadcast src", "VPBROADCASTD", []Operand{Imm(1), vreg(t, "Z1")}}, + {"VPMOVDW src", "VPMOVDW", []Operand{AX, vreg(t, "Y0")}}, + {"align arity", "VALIGND", []Operand{Imm(1), vreg(t, "Z0"), vreg(t, "Z1")}}, + // VEX-only mnemonics reject registers only EVEX can encode. + {"VPSHUFD X16", "VPSHUFD", []Operand{Imm(1), vreg(t, "X16"), vreg(t, "X17")}}, + } + for _, c := range cases { + if _, err := Encode(c.mnem, c.ops...); err == nil { + t.Errorf("%s: expected an error, got none", c.name) + } + } +} + +// TestAssembleGoFlacAVX512Kernel assembles the whole production AVX-512 +// kernel — all functions plus the file-global idx16 constant — and checks +// that the static-symbol load resolves to the right bytes in the image. +// Skipped when the sibling repository is not checked out. +func TestAssembleGoFlacAVX512Kernel(t *testing.T) { + path := "../../go-libraries/go-flac/avx512_amd64.s" + if _, err := os.Stat(path); err != nil { + t.Skip("go-libraries repository not present next to gasm-devkit") + } + src, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + f, errs := parser.Parse(path, string(src)) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + img, err := AssembleFile(f) + if err != nil { + t.Fatalf("AssembleFile: %v", err) + } + if len(img.Funcs) != 10 { + t.Errorf("functions = %d, want 10", len(img.Funcs)) + } + + // idx16 as the DATA directives define it: dwords 1..16. + idx := make([]byte, 0, 64) + for i := 1; i <= 16; i++ { + idx = append(idx, byte(i), 0, 0, 0) + } + image := img.Bytes() + base := img.Symbols["idx16"] + if base == 0 { + t.Fatal("idx16 not laid out") + } + if got := image[base : base+64]; hexCompact(got) != hexCompact(idx) { + t.Errorf("idx16 contents %x, want %x", got, idx) + } + + // The VMOVDQU32 idx16(SB), Z13 load (62 71 7e 48 6f 2d + rel32) must + // resolve to idx16 within the image. + loads := 0 + for _, fn := range img.Funcs { + code := img.Code[fn.Offset : fn.Offset+fn.Size] + pat := []byte{0x62, 0x71, 0x7e, 0x48, 0x6f, 0x2d} + for pos := 0; ; { + i := indexOf(code[pos:], pat) + if i < 0 { + break + } + i += pos + rel := int32(uint32(code[i+6]) | uint32(code[i+7])<<8 | uint32(code[i+8])<<16 | uint32(code[i+9])<<24) + target := fn.Offset + i + 10 + int(rel) + if target != base { + t.Errorf("%s: idx16 load at +%d targets 0x%x, want 0x%x", fn.Name, i, target, base) + } + loads++ + pos = i + 10 + } + } + if loads != 1 { + t.Errorf("idx16 loads found = %d, want 1", loads) + } +} + +// hexCompact renders bytes as a lowercase hex string without separators. +func hexCompact(b []byte) string { + const hexdig = "0123456789abcdef" + out := make([]byte, len(b)*2) + for i, c := range b { + out[i*2] = hexdig[c>>4] + out[i*2+1] = hexdig[c&0xf] + } + return string(out) +} + +// indexOf returns the index of the first occurrence of pat in b, or -1. +func indexOf(b, pat []byte) int { + for i := 0; i+len(pat) <= len(b); i++ { + j := 0 + for j < len(pat) && b[i+j] == pat[j] { + j++ + } + if j == len(pat) { + return i + } + } + return -1 +} diff --git a/asm/reg.go b/asm/reg.go index ac4a707..2760a45 100644 --- a/asm/reg.go +++ b/asm/reg.go @@ -14,19 +14,24 @@ import "strings" // so the encoder keys off the register's index and lets the mnemonic supply the // size. The high flag marks the legacy high-byte registers AH/CH/DH/BH, which // occupy indices 4–7 yet take no REX prefix, unlike SPL/BPL/SIL/DIL that share -// those indices but require one. +// those indices but require one. The mask flag marks the AVX-512 opmask +// registers K0–K7. type Reg struct { idx int size int // informational width implied by the name; the mnemonic decides high bool // AH/CH/DH/BH + mask bool // K0–K7 opmask register } -// Index returns the register number (0–15). +// Index returns the register number (0–15 for GPRs, 0–31 for vectors). func (r Reg) Index() int { return r.idx } // Size returns the width in bytes implied by the register's name. func (r Reg) Size() int { return r.size } +// IsMask reports whether r is an AVX-512 opmask register (K0–K7). +func (r Reg) IsMask() bool { return r.mask } + func (r Reg) isOperand() {} // needsREX reports whether this register forces a REX prefix at the given @@ -41,45 +46,45 @@ func (r Reg) needsREX(opSize int) bool { // Register constants (the size is the width the name implies). var ( - AL = Reg{0, 1, false} - CL = Reg{1, 1, false} - DL = Reg{2, 1, false} - BL = Reg{3, 1, false} - AH = Reg{4, 1, true} - CH = Reg{5, 1, true} - DH = Reg{6, 1, true} - BH = Reg{7, 1, true} - SPL = Reg{4, 1, false} - BPL = Reg{5, 1, false} - SIL = Reg{6, 1, false} - DIL = Reg{7, 1, false} + AL = Reg{idx: 0, size: 1} + CL = Reg{idx: 1, size: 1} + DL = Reg{idx: 2, size: 1} + BL = Reg{idx: 3, size: 1} + AH = Reg{idx: 4, size: 1, high: true} + CH = Reg{idx: 5, size: 1, high: true} + DH = Reg{idx: 6, size: 1, high: true} + BH = Reg{idx: 7, size: 1, high: true} + SPL = Reg{idx: 4, size: 1} + BPL = Reg{idx: 5, size: 1} + SIL = Reg{idx: 6, size: 1} + DIL = Reg{idx: 7, size: 1} - AX = Reg{0, 2, false} - CX = Reg{1, 2, false} - DX = Reg{2, 2, false} - BX = Reg{3, 2, false} - SP = Reg{4, 2, false} - BP = Reg{5, 2, false} - SI = Reg{6, 2, false} - DI = Reg{7, 2, false} + AX = Reg{idx: 0, size: 2} + CX = Reg{idx: 1, size: 2} + DX = Reg{idx: 2, size: 2} + BX = Reg{idx: 3, size: 2} + SP = Reg{idx: 4, size: 2} + BP = Reg{idx: 5, size: 2} + SI = Reg{idx: 6, size: 2} + DI = Reg{idx: 7, size: 2} - EAX = Reg{0, 4, false} - ECX = Reg{1, 4, false} - EDX = Reg{2, 4, false} - EBX = Reg{3, 4, false} - ESP = Reg{4, 4, false} - EBP = Reg{5, 4, false} - ESI = Reg{6, 4, false} - EDI = Reg{7, 4, false} + EAX = Reg{idx: 0, size: 4} + ECX = Reg{idx: 1, size: 4} + EDX = Reg{idx: 2, size: 4} + EBX = Reg{idx: 3, size: 4} + ESP = Reg{idx: 4, size: 4} + EBP = Reg{idx: 5, size: 4} + ESI = Reg{idx: 6, size: 4} + EDI = Reg{idx: 7, size: 4} - RAX = Reg{0, 8, false} - RCX = Reg{1, 8, false} - RDX = Reg{2, 8, false} - RBX = Reg{3, 8, false} - RSP = Reg{4, 8, false} - RBP = Reg{5, 8, false} - RSI = Reg{6, 8, false} - RDI = Reg{7, 8, false} + RAX = Reg{idx: 0, size: 8} + RCX = Reg{idx: 1, size: 8} + RDX = Reg{idx: 2, size: 8} + RBX = Reg{idx: 3, size: 8} + RSP = Reg{idx: 4, size: 8} + RBP = Reg{idx: 5, size: 8} + RSI = Reg{idx: 6, size: 8} + RDI = Reg{idx: 7, size: 8} ) // regByName maps an assembly register name (case-insensitive) to a Reg. @@ -91,28 +96,28 @@ func buildRegByName() map[string]Reg { // 64-bit: RAX..RDI, R8..R15. r64 := []string{"RAX", "RCX", "RDX", "RBX", "RSP", "RBP", "RSI", "RDI"} for i, n := range r64 { - m[n] = Reg{i, 8, false} + m[n] = Reg{idx: i, size: 8} } for i := 8; i <= 15; i++ { - m["R"+itoa(i)] = Reg{i, 8, false} + m["R"+itoa(i)] = Reg{idx: i, size: 8} } // 32-bit: EAX..EDI, R8D..R15D. e32 := []string{"EAX", "ECX", "EDX", "EBX", "ESP", "EBP", "ESI", "EDI"} for i, n := range e32 { - m[n] = Reg{i, 4, false} + m[n] = Reg{idx: i, size: 4} } for i := 8; i <= 15; i++ { - m["R"+itoa(i)+"D"] = Reg{i, 4, false} + m["R"+itoa(i)+"D"] = Reg{idx: i, size: 4} } // 16-bit: AX..DI, R8W..R15W. w16 := []string{"AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI"} for i, n := range w16 { - m[n] = Reg{i, 2, false} + m[n] = Reg{idx: i, size: 2} } for i := 8; i <= 15; i++ { - m["R"+itoa(i)+"W"] = Reg{i, 2, false} + m["R"+itoa(i)+"W"] = Reg{idx: i, size: 2} } // 8-bit: AL..BH, SPL..DIL, R8B..R15B. @@ -124,25 +129,34 @@ func buildRegByName() map[string]Reg { m[n] = r } for i := 8; i <= 15; i++ { - m["R"+itoa(i)+"B"] = Reg{i, 1, false} + m["R"+itoa(i)+"B"] = Reg{idx: i, size: 1} } - // Vector: X0..X15 (128-bit, encoded size 16), Y0..Y15 (256-bit, size 32). - // Z (512-bit) and K (mask) registers arrive with EVEX/AVX-512 support. - for i := 0; i <= 15; i++ { - m["X"+itoa(i)] = Reg{i, 16, false} - m["Y"+itoa(i)] = Reg{i, 32, false} + // Vector: X0..X31 (128-bit, size 16), Y0..Y31 (256-bit, size 32), + // Z0..Z31 (512-bit, size 64). Indices 16–31 are only encodable in EVEX + // (AVX-512) instructions; the encoder validates that through its tables. + for i := 0; i <= 31; i++ { + m["X"+itoa(i)] = Reg{idx: i, size: 16} + m["Y"+itoa(i)] = Reg{idx: i, size: 32} + m["Z"+itoa(i)] = Reg{idx: i, size: 64} + } + // Opmask: K0..K7. + for i := 0; i <= 7; i++ { + m["K"+itoa(i)] = Reg{idx: i, size: 8, mask: true} } return m } -// isVec reports whether r is an XMM/YMM vector register. -func (r Reg) isVec() bool { return r.size == 16 || r.size == 32 } +// isVec reports whether r is an XMM/YMM/ZMM vector register. +func (r Reg) isVec() bool { return r.size == 16 || r.size == 32 || r.size == 64 } -// vecLenBit returns the VEX.L bit for a vector register (X=0/128-bit, -// Y=1/256-bit). +// vecLenBit returns the vector-length field for a vector register: +// 0 (128-bit, VEX.L / EVEX.L'L=00), 1 (256-bit) or 2 (512-bit, EVEX only). func (r Reg) vecLenBit() int { - if r.size == 32 { + switch r.size { + case 64: + return 2 + case 32: return 1 } return 0 diff --git a/asm/vex.go b/asm/vex.go index 1e5799d..2cdd9a0 100644 --- a/asm/vex.go +++ b/asm/vex.go @@ -39,6 +39,10 @@ const ( // source lives in the reg field, the destination in r/m — the PEXTR-style // layout. VEXTRACTI128 and VEXTRACTF128 use this shape. vexExtract + // vexRMRev is the reversed two-operand form `OP src, dst` with the source + // in ModRM.reg and the destination in r/m — the layout of the EVEX + // narrowing stores (VPMOVDW, VPMOVQD). + vexRMRev // vexZero is the no-operand form (VZEROUPPER). vexZero ) @@ -131,6 +135,9 @@ var vexTable = map[string]vexSpec{ // VEX.128.0F.W0 — no operands. "VZEROUPPER": {1, 0x77, 0, 0, -1, vexZero}, + + // VEX.128.0F.W0 — mask-register test (KTESTW k1, k2: reg = dst, rm = src). + "KTESTW": {1, 0x99, 0, 0, -1, vexRM}, } // vexVarShift maps the shift mnemonics to their variable-count opcode — the @@ -188,6 +195,13 @@ func isVex(mnemUpper string) bool { // encodeVex encodes a VEX instruction with operands in Plan 9 order. func (e *enc) encodeVex(mnemUpper string, ops []Operand) error { + // Vector register indices 16–31 exist only in EVEX encodings; fail + // loudly rather than silently truncating the index. + for _, op := range ops { + if r, ok := op.(Reg); ok && r.isVec() && r.idx >= 16 { + return fmt.Errorf("%s: vector register index %d needs an EVEX (AVX-512) instruction", mnemUpper, r.idx) + } + } if ms, ok := vexMoveTable[mnemUpper]; ok { return e.encodeVexMove(mnemUpper, ms, ops) } @@ -531,6 +545,9 @@ func validMoveOther(ms vexMoveSpec, op Operand) bool { // the given precomputed fields. It is shared by every register/rm VEX form; // immediate bytes are appended by the caller. func (e *enc) emitVexFields(spec vexSpec, l, regField, rBit, vvvvBar int, rm Operand) error { + if l > 1 { + return fmt.Errorf("ZMM operand requires an EVEX instruction") + } var modrm, sib int var disp []byte var xBit, bBit int diff --git a/cmd/gasm/main.go b/cmd/gasm/main.go index 8748010..7632700 100644 --- a/cmd/gasm/main.go +++ b/cmd/gasm/main.go @@ -26,7 +26,7 @@ import ( // version is the release version, stamped at build time via // -ldflags "-X main.version=…" (defaulting to the current release). -var version = "0.4.0" +var version = "0.5.0" func main() { if len(os.Args) < 2 { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e419643..f721f36 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -206,13 +206,16 @@ memory destination r/m), the direction-sensitive moves (`VMOVDQU`, `VMOVUPD`, `VMULPD`, `VXORPD`, `VUNPCKHPD`, the scalar `VADDSD`/`VMULSD`, `VCVTDQ2PD`, `VFMADD231PD`) and the no-operand `VZEROUPPER` — together with `VPERMD` and the scalar families (`CMOVcc`, `SETcc`, `LZCNT`/`TZCNT`, the extending moves, -`CVTSx2SD`, `IMUL3`), covering every instruction the go-flac AVX2 kernels -use. Every encoding is validated two ways: by round-trip decoding through -`golang.org/x/arch`, and byte-for-byte against the machine code the real Go -assembler emits — a comparison that holds for the whole kernel: all 17 -functions of the go-flac AVX2 file assemble to exactly the Go toolchain's -bytes, the lone exception being the displacements of the static-constant -loads, which the Go linker fills at link time. +`CVTSx2SD`, `IMUL3`) and the EVEX (AVX-512) prefix — the four-byte prefix with +5-bit register fields (Z0–Z31, X/Y 16–31), opmask registers as operands and +mask destinations, and the compressed disp8×N displacement, whose multiplier +follows the memory operand's size — covering every instruction the go-flac +AVX2 and AVX-512 kernels use. Every encoding is validated two ways: by +round-trip decoding through `golang.org/x/arch`, and byte-for-byte against +the machine code the real Go assembler emits — a comparison that holds for +whole functions: all 27 functions of both kernels assemble to exactly the Go +toolchain's bytes, the lone exception being the displacements of the +static-constant loads, which the Go linker fills at link time. File-level assembly (`AssembleFile`) goes beyond single functions: it materialises the file's static symbols (`GLOBL`/`DATA`) in a data section @@ -220,7 +223,7 @@ behind the code and resolves references to them (`mask<>(SB)`) to RIP-relative loads whose displacements point inside the resulting image, so the bytes are self-consistent at any base address. External (non-file-local) symbols are rejected: they need object-file emission, which — together with -EVEX / AVX-512 and the other architectures — is the rest of Phase 2. +EVEX masking/zeroing and the other architectures — is the rest of Phase 2. ## Extension points diff --git a/justfile b/justfile index 186dd1c..5ac0b64 100644 --- a/justfile +++ b/justfile @@ -3,7 +3,7 @@ # gasm-devkit — developer tooling for Go's Plan 9 assembler (GAsm). -version := "0.4.0" +version := "0.5.0" default: @just --list