gasm-sdk/asm/vex_test.go

// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: BSD-3-Clause

package asm

import (
	"testing"

	"golang.org/x/arch/x86/x86asm"
)

func vreg(t *testing.T, name string) Reg {
	t.Helper()
	r, ok := ParseReg(name)
	if !ok {
		t.Fatalf("unknown register %s", name)
	}
	return r
}

// TestVexNDS3 encodes `mnem Y0, Y1, Y2` for every three-operand NDS
// instruction and verifies it round-trips through the x86 decoder to the same
// mnemonic.  A wrong opcode/map/pp surfaces as a different decoded instruction.
func TestVexNDS3(t *testing.T) {
	for mnem, spec := range vexTable {
		if spec.form != vexNDS3 {
			continue
		}
		code, err := Encode(mnem, vreg(t, "Y0"), vreg(t, "Y1"), vreg(t, "Y2"))
		if err != nil {
			t.Errorf("%s: Encode: %v", mnem, err)
			continue
		}
		inst, err := x86asm.Decode(code, 64)
		if err != nil {
			t.Errorf("%s: Decode(% x): %v", mnem, code, err)
			continue
		}
		if inst.Op.String() != mnem {
			t.Errorf("%s: decoded as %s (% x)", mnem, inst.Op.String(), code)
		}
	}
}

// TestVexGoFlac checks a representative go-flac instruction sequence encodes
// and decodes as expected.
func TestVexGoFlac(t *testing.T) {
	// VPADDD Y5, Y8, Y8 → vpaddd ymm8, ymm8, ymm5.
	code, err := Encode("VPADDD", vreg(t, "Y5"), vreg(t, "Y8"), vreg(t, "Y8"))
	if err != nil {
		t.Fatalf("Encode: %v", err)
	}
	inst, err := x86asm.Decode(code, 64)
	if err != nil {
		t.Fatalf("Decode(% x): %v", code, err)
	}
	if inst.Op != x86asm.VPADDD {
		t.Fatalf("decoded %s, want VPADDD", inst.Op)
	}
}

// TestVexXMM checks the 128-bit (XMM) form selects VEX.L=0.
func TestVexXMM(t *testing.T) {
	code, err := Encode("VPXOR", vreg(t, "X7"), vreg(t, "X7"), vreg(t, "X7"))
	if err != nil {
		t.Fatalf("Encode: %v", err)
	}
	inst, err := x86asm.Decode(code, 64)
	if err != nil {
		t.Fatalf("Decode(% x): %v", code, err)
	}
	if inst.Op != x86asm.VPXOR {
		t.Fatalf("decoded %s, want VPXOR", inst.Op)
	}
	// vpxor xmm7, xmm7, xmm7 → C5 C9 EF FF (2-byte VEX, L=0).
	if code[0] != 0xC5 {
		t.Errorf("expected 2-byte VEX (C5), got % x", code)
	}
}

// TestVexRM validates the two-operand (reg=dst, rm=src, no vvvv) forms by
// round-tripping through the decoder.
func TestVexRM(t *testing.T) {
	cases := []struct {
		mnem string
		ops  []Operand
		want x86asm.Op
	}{
		{"VPMOVSXWD", []Operand{Ptr(SI, 0, 16), vreg(t, "Y0")}, x86asm.VPMOVSXWD},
		{"VPMOVSXDQ", []Operand{vreg(t, "X0"), vreg(t, "Y4")}, x86asm.VPMOVSXDQ},
		{"VPMOVZXDQ", []Operand{vreg(t, "X4"), vreg(t, "Y4")}, x86asm.VPMOVZXDQ},
		{"VPBROADCASTD", []Operand{vreg(t, "X0"), vreg(t, "Y15")}, x86asm.VPBROADCASTD},
		{"VPMOVMSKB", []Operand{vreg(t, "X11"), AX}, x86asm.VPMOVMSKB},
		{"VMOVMSKPS", []Operand{vreg(t, "Y7"), AX}, x86asm.VMOVMSKPS},
	}
	for _, c := range cases {
		code, err := Encode(c.mnem, c.ops...)
		if err != nil {
			t.Errorf("%s: Encode: %v", c.mnem, err)
			continue
		}
		inst, err := x86asm.Decode(code, 64)
		if err != nil {
			t.Errorf("%s: Decode(% x): %v", c.mnem, code, err)
			continue
		}
		if inst.Op != c.want {
			t.Errorf("%s: decoded as %s (% x)", c.mnem, inst.Op, code)
		}
	}
}

// TestVexShiftImm validates the immediate-shift form, checking the destination
// (VEX.vvvv) and source (ModRM.rm) land in the right places.
func TestVexShiftImm(t *testing.T) {
	// VPSLLD $1, Y3, Y4 → vpslld ymm4, ymm3, 1.
	code, err := Encode("VPSLLD", Imm(1), vreg(t, "Y3"), vreg(t, "Y4"))
	if err != nil {
		t.Fatalf("Encode: %v", err)
	}
	inst, err := x86asm.Decode(code, 64)
	if err != nil {
		t.Fatalf("Decode(% x): %v", code, err)
	}
	if inst.Op != x86asm.VPSLLD {
		t.Fatalf("decoded %s, want VPSLLD (% x)", inst.Op, code)
	}
	// Intel order: dst, src, imm → "vpslld ymm4, ymm3, 0x1".
	if got := x86asm.IntelSyntax(inst, 0, nil); got != "vpslld ymm4, ymm3, 0x1" {
		t.Errorf("VPSLLD syntax = %q, want \"vpslld ymm4, ymm3, 0x1\" (% x)", got, code)
	}

	// VPSRAD $31, Y3, Y3 → vpsrad ymm3, ymm3, 31.
	code, err = Encode("VPSRAD", Imm(31), vreg(t, "Y3"), vreg(t, "Y3"))
	if err != nil {
		t.Fatalf("Encode VPSRAD: %v", err)
	}
	inst, err = x86asm.Decode(code, 64)
	if err != nil || inst.Op != x86asm.VPSRAD {
		t.Fatalf("VPSRAD decoded %v (err %v), want VPSRAD", inst.Op, err)
	}
}