gasm-sdk/asm/encode_test.go

// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
// SPDX-License-Identifier: BSD-3-Clause

package asm

import (
	"fmt"
	"strings"
	"testing"

	"golang.org/x/arch/x86/x86asm"
)

// decode encodes an instruction and decodes it back, returning the decoded
// instruction and its Intel-syntax rendering.
func decode(t *testing.T, mnemonic string, ops ...Operand) (x86asm.Inst, string) {
	t.Helper()
	code, err := Encode(mnemonic, ops...)
	if err != nil {
		t.Fatalf("Encode(%s): %v", mnemonic, err)
	}
	inst, err := x86asm.Decode(code, 64)
	if err != nil {
		t.Fatalf("Decode(%x) of %s: %v", code, mnemonic, err)
	}
	if inst.Len != len(code) {
		t.Fatalf("Decode consumed %d of %d bytes for %s (%x)", inst.Len, len(code), mnemonic, code)
	}
	return inst, x86asm.IntelSyntax(inst, 0, nil)
}

// checkSyntax asserts an instruction encodes and decodes to the expected
// Intel-syntax string.
func checkSyntax(t *testing.T, want, mnemonic string, ops ...Operand) {
	t.Helper()
	_, got := decode(t, mnemonic, ops...)
	if got != want {
		t.Errorf("%s: got %q, want %q", mnemonic, got, want)
	}
}

// checkOp asserts the decoded opcode (used for relative jumps, whose rendered
// target depends on the program counter).
func checkOp(t *testing.T, want x86asm.Op, mnemonic string, ops ...Operand) {
	t.Helper()
	inst, _ := decode(t, mnemonic, ops...)
	if inst.Op != want {
		t.Errorf("%s: got op %v, want %v", mnemonic, inst.Op, want)
	}
}

func TestMov(t *testing.T) {
	checkSyntax(t, "mov rbx, rax", "MOVQ", AX, BX)
	checkSyntax(t, "mov ebx, eax", "MOVL", AX, BX)
	checkSyntax(t, "mov bl, al", "MOVB", AL, BL)
	checkSyntax(t, "mov rax, rbx", "MOVQ", BX, AX)
	checkSyntax(t, "mov rbx, qword ptr [rax]", "MOVQ", Ptr(AX, 0, 8), BX)
	checkSyntax(t, "mov qword ptr [rbx], rax", "MOVQ", AX, Ptr(BX, 0, 8))
	checkSyntax(t, "mov rbx, qword ptr [rax+0x10]", "MOVQ", Ptr(AX, 0x10, 8), BX)
	checkSyntax(t, "mov rbx, qword ptr [rsi+4*rbx]", "MOVQ", Idx(SI, BX, 4, 0, 8), BX)
	// A small positive immediate compresses to the 32-bit zero-extending
	// form (matching go tool asm), so the disassembler renders the 32-bit
	// register name even for MOVQ.
	checkSyntax(t, "mov eax, 0x5", "MOVQ", Imm(5), AX)
	checkSyntax(t, "mov r8d, 0x5", "MOVQ", Imm(5), Reg{idx: 8, size: 8})
	checkSyntax(t, "mov qword ptr [rax], 0x5", "MOVQ", Imm(5), Ptr(AX, 0, 8))
	checkSyntax(t, "mov r12, r13", "MOVQ", Reg{idx: 13, size: 8}, Reg{idx: 12, size: 8})
}

func TestALU(t *testing.T) {
	checkSyntax(t, "add rbx, rax", "ADDQ", AX, BX)
	checkSyntax(t, "add rax, 0x1", "ADDQ", Imm(1), AX)
	checkSyntax(t, "add rax, 0x12c", "ADDQ", Imm(300), AX)
	checkSyntax(t, "sub rdx, rcx", "SUBQ", CX, DX)
	checkSyntax(t, "and rbx, 0x7", "ANDQ", Imm(7), BX)
	checkSyntax(t, "or rcx, rbx", "ORQ", BX, CX)
	checkSyntax(t, "xor rax, rax", "XORQ", AX, AX)
	checkSyntax(t, "cmp rsi, r10", "CMPQ", SI, Reg{idx: 10, size: 8})
	checkSyntax(t, "add rbx, qword ptr [rax]", "ADDQ", Ptr(AX, 0, 8), BX)
	checkSyntax(t, "add qword ptr [rax], rbx", "ADDQ", BX, Ptr(AX, 0, 8))
	checkSyntax(t, "cmp rbx, -0x20", "CMPQ", Imm(-32), BX)
	// The Go assembler's own spelling: immediate second.
	checkSyntax(t, "cmp ecx, 0x1f", "CMPL", CX, Imm(31))
	checkSyntax(t, "cmp ecx, -0x80000000", "CMPL", CX, Imm(-2147483648))
	checkSyntax(t, "cmp r9, -0x80000000", "CMPQ", Reg{idx: 9, size: 8}, Imm(-2147483648))
}

func TestLea(t *testing.T) {
	checkSyntax(t, "lea r9, ptr [rsi+4*rbx]", "LEAQ", Idx(SI, BX, 4, 0, 8), Reg{idx: 9, size: 8})
	checkSyntax(t, "lea rax, ptr [rbx+0x8]", "LEAQ", Ptr(BX, 0x8, 8), AX)
}

func TestTest(t *testing.T) {
	checkSyntax(t, "test rax, rax", "TESTQ", AX, AX)
	checkSyntax(t, "test rbx, 0x7", "TESTQ", Imm(7), BX)
}

func TestPushPop(t *testing.T) {
	checkSyntax(t, "push rbx", "PUSHQ", BX)
	checkSyntax(t, "pop r12", "POPQ", Reg{idx: 12, size: 8})
	checkSyntax(t, "push 0x5", "PUSHQ", Imm(5))
}

func TestUnary(t *testing.T) {
	checkSyntax(t, "inc rax", "INCQ", AX)
	checkSyntax(t, "dec rbx", "DECQ", BX)
	checkSyntax(t, "neg rcx", "NEGQ", CX)
	checkSyntax(t, "not rdx", "NOTQ", DX)
}

func TestShift(t *testing.T) {
	checkSyntax(t, "shl rdx, 0x2", "SHLQ", Imm(2), DX)
	checkSyntax(t, "shl rdx, cl", "SHLQ", CL, DX)
	checkSyntax(t, "shl rdx, 0x1", "SHLQ", Imm(1), DX)
	checkSyntax(t, "sar rcx, 0x1f", "SARQ", Imm(31), CX)
}

func TestImul(t *testing.T) {
	checkSyntax(t, "imul rdx, rcx", "IMULQ", CX, DX)
	checkSyntax(t, "imul edx, edx, 0x3", "IMULL", Imm(3), DX, DX)
	checkSyntax(t, "imul rdx, rcx, 0x100", "IMULQ", Imm(256), CX, DX)
}

func TestControl(t *testing.T) {
	checkSyntax(t, "ret", "RET")
	checkSyntax(t, "nop", "NOP")
	checkOp(t, x86asm.JMP, "JMP", Imm(0))
	checkOp(t, x86asm.CALL, "CALL", Imm(0))
	checkOp(t, x86asm.JGE, "JGE", Imm(0))
	checkOp(t, x86asm.JNE, "JNE", Imm(0))
	checkOp(t, x86asm.JBE, "JLS", Imm(0))
}

// TestSSEMoveGroundTruth checks the legacy (non-VEX) SSE moves byte for byte
// against the Go assembler.  wantOp is the decoder's name, which differs from
// the Plan 9 spelling for the octa moves (MOVOU = MOVDQU, MOVO = MOVDQA).
func TestSSEMoveGroundTruth(t *testing.T) {
	cases := []struct {
		name   string
		mnem   string
		ops    []Operand
		want   string
		wantOp string
	}{
		{"MOVOU (SI),X1", "MOVOU", []Operand{Ptr(SI, 0, 16), vreg(t, "X1")}, "f30f6f0e", "MOVDQU"},
		{"MOVOU X3,(DI)", "MOVOU", []Operand{vreg(t, "X3"), Ptr(DI, 0, 16)}, "f30f7f1f", "MOVDQU"},
		{"MOVOU X1,X2", "MOVOU", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "f30f6fd1", "MOVDQU"},
		{"MOVOU (SI)(BX*4),X9", "MOVOU", []Operand{Idx(SI, BX, 4, 0, 16), vreg(t, "X9")}, "f3440f6f0c9e", "MOVDQU"},
		{"MOVO (SI),X1", "MOVO", []Operand{Ptr(SI, 0, 16), vreg(t, "X1")}, "660f6f0e", "MOVDQA"},
		{"MOVO X3,(DI)", "MOVO", []Operand{vreg(t, "X3"), Ptr(DI, 0, 16)}, "660f7f1f", "MOVDQA"},
		{"MOVUPS (SI),X1", "MOVUPS", []Operand{Ptr(SI, 0, 16), vreg(t, "X1")}, "0f100e", "MOVUPS"},
		{"MOVAPS X3,(DI)", "MOVAPS", []Operand{vreg(t, "X3"), Ptr(DI, 0, 16)}, "0f291f", "MOVAPS"},
		{"MOVUPD (SI),X1", "MOVUPD", []Operand{Ptr(SI, 0, 16), vreg(t, "X1")}, "660f100e", "MOVUPD"},
		{"MOVAPD X3,(DI)", "MOVAPD", []Operand{vreg(t, "X3"), Ptr(DI, 0, 16)}, "660f291f", "MOVAPD"},
		{"MOVSD (SI),X1", "MOVSD", []Operand{Ptr(SI, 0, 8), vreg(t, "X1")}, "f20f100e", "MOVSD_XMM"},
		{"MOVSD X1,X2", "MOVSD", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "f20f10d1", "MOVSD_XMM"},
		{"MOVSS X3,(DI)", "MOVSS", []Operand{vreg(t, "X3"), Ptr(DI, 0, 4)}, "f30f111f", "MOVSS"},
	}
	for _, c := range cases {
		code, err := Encode(c.mnem, c.ops...)
		if err != nil {
			t.Errorf("%s: Encode: %v", c.name, err)
			continue
		}
		if got := hexCompact(code); got != c.want {
			t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
			continue
		}
		inst, err := x86asm.Decode(code, 64)
		if err != nil {
			t.Errorf("%s: Decode(%x): %v", c.name, code, err)
			continue
		}
		if inst.Op.String() != c.wantOp {
			t.Errorf("%s: decoded as %s", c.name, inst.Op.String())
		}
	}
}

// TestGoFlacScalarTail encodes the scalar tail of an analyze kernel to confirm
// the encoder handles a realistic instruction sequence.
func TestGoFlacScalarTail(t *testing.T) {
	// MOVQ swin_base+0(FP), SI — modelled as MOVQ disp(reg), reg.
	checkSyntax(t, "mov rsi, qword ptr [rax+0x10]", "MOVQ", Ptr(AX, 0x10, 8), SI)
	checkSyntax(t, "lea r9, ptr [rsi+4*rbx]", "LEAQ", Idx(SI, BX, 4, 0, 8), Reg{idx: 9, size: 8})
	checkSyntax(t, "and r10, -0x8", "ANDQ", Imm(-8), Reg{idx: 10, size: 8})
}

// TestScalarGroundTruth checks the scalar instruction families the go-flac
// kernels use beyond the basic set, byte for byte against the Go assembler's
// machine code.  wantOp is the x86 decoder's name, which differs from the
// Plan 9 spelling for some of these (CMOVLGT → CMOVG, MOVBLZX → MOVZX, …).
func TestScalarGroundTruth(t *testing.T) {
	r8 := Reg{idx: 8, size: 8}
	r9 := Reg{idx: 9, size: 8}
	r9w := Reg{idx: 9, size: 2}
	r8w := Reg{idx: 8, size: 2}
	r13 := Reg{idx: 13, size: 8}
	cases := []struct {
		name   string
		mnem   string
		ops    []Operand
		want   string
		wantOp string
	}{
		{"LZCNTL AX,CX", "LZCNTL", []Operand{AX, CX}, "f30fbdc8", "LZCNT"},
		{"LZCNTQ R8,R9", "LZCNTQ", []Operand{r8, r9}, "f34d0fbdc8", "LZCNT"},
		{"LZCNTW AX,CX", "LZCNTW", []Operand{AX, CX}, "66f30fbdc8", "LZCNT"},
		{"TZCNTL AX,CX", "TZCNTL", []Operand{AX, CX}, "f30fbcc8", "TZCNT"},
		// Bit scan: BSF/BSR are the unprefixed forms of TZCNT/LZCNT's map.
		{"BSFL AX,CX", "BSFL", []Operand{AX, CX}, "0fbcc8", "BSF"},
		{"BSFQ R8,R9", "BSFQ", []Operand{r8, r9}, "4d0fbcc8", "BSF"},
		{"BSFW AX,CX", "BSFW", []Operand{AX, CX}, "660fbcc8", "BSF"},
		{"BSRL AX,CX", "BSRL", []Operand{AX, CX}, "0fbdc8", "BSR"},
		{"BSRQ AX,CX", "BSRQ", []Operand{AX, CX}, "480fbdc8", "BSR"},
		{"POPCNTL AX,CX", "POPCNTL", []Operand{AX, CX}, "f30fb8c8", "POPCNT"},
		{"POPCNTQ R8,R9", "POPCNTQ", []Operand{r8, r9}, "f34d0fb8c8", "POPCNT"},
		// A 64-bit immediate that fits a signed int32 is compressed exactly
		// as the Go assembler does: positive via B8+rd without REX.W
		// (zero-extended), negative via REX.W C7 /0 (sign-extended).
		{"MOVQ $4,BX", "MOVQ", []Operand{Imm(4), BX}, "bb04000000", "MOV"},
		{"MOVQ $4,R8", "MOVQ", []Operand{Imm(4), r8}, "41b804000000", "MOV"},
		{"MOVQ $-1,BX", "MOVQ", []Operand{Imm(-1), BX}, "48c7c3ffffffff", "MOV"},
		{"MOVQ big,BX", "MOVQ", []Operand{Imm(0x1122334455667788), BX}, "48bb8877665544332211", "MOV"},
		{"CMOVLGT CX,AX", "CMOVLGT", []Operand{CX, AX}, "0f4fc1", "CMOVG"},
		{"CMOVLEQ CX,AX", "CMOVLEQ", []Operand{CX, AX}, "0f44c1", "CMOVE"},
		{"CMOVQGT R9,R8", "CMOVQGT", []Operand{r9, r8}, "4d0f4fc1", "CMOVG"},
		{"CMOVWLS R9W,R8W", "CMOVWLS", []Operand{r9w, r8w}, "66450f46c1", "CMOVBE"},
		{"SETNE AL", "SETNE", []Operand{AL}, "0f95c0", "SETNE"},
		{"SETNE (AX)", "SETNE", []Operand{Ptr(AX, 0, 1)}, "0f9500", "SETNE"},
		{"MOVBLZX AL,CX", "MOVBLZX", []Operand{AL, CX}, "0fb6c8", "MOVZX"},
		{"MOVBLZX (SI),CX", "MOVBLZX", []Operand{Ptr(SI, 0, 1), CX}, "0fb60e", "MOVZX"},
		{"MOVWLSX (SI)(AX*1),CX", "MOVWLSX", []Operand{Idx(SI, AX, 1, 0, 2), CX}, "0fbf0c06", "MOVSX"},
		{"MOVLQSX CX,R8", "MOVLQSX", []Operand{CX, r8}, "4c63c1", "MOVSXD"},
		{"MOVBQZX AL,R8", "MOVBQZX", []Operand{AL, r8}, "4c0fb6c0", "MOVZX"},
		{"MOVWLZX AX,CX", "MOVWLZX", []Operand{AX, CX}, "0fb7c8", "MOVZX"},
		{"MOVWQZX AX,R8", "MOVWQZX", []Operand{AX, r8}, "4c0fb7c0", "MOVZX"},
		{"CVTSL2SD R8,X13", "CVTSL2SD", []Operand{r8, vreg(t, "X13")}, "f2450f2ae8", "CVTSI2SD"},
		{"CVTSL2SD AX,X0", "CVTSL2SD", []Operand{AX, vreg(t, "X0")}, "f20f2ac0", "CVTSI2SD"},
		{"CVTSQ2SD R8,X13", "CVTSQ2SD", []Operand{r8, vreg(t, "X13")}, "f24d0f2ae8", "CVTSI2SD"},
		{"INCW (R13)(AX*2)", "INCW", []Operand{Idx(r13, AX, 2, 0, 2)}, "6641ff444500", "INC"},
		// The traditional three-operand IMUL spelling.
		{"IMUL3L $31,CX,DX", "IMUL3L", []Operand{Imm(31), CX, DX}, "6bd11f", "IMUL"},
		{"IMUL3L $256,CX,DX", "IMUL3L", []Operand{Imm(256), CX, DX}, "69d100010000", "IMUL"},
		{"IMUL3Q $7,R9,R8", "IMUL3Q", []Operand{Imm(7), r9, r8}, "4d6bc107", "IMUL"},
		{"IMUL3W $5,CX,DX", "IMUL3W", []Operand{Imm(5), CX, DX}, "666bd105", "IMUL"},
		// Negative displacement with base + index (regression: the parser
		// used to drop the whole address).
		{"LEAQ -4(DX)(R9*4),R9", "LEAQ", []Operand{Idx(DX, r9, 4, -4, 8), r9}, "4e8d4c8afc", "LEA"},
		{"LEAQ 16(SI)(BX*4),R10", "LEAQ", []Operand{Idx(SI, BX, 4, 16, 8), Reg{idx: 10, size: 8}}, "4c8d549e10", "LEA"},
		// Register-to-register MOV uses the r/m←r opcode (reg = source), the
		// Go assembler's choice.
		{"MOVQ BX,R10", "MOVQ", []Operand{BX, Reg{idx: 10, size: 8}}, "4989da", "MOV"},
		{"MOVQ AX,BX", "MOVQ", []Operand{AX, BX}, "4889c3", "MOV"},
		{"MOVL AX,BX", "MOVL", []Operand{AX, BX}, "89c3", "MOV"},
		{"MOVB AL,BL", "MOVB", []Operand{AL, BL}, "88c3", "MOV"},
		{"MOVW AX,BX", "MOVW", []Operand{AX, BX}, "6689c3", "MOV"},
		{"MOVQ R12,R13", "MOVQ", []Operand{Reg{idx: 12, size: 8}, Reg{idx: 13, size: 8}}, "4d89e5", "MOV"},
		// CMP must record first − second: with a register second operand the
		// first goes in r/m, with a memory second operand the first goes in reg.
		{"CMPQ SI,R10", "CMPQ", []Operand{SI, Reg{idx: 10, size: 8}}, "4c39d6", "CMP"},
		{"CMPQ SI,(AX)", "CMPQ", []Operand{SI, Ptr(AX, 0, 8)}, "483b30", "CMP"},
		{"CMPQ (AX),SI", "CMPQ", []Operand{Ptr(AX, 0, 8), SI}, "483930", "CMP"},
		{"CMPL CX,(AX)", "CMPL", []Operand{CX, Ptr(AX, 0, 4)}, "3b08", "CMP"},
		{"CMPB AL,(BX)", "CMPB", []Operand{AL, Ptr(BX, 0, 1)}, "3a03", "CMP"},
		{"CMPW AX,BX", "CMPW", []Operand{AX, BX}, "6639d8", "CMP"},
	}
	for _, c := range cases {
		code, err := Encode(c.mnem, c.ops...)
		if err != nil {
			t.Errorf("%s: Encode: %v", c.name, err)
			continue
		}
		if got := strings.ReplaceAll(hexBytes(code), " ", ""); got != c.want {
			t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
			continue
		}
		inst, err := x86asm.Decode(code, 64)
		if err != nil {
			t.Errorf("%s: Decode(% x): %v", c.name, code, err)
			continue
		}
		if inst.Op.String() != c.wantOp {
			t.Errorf("%s: decoded as %s", c.name, inst.Op.String())
		}
	}
}

// TestScalarErrors checks that malformed conditional / extend / convert
// instructions are rejected.
func TestScalarErrors(t *testing.T) {
	cases := []struct {
		name string
		mnem string
		ops  []Operand
	}{
		{"CMOV arity", "CMOVLGT", []Operand{AX}},
		{"CMOV bare", "CMOV", []Operand{AX, BX}},
		{"CMOV bad size", "CMOVBGT", []Operand{AX, BX}},
		{"CMOV bad condition", "CMOVLXX", []Operand{AX, BX}},
		{"CMOV mem dst", "CMOVLGT", []Operand{AX, Ptr(BX, 0, 4)}},
		{"SET arity", "SETNE", []Operand{AL, BL}},
		{"SET bad condition", "SETXX", []Operand{AL}},
		{"SET bare", "SET", []Operand{AL}},
		{"LZCNT arity", "LZCNTL", []Operand{AX}},
		{"LZCNT mem dst", "LZCNTL", []Operand{AX, Ptr(BX, 0, 4)}},
		{"MOVBLZX mem dst", "MOVBLZX", []Operand{AL, Ptr(BX, 0, 4)}},
		{"CVTSL2SD gpr dst", "CVTSL2SD", []Operand{AX, BX}},
	}
	for _, c := range cases {
		if _, err := Encode(c.mnem, c.ops...); err == nil {
			t.Errorf("%s: expected an error, got none", c.name)
		}
	}
}

// TestSSEBinGroundTruth checks the legacy packed/scalar binary family
// byte for byte (no prefix / 66 / F2 / F3 variants).
func TestSSEBinGroundTruth(t *testing.T) {
	cases := []struct {
		name string
		mnem string
		ops  []Operand
		want string
	}{
		{"MULPS X0,X1", "MULPS", []Operand{vreg(t, "X0"), vreg(t, "X1")}, "0f59c8"},
		{"MULPS (DI),X1", "MULPS", []Operand{Ptr(DI, 0, 16), vreg(t, "X1")}, "0f590f"},
		{"ADDPD X1,X2", "ADDPD", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "660f58d1"},
		{"XORPS X0,X0", "XORPS", []Operand{vreg(t, "X0"), vreg(t, "X0")}, "0f57c0"},
		{"UNPCKLPS X0,X0", "UNPCKLPS", []Operand{vreg(t, "X0"), vreg(t, "X0")}, "0f14c0"},
		{"MULSD X1,X2", "MULSD", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "f20f59d1"},
		{"ADDSS (DI),X0", "ADDSS", []Operand{Ptr(DI, 0, 4), vreg(t, "X0")}, "f30f5807"},
	}
	for _, c := range cases {
		code, err := Encode(c.mnem, c.ops...)
		if err != nil {
			t.Errorf("%s: %v", c.name, err)
			continue
		}
		if got := fmt.Sprintf("%x", code); got != c.want {
			t.Errorf("%s = %s, want %s", c.name, got, c.want)
		}
	}
}

// TestSSEShuffleGroundTruth checks the imm8 shuffle family: immediate
// first in Plan 9 order, encoded last on the wire.
func TestSSEShuffleGroundTruth(t *testing.T) {
	cases := []struct {
		name string
		mnem string
		ops  []Operand
		want string
	}{
		{"SHUFPS $0,X0,X0", "SHUFPS", []Operand{Imm(0), vreg(t, "X0"), vreg(t, "X0")}, "0fc6c000"},
		{"SHUFPS $27,X1,X2", "SHUFPS", []Operand{Imm(27), vreg(t, "X1"), vreg(t, "X2")}, "0fc6d11b"},
		{"PSHUFD $0,X0,X0", "PSHUFD", []Operand{Imm(0), vreg(t, "X0"), vreg(t, "X0")}, "660f70c000"},
		{"PSHUFLW $3,(DI),X1", "PSHUFLW", []Operand{Imm(3), Ptr(DI, 0, 8), vreg(t, "X1")}, "f20f700f03"},
		{"PSHUFHW $2,X1,X2", "PSHUFHW", []Operand{Imm(2), vreg(t, "X1"), vreg(t, "X2")}, "f30f70d102"},
	}
	for _, c := range cases {
		code, err := Encode(c.mnem, c.ops...)
		if err != nil {
			t.Errorf("%s: %v", c.name, err)
			continue
		}
		if got := fmt.Sprintf("%x", code); got != c.want {
			t.Errorf("%s = %s, want %s", c.name, got, c.want)
		}
	}
}