feat(asm): byte-identical go-flac AVX2 assembly with scalar families and jump relaxation

Assisted-by: Qwen 3.8 Max Preview

asm/assemble.go

@@ -13,45 +13,85 @@ import (
 // Assemble encodes the body of a TEXT function into x86-64 machine code,
 // resolving local labels to relative jump offsets and translating the FP/SP
 // pseudo-registers onto the hardware stack pointer (matching the Go
-// assembler's default frame-pointer behaviour).  Jumps always use the 32-bit
+// assembler's default frame-pointer behaviour).  Jumps start in the short
-// relative form so instruction sizes are fixed and offsets resolve in a single
+// (rel8) form and expand to rel32 when the settled displacement does not fit;
-// layout pass.
+// sizes only grow, so the layout reaches a fixed point in a few passes.  CALL
+// has no short form and is always rel32.
 //
 // Supported operands: registers, memory (real base register), immediates,
 // FP/SP frame-relative operands, and local-label jumps.  SB (global symbol)
-// operands require relocations and are not yet supported; SIMD (VEX/EVEX)
+// operands require relocations and are not yet supported; the SIMD (VEX/AVX2)
-// instructions are pending.
+// integer and shuffle/extract/permute/move set is in.
 func Assemble(t *ast.Text) ([]byte, map[string]int, error) {
 	fi := computeFrame(t)
+	chain := jumpChain(t)
+	resolve := func(name string) string {
+		if r, ok := chain[name]; ok {
+			return r
+		}
+		return name
+	}
 
-	// Pass 1: lay out instructions (including prologue/epilogue) to fix label
+	// Layout: iterate jump sizes to a fixed point.
-	// offsets.
+	long := make([]bool, len(t.Body))
-	offsets := map[string]int{}
 	sizes := make([]int, len(t.Body))
-	pos := len(fi.prologue)
+	offsets := map[string]int{}
-	for i, stmt := range t.Body {
+	pcs := make([]int, len(t.Body))
-		switch s := stmt.(type) {
+	for {
-		case *ast.Label:
+		pos := len(fi.prologue)
-			offsets[s.Name.Text] = pos
+		for i, stmt := range t.Body {
-		case *ast.Instr:
+			switch s := stmt.(type) {
-			sz, err := instrSize(s, fi)
+			case *ast.Label:
-			if err != nil {
+				offsets[s.Name.Text] = pos
-				return nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
+			case *ast.Instr:
+				sz, err := instrSize(s, fi, long[i])
+				if err != nil {
+					return nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
+				}
+				sizes[i] = sz
+				pcs[i] = pos
+				pos += sz
 			}
-			sizes[i] = sz
+		}
-			pos += sz
+		// Expand any short jump whose displacement no longer fits rel8.
+		changed := false
+		for i, stmt := range t.Body {
+			s, ok := stmt.(*ast.Instr)
+			if !ok {
+				continue
+			}
+			mnem := strings.ToUpper(s.Mnemonic.Text)
+			if !isJumpMnemonic(mnem) || mnem == "CALL" || long[i] {
+				continue
+			}
+			name, ok := labelName(s.Operands[0])
+			if !ok {
+				continue // reported during emission
+			}
+			target, ok := offsets[resolve(name)]
+			if !ok {
+				continue // reported during emission
+			}
+			rel := int64(target - (pcs[i] + jumpSize(mnem, false)))
+			if !fits8(rel) {
+				long[i] = true
+				changed = true
+			}
+		}
+		if !changed {
+			break
 		}
 	}
 
 	// Pass 2: emit.
 	out := append([]byte(nil), fi.prologue...)
-	pos = len(fi.prologue)
+	pos := len(fi.prologue)
 	for i, stmt := range t.Body {
 		s, ok := stmt.(*ast.Instr)
 		if !ok {
 			continue
 		}
-		code, err := encodeInstr(s, pos, offsets, fi)
+		code, err := encodeInstr(s, pos, offsets, fi, long[i], resolve)
 		if err != nil {
 			return nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
 		}
@@ -64,6 +104,58 @@ func Assemble(t *ast.Text) ([]byte, map[string]int, error) {
 	return out, offsets, nil
 }
 
+// jumpChain precomputes jump-to-jump folding: a label whose first instruction
+// is an unconditional local jump redirects its own jumpers to the ultimate
+// target.  The Go toolchain chases exactly these chains (the linker's xfol
+// pass) before it encodes branches, so matching its bytes requires the same
+// redirection.
+func jumpChain(t *ast.Text) map[string]string {
+	// label → the target of its leading unconditional local JMP, if any.
+	leadsTo := map[string]string{}
+	for i, stmt := range t.Body {
+		l, ok := stmt.(*ast.Label)
+		if !ok {
+			continue
+		}
+		// Stacked labels share an address: skip to the first instruction.
+		j := i + 1
+		for j < len(t.Body) {
+			if _, isLabel := t.Body[j].(*ast.Label); !isLabel {
+				break
+			}
+			j++
+		}
+		if j >= len(t.Body) {
+			continue
+		}
+		in, ok := t.Body[j].(*ast.Instr)
+		if !ok || strings.ToUpper(in.Mnemonic.Text) != "JMP" || len(in.Operands) != 1 {
+			continue
+		}
+		if name, ok := labelName(in.Operands[0]); ok {
+			leadsTo[l.Name.Text] = name
+		}
+	}
+	// Chase each chain to its end, guarding against cycles.
+	chain := map[string]string{}
+	for name := range leadsTo {
+		visited := map[string]bool{name: true}
+		cur := name
+		for {
+			next, ok := leadsTo[cur]
+			if !ok || visited[next] {
+				break
+			}
+			visited[next] = true
+			cur = next
+		}
+		if cur != name {
+			chain[name] = cur
+		}
+	}
+	return chain
+}
+
 // frameInfo carries the frame layout derived from the TEXT directive.
 type frameInfo struct {
 	size     int   // local frame size ($framesize)
@@ -119,15 +211,15 @@ func addSP(size int) []byte { // ADDQ $size, SP
 	return append([]byte{0x48, 0x81, 0xC4}, le32(int64(size))...)
 }
 
-// instrSize returns the encoded length of an instruction (pass 1).  encodeInstr
+// instrSize returns the encoded length of an instruction (layout pass).
-// already includes the epilogue for a RET in a frame-pointer function; jumps use
+// encodeInstr already includes the epilogue for a RET in a frame-pointer
-// a fixed rel32 size (no epilogue).
+// function; jumps use their short or long form (never an epilogue).
-func instrSize(s *ast.Instr, fi frameInfo) (int, error) {
+func instrSize(s *ast.Instr, fi frameInfo, long bool) (int, error) {
 	mnem := strings.ToUpper(s.Mnemonic.Text)
 	if isJumpMnemonic(mnem) {
-		return jumpSize(mnem), nil
+		return jumpSize(mnem, long), nil
 	}
-	code, err := encodeInstr(s, 0, nil, fi)
+	code, err := encodeInstr(s, 0, nil, fi, false, nil)
 	if err != nil {
 		return 0, err
 	}
@@ -142,18 +234,27 @@ func isJumpMnemonic(mnem string) bool {
 	return ok
 }
 
-// jumpSize returns the fixed length of a rel32 jump instruction.
+// jumpSize returns the length of a jump instruction in the requested form:
-func jumpSize(mnem string) int {
+// short (rel8) where available, otherwise the rel32 form.  CALL is always
-	if mnem == "JMP" || mnem == "CALL" {
+// rel32.
+func jumpSize(mnem string, long bool) int {
+	if mnem == "CALL" {
 		return 5 // opcode + rel32
 	}
+	if !long {
+		return 2 // opcode + rel8
+	}
+	if mnem == "JMP" {
+		return 5 // E9 + rel32
+	}
 	return 6 // 0x0F 0x8x + rel32
 }
 
 // encodeInstr encodes one instruction, resolving jump targets against offsets
 // (relative to pc, the instruction's own offset).  A RET in a frame-pointer
-// function is prefixed with the epilogue.
+// function is prefixed with the epilogue.  resolve, when non-nil, redirects a
-func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo) ([]byte, error) {
+// jump label through the jump-to-jump chain before the offset lookup.
+func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo, long bool, resolve func(string) string) ([]byte, error) {
 	mnem := strings.ToUpper(s.Mnemonic.Text)
 
 	var prefix []byte
@@ -164,7 +265,7 @@ func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo) ([]
 	var code []byte
 	var err error
 	if isJumpMnemonic(mnem) {
-		code, err = encodeJump(s, mnem, pc+len(prefix), offsets)
+		code, err = encodeJump(s, mnem, pc+len(prefix), offsets, long, resolve)
 	} else {
 		code, err = encodeNormal(s, fi)
 	}
@@ -190,9 +291,9 @@ func encodeNormal(s *ast.Instr, fi frameInfo) ([]byte, error) {
 	return Encode(s.Mnemonic.Text, ops...)
 }
 
-// encodeJump encodes a JMP/CALL/Jcc with a rel32 offset resolved from the
+// encodeJump encodes a JMP/CALL/Jcc with a relative offset resolved from the
-// target label.
+// target label, in the short (rel8) or long (rel32) form.
-func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int) ([]byte, error) {
+func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int, long bool, resolve func(string) string) ([]byte, error) {
 	if len(s.Operands) != 1 {
 		return nil, fmt.Errorf("jump expects 1 operand, got %d", len(s.Operands))
 	}
@@ -200,12 +301,25 @@ func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int) ([]by
 	if !ok {
 		return nil, fmt.Errorf("jump target must be a local label")
 	}
+	if resolve != nil && mnem != "CALL" {
+		name = resolve(name)
+	}
 	target, ok := offsets[name]
 	if !ok {
 		return nil, fmt.Errorf("undefined label %q", name)
 	}
-	rel := int64(target - (pc + jumpSize(mnem)))
+	rel := int64(target - (pc + jumpSize(mnem, long)))
 
+	if !long {
+		if !fits8(rel) {
+			return nil, fmt.Errorf("jump to %q does not fit the short form", name)
+		}
+		if mnem == "JMP" {
+			return []byte{0xEB, byte(int8(rel))}, nil
+		}
+		cc, _ := condCode(mnem)
+		return []byte{0x70 + byte(cc), byte(int8(rel))}, nil
+	}
 	switch mnem {
 	case "JMP":
 		return append([]byte{0xE9}, le32(rel)...), nil

asm/assemble_test.go

@@ -199,3 +199,121 @@ TEXT ·withframe(SB), NOSPLIT, $16-16
 		t.Errorf("frame translation mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
 	}
 }
+
+// TestAssembleVexKernel assembles the horizontal-sum reduction the go-flac
+// kernels end with — exercising the VEX moves, shuffle and extract forms
+// through the full parser → encoder path — and checks the output is
+// byte-identical to the Go assembler's.
+func TestAssembleVexKernel(t *testing.T) {
+	fn := firstText(t, `
+#include "textflag.h"
+TEXT ·hsum(SB), NOSPLIT, $0
+	VPADDQ       Y8, Y9, Y8
+	VEXTRACTI128 $1, Y8, X9
+	VPADDQ       X9, X8, X8
+	VPSHUFD      $0xEE, X8, X9
+	VPADDQ       X9, X8, X8
+	VMOVQ        X8, AX
+	VZEROUPPER
+	RET
+`)
+	code, _, err := Assemble(fn)
+	if err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	// From the Go-assembled function:
+	//   VPADDQ Y8, Y9, Y8      c44135d4c0
+	//   VEXTRACTI128 $1, Y8, X9  c4437d39c101
+	//   VPADDQ X9, X8, X8      c44139d4c1
+	//   VPSHUFD $0xEE, X8, X9  c4417970c8ee
+	//   VPADDQ X9, X8, X8      c44139d4c1
+	//   VMOVQ X8, AX           c461f97ec0
+	//   VZEROUPPER             c5f877
+	//   RET                    c3
+	want := []byte{
+		0xc4, 0x41, 0x35, 0xd4, 0xc0,
+		0xc4, 0x43, 0x7d, 0x39, 0xc1, 0x01,
+		0xc4, 0x41, 0x39, 0xd4, 0xc1,
+		0xc4, 0x41, 0x79, 0x70, 0xc8, 0xee,
+		0xc4, 0x41, 0x39, 0xd4, 0xc1,
+		0xc4, 0x61, 0xf9, 0x7e, 0xc0,
+		0xc5, 0xf8, 0x77,
+		0xc3,
+	}
+	if hexBytes(code) != hexBytes(want) {
+		t.Errorf("VEX kernel mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
+	}
+}
+
+// TestAssembleShortJumps checks that a tight loop settles on the short (rel8)
+// jump forms, byte for byte with the Go assembler.
+func TestAssembleShortJumps(t *testing.T) {
+	fn := firstText(t, `
+#include "textflag.h"
+TEXT ·loop(SB), NOSPLIT, $0
+	XORQ AX, AX
+l1:
+	ADDQ $1, AX
+	CMPQ AX, $10
+	JLT l1
+	RET
+`)
+	code, _, err := Assemble(fn)
+	if err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	// From the Go-assembled function:
+	//   XORQ AX, AX       4831c0
+	//   ADDQ $1, AX       4883c001
+	//   CMPQ AX, $10      4883f80a
+	//   JLT l1            7cf6      (short, rel8)
+	//   RET               c3
+	want := []byte{
+		0x48, 0x31, 0xc0,
+		0x48, 0x83, 0xc0, 0x01,
+		0x48, 0x83, 0xf8, 0x0a,
+		0x7c, 0xf6,
+		0xc3,
+	}
+	if hexBytes(code) != hexBytes(want) {
+		t.Errorf("short-jump mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
+	}
+}
+
+// TestAssembleJumpFolding checks jump-to-jump folding: a conditional jump to a
+// label that only holds an unconditional jump is redirected to the ultimate
+// target, exactly as the Go toolchain does before it encodes branches.
+func TestAssembleJumpFolding(t *testing.T) {
+	fn := firstText(t, `
+#include "textflag.h"
+TEXT ·fold(SB), NOSPLIT, $0
+	XORQ AX, AX
+	JGE done
+	INCQ AX
+done:
+	JMP end
+end:
+	RET
+`)
+	code, _, err := Assemble(fn)
+	if err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	// From the Go-assembled function: the JGE skips past the done: trampoline
+	// straight to end:
+	//   XORQ AX, AX       4831c0
+	//   JGE end           7d05      (folded past done)
+	//   INCQ AX           48ffc0
+	//   JMP end           eb00
+	//   RET               c3
+	want := []byte{
+		0x48, 0x31, 0xc0,
+		0x7d, 0x05,
+		0x48, 0xff, 0xc0,
+		0xeb, 0x00,
+		0xc3,
+	}
+	if hexBytes(code) != hexBytes(want) {
+		t.Errorf("jump-folding mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
+	}
+}

asm/encode.go

@@ -46,6 +46,14 @@ func (e *enc) encode(mnem string, ops []Operand) error {
 		return e.encodeVex(upper, ops)
 	}
 
+	// CMOVcc and SETcc carry the condition in the mnemonic (CMOVLGT, SETNE).
+	if strings.HasPrefix(upper, "CMOV") {
+		return e.encodeCmov(upper, ops)
+	}
+	if strings.HasPrefix(upper, "SET") {
+		return e.encodeSet(upper, ops)
+	}
+
 	base, size := splitSize(upper)
 	if size == 0 {
 		size = 8 // default operand size in 64-bit mode (e.g. PUSHQ)
@@ -63,12 +71,18 @@ func (e *enc) encode(mnem string, ops []Operand) error {
 		return e.encodeUnary(unaryOp[base], ops, size)
 	case "SHL", "SHR", "SAR":
 		return e.encodeShift(shiftOp[base], ops, size)
-	case "IMUL":
+	case "IMUL", "IMUL3":
 		return e.encodeImul(ops, size)
 	case "PUSH":
 		return e.encodePushPop(ops, true)
 	case "POP":
 		return e.encodePushPop(ops, false)
+	case "LZCNT", "TZCNT":
+		return e.encodeCount(base, ops, size)
+	case "MOVBLZX", "MOVBQZX", "MOVWLZX", "MOVWQZX", "MOVWLSX", "MOVLQSX":
+		return e.encodeMovExtend(base, ops)
+	case "CVTSL2SD", "CVTSQ2SD":
+		return e.encodeCvtsi2sd(base == "CVTSQ2SD", ops)
 	}
 	return fmt.Errorf("unsupported instruction %q", mnem)
 }
@@ -100,6 +114,7 @@ type instr struct {
 	rexX      bool
 	rexB      bool
 	rexForced bool // REX needed even with all bits zero (8-bit low registers)
+	prefix    byte // legacy 0xF2/0xF3 prefix (0 = none); emitted after 0x66
 	opcode    []byte
 	modrm     int // -1 if absent
 	sib       int // -1 if absent
@@ -111,6 +126,9 @@ func (e *enc) emit(i *instr) error {
 	if i.opSize16 {
 		e.out = append(e.out, 0x66)
 	}
+	if i.prefix != 0 {
+		e.out = append(e.out, i.prefix)
+	}
 	rex := byte(0)
 	if i.rexW {
 		rex |= 0x08

asm/encode_test.go

@@ -4,6 +4,7 @@
 package asm
 
 import (
+	"strings"
 	"testing"
 
 	"golang.org/x/arch/x86/x86asm"
@@ -70,10 +71,14 @@ func TestALU(t *testing.T) {
 	checkSyntax(t, "and rbx, 0x7", "ANDQ", Imm(7), BX)
 	checkSyntax(t, "or rcx, rbx", "ORQ", BX, CX)
 	checkSyntax(t, "xor rax, rax", "XORQ", AX, AX)
-	checkSyntax(t, "cmp r10, rsi", "CMPQ", SI, Reg{idx: 10, size: 8})
+	checkSyntax(t, "cmp rsi, r10", "CMPQ", SI, Reg{idx: 10, size: 8})
 	checkSyntax(t, "add rbx, qword ptr [rax]", "ADDQ", Ptr(AX, 0, 8), BX)
 	checkSyntax(t, "add qword ptr [rax], rbx", "ADDQ", BX, Ptr(AX, 0, 8))
 	checkSyntax(t, "cmp rbx, -0x20", "CMPQ", Imm(-32), BX)
+	// The Go assembler's own spelling: immediate second.
+	checkSyntax(t, "cmp ecx, 0x1f", "CMPL", CX, Imm(31))
+	checkSyntax(t, "cmp ecx, -0x80000000", "CMPL", CX, Imm(-2147483648))
+	checkSyntax(t, "cmp r9, -0x80000000", "CMPQ", Reg{idx: 9, size: 8}, Imm(-2147483648))
 }
 
 func TestLea(t *testing.T) {
@@ -130,3 +135,116 @@ func TestGoFlacScalarTail(t *testing.T) {
 	checkSyntax(t, "lea r9, ptr [rsi+4*rbx]", "LEAQ", Idx(SI, BX, 4, 0, 8), Reg{idx: 9, size: 8})
 	checkSyntax(t, "and r10, -0x8", "ANDQ", Imm(-8), Reg{idx: 10, size: 8})
 }
+
+// TestScalarGroundTruth checks the scalar instruction families the go-flac
+// kernels use beyond the basic set, byte for byte against the Go assembler's
+// machine code.  wantOp is the x86 decoder's name, which differs from the
+// Plan 9 spelling for some of these (CMOVLGT → CMOVG, MOVBLZX → MOVZX, …).
+func TestScalarGroundTruth(t *testing.T) {
+	r8 := Reg{idx: 8, size: 8}
+	r9 := Reg{idx: 9, size: 8}
+	r9w := Reg{idx: 9, size: 2}
+	r8w := Reg{idx: 8, size: 2}
+	r13 := Reg{idx: 13, size: 8}
+	cases := []struct {
+		name   string
+		mnem   string
+		ops    []Operand
+		want   string
+		wantOp string
+	}{
+		{"LZCNTL AX,CX", "LZCNTL", []Operand{AX, CX}, "f30fbdc8", "LZCNT"},
+		{"LZCNTQ R8,R9", "LZCNTQ", []Operand{r8, r9}, "f34d0fbdc8", "LZCNT"},
+		{"LZCNTW AX,CX", "LZCNTW", []Operand{AX, CX}, "66f30fbdc8", "LZCNT"},
+		{"TZCNTL AX,CX", "TZCNTL", []Operand{AX, CX}, "f30fbcc8", "TZCNT"},
+		{"CMOVLGT CX,AX", "CMOVLGT", []Operand{CX, AX}, "0f4fc1", "CMOVG"},
+		{"CMOVLEQ CX,AX", "CMOVLEQ", []Operand{CX, AX}, "0f44c1", "CMOVE"},
+		{"CMOVQGT R9,R8", "CMOVQGT", []Operand{r9, r8}, "4d0f4fc1", "CMOVG"},
+		{"CMOVWLS R9W,R8W", "CMOVWLS", []Operand{r9w, r8w}, "66450f46c1", "CMOVBE"},
+		{"SETNE AL", "SETNE", []Operand{AL}, "0f95c0", "SETNE"},
+		{"SETNE (AX)", "SETNE", []Operand{Ptr(AX, 0, 1)}, "0f9500", "SETNE"},
+		{"MOVBLZX AL,CX", "MOVBLZX", []Operand{AL, CX}, "0fb6c8", "MOVZX"},
+		{"MOVBLZX (SI),CX", "MOVBLZX", []Operand{Ptr(SI, 0, 1), CX}, "0fb60e", "MOVZX"},
+		{"MOVWLSX (SI)(AX*1),CX", "MOVWLSX", []Operand{Idx(SI, AX, 1, 0, 2), CX}, "0fbf0c06", "MOVSX"},
+		{"MOVLQSX CX,R8", "MOVLQSX", []Operand{CX, r8}, "4c63c1", "MOVSXD"},
+		{"MOVBQZX AL,R8", "MOVBQZX", []Operand{AL, r8}, "4c0fb6c0", "MOVZX"},
+		{"MOVWLZX AX,CX", "MOVWLZX", []Operand{AX, CX}, "0fb7c8", "MOVZX"},
+		{"MOVWQZX AX,R8", "MOVWQZX", []Operand{AX, r8}, "4c0fb7c0", "MOVZX"},
+		{"CVTSL2SD R8,X13", "CVTSL2SD", []Operand{r8, vreg(t, "X13")}, "f2450f2ae8", "CVTSI2SD"},
+		{"CVTSL2SD AX,X0", "CVTSL2SD", []Operand{AX, vreg(t, "X0")}, "f20f2ac0", "CVTSI2SD"},
+		{"CVTSQ2SD R8,X13", "CVTSQ2SD", []Operand{r8, vreg(t, "X13")}, "f24d0f2ae8", "CVTSI2SD"},
+		{"INCW (R13)(AX*2)", "INCW", []Operand{Idx(r13, AX, 2, 0, 2)}, "6641ff444500", "INC"},
+		// The traditional three-operand IMUL spelling.
+		{"IMUL3L $31,CX,DX", "IMUL3L", []Operand{Imm(31), CX, DX}, "6bd11f", "IMUL"},
+		{"IMUL3L $256,CX,DX", "IMUL3L", []Operand{Imm(256), CX, DX}, "69d100010000", "IMUL"},
+		{"IMUL3Q $7,R9,R8", "IMUL3Q", []Operand{Imm(7), r9, r8}, "4d6bc107", "IMUL"},
+		{"IMUL3W $5,CX,DX", "IMUL3W", []Operand{Imm(5), CX, DX}, "666bd105", "IMUL"},
+		// Negative displacement with base + index (regression: the parser
+		// used to drop the whole address).
+		{"LEAQ -4(DX)(R9*4),R9", "LEAQ", []Operand{Idx(DX, r9, 4, -4, 8), r9}, "4e8d4c8afc", "LEA"},
+		{"LEAQ 16(SI)(BX*4),R10", "LEAQ", []Operand{Idx(SI, BX, 4, 16, 8), Reg{idx: 10, size: 8}}, "4c8d549e10", "LEA"},
+		// Register-to-register MOV uses the r/m←r opcode (reg = source), the
+		// Go assembler's choice.
+		{"MOVQ BX,R10", "MOVQ", []Operand{BX, Reg{idx: 10, size: 8}}, "4989da", "MOV"},
+		{"MOVQ AX,BX", "MOVQ", []Operand{AX, BX}, "4889c3", "MOV"},
+		{"MOVL AX,BX", "MOVL", []Operand{AX, BX}, "89c3", "MOV"},
+		{"MOVB AL,BL", "MOVB", []Operand{AL, BL}, "88c3", "MOV"},
+		{"MOVW AX,BX", "MOVW", []Operand{AX, BX}, "6689c3", "MOV"},
+		{"MOVQ R12,R13", "MOVQ", []Operand{Reg{idx: 12, size: 8}, Reg{idx: 13, size: 8}}, "4d89e5", "MOV"},
+		// CMP must record first − second: with a register second operand the
+		// first goes in r/m, with a memory second operand the first goes in reg.
+		{"CMPQ SI,R10", "CMPQ", []Operand{SI, Reg{idx: 10, size: 8}}, "4c39d6", "CMP"},
+		{"CMPQ SI,(AX)", "CMPQ", []Operand{SI, Ptr(AX, 0, 8)}, "483b30", "CMP"},
+		{"CMPQ (AX),SI", "CMPQ", []Operand{Ptr(AX, 0, 8), SI}, "483930", "CMP"},
+		{"CMPL CX,(AX)", "CMPL", []Operand{CX, Ptr(AX, 0, 4)}, "3b08", "CMP"},
+		{"CMPB AL,(BX)", "CMPB", []Operand{AL, Ptr(BX, 0, 1)}, "3a03", "CMP"},
+		{"CMPW AX,BX", "CMPW", []Operand{AX, BX}, "6639d8", "CMP"},
+	}
+	for _, c := range cases {
+		code, err := Encode(c.mnem, c.ops...)
+		if err != nil {
+			t.Errorf("%s: Encode: %v", c.name, err)
+			continue
+		}
+		if got := strings.ReplaceAll(hexBytes(code), " ", ""); got != c.want {
+			t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
+			continue
+		}
+		inst, err := x86asm.Decode(code, 64)
+		if err != nil {
+			t.Errorf("%s: Decode(% x): %v", c.name, code, err)
+			continue
+		}
+		if inst.Op.String() != c.wantOp {
+			t.Errorf("%s: decoded as %s", c.name, inst.Op.String())
+		}
+	}
+}
+
+// TestScalarErrors checks that malformed conditional / extend / convert
+// instructions are rejected.
+func TestScalarErrors(t *testing.T) {
+	cases := []struct {
+		name string
+		mnem string
+		ops  []Operand
+	}{
+		{"CMOV arity", "CMOVLGT", []Operand{AX}},
+		{"CMOV bare", "CMOV", []Operand{AX, BX}},
+		{"CMOV bad size", "CMOVBGT", []Operand{AX, BX}},
+		{"CMOV bad condition", "CMOVLXX", []Operand{AX, BX}},
+		{"CMOV mem dst", "CMOVLGT", []Operand{AX, Ptr(BX, 0, 4)}},
+		{"SET arity", "SETNE", []Operand{AL, BL}},
+		{"SET bad condition", "SETXX", []Operand{AL}},
+		{"SET bare", "SET", []Operand{AL}},
+		{"LZCNT arity", "LZCNTL", []Operand{AX}},
+		{"LZCNT mem dst", "LZCNTL", []Operand{AX, Ptr(BX, 0, 4)}},
+		{"MOVBLZX mem dst", "MOVBLZX", []Operand{AL, Ptr(BX, 0, 4)}},
+		{"CVTSL2SD gpr dst", "CVTSL2SD", []Operand{AX, BX}},
+	}
+	for _, c := range cases {
+		if _, err := Encode(c.mnem, c.ops...); err == nil {
+			t.Errorf("%s: expected an error, got none", c.name)
+		}
+	}
+}

asm/instrs.go

@@ -52,9 +52,10 @@ func (e *enc) encodeMov(ops []Operand, size int) error {
 	switch src := src.(type) {
 	case Reg:
 		if dstIsReg {
-			// MOV r, r/m: 0x8A/0x8B, reg=dst, rm=src.
+			// MOV r/m, r: 0x88/0x89, reg=src, rm=dst — the form the Go
-			i := newInstr(size, []byte{movRR(size)})
+			// assembler emits for register-to-register moves.
-			if err := setRM(i, dstReg, src, size); err != nil {
+			i := newInstr(size, []byte{movRM(size)})
+			if err := setRM(i, src, dst, size); err != nil {
 				return err
 			}
 			return e.emit(i)
@@ -136,9 +137,48 @@ func (e *enc) encodeALU(op struct {
 		return e.encodeALUImm(op.digit, dst, int64(imm), size)
 	}
 
+	// CMP accepts the immediate in the second position too — CMPL CX, $31 is
+	// the form the Go assembler itself accepts — and encodes it identically
+	// (CMP r/m, imm sets the flags as first − second).  No other ALU op takes
+	// an immediate destination.
+	if imm, ok := dst.(Imm); ok {
+		if op.digit != 7 {
+			return fmt.Errorf("immediate must be the source operand")
+		}
+		return e.encodeALUImm(op.digit, src, int64(imm), size)
+	}
+
+	// CMP records first − second without writing anywhere, so the first
+	// operand must land as the minuend; every other ALU op writes its second
+	// operand and follows the forms below.
+	cmp := op.rr == 0x39
 	dstReg, dstIsReg := dst.(Reg)
 	srcReg, srcIsReg := src.(Reg)
 	switch {
+	case cmp && dstIsReg:
+		// CMP x, reg: OP r/m, r (0x38/0x39) with rm = first operand, reg =
+		// second, matching the Go assembler.
+		opc := op.rr
+		if size == 1 {
+			opc = op.rr - 1
+		}
+		i := newInstr(size, []byte{opc})
+		if err := setRM(i, dstReg, src, size); err != nil {
+			return err
+		}
+		return e.emit(i)
+	case cmp && srcIsReg:
+		// CMP reg, mem: OP r, r/m (0x3A/0x3B) with reg = first operand, rm =
+		// second.
+		opc := op.rr + 2
+		if size == 1 {
+			opc = op.rr + 1
+		}
+		i := newInstr(size, []byte{opc})
+		if err := setRM(i, srcReg, dst, size); err != nil {
+			return err
+		}
+		return e.emit(i)
 	case srcIsReg:
 		// OP r/m, r: reg=src, rm=dst (dst is a register or memory).  This is the
 		// form the Go assembler prefers when the source is a register.
@@ -486,3 +526,152 @@ func immediate(v int64, size int, full64 bool) []byte {
 		return le32(v) // sign-extended imm32
 	}
 }
+
+// --- CMOVcc / SETcc ---------------------------------------------------------
+
+// encodeCmov encodes a conditional move: CMOV + size (W/L/Q) + condition
+// (CMOVLGT, CMOVQEQ, …).  The condition reads exactly like the Jcc spellings;
+// the instruction is 0F 40+cc with reg = dst, rm = src.
+func (e *enc) encodeCmov(upper string, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("CMOVcc expects 2 operands, got %d", len(ops))
+	}
+	rest := upper[len("CMOV"):]
+	if len(rest) < 2 {
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	var size int
+	switch rest[0] {
+	case 'W':
+		size = 2
+	case 'L':
+		size = 4
+	case 'Q':
+		size = 8
+	default:
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	cc, ok := jccMap[rest[1:]]
+	if !ok {
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	src, dst := ops[0], ops[1]
+	dstReg, ok := dst.(Reg)
+	if !ok {
+		return fmt.Errorf("CMOVcc destination must be a register")
+	}
+	i := newInstr(size, []byte{0x0F, byte(0x40 + cc)})
+	if err := setRM(i, dstReg, src, size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// encodeSet encodes a conditional byte set: SET + condition (SETNE, SETEQ, …),
+// always a byte write — 0F 90+cc /0 into a register or memory operand.
+func (e *enc) encodeSet(upper string, ops []Operand) error {
+	if len(ops) != 1 {
+		return fmt.Errorf("SETcc expects 1 operand, got %d", len(ops))
+	}
+	cond := upper[len("SET"):]
+	cc, ok := jccMap[cond]
+	if !ok || cond == "" {
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	i := &instr{opcode: []byte{0x0F, byte(0x90 + cc)}, modrm: -1, sib: -1}
+	if err := setRMDigit(i, 0, ops[0], 1); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// --- LZCNT / TZCNT ----------------------------------------------------------
+
+// encodeCount encodes LZCNT/TZCNT (leading / trailing zero count): F3 0F BD
+// or F3 0F BC, with reg = dst and rm = src.  The size suffix selects the
+// operand width (LZCNTW/LZCNTL/LZCNTQ).
+func (e *enc) encodeCount(base string, ops []Operand, size int) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("%s expects 2 operands, got %d", base, len(ops))
+	}
+	op := byte(0xBD)
+	if base == "TZCNT" {
+		op = 0xBC
+	}
+	dstReg, ok := ops[1].(Reg)
+	if !ok {
+		return fmt.Errorf("%s destination must be a register", base)
+	}
+	i := newInstr(size, []byte{0x0F, op})
+	i.prefix = 0xF3
+	if err := setRM(i, dstReg, ops[0], size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// --- mixed-width sign/zero-extending moves -----------------------------------
+
+// movExtendOp maps Go's mixed-width move names to their opcode and destination
+// width.  The source is narrower than the destination, so the plain size-suffix
+// convention does not apply to these names.
+var movExtendOp = map[string]struct {
+	op    []byte
+	dst64 bool
+}{
+	"MOVBLZX": {[]byte{0x0F, 0xB6}, false}, // byte → long, zero-extend
+	"MOVBQZX": {[]byte{0x0F, 0xB6}, true},  // byte → quad, zero-extend
+	"MOVWLZX": {[]byte{0x0F, 0xB7}, false}, // word → long, zero-extend
+	"MOVWQZX": {[]byte{0x0F, 0xB7}, true},  // word → quad, zero-extend
+	"MOVWLSX": {[]byte{0x0F, 0xBF}, false}, // word → long, sign-extend
+	"MOVLQSX": {[]byte{0x63}, true},        // long → quad, sign-extend (MOVSXD)
+}
+
+// encodeMovExtend encodes a mixed-width extending move: reg = dst (the wider
+// operand), rm = src.
+func (e *enc) encodeMovExtend(base string, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("%s expects 2 operands, got %d", base, len(ops))
+	}
+	spec := movExtendOp[base]
+	dstReg, ok := ops[1].(Reg)
+	if !ok {
+		return fmt.Errorf("%s destination must be a register", base)
+	}
+	size := 4
+	if spec.dst64 {
+		size = 8
+	}
+	i := newInstr(size, spec.op)
+	if err := setRM(i, dstReg, ops[0], size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// --- CVTSL2SD / CVTSQ2SD -----------------------------------------------------
+
+// encodeCvtsi2sd encodes a signed integer to scalar double conversion
+// (CVTSL2SD from a 32-bit, CVTSQ2SD from a 64-bit source): F2 0F 2A with
+// reg = XMM dst, rm = GPR/memory src.  The Go assembler emits the legacy SSE
+// encoding here, not the VEX form, so we match it byte for byte.
+func (e *enc) encodeCvtsi2sd(quad bool, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("CVTSx2SD expects 2 operands, got %d", len(ops))
+	}
+	src, dst := ops[0], ops[1]
+	dstReg, ok := dst.(Reg)
+	if !ok || !dstReg.isVec() {
+		return fmt.Errorf("CVTSx2SD destination must be a vector register")
+	}
+	size := 4
+	if quad {
+		size = 8
+	}
+	i := newInstr(size, []byte{0x0F, 0x2A})
+	i.prefix = 0xF2
+	if err := setRM(i, dstReg, src, size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}

asm/vex.go

@@ -7,6 +7,10 @@ import "fmt"
 
 // This file implements VEX (AVX/AVX2) instruction encoding.  EVEX (AVX-512)
 // support is a later increment.
+//
+// Every encoding choice here is validated two ways in the tests: by
+// round-trip decoding through golang.org/x/arch's x86 decoder, and by
+// byte-for-byte comparison against the output of the real Go assembler.
 
 // vexForm selects how an instruction's operands map onto the VEX.vvvv,
 // ModRM.reg and ModRM.rm fields.
@@ -17,11 +21,26 @@ const (
 	// ModRM.reg = dst (op2), VEX.vvvv = src1 (op1), ModRM.rm = src2 (op0).
 	vexNDS3 vexForm = iota
 	// vexRM is the two-operand form `OP src, dst` with no vvvv source:
-	// ModRM.reg = dst (op1), ModRM.rm = src (op0), VEX.vvvv = 1111 (unused).
+	// ModRM.reg = dst (op1), ModRM.rm = src (op0), VEX.vvvv unused.
 	vexRM
 	// vexShiftImm is the immediate-shift form `OP $imm, src, dst`: ModRM.reg =
 	// /digit, ModRM.rm = src (op1), VEX.vvvv = dst (op2), imm8 = op0.
 	vexShiftImm
+	// vexImmRM is the immediate form `OP $imm, src, dst` with no vvvv source:
+	// ModRM.reg = dst (op2), ModRM.rm = src (op1), imm8 = op0.  VPSHUFD and
+	// VPERMQ use this shape.
+	vexImmRM
+	// vexNDS3Imm is the three-operand plus immediate form `OP $imm, src2,
+	// src1, dst`: ModRM.reg = dst, VEX.vvvv = src1, ModRM.rm = src2, imm8.
+	// VSHUFPD, VPERM2I128 and VINSERTI128 use this shape.
+	vexNDS3Imm
+	// vexExtract is the lane-extract form `OP $imm, ysrc, xdst`: ModRM.reg =
+	// ysrc (op1), ModRM.rm = xdst or memory (op2), imm8 = op0.  The YMM
+	// source lives in the reg field, the destination in r/m — the PEXTR-style
+	// layout.  VEXTRACTI128 and VEXTRACTF128 use this shape.
+	vexExtract
+	// vexZero is the no-operand form (VZEROUPPER).
+	vexZero
 )
 
 // vexSpec describes one VEX instruction's encoding parameters.
@@ -34,9 +53,9 @@ type vexSpec struct {
 	form    vexForm
 }
 
-// vexTable maps an upper-case mnemonic to its VEX encoding.  It covers the
+// vexTable maps an upper-case mnemonic to its VEX encoding.  It is extended
-// AVX2 instructions used by the go-flac kernels in the three-operand NDS form;
+// incrementally; every entry is covered by a byte-for-byte ground-truth test
-// it is extended incrementally.
+// against the Go assembler.
 var vexTable = map[string]vexSpec{
 	// VEX.128/256.66.0F.WIG — integer arithmetic / logic / compare.
 	"VPADDD":      {1, 0xFE, 0, 1, -1, vexNDS3},
@@ -52,12 +71,26 @@ var vexTable = map[string]vexSpec{
 	"VPUNPCKHDQ":  {1, 0x6A, 0, 1, -1, vexNDS3},
 	"VPUNPCKLQDQ": {1, 0x6C, 0, 1, -1, vexNDS3},
 	"VPACKSSDW":   {1, 0x6B, 0, 1, -1, vexNDS3},
+	// VEX.256.66.0F38.W0 — dword permute (three-operand NDS form).
+	"VPERMD": {2, 0x36, 0, 1, -1, vexNDS3},
 	// VEX.128/256.66.0F38.WIG.
 	"VPMULLD":  {2, 0x40, 0, 1, -1, vexNDS3},
 	"VPMULDQ":  {2, 0x28, 0, 1, -1, vexNDS3},
 	"VPSHUFB":  {2, 0x00, 0, 1, -1, vexNDS3},
 	"VPCMPGTQ": {2, 0x37, 0, 1, -1, vexNDS3},
 
+	// VEX.128/256.66.0F.WIG — packed double-precision arithmetic / logic.
+	"VADDPD":    {1, 0x58, 0, 1, -1, vexNDS3},
+	"VMULPD":    {1, 0x59, 0, 1, -1, vexNDS3},
+	"VXORPD":    {1, 0x57, 0, 1, -1, vexNDS3},
+	"VUNPCKHPD": {1, 0x15, 0, 1, -1, vexNDS3},
+	// VEX.128.F2.0F.WIG — scalar double-precision arithmetic (the packed
+	// opcodes with an F2 pp).
+	"VADDSD": {1, 0x58, 0, 3, -1, vexNDS3},
+	"VMULSD": {1, 0x59, 0, 3, -1, vexNDS3},
+	// VEX.128/256.66.0F38.W1 — fused multiply-add (NDS form).
+	"VFMADD231PD": {2, 0xB8, 1, 1, -1, vexNDS3},
+
 	// VEX.128/256.66.0F38.WIG — sign/zero extend and broadcast (reg=dst, rm=src,
 	// no vvvv).
 	"VPMOVSXWD":    {2, 0x23, 0, 1, -1, vexRM},
@@ -65,6 +98,9 @@ var vexTable = map[string]vexSpec{
 	"VPMOVZXDQ":    {2, 0x35, 0, 1, -1, vexRM},
 	"VPBROADCASTD": {2, 0x58, 0, 1, -1, vexRM},
 	"VPBROADCASTQ": {2, 0x59, 0, 1, -1, vexRM},
+	// VEX.128/256.F3.0F.WIG — signed dword to packed double conversion
+	// (reg=dst, rm=src, no vvvv; the length follows the destination).
+	"VCVTDQ2PD": {1, 0xE6, 0, 2, -1, vexRM},
 	// VEX.128/256.66.0F.WIG — move mask to a GPR (reg=gpr dst, rm=vec src).
 	"VPMOVMSKB": {1, 0xD7, 0, 1, -1, vexRM},
 	"VMOVMSKPS": {1, 0x50, 0, 0, -1, vexRM}, // no 66 prefix (that would be VMOVMSKPD)
@@ -75,16 +111,97 @@ var vexTable = map[string]vexSpec{
 	"VPSRLD": {1, 0x72, 0, 1, 2, vexShiftImm},
 	"VPSRLQ": {1, 0x73, 0, 1, 2, vexShiftImm},
 	"VPSLLQ": {1, 0x73, 0, 1, 6, vexShiftImm},
+
+	// VEX.128/256.66.0F.WIG — immediate shuffle (reg=dst, rm=src, imm8).
+	"VPSHUFD": {1, 0x70, 0, 1, -1, vexImmRM},
+	// VEX.256.66.0F3A.W1 — qword permute (reg=dst, rm=src, imm8).
+	"VPERMQ": {3, 0x00, 1, 1, -1, vexImmRM},
+
+	// VEX.128/256.66.0F.WIG — two-source shuffle (reg=dst, vvvv=src1, rm=src2,
+	// imm8).
+	"VSHUFPD": {1, 0xC6, 0, 1, -1, vexNDS3Imm},
+	// VEX.256.66.0F3A.W0 — permute / insert (same shape; VINSERTI128's rm is
+	// the XMM or memory source).
+	"VPERM2I128":  {3, 0x46, 0, 1, -1, vexNDS3Imm},
+	"VINSERTI128": {3, 0x38, 0, 1, -1, vexNDS3Imm},
+
+	// VEX.256.66.0F3A.W0 — lane extract (reg=YMM src, rm=XMM/memory dst, imm8).
+	"VEXTRACTI128": {3, 0x39, 0, 1, -1, vexExtract},
+	"VEXTRACTF128": {3, 0x19, 0, 1, -1, vexExtract},
+
+	// VEX.128.0F.W0 — no operands.
+	"VZEROUPPER": {1, 0x77, 0, 0, -1, vexZero},
+}
+
+// vexVarShift maps the shift mnemonics to their variable-count opcode — the
+// form whose count comes from an XMM register or memory (VPSRLQ X0, Y8, Y8),
+// an ordinary NDS encoding rather than the /digit immediate form above.
+var vexVarShift = map[string]byte{
+	"VPSLLD": 0xF2,
+	"VPSLLQ": 0xF3,
+	"VPSRAD": 0xE2,
+	"VPSRLD": 0xD2,
+	"VPSRLQ": 0xD3,
+}
+
+// vexMoveSpec describes a VEX move, which takes different opcodes (and
+// sometimes a different VEX.W) per operand direction.  The Go assembler
+// encodes a vector→vector move with the store-form opcode (reg = source,
+// rm = destination), so regReg defaults to store when zero.
+type vexMoveSpec struct {
+	mapSel  int
+	pp      int
+	load    byte // r/m → vector: reg=dst, rm=src
+	store   byte // vector → r/m: reg=src, rm=dst
+	loadW   int
+	storeW  int
+	regReg  byte // vector → vector opcode; 0 uses store
+	regW    int
+	vecOK   bool // the non-fixed operand may be a vector register
+	gprOK   bool // the non-fixed operand may be a general-purpose register
+	xmmOnly bool // YMM registers are rejected
+}
+
+// vexMoveTable maps an upper-case move mnemonic to its encoding.
+var vexMoveTable = map[string]vexMoveSpec{
+	// VEX.128/256.F3.0F.WIG — unaligned integer move.
+	"VMOVDQU": {1, 2, 0x6F, 0x7F, 0, 0, 0, 0, true, false, false},
+	// VEX.128/256.66.0F.WIG — unaligned packed double move.
+	"VMOVUPD": {1, 1, 0x10, 0x11, 0, 0, 0, 0, true, false, false},
+	// VEX.128.66.0F.W0 — 32-bit GPR/memory ↔ XMM.
+	"VMOVD": {1, 1, 0x6E, 0x7E, 0, 0, 0, 0, false, true, true},
+	// VMOVQ — 66 6E W1 (r/m→xmm), 66 7E W1 (xmm→r/m), 66 D6 W0 (xmm→xmm).
+	"VMOVQ": {1, 1, 0x6E, 0x7E, 1, 1, 0xD6, 0, true, true, true},
+	// VEX.128.F2.0F.WIG — scalar double move, memory operands only (the
+	// register form takes three operands and is not supported yet).
+	"VMOVSD": {1, 3, 0x10, 0x11, 0, 0, 0, 0, false, false, true},
 }
 
 // isVex reports whether the mnemonic is a VEX-encoded instruction we handle.
 func isVex(mnemUpper string) bool {
-	_, ok := vexTable[mnemUpper]
+	if _, ok := vexTable[mnemUpper]; ok {
+		return true
+	}
+	_, ok := vexMoveTable[mnemUpper]
 	return ok
 }
 
 // encodeVex encodes a VEX instruction with operands in Plan 9 order.
 func (e *enc) encodeVex(mnemUpper string, ops []Operand) error {
+	if ms, ok := vexMoveTable[mnemUpper]; ok {
+		return e.encodeVexMove(mnemUpper, ms, ops)
+	}
+	// The shifts come in two shapes under one mnemonic: an immediate count
+	// ($imm, src, dst) and a variable count in an XMM register or memory
+	// (count, src, dst), the latter an ordinary NDS form.
+	if op, ok := vexVarShift[mnemUpper]; ok && len(ops) == 3 {
+		if _, isImm := ops[0].(Imm); !isImm {
+			if !vecOrMem(ops[0]) {
+				return fmt.Errorf("%s: shift count must be an immediate, a vector register or memory", mnemUpper)
+			}
+			return e.encodeVexNDS3(vexSpec{mapSel: 1, opcode: op, pp: 1, opdigit: -1, form: vexNDS3}, ops)
+		}
+	}
 	spec := vexTable[mnemUpper]
 	switch spec.form {
 	case vexNDS3:
@@ -93,6 +210,14 @@ func (e *enc) encodeVex(mnemUpper string, ops []Operand) error {
 		return e.encodeVexRM(spec, ops)
 	case vexShiftImm:
 		return e.encodeVexShiftImm(spec, ops)
+	case vexImmRM:
+		return e.encodeVexImmRM(spec, ops)
+	case vexNDS3Imm:
+		return e.encodeVexNDS3Imm(spec, ops)
+	case vexExtract:
+		return e.encodeVexExtract(spec, ops)
+	case vexZero:
+		return e.encodeVexZero(mnemUpper, spec, ops)
 	}
 	return fmt.Errorf("unhandled VEX form for %s", mnemUpper)
 }
@@ -151,7 +276,9 @@ func (e *enc) encodeVexRM(spec vexSpec, ops []Operand) error {
 		l = srcReg.vecLenBit()
 	}
 
-	return e.emitVexFields(spec, l, regField, rBit, 0, src) // vvvv unused → vvvvBar=0
+	// An unused vvvv field must be stored as all ones (v̄vvv = 1111); the
+	// hardware raises #UD on any other value.
+	return e.emitVexFields(spec, l, regField, rBit, 15, src)
 }
 
 // encodeVexShiftImm encodes an immediate-shift instruction: OP $imm, src, dst.
@@ -176,27 +303,232 @@ func (e *enc) encodeVexShiftImm(spec vexSpec, ops []Operand) error {
 	}
 
 	vvvvBar := 15 - (dstReg.idx & 15)
-	l := dstReg.vecLenBit()
+	if err := e.emitVexFields(spec, dstReg.vecLenBit(), spec.opdigit, 0, vvvvBar, srcReg); err != nil {
-	rmField := srcReg.idx & 7
+		return err
-	bBit := 0
-	if srcReg.idx >= 8 {
-		bBit = 1
 	}
-	modrm := 0xC0 | spec.opdigit<<3 | rmField
+	immByte, err := imm8(int64(immVal))
-
+	if err != nil {
-	if spec.mapSel == 1 && bBit == 0 && spec.w == 0 {
+		return err
-		e.out = append(e.out, 0xC5, byte(1<<7|vvvvBar<<3|l<<2|spec.pp))
-	} else {
-		e.out = append(e.out, 0xC4,
-			byte(1<<7|1<<6|(1-bBit)<<5|spec.mapSel),
-			byte(spec.w<<7|vvvvBar<<3|l<<2|spec.pp))
 	}
-	e.out = append(e.out, spec.opcode, byte(modrm), byte(int8(immVal)))
+	e.out = append(e.out, immByte)
 	return nil
 }
 
+// imm8 range-checks an immediate for an 8-bit field.  Shuffle controls are
+// unsigned bit masks, but the negative spelling ($-1 = all bits set) is
+// accepted, so the accepted span is -128..255.
+func imm8(v int64) (byte, error) {
+	if v < -128 || v > 255 {
+		return 0, fmt.Errorf("immediate $%d does not fit in 8 bits", v)
+	}
+	return byte(v), nil
+}
+
+// encodeVexImmRM encodes an immediate form with no vvvv source: OP $imm, src,
+// dst (VPSHUFD, VPERMQ).  ModRM.reg = dst, ModRM.rm = src, imm8 appended.
+func (e *enc) encodeVexImmRM(spec vexSpec, ops []Operand) error {
+	if len(ops) != 3 {
+		return fmt.Errorf("shuffle expects 3 operands ($imm, src, dst), got %d", len(ops))
+	}
+	imm, src, dst := ops[0], ops[1], ops[2]
+	immVal, ok := imm.(Imm)
+	if !ok {
+		return fmt.Errorf("shuffle control must be an immediate")
+	}
+	dstReg, ok := dst.(Reg)
+	if !ok || !dstReg.isVec() {
+		return fmt.Errorf("shuffle destination must be a vector register")
+	}
+
+	// The vector length follows the source when it is a vector register,
+	// otherwise the destination (a memory source carries no length).
+	l := dstReg.vecLenBit()
+	if srcReg, ok := src.(Reg); ok && srcReg.isVec() {
+		l = srcReg.vecLenBit()
+	}
+	regField := dstReg.idx & 7
+	rBit := 0
+	if dstReg.idx >= 8 {
+		rBit = 1
+	}
+	if err := e.emitVexFields(spec, l, regField, rBit, 15, src); err != nil {
+		return err
+	}
+	immByte, err := imm8(int64(immVal))
+	if err != nil {
+		return err
+	}
+	e.out = append(e.out, immByte)
+	return nil
+}
+
+// encodeVexNDS3Imm encodes the three-operand plus immediate form: OP $imm,
+// src2, src1, dst (VSHUFPD, VPERM2I128, VINSERTI128).  ModRM.reg = dst,
+// VEX.vvvv = src1, ModRM.rm = src2, imm8 appended.
+func (e *enc) encodeVexNDS3Imm(spec vexSpec, ops []Operand) error {
+	if len(ops) != 4 {
+		return fmt.Errorf("instruction expects 4 operands ($imm, src2, src1, dst), got %d", len(ops))
+	}
+	imm, src2, src1, dst := ops[0], ops[1], ops[2], ops[3]
+	immVal, ok := imm.(Imm)
+	if !ok {
+		return fmt.Errorf("shuffle control must be an immediate")
+	}
+	dstReg, ok := dst.(Reg)
+	if !ok || !dstReg.isVec() {
+		return fmt.Errorf("destination must be a vector register")
+	}
+	vvvvReg, ok := src1.(Reg)
+	if !ok || !vvvvReg.isVec() {
+		return fmt.Errorf("second source must be a vector register")
+	}
+
+	regField := dstReg.idx & 7
+	rBit := 0
+	if dstReg.idx >= 8 {
+		rBit = 1
+	}
+	vvvvBar := 15 - (vvvvReg.idx & 15)
+	if err := e.emitVexFields(spec, dstReg.vecLenBit(), regField, rBit, vvvvBar, src2); err != nil {
+		return err
+	}
+	immByte, err := imm8(int64(immVal))
+	if err != nil {
+		return err
+	}
+	e.out = append(e.out, immByte)
+	return nil
+}
+
+// encodeVexExtract encodes a lane extract: OP $imm, ysrc, xdst
+// (VEXTRACTI128, VEXTRACTF128).  The YMM source occupies ModRM.reg and the
+// XMM (or memory) destination ModRM.rm; imm8 selects the lane.
+func (e *enc) encodeVexExtract(spec vexSpec, ops []Operand) error {
+	if len(ops) != 3 {
+		return fmt.Errorf("extract expects 3 operands ($imm, ysrc, xdst), got %d", len(ops))
+	}
+	imm, src, dst := ops[0], ops[1], ops[2]
+	immVal, ok := imm.(Imm)
+	if !ok {
+		return fmt.Errorf("extract lane must be an immediate")
+	}
+	srcReg, ok := src.(Reg)
+	if !ok || !srcReg.isVec() {
+		return fmt.Errorf("extract source must be a vector register")
+	}
+
+	regField := srcReg.idx & 7
+	rBit := 0
+	if srcReg.idx >= 8 {
+		rBit = 1
+	}
+	if err := e.emitVexFields(spec, srcReg.vecLenBit(), regField, rBit, 15, dst); err != nil {
+		return err
+	}
+	immByte, err := imm8(int64(immVal))
+	if err != nil {
+		return err
+	}
+	e.out = append(e.out, immByte)
+	return nil
+}
+
+// encodeVexZero encodes a no-operand instruction (VZEROUPPER).
+func (e *enc) encodeVexZero(mnem string, spec vexSpec, ops []Operand) error {
+	if len(ops) != 0 {
+		return fmt.Errorf("%s expects no operands, got %d", mnem, len(ops))
+	}
+	// 2-byte VEX: R̄ = 1, v̄vvv = 1111 (unused), L = 0.
+	e.out = append(e.out, 0xC5, byte(1<<7|15<<3|spec.pp), spec.opcode)
+	return nil
+}
+
+// encodeVexMove encodes a two-operand move (VMOVDQU, VMOVUPD, VMOVD, VMOVQ,
+// VMOVSD), picking the direction-specific opcode and VEX.W.  A vector→vector
+// move uses the store-form layout (reg = source, rm = destination), matching
+// the Go assembler.
+func (e *enc) encodeVexMove(mnem string, ms vexMoveSpec, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("VEX move expects 2 operands, got %d", len(ops))
+	}
+	src, dst := ops[0], ops[1]
+	srcReg, srcIsVec := vecReg(src)
+	dstReg, dstIsVec := vecReg(dst)
+
+	var reg Reg
+	var rm Operand
+	op, w := ms.store, ms.storeW
+	switch {
+	case srcIsVec && dstIsVec:
+		if !ms.vecOK {
+			return fmt.Errorf("%s does not take two vector registers", mnem)
+		}
+		if ms.xmmOnly && (srcReg.size == 32 || dstReg.size == 32) {
+			return fmt.Errorf("%s operates on XMM registers only", mnem)
+		}
+		if ms.regReg != 0 {
+			op, w = ms.regReg, ms.regW
+		}
+		reg, rm = srcReg, dst // store form: reg = source, rm = destination.
+	case srcIsVec:
+		// vector → memory, or → GPR (VMOVD/VMOVQ only).
+		if !validMoveOther(ms, dst) {
+			return fmt.Errorf("%s: invalid destination operand", mnem)
+		}
+		reg, rm = srcReg, dst
+	case dstIsVec:
+		// memory → vector, or GPR → vector (VMOVD/VMOVQ only).
+		if !validMoveOther(ms, src) {
+			return fmt.Errorf("%s: invalid source operand", mnem)
+		}
+		op, w = ms.load, ms.loadW
+		reg, rm = dstReg, src
+	default:
+		return fmt.Errorf("%s needs a vector register operand", mnem)
+	}
+	if ms.xmmOnly && reg.size == 32 {
+		return fmt.Errorf("%s operates on XMM registers only", mnem)
+	}
+
+	regField := reg.idx & 7
+	rBit := 0
+	if reg.idx >= 8 {
+		rBit = 1
+	}
+	spec := vexSpec{mapSel: ms.mapSel, opcode: op, w: w, pp: ms.pp, opdigit: -1}
+	return e.emitVexFields(spec, reg.vecLenBit(), regField, rBit, 15, rm)
+}
+
+// vecReg extracts a vector register from an operand.
+func vecReg(op Operand) (Reg, bool) {
+	r, ok := op.(Reg)
+	return r, ok && r.isVec()
+}
+
+// vecOrMem reports whether op is a vector register or a memory reference.
+func vecOrMem(op Operand) bool {
+	if _, ok := op.(Mem); ok {
+		return true
+	}
+	r, ok := op.(Reg)
+	return ok && r.isVec()
+}
+
+// validMoveOther reports whether the non-vector operand of a move is
+// acceptable: memory always is, a GPR only for VMOVD/VMOVQ.
+func validMoveOther(ms vexMoveSpec, op Operand) bool {
+	switch o := op.(type) {
+	case Mem:
+		return true
+	case Reg:
+		return ms.gprOK && !o.isVec()
+	}
+	return false
+}
+
 // emitVexFields emits the VEX prefix, opcode, ModR/M, SIB and displacement for
-// the given precomputed fields.  It is shared by the NDS and RM forms.
+// the given precomputed fields.  It is shared by every register/rm VEX form;
+// immediate bytes are appended by the caller.
 func (e *enc) emitVexFields(spec vexSpec, l, regField, rBit, vvvvBar int, rm Operand) error {
 	var modrm, sib int
 	var disp []byte

asm/vex_test.go

@@ -4,6 +4,7 @@
 package asm
 
 import (
+	"strings"
 	"testing"
 
 	"golang.org/x/arch/x86/x86asm"
@@ -26,7 +27,12 @@ func TestVexNDS3(t *testing.T) {
 		if spec.form != vexNDS3 {
 			continue
 		}
-		code, err := Encode(mnem, vreg(t, "Y0"), vreg(t, "Y1"), vreg(t, "Y2"))
+		// Scalar (F2/F3 pp) instructions exist only in the 128-bit form.
+		vec := "Y"
+		if spec.pp >= 2 {
+			vec = "X"
+		}
+		code, err := Encode(mnem, vreg(t, vec+"0"), vreg(t, vec+"1"), vreg(t, vec+"2"))
 		if err != nil {
 			t.Errorf("%s: Encode: %v", mnem, err)
 			continue
@@ -72,7 +78,7 @@ func TestVexXMM(t *testing.T) {
 	if inst.Op != x86asm.VPXOR {
 		t.Fatalf("decoded %s, want VPXOR", inst.Op)
 	}
-	// vpxor xmm7, xmm7, xmm7 → C5 C9 EF FF (2-byte VEX, L=0).
+	// vpxor xmm7, xmm7, xmm7 → C5 C1 EF FF (2-byte VEX, L=0).
 	if code[0] != 0xC5 {
 		t.Errorf("expected 2-byte VEX (C5), got % x", code)
 	}
@@ -140,3 +146,187 @@ func TestVexShiftImm(t *testing.T) {
 		t.Fatalf("VPSRAD decoded %v (err %v), want VPSRAD", inst.Op, err)
 	}
 }
+
+// TestVexGroundTruth checks byte-for-byte agreement with the real Go
+// assembler.  The expected bytes were extracted from the machine code the Go
+// toolchain produced for exactly these instructions (go build + a .text
+// section dump of the resulting binary), never from a disassembler's
+// rendering.  This locks the v̄vvv = 1111 rule for unused vvvv fields (a
+// value the hardware rejects with #UD and the x86 decoder silently ignores)
+// as well as every new operand form.
+func TestVexGroundTruth(t *testing.T) {
+	cases := []struct {
+		name string
+		mnem string
+		ops  []Operand
+		want string
+	}{
+		// Three-operand NDS form.
+		{"VPADDQ Y8,Y9,Y8", "VPADDQ", []Operand{vreg(t, "Y8"), vreg(t, "Y9"), vreg(t, "Y8")}, "c44135d4c0"},
+		{"VPADDQ X9,X8,X8", "VPADDQ", []Operand{vreg(t, "X9"), vreg(t, "X8"), vreg(t, "X8")}, "c44139d4c1"},
+		{"VPXOR X7,X7,X7", "VPXOR", []Operand{vreg(t, "X7"), vreg(t, "X7"), vreg(t, "X7")}, "c5c1efff"},
+		{"VPSHUFB Y1,Y2,Y3", "VPSHUFB", []Operand{vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3")}, "c4e26d00d9"},
+		{"VPMULLD Y1,Y2,Y3", "VPMULLD", []Operand{vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3")}, "c4e26d40d9"},
+		{"VPUNPCKLDQ Y4,Y3,Y5", "VPUNPCKLDQ", []Operand{vreg(t, "Y4"), vreg(t, "Y3"), vreg(t, "Y5")}, "c5e562ec"},
+		{"VPERMD Y1,Y2,Y3", "VPERMD", []Operand{vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3")}, "c4e26d36d9"},
+		// Floating point (packed and scalar) and FMA — same NDS form, the pp
+		// bits and map select the operation.
+		{"VADDPD Y9,Y8,Y8", "VADDPD", []Operand{vreg(t, "Y9"), vreg(t, "Y8"), vreg(t, "Y8")}, "c4413d58c1"},
+		{"VADDPD X1,X2,X3", "VADDPD", []Operand{vreg(t, "X1"), vreg(t, "X2"), vreg(t, "X3")}, "c5e958d9"},
+		{"VMULPD Y12,Y12,Y12", "VMULPD", []Operand{vreg(t, "Y12"), vreg(t, "Y12"), vreg(t, "Y12")}, "c4411d59e4"},
+		{"VXORPD Y8,Y8,Y8", "VXORPD", []Operand{vreg(t, "Y8"), vreg(t, "Y8"), vreg(t, "Y8")}, "c4413d57c0"},
+		{"VUNPCKHPD X8,X8,X9", "VUNPCKHPD", []Operand{vreg(t, "X8"), vreg(t, "X8"), vreg(t, "X9")}, "c4413915c8"},
+		{"VADDSD X9,X8,X8", "VADDSD", []Operand{vreg(t, "X9"), vreg(t, "X8"), vreg(t, "X8")}, "c4413b58c1"},
+		{"VMULSD X0,X1,X1", "VMULSD", []Operand{vreg(t, "X0"), vreg(t, "X1"), vreg(t, "X1")}, "c5f359c8"},
+		{"VFMADD231PD Y14,Y12,Y8", "VFMADD231PD", []Operand{vreg(t, "Y14"), vreg(t, "Y12"), vreg(t, "Y8")}, "c4429db8c6"},
+		{"VFMADD231PD (DI),Y12,Y8", "VFMADD231PD", []Operand{Ptr(DI, 0, 32), vreg(t, "Y12"), vreg(t, "Y8")}, "c4629db807"},
+		// Two-operand reg/rm form (v̄vvv must be 1111).
+		{"VPMOVSXDQ X0,Y4", "VPMOVSXDQ", []Operand{vreg(t, "X0"), vreg(t, "Y4")}, "c4e27d25e0"},
+		{"VPMOVSXWD (SI),Y0", "VPMOVSXWD", []Operand{Ptr(SI, 0, 8), vreg(t, "Y0")}, "c4e27d2306"},
+		{"VPBROADCASTD X0,Y15", "VPBROADCASTD", []Operand{vreg(t, "X0"), vreg(t, "Y15")}, "c4627d58f8"},
+		{"VCVTDQ2PD X12,Y12", "VCVTDQ2PD", []Operand{vreg(t, "X12"), vreg(t, "Y12")}, "c4417ee6e4"},
+		{"VCVTDQ2PD (SI),Y4", "VCVTDQ2PD", []Operand{Ptr(SI, 0, 16), vreg(t, "Y4")}, "c5fee626"},
+		{"VPMOVMSKB X11,AX", "VPMOVMSKB", []Operand{vreg(t, "X11"), AX}, "c4c179d7c3"},
+		{"VMOVMSKPS Y7,AX", "VMOVMSKPS", []Operand{vreg(t, "Y7"), AX}, "c5fc50c7"},
+		// Immediate shifts.
+		{"VPSLLD $1,Y3,Y4", "VPSLLD", []Operand{Imm(1), vreg(t, "Y3"), vreg(t, "Y4")}, "c5dd72f301"},
+		{"VPSRLQ $2,Y5,Y6", "VPSRLQ", []Operand{Imm(2), vreg(t, "Y5"), vreg(t, "Y6")}, "c5cd73d502"},
+		// Variable-count shifts: the count lives in an XMM register or memory
+		// and the instruction takes the NDS form.
+		{"VPSRLQ X0,Y8,Y8", "VPSRLQ", []Operand{vreg(t, "X0"), vreg(t, "Y8"), vreg(t, "Y8")}, "c53dd3c0"},
+		{"VPSRLQ (AX),Y8,Y8", "VPSRLQ", []Operand{Ptr(AX, 0, 16), vreg(t, "Y8"), vreg(t, "Y8")}, "c53dd300"},
+		{"VPSLLD X0,Y1,Y2", "VPSLLD", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5f2d0"},
+		{"VPSRLD X0,Y1,Y2", "VPSRLD", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5d2d0"},
+		{"VPSRAD X0,Y1,Y2", "VPSRAD", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5e2d0"},
+		{"VPSLLQ X0,Y1,Y2", "VPSLLQ", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5f3d0"},
+		// Immediate shuffle (reg=dst, rm=src, imm8).
+		{"VPSHUFD $0xEE,X8,X9", "VPSHUFD", []Operand{Imm(0xEE), vreg(t, "X8"), vreg(t, "X9")}, "c4417970c8ee"},
+		{"VPSHUFD $0xEE,Y1,Y2", "VPSHUFD", []Operand{Imm(0xEE), vreg(t, "Y1"), vreg(t, "Y2")}, "c5fd70d1ee"},
+		{"VPERMQ $0x1B,Y1,Y2", "VPERMQ", []Operand{Imm(0x1B), vreg(t, "Y1"), vreg(t, "Y2")}, "c4e3fd00d11b"},
+		{"VPERMQ $0x1B,Y11,Y12", "VPERMQ", []Operand{Imm(0x1B), vreg(t, "Y11"), vreg(t, "Y12")}, "c443fd00e31b"},
+		// Three-operand + immediate (reg=dst, vvvv=src1, rm=src2, imm8).
+		{"VSHUFPD $1,X1,X2,X3", "VSHUFPD", []Operand{Imm(1), vreg(t, "X1"), vreg(t, "X2"), vreg(t, "X3")}, "c5e9c6d901"},
+		{"VSHUFPD $1,Y1,Y2,Y3", "VSHUFPD", []Operand{Imm(1), vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3")}, "c5edc6d901"},
+		{"VPERM2I128 $0x31,Y1,Y2,Y3", "VPERM2I128", []Operand{Imm(0x31), vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3")}, "c4e36d46d931"},
+		{"VINSERTI128 $1,X5,Y1,Y2", "VINSERTI128", []Operand{Imm(1), vreg(t, "X5"), vreg(t, "Y1"), vreg(t, "Y2")}, "c4e37538d501"},
+		// Lane extract (reg=YMM source, rm=XMM/memory destination, imm8).
+		{"VEXTRACTI128 $1,Y8,X9", "VEXTRACTI128", []Operand{Imm(1), vreg(t, "Y8"), vreg(t, "X9")}, "c4437d39c101"},
+		{"VEXTRACTI128 $1,Y8,(DI)", "VEXTRACTI128", []Operand{Imm(1), vreg(t, "Y8"), Ptr(DI, 0, 16)}, "c4637d390701"},
+		{"VEXTRACTF128 $1,Y8,X9", "VEXTRACTF128", []Operand{Imm(1), vreg(t, "Y8"), vreg(t, "X9")}, "c4437d19c101"},
+		// Moves — each direction picks its own opcode and VEX.W.
+		{"VMOVDQU (SI),Y1", "VMOVDQU", []Operand{Ptr(SI, 0, 32), vreg(t, "Y1")}, "c5fe6f0e"},
+		{"VMOVDQU Y3,(DI)", "VMOVDQU", []Operand{vreg(t, "Y3"), Ptr(DI, 0, 32)}, "c5fe7f1f"},
+		{"VMOVDQU X1,X2", "VMOVDQU", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "c5fa7fca"},
+		{"VMOVUPD (DI),Y14", "VMOVUPD", []Operand{Ptr(DI, 0, 32), vreg(t, "Y14")}, "c57d1037"},
+		{"VMOVUPD Y14,(DI)", "VMOVUPD", []Operand{vreg(t, "Y14"), Ptr(DI, 0, 32)}, "c57d1137"},
+		{"VMOVUPD X1,X2", "VMOVUPD", []Operand{vreg(t, "X1"), vreg(t, "X2")}, "c5f911ca"},
+		{"VMOVQ X8,AX", "VMOVQ", []Operand{vreg(t, "X8"), AX}, "c461f97ec0"},
+		{"VMOVQ AX,X9", "VMOVQ", []Operand{AX, vreg(t, "X9")}, "c461f96ec8"},
+		{"VMOVQ X8,(DI)", "VMOVQ", []Operand{vreg(t, "X8"), Ptr(DI, 0, 8)}, "c461f97e07"},
+		{"VMOVQ (SI),X9", "VMOVQ", []Operand{Ptr(SI, 0, 8), vreg(t, "X9")}, "c461f96e0e"},
+		{"VMOVQ X8,X2", "VMOVQ", []Operand{vreg(t, "X8"), vreg(t, "X2")}, "c579d6c2"},
+		{"VMOVQ X2,X8", "VMOVQ", []Operand{vreg(t, "X2"), vreg(t, "X8")}, "c4c179d6d0"},
+		{"VMOVD X0,(SI)", "VMOVD", []Operand{vreg(t, "X0"), Ptr(SI, 0, 4)}, "c5f97e06"},
+		{"VMOVD AX,X0", "VMOVD", []Operand{AX, vreg(t, "X0")}, "c5f96ec0"},
+		{"VMOVSD (SI),X8", "VMOVSD", []Operand{Ptr(SI, 0, 8), vreg(t, "X8")}, "c57b1006"},
+		{"VMOVSD X8,(SI)", "VMOVSD", []Operand{vreg(t, "X8"), Ptr(SI, 0, 8)}, "c57b1106"},
+		// No-operand.
+		{"VZEROUPPER", "VZEROUPPER", nil, "c5f877"},
+	}
+	for _, c := range cases {
+		code, err := Encode(c.mnem, c.ops...)
+		if err != nil {
+			t.Errorf("%s: Encode: %v", c.name, err)
+			continue
+		}
+		if got := strings.ReplaceAll(hexBytes(code), " ", ""); got != c.want {
+			t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
+			continue
+		}
+		inst, err := x86asm.Decode(code, 64)
+		if err != nil {
+			t.Errorf("%s: Decode(% x): %v", c.name, code, err)
+			continue
+		}
+		if inst.Len != len(code) {
+			t.Errorf("%s: Decode consumed %d of %d bytes", c.name, inst.Len, len(code))
+		}
+		if inst.Op.String() != c.mnem {
+			t.Errorf("%s: decoded as %s", c.name, inst.Op.String())
+		}
+	}
+}
+
+// TestVexNewFormsSyntax checks the decoded Intel-syntax rendering of the new
+// SIMD forms (operand order is the decoder's, confirming the fields landed).
+func TestVexNewFormsSyntax(t *testing.T) {
+	checkSyntax(t, "vpshufd xmm9, xmm8, 0xee", "VPSHUFD", Imm(0xEE), vreg(t, "X8"), vreg(t, "X9"))
+	checkSyntax(t, "vpermq ymm2, ymm1, 0x1b", "VPERMQ", Imm(0x1B), vreg(t, "Y1"), vreg(t, "Y2"))
+	checkSyntax(t, "vextracti128 xmm9, ymm8, 0x1", "VEXTRACTI128", Imm(1), vreg(t, "Y8"), vreg(t, "X9"))
+	checkSyntax(t, "vinserti128 ymm2, ymm1, xmm5, 0x1", "VINSERTI128", Imm(1), vreg(t, "X5"), vreg(t, "Y1"), vreg(t, "Y2"))
+	checkSyntax(t, "vperm2i128 ymm3, ymm2, ymm1, 0x31", "VPERM2I128", Imm(0x31), vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3"))
+	checkSyntax(t, "vpermd ymm3, ymm2, ymm1", "VPERMD", vreg(t, "Y1"), vreg(t, "Y2"), vreg(t, "Y3"))
+	checkSyntax(t, "vshufpd xmm3, xmm2, xmm1, 0x1", "VSHUFPD", Imm(1), vreg(t, "X1"), vreg(t, "X2"), vreg(t, "X3"))
+	checkSyntax(t, "vmovq rax, xmm8", "VMOVQ", vreg(t, "X8"), AX)
+	checkSyntax(t, "vmovq xmm9, rax", "VMOVQ", AX, vreg(t, "X9"))
+	checkSyntax(t, "vmovdqu ymm1, ymmword ptr [rsi]", "VMOVDQU", Ptr(SI, 0, 32), vreg(t, "Y1"))
+	checkSyntax(t, "vmovdqu ymmword ptr [rdi], ymm3", "VMOVDQU", vreg(t, "Y3"), Ptr(DI, 0, 32))
+	checkSyntax(t, "vzeroupper", "VZEROUPPER")
+}
+
+// TestVexMemoryForms round-trips the new forms with memory sources/destinations,
+// covering the SIB/indexed path through the VEX prefix emitter.
+func TestVexMemoryForms(t *testing.T) {
+	checkSyntax(t, "vpshufd ymm1, ymmword ptr [rsi], 0x4e", "VPSHUFD", Imm(0x4E), Ptr(SI, 0, 32), vreg(t, "Y1"))
+	checkSyntax(t, "vinserti128 ymm2, ymm1, xmmword ptr [rdi], 0x1", "VINSERTI128", Imm(1), Ptr(DI, 0, 16), vreg(t, "Y1"), vreg(t, "Y2"))
+	checkSyntax(t, "vmovdqu ymm1, ymmword ptr [rax+4*rbx]", "VMOVDQU", Idx(AX, BX, 4, 0, 32), vreg(t, "Y1"))
+	checkSyntax(t, "vpermq ymm2, ymmword ptr [rsi], 0x1b", "VPERMQ", Imm(0x1B), Ptr(SI, 0, 32), vreg(t, "Y2"))
+	checkSyntax(t, "vfmadd231pd ymm8, ymm12, ymm14", "VFMADD231PD", vreg(t, "Y14"), vreg(t, "Y12"), vreg(t, "Y8"))
+	checkSyntax(t, "vcvtdq2pd ymm12, xmmword ptr [rsi]", "VCVTDQ2PD", Ptr(SI, 0, 16), vreg(t, "Y12"))
+	// The top and bottom of the accepted imm8 span: $255 and $-1 both encode
+	// an all-bits-set control.
+	checkSyntax(t, "vpshufd xmm1, xmm0, 0xff", "VPSHUFD", Imm(255), vreg(t, "X0"), vreg(t, "X1"))
+	checkSyntax(t, "vpshufd xmm1, xmm0, 0xff", "VPSHUFD", Imm(-1), vreg(t, "X0"), vreg(t, "X1"))
+}
+
+// TestVexErrors checks that invalid operand shapes are rejected.
+func TestVexErrors(t *testing.T) {
+	cases := []struct {
+		name string
+		mnem string
+		ops  []Operand
+	}{
+		{"VPSHUFD arity", "VPSHUFD", []Operand{vreg(t, "X0"), vreg(t, "X1")}},
+		{"VPSHUFD non-imm control", "VPSHUFD", []Operand{vreg(t, "X0"), vreg(t, "X1"), vreg(t, "X2")}},
+		{"VPSHUFD gpr dst", "VPSHUFD", []Operand{Imm(1), vreg(t, "X0"), AX}},
+		{"VEXTRACTI128 arity", "VEXTRACTI128", []Operand{Imm(1), vreg(t, "Y0")}},
+		{"VEXTRACTI128 non-vec src", "VEXTRACTI128", []Operand{Imm(1), AX, vreg(t, "X0")}},
+		{"VINSERTI128 arity", "VINSERTI128", []Operand{Imm(1), vreg(t, "X0"), vreg(t, "Y1")}},
+		{"VINSERTI128 non-vec vvvv", "VINSERTI128", []Operand{Imm(1), vreg(t, "X0"), AX, vreg(t, "Y1")}},
+		{"VPERM2I128 non-imm control", "VPERM2I128", []Operand{AX, vreg(t, "Y0"), vreg(t, "Y1"), vreg(t, "Y2")}},
+		{"VMOVSD reg-reg", "VMOVSD", []Operand{vreg(t, "X1"), vreg(t, "X2")}},
+		{"VMOVD reg-reg", "VMOVD", []Operand{vreg(t, "X1"), vreg(t, "X2")}},
+		{"VMOVQ ymm", "VMOVQ", []Operand{vreg(t, "Y1"), AX}},
+		{"VMOVQ mixed X/Y", "VMOVQ", []Operand{vreg(t, "X1"), vreg(t, "Y2")}},
+		{"VMOVQ no vector", "VMOVQ", []Operand{AX, BX}},
+		{"VMOVDQU gpr", "VMOVDQU", []Operand{AX, vreg(t, "Y1")}},
+		{"VMOVUPD gpr", "VMOVUPD", []Operand{vreg(t, "X1"), AX}},
+		{"VZEROUPPER operands", "VZEROUPPER", []Operand{AX}},
+		{"VPSLLD non-vec dst", "VPSLLD", []Operand{Imm(1), vreg(t, "Y0"), AX}},
+		{"VPSLLD non-imm count", "VPSLLD", []Operand{AX, vreg(t, "Y0"), vreg(t, "Y1")}},
+		{"VPSLLD non-vec src", "VPSLLD", []Operand{Imm(1), AX, vreg(t, "Y1")}},
+		{"VPADDD non-vec vvvv", "VPADDD", []Operand{vreg(t, "Y0"), AX, vreg(t, "Y1")}},
+		{"VEXTRACTI128 non-imm lane", "VEXTRACTI128", []Operand{AX, vreg(t, "Y0"), vreg(t, "X0")}},
+		{"VMOVQ imm operand", "VMOVQ", []Operand{Imm(1), vreg(t, "X0")}},
+		{"VPSHUFD imm rm", "VPSHUFD", []Operand{Imm(1), Imm(2), vreg(t, "X0")}},
+		{"VPSHUFD imm range", "VPSHUFD", []Operand{Imm(256), vreg(t, "X0"), vreg(t, "X1")}},
+		{"VPERMQ imm range", "VPERMQ", []Operand{Imm(300), vreg(t, "Y0"), vreg(t, "Y1")}},
+		{"VEXTRACTI128 imm range", "VEXTRACTI128", []Operand{Imm(256), vreg(t, "Y0"), vreg(t, "X0")}},
+		{"VPSLLD imm range", "VPSLLD", []Operand{Imm(-129), vreg(t, "Y0"), vreg(t, "Y1")}},
+	}
+	for _, c := range cases {
+		if _, err := Encode(c.mnem, c.ops...); err == nil {
+			t.Errorf("%s: expected an error, got none", c.name)
+		}
+	}
+}

cmd/gasm/main.go

@@ -26,7 +26,7 @@ import (
 
 // version is the release version, stamped at build time via
 // -ldflags "-X main.version=…" (defaulting to the current release).
-var version = "0.1.0"
+var version = "0.3.0"
 
 func main() {
 	if len(os.Args) < 2 {

docs/ARCHITECTURE.md

@@ -182,22 +182,39 @@ Every encoding is validated by decoding it again with `golang.org/x/arch` — th
 one module dependency, used in tests only and never linked into the binary.
 
 On top of the encoder, `Assemble` walks a parsed `TEXT` body, converts each
-operand to an encoder operand, and lays the instructions out in two passes so
+operand to an encoder operand, and lays the instructions out so local labels
-local labels resolve to fixed rel32 jump offsets.  The `FP`/`SP` pseudo-
+resolve to relative jump offsets: jumps start in the short (rel8) form and
+expand to rel32 when the settled displacement does not fit, iterating to a
+fixed point, and jump-to-jump chains are folded (a conditional jump to a label
+whose only instruction is an unconditional jump is redirected to the ultimate
+target) exactly as the Go toolchain's linker does before it encodes branches.
+The `FP`/`SP` pseudo-
 registers are translated onto the hardware stack pointer — `x+N(FP)` becomes
 `(N+8)(SP)` for a zero-frame function and `(N+frame+16)(SP)` once a frame
 pointer is set up, with the matching Go prologue/epilogue generated — so the
 output is byte-identical to the Go assembler for these cases.  SIMD is handled
-SIMD is handled
 by a VEX (AVX/AVX2) encoder — the two- and three-byte VEX prefixes with XMM/YMM
-registers — across three operand forms (the three-operand NDS form, the
+registers — across eight operand forms: the three-operand NDS form, the
- two-operand reg/rm form, and the immediate-shift form), together covering the
+two-operand reg/rm form, the immediate-shift form (plus the variable-count
-bulk of the integer SIMD set; each encoding is validated by round-trip
+shifts, which share the NDS shape with the count in an XMM register or
- decoding.  This increment covers register / memory / immediate / FP-frame
+memory), the immediate shuffle form (`VPSHUFD`, `VPERMQ`), the
-operands, local-label jumps and these VEX SIMD forms; the remaining SIMD forms
+three-operand-plus-immediate form (`VSHUFPD`,
-(shuffles, extract/insert, permute, moves), EVEX / AVX-512, `SB` (global
+`VPERM2I128`, `VINSERTI128`), the lane-extract form (`VEXTRACTI128`,
-symbol) operands (relocations) and object-file emission are the rest of
+`VEXTRACTF128`, where the YMM source occupies the reg field and the XMM or
-Phase 2.
+memory destination r/m), the direction-sensitive moves (`VMOVDQU`, `VMOVUPD`,
+`VMOVD`, `VMOVQ`, `VMOVSD`), the floating-point and FMA arithmetic (`VADDPD`,
+`VMULPD`, `VXORPD`, `VUNPCKHPD`, the scalar `VADDSD`/`VMULSD`, `VCVTDQ2PD`,
+`VFMADD231PD`) and the no-operand `VZEROUPPER` — together with `VPERMD` and
+the scalar families (`CMOVcc`, `SETcc`, `LZCNT`/`TZCNT`, the extending moves,
+`CVTSx2SD`, `IMUL3`), covering every instruction the go-flac AVX2 kernels use
+apart from global-symbol loads.  Every encoding is validated two ways: by
+round-trip decoding through `golang.org/x/arch`, and byte-for-byte against the
+machine code the real Go assembler emits — a comparison that now holds for
+whole functions: every kernel function that avoids `SB` operands assembles to
+exactly the Go toolchain's bytes.  This increment covers register / memory /
+immediate / FP-frame operands, local-label jumps and these VEX SIMD forms;
+EVEX / AVX-512, `SB` (global symbol) operands (relocations) and object-file
+emission are the rest of Phase 2.
 
 ## Extension points
 

justfile

@@ -3,7 +3,7 @@
 
 # gasm-devkit — developer tooling for Go's Plan 9 assembler (GAsm).
 
-version := "0.1.0"
+version := "0.3.0"
 
 default:
     @just --list

parser/parser.go

@@ -400,22 +400,29 @@ func parseAddress(g []token.Token) ast.Address {
 	}
 
 	i := 0
-	// Optional leading displacement before a '(' base group.
+	// Optional leading displacement before a '(' base group.  A sign pushes
-	if isSignedNumber(g, i) && i+1 < len(g) && g[i+1].Kind == token.LParen {
+	// the parenthesis one token further out: -4(DX) has it at i+2.
-		neg := false
+	if isSignedNumber(g, i) {
-		if g[i].Kind == token.Minus {
+		paren := i + 1
-			neg = true
+		if g[i].Kind == token.Minus || g[i].Kind == token.Plus {
-			i++
+			paren = i + 2
-		} else if g[i].Kind == token.Plus {
-			i++
 		}
-		if i < len(g) && g[i].Kind == token.Number {
+		if paren < len(g) && g[paren].Kind == token.LParen {
-			addr.Offset = parseInt(g[i].Text)
+			neg := false
-			addr.HasOff = true
+			if g[i].Kind == token.Minus {
-			if neg {
+				neg = true
-				addr.Offset = -addr.Offset
+				i++
+			} else if g[i].Kind == token.Plus {
+				i++
+			}
+			if i < len(g) && g[i].Kind == token.Number {
+				addr.Offset = parseInt(g[i].Text)
+				addr.HasOff = true
+				if neg {
+					addr.Offset = -addr.Offset
+				}
+				i++
 			}
-			i++
 		}
 	}
 	// First parenthesised group: the base register.

parser/parser_test.go

@@ -34,6 +34,46 @@ func texts(f *ast.File) []*ast.Text {
 	return out
 }
 
+// TestNegativeDisplacement is a regression test for a leading negative
+// displacement with a base and index: the sign pushed the parenthesis one
+// token further out than the lookahead expected, and the whole address used
+// to parse empty.
+func TestNegativeDisplacement(t *testing.T) {
+	f, errs := Parse("neg_amd64.s", `
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	LEAQ -4(DX)(R9*4), R9
+	MOVQ +8(AX), BX
+	RET
+`)
+	if len(errs) > 0 {
+		t.Fatalf("parse: %v", errs)
+	}
+	fn := texts(f)[0]
+	var leaq, movq *ast.Instr
+	for _, s := range fn.Body {
+		if in, ok := s.(*ast.Instr); ok {
+			switch in.Mnemonic.Text {
+			case "LEAQ":
+				leaq = in
+			case "MOVQ":
+				movq = in
+			}
+		}
+	}
+	if leaq == nil || movq == nil {
+		t.Fatalf("instructions not parsed: leaq=%v movq=%v", leaq, movq)
+	}
+	a := leaq.Operands[0].Addr
+	if a.Base != "DX" || a.Index != "R9" || a.Scale != 4 || a.Offset != -4 || !a.HasOff {
+		t.Errorf("LEAQ addr = %+v, want -4(DX)(R9*4)", a)
+	}
+	b := movq.Operands[0].Addr
+	if b.Base != "AX" || b.Offset != 8 || !b.HasOff {
+		t.Errorf("MOVQ addr = %+v, want +8(AX)", b)
+	}
+}
+
 func TestParseSample(t *testing.T) {
 	f := mustParse(t, "../testdata/sample_amd64.s")