feat(asm): assemble static symbols and the whole go-flac AVX2 kernel byte-identically

Assisted-by: Qwen 3.8 Max Preview
feat(asm): byte-identical go-flac AVX2 assembly with scalar families and jump relaxation
2026-07-09 15:56:03 +02:00 · 2026-07-08 12:51:35 +02:00
15 changed files with 1198 additions and 110 deletions
@@ -13,55 +13,174 @@ import (
 // Assemble encodes the body of a TEXT function into x86-64 machine code,
 // resolving local labels to relative jump offsets and translating the FP/SP
 // pseudo-registers onto the hardware stack pointer (matching the Go
-// assembler's default frame-pointer behaviour).  Jumps always use the 32-bit
-// relative form so instruction sizes are fixed and offsets resolve in a single
-// layout pass.
+// assembler's default frame-pointer behaviour).  Jumps start in the short
+// (rel8) form and expand to rel32 when the settled displacement does not fit;
+// sizes only grow, so the layout reaches a fixed point in a few passes.  CALL
+// has no short form and is always rel32.
 //
 // Supported operands: registers, memory (real base register), immediates,
 // FP/SP frame-relative operands, and local-label jumps.  SB (global symbol)
 // operands require relocations and are not yet supported; the SIMD (VEX/AVX2)
 // integer and shuffle/extract/permute/move set is in.
 func Assemble(t *ast.Text) ([]byte, map[string]int, error) {
-	fi := computeFrame(t)
+	code, _, labels, err := assemble(t, nil)
+	return code, labels, err
+}

-	// Pass 1: lay out instructions (including prologue/epilogue) to fix label
-	// offsets.
-	offsets := map[string]int{}
+// linkInfo carries file-level symbol context into a single-function assembly:
+// the set of static symbols a GLOBL in the same file defines.  A nil link
+// rejects SB operands outright (single-function assembly cannot resolve
+// them).
+type linkInfo struct {
+	symbols map[string]bool
+}
+
+// sbPatch is a function-relative static-symbol relocation: the disp32 field
+// at off must become the symbol's address minus after, where after is the
+// function-relative address just past the instruction.
+type sbPatch struct {
+	off    int
+	after  int
+	name   string
+	addend int64
+}
+
+// assemble encodes a TEXT body, returning the machine code, the static-symbol
+// patch sites (for the file-level layout to resolve) and the label table.
+func assemble(t *ast.Text, link *linkInfo) ([]byte, []sbPatch, map[string]int, error) {
+	fi := computeFrame(t)
+	chain := jumpChain(t)
+	resolve := func(name string) string {
+		if r, ok := chain[name]; ok {
+			return r
+		}
+		return name
+	}
+
+	// Layout: iterate jump sizes to a fixed point.
+	long := make([]bool, len(t.Body))
 	sizes := make([]int, len(t.Body))
-	pos := len(fi.prologue)
-	for i, stmt := range t.Body {
-		switch s := stmt.(type) {
-		case *ast.Label:
-			offsets[s.Name.Text] = pos
-		case *ast.Instr:
-			sz, err := instrSize(s, fi)
-			if err != nil {
-				return nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
+	offsets := map[string]int{}
+	pcs := make([]int, len(t.Body))
+	for {
+		pos := len(fi.prologue)
+		for i, stmt := range t.Body {
+			switch s := stmt.(type) {
+			case *ast.Label:
+				offsets[s.Name.Text] = pos
+			case *ast.Instr:
+				sz, err := instrSize(s, fi, long[i], link)
+				if err != nil {
+					return nil, nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
+				}
+				sizes[i] = sz
+				pcs[i] = pos
+				pos += sz
 			}
-			sizes[i] = sz
-			pos += sz
+		}
+		// Expand any short jump whose displacement no longer fits rel8.
+		changed := false
+		for i, stmt := range t.Body {
+			s, ok := stmt.(*ast.Instr)
+			if !ok {
+				continue
+			}
+			mnem := strings.ToUpper(s.Mnemonic.Text)
+			if !isJumpMnemonic(mnem) || mnem == "CALL" || long[i] {
+				continue
+			}
+			name, ok := labelName(s.Operands[0])
+			if !ok {
+				continue // reported during emission
+			}
+			target, ok := offsets[resolve(name)]
+			if !ok {
+				continue // reported during emission
+			}
+			rel := int64(target - (pcs[i] + jumpSize(mnem, false)))
+			if !fits8(rel) {
+				long[i] = true
+				changed = true
+			}
+		}
+		if !changed {
+			break
 		}
 	}

 	// Pass 2: emit.
 	out := append([]byte(nil), fi.prologue...)
-	pos = len(fi.prologue)
+	var patches []sbPatch
+	pos := len(fi.prologue)
 	for i, stmt := range t.Body {
 		s, ok := stmt.(*ast.Instr)
 		if !ok {
 			continue
 		}
-		code, err := encodeInstr(s, pos, offsets, fi)
+		code, ps, err := encodeInstr(s, pos, offsets, fi, long[i], resolve, link)
 		if err != nil {
-			return nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
+			return nil, nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err)
 		}
 		if len(code) != sizes[i] {
-			return nil, nil, fmt.Errorf("%s: size mismatch (%d vs %d)", s.Mnemonic.Text, len(code), sizes[i])
+			return nil, nil, nil, fmt.Errorf("%s: size mismatch (%d vs %d)", s.Mnemonic.Text, len(code), sizes[i])
 		}
+		patches = append(patches, ps...)
 		out = append(out, code...)
 		pos += len(code)
 	}
-	return out, offsets, nil
+	return out, patches, offsets, nil
+}
+
+// jumpChain precomputes jump-to-jump folding: a label whose first instruction
+// is an unconditional local jump redirects its own jumpers to the ultimate
+// target.  The Go toolchain chases exactly these chains (the linker's xfol
+// pass) before it encodes branches, so matching its bytes requires the same
+// redirection.
+func jumpChain(t *ast.Text) map[string]string {
+	// label → the target of its leading unconditional local JMP, if any.
+	leadsTo := map[string]string{}
+	for i, stmt := range t.Body {
+		l, ok := stmt.(*ast.Label)
+		if !ok {
+			continue
+		}
+		// Stacked labels share an address: skip to the first instruction.
+		j := i + 1
+		for j < len(t.Body) {
+			if _, isLabel := t.Body[j].(*ast.Label); !isLabel {
+				break
+			}
+			j++
+		}
+		if j >= len(t.Body) {
+			continue
+		}
+		in, ok := t.Body[j].(*ast.Instr)
+		if !ok || strings.ToUpper(in.Mnemonic.Text) != "JMP" || len(in.Operands) != 1 {
+			continue
+		}
+		if name, ok := labelName(in.Operands[0]); ok {
+			leadsTo[l.Name.Text] = name
+		}
+	}
+	// Chase each chain to its end, guarding against cycles.
+	chain := map[string]string{}
+	for name := range leadsTo {
+		visited := map[string]bool{name: true}
+		cur := name
+		for {
+			next, ok := leadsTo[cur]
+			if !ok || visited[next] {
+				break
+			}
+			visited[next] = true
+			cur = next
+		}
+		if cur != name {
+			chain[name] = cur
+		}
+	}
+	return chain
 }

 // frameInfo carries the frame layout derived from the TEXT directive.
@@ -119,15 +238,15 @@ func addSP(size int) []byte { // ADDQ $size, SP
 	return append([]byte{0x48, 0x81, 0xC4}, le32(int64(size))...)
 }

-// instrSize returns the encoded length of an instruction (pass 1).  encodeInstr
-// already includes the epilogue for a RET in a frame-pointer function; jumps use
-// a fixed rel32 size (no epilogue).
-func instrSize(s *ast.Instr, fi frameInfo) (int, error) {
+// instrSize returns the encoded length of an instruction (layout pass).
+// encodeInstr already includes the epilogue for a RET in a frame-pointer
+// function; jumps use their short or long form (never an epilogue).
+func instrSize(s *ast.Instr, fi frameInfo, long bool, link *linkInfo) (int, error) {
 	mnem := strings.ToUpper(s.Mnemonic.Text)
 	if isJumpMnemonic(mnem) {
-		return jumpSize(mnem), nil
+		return jumpSize(mnem, long), nil
 	}
-	code, err := encodeInstr(s, 0, nil, fi)
+	code, _, err := encodeInstr(s, 0, nil, fi, false, nil, link)
 	if err != nil {
 		return 0, err
 	}
@@ -142,18 +261,27 @@ func isJumpMnemonic(mnem string) bool {
 	return ok
 }

-// jumpSize returns the fixed length of a rel32 jump instruction.
-func jumpSize(mnem string) int {
-	if mnem == "JMP" || mnem == "CALL" {
+// jumpSize returns the length of a jump instruction in the requested form:
+// short (rel8) where available, otherwise the rel32 form.  CALL is always
+// rel32.
+func jumpSize(mnem string, long bool) int {
+	if mnem == "CALL" {
 		return 5 // opcode + rel32
 	}
+	if !long {
+		return 2 // opcode + rel8
+	}
+	if mnem == "JMP" {
+		return 5 // E9 + rel32
+	}
 	return 6 // 0x0F 0x8x + rel32
 }

 // encodeInstr encodes one instruction, resolving jump targets against offsets
 // (relative to pc, the instruction's own offset).  A RET in a frame-pointer
-// function is prefixed with the epilogue.
-func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo) ([]byte, error) {
+// function is prefixed with the epilogue.  resolve, when non-nil, redirects a
+// jump label through the jump-to-jump chain before the offset lookup.
+func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo, long bool, resolve func(string) string, link *linkInfo) ([]byte, []sbPatch, error) {
 	mnem := strings.ToUpper(s.Mnemonic.Text)

 	var prefix []byte
@@ -162,37 +290,53 @@ func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo) ([]
 	}

 	var code []byte
+	var ps []sbPatch
 	var err error
 	if isJumpMnemonic(mnem) {
-		code, err = encodeJump(s, mnem, pc+len(prefix), offsets)
+		code, err = encodeJump(s, mnem, pc+len(prefix), offsets, long, resolve)
 	} else {
-		code, err = encodeNormal(s, fi)
+		code, ps, err = encodeNormal(s, fi, link)
 	}
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return append(prefix, code...), nil
+	// Anchor the patch fields at function-relative positions: off indexes the
+	// disp32 field, after is the address just past the instruction.
+	body := pc + len(prefix)
+	for i := range ps {
+		ps[i].off += body
+		ps[i].after = body + len(code)
+	}
+	return append(prefix, code...), ps, nil
 }

-func encodeNormal(s *ast.Instr, fi frameInfo) ([]byte, error) {
+func encodeNormal(s *ast.Instr, fi frameInfo, link *linkInfo) ([]byte, []sbPatch, error) {
 	_, size := splitSize(strings.ToUpper(s.Mnemonic.Text))
 	if size == 0 {
 		size = 8
 	}
 	ops := make([]Operand, len(s.Operands))
 	for i, op := range s.Operands {
-		o, err := operandFromAST(op, size, fi)
+		o, err := operandFromAST(op, size, fi, link)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		ops[i] = o
 	}
-	return Encode(s.Mnemonic.Text, ops...)
+	e := &enc{}
+	if err := e.encode(s.Mnemonic.Text, ops); err != nil {
+		return nil, nil, err
+	}
+	ps := make([]sbPatch, len(e.patches))
+	for i, p := range e.patches {
+		ps[i] = sbPatch{off: p.off, name: p.name, addend: p.addend}
+	}
+	return e.out, ps, nil
 }

-// encodeJump encodes a JMP/CALL/Jcc with a rel32 offset resolved from the
-// target label.
-func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int) ([]byte, error) {
+// encodeJump encodes a JMP/CALL/Jcc with a relative offset resolved from the
+// target label, in the short (rel8) or long (rel32) form.
+func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int, long bool, resolve func(string) string) ([]byte, error) {
 	if len(s.Operands) != 1 {
 		return nil, fmt.Errorf("jump expects 1 operand, got %d", len(s.Operands))
 	}
@@ -200,12 +344,25 @@ func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int) ([]by
 	if !ok {
 		return nil, fmt.Errorf("jump target must be a local label")
 	}
+	if resolve != nil && mnem != "CALL" {
+		name = resolve(name)
+	}
 	target, ok := offsets[name]
 	if !ok {
 		return nil, fmt.Errorf("undefined label %q", name)
 	}
-	rel := int64(target - (pc + jumpSize(mnem)))
+	rel := int64(target - (pc + jumpSize(mnem, long)))

+	if !long {
+		if !fits8(rel) {
+			return nil, fmt.Errorf("jump to %q does not fit the short form", name)
+		}
+		if mnem == "JMP" {
+			return []byte{0xEB, byte(int8(rel))}, nil
+		}
+		cc, _ := condCode(mnem)
+		return []byte{0x70 + byte(cc), byte(int8(rel))}, nil
+	}
 	switch mnem {
 	case "JMP":
 		return append([]byte{0xE9}, le32(rel)...), nil
@@ -231,7 +388,7 @@ var spReg = Reg{idx: 4, size: 8}

 // operandFromAST converts a parsed operand into an encoder Operand, applying
 // the frame translation to FP/SP pseudo-register operands.
-func operandFromAST(op *ast.Operand, size int, fi frameInfo) (Operand, error) {
+func operandFromAST(op *ast.Operand, size int, fi frameInfo, link *linkInfo) (Operand, error) {
 	switch op.Kind {
 	case ast.OpImmediate:
 		if op.Imm.HasVal {
@@ -257,9 +414,20 @@ func operandFromAST(op *ast.Operand, size int, fi frameInfo) (Operand, error) {
 			off := fi.spAdjust + a.Sym.Offset
 			return Mem{Base: spReg, Disp: off, HasBase: true, Size: size}, nil
 		}
-		// SB (global symbol) needs a relocation — not yet supported.
+		// SB (global symbol): a static (file-local, <>) symbol becomes a
+		// RIP-relative reference resolved by the file-level layout; anything
+		// external needs object-file emission.
 		if a.Sym != nil && a.Sym.Pseudo == "SB" {
-			return nil, fmt.Errorf("SB (global symbol) operands need relocation support (pending)")
+			if !a.Sym.Static {
+				return nil, fmt.Errorf("external symbol %q needs object-file emission", a.Sym.Name)
+			}
+			if link == nil || link.symbols == nil {
+				return nil, fmt.Errorf("static symbol %q needs file-level assembly (AssembleFile)", a.Sym.Name)
+			}
+			if !link.symbols[a.Sym.Name] {
+				return nil, fmt.Errorf("undefined symbol %q", a.Sym.Name)
+			}
+			return sbMem{size: size, name: a.Sym.Name, addend: a.Sym.Offset}, nil
 		}

 		// Memory with a real base register: (base), off(base), (base)(index*scale).
@@ -244,3 +244,76 @@ TEXT ·hsum(SB), NOSPLIT, $0
 		t.Errorf("VEX kernel mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
 	}
 }
+
+// TestAssembleShortJumps checks that a tight loop settles on the short (rel8)
+// jump forms, byte for byte with the Go assembler.
+func TestAssembleShortJumps(t *testing.T) {
+	fn := firstText(t, `
+#include "textflag.h"
+TEXT ·loop(SB), NOSPLIT, $0
+	XORQ AX, AX
+l1:
+	ADDQ $1, AX
+	CMPQ AX, $10
+	JLT l1
+	RET
+`)
+	code, _, err := Assemble(fn)
+	if err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	// From the Go-assembled function:
+	//   XORQ AX, AX       4831c0
+	//   ADDQ $1, AX       4883c001
+	//   CMPQ AX, $10      4883f80a
+	//   JLT l1            7cf6      (short, rel8)
+	//   RET               c3
+	want := []byte{
+		0x48, 0x31, 0xc0,
+		0x48, 0x83, 0xc0, 0x01,
+		0x48, 0x83, 0xf8, 0x0a,
+		0x7c, 0xf6,
+		0xc3,
+	}
+	if hexBytes(code) != hexBytes(want) {
+		t.Errorf("short-jump mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
+	}
+}
+
+// TestAssembleJumpFolding checks jump-to-jump folding: a conditional jump to a
+// label that only holds an unconditional jump is redirected to the ultimate
+// target, exactly as the Go toolchain does before it encodes branches.
+func TestAssembleJumpFolding(t *testing.T) {
+	fn := firstText(t, `
+#include "textflag.h"
+TEXT ·fold(SB), NOSPLIT, $0
+	XORQ AX, AX
+	JGE done
+	INCQ AX
+done:
+	JMP end
+end:
+	RET
+`)
+	code, _, err := Assemble(fn)
+	if err != nil {
+		t.Fatalf("Assemble: %v", err)
+	}
+	// From the Go-assembled function: the JGE skips past the done: trampoline
+	// straight to end:
+	//   XORQ AX, AX       4831c0
+	//   JGE end           7d05      (folded past done)
+	//   INCQ AX           48ffc0
+	//   JMP end           eb00
+	//   RET               c3
+	want := []byte{
+		0x48, 0x31, 0xc0,
+		0x7d, 0x05,
+		0x48, 0xff, 0xc0,
+		0xeb, 0x00,
+		0xc3,
+	}
+	if hexBytes(code) != hexBytes(want) {
+		t.Errorf("jump-folding mismatch:\n got:  %s\n want: %s", hexBytes(code), hexBytes(want))
+	}
+}
@@ -19,7 +19,16 @@ func Encode(mnemonic string, ops ...Operand) ([]byte, error) {
 }

 type enc struct {
-	out []byte
+	out     []byte
+	patches []encPatch // disp32 fields awaiting static-symbol resolution
+}
+
+// encPatch marks a 4-byte displacement field in enc.out that must receive the
+// RIP-relative offset of a static symbol once the file layout is settled.
+type encPatch struct {
+	off    int
+	name   string
+	addend int64
 }

 func (e *enc) encode(mnem string, ops []Operand) error {
@@ -46,6 +55,14 @@ func (e *enc) encode(mnem string, ops []Operand) error {
 		return e.encodeVex(upper, ops)
 	}

+	// CMOVcc and SETcc carry the condition in the mnemonic (CMOVLGT, SETNE).
+	if strings.HasPrefix(upper, "CMOV") {
+		return e.encodeCmov(upper, ops)
+	}
+	if strings.HasPrefix(upper, "SET") {
+		return e.encodeSet(upper, ops)
+	}
+
 	base, size := splitSize(upper)
 	if size == 0 {
 		size = 8 // default operand size in 64-bit mode (e.g. PUSHQ)
@@ -63,12 +80,18 @@ func (e *enc) encode(mnem string, ops []Operand) error {
 		return e.encodeUnary(unaryOp[base], ops, size)
 	case "SHL", "SHR", "SAR":
 		return e.encodeShift(shiftOp[base], ops, size)
-	case "IMUL":
+	case "IMUL", "IMUL3":
 		return e.encodeImul(ops, size)
 	case "PUSH":
 		return e.encodePushPop(ops, true)
 	case "POP":
 		return e.encodePushPop(ops, false)
+	case "LZCNT", "TZCNT":
+		return e.encodeCount(base, ops, size)
+	case "MOVBLZX", "MOVBQZX", "MOVWLZX", "MOVWQZX", "MOVWLSX", "MOVLQSX":
+		return e.encodeMovExtend(base, ops)
+	case "CVTSL2SD", "CVTSQ2SD":
+		return e.encodeCvtsi2sd(base == "CVTSQ2SD", ops)
 	}
 	return fmt.Errorf("unsupported instruction %q", mnem)
 }
@@ -100,17 +123,29 @@ type instr struct {
 	rexX      bool
 	rexB      bool
 	rexForced bool // REX needed even with all bits zero (8-bit low registers)
+	prefix    byte // legacy 0xF2/0xF3 prefix (0 = none); emitted after 0x66
 	opcode    []byte
 	modrm     int // -1 if absent
 	sib       int // -1 if absent
 	disp      []byte
 	imm       []byte
+	sb        *sbRef // static-symbol displacement in disp, awaiting resolution
+}
+
+// sbRef records that an instruction's displacement refers to a static symbol
+// rather than holding a literal value.
+type sbRef struct {
+	name   string
+	addend int64
 }

 func (e *enc) emit(i *instr) error {
 	if i.opSize16 {
 		e.out = append(e.out, 0x66)
 	}
+	if i.prefix != 0 {
+		e.out = append(e.out, i.prefix)
+	}
 	rex := byte(0)
 	if i.rexW {
 		rex |= 0x08
@@ -134,6 +169,9 @@ func (e *enc) emit(i *instr) error {
 	if i.sib >= 0 {
 		e.out = append(e.out, byte(i.sib))
 	}
+	if i.sb != nil {
+		e.patches = append(e.patches, encPatch{off: len(e.out), name: i.sb.name, addend: i.sb.addend})
+	}
 	e.out = append(e.out, i.disp...)
 	e.out = append(e.out, i.imm...)
 	return nil
@@ -181,6 +219,13 @@ func setRMReg(i *instr, regField int, rexR, regForced bool, rm Operand, opSize i
 		return nil
 	case Mem:
 		return setMem(i, regField, r)
+	case sbMem:
+		// RIP-relative reference; the displacement is patched once the static
+		// symbol's address is known.
+		i.modrm = regField<<3 | 0x05 // mod=00, rm=101 → (RIP)+disp32
+		i.disp = le32(0)
+		i.sb = &sbRef{name: r.name, addend: r.addend}
+		return nil
 	default:
 		return fmt.Errorf("invalid r/m operand %T", rm)
 	}
@@ -4,6 +4,7 @@
 package asm

 import (
+	"strings"
 	"testing"

 	"golang.org/x/arch/x86/x86asm"
@@ -70,7 +71,7 @@ func TestALU(t *testing.T) {
 	checkSyntax(t, "and rbx, 0x7", "ANDQ", Imm(7), BX)
 	checkSyntax(t, "or rcx, rbx", "ORQ", BX, CX)
 	checkSyntax(t, "xor rax, rax", "XORQ", AX, AX)
-	checkSyntax(t, "cmp r10, rsi", "CMPQ", SI, Reg{idx: 10, size: 8})
+	checkSyntax(t, "cmp rsi, r10", "CMPQ", SI, Reg{idx: 10, size: 8})
 	checkSyntax(t, "add rbx, qword ptr [rax]", "ADDQ", Ptr(AX, 0, 8), BX)
 	checkSyntax(t, "add qword ptr [rax], rbx", "ADDQ", BX, Ptr(AX, 0, 8))
 	checkSyntax(t, "cmp rbx, -0x20", "CMPQ", Imm(-32), BX)
@@ -134,3 +135,116 @@ func TestGoFlacScalarTail(t *testing.T) {
 	checkSyntax(t, "lea r9, ptr [rsi+4*rbx]", "LEAQ", Idx(SI, BX, 4, 0, 8), Reg{idx: 9, size: 8})
 	checkSyntax(t, "and r10, -0x8", "ANDQ", Imm(-8), Reg{idx: 10, size: 8})
 }
+
+// TestScalarGroundTruth checks the scalar instruction families the go-flac
+// kernels use beyond the basic set, byte for byte against the Go assembler's
+// machine code.  wantOp is the x86 decoder's name, which differs from the
+// Plan 9 spelling for some of these (CMOVLGT → CMOVG, MOVBLZX → MOVZX, …).
+func TestScalarGroundTruth(t *testing.T) {
+	r8 := Reg{idx: 8, size: 8}
+	r9 := Reg{idx: 9, size: 8}
+	r9w := Reg{idx: 9, size: 2}
+	r8w := Reg{idx: 8, size: 2}
+	r13 := Reg{idx: 13, size: 8}
+	cases := []struct {
+		name   string
+		mnem   string
+		ops    []Operand
+		want   string
+		wantOp string
+	}{
+		{"LZCNTL AX,CX", "LZCNTL", []Operand{AX, CX}, "f30fbdc8", "LZCNT"},
+		{"LZCNTQ R8,R9", "LZCNTQ", []Operand{r8, r9}, "f34d0fbdc8", "LZCNT"},
+		{"LZCNTW AX,CX", "LZCNTW", []Operand{AX, CX}, "66f30fbdc8", "LZCNT"},
+		{"TZCNTL AX,CX", "TZCNTL", []Operand{AX, CX}, "f30fbcc8", "TZCNT"},
+		{"CMOVLGT CX,AX", "CMOVLGT", []Operand{CX, AX}, "0f4fc1", "CMOVG"},
+		{"CMOVLEQ CX,AX", "CMOVLEQ", []Operand{CX, AX}, "0f44c1", "CMOVE"},
+		{"CMOVQGT R9,R8", "CMOVQGT", []Operand{r9, r8}, "4d0f4fc1", "CMOVG"},
+		{"CMOVWLS R9W,R8W", "CMOVWLS", []Operand{r9w, r8w}, "66450f46c1", "CMOVBE"},
+		{"SETNE AL", "SETNE", []Operand{AL}, "0f95c0", "SETNE"},
+		{"SETNE (AX)", "SETNE", []Operand{Ptr(AX, 0, 1)}, "0f9500", "SETNE"},
+		{"MOVBLZX AL,CX", "MOVBLZX", []Operand{AL, CX}, "0fb6c8", "MOVZX"},
+		{"MOVBLZX (SI),CX", "MOVBLZX", []Operand{Ptr(SI, 0, 1), CX}, "0fb60e", "MOVZX"},
+		{"MOVWLSX (SI)(AX*1),CX", "MOVWLSX", []Operand{Idx(SI, AX, 1, 0, 2), CX}, "0fbf0c06", "MOVSX"},
+		{"MOVLQSX CX,R8", "MOVLQSX", []Operand{CX, r8}, "4c63c1", "MOVSXD"},
+		{"MOVBQZX AL,R8", "MOVBQZX", []Operand{AL, r8}, "4c0fb6c0", "MOVZX"},
+		{"MOVWLZX AX,CX", "MOVWLZX", []Operand{AX, CX}, "0fb7c8", "MOVZX"},
+		{"MOVWQZX AX,R8", "MOVWQZX", []Operand{AX, r8}, "4c0fb7c0", "MOVZX"},
+		{"CVTSL2SD R8,X13", "CVTSL2SD", []Operand{r8, vreg(t, "X13")}, "f2450f2ae8", "CVTSI2SD"},
+		{"CVTSL2SD AX,X0", "CVTSL2SD", []Operand{AX, vreg(t, "X0")}, "f20f2ac0", "CVTSI2SD"},
+		{"CVTSQ2SD R8,X13", "CVTSQ2SD", []Operand{r8, vreg(t, "X13")}, "f24d0f2ae8", "CVTSI2SD"},
+		{"INCW (R13)(AX*2)", "INCW", []Operand{Idx(r13, AX, 2, 0, 2)}, "6641ff444500", "INC"},
+		// The traditional three-operand IMUL spelling.
+		{"IMUL3L $31,CX,DX", "IMUL3L", []Operand{Imm(31), CX, DX}, "6bd11f", "IMUL"},
+		{"IMUL3L $256,CX,DX", "IMUL3L", []Operand{Imm(256), CX, DX}, "69d100010000", "IMUL"},
+		{"IMUL3Q $7,R9,R8", "IMUL3Q", []Operand{Imm(7), r9, r8}, "4d6bc107", "IMUL"},
+		{"IMUL3W $5,CX,DX", "IMUL3W", []Operand{Imm(5), CX, DX}, "666bd105", "IMUL"},
+		// Negative displacement with base + index (regression: the parser
+		// used to drop the whole address).
+		{"LEAQ -4(DX)(R9*4),R9", "LEAQ", []Operand{Idx(DX, r9, 4, -4, 8), r9}, "4e8d4c8afc", "LEA"},
+		{"LEAQ 16(SI)(BX*4),R10", "LEAQ", []Operand{Idx(SI, BX, 4, 16, 8), Reg{idx: 10, size: 8}}, "4c8d549e10", "LEA"},
+		// Register-to-register MOV uses the r/m←r opcode (reg = source), the
+		// Go assembler's choice.
+		{"MOVQ BX,R10", "MOVQ", []Operand{BX, Reg{idx: 10, size: 8}}, "4989da", "MOV"},
+		{"MOVQ AX,BX", "MOVQ", []Operand{AX, BX}, "4889c3", "MOV"},
+		{"MOVL AX,BX", "MOVL", []Operand{AX, BX}, "89c3", "MOV"},
+		{"MOVB AL,BL", "MOVB", []Operand{AL, BL}, "88c3", "MOV"},
+		{"MOVW AX,BX", "MOVW", []Operand{AX, BX}, "6689c3", "MOV"},
+		{"MOVQ R12,R13", "MOVQ", []Operand{Reg{idx: 12, size: 8}, Reg{idx: 13, size: 8}}, "4d89e5", "MOV"},
+		// CMP must record first − second: with a register second operand the
+		// first goes in r/m, with a memory second operand the first goes in reg.
+		{"CMPQ SI,R10", "CMPQ", []Operand{SI, Reg{idx: 10, size: 8}}, "4c39d6", "CMP"},
+		{"CMPQ SI,(AX)", "CMPQ", []Operand{SI, Ptr(AX, 0, 8)}, "483b30", "CMP"},
+		{"CMPQ (AX),SI", "CMPQ", []Operand{Ptr(AX, 0, 8), SI}, "483930", "CMP"},
+		{"CMPL CX,(AX)", "CMPL", []Operand{CX, Ptr(AX, 0, 4)}, "3b08", "CMP"},
+		{"CMPB AL,(BX)", "CMPB", []Operand{AL, Ptr(BX, 0, 1)}, "3a03", "CMP"},
+		{"CMPW AX,BX", "CMPW", []Operand{AX, BX}, "6639d8", "CMP"},
+	}
+	for _, c := range cases {
+		code, err := Encode(c.mnem, c.ops...)
+		if err != nil {
+			t.Errorf("%s: Encode: %v", c.name, err)
+			continue
+		}
+		if got := strings.ReplaceAll(hexBytes(code), " ", ""); got != c.want {
+			t.Errorf("%s: bytes %s, want %s", c.name, got, c.want)
+			continue
+		}
+		inst, err := x86asm.Decode(code, 64)
+		if err != nil {
+			t.Errorf("%s: Decode(% x): %v", c.name, code, err)
+			continue
+		}
+		if inst.Op.String() != c.wantOp {
+			t.Errorf("%s: decoded as %s", c.name, inst.Op.String())
+		}
+	}
+}
+
+// TestScalarErrors checks that malformed conditional / extend / convert
+// instructions are rejected.
+func TestScalarErrors(t *testing.T) {
+	cases := []struct {
+		name string
+		mnem string
+		ops  []Operand
+	}{
+		{"CMOV arity", "CMOVLGT", []Operand{AX}},
+		{"CMOV bare", "CMOV", []Operand{AX, BX}},
+		{"CMOV bad size", "CMOVBGT", []Operand{AX, BX}},
+		{"CMOV bad condition", "CMOVLXX", []Operand{AX, BX}},
+		{"CMOV mem dst", "CMOVLGT", []Operand{AX, Ptr(BX, 0, 4)}},
+		{"SET arity", "SETNE", []Operand{AL, BL}},
+		{"SET bad condition", "SETXX", []Operand{AL}},
+		{"SET bare", "SET", []Operand{AL}},
+		{"LZCNT arity", "LZCNTL", []Operand{AX}},
+		{"LZCNT mem dst", "LZCNTL", []Operand{AX, Ptr(BX, 0, 4)}},
+		{"MOVBLZX mem dst", "MOVBLZX", []Operand{AL, Ptr(BX, 0, 4)}},
+		{"CVTSL2SD gpr dst", "CVTSL2SD", []Operand{AX, BX}},
+	}
+	for _, c := range cases {
+		if _, err := Encode(c.mnem, c.ops...); err == nil {
+			t.Errorf("%s: expected an error, got none", c.name)
+		}
+	}
+}
@@ -52,9 +52,10 @@ func (e *enc) encodeMov(ops []Operand, size int) error {
 	switch src := src.(type) {
 	case Reg:
 		if dstIsReg {
-			// MOV r, r/m: 0x8A/0x8B, reg=dst, rm=src.
-			i := newInstr(size, []byte{movRR(size)})
-			if err := setRM(i, dstReg, src, size); err != nil {
+			// MOV r/m, r: 0x88/0x89, reg=src, rm=dst — the form the Go
+			// assembler emits for register-to-register moves.
+			i := newInstr(size, []byte{movRM(size)})
+			if err := setRM(i, src, dst, size); err != nil {
 				return err
 			}
 			return e.emit(i)
@@ -77,6 +78,17 @@ func (e *enc) encodeMov(ops []Operand, size int) error {
 		}
 		return e.emit(i)

+	case sbMem:
+		if !dstIsReg {
+			return fmt.Errorf("MOV: two memory operands")
+		}
+		// MOV r, r/m: reg=dst, rm=src(static symbol).
+		i := newInstr(size, []byte{movRR(size)})
+		if err := setRM(i, dstReg, src, size); err != nil {
+			return err
+		}
+		return e.emit(i)
+
 	case Imm:
 		if dstIsReg {
 			// MOV r, imm: 0xB0+reg (8-bit) / 0xB8+reg (16/32/64, imm64 for Q).
@@ -147,9 +159,37 @@ func (e *enc) encodeALU(op struct {
 		return e.encodeALUImm(op.digit, src, int64(imm), size)
 	}

+	// CMP records first − second without writing anywhere, so the first
+	// operand must land as the minuend; every other ALU op writes its second
+	// operand and follows the forms below.
+	cmp := op.rr == 0x39
 	dstReg, dstIsReg := dst.(Reg)
 	srcReg, srcIsReg := src.(Reg)
 	switch {
+	case cmp && dstIsReg:
+		// CMP x, reg: OP r/m, r (0x38/0x39) with rm = first operand, reg =
+		// second, matching the Go assembler.
+		opc := op.rr
+		if size == 1 {
+			opc = op.rr - 1
+		}
+		i := newInstr(size, []byte{opc})
+		if err := setRM(i, dstReg, src, size); err != nil {
+			return err
+		}
+		return e.emit(i)
+	case cmp && srcIsReg:
+		// CMP reg, mem: OP r, r/m (0x3A/0x3B) with reg = first operand, rm =
+		// second.
+		opc := op.rr + 2
+		if size == 1 {
+			opc = op.rr + 1
+		}
+		i := newInstr(size, []byte{opc})
+		if err := setRM(i, srcReg, dst, size); err != nil {
+			return err
+		}
+		return e.emit(i)
 	case srcIsReg:
 		// OP r/m, r: reg=src, rm=dst (dst is a register or memory).  This is the
 		// form the Go assembler prefers when the source is a register.
@@ -251,12 +291,13 @@ func (e *enc) encodeLea(ops []Operand, size int) error {
 	if !ok {
 		return fmt.Errorf("LEA: destination must be a register")
 	}
-	mem, ok := src.(Mem)
-	if !ok {
+	switch src.(type) {
+	case Mem, sbMem:
+	default:
 		return fmt.Errorf("LEA: source must be a memory operand")
 	}
 	i := newInstr(size, []byte{0x8D})
-	if err := setRM(i, dstReg, mem, size); err != nil {
+	if err := setRM(i, dstReg, src, size); err != nil {
 		return err
 	}
 	return e.emit(i)
@@ -497,3 +538,152 @@ func immediate(v int64, size int, full64 bool) []byte {
 		return le32(v) // sign-extended imm32
 	}
 }
+
+// --- CMOVcc / SETcc ---------------------------------------------------------
+
+// encodeCmov encodes a conditional move: CMOV + size (W/L/Q) + condition
+// (CMOVLGT, CMOVQEQ, …).  The condition reads exactly like the Jcc spellings;
+// the instruction is 0F 40+cc with reg = dst, rm = src.
+func (e *enc) encodeCmov(upper string, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("CMOVcc expects 2 operands, got %d", len(ops))
+	}
+	rest := upper[len("CMOV"):]
+	if len(rest) < 2 {
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	var size int
+	switch rest[0] {
+	case 'W':
+		size = 2
+	case 'L':
+		size = 4
+	case 'Q':
+		size = 8
+	default:
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	cc, ok := jccMap[rest[1:]]
+	if !ok {
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	src, dst := ops[0], ops[1]
+	dstReg, ok := dst.(Reg)
+	if !ok {
+		return fmt.Errorf("CMOVcc destination must be a register")
+	}
+	i := newInstr(size, []byte{0x0F, byte(0x40 + cc)})
+	if err := setRM(i, dstReg, src, size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// encodeSet encodes a conditional byte set: SET + condition (SETNE, SETEQ, …),
+// always a byte write — 0F 90+cc /0 into a register or memory operand.
+func (e *enc) encodeSet(upper string, ops []Operand) error {
+	if len(ops) != 1 {
+		return fmt.Errorf("SETcc expects 1 operand, got %d", len(ops))
+	}
+	cond := upper[len("SET"):]
+	cc, ok := jccMap[cond]
+	if !ok || cond == "" {
+		return fmt.Errorf("unsupported instruction %q", upper)
+	}
+	i := &instr{opcode: []byte{0x0F, byte(0x90 + cc)}, modrm: -1, sib: -1}
+	if err := setRMDigit(i, 0, ops[0], 1); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// --- LZCNT / TZCNT ----------------------------------------------------------
+
+// encodeCount encodes LZCNT/TZCNT (leading / trailing zero count): F3 0F BD
+// or F3 0F BC, with reg = dst and rm = src.  The size suffix selects the
+// operand width (LZCNTW/LZCNTL/LZCNTQ).
+func (e *enc) encodeCount(base string, ops []Operand, size int) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("%s expects 2 operands, got %d", base, len(ops))
+	}
+	op := byte(0xBD)
+	if base == "TZCNT" {
+		op = 0xBC
+	}
+	dstReg, ok := ops[1].(Reg)
+	if !ok {
+		return fmt.Errorf("%s destination must be a register", base)
+	}
+	i := newInstr(size, []byte{0x0F, op})
+	i.prefix = 0xF3
+	if err := setRM(i, dstReg, ops[0], size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// --- mixed-width sign/zero-extending moves -----------------------------------
+
+// movExtendOp maps Go's mixed-width move names to their opcode and destination
+// width.  The source is narrower than the destination, so the plain size-suffix
+// convention does not apply to these names.
+var movExtendOp = map[string]struct {
+	op    []byte
+	dst64 bool
+}{
+	"MOVBLZX": {[]byte{0x0F, 0xB6}, false}, // byte → long, zero-extend
+	"MOVBQZX": {[]byte{0x0F, 0xB6}, true},  // byte → quad, zero-extend
+	"MOVWLZX": {[]byte{0x0F, 0xB7}, false}, // word → long, zero-extend
+	"MOVWQZX": {[]byte{0x0F, 0xB7}, true},  // word → quad, zero-extend
+	"MOVWLSX": {[]byte{0x0F, 0xBF}, false}, // word → long, sign-extend
+	"MOVLQSX": {[]byte{0x63}, true},        // long → quad, sign-extend (MOVSXD)
+}
+
+// encodeMovExtend encodes a mixed-width extending move: reg = dst (the wider
+// operand), rm = src.
+func (e *enc) encodeMovExtend(base string, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("%s expects 2 operands, got %d", base, len(ops))
+	}
+	spec := movExtendOp[base]
+	dstReg, ok := ops[1].(Reg)
+	if !ok {
+		return fmt.Errorf("%s destination must be a register", base)
+	}
+	size := 4
+	if spec.dst64 {
+		size = 8
+	}
+	i := newInstr(size, spec.op)
+	if err := setRM(i, dstReg, ops[0], size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
+
+// --- CVTSL2SD / CVTSQ2SD -----------------------------------------------------
+
+// encodeCvtsi2sd encodes a signed integer to scalar double conversion
+// (CVTSL2SD from a 32-bit, CVTSQ2SD from a 64-bit source): F2 0F 2A with
+// reg = XMM dst, rm = GPR/memory src.  The Go assembler emits the legacy SSE
+// encoding here, not the VEX form, so we match it byte for byte.
+func (e *enc) encodeCvtsi2sd(quad bool, ops []Operand) error {
+	if len(ops) != 2 {
+		return fmt.Errorf("CVTSx2SD expects 2 operands, got %d", len(ops))
+	}
+	src, dst := ops[0], ops[1]
+	dstReg, ok := dst.(Reg)
+	if !ok || !dstReg.isVec() {
+		return fmt.Errorf("CVTSx2SD destination must be a vector register")
+	}
+	size := 4
+	if quad {
+		size = 8
+	}
+	i := newInstr(size, []byte{0x0F, 0x2A})
+	i.prefix = 0xF2
+	if err := setRM(i, dstReg, src, size); err != nil {
+		return err
+	}
+	return e.emit(i)
+}
@@ -0,0 +1,162 @@
+// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
+// SPDX-License-Identifier: BSD-3-Clause
+
+package asm
+
+import (
+	"fmt"
+
+	"sourcedock.dev/petrbalvin/gasm-devkit/ast"
+)
+
+// Image is an assembled file: the function bodies laid out in source order,
+// followed by the file's static data section (GLOBL/DATA).  Static-symbol
+// references are encoded RIP-relative and resolved within the image, so the
+// bytes are self-consistent and executable at any base address.
+type Image struct {
+	Code    []byte         // concatenated function bodies
+	Data    []byte         // static data section
+	Funcs   []FuncLayout   // function positions, in source order
+	Symbols map[string]int // static symbol → byte offset within the image
+}
+
+// FuncLayout describes one assembled function within an Image.
+type FuncLayout struct {
+	Name   string
+	Offset int // start offset within the image (== offset within Code)
+	Size   int
+	Labels map[string]int // local labels, function-relative
+}
+
+// Bytes returns the whole image: code, then data.
+func (img *Image) Bytes() []byte {
+	out := make([]byte, 0, len(img.Code)+len(img.Data))
+	out = append(out, img.Code...)
+	return append(out, img.Data...)
+}
+
+// AssembleFile assembles every TEXT function of a parsed file and lays out
+// its static symbols (GLOBL/DATA) in a data section behind the code.  Each
+// static-symbol reference becomes a RIP-relative load whose displacement is
+// resolved against that layout.  External (non-file-local) symbol references
+// are rejected: they need object-file emission.
+func AssembleFile(f *ast.File) (*Image, error) {
+	syms, order, err := collectData(f)
+	if err != nil {
+		return nil, err
+	}
+	known := make(map[string]bool, len(syms))
+	for name := range syms {
+		known[name] = true
+	}
+	link := &linkInfo{symbols: known}
+
+	img := &Image{Symbols: map[string]int{}}
+	type asmFunc struct {
+		name    string
+		patches []sbPatch
+	}
+	var funcs []asmFunc
+	for _, d := range f.Decls {
+		t, ok := d.(*ast.Text)
+		if !ok {
+			continue
+		}
+		code, patches, labels, err := assemble(t, link)
+		if err != nil {
+			return nil, fmt.Errorf("%s: %w", t.Name.Name, err)
+		}
+		img.Funcs = append(img.Funcs, FuncLayout{
+			Name:   t.Name.Name,
+			Offset: len(img.Code),
+			Size:   len(code),
+			Labels: labels,
+		})
+		img.Code = append(img.Code, code...)
+		funcs = append(funcs, asmFunc{name: t.Name.Name, patches: patches})
+	}
+
+	// Lay out the data section behind the code, each symbol 16-aligned.
+	dataStart := len(img.Code)
+	for _, name := range order {
+		if pos := dataStart + len(img.Data); pos != align16(pos) {
+			img.Data = append(img.Data, make([]byte, align16(pos)-pos)...)
+		}
+		img.Symbols[name] = dataStart + len(img.Data)
+		img.Data = append(img.Data, syms[name]...)
+	}
+
+	// Resolve the RIP-relative displacements now that every address is known.
+	for i, fn := range funcs {
+		base := img.Funcs[i].Offset
+		code := img.Code[base : base+img.Funcs[i].Size]
+		for _, p := range fn.patches {
+			rel := int64(img.Symbols[p.name]) + p.addend - int64(base+p.after)
+			if rel < -1<<31 || rel >= 1<<31 {
+				return nil, fmt.Errorf("%s: displacement to %q out of rel32 range", fn.name, p.name)
+			}
+			copy(code[p.off:p.off+4], le32(rel))
+		}
+	}
+	return img, nil
+}
+
+// collectData gathers the file's static symbols (GLOBL) and their initial
+// contents (DATA) into byte buffers, in declaration order.
+func collectData(f *ast.File) (map[string][]byte, []string, error) {
+	syms := map[string][]byte{}
+	var order []string
+	for _, d := range f.Decls {
+		switch dd := d.(type) {
+		case *ast.Globl:
+			if dd.Name == nil || dd.Name.Pseudo != "SB" {
+				continue
+			}
+			name := dd.Name.Name
+			if _, dup := syms[name]; dup {
+				return nil, nil, fmt.Errorf("duplicate GLOBL %q", name)
+			}
+			size := 0
+			if dd.Size != nil && dd.Size.Imm.HasVal {
+				size = int(dd.Size.Imm.Val)
+			}
+			syms[name] = make([]byte, size)
+			order = append(order, name)
+
+		case *ast.Data:
+			if dd.Name == nil || dd.Name.Pseudo != "SB" {
+				continue
+			}
+			buf, ok := syms[dd.Name.Name]
+			if !ok {
+				return nil, nil, fmt.Errorf("DATA %q: no matching GLOBL", dd.Name.Name)
+			}
+			if dd.Value == nil || !dd.Value.Imm.HasVal {
+				return nil, nil, fmt.Errorf("DATA %q: value must be an integer immediate", dd.Name.Name)
+			}
+			w := dd.Width
+			switch w {
+			case 1, 2, 4, 8:
+			default:
+				return nil, nil, fmt.Errorf("DATA %q: invalid width %d (want 1, 2, 4 or 8)", dd.Name.Name, w)
+			}
+			off := dd.Name.Offset
+			if off < 0 || off+int64(w) > int64(len(buf)) {
+				return nil, nil, fmt.Errorf("DATA %q+%d/%d exceeds GLOBL size %d", dd.Name.Name, off, w, len(buf))
+			}
+			v := dd.Value.Imm.Val
+			if dd.Value.Imm.Neg {
+				v = -v
+			}
+			for i := 0; i < w; i++ {
+				buf[off+int64(i)] = byte(v >> (8 * i))
+			}
+		}
+	}
+	return syms, order, nil
+}
+
+// align16 rounds n up to the next multiple of 16.
+func align16(n int) int {
+	return (n + 15) &^ 15
+}
@@ -0,0 +1,196 @@
+// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
+// SPDX-License-Identifier: BSD-3-Clause
+
+package asm
+
+import (
+	"bytes"
+	"os"
+	"strings"
+	"testing"
+
+	"golang.org/x/arch/x86/x86asm"
+
+	"sourcedock.dev/petrbalvin/gasm-devkit/parser"
+)
+
+// TestAssembleFileStaticData checks the whole-image layout — code, padding
+// and the data section — and that the RIP-relative displacements of static
+// symbol loads resolve to the right bytes.
+func TestAssembleFileStaticData(t *testing.T) {
+	f, errs := parser.Parse("d_amd64.s", `
+#include "textflag.h"
+TEXT ·load(SB), NOSPLIT, $0
+	VMOVDQU mask<>(SB), X15
+	MOVL small<>(SB), AX
+	RET
+GLOBL mask<>(SB), RODATA, $16
+DATA mask<>+0(SB)/4, $0x80020100
+DATA mask<>+4(SB)/4, $0x80050403
+DATA mask<>+8(SB)/4, $0x80080706
+DATA mask<>+12(SB)/4, $0x800B0A09
+GLOBL small<>(SB), RODATA, $4
+DATA small<>+0(SB)/4, $0x1234
+`)
+	if len(errs) > 0 {
+		t.Fatalf("parse: %v", errs)
+	}
+	img, err := AssembleFile(f)
+	if err != nil {
+		t.Fatalf("AssembleFile: %v", err)
+	}
+
+	// Code (15 bytes) + 1 pad byte to align the data section to 16:
+	//   VMOVDQU mask<>(SB), X15   c5 7a 6f 3d 08 00 00 00   (disp = 16 − 8)
+	//   MOVL small<>(SB), AX      8b 05 12 00 00 00         (disp = 32 − 14)
+	//   RET                       c3
+	// Data: pad, mask (16 bytes), small (4 bytes).
+	want := "c57a6f3d080000008b0512000000c300" +
+		"000102800304058006070880090a0b80" +
+		"34120000"
+	if got := strings.ReplaceAll(hexBytes(img.Bytes()), " ", ""); got != want {
+		t.Errorf("image bytes:\n got  %s\n want %s", got, want)
+	}
+	if img.Symbols["mask"] != 16 || img.Symbols["small"] != 32 {
+		t.Errorf("symbol offsets = %v, want mask=16 small=32", img.Symbols)
+	}
+	if len(img.Funcs) != 1 || img.Funcs[0].Name != "load" || img.Funcs[0].Size != 15 {
+		t.Errorf("funcs = %+v", img.Funcs)
+	}
+}
+
+// TestAssembleFileErrors checks the static-symbol error paths.
+func TestAssembleFileErrors(t *testing.T) {
+	cases := []struct {
+		name string
+		src  string
+		want string // substring of the error
+	}{
+		{
+			"undefined symbol",
+			`
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	VMOVDQU nope<>(SB), X0
+	RET
+`,
+			"undefined symbol",
+		},
+		{
+			"DATA without GLOBL",
+			`
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	RET
+DATA orphan<>+0(SB)/4, $1
+`,
+			"no matching GLOBL",
+		},
+		{
+			"DATA exceeds size",
+			`
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	RET
+GLOBL tiny<>(SB), RODATA, $4
+DATA tiny<>+0(SB)/8, $1
+`,
+			"exceeds GLOBL size",
+		},
+		{
+			"DATA bad width",
+			`
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	RET
+GLOBL odd<>(SB), RODATA, $4
+DATA odd<>+0(SB)/3, $1
+`,
+			"invalid width",
+		},
+	}
+	for _, c := range cases {
+		f, errs := parser.Parse("e_amd64.s", c.src)
+		if len(errs) > 0 {
+			t.Fatalf("%s: parse: %v", c.name, errs)
+		}
+		if _, err := AssembleFile(f); err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("%s: error %v, want substring %q", c.name, err, c.want)
+		}
+	}
+
+	// A static-symbol operand is unresolvable in single-function assembly.
+	fn := firstText(t, `
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	MOVQ x<>(SB), AX
+	RET
+GLOBL x<>(SB), RODATA, $8
+DATA x<>+0(SB)/4, $1
+`)
+	if _, _, err := Assemble(fn); err == nil || !strings.Contains(err.Error(), "file-level assembly") {
+		t.Errorf("single-function SB: error %v, want a file-level-assembly error", err)
+	}
+}
+
+// TestAssembleGoFlacAVX2Kernel assembles the whole production AVX2 kernel —
+// all functions plus the file-local mask24 constant — and checks that every
+// static-symbol load resolves to the right bytes in the image.  Skipped when
+// the sibling repository is not checked out.
+func TestAssembleGoFlacAVX2Kernel(t *testing.T) {
+	path := "../../go-libraries/go-flac/avx2_amd64.s"
+	if _, err := os.Stat(path); err != nil {
+		t.Skip("go-libraries repository not present next to gasm-devkit")
+	}
+	src, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	f, errs := parser.Parse(path, string(src))
+	if len(errs) > 0 {
+		t.Fatalf("parse: %v", errs)
+	}
+	img, err := AssembleFile(f)
+	if err != nil {
+		t.Fatalf("AssembleFile: %v", err)
+	}
+	if len(img.Funcs) != 17 {
+		t.Errorf("functions = %d, want 17", len(img.Funcs))
+	}
+
+	// mask24 as the DATA directives define it.
+	mask := []byte{
+		0x00, 0x01, 0x02, 0x80, 0x03, 0x04, 0x05, 0x80,
+		0x06, 0x07, 0x08, 0x80, 0x09, 0x0a, 0x0b, 0x80,
+	}
+	image := img.Bytes()
+	if got := image[img.Symbols["mask24"] : img.Symbols["mask24"]+16]; !bytes.Equal(got, mask) {
+		t.Errorf("mask24 contents %x, want %x", got, mask)
+	}
+
+	// Every VMOVDQU mask24<>(SB), X15 (c5 7a 6f 3d + rel32, i.e. a VMOVDQU
+	// with a RIP-relative r/m) must land on the mask bytes within the image.
+	loads := 0
+	for _, fn := range img.Funcs {
+		code := img.Code[fn.Offset : fn.Offset+fn.Size]
+		for pc := 0; pc < len(code); {
+			inst, err := x86asm.Decode(code[pc:], 64)
+			if err != nil {
+				t.Fatalf("%s: decode at +%d: %v", fn.Name, pc, err)
+			}
+			// mod=00, rm=101 → RIP-relative.
+			if inst.Op == x86asm.VMOVDQU && inst.Len == 8 && code[pc+3]&0xC7 == 0x05 {
+				rel := int32(uint32(code[pc+4]) | uint32(code[pc+5])<<8 | uint32(code[pc+6])<<16 | uint32(code[pc+7])<<24)
+				target := fn.Offset + pc + 8 + int(rel)
+				if !bytes.Equal(image[target:target+16], mask) {
+					t.Errorf("%s: mask load at +%d lands on %x, want %x", fn.Name, pc, image[target:target+16], mask)
+				}
+				loads++
+			}
+			pc += inst.Len
+		}
+	}
+	if loads != 2 {
+		t.Errorf("mask loads found = %d, want 2", loads)
+	}
+}
@@ -41,3 +41,15 @@ func Idx(base, index Reg, scale int, disp int64, size int) Mem {
 func Rip(disp int64, size int) Mem {
 	return Mem{Disp: disp, Size: size}
 }
+
+// sbMem is a memory operand that references a static (SB) symbol.  It encodes
+// as a RIP-relative reference with a placeholder displacement; the encoder
+// records a patch site so the file-level layout can fill in the true rel32
+// once the symbol's address is known.
+type sbMem struct {
+	size   int
+	name   string // static symbol name (the GLOBL identifier)
+	addend int64  // byte offset within the symbol
+}
+
+func (sbMem) isOperand() {}
@@ -133,6 +133,17 @@ var vexTable = map[string]vexSpec{
 	"VZEROUPPER": {1, 0x77, 0, 0, -1, vexZero},
 }

+// vexVarShift maps the shift mnemonics to their variable-count opcode — the
+// form whose count comes from an XMM register or memory (VPSRLQ X0, Y8, Y8),
+// an ordinary NDS encoding rather than the /digit immediate form above.
+var vexVarShift = map[string]byte{
+	"VPSLLD": 0xF2,
+	"VPSLLQ": 0xF3,
+	"VPSRAD": 0xE2,
+	"VPSRLD": 0xD2,
+	"VPSRLQ": 0xD3,
+}
+
 // vexMoveSpec describes a VEX move, which takes different opcodes (and
 // sometimes a different VEX.W) per operand direction.  The Go assembler
 // encodes a vector→vector move with the store-form opcode (reg = source,
@@ -180,6 +191,17 @@ func (e *enc) encodeVex(mnemUpper string, ops []Operand) error {
 	if ms, ok := vexMoveTable[mnemUpper]; ok {
 		return e.encodeVexMove(mnemUpper, ms, ops)
 	}
+	// The shifts come in two shapes under one mnemonic: an immediate count
+	// ($imm, src, dst) and a variable count in an XMM register or memory
+	// (count, src, dst), the latter an ordinary NDS form.
+	if op, ok := vexVarShift[mnemUpper]; ok && len(ops) == 3 {
+		if _, isImm := ops[0].(Imm); !isImm {
+			if !vecOrMem(ops[0]) {
+				return fmt.Errorf("%s: shift count must be an immediate, a vector register or memory", mnemUpper)
+			}
+			return e.encodeVexNDS3(vexSpec{mapSel: 1, opcode: op, pp: 1, opdigit: -1, form: vexNDS3}, ops)
+		}
+	}
 	spec := vexTable[mnemUpper]
 	switch spec.form {
 	case vexNDS3:
@@ -483,11 +505,21 @@ func vecReg(op Operand) (Reg, bool) {
 	return r, ok && r.isVec()
 }

+// vecOrMem reports whether op is a vector register or a memory reference.
+func vecOrMem(op Operand) bool {
+	switch op.(type) {
+	case Mem, sbMem:
+		return true
+	}
+	r, ok := op.(Reg)
+	return ok && r.isVec()
+}
+
 // validMoveOther reports whether the non-vector operand of a move is
 // acceptable: memory always is, a GPR only for VMOVD/VMOVQ.
 func validMoveOther(ms vexMoveSpec, op Operand) bool {
 	switch o := op.(type) {
-	case Mem:
+	case Mem, sbMem:
 		return true
 	case Reg:
 		return ms.gprOK && !o.isVec()
@@ -502,6 +534,7 @@ func (e *enc) emitVexFields(spec vexSpec, l, regField, rBit, vvvvBar int, rm Ope
 	var modrm, sib int
 	var disp []byte
 	var xBit, bBit int
+	var sb *sbRef
 	switch r := rm.(type) {
 	case Reg:
 		modrm = 0xC0 | regField<<3 | (r.idx & 7)
@@ -515,6 +548,12 @@ func (e *enc) emitVexFields(spec vexSpec, l, regField, rBit, vvvvBar int, rm Ope
 		if err != nil {
 			return err
 		}
+	case sbMem:
+		// RIP-relative static-symbol reference; disp32 patched at link time.
+		modrm = regField<<3 | 0x05
+		sib = -1
+		disp = le32(0)
+		sb = &sbRef{name: r.name, addend: r.addend}
 	default:
 		return fmt.Errorf("invalid VEX r/m operand")
 	}
@@ -530,6 +569,9 @@ func (e *enc) emitVexFields(spec vexSpec, l, regField, rBit, vvvvBar int, rm Ope
 	if sib >= 0 {
 		e.out = append(e.out, byte(sib))
 	}
+	if sb != nil {
+		e.patches = append(e.patches, encPatch{off: len(e.out), name: sb.name, addend: sb.addend})
+	}
 	e.out = append(e.out, disp...)
 	return nil
 }
@@ -191,6 +191,14 @@ func TestVexGroundTruth(t *testing.T) {
 		// Immediate shifts.
 		{"VPSLLD $1,Y3,Y4", "VPSLLD", []Operand{Imm(1), vreg(t, "Y3"), vreg(t, "Y4")}, "c5dd72f301"},
 		{"VPSRLQ $2,Y5,Y6", "VPSRLQ", []Operand{Imm(2), vreg(t, "Y5"), vreg(t, "Y6")}, "c5cd73d502"},
+		// Variable-count shifts: the count lives in an XMM register or memory
+		// and the instruction takes the NDS form.
+		{"VPSRLQ X0,Y8,Y8", "VPSRLQ", []Operand{vreg(t, "X0"), vreg(t, "Y8"), vreg(t, "Y8")}, "c53dd3c0"},
+		{"VPSRLQ (AX),Y8,Y8", "VPSRLQ", []Operand{Ptr(AX, 0, 16), vreg(t, "Y8"), vreg(t, "Y8")}, "c53dd300"},
+		{"VPSLLD X0,Y1,Y2", "VPSLLD", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5f2d0"},
+		{"VPSRLD X0,Y1,Y2", "VPSRLD", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5d2d0"},
+		{"VPSRAD X0,Y1,Y2", "VPSRAD", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5e2d0"},
+		{"VPSLLQ X0,Y1,Y2", "VPSLLQ", []Operand{vreg(t, "X0"), vreg(t, "Y1"), vreg(t, "Y2")}, "c5f5f3d0"},
 		// Immediate shuffle (reg=dst, rm=src, imm8).
 		{"VPSHUFD $0xEE,X8,X9", "VPSHUFD", []Operand{Imm(0xEE), vreg(t, "X8"), vreg(t, "X9")}, "c4417970c8ee"},
 		{"VPSHUFD $0xEE,Y1,Y2", "VPSHUFD", []Operand{Imm(0xEE), vreg(t, "Y1"), vreg(t, "Y2")}, "c5fd70d1ee"},
@@ -26,7 +26,7 @@ import (

 // version is the release version, stamped at build time via
 // -ldflags "-X main.version=…" (defaulting to the current release).
-var version = "0.2.0"
+var version = "0.4.0"

 func main() {
 	if len(os.Args) < 2 {
@@ -238,20 +238,18 @@ func cmdAsm(args []string) int {
 		return 1
 	}

-	var all []byte
-	functions := 0
-	for _, d := range f.Decls {
-		txt, ok := d.(*ast.Text)
-		if !ok {
-			continue
-		}
-		code, _, err := asm.Assemble(txt)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "%s: %s: %v\n", path, txt.Name.Name, err)
-			return 1
-		}
-		functions++
-		fmt.Printf("%s: %d bytes\n", txt.Name.Name, len(code))
+	img, err := asm.AssembleFile(f)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "%s: %v\n", path, err)
+		return 1
+	}
+	if len(img.Funcs) == 0 {
+		fmt.Fprintln(os.Stderr, "gasm asm: no assemblable TEXT functions found")
+		return 1
+	}
+	for _, fn := range img.Funcs {
+		code := img.Code[fn.Offset : fn.Offset+fn.Size]
+		fmt.Printf("%s: %d bytes\n", fn.Name, fn.Size)
 		for i := 0; i < len(code); i += 16 {
 			end := i + 16
 			if end > len(code) {
@@ -263,13 +261,34 @@ func cmdAsm(args []string) int {
 			}
 			fmt.Println()
 		}
-		all = append(all, code...)
 	}
-	if functions == 0 {
-		fmt.Fprintln(os.Stderr, "gasm asm: no assemblable TEXT functions found")
-		return 1
+	if len(img.Data) > 0 {
+		fmt.Printf("data: %d bytes at 0x%x\n", len(img.Data), len(img.Code))
+		for _, d := range f.Decls {
+			g, ok := d.(*ast.Globl)
+			if !ok || g.Name == nil || g.Name.Pseudo != "SB" {
+				continue
+			}
+			size := 0
+			if g.Size != nil && g.Size.Imm.HasVal {
+				size = int(g.Size.Imm.Val)
+			}
+			fmt.Printf("  %s: %d bytes at 0x%x\n", g.Name.Name, size, img.Symbols[g.Name.Name])
+		}
+		for i := 0; i < len(img.Data); i += 16 {
+			end := i + 16
+			if end > len(img.Data) {
+				end = len(img.Data)
+			}
+			fmt.Printf("  %04x:", len(img.Code)+i)
+			for _, b := range img.Data[i:end] {
+				fmt.Printf(" %02x", b)
+			}
+			fmt.Println()
+		}
 	}
 	if *out != "" {
+		all := img.Bytes()
 		if err := os.WriteFile(*out, all, 0o644); err != nil {
 			fmt.Fprintln(os.Stderr, "gasm asm:", err)
 			return 1
@@ -182,33 +182,45 @@ Every encoding is validated by decoding it again with `golang.org/x/arch` — th
 one module dependency, used in tests only and never linked into the binary.

 On top of the encoder, `Assemble` walks a parsed `TEXT` body, converts each
-operand to an encoder operand, and lays the instructions out in two passes so
-local labels resolve to fixed rel32 jump offsets.  The `FP`/`SP` pseudo-
+operand to an encoder operand, and lays the instructions out so local labels
+resolve to relative jump offsets: jumps start in the short (rel8) form and
+expand to rel32 when the settled displacement does not fit, iterating to a
+fixed point, and jump-to-jump chains are folded (a conditional jump to a label
+whose only instruction is an unconditional jump is redirected to the ultimate
+target) exactly as the Go toolchain's linker does before it encodes branches.
+The `FP`/`SP` pseudo-
 registers are translated onto the hardware stack pointer — `x+N(FP)` becomes
 `(N+8)(SP)` for a zero-frame function and `(N+frame+16)(SP)` once a frame
 pointer is set up, with the matching Go prologue/epilogue generated — so the
 output is byte-identical to the Go assembler for these cases.  SIMD is handled
 by a VEX (AVX/AVX2) encoder — the two- and three-byte VEX prefixes with XMM/YMM
-registers — across seven operand forms: the three-operand NDS form, the
-two-operand reg/rm form, the immediate-shift form, the immediate shuffle form
-(`VPSHUFD`, `VPERMQ`), the three-operand-plus-immediate form (`VSHUFPD`,
+registers — across eight operand forms: the three-operand NDS form, the
+two-operand reg/rm form, the immediate-shift form (plus the variable-count
+shifts, which share the NDS shape with the count in an XMM register or
+memory), the immediate shuffle form (`VPSHUFD`, `VPERMQ`), the
+three-operand-plus-immediate form (`VSHUFPD`,
 `VPERM2I128`, `VINSERTI128`), the lane-extract form (`VEXTRACTI128`,
-lane-extract form (`VEXTRACTI128`,
 `VEXTRACTF128`, where the YMM source occupies the reg field and the XMM or
 memory destination r/m), the direction-sensitive moves (`VMOVDQU`, `VMOVUPD`,
 `VMOVD`, `VMOVQ`, `VMOVSD`), the floating-point and FMA arithmetic (`VADDPD`,
 `VMULPD`, `VXORPD`, `VUNPCKHPD`, the scalar `VADDSD`/`VMULSD`, `VCVTDQ2PD`,
-`VFMADD231PD`) and the no-operand `VZEROUPPER` — together with `VPERMD`,
-covering every integer, shuffle and FP instruction the go-flac AVX2 kernels
-use.  Every encoding is validated two ways: by round-trip decoding
-through `golang.org/x/arch`, and byte-for-byte against the machine code the
-real Go assembler emits (which also locks the v̄vvv = 1111 rule for unused
-vvvv fields — a value the hardware rejects with #UD and the decoder silently
-ignores).  This increment covers register / memory / immediate / FP-frame
-operands, local-label jumps and these VEX SIMD forms; EVEX / AVX-512, `SB`
-(global symbol) operands (relocations), a handful of scalar gaps the kernels
-hit (`CMOVcc`, `SETcc`, `LZCNT`, `MOVSX`/`MOVZX`) and object-file emission
-are the rest of Phase 2.
+`VFMADD231PD`) and the no-operand `VZEROUPPER` — together with `VPERMD` and
+the scalar families (`CMOVcc`, `SETcc`, `LZCNT`/`TZCNT`, the extending moves,
+`CVTSx2SD`, `IMUL3`), covering every instruction the go-flac AVX2 kernels
+use.  Every encoding is validated two ways: by round-trip decoding through
+`golang.org/x/arch`, and byte-for-byte against the machine code the real Go
+assembler emits — a comparison that holds for the whole kernel: all 17
+functions of the go-flac AVX2 file assemble to exactly the Go toolchain's
+bytes, the lone exception being the displacements of the static-constant
+loads, which the Go linker fills at link time.
+
+File-level assembly (`AssembleFile`) goes beyond single functions: it
+materialises the file's static symbols (`GLOBL`/`DATA`) in a data section
+behind the code and resolves references to them (`mask<>(SB)`) to
+RIP-relative loads whose displacements point inside the resulting image, so
+the bytes are self-consistent at any base address.  External (non-file-local)
+symbols are rejected: they need object-file emission, which — together with
+EVEX / AVX-512 and the other architectures — is the rest of Phase 2.

 ## Extension points

@@ -3,7 +3,7 @@

 # gasm-devkit — developer tooling for Go's Plan 9 assembler (GAsm).

-version := "0.2.0"
+version := "0.4.0"

 default:
    @just --list
@@ -400,22 +400,29 @@ func parseAddress(g []token.Token) ast.Address {
 	}

 	i := 0
-	// Optional leading displacement before a '(' base group.
-	if isSignedNumber(g, i) && i+1 < len(g) && g[i+1].Kind == token.LParen {
-		neg := false
-		if g[i].Kind == token.Minus {
-			neg = true
-			i++
-		} else if g[i].Kind == token.Plus {
-			i++
+	// Optional leading displacement before a '(' base group.  A sign pushes
+	// the parenthesis one token further out: -4(DX) has it at i+2.
+	if isSignedNumber(g, i) {
+		paren := i + 1
+		if g[i].Kind == token.Minus || g[i].Kind == token.Plus {
+			paren = i + 2
 		}
-		if i < len(g) && g[i].Kind == token.Number {
-			addr.Offset = parseInt(g[i].Text)
-			addr.HasOff = true
-			if neg {
-				addr.Offset = -addr.Offset
+		if paren < len(g) && g[paren].Kind == token.LParen {
+			neg := false
+			if g[i].Kind == token.Minus {
+				neg = true
+				i++
+			} else if g[i].Kind == token.Plus {
+				i++
+			}
+			if i < len(g) && g[i].Kind == token.Number {
+				addr.Offset = parseInt(g[i].Text)
+				addr.HasOff = true
+				if neg {
+					addr.Offset = -addr.Offset
+				}
+				i++
 			}
-			i++
 		}
 	}
 	// First parenthesised group: the base register.
@@ -34,6 +34,46 @@ func texts(f *ast.File) []*ast.Text {
 	return out
 }

+// TestNegativeDisplacement is a regression test for a leading negative
+// displacement with a base and index: the sign pushed the parenthesis one
+// token further out than the lookahead expected, and the whole address used
+// to parse empty.
+func TestNegativeDisplacement(t *testing.T) {
+	f, errs := Parse("neg_amd64.s", `
+#include "textflag.h"
+TEXT ·f(SB), NOSPLIT, $0
+	LEAQ -4(DX)(R9*4), R9
+	MOVQ +8(AX), BX
+	RET
+`)
+	if len(errs) > 0 {
+		t.Fatalf("parse: %v", errs)
+	}
+	fn := texts(f)[0]
+	var leaq, movq *ast.Instr
+	for _, s := range fn.Body {
+		if in, ok := s.(*ast.Instr); ok {
+			switch in.Mnemonic.Text {
+			case "LEAQ":
+				leaq = in
+			case "MOVQ":
+				movq = in
+			}
+		}
+	}
+	if leaq == nil || movq == nil {
+		t.Fatalf("instructions not parsed: leaq=%v movq=%v", leaq, movq)
+	}
+	a := leaq.Operands[0].Addr
+	if a.Base != "DX" || a.Index != "R9" || a.Scale != 4 || a.Offset != -4 || !a.HasOff {
+		t.Errorf("LEAQ addr = %+v, want -4(DX)(R9*4)", a)
+	}
+	b := movq.Operands[0].Addr
+	if b.Base != "AX" || b.Offset != 8 || !b.HasOff {
+		t.Errorf("MOVQ addr = %+v, want +8(AX)", b)
+	}
+}
+
 func TestParseSample(t *testing.T) {
 	f := mustParse(t, "../testdata/sample_amd64.s")
Author	SHA1	Message	Date
petrbalvin	56ecc39539	feat(asm): assemble static symbols and the whole go-flac AVX2 kernel byte-identically Assisted-by: Qwen 3.8 Max Preview	2026-07-09 15:56:03 +02:00
petrbalvin	a82f575aee	feat(asm): byte-identical go-flac AVX2 assembly with scalar families and jump relaxation Assisted-by: Qwen 3.8 Max Preview	2026-07-08 12:51:35 +02:00