feat(arch): add the amd64 memory-operand mechanism to the extension layer

Assisted-by: GLM 5.3 Flash

arch/amd64_ext.go

@@ -115,6 +115,68 @@ func amd64Encode(b []byte, dest, vvvv, rm int) []byte {
 	return out
 }
 
+// amd64Memory validates a memory operand of an amd64 entry: no arrangement
+// and no qualifier, a base general register inside 0-15, a signed 32-bit
+// displacement and no shift.  The base number rides the operand's Reg and
+// the displacement its Imm.
+func (in ExtInstr) amd64Memory(op ExtOperand, pos int) (base int, disp int64, err error) {
+	if op.Kind != ExtMem {
+		return 0, 0, fmt.Errorf("%s: operand %d wants a memory operand, got %s", in.Name, pos, op.Kind)
+	}
+	if op.Arr != ExtArrNone {
+		return 0, 0, fmt.Errorf("%s: operand %d carries an arrangement suffix, the amd64 layer takes none", in.Name, pos)
+	}
+	if op.Qual != ExtQualNone {
+		return 0, 0, fmt.Errorf("%s: operand %d carries a predicate qualifier, the amd64 layer takes none", in.Name, pos)
+	}
+	if op.HasShift {
+		return 0, 0, fmt.Errorf("%s: operand %d carries a shift, the amd64 memory forms take none", in.Name, pos)
+	}
+	if op.Reg < 0 || op.Reg > 15 {
+		return 0, 0, fmt.Errorf("%s: operand %d names base register %d, outside 0-15", in.Name, pos, op.Reg)
+	}
+	if op.Imm < -1<<31 || op.Imm >= 1<<31 {
+		return 0, 0, fmt.Errorf("%s: operand %d carries displacement %d, outside the signed 32-bit range", in.Name, pos, op.Imm)
+	}
+	return op.Reg, op.Imm, nil
+}
+
+// amd64EncodeMemory returns the register-form template with a base-relative
+// memory operand filled in: dest and vvvv keep their register meanings, the
+// ModR/M r/m field carries the base, and the high base bit rides EVEX.B as
+// amd64Encode lays it.  The ModR/M mod bits and the trailing SIB and
+// displacement bytes follow the canonical choices the GNU assembler makes
+// for the plain, unscaled SDM displacements: no displacement bytes at
+// displacement zero, a disp8 when the value fits a signed byte and a disp32
+// otherwise, the SIB byte 0x24 when the base is RSP or R12, whose r/m
+// encoding 100 demands it, and a forced displacement on RBP and R13, whose
+// mod-00 r/m encoding 101 means RIP-relative.  The operand must have passed
+// amd64Memory first.
+func amd64EncodeMemory(b []byte, dest, vvvv, base int, disp int64) []byte {
+	out := amd64Encode(b, dest, vvvv, base)
+	rm := base & 7
+	var tail []byte
+	mod := byte(0)
+	switch {
+	case rm == 5 || disp != 0:
+		// RBP and R13 cannot drop the displacement: mod 00 with r/m 101
+		// addresses RIP-relative, not through the base.
+		if disp >= -128 && disp <= 127 {
+			mod = 1
+			tail = []byte{byte(disp)}
+		} else {
+			mod = 2
+			tail = []byte{byte(disp), byte(disp >> 8), byte(disp >> 16), byte(disp >> 24)}
+		}
+	}
+	if rm == 4 {
+		// RSP and R12 need the SIB byte: no index, base 100.
+		tail = append([]byte{0x24}, tail...)
+	}
+	out[5] = out[5]&0x3f | mod<<6
+	return append(out, tail...)
+}
+
 // amd64PlainReg checks the invariants every amd64 register operand carries:
 // no arm64 arrangement, no predicate qualifier, and a register number inside
 // the class the instruction encodes.

arch/amd64_ext_mem_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
+// SPDX-License-Identifier: BSD-3-Clause
+
+package arch
+
+import (
+	"encoding/hex"
+	"strings"
+	"testing"
+)
+
+// The memory-operand mechanism: the base-plus-displacement validation and
+// the ModR/M, SIB and displacement byte choices.  The expected words are
+// built on the VMOVSH memory-load template, whose rows the binutils-gdb
+// assembler testsuite quotes byte for byte; the rows marked GNU match the
+// listing bytes.  Note on the quoted disp8 rows: binutils mainline encodes
+// EVEX displacements with the APX disp8*N scaling, so its source spellings
+// (254 for the m16 rows, 8128 for the m512 ones) are N times the plain SDM
+// displacement the disp8 bytes carry; the words here pin the bytes with the
+// plain displacement those bytes encode.
+func TestAmd64ExtMemoryEncoding(t *testing.T) {
+	load := []byte{0x62, 0x05, 0x06, 0x00, 0x10, 0xC0}
+	for _, tt := range []struct {
+		name      string
+		base      int
+		disp      int64
+		want      string
+		gnuSource string // the binutils source line the bytes serve, empty for a derived row
+	}{
+		{"zero displacement drops the disp bytes", 9, 0,
+			"62457e081031", "vmovsh (%r9),%xmm30"},
+		{"positive disp8", 1, 127,
+			"62657e0810717f", "vmovsh 254(%rcx),%xmm30 (Disp8(7f) under the disp8*N scaling)"},
+		{"negative disp8", 2, -128,
+			"62657e08107280", "vmovsh -256(%rdx),%xmm30 (Disp8(80) under the disp8*N scaling)"},
+		{"disp32 past the disp8 range", 2, 8128,
+			"62657e0810b2c01f0000", ""},
+		{"negative disp32", 13, -200,
+			"62457e0810b538ffffff", ""},
+		{"RSP base takes the SIB byte", 12, 0,
+			"62457e08103424", ""},
+		{"RSP base with a disp8", 12, 4,
+			"62457e0810742404", ""},
+		{"RBP base keeps a zero displacement explicit", 5, 0,
+			"62657e08107500", ""},
+	} {
+		got := amd64EncodeMemory(load, 30, -1, tt.base, tt.disp)
+		if hex.EncodeToString(got) != tt.want {
+			t.Errorf("%s:\n  got  %x\n  want %s", tt.name, got, tt.want)
+		}
+	}
+}
+
+// TestAmd64ExtMemoryRejects checks the validation around the memory operand:
+// the kinds and ranges the layer refuses before a byte is laid down.
+func TestAmd64ExtMemoryRejects(t *testing.T) {
+	in := ExtInstr{Name: "TEST"}
+	for _, tt := range []struct {
+		name  string
+		op    ExtOperand
+		quote string
+	}{
+		{"a register where the memory operand belongs", ExtXmm(3),
+			"wants a memory operand"},
+		{"base beyond r15", ExtMemory(16, 0),
+			"outside 0-15"},
+		{"base under r0", ExtMemory(-1, 0),
+			"outside 0-15"},
+		{"displacement past the signed 32-bit ceiling", ExtMemory(8, 1<<31),
+			"outside the signed 32-bit range"},
+		{"displacement past the signed 32-bit floor", ExtMemory(8, -1<<31-1),
+			"outside the signed 32-bit range"},
+		{"shift on the memory operand", ExtOperand{Kind: ExtMem, Reg: 8, Imm: 4, Shift: 2, HasShift: true},
+			"take none"},
+		{"arrangement suffix on the memory operand", ExtOperand{Kind: ExtMem, Reg: 8, Arr: ExtArrH},
+			"arrangement"},
+		{"predicate qualifier on the memory operand", ExtOperand{Kind: ExtMem, Reg: 8, Qual: ExtQualZeroing},
+			"predicate qualifier"},
+	} {
+		_, _, err := in.amd64Memory(tt.op, 1)
+		if err == nil {
+			t.Errorf("%s: validation succeeded, want an error", tt.name)
+			continue
+		}
+		if !strings.Contains(err.Error(), tt.quote) {
+			t.Errorf("%s: error %q lacks %q", tt.name, err, tt.quote)
+		}
+	}
+}
+
+// TestAmd64ExtMemoryVocabulary pins the names the shared layer gives the
+// memory operand and its two forms.
+func TestAmd64ExtMemoryVocabulary(t *testing.T) {
+	if got := ExtMem.String(); got != "memory operand" {
+		t.Errorf("ExtMem = %q, want %q", got, "memory operand")
+	}
+	if got := ExtFormAmdMemVec.String(); got != "memory into a vector" {
+		t.Errorf("ExtFormAmdMemVec = %q, want %q", got, "memory into a vector")
+	}
+	if got := ExtFormAmdVecMem.String(); got != "a vector into memory" {
+		t.Errorf("ExtFormAmdVecMem = %q, want %q", got, "a vector into memory")
+	}
+	for _, f := range []ExtForm{ExtFormAmdMemVec, ExtFormAmdVecMem} {
+		if got := f.Arity(); got != 2 {
+			t.Errorf("%s takes %d operands, want 2", f, got)
+		}
+	}
+	if got := ExtMemory(9, 4096); got.Kind != ExtMem || got.Reg != 9 || got.Imm != 4096 {
+		t.Errorf("ExtMemory(9, 4096) = %+v, want base 9 with displacement 4096", got)
+	}
+}

arch/arm64_ext.go

@@ -34,6 +34,7 @@ const (
 	ExtKReg // opmask register K0-K7
 	ExtR32  // 32-bit general register EAX-R15D
 	ExtR64  // 64-bit general register RAX-R15
+	ExtMem  // base-relative memory operand, 4660(R8) style
 )
 
 // String returns a short label for the kind.
@@ -57,6 +58,8 @@ func (k ExtOperandKind) String() string {
 		return "32-bit general register"
 	case ExtR64:
 		return "64-bit general register"
+	case ExtMem:
+		return "memory operand"
 	default:
 		return "operand"
 	}
@@ -153,10 +156,14 @@ func (q ExtQualifier) String() string {
 // from the parsed statement; Encode consumes them.
 type ExtOperand struct {
 	Kind ExtOperandKind
-	Reg  int            // register number (Z: 0..31, P: 0..15)
+	// Reg is the register number (Z: 0..31, P: 0..15); under ExtMem it is
+	// the base general register, 0..15.
+	Reg  int
 	Arr  ExtArrangement // element-size suffix; ExtArrNone when bare
 	Qual ExtQualifier   // predicate qualifier; ExtQualNone elsewhere
-	Imm  int64          // immediate value (ExtImm only)
+	// Imm is the immediate value under ExtImm; under ExtMem it is the
+	// signed displacement the base carries.
+	Imm int64
 	// Shift carries the LSL amount an immediate form shifts the constant by
 	// before use (0 or 8 in the SVE add/subtract immediate class).  HasShift
 	// separates a spelled shift (validated as written) from an unshifted
@@ -185,6 +192,14 @@ func ExtShiftedImmediate(v int64, shift int) ExtOperand {
 	return ExtOperand{Kind: ExtImm, Imm: v, Shift: shift, HasShift: true}
 }
 
+// ExtMemory builds a base-relative memory operand, 4660(R8) style: the base
+// is a 64-bit general register number, 0..15, and the displacement rides the
+// ModR/M disp8 or disp32 form the encoder picks.  No index register and no
+// scale: the base-plus-displacement shape alone.
+func ExtMemory(base int, disp int64) ExtOperand {
+	return ExtOperand{Kind: ExtMem, Reg: base, Imm: disp}
+}
+
 // ExtField is one named field of the 32-bit encoding word: a bit offset from
 // the least significant end and the field's width.
 type ExtField struct {
@@ -282,6 +297,13 @@ const (
 	// where dest is an opmask register and the immediate's layout is named
 	// by the entry's Imm8 kind.
 	ExtFormAmdMask2Imm
+	// ExtFormAmdMemVec is the memory-load form: VMOVSH X30, 4660(R8) shape,
+	// the manual's xmm1, m16 lines that stand beside the register form.
+	// Operands: mem, dest.
+	ExtFormAmdMemVec
+	// ExtFormAmdVecMem is the memory-store form: VMOVSH 4660(R9), X29
+	// shape, the manual's m16, xmm1 lines.  Operands: src, mem.
+	ExtFormAmdVecMem
 )
 
 // Arity returns the operand count the form takes.
@@ -297,6 +319,8 @@ func (f ExtForm) Arity() int {
 		return 2
 	case ExtFormAmdVec3Imm, ExtFormAmdMask2Imm:
 		return 4
+	case ExtFormAmdMemVec, ExtFormAmdVecMem:
+		return 2
 	default:
 		return 0
 	}
@@ -350,6 +374,10 @@ func (f ExtForm) String() string {
 		return "immediate, three vectors"
 	case ExtFormAmdMask2Imm:
 		return "immediate, two vectors into an opmask"
+	case ExtFormAmdMemVec:
+		return "memory into a vector"
+	case ExtFormAmdVecMem:
+		return "a vector into memory"
 	default:
 		return "unknown form"
 	}
@@ -431,6 +459,14 @@ type ExtInstr struct {
 	// operand carries, ExtImm8None when the form takes none.  The arm64
 	// entries all carry the zero value.
 	Imm8 ExtImm8Kind
+	// Mem names the 1-based operand position that may carry a memory
+	// operand beside the register the position normally takes: 2 on the
+	// arithmetic whose second source the manual spells xmm3/m16, 1 on the
+	// narrow BF16 convert whose source it spells m256/m512.  Zero means
+	// the form takes registers alone.  The load and store shapes are forms
+	// of their own, ExtFormAmdMemVec and ExtFormAmdVecMem, and need no
+	// flag.  The arm64 entries all carry the zero value.
+	Mem int
 }
 
 // Encode assembles the operands into the 4 little-endian bytes of the
@@ -453,7 +489,7 @@ func (in ExtInstr) Encode(ops []ExtOperand) ([]byte, error) {
 		return in.encodeSignedImmediate(ops)
 	case ExtFormAmdVec3, ExtFormAmdVec2, ExtFormAmdVec2Half, ExtFormAmdMask2,
 		ExtFormAmdVecGprVec, ExtFormAmdGprVec, ExtFormAmdVecGpr,
-		ExtFormAmdVec3Imm, ExtFormAmdMask2Imm:
+		ExtFormAmdVec3Imm, ExtFormAmdMask2Imm, ExtFormAmdMemVec, ExtFormAmdVecMem:
 		return in.encodeAmd64(ops)
 	default:
 		return nil, fmt.Errorf("%s: unknown form %d", in.Name, in.Form)