feat(arch): add the amd64 memory-operand mechanism to the extension layer

Assisted-by: GLM 5.3 Flash

arch/amd64_ext_mem_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2026 Petr Balvín <opensource@petrbalvin.org> (https://petrbalvin.org)
+// SPDX-License-Identifier: BSD-3-Clause
+
+package arch
+
+import (
+	"encoding/hex"
+	"strings"
+	"testing"
+)
+
+// The memory-operand mechanism: the base-plus-displacement validation and
+// the ModR/M, SIB and displacement byte choices.  The expected words are
+// built on the VMOVSH memory-load template, whose rows the binutils-gdb
+// assembler testsuite quotes byte for byte; the rows marked GNU match the
+// listing bytes.  Note on the quoted disp8 rows: binutils mainline encodes
+// EVEX displacements with the APX disp8*N scaling, so its source spellings
+// (254 for the m16 rows, 8128 for the m512 ones) are N times the plain SDM
+// displacement the disp8 bytes carry; the words here pin the bytes with the
+// plain displacement those bytes encode.
+func TestAmd64ExtMemoryEncoding(t *testing.T) {
+	load := []byte{0x62, 0x05, 0x06, 0x00, 0x10, 0xC0}
+	for _, tt := range []struct {
+		name      string
+		base      int
+		disp      int64
+		want      string
+		gnuSource string // the binutils source line the bytes serve, empty for a derived row
+	}{
+		{"zero displacement drops the disp bytes", 9, 0,
+			"62457e081031", "vmovsh (%r9),%xmm30"},
+		{"positive disp8", 1, 127,
+			"62657e0810717f", "vmovsh 254(%rcx),%xmm30 (Disp8(7f) under the disp8*N scaling)"},
+		{"negative disp8", 2, -128,
+			"62657e08107280", "vmovsh -256(%rdx),%xmm30 (Disp8(80) under the disp8*N scaling)"},
+		{"disp32 past the disp8 range", 2, 8128,
+			"62657e0810b2c01f0000", ""},
+		{"negative disp32", 13, -200,
+			"62457e0810b538ffffff", ""},
+		{"RSP base takes the SIB byte", 12, 0,
+			"62457e08103424", ""},
+		{"RSP base with a disp8", 12, 4,
+			"62457e0810742404", ""},
+		{"RBP base keeps a zero displacement explicit", 5, 0,
+			"62657e08107500", ""},
+	} {
+		got := amd64EncodeMemory(load, 30, -1, tt.base, tt.disp)
+		if hex.EncodeToString(got) != tt.want {
+			t.Errorf("%s:\n  got  %x\n  want %s", tt.name, got, tt.want)
+		}
+	}
+}
+
+// TestAmd64ExtMemoryRejects checks the validation around the memory operand:
+// the kinds and ranges the layer refuses before a byte is laid down.
+func TestAmd64ExtMemoryRejects(t *testing.T) {
+	in := ExtInstr{Name: "TEST"}
+	for _, tt := range []struct {
+		name  string
+		op    ExtOperand
+		quote string
+	}{
+		{"a register where the memory operand belongs", ExtXmm(3),
+			"wants a memory operand"},
+		{"base beyond r15", ExtMemory(16, 0),
+			"outside 0-15"},
+		{"base under r0", ExtMemory(-1, 0),
+			"outside 0-15"},
+		{"displacement past the signed 32-bit ceiling", ExtMemory(8, 1<<31),
+			"outside the signed 32-bit range"},
+		{"displacement past the signed 32-bit floor", ExtMemory(8, -1<<31-1),
+			"outside the signed 32-bit range"},
+		{"shift on the memory operand", ExtOperand{Kind: ExtMem, Reg: 8, Imm: 4, Shift: 2, HasShift: true},
+			"take none"},
+		{"arrangement suffix on the memory operand", ExtOperand{Kind: ExtMem, Reg: 8, Arr: ExtArrH},
+			"arrangement"},
+		{"predicate qualifier on the memory operand", ExtOperand{Kind: ExtMem, Reg: 8, Qual: ExtQualZeroing},
+			"predicate qualifier"},
+	} {
+		_, _, err := in.amd64Memory(tt.op, 1)
+		if err == nil {
+			t.Errorf("%s: validation succeeded, want an error", tt.name)
+			continue
+		}
+		if !strings.Contains(err.Error(), tt.quote) {
+			t.Errorf("%s: error %q lacks %q", tt.name, err, tt.quote)
+		}
+	}
+}
+
+// TestAmd64ExtMemoryVocabulary pins the names the shared layer gives the
+// memory operand and its two forms.
+func TestAmd64ExtMemoryVocabulary(t *testing.T) {
+	if got := ExtMem.String(); got != "memory operand" {
+		t.Errorf("ExtMem = %q, want %q", got, "memory operand")
+	}
+	if got := ExtFormAmdMemVec.String(); got != "memory into a vector" {
+		t.Errorf("ExtFormAmdMemVec = %q, want %q", got, "memory into a vector")
+	}
+	if got := ExtFormAmdVecMem.String(); got != "a vector into memory" {
+		t.Errorf("ExtFormAmdVecMem = %q, want %q", got, "a vector into memory")
+	}
+	for _, f := range []ExtForm{ExtFormAmdMemVec, ExtFormAmdVecMem} {
+		if got := f.Arity(); got != 2 {
+			t.Errorf("%s takes %d operands, want 2", f, got)
+		}
+	}
+	if got := ExtMemory(9, 4096); got.Kind != ExtMem || got.Reg != 9 || got.Imm != 4096 {
+		t.Errorf("ExtMemory(9, 4096) = %+v, want base 9 with displacement 4096", got)
+	}
+}