feat(arch): add the amd64 memory-operand mechanism to the extension layer

Assisted-by: GLM 5.3 Flash

arch/amd64_ext.go

@@ -115,6 +115,68 @@ func amd64Encode(b []byte, dest, vvvv, rm int) []byte {
 	return out
 }
 
+// amd64Memory validates a memory operand of an amd64 entry: no arrangement
+// and no qualifier, a base general register inside 0-15, a signed 32-bit
+// displacement and no shift.  The base number rides the operand's Reg and
+// the displacement its Imm.
+func (in ExtInstr) amd64Memory(op ExtOperand, pos int) (base int, disp int64, err error) {
+	if op.Kind != ExtMem {
+		return 0, 0, fmt.Errorf("%s: operand %d wants a memory operand, got %s", in.Name, pos, op.Kind)
+	}
+	if op.Arr != ExtArrNone {
+		return 0, 0, fmt.Errorf("%s: operand %d carries an arrangement suffix, the amd64 layer takes none", in.Name, pos)
+	}
+	if op.Qual != ExtQualNone {
+		return 0, 0, fmt.Errorf("%s: operand %d carries a predicate qualifier, the amd64 layer takes none", in.Name, pos)
+	}
+	if op.HasShift {
+		return 0, 0, fmt.Errorf("%s: operand %d carries a shift, the amd64 memory forms take none", in.Name, pos)
+	}
+	if op.Reg < 0 || op.Reg > 15 {
+		return 0, 0, fmt.Errorf("%s: operand %d names base register %d, outside 0-15", in.Name, pos, op.Reg)
+	}
+	if op.Imm < -1<<31 || op.Imm >= 1<<31 {
+		return 0, 0, fmt.Errorf("%s: operand %d carries displacement %d, outside the signed 32-bit range", in.Name, pos, op.Imm)
+	}
+	return op.Reg, op.Imm, nil
+}
+
+// amd64EncodeMemory returns the register-form template with a base-relative
+// memory operand filled in: dest and vvvv keep their register meanings, the
+// ModR/M r/m field carries the base, and the high base bit rides EVEX.B as
+// amd64Encode lays it.  The ModR/M mod bits and the trailing SIB and
+// displacement bytes follow the canonical choices the GNU assembler makes
+// for the plain, unscaled SDM displacements: no displacement bytes at
+// displacement zero, a disp8 when the value fits a signed byte and a disp32
+// otherwise, the SIB byte 0x24 when the base is RSP or R12, whose r/m
+// encoding 100 demands it, and a forced displacement on RBP and R13, whose
+// mod-00 r/m encoding 101 means RIP-relative.  The operand must have passed
+// amd64Memory first.
+func amd64EncodeMemory(b []byte, dest, vvvv, base int, disp int64) []byte {
+	out := amd64Encode(b, dest, vvvv, base)
+	rm := base & 7
+	var tail []byte
+	mod := byte(0)
+	switch {
+	case rm == 5 || disp != 0:
+		// RBP and R13 cannot drop the displacement: mod 00 with r/m 101
+		// addresses RIP-relative, not through the base.
+		if disp >= -128 && disp <= 127 {
+			mod = 1
+			tail = []byte{byte(disp)}
+		} else {
+			mod = 2
+			tail = []byte{byte(disp), byte(disp >> 8), byte(disp >> 16), byte(disp >> 24)}
+		}
+	}
+	if rm == 4 {
+		// RSP and R12 need the SIB byte: no index, base 100.
+		tail = append([]byte{0x24}, tail...)
+	}
+	out[5] = out[5]&0x3f | mod<<6
+	return append(out, tail...)
+}
+
 // amd64PlainReg checks the invariants every amd64 register operand carries:
 // no arm64 arrangement, no predicate qualifier, and a register number inside
 // the class the instruction encodes.