feat(arch): add the VEX encoder to the extension layer

Assisted-by: GLM 5.3 Flash

arch/amd64_ext_test.go

@@ -1499,7 +1499,11 @@ func amd64ResolveEntry(mnem string, ops []ExtOperand) (ExtInstr, bool) {
 func amd64ExtInstr(t *testing.T, mnem string, class ExtOperandKind, preds ...func(ExtInstr) bool) ExtInstr {
 	t.Helper()
 	for _, in := range Extensions(AMD64) {
-		if in.Name != mnem || amd64LengthClass(in.Bytes) != class {
+		length := amd64LengthClass(in.Bytes)
+		if in.Vex {
+			length = amd64VexLengthClass(in.Bytes)
+		}
+		if in.Name != mnem || length != class {
 			continue
 		}
 		match := true
@@ -1548,7 +1552,10 @@ func TestAmd64ExtGoldenBytes(t *testing.T) {
 // TestAmd64ExtTemplateIntegrity checks the metadata contract: every entry
 // names its manual reference, summary and feature, and every template carries
 // the fixed shape of an EVEX register form with the register-derived bits
-// zero, so a slip in the table is an error and not a stray byte.
+// zero, so a slip in the table is an error and not a stray byte.  A VEX row
+// is held to the C4 layout instead: five bytes, the map beside three zero
+// register bits in byte one, W and vvvv zero in byte two and mod 11 in the
+// ModR/M.
 func TestAmd64ExtTemplateIntegrity(t *testing.T) {
 	features := map[ExtFeature]bool{
 		ExtFeatureBF16:         true,
@@ -1562,6 +1569,34 @@ func TestAmd64ExtTemplateIntegrity(t *testing.T) {
 		if !features[in.Feature] {
 			t.Errorf("%s: feature %q is not an amd64 extension feature", in.Name, in.Feature)
 		}
+		if in.Form.Arity() < 2 || in.Form.Arity() > 4 {
+			t.Errorf("%s: form %s carries an unusable arity %d", in.Name, in.Form, in.Form.Arity())
+		}
+		if in.Mem > in.Form.Arity() {
+			t.Errorf("%s: Mem names operand %d, outside the form's %d positions", in.Name, in.Mem, in.Form.Arity())
+		}
+		if in.Vex {
+			if len(in.Bytes) != 5 {
+				t.Errorf("%s: the template is %d bytes, want the 5-byte VEX register form", in.Name, len(in.Bytes))
+				continue
+			}
+			if in.Bytes[0] != 0xC4 {
+				t.Errorf("%s: the template opens with %02x, want the VEX escape c4", in.Name, in.Bytes[0])
+			}
+			if in.Bytes[1]&0xe0 != 0 {
+				t.Errorf("%s: byte one carries register bits %03b, want them zero", in.Name, in.Bytes[1]>>5)
+			}
+			if in.Bytes[1]&0x1f != 0x02 {
+				t.Errorf("%s: byte one carries the map %05b, want the 0f38 escape 00010", in.Name, in.Bytes[1]&0x1f)
+			}
+			if in.Bytes[2]&0xf8 != 0 {
+				t.Errorf("%s: byte two carries W or vvvv bits, want them zero: %08b", in.Name, in.Bytes[2])
+			}
+			if in.Bytes[4]&0x3f != 0 || in.Bytes[4]&0xc0 != 0xc0 {
+				t.Errorf("%s: byte four is %08b, want mod 11 with the reg and rm fields zero", in.Name, in.Bytes[4])
+			}
+			continue
+		}
 		if len(in.Bytes) != 6 {
 			t.Errorf("%s: the template is %d bytes, want the 6-byte EVEX register form", in.Name, len(in.Bytes))
 			continue
@@ -1584,12 +1619,39 @@ func TestAmd64ExtTemplateIntegrity(t *testing.T) {
 		if in.Bytes[5]&0x3f != 0 || in.Bytes[5]&0xc0 != 0xc0 {
 			t.Errorf("%s: byte five is %08b, want mod 11 with the reg and rm fields zero", in.Name, in.Bytes[5])
 		}
-		if in.Form.Arity() < 2 || in.Form.Arity() > 4 {
-			t.Errorf("%s: form %s carries an unusable arity %d", in.Name, in.Form, in.Form.Arity())
-		}
-		if in.Mem > in.Form.Arity() {
-			t.Errorf("%s: Mem names operand %d, outside the form's %d positions", in.Name, in.Mem, in.Form.Arity())
-		}
+	}
+}
+
+// TestAmd64VexEncoder pins the VEX plumbing over a hand-built template: the
+// register complements of R, X and B in byte one, vvvv's complement in byte
+// two, the vector length bit two of byte two names and the memory shape's
+// mod, SIB and displacement bytes, over the words the AVX-VNNI-INT16 dot
+// product the mechanism serves quotes from GNU as.
+func TestAmd64VexEncoder(t *testing.T) {
+	template := []byte{0xC4, 0x02, 0x02, 0xD2, 0xC0}
+	if got := amd64VexLengthClass(template); got != ExtXMM {
+		t.Errorf("the template reads as %s, want XMM", got)
+	}
+	if got := amd64VexLengthClass([]byte{0xC4, 0x02, 0x06, 0xD2, 0xC0}); got != ExtYMM {
+		t.Errorf("the wide template reads as %s, want YMM", got)
+	}
+	if got := hex.EncodeToString(amd64EncodeVex(template, 3, 2, 1)); got != "c4e26ad2d9" {
+		t.Errorf("amd64EncodeVex: got %s, want c4e26ad2d9", got)
+	}
+	// The registers above 7 clear R bar and B bar; vvvv keeps its complement.
+	if got := hex.EncodeToString(amd64EncodeVex(template, 9, 11, 12)); got != "c44222d2cc" {
+		t.Errorf("amd64EncodeVex, high registers: got %s, want c44222d2cc", got)
+	}
+	if got := hex.EncodeToString(amd64EncodeVexMemory(template, 1, 2, 1, 127)); got != "c4e26ad2497f" {
+		t.Errorf("amd64EncodeVexMemory: got %s, want c4e26ad2497f", got)
+	}
+	// The RSP base demands the SIB byte, the RBP base a scaled index form
+	// whose X bar clears under the index above 7.
+	if got := hex.EncodeToString(amd64EncodeVexMemory(template, 1, 2, 4, 0)); got != "c4e26ad20c24" {
+		t.Errorf("amd64EncodeVexMemory over RSP: got %s, want c4e26ad20c24", got)
+	}
+	if got := hex.EncodeToString(amd64EncodeVexScaledMemory(template, 1, 2, 5, 14, 8, 127)); got != "c4a26ad24cf57f" {
+		t.Errorf("amd64EncodeVexScaledMemory: got %s, want c4a26ad24cf57f", got)
 	}
 }