From 4632ac1bb9856d4c1462e47366d8eb60126e1477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Balv=C3=ADn?= Date: Wed, 7 Oct 2026 19:46:07 +0200 Subject: [PATCH] feat(arch): add the AVX-VNNI-INT16 dot products to the extension layer Assisted-by: GLM 5.3 Flash --- arch/amd64_ext.go | 34 ++++++++++++++++++ arch/amd64_ext_test.go | 71 +++++++++++++++++++++++++++++++++++-- asm/extension_amd64_test.go | 17 +++++++-- 3 files changed, 118 insertions(+), 4 deletions(-) diff --git a/arch/amd64_ext.go b/arch/amd64_ext.go index 018c718..182fc6c 100644 --- a/arch/amd64_ext.go +++ b/arch/amd64_ext.go @@ -48,6 +48,7 @@ const ( ExtFeatureBF16 ExtFeature = "avx512bf16" ExtFeatureVP2INTERSECT ExtFeature = "avx512vp2intersect" ExtFeatureFP16 ExtFeature = "avx512fp16" + ExtFeatureVnniInt16 ExtFeature = "avxvnniint16" ) // ExtXmm, ExtYmm and ExtZmm build vector operands of the three EVEX register @@ -1798,4 +1799,37 @@ var amd64Extensions = []ExtInstr{ {Name: "VMINMAXSH", Summary: "Return the minimum or maximum of scalar FP16 values under an imm8 control", Bytes: []byte{0x62, 0x03, 0x04, 0x00, 0x53, 0xC0}, Form: ExtFormAmdVec3Imm, Mem: 3, Feature: ExtFeatureFP16, Ref: "Intel SDM Vol. 2C, VMINMAXSH (EVEX.NDS.LIG.0F3A.W0 53 /r /ib)"}, + + // The AVX-VNNI-INT16 dot products: VPDPWSUD and VPDPWSUDS, the + // unsigned by signed word pairs whose accumulation saturates under the + // S suffix, and VPDPWUSD and VPDPWUSDS, the signed by unsigned ones. + // VEX.NDS.F3.0F38.W0 D2 and D3, and the 66-prefixed pair beside them, + // the quartet the VEX mechanism carries: no AVX-512 form exists, so no + // register above 15, no write mask, no broadcast and no rounding ever + // applies, and the memory shape reads its second source plain, the + // m128 and m256 the manual spells. + {Name: "VPDPWSUD", Summary: "Dot product of unsigned and signed 16-bit integers with 32-bit accumulation", + Bytes: []byte{0xC4, 0x02, 0x02, 0xD2, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWSUD (VEX.NDS.128.F3.0F38.W0 D2 /r)"}, + {Name: "VPDPWSUD", Summary: "Dot product of unsigned and signed 16-bit integers with 32-bit accumulation", + Bytes: []byte{0xC4, 0x02, 0x06, 0xD2, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWSUD (VEX.NDS.256.F3.0F38.W0 D2 /r)"}, + {Name: "VPDPWSUDS", Summary: "Dot product of unsigned and signed 16-bit integers, the accumulation saturating", + Bytes: []byte{0xC4, 0x02, 0x02, 0xD3, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWSUDS (VEX.NDS.128.F3.0F38.W0 D3 /r)"}, + {Name: "VPDPWSUDS", Summary: "Dot product of unsigned and signed 16-bit integers, the accumulation saturating", + Bytes: []byte{0xC4, 0x02, 0x06, 0xD3, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWSUDS (VEX.NDS.256.F3.0F38.W0 D3 /r)"}, + {Name: "VPDPWUSD", Summary: "Dot product of signed and unsigned 16-bit integers with 32-bit accumulation", + Bytes: []byte{0xC4, 0x02, 0x01, 0xD2, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWUSD (VEX.NDS.128.66.0F38.W0 D2 /r)"}, + {Name: "VPDPWUSD", Summary: "Dot product of signed and unsigned 16-bit integers with 32-bit accumulation", + Bytes: []byte{0xC4, 0x02, 0x05, 0xD2, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWUSD (VEX.NDS.256.66.0F38.W0 D2 /r)"}, + {Name: "VPDPWUSDS", Summary: "Dot product of signed and unsigned 16-bit integers, the accumulation saturating", + Bytes: []byte{0xC4, 0x02, 0x01, 0xD3, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWUSDS (VEX.NDS.128.66.0F38.W0 D3 /r)"}, + {Name: "VPDPWUSDS", Summary: "Dot product of signed and unsigned 16-bit integers, the accumulation saturating", + Bytes: []byte{0xC4, 0x02, 0x05, 0xD3, 0xC0}, Form: ExtFormAmdVec3, Mem: 2, Vex: true, Feature: ExtFeatureVnniInt16, + Ref: "Intel SDM Vol. 2C, VPDPWUSDS (VEX.NDS.256.66.0F38.W0 D3 /r)"}, } diff --git a/arch/amd64_ext_test.go b/arch/amd64_ext_test.go index 9af5c11..8bd5e90 100644 --- a/arch/amd64_ext_test.go +++ b/arch/amd64_ext_test.go @@ -1472,6 +1472,63 @@ var amd64GoldenRows = []amd64GoldenRow{ {"vminmaxsh memory source disp8", "VMINMAXSH", []ExtOperand{ExtImmediate(0x88), ExtXmm(28), ExtMemory(1, 127), ExtXmm(29)}, "62631c0053697f88", "62 63 1c 00 53 69 7f 88 vminmaxsh $0x88,0xfe(%rcx),%xmm28,%xmm29 (Disp8(7f))"}, + + // The AVX-VNNI-INT16 dot products, VEX.NDS.F3 and 66.0F38.W0 D2 and D3: + // the words the VEX mechanism encodes, five bytes over the C4 escape. + // The registers run 0..15, and the word is the three-operand shape the + // AVX dot products carry: dest, src1 and the second source the memory + // form reads plain. + {"vpdpwsud", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtXmm(1), ExtXmm(3)}, + "c4e26ad2d9", "c4 e2 6a d2 d9 vpdpwsud %xmm1,%xmm2,%xmm3"}, + {"vpdpwsuds", "VPDPWSUDS", + []ExtOperand{ExtXmm(2), ExtXmm(1), ExtXmm(3)}, + "c4e26ad3d9", "c4 e2 6a d3 d9 vpdpwsuds %xmm1,%xmm2,%xmm3"}, + {"vpdpwusd", "VPDPWUSD", + []ExtOperand{ExtXmm(2), ExtXmm(1), ExtXmm(3)}, + "c4e269d2d9", "c4 e2 69 d2 d9 vpdpwusd %xmm1,%xmm2,%xmm3"}, + {"vpdpwusds", "VPDPWUSDS", + []ExtOperand{ExtXmm(2), ExtXmm(1), ExtXmm(3)}, + "c4e269d3d9", "c4 e2 69 d3 d9 vpdpwusds %xmm1,%xmm2,%xmm3"}, + {"vpdpwsud ymm", "VPDPWSUD", + []ExtOperand{ExtYmm(2), ExtYmm(1), ExtYmm(3)}, + "c4e26ed2d9", "c4 e2 6e d2 d9 vpdpwsud %ymm1,%ymm2,%ymm3"}, + {"vpdpwsuds ymm", "VPDPWSUDS", + []ExtOperand{ExtYmm(2), ExtYmm(1), ExtYmm(3)}, + "c4e26ed3d9", "c4 e2 6e d3 d9 vpdpwsuds %ymm1,%ymm2,%ymm3"}, + {"vpdpwusd ymm", "VPDPWUSD", + []ExtOperand{ExtYmm(2), ExtYmm(1), ExtYmm(3)}, + "c4e26dd2d9", "c4 e2 6d d2 d9 vpdpwusd %ymm1,%ymm2,%ymm3"}, + {"vpdpwusds ymm", "VPDPWUSDS", + []ExtOperand{ExtYmm(2), ExtYmm(1), ExtYmm(3)}, + "c4e26dd3d9", "c4 e2 6d d3 d9 vpdpwusds %ymm1,%ymm2,%ymm3"}, + {"vpdpwsud high registers", "VPDPWSUD", + []ExtOperand{ExtXmm(11), ExtXmm(12), ExtXmm(9)}, + "c44222d2cc", "c4 42 22 d2 cc vpdpwsud %xmm12,%xmm11,%xmm9"}, + {"vpdpwusds ymm high registers", "VPDPWUSDS", + []ExtOperand{ExtYmm(10), ExtYmm(15), ExtYmm(8)}, + "c4422dd3c7", "c4 42 2d d3 c7 vpdpwusds %ymm15,%ymm10,%ymm8"}, + {"vpdpwusds ymm, the destination above 7", "VPDPWUSDS", + []ExtOperand{ExtYmm(10), ExtYmm(8), ExtYmm(15)}, + "c4422dd3f8", "c4 42 2d d3 f8 vpdpwusds %ymm8,%ymm10,%ymm15"}, + {"vpdpwsud memory source", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtMemory(1, 127), ExtXmm(1)}, + "c4e26ad2497f", "c4 e2 6a d2 49 7f vpdpwsud 0x7f(%rcx),%xmm2,%xmm1"}, + {"vpdpwsud memory source negative disp8", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtMemory(2, -128), ExtXmm(1)}, + "c4e26ad24a80", "c4 e2 6a d2 4a 80 vpdpwsud -0x80(%rdx),%xmm2,%xmm1"}, + {"vpdpwsud memory source disp32", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtMemory(1, 8128), ExtXmm(1)}, + "c4e26ad289c01f0000", "c4 e2 6a d2 89 c0 1f 00 00 vpdpwsud 0x1fc0(%rcx),%xmm2,%xmm1"}, + {"vpdpwsud memory source over an r9 base", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtMemory(9, 0), ExtXmm(12)}, + "c4426ad221", "c4 42 6a d2 21 vpdpwsud (%r9),%xmm2,%xmm12"}, + {"vpdpwsud memory source over an rsp base", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtMemory(4, 0), ExtXmm(1)}, + "c4e26ad20c24", "c4 e2 6a d2 0c 24 vpdpwsud (%rsp),%xmm2,%xmm1"}, + {"vpdpwusd memory source over a scaled index", "VPDPWUSD", + []ExtOperand{ExtXmm(2), ExtScaledMemory(5, 14, 8, 127), ExtXmm(1)}, + "c4a269d24cf57f", "c4 a2 69 d2 4c f5 7f vpdpwusd 0x7f(%rbp,%r14,8),%xmm2,%xmm1"}, } // amd64ResolveEntry finds the table entry a golden row exercises: the entry @@ -1561,6 +1618,7 @@ func TestAmd64ExtTemplateIntegrity(t *testing.T) { ExtFeatureBF16: true, ExtFeatureVP2INTERSECT: true, ExtFeatureFP16: true, + ExtFeatureVnniInt16: true, } for _, in := range Extensions(AMD64) { if in.Name == "" || in.Summary == "" || in.Ref == "" { @@ -1891,6 +1949,15 @@ func TestAmd64ExtRejects(t *testing.T) { {"broadcast on the scalar minimum-maximum's memory source", "VMINMAXSH", []ExtOperand{ExtImmediate(0x88), ExtXmm(29), ExtBroadcast(9, 0), ExtXmm(30)}, "the entry's memory operand takes none"}, + {"register beyond r15 on the dot product", "VPDPWSUD", + []ExtOperand{ExtXmm(16), ExtXmm(1), ExtXmm(2)}, + "outside 0-15"}, + {"write mask on the dot product", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtXmm(1), ExtWriteMasked(ExtXmm(3), 3, true)}, + "the entry's destination takes none"}, + {"broadcast on the dot product's memory source", "VPDPWSUD", + []ExtOperand{ExtXmm(2), ExtBroadcast(9, 0), ExtXmm(1)}, + "the entry's memory operand takes none"}, {"a register where the multiply-add reads memory", "VFMSUB231PH", []ExtOperand{ExtZmm(29), ExtYmm(4), ExtZmm(30)}, "wants a ZMM register"}, @@ -2041,7 +2108,7 @@ func TestAmd64ExtArchBinding(t *testing.T) { t.Errorf("Extensions(%s) carries %d instructions, want none", a, len(got)) } } - if got := Extensions(AMD64); len(got) != 183 { - t.Errorf("the amd64 layer registers %d instructions, want 183", len(got)) + if got := Extensions(AMD64); len(got) != 191 { + t.Errorf("the amd64 layer registers %d instructions, want 191", len(got)) } } diff --git a/asm/extension_amd64_test.go b/asm/extension_amd64_test.go index d6b914d..1252362 100644 --- a/asm/extension_amd64_test.go +++ b/asm/extension_amd64_test.go @@ -86,6 +86,10 @@ func TestAmd64ExtensionRegistry(t *testing.T) { {"VFCMADDCSH", 1}, {"VMINMAXPH", 3}, {"VMINMAXSH", 1}, + {"VPDPWSUD", 2}, + {"VPDPWSUDS", 2}, + {"VPDPWUSD", 2}, + {"VPDPWUSDS", 2}, } { cands, ok := LookupExtension(arch.AMD64, tt.mnem) if !ok { @@ -99,8 +103,8 @@ func TestAmd64ExtensionRegistry(t *testing.T) { t.Errorf("the %s lookup is not case-insensitive", tt.mnem) } } - if got := arch.Extensions(arch.AMD64); len(got) != 183 { - t.Errorf("the amd64 layer registers %d instructions, want 183", len(got)) + if got := arch.Extensions(arch.AMD64); len(got) != 191 { + t.Errorf("the amd64 layer registers %d instructions, want 191", len(got)) } if _, ok := LookupExtension(arch.AMD64, "NOSUCHINSTR"); ok { t.Error("a non-extended mnemonic resolved") @@ -221,6 +225,15 @@ func TestEncodeExtensionAmd64(t *testing.T) { {"scalar minimum or maximum out of memory", "VMINMAXSH", []arch.ExtOperand{arch.ExtImmediate(0x88), arch.ExtXmm(28), arch.ExtMemory(9, 0), arch.ExtXmm(29)}, "62431c00532988"}, + {"vnni dot product through the VEX word", "VPDPWSUD", + []arch.ExtOperand{arch.ExtXmm(2), arch.ExtXmm(1), arch.ExtXmm(3)}, + "c4e26ad2d9"}, + {"saturating dot product, high registers", "VPDPWUSDS", + []arch.ExtOperand{arch.ExtYmm(10), arch.ExtYmm(15), arch.ExtYmm(8)}, + "c4422dd3c7"}, + {"dot product out of memory", "VPDPWSUD", + []arch.ExtOperand{arch.ExtXmm(2), arch.ExtMemory(1, 127), arch.ExtXmm(1)}, + "c4e26ad2497f"}, } { got, err := EncodeExtension(arch.AMD64, tt.mnem, tt.ops...) if err != nil {