diff --git a/arch/arm64_ext.go b/arch/arm64_ext.go index d852967..e2917bd 100644 --- a/arch/arm64_ext.go +++ b/arch/arm64_ext.go @@ -235,6 +235,12 @@ type ExtOperand struct { // the destinations of the entries that carry Er or Sae accept it. The // arm64 entries all carry the zero value. Round ExtRounding + // List carries the register count of a multi-vector list operand, the + // [Z13.B, Z14.B, Z15.B] shape of the SVE LD2-LD4 and ST2-ST4 + // multiple-structure loads and stores: consecutive registers under one + // arrangement, of which the encoding carries the first alone. A list + // of one register carries 0, and the amd64 operands never more. + List int } // ExtVector builds a scalable vector operand, ADD Z1.S style. @@ -3486,6 +3492,106 @@ var arm64Extensions = []ExtInstr{ {Name: "ZBFSUB", Summary: "Subtract BFloat16 halfwords, unpredicated", Word: 0x65000400, Form: ExtFormVectorsZm, Arr: ExtArrH, Feature: ExtFeatureSVE2, Ref: "ARM DDI 0487J SVE2: ZBFSUB; inst_gen.go + arm64sveenc.s"}, + // --- the SVE multiple-structure loads and stores -------------------------- + // + // The two-, three- and four-register list loads and stores over the + // two-general-register addressing mode, the shift scaling the base by + // the list element size. The list operand carries consecutive + // registers under one arrangement; the encoding takes the first alone + // and the conversion validates the rest against the instruction's own + // digit. The MUL VL immediate-offset variant of the same shapes stays + // unwired. Provenance as above: the toolchain encoding table and its + // generated corpus, both produced from Arm's official ISA description. + {Name: "ZLD2B", Summary: "Load 2 scalable vector registers, .B list", + Word: 0xa420c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 0, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD2B (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD2D", Summary: "Load 2 scalable vector registers, .D list", + Word: 0xa5a0c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 3, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD2D (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD2H", Summary: "Load 2 scalable vector registers, .H list", + Word: 0xa4a0c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 1, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD2H (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD2Q", Summary: "Load 2 scalable vector registers, .Q list", + Word: 0xa4a08000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 4, Arr: ExtArrQ, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD2Q (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD2W", Summary: "Load 2 scalable vector registers, .W list", + Word: 0xa520c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 2, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD2W (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD3B", Summary: "Load 3 scalable vector registers, .B list", + Word: 0xa440c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 0, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD3B (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD3D", Summary: "Load 3 scalable vector registers, .D list", + Word: 0xa5c0c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 3, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD3D (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD3H", Summary: "Load 3 scalable vector registers, .H list", + Word: 0xa4c0c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 1, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD3H (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD3Q", Summary: "Load 3 scalable vector registers, .Q list", + Word: 0xa5208000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 4, Arr: ExtArrQ, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD3Q (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD3W", Summary: "Load 3 scalable vector registers, .W list", + Word: 0xa540c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 2, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD3W (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD4B", Summary: "Load 4 scalable vector registers, .B list", + Word: 0xa460c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 0, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD4B (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD4D", Summary: "Load 4 scalable vector registers, .D list", + Word: 0xa5e0c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 3, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD4D (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD4H", Summary: "Load 4 scalable vector registers, .H list", + Word: 0xa4e0c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 1, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD4H (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD4Q", Summary: "Load 4 scalable vector registers, .Q list", + Word: 0xa5a08000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 4, Arr: ExtArrQ, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD4Q (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZLD4W", Summary: "Load 4 scalable vector registers, .W list", + Word: 0xa560c000, Form: ExtFormSveLoad, SveMode: extMemPair, SveShift: 2, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZLD4W (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST2B", Summary: "Store 2 scalable vector registers, .B list", + Word: 0xe4206000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 0, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST2B (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST2D", Summary: "Store 2 scalable vector registers, .D list", + Word: 0xe5a06000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 3, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST2D (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST2H", Summary: "Store 2 scalable vector registers, .H list", + Word: 0xe4a06000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 1, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST2H (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST2Q", Summary: "Store 2 scalable vector registers, .Q list", + Word: 0xe4600000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 4, Arr: ExtArrQ, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST2Q (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST2W", Summary: "Store 2 scalable vector registers, .W list", + Word: 0xe5206000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 2, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST2W (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST3B", Summary: "Store 3 scalable vector registers, .B list", + Word: 0xe4406000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 0, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST3B (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST3D", Summary: "Store 3 scalable vector registers, .D list", + Word: 0xe5c06000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 3, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST3D (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST3H", Summary: "Store 3 scalable vector registers, .H list", + Word: 0xe4c06000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 1, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST3H (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST3Q", Summary: "Store 3 scalable vector registers, .Q list", + Word: 0xe4a00000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 4, Arr: ExtArrQ, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST3Q (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST3W", Summary: "Store 3 scalable vector registers, .W list", + Word: 0xe5406000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 2, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST3W (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST4B", Summary: "Store 4 scalable vector registers, .B list", + Word: 0xe4606000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 0, Arr: ExtArrB, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST4B (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST4D", Summary: "Store 4 scalable vector registers, .D list", + Word: 0xe5e06000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 3, Arr: ExtArrD, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST4D (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST4H", Summary: "Store 4 scalable vector registers, .H list", + Word: 0xe4e06000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 1, Arr: ExtArrH, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST4H (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST4Q", Summary: "Store 4 scalable vector registers, .Q list", + Word: 0xe4e00000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 4, Arr: ExtArrQ, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST4Q (multiple structures); inst_gen.go + arm64sveenc.s"}, + {Name: "ZST4W", Summary: "Store 4 scalable vector registers, .W list", + Word: 0xe5606000, Form: ExtFormSveStore, SveMode: extMemPair, SveShift: 2, Arr: ExtArrS, Feature: ExtFeatureSVE, + Ref: "ARM DDI 0487J: ZST4W (multiple structures); inst_gen.go + arm64sveenc.s"}, // --- the SVE2 crypto family ---------------------------------------------- // diff --git a/arch/arm64_ext_test.go b/arch/arm64_ext_test.go index 38bc3fe..3918809 100644 --- a/arch/arm64_ext_test.go +++ b/arch/arm64_ext_test.go @@ -1551,6 +1551,16 @@ func TestArm64ExtStage4Golden(t *testing.T) { {"ZBFSUB Z7.H, Z6.H, Z23.H", "ZBFSUB", ExtFormVectorsZm, ExtQualNone, ExtArrH, []ExtOperand{ExtVector(7, ExtArrH), ExtVector(6, ExtArrH), ExtVector(23, ExtArrH)}, 0x650704d7}, + // The multiple-structure loads and stores. + {"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B]", "ZLD2B", ExtFormSveLoad, ExtQualNone, ExtArrB, + []ExtOperand{ExtSveMemPair(6, 0, 14, false), ExtPredicate(4, ExtQualZeroing), ExtOperand{Kind: ExtZReg, Reg: 13, Arr: ExtArrB, List: 2}}, + 0xa426d1cd}, + {"ZLD4D (R6<<3)(R14), P4.Z, [Z13.D, Z14.D, Z15.D, Z16.D]", "ZLD4D", ExtFormSveLoad, ExtQualNone, ExtArrD, + []ExtOperand{ExtSveMemPair(6, 3, 14, false), ExtPredicate(4, ExtQualZeroing), ExtOperand{Kind: ExtZReg, Reg: 13, Arr: ExtArrD, List: 4}}, + 0xa5e6d1cd}, + {"ZST2Q [Z8.Q, Z9.Q], P3, (R6<<4)(RSP)", "ZST2Q", ExtFormSveStore, ExtQualNone, ExtArrQ, + []ExtOperand{ExtOperand{Kind: ExtZReg, Reg: 8, Arr: ExtArrQ, List: 2}, ExtPredicateSized(3, ExtArrNone), ExtSveMemPair(6, 4, 31, true)}, + 0xe4660fe8}, } { in := extInstructionMeta(t, tt.mnem, tt.form, tt.qual, tt.arr) got, err := in.Encode(tt.ops) diff --git a/asm/arm64_ext.go b/asm/arm64_ext.go index f3f76a9..f9b5400 100644 --- a/asm/arm64_ext.go +++ b/asm/arm64_ext.go @@ -52,6 +52,7 @@ func arm64ExtStatement(mnem string, ops []*ast.Operand) (extops []arch.ExtOperan if !arm64ExtPinned(mnem, ops) { return nil, false, nil } + ops = arm64ExtMergeLists(ops) out := make([]arch.ExtOperand, 0, len(ops)) for i, op := range ops { text := strings.Join(strings.Fields(op.Raw), "") @@ -79,10 +80,11 @@ func arm64ExtStatement(mnem string, ops []*ast.Operand) (extops []arch.ExtOperan out = append(out, ext) continue } - // The gather/scatter destination list, [Z13.B]: one scalable vector - // in brackets, its arrangement part of the instruction's identity. + // The load and store destination list, [Z13.B] or the multi-register + // [Z13.B, Z14.B, Z15.B] of the multiple-structure shapes, its + // arrangement part of the instruction's identity. if strings.HasPrefix(text, "[") && strings.HasSuffix(text, "]") { - if ext, ok := arm64ExtVector(strings.Trim(text, "[]")); ok { + if ext, ok := arm64ExtVectorList(mnem, strings.Trim(text, "[]")); ok { out = append(out, ext) continue } @@ -221,6 +223,82 @@ func arm64ExtVector(text string) (arch.ExtOperand, bool) { return arch.ExtOperand{Kind: arch.ExtZReg, Reg: reg, Arr: arr}, true } +// arm64ExtMergeLists rejoins the bracketed vector lists the parser reads as +// separate operands: the comma inside [Z13.B, Z14.B, Z15.B] is an operand +// boundary to the parser, so the list arrives as two or more pieces and the +// multiple-structure loads and stores need it whole. Pieces from an opening +// bracket to the one carrying the closing bracket rejoin over their commas; +// everything else passes through untouched. +func arm64ExtMergeLists(ops []*ast.Operand) []*ast.Operand { + closed := func(op *ast.Operand) bool { + return strings.HasSuffix(strings.Join(strings.Fields(op.Raw), ""), "]") + } + merged := make([]*ast.Operand, 0, len(ops)) + for i := 0; i < len(ops); i++ { + text := strings.Join(strings.Fields(ops[i].Raw), "") + if !strings.HasPrefix(text, "[") || closed(ops[i]) { + merged = append(merged, ops[i]) + continue + } + parts := []string{ops[i].Raw} + kind := ops[i].Kind + for i+1 < len(ops) { + i++ + parts = append(parts, ops[i].Raw) + if closed(ops[i]) { + break + } + } + merged = append(merged, &ast.Operand{Kind: kind, Raw: strings.Join(parts, ",")}) + } + return merged +} + +// arm64ExtVectorList parses the bracketed vector list of the loads and +// stores: a single register, [Z13.B], or the multi-register lists of the +// LD2-LD4 and ST2-ST4 multiple-structure shapes, [Z13.B, Z14.B, Z15.B], +// consecutive registers under one arrangement. The instruction's own digit +// names the list's length where it carries one, so a two-register list +// under ZLD3 fails here. The encoding carries the first register alone; +// the length rides the operand for the encode side. +func arm64ExtVectorList(mnem, body string) (arch.ExtOperand, bool) { + regs := strings.Split(body, ",") + first, ok := arm64ExtVector(regs[0]) + if !ok { + return arch.ExtOperand{}, false + } + for i, reg := range regs[1:] { + op, ok := arm64ExtVector(reg) + if !ok || op.Arr != first.Arr || op.Reg != first.Reg+i+1 { + return arch.ExtOperand{}, false + } + } + if count := arm64ExtListCount(mnem); count != len(regs) { + return arch.ExtOperand{}, false + } + if len(regs) > 1 { + first.List = len(regs) + } + return first, true +} + +// arm64ExtListCount reads the list length a load or store mnemonic names, +// the digit straight after its ZLD or ZST prefix; the loads and stores +// without one carry a single register. +func arm64ExtListCount(mnem string) int { + rest, ok := strings.CutPrefix(mnem, "ZLD") + if !ok { + rest, ok = strings.CutPrefix(mnem, "ZST") + } + if !ok || rest == "" { + return 1 + } + if c := rest[0]; c >= '2' && c <= '4' { + return int(c - '0') + } + return 1 +} + // arm64ExtPredicate parses a predicate register operand: P0..P15 with an // optional element-size suffix (P0.B) and an optional qualifier in either // spelling the corpus and the wired forms use, P0/M and P0.Z. diff --git a/asm/arm64_ext_asm_test.go b/asm/arm64_ext_asm_test.go index 3c8cde2..a83e50a 100644 --- a/asm/arm64_ext_asm_test.go +++ b/asm/arm64_ext_asm_test.go @@ -822,6 +822,17 @@ func TestArm64AssembleExtensionStage4Golden(t *testing.T) { {"ZBFMUL Z7.H, Z6.H, Z23.H", 0x650708d7}, {"ZBFSUB Z23.H, Z13.H, P1.M, Z13.H", 0x650186ed}, {"ZBFSUB Z7.H, Z6.H, Z23.H", 0x650704d7}, + // The multiple-structure loads and stores. + {"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.B]", 0xa426d1cd}, + {"ZLD2H (R6<<1)(R14), P4.Z, [Z13.H, Z14.H]", 0xa4a6d1cd}, + {"ZLD2Q (R6<<4)(R14), P4.Z, [Z13.Q, Z14.Q]", 0xa4a691cd}, + {"ZLD3W (R6<<2)(R14), P4.Z, [Z13.S, Z14.S, Z15.S]", 0xa546d1cd}, + {"ZLD4B (R6)(R14), P4.Z, [Z13.B, Z14.B, Z15.B, Z16.B]", 0xa466d1cd}, + {"ZLD4D (R6<<3)(R14), P4.Z, [Z13.D, Z14.D, Z15.D, Z16.D]", 0xa5e6d1cd}, + {"ZST2B [Z8.B, Z9.B], P3, (R6)(RSP)", 0xe4266fe8}, + {"ZST2Q [Z8.Q, Z9.Q], P3, (R6<<4)(RSP)", 0xe4660fe8}, + {"ZST3D [Z8.D, Z9.D, Z10.D], P3, (R6<<3)(RSP)", 0xe5c66fe8}, + {"ZST4Q [Z8.Q, Z9.Q, Z10.Q, Z11.Q], P3, (R6<<4)(RSP)", 0xe4e60fe8}, } for _, tt := range tests { words := assembleArm64Words(t, arm64ExtProbeHead+"\t"+tt.stmt+"\n\tRET\n") @@ -858,6 +869,10 @@ func TestArm64AssembleExtensionStage4Refusals(t *testing.T) { {"ZADDQV Z25.S, P3/M, V5.S4", "takes no qualifier"}, {"ZADDQV Z25.S, P3, V5", "no quadword suffix"}, {"ZADDQV Z25.S, P3, V5.D2", "carries arrangement .D, want .S"}, + // The multiple-structure lists. + {"ZLD3B (R6)(R14), P4.Z, [Z13.B, Z14.B]", "is not an extended-layer operand"}, + {"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z15.B]", "is not an extended-layer operand"}, + {"ZLD2B (R6)(R14), P4.Z, [Z13.B, Z14.S]", "is not an extended-layer operand"}, } for _, tt := range tests { got := assembleArm64SrcError(t, arm64ExtProbeHead+"\t"+tt.stmt+"\n\tRET\n") diff --git a/asm/extension_test.go b/asm/extension_test.go index 616779a..4d7241f 100644 --- a/asm/extension_test.go +++ b/asm/extension_test.go @@ -327,6 +327,36 @@ func TestExtensionNamesARM64(t *testing.T) { "ZBFMINNM", "ZBFMUL", "ZBFSUB", + "ZLD2B", + "ZLD2D", + "ZLD2H", + "ZLD2Q", + "ZLD2W", + "ZLD3B", + "ZLD3D", + "ZLD3H", + "ZLD3Q", + "ZLD3W", + "ZLD4B", + "ZLD4D", + "ZLD4H", + "ZLD4Q", + "ZLD4W", + "ZST2B", + "ZST2D", + "ZST2H", + "ZST2Q", + "ZST2W", + "ZST3B", + "ZST3D", + "ZST3H", + "ZST3Q", + "ZST3W", + "ZST4B", + "ZST4D", + "ZST4H", + "ZST4Q", + "ZST4W", // The stage-three families: the SVE2 crypto group, the predicate // counters and loop terminators with the 32-bit while compares, and // the reductions into a SIMD register. @@ -439,7 +469,7 @@ func TestExtensionNamesARM64(t *testing.T) { if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("ExtensionNames(ARM64) = %v, want %v", got, want) } - if n := len(arch.Extensions(arch.ARM64)); n != 440 { - t.Errorf("the family registers %d instructions, want 440", n) + if n := len(arch.Extensions(arch.ARM64)); n != 470 { + t.Errorf("the family registers %d instructions, want 470", n) } }