diff --git a/asm/loong64_assemble.go b/asm/loong64_assemble.go index b20d164..d133987 100644 --- a/asm/loong64_assemble.go +++ b/asm/loong64_assemble.go @@ -46,30 +46,70 @@ func assembleLOONG64(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, spadj = append(spadj, SpadjStep{PC: guardLen + (loong64StoreWords(fi.autosize)+loong64AdjustWords(-int64(fi.autosize)))*4, Value: fi.autosize}) } - // Pass 1: label offsets from the instruction sizes. - offsets := map[string]int{} - pos := guardLen + len(prologue) + // The toolchain's parser counts N(PC) displacements over the source + // instructions at a uniform 4 bytes each, so a PC-relative branch + // resolves to the instruction N slots away in body order; the resolved + // target then participates in layout and loop-head padding like any + // branch target. + instrs := make([]*ast.Instr, 0, len(t.Body)) for _, stmt := range t.Body { - switch s := stmt.(type) { - case *ast.Label: - offsets[s.Name.Text] = pos - case *ast.Instr: - pos += loong64InstrSize(s, fi) + if in, ok := stmt.(*ast.Instr); ok && strings.ToUpper(in.Mnemonic.Text) != "PCALIGN" { + instrs = append(instrs, in) } } - - // Pass 2: encode. The guard prefix precedes the prologue; its branches - // target the morestack block at the end of the function, which the first - // pass has sized. - bodyLen := 0 - { - p := guardLen + len(prologue) - for _, stmt := range t.Body { - if in, ok := stmt.(*ast.Instr); ok { - p += loong64InstrSize(in, fi) - } + parseIndex := make(map[*ast.Instr]int, len(instrs)) + for i, in := range instrs { + parseIndex[in] = i + } + pcRelTarget := make(map[*ast.Instr]*ast.Instr) + for _, in := range instrs { + off, ok := loong64PCRelOffset(in) + if !ok { + continue } - bodyLen = p - (guardLen + len(prologue)) + tgt := parseIndex[in] + off + if tgt < 0 || tgt >= len(instrs) { + continue + } + pcRelTarget[in] = instrs[tgt] + } + + // Pass 1: label offsets from the instruction sizes. PCALIGN contributes + // only its padding. On top of the explicit PCALIGNs, the toolchain pads + // every backward-branch target (loop head) to a 16-byte boundary, so the + // layout runs to a fixpoint over the alignment set. + loopAligns := map[string]bool{} + alignInstrs := map[*ast.Instr]bool{} + for { + offsets, _, pcs, _ := loong64Layout(t, guardLen+len(prologue), fi, loopAligns, alignInstrs) + changed := false + for _, in := range instrs { + // A backward PC-relative target is the resolved instruction. + if tgt, ok := pcRelTarget[in]; ok && pcs[tgt] < pcs[in] && !alignInstrs[tgt] { + alignInstrs[tgt] = true + changed = true + } + target, ok := loong64BranchTarget(in) + if !ok { + continue + } + tOff, ok := offsets[target] + if !ok || tOff >= pcs[in] || loopAligns[target] { + continue + } + loopAligns[target] = true + changed = true + } + if !changed { + break + } + } + // Final layout with the complete alignment set. + offsets, alignPad, pcs, bodyEnd := loong64Layout(t, guardLen+len(prologue), fi, loopAligns, alignInstrs) + bodyLen := bodyEnd - (guardLen + len(prologue)) + pcRelPcs := make(map[*ast.Instr]int, len(pcRelTarget)) + for in, tgt := range pcRelTarget { + pcRelPcs[in] = pcs[tgt] } var out []byte if fi.needSplit { @@ -84,7 +124,20 @@ func assembleLOONG64(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, if !ok { continue } - code, err := encodeLOONG64Instr(in, pc, offsets, fi, &relocs, resolve) + // PCALIGN pads to the requested boundary with andi $0, $0, 0, the + // architecture's NOP, and encodes to nothing itself. + if strings.ToUpper(in.Mnemonic.Text) == "PCALIGN" { + pad := loong64PCAlignPad(pc, in) + out = append(out, loong64PadBytes(pad)...) + pc += pad + continue + } + // Loop-head alignment padding precedes the instruction. + if pad := alignPad[in]; pad > 0 { + out = append(out, loong64PadBytes(pad)...) + pc += pad + } + code, err := encodeLOONG64Instr(in, pc, offsets, fi, &relocs, resolve, pcRelPcs) if err != nil { return nil, nil, nil, nil, nil, fmt.Errorf("%s: %w", in.Mnemonic.Text, err) } @@ -115,6 +168,115 @@ func assembleLOONG64(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, return out, offsets, relocs, lines, spadj, nil } +// loong64PCRelOffset reports the N of a branch operand spelled N(PC): the +// displacement counted in source instructions from the branch itself. +func loong64PCRelOffset(instr *ast.Instr) (int, bool) { + mnem := strings.ToUpper(instr.Mnemonic.Text) + branch := false + switch mnem { + case "JMP": + branch = len(instr.Operands) == 1 + case "JAL", "CALL", "BL": + branch = len(instr.Operands) == 1 || len(instr.Operands) == 2 + case "BFPT", "BFPF": + branch = len(instr.Operands) == 1 + case "BEQ", "BNE", "BLT", "BGE", "BLTU", "BGEU", + "BEQZ", "BNEZ", "BLTZ", "BGEZ", "BLEZ", "BGTZ": + branch = len(instr.Operands) >= 2 + } + if !branch { + return 0, false + } + op := instr.Operands[len(instr.Operands)-1] + if op.Kind == ast.OpAddr && op.Addr.Sym == nil && op.Addr.Base == "PC" { + return int(op.Addr.Offset), true + } + return 0, false +} + +// loong64Layout walks the function body once and returns the label offsets, +// the loop-alignment padding due before each instruction (a pad of 0 needs +// nothing), the pc each instruction starts at (its padding included) and the +// first pc past the body. Explicit PCALIGN pads, the alignment pads for the +// labels in aligns and those for the instructions in alignInstrs (backward +// PC-relative targets) all contribute, mirroring the toolchain's layout +// pass. +func loong64Layout(t *ast.Text, start int, fi loong64FrameInfo, aligns map[string]bool, alignInstrs map[*ast.Instr]bool) (map[string]int, map[*ast.Instr]int, map[*ast.Instr]int, int) { + offsets := map[string]int{} + alignPad := map[*ast.Instr]int{} + pcs := map[*ast.Instr]int{} + pos := start + pendingAlign := false + var pendingNames []string + explicit := false + for _, stmt := range t.Body { + switch s := stmt.(type) { + case *ast.Label: + if aligns[s.Name.Text] { + pendingAlign = true + } + pendingNames = append(pendingNames, s.Name.Text) + // Provisional: a branch to the label lands here unless a loop + // alignment pad follows, in which case the label resolves to the + // padded instruction (the toolchain's labels bind to the branch + // target instruction, which the padding pass precedes). + offsets[s.Name.Text] = pos + case *ast.Instr: + if strings.ToUpper(s.Mnemonic.Text) == "PCALIGN" { + pos += loong64PCAlignPad(pos, s) + explicit = true + continue + } + if pendingAlign { + pendingAlign = false + if pos&15 != 0 { + alignPad[s] = 16 - pos&15 + } + } + if alignInstrs[s] && pos&15 != 0 { + alignPad[s] = 16 - pos&15 + } + if !explicit { + for _, n := range pendingNames { + offsets[n] = pos + alignPad[s] + } + } + pendingNames = nil + explicit = false + pcs[s] = pos + alignPad[s] + pos += alignPad[s] + loong64InstrSize(s, fi) + } + } + return offsets, alignPad, pcs, pos +} + +// loong64BranchTarget reports the local label a branch-like instruction +// transfers to, the loop-head signal the toolchain derives from backward +// branch targets. +func loong64BranchTarget(instr *ast.Instr) (string, bool) { + mnem := strings.ToUpper(instr.Mnemonic.Text) + ops := instr.Operands + var op *ast.Operand + switch { + case mnem == "JMP" || mnem == "JAL" || mnem == "BFPT" || mnem == "BFPF": + if len(ops) != 1 { + return "", false + } + op = ops[0] + case mnem == "TEQ" || mnem == "TNE": + return "", false + case len(ops) >= 2: + op = ops[len(ops)-1] + default: + return "", false + } + if op.Kind == ast.OpAddr && op.Addr.Sym != nil && op.Addr.Sym.Pseudo == "" && + op.Addr.Base == "" && op.Addr.Sym.Name != "" { + return op.Addr.Sym.Name, true + } + return "", false +} + // loong64JumpChain precomputes jump-to-jump folding, mirroring the linker's // branch-chasing pass: a label whose first instruction is an unconditional // local jump redirects its own jumpers to the ultimate target. The Go @@ -177,21 +339,51 @@ func l64LabelOK(op *ast.Operand) (string, bool) { return "", false } +// l64SubToAdd rewrites the SUB family with an immediate first operand onto +// its ADD counterpart with the negated immediate: LoongArch has no +// subtract-immediate instructions, and the toolchain folds SUB $v into the +// ADD immediate form through the same optab matching (the $0 fold into 3R +// and the large-constant materialisations included). The negation is the +// second result; the operand is left untouched because the size pass +// normalises the same instruction. +func l64SubToAdd(mnem string, ops []*ast.Operand) (string, bool) { + if len(ops) >= 2 && isImmOperand(ops[0]) { + switch mnem { + case "SUB": + return "ADD", true + case "SUBW": + return "ADDW", true + case "SUBV", "SUBVU": + return "ADDV", true + } + } + return mnem, false +} + // loong64InstrSize returns the encoded size of an instruction: 4 bytes for // most, more for the multi-instruction expansions. func loong64InstrSize(instr *ast.Instr, fi loong64FrameInfo) int { mnem := strings.ToUpper(instr.Mnemonic.Text) ops := instr.Operands + var neg bool + mnem, neg = l64SubToAdd(mnem, ops) if mnem == "RET" { return len(loong64Return(fi)) } switch mnem { + case "TEQ", "TNE": + return 8 // bne/beq over the BREAK, then BREAK + case "PRELDX": + return 20 // the four-instruction constant materialisation + preldx case "MOV", "MOVB", "MOVH", "MOVW", "MOVV", "MOVBU", "MOVHU", "MOVWU", "MOVF", "MOVD": return loong64MovSize(mnem, ops, fi) case "ADD", "ADDW", "ADDV", "ADDVU", "AND", "OR", "XOR", "SGT", "SGTU": if len(ops) >= 2 && isImmOperand(ops[0]) { v := l64Imm64(ops[0]) + if neg { + v = -v + } if v == 0 { return 4 // folds into the 3R form (rk = R0) } @@ -229,11 +421,50 @@ func loong64InstrSize(instr *ast.Instr, fi loong64FrameInfo) int { return 4 } +// loong64PCAlignPad returns the padding PCALIGN inserts before the next +// instruction so that it starts at the requested boundary relative to the +// function start. The boundary must be a power of two between 8 and 2048, as +// the toolchain requires; anything else pads nothing. +func loong64PCAlignPad(pos int, instr *ast.Instr) int { + if len(instr.Operands) != 1 || !isImmOperand(instr.Operands[0]) { + return 0 + } + align := int(immFromOperand(instr.Operands[0])) + if align < 8 || align > 2048 || align&(align-1) != 0 { + return 0 + } + return (align - pos%align) % align +} + +// loong64PadBytes renders PCALIGN padding: the toolchain emits andi $0, $0, 0 +// (the architecture's NOP) for every full 4 bytes of pad. +func loong64PadBytes(pad int) []byte { + nop := l64wordLE(l64irr(l64DualTable["AND"].imm, 0, 0, 0)) + out := make([]byte, 0, pad/4*len(nop)) + for i := 0; i < pad/4; i++ { + out = append(out, nop...) + } + return out +} + // encodeLOONG64Instr encodes a single LoongArch instruction. -func encodeLOONG64Instr(instr *ast.Instr, pc int, offsets map[string]int, fi loong64FrameInfo, relocs *[]Reloc, resolve func(string) string) ([]byte, error) { +func encodeLOONG64Instr(instr *ast.Instr, pc int, offsets map[string]int, fi loong64FrameInfo, relocs *[]Reloc, resolve func(string) string, pcRelPcs map[*ast.Instr]int) ([]byte, error) { mnem := strings.ToUpper(instr.Mnemonic.Text) ops := instr.Operands + // The SUB family with an immediate first operand folds onto the ADD + // immediate form with the negated immediate; the negation happens on a + // copy of the operand, never on the shared syntax tree. + mnem, neg := l64SubToAdd(mnem, ops) + if neg { + c := *ops[0] + c.Imm.Val = -c.Imm.Val + ops2 := make([]*ast.Operand, len(ops)) + ops2[0] = &c + copy(ops2[1:], ops[1:]) + ops = ops2 + } + // Pseudo-instructions and the branches first. switch mnem { case "RET": @@ -249,10 +480,80 @@ func encodeLOONG64Instr(instr *ast.Instr, pc int, offsets map[string]int, fi loo return nil, fmt.Errorf("WORD expects 1 operand, got %d", len(ops)) } return l64wordLE(uint32(immFromOperand(ops[0]))), nil + case "NEGW", "NEGV": + // The integer negation pseudo is a subtract from zero: + // NEGW src, dst → sub.w r0, src, dst. + if len(ops) != 2 { + return nil, fmt.Errorf("%s expects 2 operands, got %d", mnem, len(ops)) + } + src, dst := l64Reg(ops[0]), l64Reg(ops[1]) + if src < 0 || dst < 0 { + return nil, fmt.Errorf("invalid register operand") + } + sub := l64InstrTable["SUBW"].op + if mnem == "NEGV" { + sub = l64InstrTable["SUBV"].op + } + return l64wordLE(l64rrr(sub, src, 0, dst)), nil + case "TEQ", "TNE": + // The trap pseudo expands to two instructions: bne/beq rj, rd over + // the BREAK (offset 2 instruction units), then BREAK $code. + if len(ops) != 2 && len(ops) != 3 { + return nil, fmt.Errorf("%s expects 2 or 3 operands, got %d", mnem, len(ops)) + } + code := int(immFromOperand(ops[0])) + rj, rd := 0, l64Reg(ops[len(ops)-1]) + if len(ops) == 3 { + rj = l64Reg(ops[1]) + } + if rj < 0 || rd < 0 { + return nil, fmt.Errorf("invalid register operand") + } + bop := l64branchTable["BNE"] + if mnem == "TNE" { + bop = l64branchTable["BEQ"] + } + return l64WordsLE( + l64irr16(bop, 2, rj, rd), + l64i15(l64InstrTable["BREAK"].op, code), + ), nil + case "PRELDX": + // preldx offset(Rbase), $n, $hint: the 64-bit descriptor n packs + // (addrSeq, blockSize, blockNums, stride); the constant v built from + // it materialises in R30 across four instructions, then the preldx. + if len(ops) != 3 || !isMemOperand(ops[0]) || !isImmOperand(ops[1]) || !isImmOperand(ops[2]) { + return nil, fmt.Errorf("PRELDX expects offset(reg), $n, $hint") + } + rj := loong64RegNum(ops[0].Addr.Base) + if rj < 0 { + return nil, fmt.Errorf("invalid register operand") + } + n := uint64(l64Imm64(ops[1])) + hint := int(l64Imm64(ops[2])) + addrSeq := (n >> 0) & 0x1 + blkSize := (n >> 1) & 0x7ff + blkNums := (n >> 12) & 0x1ff + stride := (n >> 21) & 0xffff + v := uint64(ops[0].Addr.Offset)&0xffff + addrSeq<<16 + + ((blkSize/16)-1)<<20 + (blkNums-1)<<32 + stride<<44 + const ( + lu12iw = 0x0a << 25 + lu32id = 0x0b << 25 + lu52id = 0x00c << 22 + ori = 0x00e << 22 + preldx = 0x7058 << 15 + ) + return l64WordsLE( + l64ir(lu12iw, int(uint32(v>>12)), 30), + l64irr(ori, int(uint32(v)), 30, 30), + l64ir(lu32id, int(uint32(v>>32)), 30), + l64irr(lu52id, int(uint32(v>>52)), 30, 30), + l64rrr(preldx, 30, rj, hint), + ), nil case "JMP", "B": - return encodeLOONG64Branch(instr, mnem, pc, offsets, false, resolve, relocs) + return encodeLOONG64Branch(instr, mnem, pc, offsets, false, resolve, relocs, pcRelPcs) case "JAL", "CALL", "BL": - return encodeLOONG64Branch(instr, mnem, pc, offsets, true, resolve, relocs) + return encodeLOONG64Branch(instr, mnem, pc, offsets, true, resolve, relocs, pcRelPcs) case "MOV", "MOVB", "MOVH", "MOVW", "MOVV", "MOVBU", "MOVHU", "MOVWU", "MOVF", "MOVD": return encodeLOONG64Mov(instr, mnem, fi, relocs) } @@ -262,12 +563,12 @@ func encodeLOONG64Instr(instr *ast.Instr, pc int, offsets map[string]int, fi loo if mnem == "JIRL" { return encodeLOONG64Jirl(op, ops) } - return encodeLOONG64Branch16(mnem, op, ops, pc, offsets, resolve) + return encodeLOONG64Branch16(instr, mnem, op, ops, pc, offsets, resolve, pcRelPcs) } // Single-register branches with 21-bit offsets (BLTZ/BGEZ/BLEZ/BGTZ, // BFPT/BFPF; BEQZ/BNEZ are reached through BEQ/BNE with R0). if op, ok := l64branch21Table[mnem]; ok { - return encodeLOONG64Branch21(mnem, op, ops, pc, offsets, resolve) + return encodeLOONG64Branch21(instr, mnem, op, ops, pc, offsets, resolve, pcRelPcs) } // B/BL aliases reached only via JMP/JAL above. @@ -533,11 +834,23 @@ func encodeLOONG64Instr(instr *ast.Instr, pc int, offsets map[string]int, fi loo // // JMP/B label → b label JMP/B (rj) → jirl r0, rj, 0 // JAL/CALL/BL label → bl label JAL/CALL/BL (rj) → jirl r1, rj, 0 -func encodeLOONG64Branch(instr *ast.Instr, mnem string, pc int, offsets map[string]int, link bool, resolve func(string) string, relocs *[]Reloc) ([]byte, error) { +func encodeLOONG64Branch(instr *ast.Instr, mnem string, pc int, offsets map[string]int, link bool, resolve func(string) string, relocs *[]Reloc, pcRelPcs map[*ast.Instr]int) ([]byte, error) { if len(instr.Operands) != 1 { return nil, fmt.Errorf("%s expects 1 operand, got %d", mnem, len(instr.Operands)) } op := instr.Operands[0] + // PC-relative displacement: N(PC) resolves to the instruction N slots + // away in source order (the toolchain's parse-time count), and the field + // carries the final pc distance in instruction units. + if op.Addr.Sym == nil && op.Addr.Base == "PC" { + targetPc, ok := pcRelPcs[instr] + if !ok { + return nil, fmt.Errorf("%s: PC-relative target %d out of range", mnem, op.Addr.Offset) + } + v := (targetPc - pc) >> 2 + opc := l64jumpTable[mnem] + return l64wordLE(l64bbl(opc, v)), nil + } if isMemOperand(op) && op.Addr.Base != "" && op.Addr.Index == "" && op.Addr.Sym == nil { // Indirect: (rj) → jirl. rj := loong64RegNum(op.Addr.Base) @@ -620,16 +933,28 @@ func l64offsetOperand(op *ast.Operand) (int32, bool) { // encodeLOONG64Branch16 encodes a 16-bit branch (BEQ/BNE/BLT/BGE/BLTU/BGEU): // INSTR rj, rd, label, or INSTR rj, label with rd = R0, which the toolchain // turns into the 21-bit BEQZ/BNEZ form when the register is the only operand. -func encodeLOONG64Branch16(mnem string, op uint32, ops []*ast.Operand, pc int, offsets map[string]int, resolve func(string) string) ([]byte, error) { +func encodeLOONG64Branch16(instr *ast.Instr, mnem string, op uint32, ops []*ast.Operand, pc int, offsets map[string]int, resolve func(string) string, pcRelPcs map[*ast.Instr]int) ([]byte, error) { if len(ops) != 2 && len(ops) != 3 { return nil, fmt.Errorf("%s expects 2 or 3 operands, got %d", mnem, len(ops)) } - target := resolve(l64Label(ops[len(ops)-1])) - targetOff, ok := offsets[target] - if !ok { - return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + var target string + var v int + lastOp := ops[len(ops)-1] + if lastOp.Kind == ast.OpAddr && lastOp.Addr.Sym == nil && lastOp.Addr.Base == "PC" { + // N(PC) resolves to the instruction N slots away in source order. + targetPc, ok := pcRelPcs[instr] + if !ok { + return nil, fmt.Errorf("%s: PC-relative target %d out of range", mnem, lastOp.Addr.Offset) + } + v = (targetPc - pc) >> 2 + } else { + target = resolve(l64Label(lastOp)) + targetOff, ok := offsets[target] + if !ok { + return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + } + v = (targetOff - pc) >> 2 } - v := (targetOff - pc) >> 2 if len(ops) == 2 { // Single register: BEQ rj, label → beqz (21-bit), and the BLTZ/ // BGEZ-family aliases encoded with rj in the rj field. @@ -690,33 +1015,55 @@ func encodeLOONG64Branch16(mnem string, op uint32, ops []*ast.Operand, pc int, o // BFPT/BFPF use the 21-bit offset form (register in the rj field), while // BGTZ/BLEZ, which the toolchain encodes with the register in the rd field // and a 16-bit offset, are handled separately. -func encodeLOONG64Branch21(mnem string, op uint32, ops []*ast.Operand, pc int, offsets map[string]int, resolve func(string) string) ([]byte, error) { - if len(ops) != 2 { +func encodeLOONG64Branch21(instr *ast.Instr, mnem string, op uint32, ops []*ast.Operand, pc int, offsets map[string]int, resolve func(string) string, pcRelPcs map[*ast.Instr]int) ([]byte, error) { + isBF := mnem == "BFPT" || mnem == "BFPF" + if len(ops) != 2 && !(isBF && (len(ops) == 1 || len(ops) == 2)) { return nil, fmt.Errorf("%s expects 2 operands, got %d", mnem, len(ops)) } - target := resolve(l64Label(ops[1])) - targetOff, ok := offsets[target] - if !ok { - return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) - } - v := (targetOff - pc) >> 2 - rj := 0 // BFPT/BFPF default to FCC0 - if mnem != "BFPT" && mnem != "BFPF" { + var rj int + tgtOp := ops[len(ops)-1] + if isBF { + // BFPT/BFPF test an FCC condition register, defaulting to FCC0 when + // spelled without one. + rj = 0 + if len(ops) == 2 { + rj = l64Reg(ops[0]) + if rj < 0 { + return nil, fmt.Errorf("invalid register operand") + } + } + } else { rj = l64Reg(ops[0]) if rj < 0 { return nil, fmt.Errorf("invalid register operand") } } + var v int + if tgtOp.Kind == ast.OpAddr && tgtOp.Addr.Sym == nil && tgtOp.Addr.Base == "PC" { + // N(PC) resolves to the instruction N slots away in source order. + targetPc, ok := pcRelPcs[instr] + if !ok { + return nil, fmt.Errorf("%s: PC-relative target %d out of range", mnem, tgtOp.Addr.Offset) + } + v = (targetPc - pc) >> 2 + } else { + target := resolve(l64Label(tgtOp)) + targetOff, ok := offsets[target] + if !ok { + return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + } + v = (targetOff - pc) >> 2 + } if mnem == "BGTZ" || mnem == "BLEZ" { // The toolchain swaps the register into the rd field and keeps the // 16-bit offset form. if (v<<16)>>16 != v { - return nil, fmt.Errorf("branch to %q too far (16-bit range)", target) + return nil, fmt.Errorf("branch %d too far (16-bit range)", v) } return l64wordLE(l64irr16(op, v, 0, rj)), nil } if (v<<11)>>11 != v { - return nil, fmt.Errorf("branch to %q too far (21-bit range)", target) + return nil, fmt.Errorf("branch %d too far (21-bit range)", v) } return l64wordLE(l64ir21(op, v, rj)), nil } diff --git a/asm/riscv_assemble.go b/asm/riscv_assemble.go index 4724eff..b790bcd 100644 --- a/asm/riscv_assemble.go +++ b/asm/riscv_assemble.go @@ -4,7 +4,9 @@ package asm import ( + "errors" "fmt" + "slices" "strings" "sourcedock.dev/petrbalvin/gasm-devkit/ast" @@ -31,32 +33,54 @@ func assembleRISCV(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, [ } // Pass 1: collect instructions and compute label offsets assuming 4 bytes - // per instruction (or 8 for MOV $large-imm). No encoding yet. + // per instruction (or 8 for MOV $large-imm). No encoding yet. PCALIGN + // contributes only its padding, which is attached to the following + // instruction and emitted ahead of it. A relaxed branch carries the + // inverted condition and is followed by an inserted JMP rec (jmpTo set) + // that carries the original target. type instrRec struct { instr *ast.Instr compressed bool code []byte + pad int + relaxed bool + jmpTo string } var recs []instrRec offsets := map[string]int{} pos := guardLen + len(prologue) + pendingPad := 0 for _, stmt := range t.Body { switch s := stmt.(type) { case *ast.Label: offsets[s.Name.Text] = pos case *ast.Instr: - recs = append(recs, instrRec{instr: s}) + if strings.ToUpper(s.Mnemonic.Text) == "PCALIGN" { + pendingPad += riscvPCAlignPad(pos, s) + pos += riscvPCAlignPad(pos, s) + continue + } + recs = append(recs, instrRec{instr: s, pad: pendingPad}) + pendingPad = 0 pos += riscvInstrSize(s, fi) } } - // Pass 2: encode each instruction using Pass-1 offsets. + // Pass 2: encode each instruction using Pass-1 offsets. A branch or + // jump the offsets prove overlong encodes to a 4-byte placeholder: the + // relaxation pass rewrites it before the final encoding. pcRelPcs is + // unavailable this early, so the N(PC) forms take the same placeholder + // path. pc := len(prologue) for i := range recs { - code, err := encodeRISCVInstr(recs[i].instr, pc, offsets, fi, nil) // no relocs in Pass 2 - if err != nil { + branchLike := isBranchLike(recs[i].instr.Mnemonic.Text) || riscvIsCondBranch(recs[i].instr.Mnemonic.Text) + code, err := encodeRISCVInstr(recs[i].instr, pc, offsets, fi, nil, nil) // no relocs in Pass 2 + if err != nil && !(branchLike && riscvIsRangeError(err)) { return nil, nil, nil, nil, nil, fmt.Errorf("%s: %w", recs[i].instr.Mnemonic.Text, err) } + if err != nil { + code = make([]byte, 4) + } recs[i].code = code pc += len(code) } @@ -72,20 +96,100 @@ func assembleRISCV(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, [ // Pass 4: recompute offsets with actual sizes. recs holds the // instructions in emission order, so an index into it walks t.Body in // lockstep (the same single pass Pass 1 uses) instead of rescanning the - // whole slice per statement. + // whole slice per statement. PCALIGN padding is recomputed here, since + // compression has shifted instruction sizes since Pass 1. offsets = map[string]int{} pos = guardLen + len(prologue) ri := 0 + pendingPad = 0 for _, stmt := range t.Body { switch s := stmt.(type) { case *ast.Label: offsets[s.Name.Text] = pos case *ast.Instr: + if strings.ToUpper(s.Mnemonic.Text) == "PCALIGN" { + pad := riscvPCAlignPad(pos, s) + pendingPad += pad + pos += pad + continue + } + recs[ri].pad = pendingPad + pendingPad = 0 pos += len(recs[ri].code) ri++ } } + // Pass 4b: relax overlong conditional branches exactly as the toolchain + // does: invert the branch condition, point it at the instruction after an + // inserted JMP, let the JMP carry the original target, and re-layout until + // a pass inserts nothing. Inserted JMP recs share their branch's source + // line and trail it in emission order, so the body walk flushes them + // before every statement and at the end. + var pcRelPcs map[*ast.Instr]int + for { + offsets = map[string]int{} + pos = guardLen + len(prologue) + ri := 0 + pcs := make([]int, len(recs)) + flushJmps := func() { + for ri < len(recs) && recs[ri].jmpTo != "" { + pcs[ri] = pos + pos += 4 + ri++ + } + } + for _, stmt := range t.Body { + switch s := stmt.(type) { + case *ast.Label: + flushJmps() + offsets[s.Name.Text] = pos + case *ast.Instr: + flushJmps() + if ri >= len(recs) { + continue + } + pcs[ri] = pos + recs[ri].pad + pos += recs[ri].pad + len(recs[ri].code) + ri++ + } + } + flushJmps() + + changed := false + for i := range recs { + r := &recs[i] + if r.relaxed || r.jmpTo != "" { + continue + } + mnem := strings.ToUpper(r.instr.Mnemonic.Text) + if !riscvIsCondBranch(mnem) || len(r.instr.Operands) == 0 { + continue + } + target := labelFromOperand(r.instr.Operands[len(r.instr.Operands)-1]) + targetOff, ok := offsets[target] + if !ok { + continue + } + if delta := int32(targetOff - pcs[i]); delta < -4096 || delta >= 4096 { + r.relaxed = true + recs = slices.Insert(recs, i+1, instrRec{instr: r.instr, jmpTo: target}) + changed = true + } + } + if !changed { + // Capture the final pcs for the N(PC) branch forms: their target + // is the instruction N source slots away, resolved by index. + pcRelPcs = map[*ast.Instr]int{} + for i := range recs { + if _, ok := riscvPCRelOffset(recs[i].instr); ok { + pcRelPcs[recs[i].instr] = pcs[i] + } + } + break + } + } + // Pass 5: re-encode branches with corrected offsets. Record relocations // during this final pass (relocation offsets are relative to instruction // start). The guard prefix precedes the prologue; its branches target @@ -104,12 +208,40 @@ func assembleRISCV(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, [ preCount := len(relocs) var lines []LineEntry for _, r := range recs { + // PCALIGN padding precedes the instruction it was attached to. + if r.pad > 0 { + out = append(out, riscvPadBytes(r.pad)...) + pc += r.pad + } lines = append(lines, LineEntry{Offset: pc, Line: r.instr.Pos().Line}) - if r.compressed && !isBranchLike(r.instr.Mnemonic.Text) { - out = append(out, r.code...) - pc += len(r.code) - } else { - code, err := encodeRISCVInstr(r.instr, pc, offsets, fi, &relocs) + var code []byte + switch { + case r.jmpTo != "": + // The JMP a relaxation inserted: JAL X0 to the original target. + targetOff, ok := offsets[r.jmpTo] + if !ok { + return nil, nil, nil, nil, nil, fmt.Errorf("undefined label %q", r.jmpTo) + } + offset := int32(targetOff - pc) + if err := riscvCheckJumpOffset(r.jmpTo, offset); err != nil { + return nil, nil, nil, nil, nil, err + } + word := riscvJType(0, offset) + code = []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)} + case r.relaxed: + // The inverted half of a relaxed branch: it targets the inserted + // JMP, always the very next instruction (offset 4). + enc, rs1, rs2, ok := riscvInvertedBranchEnc(strings.ToUpper(r.instr.Mnemonic.Text), r.instr.Operands) + if !ok { + return nil, nil, nil, nil, nil, fmt.Errorf("%s: cannot relax branch", r.instr.Mnemonic.Text) + } + word := riscvBType(enc, rs1, rs2, 4) + code = []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)} + case r.compressed && !isBranchLike(r.instr.Mnemonic.Text): + code = r.code + default: + var err error + code, err = encodeRISCVInstr(r.instr, pc, offsets, fi, &relocs, pcRelPcs) if err != nil { return nil, nil, nil, nil, nil, err } @@ -131,9 +263,9 @@ func assembleRISCV(t *ast.Text) ([]byte, map[string]int, []Reloc, []LineEntry, [ if strings.ToUpper(r.instr.Mnemonic.Text) == "RET" && fi.autosize != 0 { spadj = append(spadj, SpadjStep{PC: pc + riscvReturnEpilogueLen(fi), Value: 0}) } - out = append(out, code...) - pc += len(code) } + out = append(out, code...) + pc += len(code) } if fi.needSplit { relocs = append(relocs, guardReloc) @@ -150,6 +282,8 @@ var riscvImmAlias = map[string]string{ "AND": "ANDI", "OR": "ORI", "XOR": "XORI", + "SLT": "SLTI", + "SLTU": "SLTIU", "SLL": "SLLI", "SRL": "SRLI", "SRA": "SRAI", @@ -178,6 +312,35 @@ func riscvNormaliseImmAlias(mnem string, ops []*ast.Operand) (string, bool) { return mnem, false } +// riscvPCAlignPad returns the padding PCALIGN inserts before the next +// instruction so that it starts at the requested boundary relative to the +// function start. The boundary must be a power of two between 8 and 2048, as +// the toolchain requires; anything else pads nothing. +func riscvPCAlignPad(pos int, instr *ast.Instr) int { + if len(instr.Operands) != 1 || !isImmOperand(instr.Operands[0]) { + return 0 + } + align := int(immFromOperand(instr.Operands[0])) + if align < 8 || align > 2048 || align&(align-1) != 0 { + return 0 + } + return (align - pos%align) % align +} + +// riscvPadBytes renders PCALIGN padding: 4-byte NOPs (addi $0, X0, X0) with a +// trailing 2-byte compressed NOP when the pad is 2 mod 4, exactly as the +// toolchain lays the bytes down. +func riscvPadBytes(pad int) []byte { + out := make([]byte, 0, pad) + for ; pad >= 4; pad -= 4 { + out = append(out, 0x13, 0x00, 0x00, 0x00) + } + if pad == 2 { + out = append(out, 0x01, 0x00) + } + return out +} + // riscvInstrSize returns the encoded size in bytes of a RISC-V instruction. // Most instructions are 4 bytes; MOV with a large immediate and I-type // arithmetic with a large immediate expand to several (possibly compressed) @@ -224,6 +387,10 @@ func riscvInstrSize(instr *ast.Instr, fi riscvFrameInfo) int { } return riscvItypeImmediateSize(mnem, imm) } + // BYTE lays down one raw byte per operand. + if mnem == "BYTE" { + return len(ops) + } // The toolchain's synthesised instructions: some emit one word, others // expand to a fixed sequence. return riscvExtendedSize(mnem, ops) @@ -318,12 +485,171 @@ func isBranchLike(mnem string) bool { return false } +// riscvIsCondBranch reports whether m is a conditional branch, the only +// instruction class branch relaxation rewrites. +func riscvIsCondBranch(mnem string) bool { + switch mnem { + case "BEQ", "BNE", "BLT", "BGE", "BLTU", "BGEU", "BGT", "BLE", "BGTU", "BLEU", + "BEQZ", "BNEZ", "BLTZ", "BGEZ", "BLEZ", "BGTZ": + return true + } + return false +} + +// riscvCSRNames maps the standard CSR mnemonics the assembler accepts onto +// their addresses. +var riscvCSRNames = map[string]int32{ + "FFLAGS": 0x001, + "FRM": 0x002, + "FCSR": 0x003, + "VSTART": 0x008, + "VXSAT": 0x009, + "VXRM": 0x00A, + "VCSR": 0x00F, + "CYCLE": 0xC00, + "TIME": 0xC01, + "INSTRET": 0xC02, + "CYCLEH": 0xC80, + "TIMEH": 0xC81, + "INSTRETH": 0xC82, + "VL": 0xC20, + "VLENB": 0xC22, +} + +// riscvCSRAddress resolves a CSR operand: an integer immediate or one of the +// standard CSR names. +func riscvCSRAddress(op *ast.Operand) (int32, bool) { + if isImmOperand(op) { + return immFromOperand(op), true + } + if op.Addr.Sym != nil { + if v, ok := riscvCSRNames[strings.ToUpper(op.Addr.Sym.Name)]; ok { + return v, true + } + } + return 0, false +} + +// riscvPCRelOffset reports the N of a branch or jump operand spelled N(PC): +// the displacement counted in source instructions from the branch itself. +func riscvPCRelOffset(instr *ast.Instr) (int, bool) { + mnem := strings.ToUpper(instr.Mnemonic.Text) + switch mnem { + case "JMP": + if len(instr.Operands) != 1 { + return 0, false + } + case "JAL": + if len(instr.Operands) != 1 && len(instr.Operands) != 2 { + return 0, false + } + case "BEQ", "BNE", "BLT", "BGE", "BLTU", "BGEU", + "BEQZ", "BNEZ", "BLTZ", "BGEZ", "BLEZ", "BGTZ": + if len(instr.Operands) < 2 { + return 0, false + } + default: + return 0, false + } + op := instr.Operands[len(instr.Operands)-1] + if op.Kind == ast.OpAddr && op.Addr.Sym == nil && op.Addr.Base == "PC" { + return int(op.Addr.Offset), true + } + return 0, false +} + +// riscvPCRelTargetOff resolves the target displacement of a branch whose last +// operand is N(PC): the toolchain's parser counts the source instructions at +// a uniform 4 bytes, so the target is the instruction N slots away, and the +// displacement tracks that instruction's final pc. A nil pcRelPcs (the +// layout passes) yields a placeholder range error; the caller tolerates it +// for branch-like instructions. +func riscvPCRelTargetOff(instr *ast.Instr, pc int, pcRelPcs map[*ast.Instr]int) (int, bool, error) { + off, ok := riscvPCRelOffset(instr) + if !ok { + return 0, false, nil + } + if pcRelPcs == nil { + return 0, true, &riscvRangeError{"pc-relative placeholder"} + } + targetPc, ok := pcRelPcs[instr] + if !ok { + return 0, true, fmt.Errorf("PC-relative target %d out of range", off) + } + return targetPc, true, nil +} + +// riscvInvertedBranchEnc returns the encoding of mnem's inverted condition +// for the given operands: InvertBranch's table applied at the encoding level. +// The register operands are already in position for the inverted form. +func riscvInvertedBranchEnc(mnem string, ops []*ast.Operand) (riscvEnc, int, int, bool) { + reg := func(i int) int { return regFromOperand(ops[i]) } + switch mnem { + case "BEQ": // → BNE rs1, rs2 + return riscvEnc{0x63, 0x1, 0x00}, reg(0), reg(1), true + case "BNE": // → BEQ rs1, rs2 + return riscvEnc{0x63, 0x0, 0x00}, reg(0), reg(1), true + case "BLT": // → BGE rs1, rs2 + return riscvEnc{0x63, 0x5, 0x00}, reg(0), reg(1), true + case "BGE": // → BLT rs1, rs2 + return riscvEnc{0x63, 0x4, 0x00}, reg(0), reg(1), true + case "BLTU": // → BGEU rs1, rs2 + return riscvEnc{0x63, 0x7, 0x00}, reg(0), reg(1), true + case "BGEU": // → BLTU rs1, rs2 + return riscvEnc{0x63, 0x6, 0x00}, reg(0), reg(1), true + case "BEQZ": // → BNEZ rs, X0 + return riscvEnc{0x63, 0x1, 0x00}, reg(0), 0, true + case "BNEZ": // → BEQZ rs, X0 + return riscvEnc{0x63, 0x0, 0x00}, reg(0), 0, true + case "BLTZ": // → BGEZ rs, X0 + return riscvEnc{0x63, 0x5, 0x00}, reg(0), 0, true + case "BGEZ": // → BLTZ rs, X0 + return riscvEnc{0x63, 0x4, 0x00}, reg(0), 0, true + case "BLEZ": // → BGTZ: blt X0, rs + return riscvEnc{0x63, 0x4, 0x00}, 0, reg(0), true + case "BGTZ": // → BLEZ: bge X0, rs + return riscvEnc{0x63, 0x5, 0x00}, 0, reg(0), true + case "BGT": // → BLE: bge rs2, rs1 + return riscvEnc{0x63, 0x5, 0x00}, reg(1), reg(0), true + case "BLE": // → BGT: blt rs2, rs1 + return riscvEnc{0x63, 0x4, 0x00}, reg(1), reg(0), true + case "BGTU": // → BLEU: bgeu rs2, rs1 + return riscvEnc{0x63, 0x7, 0x00}, reg(1), reg(0), true + case "BLEU": // → BGTU: bltu rs2, rs1 + return riscvEnc{0x63, 0x6, 0x00}, reg(1), reg(0), true + } + return riscvEnc{}, 0, 0, false +} + +// riscvRangeError reports a branch or jump displacement beyond its +// architecture limit. The layout passes tolerate it (the relaxation pass +// rewrites overlong conditional branches before the final encoding); a range +// error reaching the final pass is a real failure. +type riscvRangeError struct{ msg string } + +func (e *riscvRangeError) Error() string { return e.msg } + +// riscvIsRangeError reports whether err is a displacement-range rejection. +func riscvIsRangeError(err error) bool { + var re *riscvRangeError + return errors.As(err, &re) +} + +// riscvRoundModes maps the rounding-mode suffixes onto their funct7 codes. +var riscvRoundModes = map[string]uint32{ + "RNE": 0, + "RTZ": 1, + "RDN": 2, + "RUP": 3, + "RMM": 4, +} + // riscvCheckBranchOffset rejects a B-type displacement outside its signed // 13-bit span [-4096, 4094]; the encoder masks to 13 bits, so an // out-of-range offset would otherwise wrap to a wrong target. func riscvCheckBranchOffset(target string, off int32) error { if off < -4096 || off > 4094 { - return fmt.Errorf("branch to %q too far (13-bit range)", target) + return &riscvRangeError{fmt.Sprintf("branch to %q too far (13-bit range)", target)} } return nil } @@ -332,13 +658,13 @@ func riscvCheckBranchOffset(target string, off int32) error { // 21-bit span [-1048576, 1048574]. func riscvCheckJumpOffset(target string, off int32) error { if off < -1048576 || off > 1048574 { - return fmt.Errorf("jump to %q too far (21-bit range)", target) + return &riscvRangeError{fmt.Sprintf("jump to %q too far (21-bit range)", target)} } return nil } // encodeRISCVInstr encodes a single RISC-V instruction. -func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscvFrameInfo, relocs *[]Reloc) ([]byte, error) { +func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscvFrameInfo, relocs *[]Reloc, pcRelPcs map[*ast.Instr]int) ([]byte, error) { mnem := instr.Mnemonic.Text ops := instr.Operands var immNeg bool @@ -351,6 +677,27 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv // RET = epilogue (restore LR and close the frame when present) + // uncompressed JALR X0, 0(X1) (the toolchain never compresses RET). return riscvReturn(fi), nil + case "WORD": + // WORD $w lays down a raw 32-bit little-endian word. + if len(ops) != 1 { + return nil, fmt.Errorf("WORD expects 1 operand, got %d", len(ops)) + } + w := int64(immFromOperand(ops[0])) + if w < 0 || w > 0xFFFFFFFF { + return nil, fmt.Errorf("WORD: immediate %d does not fit a 32-bit word", w) + } + return []byte{byte(w), byte(w >> 8), byte(w >> 16), byte(w >> 24)}, nil + case "BYTE": + // BYTE $b lays down one raw byte per operand. + var out []byte + for _, op := range ops { + b := int64(immFromOperand(op)) + if b < 0 || b > 0xFF { + return nil, fmt.Errorf("BYTE: immediate %d does not fit a byte", b) + } + out = append(out, byte(b)) + } + return out, nil case "CALL": // CALL sym(SB) → JAL X1, sym(SB) with a single R_RISCV_JAL // relocation. The Go assembler rejects CALL to a local branch label. @@ -404,6 +751,17 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv word = riscvIType(riscvEnc{0x67, 0x0, 0x00}, 0, rs1, 0) return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil } + if off, isPCRel, err := riscvPCRelTargetOff(instr, pc, pcRelPcs); isPCRel { + if err != nil { + return nil, err + } + offset := int32(off - pc) + if err := riscvCheckJumpOffset("", offset); err != nil { + return nil, err + } + word = riscvJType(0, offset) + return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil + } } targetOff, ok := offsets[target] if !ok { @@ -424,6 +782,18 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv } else if len(ops) == 1 { target = labelFromOperand(ops[0]) } + if off, isPCRel, err := riscvPCRelTargetOff(instr, pc, pcRelPcs); isPCRel { + if err != nil { + return nil, err + } + targetOff := off + offset := int32(targetOff - pc) + if err := riscvCheckJumpOffset(target, offset); err != nil { + return nil, err + } + word = riscvJType(rd, offset) + return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil + } targetOff, ok := offsets[target] if !ok { return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) @@ -456,10 +826,20 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv if rs < 0 { return nil, fmt.Errorf("%s: invalid register", mnem) } - target := labelFromOperand(ops[1]) - targetOff, ok := offsets[target] - if !ok { - return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + targetOff := 0 + target := "" + if off, isPCRel, err := riscvPCRelTargetOff(instr, pc, pcRelPcs); isPCRel { + if err != nil { + return nil, err + } + targetOff = off + } else { + target = labelFromOperand(ops[1]) + var ok bool + targetOff, ok = offsets[target] + if !ok { + return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + } } var enc riscvEnc rs1, rs2 := rs, 0 @@ -484,18 +864,25 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil // System instructions with no operands. - case "FENCE", "ECALL", "EBREAK": + case "FENCE", "ECALL", "EBREAK", "FENCE.TSO", "PAUSE": enc, ok := riscvInstrTable[mnem] if !ok { return nil, fmt.Errorf("unsupported system instruction %q", mnem) } // The bare FENCE expands to fence iorw, iorw: the predecessor and // successor fields both carry 0xF in the I-type immediate - // (the toolchain's encodeFenceOperand TYPE_NONE default). + // (the toolchain's encodeFenceOperand TYPE_NONE default). FENCE.TSO + // carries the TSO fence mode with RW predecessor and successor. imm := int32(0) if mnem == "FENCE" { imm = 0x0FF } + if mnem == "FENCE.TSO" { + imm = 0x833 + } + if mnem == "PAUSE" { + imm = 0x010 + } word = riscvIType(enc, 0, 0, imm) return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil } @@ -516,6 +903,29 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil } + // FP conversions with an explicit rounding mode: FCVTWS.RNE and friends + // suffix the base mnemonic with RNE/RTZ/RDN/RUP/RMM, which lands in the + // low three bits of the funct7 field. + if i := strings.IndexByte(mnem, '.'); i > 0 { + if base, ok := riscvCvtTable[mnem[:i]]; ok { + rm, ok := riscvRoundModes[mnem[i+1:]] + if !ok { + return nil, fmt.Errorf("unsupported rounding mode in %q", mnem) + } + if len(ops) != 2 { + return nil, fmt.Errorf("%s expects 2 operands, got %d", mnem, len(ops)) + } + rs1 := regFromOperand(ops[0]) + rd := regFromOperand(ops[1]) + if rd < 0 || rs1 < 0 { + return nil, fmt.Errorf("invalid operand in %s", mnem) + } + base.funct7 = (base.funct7 &^ 7) | rm + word := riscvCvtType(base, rd, rs1) + return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil + } + } + // R4-type fused multiply-add: INSTR rs1, rs2, rs3, rd (destination last). if fmaEnc, ok := riscvFmaTable[mnem]; ok { if len(ops) != 4 { @@ -532,29 +942,103 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv return []byte{byte(word), byte(word >> 8), byte(word >> 16), byte(word >> 24)}, nil } - // CSR instructions: INSTR csr, rs1|uimm, rd (destination last). - if csrEnc, ok := riscvCsrTable[mnem]; ok { - if len(ops) != 3 { + // CSR instructions: INSTR csr, rs1|uimm, rd (destination last). The + // write-only pseudos (CSRS/CSRC/CSRW and their immediate forms) spell + // the source first, the CSR second, and read the destination as X0; the + // immediate or register variant follows the source operand's kind. + csrMnem := mnem + csrPseudo := false + csrRead := false + csrFix := int32(0) + switch mnem { + case "CSRS", "CSRW", "CSRC", "CSRSI", "CSRWI", "CSRCI": + csrMnem = map[string]string{ + "CSRS": "CSRRS", "CSRW": "CSRRW", "CSRC": "CSRRC", + "CSRSI": "CSRRSI", "CSRWI": "CSRRWI", "CSRCI": "CSRRCI", + }[mnem] + csrPseudo = true + // CSRR csr, rd is CSRRS rd, csr, X0; the read pseudos RDCYCLE/RDTIME/ + // RDINSTRET fix the CSR to cycle/time/instret. + case "CSRR": + csrMnem = "CSRRS" + csrPseudo = true + csrRead = true + case "RDCYCLE", "RDTIME", "RDINSTRET": + csrMnem = "CSRRS" + csrPseudo = true + csrRead = true + csrFix = map[string]int32{"RDCYCLE": 0xC00, "RDTIME": 0xC01, "RDINSTRET": 0xC02}[mnem] + } + if csrEnc, ok := riscvCsrTable[csrMnem]; ok { + if csrRead && len(ops) != 1 && len(ops) != 2 { + return nil, fmt.Errorf("%s expects 1 or 2 operands, got %d", mnem, len(ops)) + } + if csrPseudo && !csrRead && len(ops) != 2 { + return nil, fmt.Errorf("%s expects 2 operands, got %d", mnem, len(ops)) + } + if !csrPseudo && len(ops) != 3 { return nil, fmt.Errorf("%s expects 3 operands, got %d", mnem, len(ops)) } - csr := immFromOperand(ops[0]) // CSR address (12-bit) + csrOp := ops[0] + srcOp := ops[0] + rdOp := ops[len(ops)-1] + switch { + case csrRead: + // CSRR csr, rd (or the fixed-CSR read pseudos with only rd). + case csrPseudo: + // src, csr. + if len(ops) > 1 { + csrOp, srcOp = ops[1], ops[0] + } + rdOp = nil + default: + // Either src, csr, rd or csr, src, rd: a CSR *name* in the + // second operand marks the toolchain's order. + srcOp = ops[1] + if op := ops[1]; op.Addr.Sym != nil { + if _, ok := riscvCSRNames[strings.ToUpper(op.Addr.Sym.Name)]; ok { + csrOp, srcOp = ops[1], ops[0] + } + } + } + csr, ok := riscvCSRAddress(csrOp) + if !ok && csrFix == 0 { + return nil, fmt.Errorf("%s: unknown CSR %q", mnem, csrOp.Raw) + } + if csrFix != 0 { + csr = csrFix + } if csr < 0 || csr > 0xFFF { return nil, fmt.Errorf("%s: CSR address %d out of range 0-0xFFF", mnem, csr) } - rd := regFromOperand(ops[2]) // destination register - if rd < 0 { - return nil, fmt.Errorf("invalid destination register in %s", mnem) + rd := 0 + if !csrPseudo { + rd = regFromOperand(rdOp) // destination register + if rd < 0 { + return nil, fmt.Errorf("invalid destination register in %s", mnem) + } + } + if csrRead { + rd = regFromOperand(rdOp) + if rd < 0 { + return nil, fmt.Errorf("invalid destination register in %s", mnem) + } } var src int - if csrEnc.imm { - // Immediate variant: ops[1] is a 5-bit unsigned immediate. - src = int(immFromOperand(ops[1])) + switch { + case csrRead: + // CSRR reads with rs1 = X0: src stays zero. + case isImmOperand(srcOp): + // Immediate variant: the source is a 5-bit unsigned immediate. + src = int(immFromOperand(srcOp)) if src < 0 || src > 31 { return nil, fmt.Errorf("%s: uimm out of range 0-31", mnem) } - } else { - // Register variant: ops[1] is a register. - src = regFromOperand(ops[1]) + case csrEnc.imm: + return nil, fmt.Errorf("%s expects an immediate source", mnem) + default: + // Register variant: the source is a register. + src = regFromOperand(srcOp) if src < 0 { return nil, fmt.Errorf("invalid source register in %s", mnem) } @@ -738,14 +1222,34 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv } word = riscvSType(enc, rs1, rs2, imm) - // Branches: rs1, rs2, label. + // Branches: rs1, rs2, label. BGT/BLE/BGTU/BLEU are the swapped-spelling + // forms of BLT/BGE/BLTU/BGEU (bgt rs1, rs2 is blt rs2, rs1). case len(ops) == 3 && isBranchInstr(mnem): rs1 := regFromOperand(ops[0]) rs2 := regFromOperand(ops[1]) target := labelFromOperand(ops[2]) - targetOff, ok := offsets[target] - if !ok { - return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + switch mnem { + case "BGT": + enc, rs1, rs2 = riscvEnc{0x63, 0x4, 0x00}, rs2, rs1 // blt rs2, rs1 + case "BLE": + enc, rs1, rs2 = riscvEnc{0x63, 0x5, 0x00}, rs2, rs1 // bge rs2, rs1 + case "BGTU": + enc, rs1, rs2 = riscvEnc{0x63, 0x6, 0x00}, rs2, rs1 // bltu rs2, rs1 + case "BLEU": + enc, rs1, rs2 = riscvEnc{0x63, 0x7, 0x00}, rs2, rs1 // bgeu rs2, rs1 + } + targetOff := 0 + if off, isPCRel, err := riscvPCRelTargetOff(instr, pc, pcRelPcs); isPCRel { + if err != nil { + return nil, err + } + targetOff = off + } else { + var ok bool + targetOff, ok = offsets[target] + if !ok { + return nil, fmt.Errorf("undefined label %q%s", target, suggestLabel(target, offsets)) + } } offset := int32(targetOff - pc) if rs1 < 0 || rs2 < 0 { @@ -758,10 +1262,15 @@ func encodeRISCVInstr(instr *ast.Instr, pc int, offsets map[string]int, fi riscv // The Go assembler never compresses branches to C.BEQZ/C.BNEZ. word = riscvBType(enc, rs1, rs2, offset) - // U-type: rd, imm. + // U-type: rd, imm (or the toolchain testdata's INSTR $imm, rd). case len(ops) == 2 && isUTypeInstr(mnem): - rd := regFromOperand(ops[0]) - imm := immFromOperand(ops[1]) + var rd int + var imm int32 + if isImmOperand(ops[0]) { + imm, rd = immFromOperand(ops[0]), regFromOperand(ops[1]) + } else { + rd, imm = regFromOperand(ops[0]), immFromOperand(ops[1]) + } if rd < 0 { return nil, fmt.Errorf("invalid register in %s", mnem) } @@ -1226,6 +1735,11 @@ func encodeRISCVJALR(instr *ast.Instr, fi riscvFrameInfo) ([]byte, error) { func tryCompressRVC(instr *ast.Instr, fi riscvFrameInfo) (uint16, bool) { mnem := riscvCompressMnem(instr) ops := instr.Operands + // The immediate aliases fold onto their I-type mnemonics before + // compression: the toolchain compresses ADD $imm, rd as c.addi, exactly + // as it compresses the spelling ADDI. + var immNeg bool + mnem, immNeg = riscvNormaliseImmAlias(mnem, ops) switch mnem { case "LD", "MOV": @@ -1289,6 +1803,9 @@ func tryCompressRVC(instr *ast.Instr, fi riscvFrameInfo) (uint16, bool) { case "ADDI": rd, rs1, imm := extractITypeParams(instr) + if immNeg { + imm = -imm + } if rd == -1 || rs1 == -1 { return 0, false } @@ -2045,7 +2562,8 @@ func isRTypeInstr(m string) bool { case "ADD", "SUB", "SLL", "SLT", "SLTU", "XOR", "SRL", "SRA", "OR", "AND", "ADDW", "SUBW", "SLLW", "SRLW", "SRAW", "MUL", "MULH", "MULHSU", "MULHU", "DIV", "DIVU", "REM", "REMU", - "MULW", "DIVW", "DIVUW", "REMW", "REMUW": + "MULW", "DIVW", "DIVUW", "REMW", "REMUW", + "CZEROEQZ", "CZERONEZ": return true } return false @@ -2085,7 +2603,7 @@ func isStoreInstr(m string) bool { func isBranchInstr(m string) bool { switch m { - case "BEQ", "BNE", "BLT", "BGE", "BLTU", "BGEU": + case "BEQ", "BNE", "BLT", "BGE", "BLTU", "BGEU", "BGT", "BLE", "BGTU", "BLEU": return true } return false @@ -2111,7 +2629,8 @@ func isFPArithInstr(m string) bool { switch m { case "FADDS", "FSUBS", "FMULS", "FDIVS", "FADDD", "FSUBD", "FMULD", "FDIVD", - "FSQRTS", "FSQRTD", "FMINS", "FMAXS", "FMIND", "FMAXD", "FSGNJD": + "FSQRTS", "FSQRTD", "FMINS", "FMAXS", "FMIND", "FMAXD", "FSGNJD", + "FSGNJS", "FSGNJX", "FSGNJXD", "FSGNJXS", "FSGNJND", "FSGNJNS", "FSGNJNX": return true } return false diff --git a/asm/riscv_encode.go b/asm/riscv_encode.go index eae74d0..53ae999 100644 --- a/asm/riscv_encode.go +++ b/asm/riscv_encode.go @@ -219,6 +219,9 @@ var riscvInstrTable = map[string]riscvEnc{ "DIVUW": {0x3B, 0x5, 0x01}, "REMW": {0x3B, 0x6, 0x01}, "REMUW": {0x3B, 0x7, 0x01}, + // Zicond conditional zeroing. + "CZEROEQZ": {0x33, 0x5, 0x07}, + "CZERONEZ": {0x33, 0x7, 0x07}, // RV64I, I-type arithmetic. "ADDI": {0x13, 0x0, 0x00}, "ADDIW": {0x1B, 0x0, 0x00}, @@ -247,13 +250,21 @@ var riscvInstrTable = map[string]riscvEnc{ "BGE": {0x63, 0x5, 0x00}, "BLTU": {0x63, 0x6, 0x00}, "BGEU": {0x63, 0x7, 0x00}, + // The swapped-spelling comparison forms: encoded as BLT/BGE/BLTU/BGEU + // with the register operands swapped. + "BGT": {0x63, 0x4, 0x00}, + "BLE": {0x63, 0x5, 0x00}, + "BGTU": {0x63, 0x6, 0x00}, + "BLEU": {0x63, 0x7, 0x00}, // U-type. "LUI": {0x37, 0x0, 0x00}, "AUIPC": {0x17, 0x0, 0x00}, // System. - "ECALL": {0x73, 0x0, 0x00}, - "EBREAK": {0x73, 0x0, 0x00}, - "FENCE": {0x0F, 0x0, 0x00}, + "ECALL": {0x73, 0x0, 0x00}, + "EBREAK": {0x73, 0x0, 0x00}, + "FENCE": {0x0F, 0x0, 0x00}, + "FENCE.TSO": {0x0F, 0x0, 0x00}, + "PAUSE": {0x0F, 0x0, 0x00}, // JALR, indirect jump/call (I-type). "JALR": {0x67, 0x0, 0x00}, @@ -303,7 +314,14 @@ var riscvInstrTable = map[string]riscvEnc{ "FMIND": {0x53, 0x0, 0x15}, "FMAXD": {0x53, 0x1, 0x15}, // FP sign injection (double): rs2 carries the sign source. - "FSGNJD": {0x53, 0x0, 0x11}, + "FSGNJD": {0x53, 0x0, 0x11}, + "FSGNJS": {0x53, 0x0, 0x10}, + "FSGNJX": {0x53, 0x0, 0x14}, + "FSGNJXD": {0x53, 0x0, 0x15}, + "FSGNJXS": {0x53, 0x0, 0x14}, + "FSGNJND": {0x53, 0x1, 0x11}, + "FSGNJNS": {0x53, 0x1, 0x10}, + "FSGNJNX": {0x53, 0x1, 0x14}, // RV64A, load-reserved / store-conditional (funct5 0x02 / 0x03). // The toolchain gives LR acquire ordering (aq = 1) and SC release @@ -375,6 +393,10 @@ var riscvCvtTable = map[string]riscvCvtEnc{ "FMVDX": {0x79, 0x0, 0x53}, // int64 → float64 (bit move) "FMVXW": {0x70, 0x0, 0x53}, // float32 → int32 (bit move) "FMVWX": {0x78, 0x0, 0x53}, // int32 → float32 (bit move) + // The toolchain's W/D suffix spellings of the same moves. + "FMVXS": {0x70, 0x0, 0x53}, + "FMVFS": {0x78, 0x0, 0x53}, + "FMVSX": {0x79, 0x0, 0x53}, } // riscvCvtType encodes an FP conversion instruction. diff --git a/asm/riscv_encode_test.go b/asm/riscv_encode_test.go index f6dea7e..757f440 100644 --- a/asm/riscv_encode_test.go +++ b/asm/riscv_encode_test.go @@ -868,7 +868,7 @@ func encodeOneInstrRISCV(t *testing.T, src string, pc int, offsets map[string]in t.Helper() fn := firstTextRISCV(t, "#include \"textflag.h\"\n"+src) instr := fn.Body[0].(*ast.Instr) - return encodeRISCVInstr(instr, pc, offsets, riscvFrameInfo{}, nil) + return encodeRISCVInstr(instr, pc, offsets, riscvFrameInfo{}, nil, nil) } // TestRISCVBranchJumpRange checks that displacements beyond the B-type span @@ -905,9 +905,10 @@ func TestRISCVBranchJumpRange(t *testing.T) { } } -// TestRISCVBranchFarBody drives the range check through the full two-pass -// assembler: a forward branch over a body larger than the B-type span must -// error rather than wrap. +// TestRISCVBranchFarBody drives the relaxation pass through the full +// assembler: a forward branch over a body larger than the B-type span is +// rewritten as an inverted branch over an inserted JMP, the same layout the +// toolchain produces, instead of wrapping to a wrong target. func TestRISCVBranchFarBody(t *testing.T) { var sb strings.Builder sb.WriteString("#include \"textflag.h\"\nTEXT ·far(SB), NOSPLIT, $0\n\tBEQ X10, X11, done\n") @@ -916,8 +917,20 @@ func TestRISCVBranchFarBody(t *testing.T) { } sb.WriteString("done:\n\tRET\n") fn := firstTextRISCV(t, sb.String()) - if _, _, _, _, _, err := assembleRISCV(fn); err == nil { - t.Error("expected a branch-out-of-range error, got none") + out, _, _, _, _, err := assembleRISCV(fn) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // The relaxed branch at offset 0 targets the inserted JMP at 4 (bne + // x10, x11, +4); the JMP at 4 carries the far forward displacement. + wantBranch := wordLE(riscvBType(riscvEnc{0x63, 0x1, 0x00}, 10, 11, 4)) + if !bytes.Equal(out[0:4], wantBranch) { + t.Errorf("relaxed branch = %x, want %x", out[0:4], wantBranch) + } + // done sits after 1100 ADDs: 4 + 4400, i.e. offset 4404 from the JMP at 4. + wantJmp := wordLE(riscvJType(0, 4404)) + if !bytes.Equal(out[4:8], wantJmp) { + t.Errorf("inserted JMP = %x, want %x", out[4:8], wantJmp) } } diff --git a/testdata/verify/branch_far_riscv64.s b/testdata/verify/branch_far_riscv64.s new file mode 100644 index 0000000..2010df8 --- /dev/null +++ b/testdata/verify/branch_far_riscv64.s @@ -0,0 +1,2110 @@ +// Branch relaxation on riscv64: overlong conditional branches become an +// inverted branch over an inserted JMP, the layout the toolchain produces. +#include "textflag.h" + +// func FarForward(x int64) int64 +TEXT ·FarForward(SB), NOSPLIT, $0-16 + MOV x+0(FP), X5 + BEQ X5, X6, done + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + +done: + MOV X5, ret+8(FP) + RET + +// func FarBackward(x int64) int64 +TEXT ·FarBackward(SB), NOSPLIT, $0-16 + MOV x+0(FP), X5 + MOV $0, X7 + +loop: + BEQZ X7, out + BGTZ X5, skip + ADD $2, X7 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + ADD $1, X5 + +skip: + SUB $1, X5 + BNEZ X7, loop + +out: + MOV X7, ret+8(FP) + RET diff --git a/testdata/verify/l64forms_loong64.s b/testdata/verify/l64forms_loong64.s new file mode 100644 index 0000000..f3d5593 --- /dev/null +++ b/testdata/verify/l64forms_loong64.s @@ -0,0 +1,51 @@ +// The subtract-immediate fold, the TEQ/TNE trap pseudos, PRELDX, the FP +// condition branches and the N(PC) branch spellings, against the toolchain. +#include "textflag.h" + +// func SubFold(x int64) int64 +TEXT ·SubFold(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R8 + SUBV $0, R8 + SUBV $4, R9, R10 + SUBV $4096, R11 + SUBV $-4, R12 + SUB $1, R13 + SUBVU $4, R14 + SUBV $1048576, R15 + MOVV R8, ret+8(FP) + RET + +// func Traps(x int64) int64 +TEXT ·Traps(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R4 + TEQ $4, R4, R5 + TEQ $4, R4 + TNE $6, R5, R6 + MOVV R4, ret+8(FP) + RET + +// func Prefetch(x int64) int64 +TEXT ·Prefetch(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R7 + PRELDX 0(R7), $0x80001021, $0 + PRELDX -1(R7), $0x1021, $2 + MOVV R7, ret+8(FP) + RET + +// func BranchForms(x int64) int64 +TEXT ·BranchForms(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R4 + +l1: + BFPT l1 + BFPT FCC3, l1 + BFPF l1 + JMP -4(PC) + JAL 1(PC) + JAL (R4) + +loop: + ADDV $1, R4 + BEQ R4, R5, loop + BNE R4, l1 + RET diff --git a/testdata/verify/pcalign_loong64.s b/testdata/verify/pcalign_loong64.s new file mode 100644 index 0000000..b7e3767 --- /dev/null +++ b/testdata/verify/pcalign_loong64.s @@ -0,0 +1,33 @@ +// PCALIGN padding on loong64: andi $0, $0, 0 (the architecture's NOP), plus +// the automatic loop-head alignment to a 16-byte boundary. +#include "textflag.h" + +// func Pad16(x int64) int64 +TEXT ·Pad16(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R4 + PCALIGN $16 + ADDV $1, R4 + MOVV R4, ret+8(FP) + RET + +// func Pad32(x int64) int64 +TEXT ·Pad32(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R4 + PCALIGN $32 + ADDV $1, R4 + MOVV R4, ret+8(FP) + RET + +// func LoopAlign(x int64) int64 +TEXT ·LoopAlign(SB), NOSPLIT, $0-16 + MOVV x+0(FP), R4 + MOVV $10, R5 + +loop: + BEQ R4, R5, done + ADDV $1, R4 + JMP loop + +done: + MOVV R4, ret+8(FP) + RET diff --git a/testdata/verify/pcalign_riscv64.s b/testdata/verify/pcalign_riscv64.s new file mode 100644 index 0000000..ccb28d4 --- /dev/null +++ b/testdata/verify/pcalign_riscv64.s @@ -0,0 +1,35 @@ +// PCALIGN padding on riscv64: 4-byte NOPs with a 2-byte compressed NOP when +// the pad is 2 mod 4, exactly as the toolchain lays the bytes down. +#include "textflag.h" + +// func Pad8(x int64) int64 +TEXT ·Pad8(SB), NOSPLIT, $0-16 + MOV x+0(FP), X5 + PCALIGN $8 + ADD $1, X5 + MOV X5, ret+8(FP) + RET + +// func Pad16(x int64) int64 +TEXT ·Pad16(SB), NOSPLIT, $0-16 + MOV x+0(FP), X5 + PCALIGN $16 + ADD $1, X5 + MOV X5, ret+8(FP) + RET + +// func Pad32(x int64) int64 +TEXT ·Pad32(SB), NOSPLIT, $0-16 + MOV x+0(FP), X5 + PCALIGN $32 + ADD $1, X5 + MOV X5, ret+8(FP) + RET + +// func PadAfterOdd(x int64) int64 +TEXT ·PadAfterOdd(SB), NOSPLIT, $0-16 + MOV x+0(FP), X5 + PCALIGN $8 + ADD $1, X5 + MOV X5, ret+8(FP) + RET