diff --git a/CHANGELOG.md b/CHANGELOG.md index 76b07d1..e7a5e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [development] +### Fixed + +- **The operand forms GOROOT writes.** Numeric PC-relative jumps + (`JEQ 2(PC)`, the park loop `JMP 0(PC)`) resolve with the toolchain's + own instruction counting and fold jump-to-jump chains exactly as its + branch optimiser does; symbol immediates (`MOVQ $sym(SB), AX`) + assemble to the toolchain's RIP-relative LEA with an R_PCREL + relocation; negated constant expressions in operands (`ADJSP + $-(REGS - 8)`, the shape the cgo ABI macros write) fold; and `gasm + asm` predefines the `GOARCH_` and `GOOS_` macros the go + command passes to `go tool asm`, so GOROOT headers' `#ifdef + GOARCH_amd64` platform blocks (`go_tls.h`'s `get_tls` and friends) + select as intended. The GOROOT corpus measure moves to 261 of 353 + files assembling for every target architecture (73.9 %), 87.5 % of + the real-code corpus, from 70.8 % and 82.2 %. + ### Added - **Per-architecture reference pages.** [docs/asm/](docs/asm/README.md) diff --git a/asm/assemble.go b/asm/assemble.go index 12bdcc4..36b0feb 100644 --- a/asm/assemble.go +++ b/asm/assemble.go @@ -86,6 +86,10 @@ func assemble(t *ast.Text, link *linkInfo) ([]byte, []sbPatch, map[string]int, [ // outgrows the short form. long := make([]bool, len(t.Body)) sizes := make([]int, len(t.Body)) + numTargets := make([]int, len(t.Body)) + for i := range numTargets { + numTargets[i] = -1 + } offsets := map[string]int{} pcs := make([]int, len(t.Body)) var guardJBlong, guardJBElong, moreJMPlong bool @@ -94,6 +98,10 @@ func assemble(t *ast.Text, link *linkInfo) ([]byte, []sbPatch, map[string]int, [ for { guard := fi.guardLen(guardJBlong, guardJBElong) pos := guard + len(fi.prologue) + for i := range numTargets { + numTargets[i] = -1 + } + idxAtPc := map[int]int{} for i, stmt := range t.Body { switch s := stmt.(type) { case *ast.Label: @@ -105,12 +113,78 @@ func assemble(t *ast.Text, link *linkInfo) ([]byte, []sbPatch, map[string]int, [ } sizes[i] = sz pcs[i] = pos + idxAtPc[pos] = i pos += sz } } bodyLen := pos - (guard + len(fi.prologue)) // Expand any short jump whose displacement no longer fits rel8. changed := false + // Numeric ±N(PC) jumps resolve against this iteration's layout; the + // emission pass reads the same table after the loop converges. A + // target that is itself an unconditional local JMP is chased to the + // ultimate target: the toolchain's brloop pass collapses branch-to- + // branch chains before it encodes, so matching its bytes requires + // the same redirection. + for i := range numTargets { + numTargets[i] = -1 + } + for i, stmt := range t.Body { + s, ok := stmt.(*ast.Instr) + if !ok { + continue + } + if len(s.Operands) == 1 { + if n, isNum := pcJumpOffset(s.Operands[0]); isNum { + if target, okT := pcJumpTarget(t, i, n, pcs); okT { + numTargets[i] = target + } + } + } + } + for i := range numTargets { + if numTargets[i] < 0 { + continue + } + tgt := numTargets[i] + for hop := 0; hop < len(t.Body); hop++ { + idx, ok := idxAtPc[tgt] + if !ok { + break + } + in, ok := t.Body[idx].(*ast.Instr) + if !ok || strings.ToUpper(in.Mnemonic.Text) != "JMP" || len(in.Operands) != 1 { + break + } + if name, isLabel := labelName(in.Operands[0]); isLabel { + tgt = offsets[resolve(name)] + continue + } + if n, isNum := pcJumpOffset(in.Operands[0]); isNum { + next, okT := pcJumpTarget(t, idx, n, pcs) + if !okT { + break + } + tgt = next + continue + } + break // JMP through a register or memory: the chain ends + } + numTargets[i] = tgt + } + for i, stmt := range t.Body { + s, ok := stmt.(*ast.Instr) + if !ok { + continue + } + if numTargets[i] >= 0 && !long[i] { + rel := int64(numTargets[i] - (pcs[i] + jumpSize(strings.ToUpper(s.Mnemonic.Text), false))) + if !fits8(rel) { + long[i] = true + changed = true + } + } + } for i, stmt := range t.Body { s, ok := stmt.(*ast.Instr) if !ok { @@ -232,7 +306,7 @@ func assemble(t *ast.Text, link *linkInfo) ([]byte, []sbPatch, map[string]int, [ spadjStep{pos + epi, 0}, ) } - code, ps, pool, err := encodeInstr(s, pos, offsets, fi, long[i], resolve, link) + code, ps, pool, err := encodeInstr(s, pos, offsets, fi, long[i], resolve, link, numTargets[i]) if err != nil { return nil, nil, nil, nil, nil, nil, fmt.Errorf("%s: %w", s.Mnemonic.Text, err) } @@ -428,6 +502,37 @@ func computeFrame(t *ast.Text) frameInfo { return fi } +// pcJumpOffset recognises the numeric relative jump operand ±N(PC) and +// returns N: the toolchain counts instructions, not bytes, so +2(PC) targets +// the second instruction boundary after the branch. +func pcJumpOffset(op *ast.Operand) (int, bool) { + if op.Kind != ast.OpAddr || op.Addr.Base != "PC" { + return 0, false + } + return int(op.Addr.Offset), true +} + +// pcJumpTarget resolves a numeric jump at statement index j: N counts the +// instruction statements after the jump itself (N = 0 is the jump's own +// address, the classic park loop), and the target is the start of the Nth +// one. It reports false when the count runs past the end of the function. +func pcJumpTarget(t *ast.Text, j, n int, pcs []int) (int, bool) { + if n == 0 { + return pcs[j], true + } + seen := 0 + for k := j + 1; k < len(t.Body); k++ { + if _, ok := t.Body[k].(*ast.Instr); !ok { + continue + } + seen++ + if seen == n { + return pcs[k], true + } + } + return 0, false +} + // hasCall reports whether the function body contains a CALL instruction. func hasCall(t *ast.Text) bool { for _, stmt := range t.Body { @@ -602,7 +707,7 @@ func instrSize(s *ast.Instr, fi frameInfo, long bool, link *linkInfo) (int, erro } return jumpSize(mnem, long), nil } - code, _, _, err := encodeInstr(s, 0, nil, fi, false, nil, link) + code, _, _, err := encodeInstr(s, 0, nil, fi, false, nil, link, -1) if err != nil { return 0, err } @@ -637,7 +742,7 @@ func jumpSize(mnem string, long bool) int { // (relative to pc, the instruction's own offset). A RET in a frame-pointer // function is prefixed with the epilogue. resolve, when non-nil, redirects a // jump label through the jump-to-jump chain before the offset lookup. -func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo, long bool, resolve func(string) string, link *linkInfo) ([]byte, []sbPatch, []floatPoolEntry, error) { +func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo, long bool, resolve func(string) string, link *linkInfo, numTarget int) ([]byte, []sbPatch, []floatPoolEntry, error) { mnem := strings.ToUpper(s.Mnemonic.Text) var prefix []byte @@ -677,7 +782,7 @@ func encodeInstr(s *ast.Instr, pc int, offsets map[string]int, fi frameInfo, lon } return append(prefix, code...), nil, nil, nil } - code, err = encodeJump(s, mnem, pc+len(prefix), offsets, long, resolve) + code, err = encodeJump(s, mnem, pc+len(prefix), offsets, long, resolve, numTarget) } else { code, ps, pool, err = encodeNormal(s, fi, link) } @@ -703,6 +808,32 @@ func encodeNormal(s *ast.Instr, fi frameInfo, link *linkInfo) ([]byte, []sbPatch } return code, nil, nil, nil } + // MOVQ $sym±off(SB), r64: the toolchain assembles a symbol immediate as + // LEAQ disp32(RIP), r64 with an R_PCREL relocation at the disp32 field, + // never as a 64-bit absolute immediate (verified against go tool asm). + // MOVD is the MOVQ alias; the narrower widths reject the form outright. + if (mnemUpper == "MOVQ" || mnemUpper == "MOVD") && len(s.Operands) == 2 && + s.Operands[0].Kind == ast.OpImmediate && s.Operands[0].Imm.Sym != nil && + s.Operands[0].Imm.Sym.Pseudo == "SB" { + mem := &ast.Operand{Kind: ast.OpAddr, Addr: ast.Address{Sym: s.Operands[0].Imm.Sym}} + src, err := operandFromAST(mnemUpper, mem, 8, fi, link) + if err != nil { + return nil, nil, nil, err + } + dst, err := operandFromAST(mnemUpper, s.Operands[1], 8, fi, link) + if err != nil { + return nil, nil, nil, err + } + e := &enc{} + if err := e.encodeLea([]Operand{src, dst}, 8); err != nil { + return nil, nil, nil, err + } + ps := make([]sbPatch, len(e.patches)) + for i, p := range e.patches { + ps[i] = sbPatch{off: p.off, name: p.name, addend: p.addend} + } + return e.out, ps, nil, nil + } _, size := splitSize(mnemUpper) if size == 0 { size = 8 @@ -753,21 +884,29 @@ func encodeBookkeeping(upper string, s *ast.Instr) ([]byte, error) { } // encodeJump encodes a JMP/CALL/Jcc with a relative offset resolved from the -// target label, in the short (rel8) or long (rel32) form. -func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int, long bool, resolve func(string) string) ([]byte, error) { +// target label or from a numeric ±N(PC) instruction count, in the short +// (rel8) or long (rel32) form. numTarget is the resolved byte offset of a +// numeric operand, negative when the operand is not one. +func encodeJump(s *ast.Instr, mnem string, pc int, offsets map[string]int, long bool, resolve func(string) string, numTarget int) ([]byte, error) { if len(s.Operands) != 1 { return nil, fmt.Errorf("jump expects 1 operand, got %d", len(s.Operands)) } - name, ok := labelName(s.Operands[0]) - if !ok { + name, isLabel := labelName(s.Operands[0]) + if !isLabel && numTarget < 0 { return nil, fmt.Errorf("jump target must be a local label") } - if resolve != nil && mnem != "CALL" { - name = resolve(name) - } - target, ok := offsets[name] - if !ok { - return nil, fmt.Errorf("undefined label %q", name) + var target int + if isLabel { + if resolve != nil && mnem != "CALL" { + name = resolve(name) + } + t, ok := offsets[name] + if !ok { + return nil, fmt.Errorf("undefined label %q", name) + } + target = t + } else { + target = numTarget } rel := int64(target - (pc + jumpSize(mnem, long))) @@ -841,6 +980,11 @@ func indirectJumpTarget(s *ast.Instr) bool { return false } a := s.Operands[0].Addr + // ±N(PC) is the numeric relative form, the PC counts instructions from + // the branch: relative, not indirect. + if a.Base == "PC" || a.Index == "PC" { + return false + } if a.Base != "" || a.Index != "" { return true } diff --git a/asm/kernels_differential_test.go b/asm/kernels_differential_test.go index 0938783..7c7ef3a 100644 --- a/asm/kernels_differential_test.go +++ b/asm/kernels_differential_test.go @@ -130,6 +130,7 @@ func TestDifferentialKernels(t *testing.T) { {filepath.Join("..", "testdata", "verify", "quadreg_amd64.s"), "", false}, {filepath.Join("..", "testdata", "verify", "floatimm_amd64.s"), "", false}, {filepath.Join("..", "testdata", "verify", "bookkeep_amd64.s"), "", false}, + {filepath.Join("..", "testdata", "verify", "forms_amd64.s"), "", false}, {filepath.Join("..", "testdata", "verify", "datarel_arm64.s"), "arm64", true}, {filepath.Join("..", "testdata", "verify", "divslash_arm64.s"), "arm64", true}, } { diff --git a/asm/link.go b/asm/link.go index bfbb1d5..3e30c03 100644 --- a/asm/link.go +++ b/asm/link.go @@ -166,6 +166,13 @@ func AssembleFile(f *ast.File) (*Image, error) { for _, d := range dataSyms { known[d.name] = true } + // TEXT symbols are file-level definitions too: a symbol immediate + // ($fn(SB)) may name one, exactly as a data reference names a GLOBL. + for _, d := range f.Decls { + if t, ok := d.(*ast.Text); ok { + known[t.Name.Name] = true + } + } link := &linkInfo{symbols: known, allowExternal: true} poolSeen := map[string]bool{} diff --git a/cmd/gasm/audit.go b/cmd/gasm/audit.go index d2c8569..f2b8b93 100644 --- a/cmd/gasm/audit.go +++ b/cmd/gasm/audit.go @@ -664,6 +664,7 @@ func runCorpusAudit(root string, dirs includeDirs) (*corpusStats, error) { f, errs := parser.ParseWithOptions(path, src, parser.Options{ Expand: true, IncludeDirs: append(slices.Clone(dirs), hdrDir), + Predefines: platformPredefinesFor(goarchName(tg.a), goos), }) if len(errs) > 0 { ok = false @@ -683,12 +684,19 @@ func runCorpusAudit(root string, dirs includeDirs) (*corpusStats, error) { continue } - f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs}) - ok := true for _, i := range wanted { tg, t := targets[i], tallies[i] t.attempted++ + // The parse carries the target's platform predefines, so it + // cannot be shared across targets the way a header-free file's + // could: a #ifdef GOARCH_arm block must be live on arm64 and + // dead everywhere else. + f, errs := parser.ParseWithOptions(path, src, parser.Options{ + Expand: true, + IncludeDirs: dirs, + Predefines: platformPredefinesFor(goarchName(tg.a), goos), + }) var err error if len(errs) > 0 { err = errs[0] // a parse failure is a failure for every target diff --git a/cmd/gasm/main.go b/cmd/gasm/main.go index f34f0e4..389c429 100644 --- a/cmd/gasm/main.go +++ b/cmd/gasm/main.go @@ -574,7 +574,7 @@ naming the package. defer cleanup() dirs = append(dirs, hdrDir) } - f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs}) + f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs, Predefines: platformPredefinesFor(string(targetArch), goos)}) for _, e := range errs { fmt.Fprintf(os.Stderr, "%s: %v\n", path, e) } @@ -789,6 +789,29 @@ e.g. --map wideCopyAVX2=wideCopyAVX512 pairs the two regardless of suffix. return 1 } +// platformPredefines mirrors the go command's assembler invocation, which +// defines GOOS_ and GOARCH_ as -D macros: GOROOT headers +// (go_tls.h, asm_riscv64.h) select their platform blocks with #ifdef on +// exactly those names, so an assembler without them cannot see the platform +// definitions at all. +func platformPredefines(goarch, goos string) map[string]string { + return map[string]string{ + "GOARCH_" + goarch: "1", + "GOOS_" + goos: "1", + } +} + +// platformPredefinesFor resolves the ambient GOOS the way a build would: a +// file whose name carries one (sys_darwin_arm64.s) is compiled for that GOOS +// and nothing else. +func platformPredefinesFor(goarch string, fileGoos string) map[string]string { + goos := fileGoos + if goos == "" { + goos = runtime.GOOS + } + return platformPredefines(goarch, goos) +} + // assembleFile assembles a parsed file for the given architecture and returns the image. func assembleFile(targetArch arch.Arch, f *ast.File) (*asm.Image, error) { switch targetArch { @@ -813,17 +836,17 @@ func assemblePath(path string, forced arch.Arch, dirs includeDirs) (*asm.Image, if err != nil { return nil, err } - f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs}) + target := forced + if target == arch.Unknown { + target = arch.FromFilename(path) + } + f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs, Predefines: platformPredefinesFor(string(target), "")}) for _, e := range errs { fmt.Fprintf(os.Stderr, "%s: %v\n", path, e) } if len(errs) > 0 { return nil, fmt.Errorf("parse errors") } - target := forced - if target == arch.Unknown { - target = arch.FromFilename(path) - } return assembleFile(target, f) } diff --git a/parser/parser.go b/parser/parser.go index 0601c2c..9f7cf45 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -489,6 +489,17 @@ func parseImmediate(g []token.Token) ast.Immediate { } else if g[i].Kind == token.Plus { i++ } + // A constant expression after the sign: $-(R - 8), $+(32-shift). The + // toolchain folds the negated value in place (the cgo ABI macros write + // ADJSP $-(REGS_HOST_TO_ABI0_STACK - 8)), so the sign applies to the + // folded value exactly as it does to a bare literal. + if i < len(g) && (g[i].Kind == token.LParen || g[i].Kind == token.Tilde) { + if v, rest, ok := foldExpr(g[i:]); ok && len(rest) == 0 { + imm.Val = v + imm.HasVal = true + return imm + } + } if i < len(g) && g[i].Kind == token.Number { text := g[i].Text if v, ok := tryInt(text); ok { diff --git a/parser/preproc.go b/parser/preproc.go index 313bc70..9f26079 100644 --- a/parser/preproc.go +++ b/parser/preproc.go @@ -34,6 +34,12 @@ type Options struct { // Expand enables macro expansion, include splicing and the // statement-separator reading of ';' that the expanded bodies rely on. Expand bool + // Predefines names the macros defined before the file is read. The + // go command drives go tool asm with -D GOOS_ -D GOARCH_, + // and GOROOT's own headers (go_tls.h, asm_riscv64.h) select their + // platform blocks with #ifdef on exactly those names, so an assembler + // without them cannot see the platform definitions at all. + Predefines map[string]string } // ParseWithOptions parses src like Parse, optionally preprocessing it first. @@ -44,6 +50,9 @@ func ParseWithOptions(path, src string, opts Options) (*ast.File, []error) { var errs []error if opts.Expand { pp := &preproc{opts: opts, macros: map[string]*macroDef{}} + for name, value := range opts.Predefines { + pp.macros[name] = ¯oDef{name: name, body: lexer.Tokenize(value)} + } lines = pp.fileLines(path, tokens, token.Position{}) errs = pp.errs } else { diff --git a/testdata/verify/forms_amd64.s b/testdata/verify/forms_amd64.s new file mode 100644 index 0000000..61dcc16 --- /dev/null +++ b/testdata/verify/forms_amd64.s @@ -0,0 +1,36 @@ +// Kernel: the operand forms the GOROOT campaign surfaced — numeric +// PC-relative jumps, symbol-immediate materialisation (the toolchain rewrites +// MOVQ $sym(SB) into a RIP-relative LEA) and the negated constant-expression +// ADJSP the cgo ABI macros write. Bytes are pinned against go tool asm by +// TestDifferentialKernels. +#include "textflag.h" + +DATA sd<>(SB)/4, $7 +GLOBL sd<>(SB), RODATA, $4 + +// func Jumps(flag int64) int64 +TEXT ·Jumps(SB), NOSPLIT, $0-16 + MOVQ flag+0(FP), AX + TESTQ AX, AX + JEQ 2(PC) + MOVQ $1, AX + JMP 3(PC) + MOVQ $2, AX + MOVQ AX, ret+0(FP) + RET + +// func SymImm() int64 +TEXT ·SymImm(SB), NOSPLIT, $0-16 + MOVQ $sd<>(SB), AX + MOVQ $·SymImm(SB), CX + MOVQ AX, ret+0(FP) + RET + +// func Frame() +TEXT ·Frame(SB), NOSPLIT, $0 + PUSHFQ + CLD + ADJSP $(64 - 8) + ADJSP $-(64 - 8) + POPFQ + RET