diff --git a/cmd/gasm/audit.go b/cmd/gasm/audit.go index 50110b0..9df6635 100644 --- a/cmd/gasm/audit.go +++ b/cmd/gasm/audit.go @@ -37,7 +37,7 @@ import ( // construction and are excluded from the diff; the other architectures list // their conditional branches outright. func cmdAuditInstructions(args []string) error { - fs := newCommand("audit-instructions", "gasm audit-instructions [--corpus [dir]] [amd64|arm64|riscv64|loong64]", ` + fs := newCommand("audit-instructions", "gasm audit-instructions [--corpus [dir]] [-I dir] [amd64|arm64|riscv64|loong64]", ` Compare the gasm encoder for the given architecture (default amd64) against go tool asm and print the diff: superset encodings (gasm-only, shippable via gasm asm --format goobj) and known-but-unencodable names (the backlog). The @@ -57,11 +57,13 @@ per-architecture pass rates and the most common failure reasons, which drive the encodability backlog by frequency rather than by table order. `) corpus := fs.Bool("corpus", false, "assemble a corpus of .s files and report pass rates and failure reasons") + var dirs includeDirs + fs.Var(&dirs, "I", "directory to search for #include files (may be repeated)") if err := fs.Parse(args); err != nil { return err } if *corpus { - return cmdAuditCorpus(fs.Args()) + return cmdAuditCorpus(fs.Args(), dirs) } archName := "amd64" switch n := len(fs.Args()); { @@ -395,8 +397,11 @@ func (t *corpusTally) fail(path, reason string) { } } -// cmdAuditCorpus implements audit-instructions --corpus. -func cmdAuditCorpus(args []string) error { +// cmdAuditCorpus implements audit-instructions --corpus. The include +// directories carry #include resolution over a corpus whose files refer to +// headers such as GOROOT/pkg/include, the same -I a toolchain comparison +// needs. +func cmdAuditCorpus(args []string, dirs includeDirs) error { if len(args) > 1 { return &usageError{fmt.Errorf("audit-instructions --corpus takes at most one directory argument")} } @@ -410,7 +415,25 @@ func cmdAuditCorpus(args []string) error { } root = filepath.Join(strings.TrimSpace(string(out)), "src") } - stats, err := runCorpusAudit(root) + // The toolchain's shipped headers (funcdata.h and friends) define the + // macros GOROOT files include; a corpus audit measures those files, so + // the header directory joins the search path automatically. go_asm.h + // is compiler-generated per package and stays unresolvable on purpose. + if out, err := exec.Command("go", "env", "GOROOT").Output(); err == nil { + pkgInclude := filepath.Join(strings.TrimSpace(string(out)), "pkg", "include") + if fi, err := os.Stat(pkgInclude); err == nil && fi.IsDir() { + seen := false + for _, d := range dirs { + if d == pkgInclude { + seen = true + } + } + if !seen { + dirs = append(dirs, pkgInclude) + } + } + } + stats, err := runCorpusAudit(root, dirs) if err != nil { return err } @@ -454,7 +477,7 @@ func otherPortFile(path string) bool { return false } -func runCorpusAudit(root string) (*corpusStats, error) { +func runCorpusAudit(root string, dirs includeDirs) (*corpusStats, error) { files, err := asmFiles(root) if err != nil { return nil, err @@ -479,7 +502,7 @@ func runCorpusAudit(root string) (*corpusStats, error) { if err != nil { return nil, err } - f, errs := parser.Parse(path, src) + f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs}) var wanted []int // indexes into targets if a := arch.FromFilename(path); a != arch.Unknown { diff --git a/cmd/gasm/main.go b/cmd/gasm/main.go index a5fa4c2..fa7c2f9 100644 --- a/cmd/gasm/main.go +++ b/cmd/gasm/main.go @@ -240,6 +240,16 @@ func readSource(path string) (string, error) { return string(b), err } +// includeDirs collects repeatable -I flags: the directories searched for +// #include files during macro expansion and include splicing. +type includeDirs []string + +func (d *includeDirs) String() string { return strings.Join(*d, ",") } +func (d *includeDirs) Set(v string) error { + *d = append(*d, v) + return nil +} + func cmdTokens(args []string) int { fs := newCommand("tokens", "gasm tokens ", ` Print the lexical token stream of FILE: position, token kind and text, one @@ -476,7 +486,7 @@ hover, document symbols, diagnostics and semantic-token highlighting. } func cmdAsm(args []string) int { - fs := newCommand("asm", "gasm asm [--format raw|elf|goobj] [-p pkg] [-GOARCH arch] [-o out] ", ` + fs := newCommand("asm", "gasm asm [--format raw|elf|goobj] [-I dir] [-p pkg] [-GOARCH arch] [-o out] ", ` Assemble FILE without the Go toolchain: every TEXT function is encoded to machine code and printed as a hex dump. Supported architectures: amd64 (including VEX/AVX2 and EVEX/AVX-512), arm64 (AArch64 integer, FP, @@ -498,9 +508,11 @@ and the format version from go version). format := fs.String("format", "raw", "output format: raw (concatenated image), elf or goobj (Go object)") pkg := fs.String("p", "", "package path for --format goobj (qualifies the exported symbols)") archName := fs.String("GOARCH", "", "target architecture: amd64, arm64, riscv64 or loong64 (overrides the file-name suffix)") + var dirs includeDirs + fs.Var(&dirs, "I", "directory to search for #include files (may be repeated)") fs.Parse(args) if fs.NArg() != 1 { - fmt.Fprintln(os.Stderr, "usage: gasm asm [--format raw|elf|goobj] [-p pkg] [-GOARCH arch] [-o out] ") + fmt.Fprintln(os.Stderr, "usage: gasm asm [--format raw|elf|goobj] [-I dir] [-p pkg] [-GOARCH arch] [-o out] ") return 2 } // The format is validated before anything else, so a bogus value exits 2 @@ -526,7 +538,7 @@ and the format version from go version). fmt.Fprintln(os.Stderr, "gasm:", err) return 1 } - f, errs := parser.Parse(path, src) + f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs}) for _, e := range errs { fmt.Fprintf(os.Stderr, "%s: %v\n", path, e) } @@ -634,7 +646,7 @@ and the format version from go version). // cmdDiff compares the machine code of two assembly files. func cmdDiff(args []string) int { - set := newCommand("diff", "gasm diff [-GOARCH arch] ", ` + set := newCommand("diff", "gasm diff [-GOARCH arch] [-I dir] ", ` Compare the machine code produced by assembling two files. Shows which functions differ and the byte-level differences. Useful for verifying that two implementations produce identical code, @@ -645,9 +657,11 @@ e.g. --map wideCopyAVX2=wideCopyAVX512 pairs the two regardless of suffix. `) mapSpec := set.String("map", "", "comma-separated old=new pairs to match functions with different names") archName := set.String("GOARCH", "", "target architecture for both files: amd64, arm64, riscv64 or loong64") + var dirs includeDirs + set.Var(&dirs, "I", "directory to search for #include files (may be repeated)") set.Parse(args) if set.NArg() != 2 { - fmt.Fprintln(os.Stderr, "usage: gasm diff [-GOARCH arch] ") + fmt.Fprintln(os.Stderr, "usage: gasm diff [-GOARCH arch] [-I dir] ") return 2 } path1, path2 := set.Arg(0), set.Arg(1) @@ -675,12 +689,12 @@ e.g. --map wideCopyAVX2=wideCopyAVX512 pairs the two regardless of suffix. } // Assemble both files. - img1, err := assemblePath(path1, forced) + img1, err := assemblePath(path1, forced, dirs) if err != nil { fmt.Fprintf(os.Stderr, "gasm diff: %s: %v\n", path1, err) return 1 } - img2, err := assemblePath(path2, forced) + img2, err := assemblePath(path2, forced, dirs) if err != nil { fmt.Fprintf(os.Stderr, "gasm diff: %s: %v\n", path2, err) return 1 @@ -755,14 +769,15 @@ func assembleFile(targetArch arch.Arch, f *ast.File) (*asm.Image, error) { } } -// assemblePath reads, parses and assembles a file (used by cmdDiff). A -// non-Unknown forced architecture overrides the file-name suffix. -func assemblePath(path string, forced arch.Arch) (*asm.Image, error) { +// assemblePath reads, preprocesses, parses and assembles a file (used by +// cmdDiff). A non-Unknown forced architecture overrides the file-name +// suffix. +func assemblePath(path string, forced arch.Arch, dirs includeDirs) (*asm.Image, error) { src, err := readSource(path) if err != nil { return nil, err } - f, errs := parser.Parse(path, src) + f, errs := parser.ParseWithOptions(path, src, parser.Options{Expand: true, IncludeDirs: dirs}) for _, e := range errs { fmt.Fprintf(os.Stderr, "%s: %v\n", path, e) } diff --git a/cmd/gasm/main_test.go b/cmd/gasm/main_test.go index 0f6bf4b..13c1707 100644 --- a/cmd/gasm/main_test.go +++ b/cmd/gasm/main_test.go @@ -403,7 +403,7 @@ func TestRunCorpusAudit(t *testing.T) { write("generic.s", "#include \"textflag.h\"\nTEXT ·g(SB), NOSPLIT, $0-0\n\tRET\n") write("broken.s", "#include \"textflag.h\"\nTEXT ·b(SB), NOSPLIT, $0-0\n\tJMP nowhere\n\tRET\n") - stats, err := runCorpusAudit(dir) + stats, err := runCorpusAudit(dir, nil) if err != nil { t.Fatalf("runCorpusAudit: %v", err) } diff --git a/cmd/gasm/preproc_test.go b/cmd/gasm/preproc_test.go new file mode 100644 index 0000000..bde28e8 --- /dev/null +++ b/cmd/gasm/preproc_test.go @@ -0,0 +1,109 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: BSD-3-Clause + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTree writes a directory of files and returns its root. +func writeTree(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for name, content := range files { + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// TestAsmMacroAndIncludeEndToEnd drives `gasm asm` over a source with an +// in-file parameterised macro and an include resolved through -I, and checks +// the assembled bytes came from the expansion (the loop body counts six +// increments, two per expanded iteration). +func TestAsmMacroAndIncludeEndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("runs the assembler end to end") + } + dir := writeTree(t, map[string]string{ + "inc/consts.h": "#define NITER 3\n", + "main_amd64.s": "#include \"textflag.h\"\n" + + "#include \"consts.h\"\n" + + "#define STEP(r) ADDQ $1, r; ADDQ $1, r\n" + + "TEXT ·f(SB), NOSPLIT, $0-8\n" + + "\tXORQ AX, AX\n" + + "\tMOVQ $NITER, CX\n" + + "loop:\n" + + "\tSTEP(AX)\n" + + "\tDECQ CX\n" + + "\tJNZ loop\n" + + "\tMOVQ AX, ret+0(FP)\n" + + "\tRET\n", + }) + stdout, stderr, code := capture(func() int { + return cmdAsm([]string{"-I", filepath.Join(dir, "inc"), "-GOARCH", "amd64", filepath.Join(dir, "main_amd64.s")}) + }) + if code != 0 { + t.Fatalf("gasm asm exited %d: %s%s", code, stdout, stderr) + } + // The macro expanded to two ADDQ $1 encodings in the static body; the + // iteration count lives in the runtime loop. + if n := strings.Count(stdout, "83 c0 01"); n != 2 { + t.Errorf("found %d ADDQ $1 encodings in the image, want 2:\n%s", n, stdout) + } +} + +// TestAsmIncludeResolutionOrder pins the -I search order end to end: the +// including file's directory wins over the -I directories. +func TestAsmIncludeResolutionOrder(t *testing.T) { + if testing.Short() { + t.Skip("runs the assembler end to end") + } + dir := writeTree(t, map[string]string{ + "src/main_amd64.s": "#include \"textflag.h\"\n" + + "#include \"vals.h\"\n" + + "TEXT ·f(SB), NOSPLIT, $0\n" + + "\tMOVQ $VAL, AX\n" + + "\tRET\n", + "src/vals.h": "#define VAL 1\n", + "late/vals.h": "#define VAL 2\n", + "early/vals.h": "#define VAL 3\n", + }) + stdout, stderr, code := capture(func() int { + return cmdAsm([]string{"-I", filepath.Join(dir, "early"), "-I", filepath.Join(dir, "late"), + "-GOARCH", "amd64", filepath.Join(dir, "src", "main_amd64.s")}) + }) + if code != 0 { + t.Fatalf("gasm asm exited %d: %s%s", code, stdout, stderr) + } + // VAL came from src/vals.h, not from either -I directory: the image + // loads the immediate 1. + if !strings.Contains(stdout, "b8 01 00 00 00") { + t.Errorf("expected the source-directory VAL (immediate 1) in:\n%s", stdout) + } +} + +// TestAsmMissingIncludeIsAnError pins the diagnostic for an include that +// resolves nowhere on the assembly path. +func TestAsmMissingIncludeIsAnError(t *testing.T) { + if testing.Short() { + t.Skip("runs the assembler end to end") + } + path := writeTemp(t, "main_amd64.s", "#include \"textflag.h\"\n#include \"nothere.h\"\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n") + _, stderr, code := capture(func() int { return cmdAsm([]string{"-GOARCH", "amd64", path}) }) + if code == 0 { + t.Fatal("gasm asm accepted a file whose include resolves nowhere") + } + if !strings.Contains(stderr, `#include "nothere.h"`) { + t.Errorf("stderr does not name the failing include: %s", stderr) + } +} diff --git a/docs/CLI.md b/docs/CLI.md index ba4e0b3..1966ebb 100644 --- a/docs/CLI.md +++ b/docs/CLI.md @@ -141,12 +141,13 @@ gasm lint kernel_amd64.s ## asm ```text -Usage: gasm asm [--format raw|elf|goobj] [-p pkg] [-GOARCH arch] [-o out] +Usage: gasm asm [--format raw|elf|goobj] [-I dir] [-p pkg] [-GOARCH arch] [-o out] ``` | Flag | Default | Effect | |---|---|---| | `-format` | `raw` | output format: `raw` (concatenated image), `elf` or `goobj` (Go object) | +| `-I` | empty | directory to search for `#include` files; may be repeated, searched in order after the source directory | | `-p` | empty | package path for `--format goobj`, qualifying the exported symbols | | `-GOARCH` | empty | target architecture: `amd64`, `arm64`, `riscv64` or `loong64`; overrides the file-name suffix | | `-o` | empty | write the output to this file instead of a hex dump on stdout | @@ -162,6 +163,13 @@ system toolchain; `goobj` emits the Go toolchain's own object format, which installed: the object preamble is captured from `go tool asm` and the format version from `go version`. `raw` and `elf` need no toolchain at all. +Assembly preprocessing matches the toolchain's: `#define` macros (object and +parameterised) expand at the point of use, `#undef`, `#ifdef`, `#ifndef`, +`#else` and `#endif` behave as in `go tool asm`, `;` separates statements, +and `#include "file"` splices the named file in, resolved against the source +directory and then each `-I` directory in order. `textflag.h` is the one +header that is not spliced: gasm consumes its flag names natively. + ```sh gasm asm hello_amd64.s ``` @@ -305,12 +313,13 @@ gasm debug --func add --cover hello_amd64.s ## diff ```text -Usage: gasm diff [-GOARCH arch] +Usage: gasm diff [-GOARCH arch] [-I dir] ``` | Flag | Default | Effect | |---|---|---| | `-GOARCH` | empty | target architecture for both files, overriding the file-name suffixes | +| `-I` | empty | directory to search for `#include` files; may be repeated, searched in order after the source directory | | `-map` | empty | comma-separated `old=new` pairs to match functions with different names | Functions are paired by exact name unless `--map` says otherwise, so @@ -348,7 +357,7 @@ add: 16 bytes, args=24, frame=0 NOSPLIT ## audit-instructions ```text -Usage: gasm audit-instructions [--corpus [dir]] [amd64|arm64|riscv64|loong64] +Usage: gasm audit-instructions [--corpus [dir]] [-I dir] [amd64|arm64|riscv64|loong64] ``` Compare the gasm encoder for the given architecture (default amd64) against the diff --git a/docs/man/gasm-asm.1 b/docs/man/gasm-asm.1 index 33d8b30..0aae0c1 100644 --- a/docs/man/gasm-asm.1 +++ b/docs/man/gasm-asm.1 @@ -2,7 +2,7 @@ .SH NAME gasm-asm \- assemble Plan 9 assembly without the Go toolchain .SH SYNOPSIS -.B gasm asm [\-\-format raw|elf|goobj] [\-p pkg] [\-GOARCH arch] [\-o out] +.B gasm asm [\-\-format raw|elf|goobj] [\-I dir] [\-p pkg] [\-GOARCH arch] [\-o out] .SH DESCRIPTION Assemble FILE without the Go toolchain: every TEXT function is encoded to machine code and printed as a hex dump. Supported architectures: @@ -47,6 +47,10 @@ functions link too. .B \-\-format \fIraw|elf|goobj\fR Output format; the default is raw. .TP +.B \-I \fIdir\fR +Directory to search for #include files; may be repeated, searched in +order after the source directory. +.TP .B \-p \fIpkg\fR Package path for --format goobj, qualifying the exported symbols. .TP diff --git a/docs/man/gasm-audit-instructions.1 b/docs/man/gasm-audit-instructions.1 index 5c58c71..82f85d5 100644 --- a/docs/man/gasm-audit-instructions.1 +++ b/docs/man/gasm-audit-instructions.1 @@ -2,7 +2,7 @@ .SH NAME gasm-audit-instructions \- diff the encoder against the Go toolchain, or measure a corpus .SH SYNOPSIS -.B gasm audit\-instructions [\-\-corpus [\fIdir\fR]] [amd64|arm64|riscv64|loong64] +.B gasm audit\-instructions [\-\-corpus [\fIdir\fR]] [\-I dir] [amd64|arm64|riscv64|loong64] .SH DESCRIPTION Compare the gasm encoder for the given architecture (default amd64) against @@ -38,6 +38,12 @@ second. .B \-\-corpus [\fIdir\fR] Assemble a corpus of .s files and report pass rates and failure reasons. +.TP +.B \-I \fIdir\fR +Directory to search for #include files; may be repeated, searched in +order after the source directory. A corpus run whose files include +toolchain headers (such as GOROOT/pkg/include) needs it, the same -I a +toolchain comparison takes. .SH EXIT STATUS The mnemonic-diff mode reports through its output and exits 0; a failed probe or an unknown architecture exits non-zero. diff --git a/docs/man/gasm-diff.1 b/docs/man/gasm-diff.1 index ab15a35..b9fb5b2 100644 --- a/docs/man/gasm-diff.1 +++ b/docs/man/gasm-diff.1 @@ -2,7 +2,7 @@ .SH NAME gasm-diff \- compare the machine code of two assembly files .SH SYNOPSIS -.B gasm diff [\-GOARCH arch] +.B gasm diff [\-GOARCH arch] [\-I dir] .SH DESCRIPTION Compare the machine code produced by assembling two files. Shows which functions differ and the byte-level differences. Useful for verifying @@ -20,6 +20,10 @@ pairs two variants regardless of suffix. Target architecture for both files: amd64, arm64, riscv64 or loong64; overrides the file-name suffixes. .TP +.B \-I \fIdir\fR +Directory to search for #include files; may be repeated, searched in +order after the source directory. +.TP .B \-\-map \fIspec\fR Comma-separated old=new pairs to match functions with different names. .SH EXIT STATUS diff --git a/lexer/lexer.go b/lexer/lexer.go index 0fe7e3c..be38828 100644 --- a/lexer/lexer.go +++ b/lexer/lexer.go @@ -119,7 +119,9 @@ func (l *Lexer) Next() token.Token { // is a C-preprocessor line continuation (used by #define macros in the // runtime .s files): splice the lines together by consuming both, so // the whole macro becomes one logical line that the parser treats as an - // opaque preprocessor directive. + // opaque preprocessor directive. The backslash may also reach its + // newline across whitespace and a trailing comment ("…; \ // note\n"), + // which the toolchain's scanner skips the same way. for { c := l.cur() if c == ' ' || c == '\t' || c == '\r' { @@ -136,6 +138,16 @@ func (l *Lexer) Next() token.Token { } continue } + if c == '\\' && l.continuationAhead() { + l.advance() // backslash, then the runes the scan saw + for !l.atEnd() && l.cur() != '\n' { + l.advance() + } + if !l.atEnd() { + l.advance() // the newline that closes the continuation + } + continue + } break } @@ -185,6 +197,28 @@ func (l *Lexer) Next() token.Token { } } +// continuationAhead reports, without consuming anything, whether the +// backslash at the current position closes onto a newline through nothing +// but horizontal whitespace and one line comment. Positions after the +// backslash are inspected directly on the rune slice so a non-match leaves +// the scanner state untouched. +func (l *Lexer) continuationAhead() bool { + i := l.i + 1 + for i < len(l.src) { + switch r := l.src[i]; { + case r == ' ' || r == '\t' || r == '\r': + i++ + case r == '/' && i+1 < len(l.src) && l.src[i+1] == '/': + for i < len(l.src) && l.src[i] != '\n' { + i++ + } + default: + return r == '\n' + } + } + return false +} + // lineComment consumes a // comment up to, but not including, the newline. A // trailing run of \r, spaces and tabs is line-ending whitespace rather than // comment content, so it never enters the token text. Trimming only a \r @@ -403,6 +437,15 @@ func (l *Lexer) punct(start token.Position) token.Token { case '|': l.advance() return l.make(token.Pipe, start, "|") + case ';': + l.advance() + return l.make(token.Semicolon, start, ";") + case '&': + l.advance() + return l.make(token.Ampersand, start, "&") + case '~': + l.advance() + return l.make(token.Tilde, start, "~") default: // Unknown rune: emit it as Illegal and move on. l.advance() diff --git a/parser/expr.go b/parser/expr.go new file mode 100644 index 0000000..9c066dd --- /dev/null +++ b/parser/expr.go @@ -0,0 +1,146 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: BSD-3-Clause + +// Constant-expression folding for operands. The toolchain's assembler +// evaluates arithmetic in every operand position, and macro-heavy GOROOT +// sources lean on it: parameterised bodies carry offsets like +// ((index*4)+0)(base), immediates like $(32-shift) and masks like +// $~63 or $(1<<0|1<<9). Substituting the parameters textually therefore +// leaves constant arithmetic behind, and the parser folds it here, keeping +// the operand AST identical to what the same literals written out would +// produce. Anything that is not a closed integer expression fails to fold +// and falls through to the ordinary operand paths. +package parser + +import ( + "sourcedock.dev/petrbalvin/gasm-devkit/token" +) + +// foldExpr evaluates the constant integer expression at the head of ts and +// returns its value together with the unconsumed tokens. ok is false when +// the tokens do not form an expression, which is the callers' signal to use +// the ordinary parsing paths. +func foldExpr(ts []token.Token) (val int64, rest []token.Token, ok bool) { + v, rest, ok := foldAdd(ts) + if !ok { + return 0, ts, false + } + return v, rest, true +} + +// foldAdd parses addition-level expressions: +, - and | bind loosest, the +// Plan 9 convention that makes x<<1|3 read as (x<<1)|3. +func foldAdd(ts []token.Token) (int64, []token.Token, bool) { + v, rest, ok := foldMul(ts) + if !ok { + return 0, ts, false + } + for len(rest) > 0 { + kind := rest[0].Kind + if kind != token.Plus && kind != token.Minus && kind != token.Pipe { + return v, rest, true + } + w, r2, ok := foldMul(rest[1:]) + if !ok { + return v, rest, true + } + switch kind { + case token.Plus: + v += w + case token.Minus: + v -= w + case token.Pipe: + v |= w + } + rest = r2 + } + return v, rest, true +} + +// foldMul parses multiplication-level expressions: *, / and the bit +// operators &, << and >>. +func foldMul(ts []token.Token) (int64, []token.Token, bool) { + v, rest, ok := foldFactor(ts) + if !ok { + return 0, ts, false + } + for len(rest) > 0 { + switch rest[0].Kind { + case token.Star: + w, r2, ok := foldFactor(rest[1:]) + if !ok { + return v, rest, true + } + v *= w + rest = r2 + case token.Slash: + w, r2, ok := foldFactor(rest[1:]) + if !ok || w == 0 { + return v, rest, true + } + v /= w + rest = r2 + case token.Ampersand: + w, r2, ok := foldFactor(rest[1:]) + if !ok { + return v, rest, true + } + v &= w + rest = r2 + case token.LShift: + w, r2, ok := foldFactor(rest[1:]) + if !ok || w < 0 || w >= 64 { + return v, rest, true + } + v <<= uint(w) + rest = r2 + case token.RShift: + w, r2, ok := foldFactor(rest[1:]) + if !ok || w < 0 || w >= 64 { + return v, rest, true + } + v >>= uint(w) + rest = r2 + default: + return v, rest, true + } + } + return v, rest, true +} + +// foldFactor parses a number, a parenthesised expression, or a unary sign +// or complement. +func foldFactor(ts []token.Token) (int64, []token.Token, bool) { + if len(ts) == 0 { + return 0, ts, false + } + switch ts[0].Kind { + case token.Number: + v, ok := tryInt(ts[0].Text) + if !ok { + return 0, ts, false + } + return v, ts[1:], true + case token.LParen: + v, rest, ok := foldAdd(ts[1:]) + if !ok || len(rest) == 0 || rest[0].Kind != token.RParen { + return 0, ts, false + } + return v, rest[1:], true + case token.Minus: + v, rest, ok := foldFactor(ts[1:]) + if !ok { + return 0, ts, false + } + return -v, rest, true + case token.Plus: + return foldFactor(ts[1:]) + case token.Tilde: + v, rest, ok := foldFactor(ts[1:]) + if !ok { + return 0, ts, false + } + return ^v, rest, true + } + return 0, ts, false +} diff --git a/parser/parser.go b/parser/parser.go index 6ec3bc3..148de2c 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -408,6 +408,18 @@ func parseImmediate(g []token.Token) ast.Immediate { return imm } } + // A constant expression introduced by '(' or '~'. Textual macro + // substitution leaves arithmetic such as $(32-shift) and $~63 behind, + // and the toolchain evaluates it in place; only shapes the ordinary + // paths below cannot read reach the folder, so every existing form + // keeps its exact parse. + if g[0].Kind == token.LParen || g[0].Kind == token.Tilde { + if v, rest, ok := foldExpr(g); ok && len(rest) == 0 { + imm.Val = v + imm.HasVal = true + return imm + } + } i := 0 if g[i].Kind == token.Minus { imm.Neg = true @@ -459,6 +471,17 @@ func parseAddress(g []token.Token) ast.Address { } i := 0 + // A parenthesised constant expression as the displacement: substituted + // macro bodies carry ((index*4)+0)(base) shapes. As with the signed + // number path below, the value is committed only when a base group + // follows. + if i < len(g) && g[i].Kind == token.LParen { + if v, rest, ok := foldExpr(g[i:]); ok && len(rest) > 0 && rest[0].Kind == token.LParen { + addr.Offset = v + addr.HasOff = true + i = len(g) - len(rest) + } + } // Optional leading displacement before a '(' base group. A sign pushes // the parenthesis one token further out: -4(DX) has it at i+2. if isSignedNumber(g, i) { diff --git a/parser/preproc.go b/parser/preproc.go new file mode 100644 index 0000000..721b45e --- /dev/null +++ b/parser/preproc.go @@ -0,0 +1,475 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: BSD-3-Clause + +// The preprocessor turns #define and #include directives into the token +// stream the parser really sees, the way the Go toolchain's assembler does: +// object and parameterised macros expand at the point of use, and an +// #include splices the named file's lines in place of the directive. The +// pass runs only on the assembly path (gasm asm, diff, the corpus audit), +// where the result is machine code; parsing for the linter, formatter and +// language server keeps the raw file so their view of #define lines, and +// therefore their macro-aware behaviour, is unchanged. +package parser + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "unicode/utf8" + + "sourcedock.dev/petrbalvin/gasm-devkit/ast" + "sourcedock.dev/petrbalvin/gasm-devkit/lexer" + "sourcedock.dev/petrbalvin/gasm-devkit/token" +) + +// Options controls the optional preprocessing applied before a file is +// parsed. The zero value reproduces Parse exactly. +type Options struct { + // IncludeDirs lists the -I directories searched for #include files, + // in order, after the including file's own directory. + IncludeDirs []string + // Expand enables macro expansion, include splicing and the + // statement-separator reading of ';' that the expanded bodies rely on. + Expand bool +} + +// ParseWithOptions parses src like Parse, optionally preprocessing it first. +// The returned file is usable even when errors is non-empty. +func ParseWithOptions(path, src string, opts Options) (*ast.File, []error) { + tokens := lexer.Tokenize(src) + var lines [][]token.Token + var errs []error + if opts.Expand { + pp := &preproc{opts: opts, macros: map[string]*macroDef{}} + lines = pp.fileLines(path, tokens, token.Position{}) + errs = pp.errs + } else { + lines = splitLines(tokens) + } + p := &state{path: path} + p.parse(lines) + return p.file, append(errs, p.errs...) +} + +// maxExpansionDepth bounds recursive macro expansion; the toolchain's +// assembler gives up after 100 nested invocations without producing a token. +const maxExpansionDepth = 100 + +// textflagHeader names the one header gasm does not splice: its flag macros +// (NOSPLIT, RODATA, …) are consumed by name throughout gasm's parser, +// encoders and linter, and expanding them to their numeric constants would +// leave every consumer blind to them. +const textflagHeader = "textflag.h" + +// macroDef is one #define. A nil args slice is an object macro; a non-nil +// (possibly empty) one is parameterised, the C distinction between +// "#define A(x)" and "#define A (x)". +type macroDef struct { + name string + args []string + body []token.Token +} + +// preproc carries the state of one expansion pass: the live macro table, the +// chain of files currently being read, for cycle detection, and the +// conditional-inclusion stack of #ifdef regions. +type preproc struct { + opts Options + macros map[string]*macroDef + errs []error + stack []string // absolute paths of files being read, innermost last + ifdefStack []bool // one entry per open #ifdef/#ifndef, its truth +} + +// enabled reports whether the position being read is inside a live +// conditional branch. Directives inside a disabled branch contribute +// nothing, and its content lines are dropped, exactly as the toolchain's +// input stack does. +func (pp *preproc) enabled() bool { + return len(pp.ifdefStack) == 0 || pp.ifdefStack[len(pp.ifdefStack)-1] +} + +func (pp *preproc) errorf(pos token.Position, format string, args ...any) { + pp.errs = append(pp.errs, Error{Pos: pos, Msg: fmt.Sprintf(format, args...)}) +} + +// fileLines tokenizes and preprocesses one file into logical lines. +// Directive lines are kept (the parser records them for the tooling); +// #include lines are replaced by the included file's lines. includePos is +// the position of the #include that pulled this file in, zero for the +// top-level file, and only serves cycle diagnostics. +func (pp *preproc) fileLines(path string, tokens []token.Token, includePos token.Position) [][]token.Token { + abs, err := filepath.Abs(path) + if err != nil { + abs = filepath.Clean(path) + } + if slices.Contains(pp.stack, abs) { + if includePos.IsValid() { + pp.errorf(includePos, "#include %q: include cycle (%s is already being read)", path, filepath.Base(path)) + } + return nil + } + pp.stack = append(pp.stack, abs) + + var out [][]token.Token + for _, line := range splitLines(tokens) { + if len(line) == 0 { + out = append(out, line) + continue + } + if line[0].Kind == token.Hash { + out = append(out, pp.directive(line, filepath.Dir(path))...) + continue + } + if !pp.enabled() { + continue + } + out = append(out, splitOnSemicolons(pp.expandTokens(line))...) + } + pp.stack = pp.stack[:len(pp.stack)-1] + if len(pp.stack) == 0 && len(pp.ifdefStack) > 0 { + // The stack is per-input, shared across includes, so only the + // top-level file's end can decide the input was left unclosed. + pp.errorf(token.Position{Line: 1, Column: 1}, "unclosed #ifdef or #ifndef") + } + return out +} + +// directive processes one '#' line and returns the lines to keep in the +// stream: every directive line is kept as-is for the parser (which records +// it), except #include, which is replaced by the spliced content. +// Conditionals are tracked on every line; every other directive is inert +// inside a disabled branch. +func (pp *preproc) directive(line []token.Token, dir string) [][]token.Token { + if len(line) < 2 || line[1].Kind != token.Ident { + return [][]token.Token{line} + } + switch line[1].Text { + case "ifdef", "ifndef": + pp.ifdef(line, line[1].Text == "ifndef") + case "else": + pp.elseBranch(line) + case "endif": + pp.endif(line) + case "define": + if pp.enabled() { + pp.define(line) + } + case "undef": + if pp.enabled() { + pp.undef(line) + } + case "include": + if pp.enabled() { + return pp.include(line, dir) + } + default: + // #line and unknown directives are recorded but not interpreted: + // conservative support keeps the parser's view intact and files + // using them fail on their content, not silently. + } + return [][]token.Token{line} +} + +// ifdef handles "#ifdef NAME" and "#ifndef NAME", pushing the branch's truth +// onto the conditional stack. A branch opened inside a disabled region is +// itself disabled, however the name resolves. +func (pp *preproc) ifdef(line []token.Token, inverted bool) { + truth := false + if len(line) >= 3 && line[2].Kind == token.Ident { + _, defined := pp.macros[line[2].Text] + truth = defined != inverted + } else { + pp.errorf(line[0].Pos, "expected identifier after #%s", line[1].Text) + } + if !pp.enabled() { + truth = false + } + pp.ifdefStack = append(pp.ifdefStack, truth) +} + +// elseBranch flips the innermost conditional's truth, but only when the +// region enclosing it is itself live: the toolchain keeps outer overrides. +func (pp *preproc) elseBranch(line []token.Token) { + if len(pp.ifdefStack) == 0 { + pp.errorf(line[0].Pos, "unmatched #else") + return + } + if len(pp.ifdefStack) == 1 || pp.ifdefStack[len(pp.ifdefStack)-2] { + pp.ifdefStack[len(pp.ifdefStack)-1] = !pp.ifdefStack[len(pp.ifdefStack)-1] + } +} + +// endif closes the innermost conditional. +func (pp *preproc) endif(line []token.Token) { + if len(pp.ifdefStack) == 0 { + pp.errorf(line[0].Pos, "unmatched #endif") + return + } + pp.ifdefStack = pp.ifdefStack[:len(pp.ifdefStack)-1] +} + +// define parses "#define NAME[(formals)] body" into the macro table. The +// body runs to the end of the logical line (the lexer has already spliced +// backslash continuations) and stops at a comment, which never expands. +func (pp *preproc) define(line []token.Token) { + if len(line) < 3 || line[2].Kind != token.Ident { + return + } + name := line[2] + args := []string(nil) + body := line[3:] + // The definition is parameterised only when '(' follows the name + // directly; the toolchain separates "#define A(x)" from + // "#define A (x)" by adjacency, and so does the column check here. + if len(body) > 0 && body[0].Kind == token.LParen && + body[0].Pos.Column == name.Pos.Column+utf8.RuneCountInString(name.Text) { + args = []string{} + i := 1 + for i < len(body) && body[i].Kind != token.RParen { + if body[i].Kind == token.Ident { + args = append(args, body[i].Text) + } + i++ + } + if i < len(body) { + body = body[i+1:] + } else { + body = nil + } + } + if i := slices.IndexFunc(body, func(t token.Token) bool { return t.Kind == token.Comment }); i >= 0 { + body = body[:i] + } + if _, exists := pp.macros[name.Text]; exists { + // The toolchain refuses redefinition, so a file the oracle accepts + // never redefines; failing here keeps that contract visible. + pp.errorf(name.Pos, "redefinition of macro %s", name.Text) + } + pp.macros[name.Text] = ¯oDef{name: name.Text, args: args, body: pp.bodyWithBreaks(body)} + +} + +// bodyWithBreaks records the statement boundaries the continuations carry. +// The lexer splices backslash-continued lines into one logical line, but the +// toolchain keeps the newline as a token in the stored body, which is how a +// multi-instruction body without semicolons (the arm64 style) still splits +// into statements on expansion. A line change inside the logical line is +// exactly a continuation, so the boundary is restored from the positions. +func (pp *preproc) bodyWithBreaks(body []token.Token) []token.Token { + out := make([]token.Token, 0, len(body)) + for i, t := range body { + if i > 0 && t.Pos.Line != body[i-1].Pos.Line { + out = append(out, token.Token{Kind: token.Newline, Text: "\n", Pos: t.Pos, End: t.Pos}) + } + out = append(out, t) + } + return out +} + +// undef handles "#undef NAME", which the toolchain honours and requires to +// name a defined macro. +func (pp *preproc) undef(line []token.Token) { + if len(line) < 3 || line[2].Kind != token.Ident { + return + } + if _, ok := pp.macros[line[2].Text]; !ok { + pp.errorf(line[2].Pos, "#undef for undefined macro %s", line[2].Text) + return + } + delete(pp.macros, line[2].Text) +} + +// include resolves and splices "#include \"file\"". A header that cannot be +// read keeps the directive line in the stream, with a diagnostic. +func (pp *preproc) include(line []token.Token, dir string) [][]token.Token { + if len(line) < 3 || line[2].Kind != token.String { + return [][]token.Token{line} + } + header := line[2] + name, err := strconv.Unquote(header.Text) + if err != nil { + pp.errorf(header.Pos, "unquoting include file name: %v", err) + return [][]token.Token{line} + } + if filepath.Base(name) == textflagHeader { + // Flag macros are handled natively (see textflagHeader); the + // directive stays so tools still see the include. + return [][]token.Token{line} + } + resolved, ok := pp.resolve(name, dir) + if !ok { + searched := append([]string{dir}, pp.opts.IncludeDirs...) + pp.errorf(header.Pos, "#include %q: file not found (searched %s)", name, strings.Join(searched, ", ")) + return [][]token.Token{line} + } + src, err := os.ReadFile(resolved) + if err != nil { + pp.errorf(header.Pos, "#include %q: %v", name, err) + return [][]token.Token{line} + } + return pp.fileLines(resolved, lexer.Tokenize(string(src)), header.Pos) +} + +// resolve looks an include name up the way the toolchain does: as written +// (relative to the working directory), then relative to the including +// file's directory, then in each -I directory in order. +func (pp *preproc) resolve(name, dir string) (string, bool) { + candidates := []string{name} + if !filepath.IsAbs(name) { + candidates = append(candidates, filepath.Join(dir, name)) + for _, d := range pp.opts.IncludeDirs { + candidates = append(candidates, filepath.Join(d, name)) + } + } + for _, c := range candidates { + if st, err := os.Stat(c); err == nil && !st.IsDir() { + return c, true + } + } + return "", false +} + +// expandTokens expands every macro invocation in a token sequence, +// recursively, with a depth guard. A body is spliced into the sequence in +// place and rescanned, the way the toolchain's input stack re-reads pushed +// tokens: an object macro may name a parameterised one, and the argument +// list of the expansion may then come from the tokens that follow. +func (pp *preproc) expandTokens(in []token.Token) []token.Token { + s := in + i := 0 + consecutive := 0 + for i < len(s) { + t := s[i] + if t.Kind != token.Ident { + i++ + consecutive = 0 + continue + } + def := pp.macros[t.Text] + if def == nil { + i++ + consecutive = 0 + continue + } + // The guard mirrors the toolchain's: 100 nested invocations in a + // row without a plain token between them means recursion. + consecutive++ + if consecutive > maxExpansionDepth { + pp.errorf(t.Pos, "recursive macro invocation (deeper than %d levels)", maxExpansionDepth) + return nil + } + if def.args == nil { + s = append(s[:i], append(restamp(def.body, t.Pos), s[i+1:]...)...) + continue + } + // A parameterised macro invoked without its parentheses stands + // unexpanded, naming itself, as in the toolchain. + if i+1 >= len(s) || s[i+1].Kind != token.LParen { + i++ + consecutive = 0 + continue + } + args, next := pp.collectArgs(s, i+1, t) + if args == nil { + return nil + } + // A zero-argument macro may be invoked as NAME(). + if len(def.args) == 0 && len(args) == 1 && len(args[0]) == 0 { + args = nil + } + if len(args) != len(def.args) { + pp.errorf(t.Pos, "wrong arg count for macro %s: got %d, want %d", t.Text, len(args), len(def.args)) + i = next + consecutive = 0 + continue + } + sub := make([]token.Token, 0, len(def.body)) + for _, bt := range def.body { + if bt.Kind == token.Ident { + if k := slices.Index(def.args, bt.Text); k >= 0 { + sub = append(sub, restamp(args[k], t.Pos)...) + continue + } + } + sub = append(sub, bt) + } + s = append(s[:i], append(sub, s[next:]...)...) + } + return s +} + +// collectArgs reads the actual argument tokens of an invocation; the opening +// parenthesis is at start. Commas separate arguments except inside nested +// parentheses. A nil result means the list was unterminated, which is a +// diagnostic. +func (pp *preproc) collectArgs(in []token.Token, start int, name token.Token) ([][]token.Token, int) { + var args [][]token.Token + var cur []token.Token + nesting := 0 + for i := start + 1; i < len(in); i++ { + t := in[i] + switch t.Kind { + case token.LParen: + nesting++ + cur = append(cur, t) + case token.RParen: + if nesting == 0 { + return append(args, cur), i + 1 + } + nesting-- + cur = append(cur, t) + case token.Comma: + if nesting == 0 { + args = append(args, cur) + cur = nil + continue + } + cur = append(cur, t) + case token.Comment: + pp.errorf(name.Pos, "unterminated arg list invoking macro %s", name.Text) + return nil, i + default: + cur = append(cur, t) + } + } + pp.errorf(name.Pos, "unterminated arg list invoking macro %s", name.Text) + return nil, len(in) +} + +// restamp copies body tokens to the invocation's position, so diagnostics +// and the line table point where the macro was used, as the toolchain's +// input stack does. +func restamp(body []token.Token, pos token.Position) []token.Token { + out := make([]token.Token, len(body)) + for i, t := range body { + t.Pos, t.End = pos, pos + out[i] = t + } + return out +} + +// splitOnSemicolons breaks a token sequence at ';' statement separators and +// at the Newline markers that record continuation boundaries inside macro +// bodies, producing the logical lines the parser expects. The separators +// carry no meaning beyond the break, so the pieces are exactly what the same +// statements on separate lines would produce. +func splitOnSemicolons(ts []token.Token) [][]token.Token { + var out [][]token.Token + start := 0 + for i, t := range ts { + if t.Kind == token.Semicolon || t.Kind == token.Newline { + if i > start { + out = append(out, ts[start:i]) + } + start = i + 1 + } + } + if start < len(ts) { + out = append(out, ts[start:]) + } + return out +} diff --git a/parser/preproc_test.go b/parser/preproc_test.go new file mode 100644 index 0000000..3dff224 --- /dev/null +++ b/parser/preproc_test.go @@ -0,0 +1,552 @@ +// Copyright (c) 2026 Petr Balvín (https://petrbalvin.org) +// SPDX-License-Identifier: BSD-3-Clause + +package parser + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "sourcedock.dev/petrbalvin/gasm-devkit/ast" +) + +// expand parses src with preprocessing enabled and returns the first TEXT's +// body instructions as "MNEMONIC operand|operand" strings, the shape the +// expansion assertions below compare against. Runs of spaces are +// collapsed: Raw renders a token group as its tokens joined with single +// spaces, so "$(32-7)" arrives as "$ ( 32 - 7 )" and the comparison must +// not depend on that spelling. +func expand(t *testing.T, src string) (*ast.File, []string) { + t.Helper() + f, errs := ParseWithOptions("t_amd64.s", src, Options{Expand: true}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + ts := texts(f) + if len(ts) == 0 { + t.Fatalf("no TEXT in:\n%s", src) + } + var got []string + for _, s := range ts[0].Body { + in, ok := s.(*ast.Instr) + if !ok { + continue + } + var ops []string + for _, op := range in.Operands { + ops = append(ops, op.Raw) + } + line := in.Mnemonic.Text + " " + strings.Join(ops, ", ") + got = append(got, strings.ReplaceAll(line, " ", "")) + } + return f, got +} + +func wantLines(t *testing.T, got []string, want ...string) { + t.Helper() + strip := func(lines []string) string { + var out []string + for _, l := range lines { + out = append(out, strings.ReplaceAll(l, " ", "")) + } + return strings.Join(out, "\n") + } + if strip(got) != strip(want) { + t.Errorf("expanded body:\n %s\nwant:\n %s", strings.Join(got, "\n "), strings.Join(want, "\n ")) + } +} + +func TestObjectMacroExpandsAtUse(t *testing.T) { + _, got := expand(t, ` +#define REGTMP CX +#define TWICE ADDQ CX, AX; ADDQ CX, AX +TEXT ·f(SB), NOSPLIT, $0 + MOVQ 8(SP), REGTMP + TWICE + RET +`) + wantLines(t, got, + "MOVQ 8(SP), CX", + "ADDQ CX, AX", + "ADDQ CX, AX", + "RET", + ) +} + +func TestParameterisedMacroSubstitutesArguments(t *testing.T) { + f, errs := ParseWithOptions("t_amd64.s", ` +#define ROUND1(a, index, const, shift) \ + ADDQ $const, a; \ + MOVW (index*4)(SP), a; \ + RORQ $(32-shift), a +TEXT ·f(SB), NOSPLIT, $0 + ROUND1(AX, 3, 0xd76aa478, 7) + RET +`, Options{Expand: true}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + body := texts(f)[0].Body + add := body[0].(*ast.Instr) + if add.Mnemonic.Text != "ADDQ" || !add.Operands[0].Imm.HasVal || + add.Operands[0].Imm.Val != 0xd76aa478 || add.Operands[1].Addr.Sym == nil || + add.Operands[1].Addr.Sym.Name != "AX" { + t.Errorf("ADDQ operands substituted wrong: %+v %+v", add.Operands[0].Imm, add.Operands[1].Addr) + } + mov := body[1].(*ast.Instr) + if addr := mov.Operands[0].Addr; !addr.HasOff || addr.Offset != 12 { + t.Errorf("MOVW offset = %+v, want 12 from 3*4", addr) + } + ror := body[2].(*ast.Instr) + if !ror.Operands[0].Imm.HasVal || ror.Operands[0].Imm.Val != 25 { + t.Errorf("RORQ immediate = %+v, want 25 from (32-7)", ror.Operands[0].Imm) + } +} + +func TestMacroArgumentsKeepCommasInParens(t *testing.T) { + // An argument may itself be an unparenthesised expression: the tokens + // substitute verbatim and the parser folds the result, as the + // toolchain's parser does. + f, errs := ParseWithOptions("t_amd64.s", ` +#define LOAD(dst, off) MOVQ off(SP), dst +TEXT ·f(SB), NOSPLIT, $0 + LOAD(AX, 1*8) + RET +`, Options{Expand: true}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + in := texts(f)[0].Body[0].(*ast.Instr) + addr := in.Operands[0].Addr + if !addr.HasOff || addr.Offset != 8 { + t.Errorf("offset = %+v, want 8", addr) + } + if sym := in.Operands[1].Addr.Sym; sym == nil || sym.Name != "AX" { + t.Errorf("destination = %+v, want AX", in.Operands[1].Addr) + } +} + +func TestNestedMacroInvocations(t *testing.T) { + // An object macro naming a parameterised one, and a parameterised body + // invoking another parameterised macro: the toolchain's input stack + // rescans substituted tokens, and so does expansion here. + _, got := expand(t, ` +#define DOUBLE(x) ADDQ x, x +#define TWICE2 DOUBLE +#define FOUR(a, b) DOUBLE(a); DOUBLE(b) +TEXT ·f(SB), NOSPLIT, $0 + TWICE2(AX) + FOUR(AX, CX) + RET +`) + wantLines(t, got, + "ADDQ AX, AX", + "ADDQ AX, AX", + "ADDQ CX, CX", + "RET", + ) +} + +func TestMultiLineBodySplitsWithoutSemicolons(t *testing.T) { + // The arm64 style: backslash-continued lines with no semicolons. The + // continuation newline is a statement boundary, as in the toolchain. + _, got := expand(t, ` +#define PAIR \ + ADDQ AX, AX \ + MOVQ AX, CX +TEXT ·f(SB), NOSPLIT, $0 + PAIR + RET +`) + wantLines(t, got, + "ADDQ AX, AX", + "MOVQ AX, CX", + "RET", + ) +} + +func TestZeroArgumentMacro(t *testing.T) { + _, got := expand(t, ` +#define BARRIER() +TEXT ·f(SB), NOSPLIT, $0 + BARRIER() + RET +`) + wantLines(t, got, "RET") +} + +func TestParameterisedWithoutParensStandsAsName(t *testing.T) { + // A parameterised macro invoked without its parentheses names itself, + // which the parser then reports as an unknown instruction rather than + // silently expanding nothing. + f, errs := ParseWithOptions("t_amd64.s", ` +#define M(x) ADDQ x, x +TEXT ·f(SB), NOSPLIT, $0 + M + RET +`, Options{Expand: true}) + if len(errs) != 0 { + t.Fatalf("parse: %v", errs) + } + fn := texts(f)[0] + if len(fn.Body) == 0 { + t.Fatal("body empty") + } + in, ok := fn.Body[0].(*ast.Instr) + if !ok || in.Mnemonic.Text != "M" { + t.Fatalf("bare parameterised macro did not stand as its name: %+v", fn.Body[0]) + } +} + +func TestDefinitionScoping(t *testing.T) { + // A definition applies from its point onward: the use before the + // #define stays untouched. + _, got := expand(t, ` +TEXT ·f(SB), NOSPLIT, $0 + SPECIAL +#define SPECIAL ADDQ AX, AX + SPECIAL + RET +`) + wantLines(t, got, + "SPECIAL", + "ADDQ AX, AX", + "RET", + ) +} + +func TestUndefRemovesMacro(t *testing.T) { + _, got := expand(t, ` +#define TEMP AX +TEXT ·f(SB), NOSPLIT, $0 + TEMP +#undef TEMP + TEMP + RET +`) + wantLines(t, got, + "AX", + "TEMP", + "RET", + ) +} + +func TestUndefUndefinedMacroIsAnError(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#undef NOSUCH\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "undefined macro NOSUCH") { + t.Fatalf("#undef of an undefined macro: got %v, want an error naming it", errs) + } +} + +func TestRedefinitionIsAnError(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#define A X\n#define A Y\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "redefinition of macro A") { + t.Fatalf("redefinition: got %v, want an error", errs) + } +} + +func TestRecursiveMacroIsAnError(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#define A B\n#define B A\nTEXT ·f(SB), NOSPLIT, $0\n\tA\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "recursive macro invocation") { + t.Fatalf("recursion: got %v, want a recursive-macro error, not a hang", errs) + } +} + +func TestWrongArgumentCountIsAnError(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#define M(a, b) ADDQ a, b\nTEXT ·f(SB), NOSPLIT, $0\n\tM(AX)\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "wrong arg count for macro M") { + t.Fatalf("arg count: got %v, want an error", errs) + } +} + +func TestConditionalsSelectOneBranch(t *testing.T) { + _, got := expand(t, ` +#define MODE2 +TEXT ·f(SB), NOSPLIT, $0 +#ifdef MODE2 + ADDQ AX, AX +#else + SUBQ AX, AX +#endif +#ifndef MODE2 + SUBQ CX, CX +#else + ADDQ CX, CX +#endif + RET +`) + wantLines(t, got, + "ADDQ AX, AX", + "ADDQ CX, CX", + "RET", + ) +} + +func TestConditionalsHideDefinitionsAndIncludes(t *testing.T) { + // A definition inside a disabled branch must not exist, and an + // unresolvable include there must not be followed. + _, got := expand(t, ` +TEXT ·f(SB), NOSPLIT, $0 +#ifdef NOTDEFINED +#define HIDEN ADDQ AX, AX +#include "nowhere.h" +#endif + HIDEN + RET +`) + wantLines(t, got, "HIDEN", "RET") +} + +func TestUnclosedConditionalIsAnError(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#ifdef X\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "unclosed #ifdef") { + t.Fatalf("unclosed conditional: got %v, want an error", errs) + } +} + +func TestUnmatchedConditionalDelimitersAreErrors(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#endif\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "unmatched #endif") { + t.Fatalf("unmatched #endif: got %v, want an error", errs) + } + _, errs = ParseWithOptions("t_amd64.s", "#else\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "unmatched #else") { + t.Fatalf("unmatched #else: got %v, want an error", errs) + } +} + +// includeTree writes a directory of include files and returns its path. +func includeTree(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for name, content := range files { + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +func TestIncludeSplicesAndDefinesAreShared(t *testing.T) { + dir := includeTree(t, map[string]string{ + "consts.h": "#define KONST $42\n", + }) + f, errs := ParseWithOptions("t_amd64.s", ` +#include "consts.h" +TEXT ·f(SB), NOSPLIT, $0 + MOVQ KONST, AX + RET +`, Options{Expand: true, IncludeDirs: []string{dir}}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + in := texts(f)[0].Body[0].(*ast.Instr) + if in.Mnemonic.Text != "MOVQ" || strings.ReplaceAll(in.Operands[0].Raw, " ", "") != "$42" { + t.Fatalf("include splicing failed: %+v", in) + } +} + +func TestIncludeResolutionOrder(t *testing.T) { + // The including file's directory wins over the -I list, and the -I list + // is searched in order. + src := includeTree(t, map[string]string{ + "inc/main.s": "#include \"which.h\"\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", + "inc/which.h": "#define WHO ONE\n", + "first/which.h": "#define WHO TWO\n", + "second/which.h": "#define WHO THREE\n", + }) + main := filepath.Join(src, "inc", "main.s") + body, err := os.ReadFile(main) + if err != nil { + t.Fatal(err) + } + // The header exists in the including file's directory and in two -I + // directories; the source-directory copy must win. + f, errs := ParseWithOptions(main, string(body), Options{Expand: true, IncludeDirs: []string{ + filepath.Join(src, "first"), filepath.Join(src, "second"), + }}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + found := false + for _, d := range f.Decls { + if pp, ok := d.(*ast.Preproc); ok && strings.Contains(pp.Raw, "define WHO ONE") { + found = true + } + } + if !found { + t.Error("the including file's directory did not win include resolution") + } +} + +func TestIncludeSearchesIncludeDirsInOrder(t *testing.T) { + src := includeTree(t, map[string]string{ + "inc/main.s": "#include \"which.h\"\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", + "first/which.h": "#define WHO TWO\n", + "second/which.h": "#define WHO THREE\n", + }) + main := filepath.Join(src, "inc", "main.s") + body, err := os.ReadFile(main) + if err != nil { + t.Fatal(err) + } + f, errs := ParseWithOptions(main, string(body), Options{Expand: true, IncludeDirs: []string{ + filepath.Join(src, "first"), filepath.Join(src, "second"), + }}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + for _, d := range f.Decls { + if pp, ok := d.(*ast.Preproc); ok && strings.Contains(pp.Raw, "define WHO THREE") { + t.Error("the second -I directory was searched before the first") + } + } +} + +func TestIncludeCycleIsDetected(t *testing.T) { + src := includeTree(t, map[string]string{ + "a.s": "#include \"b.s\"\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", + "b.s": "#include \"a.s\"\n", + }) + _, errs := ParseWithOptions(filepath.Join(src, "a.s"), "#include \"b.s\"\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", + Options{Expand: true}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), "include cycle") { + t.Fatalf("include cycle: got %v, want a cycle diagnostic, not a hang", errs) + } +} + +func TestUnresolvableIncludeIsAnError(t *testing.T) { + _, errs := ParseWithOptions("t_amd64.s", "#include \"nothere.h\"\nTEXT ·f(SB), NOSPLIT, $0\n\tRET\n", + Options{Expand: true, IncludeDirs: []string{t.TempDir()}}) + if len(errs) == 0 || !strings.Contains(errs[0].Error(), `#include "nothere.h"`) { + t.Fatalf("missing include: got %v, want a clear diagnostic", errs) + } +} + +func TestTextflagHeaderIsNeverSpliced(t *testing.T) { + // textflag.h resolves nowhere here, yet the file must parse: the flag + // names are consumed natively and the include stays in the tree. + f, errs := ParseWithOptions("t_amd64.s", ` +#include "textflag.h" +TEXT ·f(SB), NOSPLIT, $0 + RET +`, Options{Expand: true}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + hasInclude := false + for _, d := range f.Decls { + if _, ok := d.(*ast.Include); ok { + hasInclude = true + } + } + if !hasInclude { + t.Error("textflag.h include was dropped from the tree") + } +} + +func TestSemicolonSplitsRawLinesToo(t *testing.T) { + _, got := expand(t, ` +TEXT ·f(SB), NOSPLIT, $0 + BYTE $0x0f; BYTE $0x1f + RET +`) + wantLines(t, got, "BYTE $0x0f", "BYTE $0x1f", "RET") +} + +func TestParseUnchangedWithoutExpand(t *testing.T) { + // Without Expand the preprocessor must not exist: a macro invocation + // stays an unexpanded instruction line and ';' keeps the old parse. + f, errs := Parse("t_amd64.s", ` +#define TWICE ADDQ AX, AX +TEXT ·f(SB), NOSPLIT, $0 + TWICE + BYTE $0x0f; BYTE $0x1f + RET +`) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + fn := texts(f)[0] + var mnemonics []string + for _, s := range fn.Body { + if in, ok := s.(*ast.Instr); ok { + mnemonics = append(mnemonics, in.Mnemonic.Text) + } + } + if strings.Join(mnemonics, " ") != "TWICE BYTE RET" { + t.Errorf("non-expanding parse changed: %v", mnemonics) + } +} + +func TestConstantExpressionFolding(t *testing.T) { + // The shapes substituted macro bodies leave behind: parenthesised + // arithmetic in immediates and displacements, tilde complements. The + // assertions read the semantic fields; Raw keeps the operand's tokens + // in the canonicalised rendering, not the folded values. + f, errs := ParseWithOptions("t_amd64.s", ` +TEXT ·f(SB), NOSPLIT, $0 + RORQ $(32-7), AX + ANDQ $~63, AX + MOVQ ((2*4)+0)(SP), AX + MOVQ $((1<<3)|(1<<1)), AX + RET +`, Options{Expand: true}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + body := texts(f)[0].Body + ror := body[0].(*ast.Instr) + if !ror.Operands[0].Imm.HasVal || ror.Operands[0].Imm.Val != 25 { + t.Errorf("RORQ immediate = %+v, want 25", ror.Operands[0].Imm) + } + and := body[1].(*ast.Instr) + if !and.Operands[0].Imm.HasVal || and.Operands[0].Imm.Val != -64 { + t.Errorf("ANDQ immediate = %+v, want -64", and.Operands[0].Imm) + } + mov := body[2].(*ast.Instr) + addr := mov.Operands[0].Addr + if !addr.HasOff || addr.Offset != 8 || addr.Base != "SP" { + t.Errorf("MOVQ address = %+v, want 8(SP)", addr) + } + mov2 := body[3].(*ast.Instr) + if !mov2.Operands[0].Imm.HasVal || mov2.Operands[0].Imm.Val != 10 { + t.Errorf("MOVQ immediate = %+v, want 10", mov2.Operands[0].Imm) + } +} + +func TestConstantExpressionFoldsWithoutExpand(t *testing.T) { + // Folding is a parser capability, not a preprocessing one: a + // hand-written $(32-7) folds the same way with expansion off. + f, errs := ParseWithOptions("t_amd64.s", "TEXT ·f(SB), NOSPLIT, $0\n\tRORQ $(32-7), AX\n\tRET\n", Options{}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + in := texts(f)[0].Body[0].(*ast.Instr) + if !in.Operands[0].Imm.HasVal || in.Operands[0].Imm.Val != 25 { + t.Errorf("Imm = %+v, want 25", in.Operands[0].Imm) + } +} + +func TestNotAnExpressionFallsBack(t *testing.T) { + // Symbol immediates and floats must keep their ordinary parse. + f, errs := ParseWithOptions("t_amd64.s", "TEXT ·f(SB), NOSPLIT, $0\n\tMOVQ $1.5, AX\n\tMOVQ $·sym(SB), AX\n\tRET\n", Options{}) + if len(errs) > 0 { + t.Fatalf("parse: %v", errs) + } + fn := texts(f)[0] + mov1 := fn.Body[0].(*ast.Instr) + if mov1.Operands[0].Imm.HasVal || mov1.Operands[0].Imm.Float != "1.5" { + t.Errorf("float immediate parsed as %+v", mov1.Operands[0].Imm) + } + mov2 := fn.Body[1].(*ast.Instr) + if mov2.Operands[0].Imm.Sym == nil { + t.Errorf("symbol immediate parsed as %+v", mov2.Operands[0].Imm) + } +} diff --git a/token/token.go b/token/token.go index 5de01c5..65a6974 100644 --- a/token/token.go +++ b/token/token.go @@ -43,6 +43,13 @@ const ( At // @ Hash // # Pipe // | + + // Semicolon separates statements on one line (a Plan 9 statement + // terminator); Ampersand and Tilde are the expression operators & and ~ + // of constant expressions. All three appear mostly inside macro bodies. + Semicolon // ; + Ampersand // & + Tilde // ~ ) var kindNames = map[Kind]string{ @@ -71,6 +78,10 @@ var kindNames = map[Kind]string{ At: "@", Hash: "#", Pipe: "|", + + Semicolon: ";", + Ampersand: "&", + Tilde: "~", } // String returns a human-readable name for the kind.