// Copyright (c) 2026 Petr BalvĂ­n (https://petrbalvin.org) // SPDX-License-Identifier: BSD-3-Clause // Package token defines the lexical tokens of Go's Plan 9 assembler (GAsm) // and the source positions attached to them. It has no dependencies and is // shared by the lexer, parser, formatter, linter and language server. package token import "strconv" // Kind classifies a lexical token. type Kind int // The token kinds. The zero value is Illegal so that an uninitialised Token // is obviously invalid. const ( Illegal Kind = iota EOF Newline Comment // Literals and names. Ident // instruction mnemonic, label, register or symbol name Number // integer or floating-point literal (sign carried separately) String // "..." Rune // '.' // Punctuation and operators. LParen // ( RParen // ) Comma // , Plus // + Minus // - Star // * Slash // / Colon // : Dollar // $ LAngle // < RAngle // > LShift // << RShift // >> Arrow // -> At // @ Hash // # ) var kindNames = map[Kind]string{ Illegal: "ILLEGAL", EOF: "EOF", Newline: "NEWLINE", Comment: "COMMENT", Ident: "IDENT", Number: "NUMBER", String: "STRING", Rune: "RUNE", LParen: "(", RParen: ")", Comma: ",", Plus: "+", Minus: "-", Star: "*", Slash: "/", Colon: ":", Dollar: "$", LAngle: "<", RAngle: ">", LShift: "<<", RShift: ">>", Arrow: "->", At: "@", Hash: "#", } // String returns a human-readable name for the kind. func (k Kind) String() string { if s, ok := kindNames[k]; ok { return s } return "Kind(" + strconv.Itoa(int(k)) + ")" } // Position is a byte offset plus one-based line and column within a file. type Position struct { Offset int // byte offset, zero-based Line int // line number, one-based Column int // column number, one-based (in runes) } // String renders the position as "line:column". func (p Position) String() string { return strconv.Itoa(p.Line) + ":" + strconv.Itoa(p.Column) } // IsValid reports whether the position carries a real line number. func (p Position) IsValid() bool { return p.Line > 0 } // Token is a single lexical token together with its literal text and span. type Token struct { Kind Kind Text string Pos Position // inclusive start End Position // exclusive end } // String renders the token for diagnostics and debugging. func (t Token) String() string { return t.Pos.String() + " " + t.Kind.String() + " " + strconv.Quote(t.Text) }