// Flash lexer — source bytes to a token stream for Flash.
//
// A single forward pass, zero allocation: tokens carry byte spans into the
// caller's source buffer rather than copied text (see token.flash). The
// grammar is small enough that a table-driven scanner would be more machinery
// than the language earns today; this stays a readable switch the parser can
// trust.
//
// Trivia handling: only spaces, tabs, CR and LF are skipped between tokens.
// Newlines bump the line counter, which the parser reads both for diagnostics
// and for the semicolon-free statement boundary (a binary/postfix operator
// opening a new line begins a new statement). Comments are NOT trivia: a `///`
// doc comment is a content-bearing `doc_comment` token, and every other `//…`
// shape is a `line_comment` token (the parser filters those out of the descent
// stream into `p.comments` for the formatter). The one other line-bound token
// is a `\\…` multiline-string line — its span ends at the newline, and the
// parser folds a run of consecutive lines into a single string value.

use "token"
use "support" as sup

const Kind = token.Kind
const Token = token.Token

pub const Lexer = struct {
    src []u8,
    pos u32 = 0,
    line u32 = 1,

    pub fn init(src []u8) Lexer {
        return .{ .src = src }
    }

    fn at(self *Lexer) u8 {
        return if (self.pos < self.src.len) self.src[self.pos] else 0
    }

    fn peek(self *Lexer, n u32) u8 {
        i := self.pos + n
        return if (i < self.src.len) self.src[i] else 0
    }

    // `///` (exactly three slashes) marks a doc comment — a content-bearing
    // token. A bare `//`, four-or-more slashes (`////…`), and the top-level
    // `//!` are all ordinary line comments (also tokens — see lexLineComment).
    // next() uses this to route the `///` shape to lexDocComment and every
    // other `//…` to lexLineComment, so the doc-vs-comment rule lives in one
    // place.
    fn atDocComment(self *Lexer) bool {
        return self.at() == '/' && self.peek(1) == '/' && self.peek(2) == '/' && self.peek(3) != '/'
    }

    // Advance past whitespace only — spaces, tabs, carriage returns and
    // newlines. Comments are not trivia: both the `///` doc comment and
    // every other `//…` line comment surface as tokens, lexed by next().
    fn skipTrivia(self *mut Lexer) {
        while self.pos < self.src.len {
            switch self.src[self.pos] {
                ' ', '\t', '\r' => {
                    self.pos += 1
                },
                '\n' => {
                    self.line += 1
                    self.pos += 1
                },
                else => return,
            }
        }
    }

    pub fn next(self *mut Lexer) Token {
        self.skipTrivia()
        start := self.pos
        line := self.line
        if self.pos >= self.src.len {
            return self.make(.eof, start, line)
        }

        c := self.src[self.pos]
        if isIdentStart(c) {
            return self.lexIdent(start, line)
        }
        if isDigit(c) {
            return self.lexNumber(start, line)
        }
        if c == '"' {
            return self.lexString(start, line)
        }
        if c == '\'' {
            return self.lexChar(start, line)
        }
        if c == '#' {
            return self.lexBuiltin(start, line)
        }
        if c == '\\' && self.peek(1) == '\\' {
            return self.lexMultilineStr(start, line)
        }
        if self.atDocComment() {
            return self.lexDocComment(start, line)
        }
        if c == '/' && self.peek(1) == '/' {
            return self.lexLineComment(start, line)
        }

        // single- and two-character punctuation / operators
        self.pos += 1
        const k Kind = switch c {
            '(' => .l_paren,
            ')' => .r_paren,
            '{' => .l_brace,
            '}' => .r_brace,
            '[' => .l_bracket,
            ']' => .r_bracket,
            ',' => .comma,
            '+' => self.lexPlus(),
            '*' => self.lexStar(),
            '/' => self.pick('=', .slash_eq, .slash),
            '%' => self.pick('=', .percent_eq, .percent),
            '?' => .question,
            ':' => self.pick('=', .colon_equal, .colon),
            '.' => self.lexDot(),
            '=' => self.lexEq(),
            '!' => self.pick('=', .bang_eq, .bang),
            '<' => self.lexLt(),
            '>' => self.lexGt(),
            '&' => self.lexAmp(),
            '|' => self.lexPipe(),
            '^' => self.pick('=', .caret_eq, .caret),
            '~' => .tilde, // unary bitwise-NOT; Zig has no `~=`, so it is one-way
            '-' => self.lexMinus(),
            else => .invalid,
        }
        return self.make(k, start, line)
    }

    // If the byte after the just-consumed one is `second`, consume it and
    // return `both` (the two-char token); otherwise return `one`.
    fn pick(self *mut Lexer, second u8, both Kind, one Kind) Kind {
        if self.at() == second {
            self.pos += 1
            return both
        }
        return one
    }

    // '+' is five-way and so cannot use the two-way `pick`: "++" is
    // concatenation, "+%=" is wrapping add-assign, "+%" is wrapping addition,
    // "+=" is add-assign, and a lone "+" is addition.
    fn lexPlus(self *mut Lexer) Kind {
        switch self.at() {
            '+' => {
                self.pos += 1
                return .plus_plus
            },
            '%' => {
                self.pos += 1
                return self.pick('=', .plus_percent_eq, .plus_percent)
            },
            '=' => {
                self.pos += 1
                return .plus_eq
            },
            else => return .plus,
        }
    }

    // '-' is five-way and so cannot use the two-way `pick`: "->" is the
    // return arrow, "-%=" is wrapping subtract-assign, "-%" is wrapping
    // subtraction, "-=" is subtract-assign, and a lone "-" is negation /
    // subtraction.
    fn lexMinus(self *mut Lexer) Kind {
        switch self.at() {
            '>' => {
                self.pos += 1
                return .arrow
            },
            '%' => {
                self.pos += 1
                return self.pick('=', .minus_percent_eq, .minus_percent)
            },
            '=' => {
                self.pos += 1
                return .minus_eq
            },
            else => return .minus,
        }
    }

    // '*' is five-way (like lexPlus): "**" is array repetition, "*%=" is
    // wrapping multiply-assign, "*%" is wrapping multiplication, "*=" is
    // multiply-assign, and a lone "*" is multiplication (or a pointer sigil —
    // the parser tells those apart by position). It cannot use the two-way
    // `pick`.
    fn lexStar(self *mut Lexer) Kind {
        switch self.at() {
            '*' => {
                self.pos += 1
                return .star_star
            },
            '%' => {
                self.pos += 1
                return self.pick('=', .star_percent_eq, .star_percent)
            },
            '=' => {
                self.pos += 1
                return .star_eq
            },
            else => return .star,
        }
    }

    // '=' is three-way (like lexMinus): "=>" is the switch prong arrow, "==" is
    // equality, and a lone "=" is assignment / a binding.
    fn lexEq(self *mut Lexer) Kind {
        switch self.at() {
            '>' => {
                self.pos += 1
                return .fat_arrow
            },
            '=' => {
                self.pos += 1
                return .eq_eq
            },
            else => return .equal,
        }
    }

    // '.' is three-way: "..." is an inclusive switch range, ".." a slice / range
    // bound, and a lone "." is member access or a literal prefix.
    fn lexDot(self *mut Lexer) Kind {
        if self.at() == '.' {
            self.pos += 1
            if self.at() == '.' {
                self.pos += 1
                return .ellipsis3
            }
            return .dot_dot
        }
        return .dot
    }

    // '&' and '|' are three-way (like lexMinus): "&&"/"||" are the logical
    // operators, "&="/"|=" the bitwise compound assignments, and a lone "&"/"|"
    // is bitwise-and/or (a prefix "&" is address-of, told apart by the parser).
    fn lexAmp(self *mut Lexer) Kind {
        switch self.at() {
            '&' => {
                self.pos += 1
                return .amp_amp
            },
            '=' => {
                self.pos += 1
                return .amp_eq
            },
            else => return .amp,
        }
    }

    fn lexPipe(self *mut Lexer) Kind {
        switch self.at() {
            '|' => {
                self.pos += 1
                return .pipe_pipe
            },
            '=' => {
                self.pos += 1
                return .pipe_eq
            },
            else => return .pipe,
        }
    }

    // '<' and '>' are four-way: "<<="/">>=" are the shift compound assignments,
    // "<<"/">>" the shifts, "<="/">=" the comparisons, and a lone "<"/">"
    // less/greater. The shift case consumes its second angle then reuses `pick`
    // for the trailing '='. Maximal munch means a spaced `< <` stays two
    // compares and `<< =` a shift then a store.
    fn lexLt(self *mut Lexer) Kind {
        switch self.at() {
            '<' => {
                self.pos += 1
                return self.pick('=', .lt_lt_eq, .lt_lt)
            },
            '=' => {
                self.pos += 1
                return .lt_eq
            },
            else => return .lt,
        }
    }

    fn lexGt(self *mut Lexer) Kind {
        switch self.at() {
            '>' => {
                self.pos += 1
                return self.pick('=', .gt_gt_eq, .gt_gt)
            },
            '=' => {
                self.pos += 1
                return .gt_eq
            },
            else => return .gt,
        }
    }

    fn lexIdent(self *mut Lexer, start u32, line u32) Token {
        while self.pos < self.src.len && isIdentCont(self.src[self.pos]) {
            self.pos += 1
        }
        text := self.src[start..self.pos]
        if text.len == 1 && text[0] == '_' {
            return self.make(.underscore, start, line)
        }
        const k Kind = token.keywords.get(text) orelse Kind.ident
        return self.make(k, start, line)
    }

    // Integer and float literals. Handles decimal, hexadecimal (0x…), octal
    // (0o…), and binary (0b…) integers with optional `_` digit separators, and
    // decimal float literals (`3.14`, `1.0e-10`). The value passes through to
    // the emitted Zig verbatim; the exact digit content is validated downstream.
    // A letter or digit immediately adjacent (no whitespace) after the literal
    // is a lexer error — the guard that kills the old silent-mis-lex hazard
    // (`0o755` once split as two tokens, `1_000` as `1` + `_000`).
    fn lexNumber(self *mut Lexer, start u32, line u32) Token {
        if self.at() == '0' {
            switch self.peek(1) {
                'x', 'X' => {
                    self.pos += 2
                    while self.pos < self.src.len && (isHexDigit(self.src[self.pos]) || self.src[self.pos] == '_') {
                        self.pos += 1
                    }
                    return self.lexNumEnd(start, line, .int)
                },
                'b', 'B' => {
                    self.pos += 2
                    while self.pos < self.src.len && (isBinaryDigit(self.src[self.pos]) || self.src[self.pos] == '_') {
                        self.pos += 1
                    }
                    return self.lexNumEnd(start, line, .int)
                },
                'o', 'O' => {
                    self.pos += 2
                    while self.pos < self.src.len && (isOctalDigit(self.src[self.pos]) || self.src[self.pos] == '_') {
                        self.pos += 1
                    }
                    return self.lexNumEnd(start, line, .int)
                },
                else => {},
            }
        }
        // Decimal: consume the integer part (digits and `_` separators).
        while self.pos < self.src.len && (isDigit(self.src[self.pos]) || self.src[self.pos] == '_') {
            self.pos += 1
        }
        // A `.` followed by a digit starts the fractional part of a float literal.
        // A lone `.` or `..` stays a punctuation token (member access / range).
        if self.at() == '.' && isDigit(self.peek(1)) {
            self.pos += 1 // '.'
            while self.pos < self.src.len && (isDigit(self.src[self.pos]) || self.src[self.pos] == '_') {
                self.pos += 1
            }
            if self.at() == 'e' || self.at() == 'E' {
                self.pos += 1
                if self.at() == '+' || self.at() == '-' {
                    self.pos += 1
                }
                while self.pos < self.src.len && (isDigit(self.src[self.pos]) || self.src[self.pos] == '_') {
                    self.pos += 1
                }
            }
            return self.lexNumEnd(start, line, .float)
        }
        return self.lexNumEnd(start, line, .int)
    }

    // Emit `k` or `.invalid` when an identifier-start or digit immediately
    // follows the literal (no whitespace). On error the bad suffix is consumed
    // so the error span is informative for diagnostics.
    fn lexNumEnd(self *mut Lexer, start u32, line u32, k Kind) Token {
        if self.pos < self.src.len && (isIdentStart(self.src[self.pos]) || isDigit(self.src[self.pos])) {
            while self.pos < self.src.len && (isIdentCont(self.src[self.pos]) || isDigit(self.src[self.pos])) {
                self.pos += 1
            }
            return self.make(.invalid, start, line)
        }
        return self.make(k, start, line)
    }

    // Char literals: `'c'`, `'\n'`, `'\xNN'`, `'\u{NNNNNN}'`. The lexer
    // validates termination and escape structure; the byte value passes through
    // to the emitted Zig verbatim (Zig uses the same spellings). An unterminated
    // or malformed literal emits `.invalid`.
    fn lexChar(self *mut Lexer, start u32, line u32) Token {
        self.pos += 1 // opening quote
        if self.at() == '\\' {
            self.pos += 1 // backslash
            switch self.at() {
                'x' => {
                    self.pos += 1 // 'x'
                    var i usize = 0
                    while i < 2 && self.pos < self.src.len && isHexDigit(self.src[self.pos]) {
                        self.pos += 1
                        i += 1
                        // 'u'
                        // '{'
                        // single-char escape: \n \r \t \0 \\ \' \"
                    }
                },
                'u' => {
                    self.pos += 1
                    if self.at() == '{' {
                        self.pos += 1
                        while self.pos < self.src.len && isHexDigit(self.src[self.pos]) {
                            self.pos += 1
                        }
                        if self.at() == '}' {
                            self.pos += 1
                        }
                    }
                },
                else => {
                    if self.pos < self.src.len {
                        self.pos += 1
                    }
                },
            }
        } else if self.pos < self.src.len {
            self.pos += 1 // the single byte
        }
        if self.at() == '\'' {
            self.pos += 1
            return self.make(.char, start, line)
        }
        return self.make(.invalid, start, line)
    }

    // #name builtins (#intCast, #as, …). Lexed as one token carrying the
    // leading '#'; the parser strips the sigil and lowering re-sigils to Zig's
    // '@' (Tier-0 passthrough of the intrinsic).
    fn lexBuiltin(self *mut Lexer, start u32, line u32) Token {
        self.pos += 1 // '#'
        while self.pos < self.src.len && isIdentCont(self.src[self.pos]) {
            self.pos += 1
        }
        return self.make(.builtin, start, line)
    }

    // Single-line strings with escape sequences: `\n \r \t \0 \\ \" \xNN
    // \u{NNNNNN}`. The lexer validates termination and escape structure; escape
    // *values* pass through to the emitted Zig verbatim (Zig uses the same set).
    fn lexString(self *mut Lexer, start u32, line u32) Token {
        self.pos += 1 // opening quote
        while self.pos < self.src.len {
            c := self.src[self.pos]
            if c == '\\' {
                self.pos += 1 // backslash
                if self.pos >= self.src.len {
                    break
                }
                switch self.src[self.pos] {
                    'x' => {
                        self.pos += 1 // 'x'
                        var i usize = 0
                        while i < 2 && self.pos < self.src.len && isHexDigit(self.src[self.pos]) {
                            self.pos += 1
                            i += 1
                            // 'u'
                            // '{'
                            // single-char escape or unknown (validated downstream)
                        }
                    },
                    'u' => {
                        self.pos += 1
                        if self.pos < self.src.len && self.src[self.pos] == '{' {
                            self.pos += 1
                            while self.pos < self.src.len && isHexDigit(self.src[self.pos]) {
                                self.pos += 1
                            }
                            if self.pos < self.src.len && self.src[self.pos] == '}' {
                                self.pos += 1
                            }
                        }
                    },
                    else => {
                        self.pos += 1
                    },
                }
                continue
            }
            if c == '"' {
                self.pos += 1
                return self.make(.string, start, line)
            }
            if c == '\n' {
                break
                // unterminated on this line
            }
            self.pos += 1
        }
        return self.make(.invalid, start, line)
    }

    // A Zig-style multiline / raw string line: `\\` followed by the rest of the
    // physical line, with no escape processing. The token spans `\\` through the
    // last byte before the newline (the newline itself stays trivia). Each line
    // is one token; the parser folds a run of consecutive lines into one string
    // value, so a single multiline literal is several of these tokens.
    fn lexMultilineStr(self *mut Lexer, start u32, line u32) Token {
        self.pos += 2 // the two backslashes
        while self.pos < self.src.len && self.src[self.pos] != '\n' {
            self.pos += 1
        }
        return self.make(.multiline_str, start, line)
    }

    // A `///` doc-comment line: the three slashes plus the rest of the physical
    // line (the newline stays trivia). One token per line; the parser folds a run
    // of consecutive lines into one doc block and re-emits it before the
    // declaration. Regular `//`, `////…` and `//!` never reach here — next()
    // routes them to lexLineComment (see atDocComment).
    fn lexDocComment(self *mut Lexer, start u32, line u32) Token {
        self.pos += 3 // the three slashes
        while self.pos < self.src.len && self.src[self.pos] != '\n' {
            self.pos += 1
        }
        return self.make(.doc_comment, start, line)
    }

    // A `//…` line comment in any non-doc shape — a bare `//`, the four-or-more
    // slash `////…`, or the module-head `//!`. The token spans the slashes
    // through the last byte before the newline (the newline stays trivia). A
    // trailing `\r` is trimmed from the span so re-emission on a CRLF input
    // embeds no carriage return mid-line. One token per line; the parser filters
    // these out of the descent stream into `p.comments` (the formatter's input),
    // so the grammar never sees them. The exactly-three-slash `///` is the
    // separate doc_comment token (see atDocComment), routed before this in next().
    fn lexLineComment(self *mut Lexer, start u32, line u32) Token {
        self.pos += 2 // the two leading slashes
        while self.pos < self.src.len && self.src[self.pos] != '\n' {
            self.pos += 1
        }
        var end u32 = self.pos
        if end > start && self.src[end - 1] == '\r' {
            end -= 1
        }
        return .{ .kind = .line_comment, .start = start, .end = end, .line = line }
    }

    fn make(self *Lexer, k Kind, start u32, line u32) Token {
        return .{ .kind = k, .start = start, .end = self.pos, .line = line }
    }
}

fn isDigit(c u8) bool {
    return c >= '0' && c <= '9'
}
fn isHexDigit(c u8) bool {
    return isDigit(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')
}
fn isOctalDigit(c u8) bool {
    return c >= '0' && c <= '7'
}
fn isBinaryDigit(c u8) bool {
    return c == '0' || c == '1'
}
fn isIdentStart(c u8) bool {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'
}
fn isIdentCont(c u8) bool {
    return isIdentStart(c) || isDigit(c)
}

// --- tests ---------------------------------------------------------------

fn collect(src []u8, out []mut Kind) []Kind {
    var lx = Lexer.init(src)
    var i usize = 0
    while true {
        t := lx.next()
        out[i] = t.kind
        i += 1
        if t.kind == .eof {
            break
        }
    }
    return out[0..i]
}

test "keywords and identifiers" {
    var buf [32]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_use, .ident, .kw_fn, .ident, .eof }, collect("use flibc fn main", &buf))
}

test "underscore is its own token, not an identifier" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .underscore, .ident, .eof }, collect("_ _x", &buf))
}

test "colon-equal is one token; a lone colon is its own" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .colon_equal, .colon, .eof }, collect(":= :", &buf))
}

test "line comments surface as tokens between real tokens" {
    var buf [16]Kind = undefined
    got := collect("// header\nfn main // trailing\n{ }", &buf)
    try sup.expectEqualSlices(Kind, &[_]Kind{ .line_comment, .kw_fn, .ident, .line_comment, .l_brace, .r_brace, .eof }, got)
}

test "string literal with an escape" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .string, .eof }, collect("\"hello\\n\"", &buf))
}

test "operators: one- and two-character forms" {
    var buf [32]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .plus, .minus, .star, .slash, .percent, .eq_eq, .bang_eq, .lt, .lt_eq, .gt, .gt_eq, .amp_amp, .pipe_pipe, .bang, .amp, .question, .dot_dot, .arrow, .colon_equal, .tilde, .eof }, collect("+ - * / % == != < <= > >= && || ! & ? .. -> := ~", &buf))
}

test "bitwise and shift operators, and shift vs spaced compares" {
    var buf [16]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .amp, .pipe, .caret, .lt_lt, .gt_gt, .eof }, collect("& | ^ << >>", &buf))
    // Maximal munch: `<<` is one shift token, but a spaced `< <` stays two
    // compares (and likewise `> >`), so a comparison never absorbs a shift.
    var buf2 [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .lt_lt, .lt, .lt, .eof }, collect("<< < <", &buf2))
}

test "char literal, hex int, and #builtin" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .char, .int, .builtin, .eof }, collect("'0' 0xFF #intCast", &buf))
}

test "control-flow keywords lex to their kinds" {
    var buf [16]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_if, .kw_else, .kw_while, .kw_for, .kw_in, .kw_break, .kw_continue, .kw_return, .eof }, collect("if else while for in break continue return", &buf))
}

test "error-union keywords lex to their kinds" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_try, .kw_catch, .kw_defer, .kw_errdefer, .eof }, collect("try catch defer errdefer", &buf))
}

test "type-definition keywords lex to their kinds" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_struct, .kw_enum, .kw_union, .eof }, collect("struct enum union", &buf))
}

test "the test keyword lexes to kw_test" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_test, .string, .l_brace, .r_brace, .eof }, collect("test \"adds\" {}", &buf))
}

test "error is a keyword heading a set or an origination" {
    var buf [16]Kind = undefined
    // `error` reserves its own kind so it can head an `error{…}` set or an
    // `error.Name` origination; the lexer recognises it through the keyword map.
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_error, .l_brace, .ident, .r_brace, .kw_error, .dot, .ident, .eof }, collect("error{ Bad } error.Bad", &buf))
}

test "asm is a keyword; volatile stays a contextual identifier" {
    var buf [8]Kind = undefined
    // `asm` reserves its own kind so it can head an expression; `volatile` is
    // recognised positionally by the parser, so it still lexes as a plain ident.
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_asm, .ident, .eof }, collect("asm volatile", &buf))
}

test "a tagged-union variant list lexes name-then-type" {
    var buf [16]Kind = undefined
    // `union` is its own keyword; `(enum)` reuses kw_enum inside the parens; a
    // bare variant is a lone ident, a typed variant lexes name-then-type.
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_union, .l_paren, .kw_enum, .r_paren, .l_brace, .ident, .comma, .ident, .ident, .r_brace, .eof }, collect("union(enum) { empty, single usize }", &buf))
}

test "visibility and declaration-modifier keywords lex to their kinds" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_pub, .kw_export, .kw_inline, .kw_fn, .eof }, collect("pub export inline fn", &buf))
}

test "each multiline-string line is its own token" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .kw_const, .ident, .equal, .multiline_str, .multiline_str, .eof }, collect("const usage =\n    \\\\line one\n    \\\\line two", &buf))
}

test "a multiline-string token's content is the bytes after the backslashes" {
    var lx = Lexer.init("    \\\\  hi there")
    t := lx.next()
    try sup.expectEqual(Kind.multiline_str, t.kind)
    // The lexeme keeps the leading `\\`; the parser strips it to get the content.
    try sup.expectEqualStrings("\\\\  hi there", t.lexeme(lx.src))
}

test "compound-assign operators and the three-way minus" {
    var buf [16]Kind = undefined
    // `-` is three-way: "->" arrow, "-=" minus_eq, and a lone "-" minus.
    try sup.expectEqualSlices(Kind, &[_]Kind{ .plus_eq, .minus_eq, .star_eq, .slash_eq, .percent_eq, .arrow, .minus_eq, .minus, .eof }, collect("+= -= *= /= %= -> -= -", &buf))
}

test "wrapping compound-assignment operators munch maximally" {
    var buf [16]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .plus_percent_eq, .minus_percent_eq, .star_percent_eq, .eof }, collect("+%= -%= *%=", &buf))
    // Maximal munch around the three-byte forms: `+% =` (spaced) is a wrapping
    // add then a store, and a lone `+%` stays the wrapping binop — a wrapping
    // compound assign never forms across whitespace.
    var buf2 [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .plus_percent, .equal, .minus_percent, .equal, .star_percent, .eof }, collect("+% = -% = *%", &buf2))
}

test "star family: repetition, wrapping forms, assign, and the lone star" {
    var buf [16]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .star_star, .star_percent, .star_percent_eq, .star_eq, .star, .eof }, collect("** *% *%= *= *", &buf))
    // A spaced `* *` stays two stars (deref-then-multiply territory); `**`
    // never forms across whitespace.
    var buf2 [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .star, .star, .eof }, collect("* *", &buf2))
}

test "bitwise and shift compound-assignment operators" {
    var buf [16]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .amp_eq, .pipe_eq, .caret_eq, .lt_lt_eq, .gt_gt_eq, .eof }, collect("&= |= ^= <<= >>=", &buf))
    // Maximal munch around the three-byte shift-assign: `<< =` (spaced) is a
    // shift then a store, and a lone `<<` stays a bare shift — a shift-assign
    // never forms across whitespace.
    var buf2 [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .lt_lt, .equal, .lt_lt, .eof }, collect("<< = <<", &buf2))
}

test "clear.flash lexes without an invalid token" {
    var lx = Lexer.init("use flibc\nfn sink(bytes []u8) {\n    _ = flibc.sys.write_fd(1, bytes.ptr, bytes.len)\n}")
    while true {
        t := lx.next()
        try sup.expect(t.kind != .invalid)
        if t.kind == .eof {
            break
        }
    }
}

test "/// is a doc_comment; //, ////, //! are line_comments" {
    var buf [16]Kind = undefined
    // The exactly-three-slash line is the content-bearing doc_comment; a bare
    // `//`, the four-slash `////`, and the top-level `//!` are each a
    // line_comment. All four are tokens (none are trivia).
    try sup.expectEqualSlices(Kind, &[_]Kind{ .line_comment, .line_comment, .line_comment, .doc_comment, .kw_pub, .kw_fn, .ident, .eof }, collect("// regular\n//! module\n//// not a doc\n/// a doc\npub fn f", &buf))
}

test "a doc-comment token's content is the bytes after the three slashes" {
    var lx = Lexer.init("/// hello")
    t := lx.next()
    try sup.expectEqual(Kind.doc_comment, t.kind)
    // The lexeme keeps the leading `///`; the parser strips it to get the content.
    try sup.expectEqualStrings("/// hello", t.lexeme(lx.src))
}

test "line comments surface as tokens, lexeme verbatim across shapes" {
    // Every non-doc `//…` shape is one line_comment token whose lexeme is the
    // full text from the slashes to end of line — the content is never edited.
    var lx1 = Lexer.init("// x")
    t1 := lx1.next()
    try sup.expectEqual(Kind.line_comment, t1.kind)
    try sup.expectEqualStrings("// x", t1.lexeme(lx1.src))

    var lx2 = Lexer.init("//! mod")
    t2 := lx2.next()
    try sup.expectEqual(Kind.line_comment, t2.kind)
    try sup.expectEqualStrings("//! mod", t2.lexeme(lx2.src))

    var lx3 = Lexer.init("//// rule")
    t3 := lx3.next()
    try sup.expectEqual(Kind.line_comment, t3.kind)
    try sup.expectEqualStrings("//// rule", t3.lexeme(lx3.src))
}

test "a line comment's trailing carriage return is trimmed from its span" {
    // CRLF input: the `\r` before the `\n` is not part of the comment lexeme,
    // so re-emission embeds no carriage return mid-line.
    var lx = Lexer.init("// x\r\nfn")
    t := lx.next()
    try sup.expectEqual(Kind.line_comment, t.kind)
    try sup.expectEqualStrings("// x", t.lexeme(lx.src))
}

test "binary and octal integer literals lex to .int" {
    var buf [4]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0b101", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0B110", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0o755", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0O644", &buf))
}

test "digit-separator _ in integer literals" {
    var buf [4]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("1_000_000", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0xFF_AA", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0b1010_1010", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .eof }, collect("0o7_7_7", &buf))
}

test "float literals lex to .float" {
    var buf [4]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .float, .eof }, collect("3.14", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .float, .eof }, collect("1.0e10", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .float, .eof }, collect("1.5e-3", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .float, .eof }, collect("1.5e+3", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .float, .eof }, collect("1_000.5", &buf))
}

test "float literal lexeme is verbatim" {
    var lx = Lexer.init("3.14")
    t := lx.next()
    try sup.expectEqual(Kind.float, t.kind)
    try sup.expectEqualStrings("3.14", t.lexeme(lx.src))
}

test "3.method is not a float — dot floats only when followed by a digit" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .dot, .ident, .eof }, collect("3.method", &buf))
}

test "1..2 is a range, not a float" {
    var buf [8]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .int, .dot_dot, .int, .eof }, collect("1..2", &buf))
}

test "literal-adjacency guard: adjacent ident or wrong-base digit after a number is .invalid" {
    var buf [4]Kind = undefined
    // Old silent mis-lex: `0o755` used to split as int(0) + ident(o755).
    // An ident after any number is now a lexer error.
    try sup.expectEqualSlices(Kind, &[_]Kind{ .invalid, .eof }, collect("123abc", &buf))
    // Wrong-base digits: `2` is not a binary digit; `8`/`9` not octal.
    try sup.expectEqualSlices(Kind, &[_]Kind{ .invalid, .eof }, collect("0b102", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .invalid, .eof }, collect("0o89", &buf))
}

test "char literal with hex escape \\xNN" {
    var buf [4]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .char, .eof }, collect("'\\x1b'", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .char, .eof }, collect("'\\x00'", &buf))
}

test "char literal with unicode escape \\u{N}" {
    var buf [4]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .char, .eof }, collect("'\\u{41}'", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .char, .eof }, collect("'\\u{1F600}'", &buf))
}

test "string literal with hex and unicode escapes" {
    var buf [4]Kind = undefined
    try sup.expectEqualSlices(Kind, &[_]Kind{ .string, .eof }, collect("\"\\x1b[0m\"", &buf))
    try sup.expectEqualSlices(Kind, &[_]Kind{ .string, .eof }, collect("\"\\u{263A}\"", &buf))
}