rl_lang/lexer/
tokentypes.rs

1//! [`Token`] and [`TokenType`] - the complete vocabulary of the lexer.
2//!
3//! Every variant the lexer can produce is defined here. Literal-carrying
4//! variants (`NumberLiteral`, `StringLiteral`, etc.) embed their parsed value
5//! directly so downstream passes never need to re-parse raw text.
6use crate::utils::span::Span;
7
8/// Represents every token type the lexer can produce.
9///
10/// Variants are grouped into:
11/// - **Delimiters** - brackets, braces, parens
12/// - **Punctuation** - dots, colons, commas, semicolons
13/// - **Operators** - arithmetic, comparison, assignment, logical
14/// - **Literals** - carry their parsed value directly
15/// - **Keywords** - reserved words of the language
16/// - **Special** - [`TokenType::Newline`], [`TokenType::Eof`]
17#[derive(Debug, Clone, PartialEq)]
18pub enum TokenType {
19    // -- delimiters --
20    LeftParen,
21    RightParen,
22    LeftBrace,
23    RightBrace,
24    LeftBracket,
25    RightBracket,
26
27    // -- punctuation --
28    Dot,
29    DotDot,
30    Colon,
31    ColonColon,
32    Semicolon,
33    Comma,
34    Question,
35
36    // -- arithmetic --
37    Plus,
38    Minus,
39    Slash,
40    Star,
41
42    // -- compound assignment --
43    PlusEqual,
44    MinusEqual,
45    SlashEqual,
46    StarEqual,
47
48    // -- assignment & comparison --
49    Assign,
50    Compare,
51
52    // -- logical --
53    Bang,
54    BangEqual,
55    BangHash,
56    Or,
57    And,
58
59    // -- relational --
60    Less,
61    LessEqual,
62    Greater,
63    GreaterEqual,
64
65    // -- special operators --
66    Hash,
67    Arrow,
68    FatArrow,
69    Wildcard,
70
71    // -- literals --
72    /// A 64-bit signed integer e.g. `1000`
73    NumberLiteral(i64),
74    /// A single byte (u8) e.g. `1`
75    ByteLiteral(u8),
76    /// A UTF-8 string e.g. `"hello"`
77    StringLiteral(String),
78    /// A single character e.g. `'a'`
79    CharacterLiteral(char),
80    /// A 64-bit float e.g. `3.14`
81    FloatLiteral(f64),
82    /// `true` or `false`
83    BoolLiteral(bool),
84
85    // -- identifiers --
86    /// Any user-defined name e.g. `foo`, `my_var`
87    Identifier(String),
88
89    // -- keywords --
90    Null,
91    Fn,
92    In,
93    For,
94    While,
95    Return,
96    Break,
97    Continue,
98    Get,
99    From,
100    If,
101    Else,
102    Const,
103    Dec,
104    As,
105    Ok,
106    Err,
107    Match,
108    Record,
109    Tag,
110
111    // -- type keywords --
112    Int,
113    Float,
114    Bool,
115    String,
116    Byte,
117    Char,
118    Array,
119    Error,
120    Result,
121    Map,
122    Set,
123
124    // -- special --
125    /// Emitted for each newline in the source
126    Newline,
127    /// Always the last token in the stream
128    Eof,
129}
130
131/// A single token produced by the lexer.
132///
133/// Carries the token type, the original source text ([`Token::lexeme`]),
134/// the line it appeared on, and a [`Span`] for error reporting.
135pub struct Token {
136    /// The classified token type, with literal values inlined for literal variants.
137    pub token: TokenType,
138    /// The line number in the source file (1-indexed).
139    pub line: usize,
140    /// The raw source text that produced this token.
141    pub lexeme: String,
142    /// Byte offsets into the source for error reporting.
143    pub span: Span,
144}
145
146impl Token {
147    /// Creates a new [`Token`].
148    pub fn new(token: TokenType, lexeme: String, line: usize, span: Span) -> Self {
149        Token {
150            token,
151            lexeme,
152            line,
153            span,
154        }
155    }
156}
rl_lang/lexer/tokentypes.rs

rl_lang/lexer/
tokentypes.rs
