rl_lang/lexer/
tokenizer.rs

1//! [`Tokenizer`] struct and the main character-by-character scan loop.
2//!
3//! Holds the source text, the current byte cursor, and the accumulated token
4//! list. The scan loop in here dispatches each character to the appropriate
5//! sub-scanner in `types/` or handles single/double-character operators inline.
6use crate::lexer::tokentypes::{Token, TokenType};
7use crate::utils::errors::Error;
8use crate::utils::source::SourceFile;
9use crate::utils::span::Span;
10
11/// Converts raw source text into a flat list of [`Token`]s.
12///
13/// Operates character by character, grouping lexemes into tokens.
14/// The main entry point is [`Tokenizer::lex`].
15pub struct Tokenizer {
16    /// The source file (text + name) used for error reporting.
17    pub source_file: SourceFile,
18    /// The source text as a sequence of characters.
19    pub source: Vec<char>,
20    /// The accumulated token list built during lexing.
21    pub tokens: Vec<super::tokentypes::Token>,
22    /// Index of the current character being examined.
23    pub current: usize,
24    /// Index of the first character of the current token.
25    pub start: usize,
26    /// Current line number, incremented on every `\n`.
27    pub line: usize,
28}
29
30impl Tokenizer {
31    /// The main entry point: lexes a [`SourceFile`] into a [`Vec<Token>`].
32    ///
33    /// Drives [`Tokenizer::scan_tokens`] in a loop until the source is exhausted,
34    /// then appends a [`TokenType::Eof`] so the parser always has a clean terminator.
35    ///
36    /// # Errors
37    ///
38    /// Returns [`Error`] if the source contains an unrecognized character,
39    /// unterminated string, or invalid literal.
40    ///
41    /// # Examples
42    ///
43    /// ```rust
44    /// use rl_lang::{
45    ///     lexer::{
46    ///         tokenizer::Tokenizer,
47    ///         tokentypes::TokenType,
48    ///     },
49    ///     utils::source::SourceFile,
50    /// };
51    ///
52    /// let tokens = match Tokenizer::lex(SourceFile::new("source", "1 == 1".to_string())) {
53    ///     Ok(tokens) => tokens,
54    ///     Err(error) => {
55    ///         error.report_to_stderr();
56    ///         std::process::exit(1);
57    ///     },
58    /// };
59    ///
60    /// assert_eq!(tokens[0].token, TokenType::NumberLiteral(1));
61    /// assert_eq!(tokens[1].token, TokenType::Compare);
62    /// assert_eq!(tokens[2].token, TokenType::NumberLiteral(1));
63    /// assert_eq!(tokens[3].token, TokenType::Eof);
64    /// ```
65    pub fn lex(source_file: SourceFile) -> Result<Vec<Token>, Error> {
66        let chars: Vec<char> = source_file.text.chars().collect();
67        let eof_char_index = chars.len();
68
69        let mut lexer = Tokenizer {
70            source_file,
71            source: chars,
72            tokens: Vec::new(),
73            current: 0,
74            start: 0,
75            line: 1,
76        };
77
78        while !lexer.is_at_end() {
79            lexer.start = lexer.current;
80            lexer.scan_tokens()?;
81        }
82
83        lexer.tokens.push(Token::new(
84            TokenType::Eof,
85            String::new(),
86            lexer.line,
87            Span::new(eof_char_index, eof_char_index),
88        ));
89
90        #[cfg(feature = "debug")]
91        log::debug!("Recognized {} token(s)", lexer.tokens.len());
92        Ok(lexer.tokens)
93    }
94
95    /// Returns a [`Span`] covering the current token in character indices.
96    ///
97    /// Ariadne's `Source::from(&str)` indexes by character, so spans must be
98    /// char-indexed - passing byte offsets misaligns reports for multi-byte characters.
99    pub fn current_span(&self) -> Span {
100        Span::new(self.start, self.current)
101    }
102
103    /// Builds a [`Reason::Lexer`] error anchored at `span`.
104    ///
105    /// Attaches the source file so Ariadne can render the relevant source line
106    /// alongside the error message.
107    pub fn err(&self, message: impl Into<String>, span: Span) -> Error {
108        Error::at(crate::utils::errors::Reason::Lexer, message, span)
109            .with_source_file(&self.source_file)
110    }
111}
rl_lang/lexer/tokenizer.rs

rl_lang/lexer/
tokenizer.rs
