rl_lang/lexer/utils/
helper.rs

1//! Low-level character cursor primitives used across all sub-scanners.
2//!
3//! Provides `is_at_end`, `peek`, `advance`, and similar helpers that operate
4//! directly on the [`Tokenizer`]'s byte position.
5use super::super::tokenizer::Tokenizer;
6use super::super::tokentypes::{Token, TokenType};
7
8impl Tokenizer {
9    /// Returns `true` if all characters have been consumed.
10    pub fn is_at_end(&self) -> bool {
11        self.current >= self.source.len()
12    }
13
14    /// Returns the current character without consuming it.
15    ///
16    /// Returns `'\0'` if at end of source.
17    pub fn peek(&mut self) -> char {
18        if self.is_at_end() {
19            '\0'
20        } else {
21            self.source[self.current]
22        }
23    }
24
25    /// Returns the character after the current one without consuming it.
26    ///
27    /// Returns `'\0'` if at end of source.
28    pub fn peek_next(&mut self) -> char {
29        if self.current + 1 >= self.source.len() {
30            '\0'
31        } else {
32            self.source[self.current + 1]
33        }
34    }
35
36    /// Consumes and returns the current character, advancing the cursor.
37    ///
38    /// Returns `'\0'` if at end of source.
39    pub fn advance(&mut self) -> char {
40        if self.is_at_end() {
41            return '\0';
42        }
43        let character = self.source[self.current];
44        self.current += 1;
45        character
46    }
47
48    /// Constructs a [`Token`] from `start..current` and pushes it onto the token list.
49    pub fn add_token(&mut self, tokentype: TokenType) {
50        let lexeme = self.source[self.start..self.current].iter().collect();
51        let span = self.current_span();
52        self.tokens
53            .push(Token::new(tokentype, lexeme, self.line, span));
54    }
55}
rl_lang/lexer/utils/helper.rs

rl_lang/lexer/utils/
helper.rs
