rl_lang/lexer/
scanner.rs

1//! Top-level scan driver.
2//!
3//! Exposes the single entry point that the pipeline calls to turn a source
4//! string into a token stream. Delegates all real work to [`Tokenizer`].
5use crate::lexer::{tokenizer::Tokenizer, tokentypes::TokenType};
6use crate::utils::errors::Error;
7
8impl Tokenizer {
9    /// Scans the source file and return token stream
10    ///
11    /// Iterates over every character in the source file, identifies lexemes,
12    /// and returns a flat list of [`Token`]s terminated by [`TokenType::EOF`].
13    ///
14    /// For multi-character tokens (e.g. `==`, `!=`, `<=`) the scanner peeks
15    /// ahead one character to decide which token to emit.
16    ///
17    /// # Tokens
18    ///
19    /// | category       | examples                                      |
20    /// |----------------|-----------------------------------------------|
21    /// | literals       | `1`, `3.14`, `'a'`, `"hello"`                 |
22    /// | identifiers    | `foo`, `my_var`, `_private`                   |
23    /// | arithmetic     | `+`, `-`, `*`, `/`                            |
24    /// | comparison     | `==`, `!=`, `<`, `>`, `<=`, `>=`              |
25    /// | assignment     | `=`, `+=`, `-=`, `*=`, `/=`                   |
26    /// | delimiters     | `(`, `)`, `{`, `}`, `[`, `]`                  |
27    /// | punctuation    | `,`, `;`, `.`, `..`, `:`, `::`                |
28    /// | special        | `#`, `!#`, `->`, `//` (comment)               |
29    /// | whitespace     | newlines (emitted), spaces/tabs (skipped)     |
30    ///
31    ///
32    /// # Errors
33    ///
34    /// - unterminated character literal
35    /// - unknown escape sequence
36    /// - unterminated string
37    /// - unexpected character
38    pub fn scan_tokens(&mut self) -> Result<(), Error> {
39        let current_character = self.advance();
40        match current_character {
41            // single character
42            '{' => self.add_token(TokenType::LeftBrace),
43            '}' => self.add_token(TokenType::RightBrace),
44            '[' => self.add_token(TokenType::LeftBracket),
45            ']' => self.add_token(TokenType::RightBracket),
46            '(' => self.add_token(TokenType::LeftParen),
47            ')' => self.add_token(TokenType::RightParen),
48            '#' => self.add_token(TokenType::Hash),
49            ',' => self.add_token(TokenType::Comma),
50            ';' => self.add_token(TokenType::Semicolon),
51            '?' => self.add_token(TokenType::Question),
52
53            // multi character
54            ':' => {
55                if self.peek() == ':' {
56                    self.advance();
57                    self.add_token(TokenType::ColonColon);
58                } else {
59                    self.add_token(TokenType::Colon);
60                }
61            }
62
63            '.' => {
64                if self.peek() == '.' {
65                    self.advance();
66                    self.add_token(TokenType::DotDot);
67                } else {
68                    self.add_token(TokenType::Dot);
69                }
70            }
71
72            '/' => {
73                if self.peek() == '/' {
74                    while self.peek() != '\n' {
75                        self.advance();
76                    }
77                } else if self.peek() == '=' {
78                    self.advance();
79                    self.add_token(TokenType::SlashEqual);
80                } else {
81                    self.add_token(TokenType::Slash);
82                }
83            }
84
85            '=' => {
86                if self.peek() == '=' {
87                    self.advance();
88                    self.add_token(TokenType::Compare);
89                } else if self.peek() == '>' {
90                    self.advance();
91                    self.add_token(TokenType::FatArrow);
92                } else {
93                    self.add_token(TokenType::Assign);
94                }
95            }
96
97            '!' => {
98                if self.peek() == '=' {
99                    self.advance();
100                    self.add_token(TokenType::BangEqual);
101                } else if self.peek() == '#' {
102                    self.advance();
103                    self.add_token(TokenType::BangHash);
104                } else {
105                    self.add_token(TokenType::Bang);
106                }
107            }
108
109            '>' => {
110                if self.peek() == '=' {
111                    self.advance();
112                    self.add_token(TokenType::GreaterEqual);
113                } else {
114                    self.add_token(TokenType::Greater);
115                }
116            }
117
118            '<' => {
119                if self.peek() == '=' {
120                    self.advance();
121                    self.add_token(TokenType::LessEqual);
122                } else {
123                    self.add_token(TokenType::Less);
124                }
125            }
126
127            '+' => {
128                if self.peek() == '=' {
129                    self.advance();
130                    self.add_token(TokenType::PlusEqual);
131                } else {
132                    self.add_token(TokenType::Plus);
133                }
134            }
135
136            '-' => {
137                if self.peek() == '=' {
138                    self.advance();
139                    self.add_token(TokenType::MinusEqual);
140                } else if self.peek() == '>' {
141                    self.advance();
142                    self.add_token(TokenType::Arrow);
143                } else {
144                    self.add_token(TokenType::Minus);
145                }
146            }
147
148            '*' => {
149                if self.peek() == '=' {
150                    self.advance();
151                    self.add_token(TokenType::StarEqual);
152                } else {
153                    self.add_token(TokenType::Star);
154                }
155            }
156
157            // whitespaces
158            ' ' | '\t' | '\r' => {}
159
160            '\n' => {
161                self.line += 1;
162                self.add_token(TokenType::Newline)
163            }
164
165            // literals
166            '\'' => self.character_literal()?,
167            '"' => self.string_literal()?,
168
169            '0'..='9' => self.number_literal(),
170
171            '_' | 'a'..='z' | 'A'..='Z' => self.identifier(),
172
173            other => {
174                return Err(self.err(
175                    format!("unexpected character `{}`", other),
176                    self.current_span(),
177                ));
178            }
179        }
180        Ok(())
181    }
182}
rl_lang/lexer/scanner.rs

rl_lang/lexer/
scanner.rs
