rl_lang/lexer/types/
string_literal.rs

1//! Double-quoted string literal scanner.
2//!
3//! Consumes everything between `"…"`, handling escape sequences, and emits
4//! [`TokenType::StringLiteral`].
5use crate::lexer::{tokenizer::Tokenizer, tokentypes::TokenType};
6use crate::utils::errors::Error;
7
8impl Tokenizer {
9    /// Scans a double-quoted string literal and emits [`TokenType::StringLiteral`].
10    ///
11    /// Supports multi-line strings and the following escape sequences:
12    ///
13    /// | Sequence | Meaning        |
14    /// |----------|----------------|
15    /// | `\n`     | newline        |
16    /// | `\t`     | tab            |
17    /// | `\r`     | carriage return|
18    /// | `\0`     | null           |
19    /// | `\\`     | backslash      |
20    /// | `\"`     | double quote   |
21    /// | `\'`     | single quote   |
22    ///
23    /// # Errors
24    ///
25    /// - `unterminated string` -> if EOF is reached before the closing `"`
26    /// - `unknown escape sequence` -> if `\` is followed by an unrecognized character
27    pub fn string_literal(&mut self) -> Result<(), Error> {
28        // construct new string
29        let mut value = String::new();
30
31        // while not the end of string which is determinated by "
32        while !self.is_at_end() && self.peek() != '"' {
33            // cache next character in ch
34            let ch = self.peek();
35            // if it is new line (e.g. pressed enter)
36            // increase line count and push escaped sequence into string
37            // then advance
38            if ch == '\n' {
39                self.line += 1;
40                value.push(ch);
41                self.advance();
42                continue;
43            }
44
45            // is it escape sequence?
46            if ch == '\\' {
47                // consume the first \
48                self.advance();
49                // safety check for end of file
50                if self.is_at_end() {
51                    return Err(self.err("unterminated string", self.current_span()));
52                }
53
54                // cache the next escaped character
55                let escaped_ch = self.peek();
56                // is it recognized escaped sequence?
57                let resolved_escape = match escaped_ch {
58                    'n' => '\n',
59                    't' => '\t',
60                    'r' => '\r',
61                    '0' => '\0',
62                    '\\' => '\\',
63                    '"' => '"',
64                    '\'' => '\'',
65                    other => {
66                        return Err(self.err(
67                            format!("unknown escape sequence '\\{}'", other),
68                            self.current_span(),
69                        ));
70                    }
71                };
72
73                // add the escaped sequence into value then advance
74                value.push(resolved_escape);
75                self.advance();
76                continue;
77            }
78
79            // if not escape sequence nor " then add to value and advance
80            value.push(ch);
81            self.advance();
82        }
83
84        // are we at end of file or there is "?
85        if self.is_at_end() {
86            return Err(self.err("unterminated string", self.current_span()));
87        }
88        // consume the "
89        self.advance();
90
91        // add the constructed string value
92        self.add_token(TokenType::StringLiteral(value));
93        Ok(())
94    }
95}
rl_lang/lexer/types/string_literal.rs

rl_lang/lexer/types/
string_literal.rs
