rl_lang/lexer/mod.rs
1//! The lexer - converts raw source text into a flat [`Vec<Token>`].
2//!
3//! # Pipeline position
4//! ```text
5//! source -> Lexer -> [Token] -> Parser -> [Statement] -> Checker -> Evaluator
6//! ```
7//!
8//! # Module layout
9//! - [`tokenizer`] - the [`Tokenizer`] struct and its main scanning loop
10//! - [`tokentypes`] - the [`Token`] and [`TokenType`] definitions
11//! - `scanner` - top-level scan driver called by the pipeline
12//! - `types` - sub-scanners for each literal kind (string, char, number, identifier)
13//! - `utils` - shared cursor helpers used across the sub-scanners
14mod scanner;
15pub mod tokenizer;
16pub mod tokentypes;
17mod types;
18mod utils;