rl_lang/interpreter/stdlib/rl/
lex.rs

1use std::rc::Rc;
2
3use crate::ast::statements::TypeAnnotation;
4use crate::interpreter::stdlib::common::vi;
5use crate::interpreter::{
6    evaluator::Evaluator,
7    stdlib::common::{extract_string, verr, vok, vs},
8    values::Value,
9};
10use crate::lexer::tokenizer::Tokenizer;
11use crate::utils::source::SourceFile;
12
13pub fn func(_: &mut Evaluator, value: Value) -> Value {
14    let code = match extract_string(value, "lex") {
15        Ok(s) => s,
16        Err(e) => return verr!(vs!(e)),
17    };
18
19    let source = SourceFile::new("<lex>", code);
20
21    let tokens = match Tokenizer::lex(source) {
22        Ok(t) => t,
23        Err(e) => return verr!(vs!(e.message().to_string())),
24    };
25
26    let items: Vec<Value> = tokens
27        .into_iter()
28        .map(|t| {
29            let kind = format!("{:?}", t.token);
30            let kind = kind.split('(').next().unwrap_or(&kind).to_string();
31            Value::Tuple(vec![vs!(kind), vs!(t.lexeme), vi!(t.line as i64)])
32        })
33        .collect();
34
35    let items_type = TypeAnnotation::Tuple(Rc::new(vec![
36        TypeAnnotation::String,
37        TypeAnnotation::String,
38        TypeAnnotation::Int,
39    ]));
40
41    vok!(Value::Values { items_type, items })
42}
rl_lang/interpreter/stdlib/rl/lex.rs

rl_lang/interpreter/stdlib/rl/
lex.rs
