1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
use crate::errors::{Error, ErrorKind};
#[derive(Debug)]
pub enum TokenKind {
// Single-character tokens.
LeftParen,
RightParen,
LeftBrace,
RightBrace,
Comma,
Dot,
Minus,
Plus,
Semicolon,
Slash,
Star,
// One or two character tokens.
Bang,
BangEqual,
Equal,
EqualEqual,
Greater,
GreaterEqual,
Less,
LessEqual,
// Literals.
Identifier,
String,
Number,
// Keywords.
And,
Class,
Else,
False,
Fun,
For,
If,
Nil,
Or,
Print,
Return,
Super,
This,
True,
Var,
While,
// Special things
Eof,
}
#[derive(Debug)]
pub struct Token<'a> {
kind: TokenKind,
lexeme: &'a str,
// literal: Object, // TODO(tazjin): Uhh?
line: usize,
}
struct Scanner<'a> {
source: &'a str,
tokens: Vec<Token<'a>>,
errors: Vec<Error>,
start: usize, // offset of first character in current lexeme
current: usize, // current offset into source
line: usize, // current line in source
}
impl<'a> Scanner<'a> {
fn is_at_end(&self) -> bool {
return self.current >= self.source.len();
}
fn advance(&mut self) -> char {
self.current += 1;
// TODO(tazjin): Due to utf8-safety, this is a bit annoying.
// Since string iteration is not the point here I'm just
// leaving this as is for now.
self.source.chars().nth(self.current - 1).unwrap()
}
fn add_token(&mut self, kind: TokenKind) {
let lexeme = &self.source[self.start..self.current];
self.tokens.push(Token {
kind,
lexeme,
line: self.line,
})
}
fn scan_token(&mut self) {
match self.advance() {
'(' => self.add_token(TokenKind::LeftParen),
')' => self.add_token(TokenKind::RightParen),
'{' => self.add_token(TokenKind::LeftBrace),
'}' => self.add_token(TokenKind::RightBrace),
',' => self.add_token(TokenKind::Comma),
'.' => self.add_token(TokenKind::Dot),
'-' => self.add_token(TokenKind::Minus),
'+' => self.add_token(TokenKind::Plus),
';' => self.add_token(TokenKind::Semicolon),
'*' => self.add_token(TokenKind::Star),
unexpected => self.errors.push(Error {
line: self.line,
kind: ErrorKind::UnexpectedChar(unexpected),
}),
};
}
fn scan_tokens(mut self) -> Vec<Token<'a>> {
while !self.is_at_end() {
self.start = self.current;
self.scan_token();
}
return self.tokens;
}
}
|
