about summary refs log tree commit diff
diff options
context:
space:
mode:
authorVincent Ambo <mail@tazj.in>2020-11-27T16·55+0100
committertazjin <mail@tazj.in>2020-11-27T17·29+0000
commit6363efbebf2c9f567fb5f007b45aab619c4f7e25 (patch)
treea332a8f9f2cd9b2b0f91dbe4239c50931f381c9b
parentb595553f6311a1b278d937d0aafd070767ed363b (diff)
refactor(tazjin/rlox): Use &[char] instead of &str in scanner r/1925
This makes it easier to work with the Unicode issue. The original
string representation can be discarded.

Change-Id: I740be4cb9654679ea7950f3899c5c709b1e7a739
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2160
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI
-rw-r--r--users/tazjin/rlox/src/scanner.rs10
1 files changed, 3 insertions, 7 deletions
diff --git a/users/tazjin/rlox/src/scanner.rs b/users/tazjin/rlox/src/scanner.rs
index c1809010547a..78a019592ca4 100644
--- a/users/tazjin/rlox/src/scanner.rs
+++ b/users/tazjin/rlox/src/scanner.rs
@@ -55,13 +55,13 @@ pub enum TokenKind {
 #[derive(Debug)]
 pub struct Token<'a> {
     kind: TokenKind,
-    lexeme: &'a str,
+    lexeme: &'a [char],
     // literal: Object, // TODO(tazjin): Uhh?
     line: usize,
 }
 
 struct Scanner<'a> {
-    source: &'a str,
+    source: &'a [char],
     tokens: Vec<Token<'a>>,
     errors: Vec<Error>,
     start: usize,   // offset of first character in current lexeme
@@ -76,11 +76,7 @@ impl<'a> Scanner<'a> {
 
     fn advance(&mut self) -> char {
         self.current += 1;
-
-        // TODO(tazjin): Due to utf8-safety, this is a bit annoying.
-        // Since string iteration is not the point here I'm just
-        // leaving this as is for now.
-        self.source.chars().nth(self.current - 1).unwrap()
+        self.source[self.current-1]
     }
 
     fn add_token(&mut self, kind: TokenKind) {