refactor(tazjin/rlox): Use &[char] instead of &str in scanner r/1925

This makes it easier to work with the Unicode issue. The original
string representation can be discarded.

Change-Id: I740be4cb9654679ea7950f3899c5c709b1e7a739
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2160
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI

1 files changed, 3 insertions, 7 deletions

diff --git a/users/tazjin/rlox/src/scanner.rs b/users/tazjin/rlox/src/scanner.rs
index c1809010547a..78a019592ca4 100644
--- a/users/tazjin/rlox/src/scanner.rs
+++ b/users/tazjin/rlox/src/scanner.rs
@@ -55,13 +55,13 @@ pub enum TokenKind {
 #[derive(Debug)]
 pub struct Token<'a> {
     kind: TokenKind,
-    lexeme: &'a str,
+    lexeme: &'a [char],
     // literal: Object, // TODO(tazjin): Uhh?
     line: usize,
 }
 
 struct Scanner<'a> {
-    source: &'a str,
+    source: &'a [char],
     tokens: Vec<Token<'a>>,
     errors: Vec<Error>,
     start: usize,   // offset of first character in current lexeme
@@ -76,11 +76,7 @@ impl<'a> Scanner<'a> {
 
     fn advance(&mut self) -> char {
         self.current += 1;
-
-        // TODO(tazjin): Due to utf8-safety, this is a bit annoying.
-        // Since string iteration is not the point here I'm just
-        // leaving this as is for now.
-        self.source.chars().nth(self.current - 1).unwrap()
+        self.source[self.current-1]
     }
 
     fn add_token(&mut self, kind: TokenKind) {