From 91dc4b1f461fa76df7163f68b2a6db361a71beea Mon Sep 17 00:00:00 2001
From: Titus Wormer <tituswormer@gmail.com>
Date: Mon, 25 Jul 2022 16:48:27 +0200
Subject: Refactor to store less data

---
 src/tokenizer.rs | 90 +++++++++++++++++++++++++++-----------------------------
 1 file changed, 43 insertions(+), 47 deletions(-)

(limited to 'src')

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index d3a40d3..2c6726b 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -173,7 +173,7 @@ struct InternalState {
 #[allow(clippy::struct_excessive_bools)]
 pub struct Tokenizer<'a> {
     /// Jump between line endings.
-    column_start: Vec<(usize, usize, usize)>,
+    column_start: Vec<usize>,
     // First line.
     line_start: usize,
     /// Track whether a character is expected to be consumed, and whether it’s
@@ -292,16 +292,12 @@ impl<'a> Tokenizer<'a> {
 
     /// Define a jump between two places.
     pub fn define_skip(&mut self, point: &Point) {
-        define_skip_impl(self, point.line, (point.column, point.offset, point.index));
+        define_skip_impl(self, point.line, point.index);
     }
 
     /// Define the current place as a jump between two places.
     pub fn define_skip_current(&mut self) {
-        define_skip_impl(
-            self,
-            self.point.line,
-            (self.point.column, self.point.offset, self.point.index),
-        );
+        define_skip_impl(self, self.point.line, self.point.index);
     }
 
     /// Increment the current positional info if we’re right after a line
@@ -310,11 +306,7 @@ impl<'a> Tokenizer<'a> {
         let at = self.point.line - self.line_start;
 
         if self.point.column == 1 && at != self.column_start.len() {
-            let (column, offset, index) = &self.column_start[at];
-            self.point.column = *column;
-            self.point.offset = *offset;
-            self.point.index = *index;
-            self.index = *index;
+            self.move_to(self.column_start[at]);
         }
     }
 
@@ -329,42 +321,46 @@ impl<'a> Tokenizer<'a> {
         log::debug!("consume: `{:?}` ({:?})", code, self.point);
         assert!(!self.consumed, "expected code to not have been consumed: this might be because `x(code)` instead of `x` was returned");
 
-        self.point.index += 1;
-        self.index += 1;
+        self.move_to(self.index + 1);
 
-        match code {
-            Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
-                self.point.line += 1;
-                self.point.column = 1;
-                self.point.offset += if code == Code::CarriageReturnLineFeed {
-                    2
-                } else {
-                    1
-                };
+        self.previous = code;
+        // Mark as consumed.
+        self.consumed = true;
+    }
 
-                if self.point.line - self.line_start + 1 > self.column_start.len() {
-                    self.column_start.push((
-                        self.point.column,
-                        self.point.offset,
-                        self.point.index,
-                    ));
+    /// To do.
+    pub fn move_to(&mut self, to: usize) {
+        while self.index < to {
+            let code = &self.parse_state.codes[self.index];
+            self.point.index += 1;
+            self.index += 1;
+
+            match code {
+                Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
+                    self.point.line += 1;
+                    self.point.column = 1;
+                    self.point.offset += if *code == Code::CarriageReturnLineFeed {
+                        2
+                    } else {
+                        1
+                    };
+
+                    if self.point.line - self.line_start + 1 > self.column_start.len() {
+                        self.column_start.push(self.point.index);
+                    }
+
+                    self.account_for_potential_skip();
+                    log::debug!("position: after eol: `{:?}`", self.point);
+                }
+                Code::VirtualSpace => {
+                    // Empty.
+                }
+                _ => {
+                    self.point.column += 1;
+                    self.point.offset += 1;
                 }
-
-                self.account_for_potential_skip();
-                log::debug!("position: after eol: `{:?}`", self.point);
-            }
-            Code::VirtualSpace => {
-                // Empty.
-            }
-            _ => {
-                self.point.column += 1;
-                self.point.offset += 1;
             }
         }
-
-        self.previous = code;
-        // Mark as consumed.
-        self.consumed = true;
     }
 
     /// Mark the start of a semantic label.
@@ -766,14 +762,14 @@ fn flush_impl(
 ///
 /// This defines how much columns, offsets, and the `index` are increased when
 /// consuming a line ending.
-fn define_skip_impl(tokenizer: &mut Tokenizer, line: usize, info: (usize, usize, usize)) {
-    log::debug!("position: define skip: {:?} -> ({:?})", line, info);
+fn define_skip_impl(tokenizer: &mut Tokenizer, line: usize, index: usize) {
+    log::debug!("position: define skip: {:?} -> ({:?})", line, index);
     let at = line - tokenizer.line_start;
 
     if at == tokenizer.column_start.len() {
-        tokenizer.column_start.push(info);
+        tokenizer.column_start.push(index);
     } else {
-        tokenizer.column_start[at] = info;
+        tokenizer.column_start[at] = index;
     }
 
     tokenizer.account_for_potential_skip();
-- 
cgit