Add basic subtokenization, string content in fenced code

1 files changed, 8 insertions, 7 deletions

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index faee8d9..35e768e 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -177,16 +177,12 @@ pub struct Tokenizer {
 
 impl Tokenizer {
     /// Create a new tokenizer.
-    pub fn new() -> Tokenizer {
+    pub fn new(point: Point, index: usize) -> Tokenizer {
         Tokenizer {
             current: Code::None,
-            index: 0,
+            index,
             consumed: true,
-            point: Point {
-                line: 1,
-                column: 1,
-                offset: 0,
-            },
+            point,
             stack: vec![],
             events: vec![],
         }
@@ -499,6 +495,11 @@ impl Tokenizer {
             }
         }
 
+        match state {
+            State::Ok => {}
+            _ => unreachable!("expected final state to be `State::Ok`"),
+        }
+
         check_statefn_result((state, None))
     }
 }