From 021d5f989ae41ae39a9b937b498141d9dc70d894 Mon Sep 17 00:00:00 2001
From: Titus Wormer <tituswormer@gmail.com>
Date: Thu, 9 Jun 2022 15:01:46 +0200
Subject: Add basic subtokenization, string content in fenced code

---
 src/tokenizer.rs | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

(limited to 'src/tokenizer.rs')

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index faee8d9..35e768e 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -177,16 +177,12 @@ pub struct Tokenizer {
 
 impl Tokenizer {
     /// Create a new tokenizer.
-    pub fn new() -> Tokenizer {
+    pub fn new(point: Point, index: usize) -> Tokenizer {
         Tokenizer {
             current: Code::None,
-            index: 0,
+            index,
             consumed: true,
-            point: Point {
-                line: 1,
-                column: 1,
-                offset: 0,
-            },
+            point,
             stack: vec![],
             events: vec![],
         }
@@ -499,6 +495,11 @@ impl Tokenizer {
             }
         }
 
+        match state {
+            State::Ok => {}
+            _ => unreachable!("expected final state to be `State::Ok`"),
+        }
+
         check_statefn_result((state, None))
     }
 }
-- 
cgit