Add basic subtokenization, string content in fenced code

2 files changed, 10 insertions, 33 deletions

diff --git a/src/content/flow.rs b/src/content/flow.rs
index 693ffb5..6f94424 100644
--- a/src/content/flow.rs
+++ b/src/content/flow.rs
@@ -26,28 +26,17 @@ use crate::construct::{
     html_flow::start as html_flow, partial_whitespace::start as whitespace,
     thematic_break::start as thematic_break,
 };
-use crate::tokenizer::{Code, Event, State, StateFnResult, TokenType, Tokenizer};
+use crate::subtokenize::subtokenize;
+use crate::tokenizer::{Code, Event, Point, State, StateFnResult, TokenType, Tokenizer};
 use crate::util::get_span;
 
 /// Turn `codes` as the flow content type into events.
 // To do: remove this `allow` when all the content types are glued together.
 #[allow(dead_code)]
-pub fn flow(codes: &[Code]) -> Vec<Event> {
-    let mut tokenizer = Tokenizer::new();
-    let (state, remainder) = tokenizer.feed(codes, Box::new(start), true);
-
-    if let Some(ref x) = remainder {
-        if !x.is_empty() {
-            unreachable!("expected no final remainder {:?}", x);
-        }
-    }
-
-    match state {
-        State::Ok => {}
-        _ => unreachable!("expected final state to be `State::Ok`"),
-    }
-
-    tokenizer.events
+pub fn flow(codes: &[Code], point: Point, index: usize) -> Vec<Event> {
+    let mut tokenizer = Tokenizer::new(point, index);
+    tokenizer.feed(codes, Box::new(start), true);
+    subtokenize(tokenizer.events, codes)
 }
 
 /// Before flow.
diff --git a/src/content/string.rs b/src/content/string.rs
index 1239a36..64f544b 100644
--- a/src/content/string.rs
+++ b/src/content/string.rs
@@ -13,26 +13,14 @@
 use crate::construct::{
     character_escape::start as character_escape, character_reference::start as character_reference,
 };
-use crate::tokenizer::{Code, Event, State, StateFnResult, TokenType, Tokenizer};
+use crate::tokenizer::{Code, Event, Point, State, StateFnResult, TokenType, Tokenizer};
 
 /// Turn `codes` as the string content type into events.
 // To do: remove this `allow` when all the content types are glued together.
 #[allow(dead_code)]
-pub fn string(codes: &[Code]) -> Vec<Event> {
-    let mut tokenizer = Tokenizer::new();
-    let (state, remainder) = tokenizer.feed(codes, Box::new(before), true);
-
-    if let Some(ref x) = remainder {
-        if !x.is_empty() {
-            unreachable!("expected no final remainder {:?}", x);
-        }
-    }
-
-    match state {
-        State::Ok => {}
-        _ => unreachable!("expected final state to be `State::Ok`"),
-    }
-
+pub fn string(codes: &[Code], point: Point, index: usize) -> Vec<Event> {
+    let mut tokenizer = Tokenizer::new(point, index);
+    tokenizer.feed(codes, Box::new(before), true);
     tokenizer.events
 }