diff options
author | Titus Wormer <tituswormer@gmail.com> | 2022-06-09 15:01:46 +0200 |
---|---|---|
committer | Titus Wormer <tituswormer@gmail.com> | 2022-06-09 15:01:46 +0200 |
commit | 021d5f989ae41ae39a9b937b498141d9dc70d894 (patch) | |
tree | 8009a01d69cbd4f8200ffd34fc4031265b67406e /src/content/flow.rs | |
parent | 344c3db875056d4aec509f24fb2dbeaf7e2a14b6 (diff) | |
download | markdown-rs-021d5f989ae41ae39a9b937b498141d9dc70d894.tar.gz markdown-rs-021d5f989ae41ae39a9b937b498141d9dc70d894.tar.bz2 markdown-rs-021d5f989ae41ae39a9b937b498141d9dc70d894.zip |
Add basic subtokenization, string content in fenced code
Diffstat (limited to 'src/content/flow.rs')
-rw-r--r-- | src/content/flow.rs | 23 |
1 files changed, 6 insertions, 17 deletions
diff --git a/src/content/flow.rs b/src/content/flow.rs index 693ffb5..6f94424 100644 --- a/src/content/flow.rs +++ b/src/content/flow.rs @@ -26,28 +26,17 @@ use crate::construct::{ html_flow::start as html_flow, partial_whitespace::start as whitespace, thematic_break::start as thematic_break, }; -use crate::tokenizer::{Code, Event, State, StateFnResult, TokenType, Tokenizer}; +use crate::subtokenize::subtokenize; +use crate::tokenizer::{Code, Event, Point, State, StateFnResult, TokenType, Tokenizer}; use crate::util::get_span; /// Turn `codes` as the flow content type into events. // To do: remove this `allow` when all the content types are glued together. #[allow(dead_code)] -pub fn flow(codes: &[Code]) -> Vec<Event> { - let mut tokenizer = Tokenizer::new(); - let (state, remainder) = tokenizer.feed(codes, Box::new(start), true); - - if let Some(ref x) = remainder { - if !x.is_empty() { - unreachable!("expected no final remainder {:?}", x); - } - } - - match state { - State::Ok => {} - _ => unreachable!("expected final state to be `State::Ok`"), - } - - tokenizer.events +pub fn flow(codes: &[Code], point: Point, index: usize) -> Vec<Event> { + let mut tokenizer = Tokenizer::new(point, index); + tokenizer.feed(codes, Box::new(start), true); + subtokenize(tokenizer.events, codes) } /// Before flow. |