From 297784cb925b1196d89479fa24c898703ae598d6 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Tue, 26 Jul 2022 12:39:38 +0200 Subject: Refactor to simplify tokenizer --- src/content/document.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src/content') diff --git a/src/content/document.rs b/src/content/document.rs index 97d1299..0c3cef7 100644 --- a/src/content/document.rs +++ b/src/content/document.rs @@ -78,7 +78,8 @@ struct DocumentInfo { pub fn document(parse_state: &mut ParseState, point: Point) -> Vec { let mut tokenizer = Tokenizer::new(point, parse_state); - tokenizer.push(0, parse_state.codes.len(), Box::new(start), true); + let state = tokenizer.push(0, parse_state.codes.len(), Box::new(start)); + tokenizer.flush(state, true); let mut index = 0; let mut definitions = vec![]; @@ -400,8 +401,7 @@ fn exit_containers( let mut current_events = tokenizer.events.split_off(info.index); let next = info.next; info.next = Box::new(flow); // This is weird but Rust needs a function there. - let result = tokenizer.flush(next); - assert!(matches!(result, State::Ok)); + tokenizer.flush(State::Fn(next), false); if *phase == Phase::Prefix { info.index = tokenizer.events.len(); -- cgit