Refactor to simplify tokenizer

1 files changed, 3 insertions, 6 deletions

diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index a1ec90a..8aa4df1 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -100,16 +100,13 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
                         _ => unreachable!("cannot be ok/nok"),
                     };
 
-                    state = tokenizer.push(
-                        enter.point.index,
-                        events[index + 1].point.index,
-                        func,
-                        link_curr.next == None,
-                    );
+                    state = tokenizer.push(enter.point.index, events[index + 1].point.index, func);
 
                     link_index = link_curr.next;
                 }
 
+                tokenizer.flush(state, true);
+
                 // Now, loop through all subevents to figure out which parts
                 // belong where and fix deep links.
                 let mut subindex = 0;