From 11304728b6607bc2a8d41a640308f3379a25b933 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Mon, 25 Jul 2022 15:29:11 +0200 Subject: Improve performance w/ a single feed loop --- src/subtokenize.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/subtokenize.rs') diff --git a/src/subtokenize.rs b/src/subtokenize.rs index 2b5d775..0c9df34 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -84,6 +84,7 @@ pub fn subtokenize(events: &mut Vec, parse_state: &ParseState) -> bool { } else { text })); + let mut size = 0; // Loop through links to pass them in order to the subtokenizer. while let Some(index) = link_index { @@ -96,7 +97,7 @@ pub fn subtokenize(events: &mut Vec, parse_state: &ParseState) -> bool { }; if link_curr.previous != None { - tokenizer.define_skip(&enter.point); + tokenizer.define_skip(&enter.point, size); } let func = match state { @@ -105,10 +106,13 @@ pub fn subtokenize(events: &mut Vec, parse_state: &ParseState) -> bool { }; state = tokenizer.push( - span::codes(&parse_state.codes, &span), + span::codes(&parse_state.codes, &span).to_vec(), func, link_curr.next == None, ); + + size += span.end_index - span.start_index; + link_index = link_curr.next; } -- cgit