From b00fafbdcba39e7e17144b07834702629b891062 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Tue, 14 Jun 2022 18:57:28 +0200 Subject: Fix support for deep subtokenization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix a couple of forgotten line ending handling in html (text) * Fix missing initial case for html (text) not having a `<` 😬 * Add line ending handling to `text` construct --- src/subtokenize.rs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) (limited to 'src/subtokenize.rs') diff --git a/src/subtokenize.rs b/src/subtokenize.rs index 35d7672..71a84e1 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -82,12 +82,12 @@ pub fn subtokenize(events: Vec, codes: &[Code]) -> (Vec, bool) { let mut subindex = 0; // Index into subevents that starts the current slice. let mut last_start = 0; - // Counter into `ends`. + // Counter into `ends`: the linked token we are at. let mut end_index = 0; let mut index_opt: Option = Some(index); while subindex < tokenizer.events.len() { - let subevent = &tokenizer.events[subindex]; + let subevent = &mut tokenizer.events[subindex]; // Find the first event that starts after the end we’re looking // for. @@ -101,11 +101,26 @@ pub fn subtokenize(events: Vec, codes: &[Code]) -> (Vec, bool) { index_opt = events[link].next; } + // If there is a `next` link in the subevents, we have to change + // its index to account for the shifted events. + // If it points to a next event, we also change the next event’s + // reference back to *this* event. + if let Some(next) = subevent.next { + // The `index` in `events` where the current link is, + // minus 2 events (the enter and exit) for each removed + // link. + let shift = index_opt.unwrap() - (end_index * 2); + + subevent.next = Some(next + shift); + let next_ev = &mut tokenizer.events[next]; + let previous = next_ev.previous.unwrap(); + next_ev.previous = Some(previous + shift); + } + subindex += 1; } - let link = index_opt.unwrap(); - link_to_info.insert(link, (index, last_start, subindex)); + link_to_info.insert(index_opt.unwrap(), (index, last_start, subindex)); head_to_tokenizer.insert(index, tokenizer); } @@ -119,11 +134,6 @@ pub fn subtokenize(events: Vec, codes: &[Code]) -> (Vec, bool) { // from each slice and slices from events? let mut index = events.len() - 1; - // To do: this is broken, because it can inject linked events, which point - // to their links through indices, and this messes with all indices. - // We should try walking front to end instead, keep a count of the shifted - // index. - // It’s a bit complex but should work? while index > 0 { let slice_opt = link_to_info.get(&index); -- cgit