Use `edit_map` in `subtokenize`

1 files changed, 40 insertions, 67 deletions

diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index ce4f788..174ddfe 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -21,11 +21,10 @@
 //! thus the whole document needs to be parsed up to the level of definitions,
 //! before any level that can include references can be parsed.
 
-use crate::content::{flow::start as flow, string::start as string, text::start as text};
+use crate::content::{string::start as string, text::start as text};
 use crate::parser::ParseState;
 use crate::tokenizer::{ContentType, Event, EventType, State, StateFn, StateFnResult, Tokenizer};
-use crate::util::span;
-use std::collections::HashMap;
+use crate::util::{edit_map::EditMap, span};
 
 /// Create a link between two [`Event`][]s.
 ///
@@ -63,16 +62,9 @@ pub fn link_to(events: &mut [Event], pevious: usize, next: usize) {
 ///
 /// Supposed to be called repeatedly, returns `1: true` when done.
 pub fn subtokenize(mut events: Vec<Event>, parse_state: &ParseState) -> (Vec<Event>, bool) {
-    let mut index = 0;
-    // Map of first chunks to their tokenizer.
-    let mut head_to_tokenizer: HashMap<usize, Tokenizer> = HashMap::new();
-    // Map of chunks to their head and corresponding range of events.
-    let mut link_to_info: HashMap<usize, (usize, usize, usize)> = HashMap::new();
+    let mut edit_map = EditMap::new();
     let mut done = true;
-
-    if events.is_empty() {
-        return (events, true);
-    }
+    let mut index = 0;
 
     while index < events.len() {
         let event = &events[index];
@@ -83,34 +75,28 @@ pub fn subtokenize(mut events: Vec<Event>, parse_state: &ParseState) -> (Vec<Eve
 
             // No need to enter linked events again.
             if event.previous == None {
-                done = false;
                 // Index into `events` pointing to a chunk.
-                let mut index_opt: Option<usize> = Some(index);
+                let mut link_index: Option<usize> = Some(index);
                 // Subtokenizer.
                 let mut tokenizer = Tokenizer::new(event.point.clone(), event.index, parse_state);
                 // Substate.
                 let mut result: StateFnResult = (
-                    State::Fn(Box::new(if *content_type == ContentType::Flow {
-                        flow
-                    } else if *content_type == ContentType::String {
+                    State::Fn(Box::new(if *content_type == ContentType::String {
                         string
                     } else {
                         text
                     })),
                     None,
                 );
-                // Indices into `codes` of each end of chunk.
-                let mut ends: Vec<usize> = vec![];
 
-                // Loop through chunks to pass them in order to the subtokenizer.
-                while let Some(index_ptr) = index_opt {
-                    let enter = &events[index_ptr];
+                // Loop through links to pass them in order to the subtokenizer.
+                while let Some(index) = link_index {
+                    let enter = &events[index];
                     assert_eq!(enter.event_type, EventType::Enter);
                     let span = span::Span {
                         start_index: enter.index,
-                        end_index: events[index_ptr + 1].index,
+                        end_index: events[index + 1].index,
                     };
-                    ends.push(span.end_index);
 
                     if enter.previous != None {
                         tokenizer.define_skip(&enter.point, enter.index);
@@ -127,32 +113,32 @@ pub fn subtokenize(mut events: Vec<Event>, parse_state: &ParseState) -> (Vec<Eve
                         enter.next == None,
                     );
                     assert!(result.1.is_none(), "expected no remainder");
-                    index_opt = enter.next;
+                    link_index = enter.next;
                 }
 
-                // Now, loop through all subevents (and `ends`), to figure out
-                // which parts belong where.
-                // Current index.
+                // Now, loop through all subevents to figure out which parts
+                // belong where and fix deep links.
                 let mut subindex = 0;
-                // Index into subevents that starts the current slice.
-                let mut last_start = 0;
-                // Counter into `ends`: the linked token we are at.
-                let mut end_index = 0;
-                let mut index_opt: Option<usize> = Some(index);
+                let mut link_index = index;
+                let mut slices = vec![];
+                let mut slice_start = 0;
 
                 while subindex < tokenizer.events.len() {
                     let subevent = &mut tokenizer.events[subindex];
 
                     // Find the first event that starts after the end we’re looking
                     // for.
-                    if subevent.event_type == EventType::Enter && subevent.index >= ends[end_index]
+                    if subevent.event_type == EventType::Enter
+                        && subevent.index >= events[link_index + 1].index
                     {
-                        let link = index_opt.unwrap();
-                        link_to_info.insert(link, (index, last_start, subindex));
+                        slices.push((link_index, slice_start));
+                        slice_start = subindex;
+                        link_index = events[link_index].next.unwrap();
+                    }
 
-                        last_start = subindex;
-                        end_index += 1;
-                        index_opt = events[link].next;
+                    if subevent.content_type.is_some() {
+                        // Need to call `subtokenize` again.
+                        done = false;
                     }
 
                     // If there is a `next` link in the subevents, we have to change
@@ -163,8 +149,7 @@ pub fn subtokenize(mut events: Vec<Event>, parse_state: &ParseState) -> (Vec<Eve
                         // The `index` in `events` where the current link is,
                         // minus 2 events (the enter and exit) for each removed
                         // link.
-                        let shift = index_opt.unwrap() - (end_index * 2);
-
+                        let shift = link_index - (slices.len() * 2);
                         subevent.next = Some(next + shift);
                         let next_ev = &mut tokenizer.events[next];
                         let previous = next_ev.previous.unwrap();
@@ -174,36 +159,24 @@ pub fn subtokenize(mut events: Vec<Event>, parse_state: &ParseState) -> (Vec<Eve
                     subindex += 1;
                 }
 
-                link_to_info.insert(index_opt.unwrap(), (index, last_start, subindex));
-                head_to_tokenizer.insert(index, tokenizer);
-            }
-        }
+                slices.push((link_index, slice_start));
 
-        index += 1;
-    }
-
-    // Now that we fed everything into a tokenizer, and we know which parts
-    // belong where, the final task is to splice the events from each
-    // tokenizer into the current events.
-    // To do: instead of splicing, it might be possible to create a new `events`
-    // from each slice and slices from events?
-    let mut index = events.len() - 1;
-
-    while index > 0 {
-        let slice_opt = link_to_info.get(&index);
-
-        if let Some(slice) = slice_opt {
-            let (head, start, end) = *slice;
-            // If there’s a slice at this index, it must also point to a head,
-            // and that head must have a tokenizer.
-            let tokenizer = head_to_tokenizer.get(&head).unwrap();
+                // Finally, inject the subevents.
+                let mut index = slices.len();
 
-            // To do: figure out a way that moves instead of clones?
-            events.splice(index..(index + 2), tokenizer.events[start..end].to_vec());
+                while index > 0 {
+                    index -= 1;
+                    edit_map.add(
+                        slices[index].0,
+                        2,
+                        tokenizer.events.split_off(slices[index].1),
+                    );
+                }
+            }
         }
 
-        index -= 1;
+        index += 1;
     }
 
-    (events, done)
+    (edit_map.consume(&mut events), done)
 }