Refactor to move `index` field to `point`

11 files changed, 49 insertions, 102 deletions

diff --git a/src/construct/attention.rs b/src/construct/attention.rs
index ac6aecf..38164a8 100644
--- a/src/construct/attention.rs
+++ b/src/construct/attention.rs
@@ -153,12 +153,8 @@ struct Sequence {
     event_index: usize,
     /// The (shifted) point where this sequence starts.
     start_point: Point,
-    /// The (shifted) index where this sequence starts.
-    start_index: usize,
     /// The (shifted) point where this sequence end.
     end_point: Point,
-    /// The (shifted) index where this sequence end.
-    end_index: usize,
     /// The number of markers we can still use.
     size: usize,
     /// Whether this sequence can open attention.
@@ -221,14 +217,14 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
             if enter.token_type == Token::AttentionSequence {
                 let end = start + 1;
                 let exit = &tokenizer.events[end];
-                let marker = MarkerKind::from_code(codes[enter.index]);
-                let before = classify_character(if enter.index > 0 {
-                    codes[enter.index - 1]
+                let marker = MarkerKind::from_code(codes[enter.point.index]);
+                let before = classify_character(if enter.point.index > 0 {
+                    codes[enter.point.index - 1]
                 } else {
                     Code::None
                 });
-                let after = classify_character(if exit.index < codes.len() {
-                    codes[exit.index]
+                let after = classify_character(if exit.point.index < codes.len() {
+                    codes[exit.point.index]
                 } else {
                     Code::None
                 });
@@ -245,10 +241,8 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                     event_index: start,
                     balance,
                     start_point: enter.point.clone(),
-                    start_index: enter.index,
                     end_point: exit.point.clone(),
-                    end_index: exit.index,
-                    size: exit.index - enter.index,
+                    size: exit.point.index - enter.point.index,
                     open: if marker == MarkerKind::Asterisk {
                         open
                     } else {
@@ -332,18 +326,12 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
 
                     let sequence_close = &mut sequences[close];
                     let close_event_index = sequence_close.event_index;
-                    let seq_close_enter = (
-                        sequence_close.start_point.clone(),
-                        sequence_close.start_index,
-                    );
+                    let seq_close_enter = sequence_close.start_point.clone();
                     sequence_close.size -= take;
                     sequence_close.start_point.column += take;
                     sequence_close.start_point.offset += take;
-                    sequence_close.start_index += take;
-                    let seq_close_exit = (
-                        sequence_close.start_point.clone(),
-                        sequence_close.start_index,
-                    );
+                    sequence_close.start_point.index += take;
+                    let seq_close_exit = sequence_close.start_point.clone();
 
                     // Stay on this closing sequence for the next iteration: it
                     // might close more things.
@@ -359,18 +347,17 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                         // other sequences, and the remainder can be on any side or
                         // somewhere in the middle.
                         let mut enter = &mut tokenizer.events[close_event_index];
-                        enter.point = seq_close_exit.0.clone();
-                        enter.index = seq_close_exit.1;
+                        enter.point = seq_close_exit.clone();
                     }
 
                     let sequence_open = &mut sequences[open];
                     let open_event_index = sequence_open.event_index;
-                    let seq_open_exit = (sequence_open.end_point.clone(), sequence_open.end_index);
+                    let seq_open_exit = sequence_open.end_point.clone();
                     sequence_open.size -= take;
                     sequence_open.end_point.column -= take;
                     sequence_open.end_point.offset -= take;
-                    sequence_open.end_index -= take;
-                    let seq_open_enter = (sequence_open.end_point.clone(), sequence_open.end_index);
+                    sequence_open.end_point.index -= take;
+                    let seq_open_enter = sequence_open.end_point.clone();
 
                     // Remove opening sequence if fully used.
                     if sequence_open.size == 0 {
@@ -381,8 +368,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                         // Shift remaining opening sequence backwards.
                         // See note above for why that happens here.
                         let mut exit = &mut tokenizer.events[open_event_index + 1];
-                        exit.point = seq_open_enter.0.clone();
-                        exit.index = seq_open_enter.1;
+                        exit.point = seq_open_enter.clone();
                     }
 
                     // Opening.
@@ -398,8 +384,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::Strong
                                 },
-                                point: seq_open_enter.0.clone(),
-                                index: seq_open_enter.1,
+                                point: seq_open_enter.clone(),
                                 link: None,
                             },
                             Event {
@@ -409,8 +394,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::StrongSequence
                                 },
-                                point: seq_open_enter.0.clone(),
-                                index: seq_open_enter.1,
+                                point: seq_open_enter.clone(),
                                 link: None,
                             },
                             Event {
@@ -420,8 +404,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::StrongSequence
                                 },
-                                point: seq_open_exit.0.clone(),
-                                index: seq_open_exit.1,
+                                point: seq_open_exit.clone(),
                                 link: None,
                             },
                             Event {
@@ -431,8 +414,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::StrongText
                                 },
-                                point: seq_open_exit.0.clone(),
-                                index: seq_open_exit.1,
+                                point: seq_open_exit.clone(),
                                 link: None,
                             },
                         ],
@@ -449,8 +431,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::StrongText
                                 },
-                                point: seq_close_enter.0.clone(),
-                                index: seq_close_enter.1,
+                                point: seq_close_enter.clone(),
                                 link: None,
                             },
                             Event {
@@ -460,8 +441,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::StrongSequence
                                 },
-                                point: seq_close_enter.0.clone(),
-                                index: seq_close_enter.1,
+                                point: seq_close_enter.clone(),
                                 link: None,
                             },
                             Event {
@@ -471,8 +451,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::StrongSequence
                                 },
-                                point: seq_close_exit.0.clone(),
-                                index: seq_close_exit.1,
+                                point: seq_close_exit.clone(),
                                 link: None,
                             },
                             Event {
@@ -482,8 +461,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                                 } else {
                                     Token::Strong
                                 },
-                                point: seq_close_exit.0.clone(),
-                                index: seq_close_exit.1,
+                                point: seq_close_exit.clone(),
                                 link: None,
                             },
                         ],
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 0e4cda5..e544064 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -216,7 +216,6 @@ pub fn resolve(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                         event_type: EventType::Enter,
                         token_type: Token::HeadingAtxText,
                         point: tokenizer.events[start].point.clone(),
-                        index: tokenizer.events[start].index,
                         link: None,
                     }],
                 );
@@ -231,7 +230,6 @@ pub fn resolve(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                         event_type: EventType::Exit,
                         token_type: Token::HeadingAtxText,
                         point: tokenizer.events[end].point.clone(),
-                        index: tokenizer.events[end].index,
                         link: None,
                     }],
                 );
diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs
index 883e314..2dd8222 100644
--- a/src/construct/label_end.rs
+++ b/src/construct/label_end.rs
@@ -216,8 +216,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
                     id: normalize_identifier(&serialize(
                         &tokenizer.parse_state.codes,
                         &Span {
-                            start_index: tokenizer.events[label_start.start.1].index,
-                            end_index: tokenizer.events[label_end_start - 1].index,
+                            start_index: tokenizer.events[label_start.start.1].point.index,
+                            end_index: tokenizer.events[label_end_start - 1].point.index,
                         },
                         false,
                     )),
@@ -538,9 +538,9 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
         let event = &events[index];
         if event.token_type == Token::ReferenceString {
             if event.event_type == EventType::Exit {
-                end = Some(event.index);
+                end = Some(event.point.index);
             } else {
-                start = Some(event.index);
+                start = Some(event.point.index);
                 break;
             }
         }
@@ -635,14 +635,12 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                     event_type: EventType::Enter,
                     token_type: Token::Data,
                     point: events[data_enter_index].point.clone(),
-                    index: events[data_enter_index].index,
                     link: None,
                 },
                 Event {
                     event_type: EventType::Exit,
                     token_type: Token::Data,
                     point: events[data_exit_index].point.clone(),
-                    index: events[data_exit_index].index,
                     link: None,
                 },
             ],
@@ -685,14 +683,12 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                         Token::Image
                     },
                     point: group_enter_event.point.clone(),
-                    index: group_enter_event.index,
                     link: None,
                 },
                 Event {
                     event_type: EventType::Enter,
                     token_type: Token::Label,
                     point: group_enter_event.point.clone(),
-                    index: group_enter_event.index,
                     link: None,
                 },
             ],
@@ -708,7 +704,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                     event_type: EventType::Enter,
                     token_type: Token::LabelText,
                     point: events[text_enter_index].point.clone(),
-                    index: events[text_enter_index].index,
                     link: None,
                 }],
             );
@@ -721,7 +716,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                     event_type: EventType::Exit,
                     token_type: Token::LabelText,
                     point: events[text_exit_index].point.clone(),
-                    index: events[text_exit_index].index,
                     link: None,
                 }],
             );
@@ -735,7 +729,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                 event_type: EventType::Exit,
                 token_type: Token::Label,
                 point: events[label_exit_index].point.clone(),
-                index: events[label_exit_index].index,
                 link: None,
             }],
         );
@@ -748,7 +741,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                 event_type: EventType::Exit,
                 token_type: Token::Link,
                 point: events[group_end_index].point.clone(),
-                index: events[group_end_index].index,
                 link: None,
             }],
         );
diff --git a/src/construct/list.rs b/src/construct/list.rs
index 12c666b..06be7ec 100644
--- a/src/construct/list.rs
+++ b/src/construct/list.rs
@@ -317,7 +317,8 @@ fn after(tokenizer: &mut Tokenizer, code: Code, blank: bool) -> StateFnResult {
             tokenizer.events.len() - 1,
             &[Token::ListItem],
         );
-        let prefix = tokenizer.index - tokenizer.events[start].index + (if blank { 1 } else { 0 });
+        let prefix = tokenizer.point.index - tokenizer.events[start].point.index
+            + (if blank { 1 } else { 0 });
 
         let container = tokenizer.container.as_mut().unwrap();
         container.blank_initial = blank;
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index 3932ad8..a2eb6ff 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -112,10 +112,8 @@ pub fn resolve(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
                 // Add Exit:LineEnding position info to Exit:Data.
                 let line_ending_exit = &tokenizer.events[exit_index + 2];
                 let line_ending_point = line_ending_exit.point.clone();
-                let line_ending_index = line_ending_exit.index;
                 let data_exit = &mut tokenizer.events[exit_index - 1];
                 data_exit.point = line_ending_point;
-                data_exit.index = line_ending_index;
 
                 // Link Enter:Data on the previous line to Enter:Data on this line.
                 if let Some(link) = &mut tokenizer.events[exit_index - 2].link {
diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs
index ea71bcf..ceeb89b 100644
--- a/src/construct/partial_data.rs
+++ b/src/construct/partial_data.rs
@@ -99,11 +99,7 @@ pub fn resolve_data(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
 
                 // Change positional info.
                 let exit_far = &tokenizer.events[exit_far_index];
-                let point_end = exit_far.point.clone();
-                let index_end = exit_far.index;
-                let exit = &mut tokenizer.events[exit_index];
-                exit.point = point_end;
-                exit.index = index_end;
+                tokenizer.events[exit_index].point = exit_far.point.clone();
                 index = exit_far_index;
 
                 continue;
diff --git a/src/content/document.rs b/src/content/document.rs
index 46da830..1a0ffee 100644
--- a/src/content/document.rs
+++ b/src/content/document.rs
@@ -77,8 +77,8 @@ struct DocumentInfo {
 }
 
 /// Turn `codes` as the document content type into events.
-pub fn document(parse_state: &mut ParseState, point: Point, index: usize) -> Vec<Event> {
-    let mut tokenizer = Tokenizer::new(point, index, parse_state);
+pub fn document(parse_state: &mut ParseState, point: Point) -> Vec<Event> {
+    let mut tokenizer = Tokenizer::new(point, parse_state);
 
     tokenizer.push(&parse_state.codes, Box::new(start), true);
 
@@ -464,7 +464,6 @@ fn exit_containers(
             token_type: token_type.clone(),
             // Note: positions are fixed later.
             point: tokenizer.point.clone(),
-            index: tokenizer.index,
             link: None,
         });
     }
@@ -514,18 +513,12 @@ fn resolve(tokenizer: &mut Tokenizer, info: &mut DocumentInfo) {
         } else {
             &tokenizer.point
         };
-        let index_rel = if let Some(index) = first_line_ending_in_run {
-            tokenizer.events[index].index
-        } else {
-            tokenizer.index
-        };
 
         let close_index = first_line_ending_in_run.unwrap_or(index);
 
         let mut subevent_index = 0;
         while subevent_index < after.len() {
             after[subevent_index].point = point_rel.clone();
-            after[subevent_index].index = index_rel;
             subevent_index += 1;
         }
 
diff --git a/src/parser.rs b/src/parser.rs
index 1e689ee..0f71daf 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -34,8 +34,8 @@ pub fn parse<'a>(value: &str, options: &'a Options) -> (Vec<Event>, ParseState<'
             line: 1,
             column: 1,
             offset: 0,
+            index: 0,
         },
-        0,
     );
 
     (events, parse_state)
diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index ad13022..1f4c6e3 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -75,7 +75,7 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
                 // Index into `events` pointing to a chunk.
                 let mut link_index: Option<usize> = Some(index);
                 // Subtokenizer.
-                let mut tokenizer = Tokenizer::new(event.point.clone(), event.index, parse_state);
+                let mut tokenizer = Tokenizer::new(event.point.clone(), parse_state);
                 // Substate.
                 let mut result: StateFnResult = (
                     State::Fn(Box::new(if link.content_type == ContentType::String {
@@ -92,12 +92,12 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
                     let link_curr = enter.link.as_ref().expect("expected link");
                     assert_eq!(enter.event_type, EventType::Enter);
                     let span = span::Span {
-                        start_index: enter.index,
-                        end_index: events[index + 1].index,
+                        start_index: enter.point.index,
+                        end_index: events[index + 1].point.index,
                     };
 
                     if link_curr.previous != None {
-                        tokenizer.define_skip(&enter.point, enter.index);
+                        tokenizer.define_skip(&enter.point);
                     }
 
                     let func: Box<StateFn> = match result.0 {
@@ -127,7 +127,7 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
                     // Find the first event that starts after the end we’re looking
                     // for.
                     if subevent.event_type == EventType::Enter
-                        && subevent.index >= events[link_index + 1].index
+                        && subevent.point.index >= events[link_index + 1].point.index
                     {
                         slices.push((link_index, slice_start));
                         slice_start = subindex;
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index b745ac8..f5ac1af 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -55,6 +55,8 @@ pub struct Point {
     pub column: usize,
     /// 0-indexed position in the document.
     pub offset: usize,
+    /// Index into `codes`.
+    pub index: usize,
 }
 
 /// Possible event types.
@@ -80,7 +82,6 @@ pub struct Event {
     pub event_type: EventType,
     pub token_type: Token,
     pub point: Point,
-    pub index: usize,
     pub link: Option<Link>,
 }
 
@@ -168,8 +169,6 @@ struct InternalState {
     previous: Code,
     /// Current code.
     current: Code,
-    /// `index` in codes of the current code.
-    index: usize,
     /// Current relative and absolute position in the file.
     point: Point,
 }
@@ -198,8 +197,6 @@ pub struct Tokenizer<'a> {
     pub previous: Code,
     /// Current character code.
     current: Code,
-    /// `index` in codes of the current code.
-    pub index: usize,
     /// Current relative and absolute place in the file.
     pub point: Point,
     /// List of attached resolvers, which will be called when done feeding,
@@ -240,14 +237,13 @@ pub struct Tokenizer<'a> {
 
 impl<'a> Tokenizer<'a> {
     /// Create a new tokenizer.
-    pub fn new(point: Point, index: usize, parse_state: &'a ParseState) -> Tokenizer<'a> {
+    pub fn new(point: Point, parse_state: &'a ParseState) -> Tokenizer<'a> {
         Tokenizer {
             previous: Code::None,
             current: Code::None,
             // To do: reserve size when feeding?
             column_start: vec![],
             line_start: point.line,
-            index,
             consumed: true,
             drained: false,
             point,
@@ -292,8 +288,8 @@ impl<'a> Tokenizer<'a> {
     }
 
     /// Define a jump between two places.
-    pub fn define_skip(&mut self, point: &Point, index: usize) {
-        define_skip_impl(self, point.line, (point.column, point.offset, index));
+    pub fn define_skip(&mut self, point: &Point) {
+        define_skip_impl(self, point.line, (point.column, point.offset, point.index));
     }
 
     /// Define the current place as a jump between two places.
@@ -301,7 +297,7 @@ impl<'a> Tokenizer<'a> {
         define_skip_impl(
             self,
             self.point.line,
-            (self.point.column, self.point.offset, self.index),
+            (self.point.column, self.point.offset, self.point.index),
         );
     }
 
@@ -316,7 +312,7 @@ impl<'a> Tokenizer<'a> {
                 Some((column, offset, index)) => {
                     self.point.column = *column;
                     self.point.offset = *offset;
-                    self.index = *index;
+                    self.point.index = *index;
                 }
             };
         }
@@ -333,7 +329,7 @@ impl<'a> Tokenizer<'a> {
         log::debug!("consume: `{:?}` ({:?})", code, self.point);
         assert!(!self.consumed, "expected code to not have been consumed: this might be because `x(code)` instead of `x` was returned");
 
-        self.index += 1;
+        self.point.index += 1;
 
         match code {
             Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
@@ -383,7 +379,6 @@ impl<'a> Tokenizer<'a> {
             event_type: EventType::Enter,
             token_type: token_type.clone(),
             point: self.point.clone(),
-            index: self.index,
             link,
         });
         self.stack.push(token_type);
@@ -399,11 +394,10 @@ impl<'a> Tokenizer<'a> {
         );
 
         let previous = self.events.last().expect("cannot close w/o open event");
-        let mut index = self.index;
         let mut point = self.point.clone();
 
         assert!(
-            current_token != previous.token_type || previous.index != index,
+            current_token != previous.token_type || previous.point.index != point.index,
             "expected non-empty token"
         );
 
@@ -429,7 +423,7 @@ impl<'a> Tokenizer<'a> {
                 } else {
                     1
                 };
-            index = previous.index + 1;
+            point.index = previous.point.index + 1;
         }
 
         log::debug!("exit: `{:?}` ({:?})", token_type, point);
@@ -437,7 +431,6 @@ impl<'a> Tokenizer<'a> {
             event_type: EventType::Exit,
             token_type,
             point,
-            index,
             link: None,
         });
     }
@@ -445,7 +438,6 @@ impl<'a> Tokenizer<'a> {
     /// Capture the internal state.
     fn capture(&mut self) -> InternalState {
         InternalState {
-            index: self.index,
             previous: self.previous,
             current: self.current,
             point: self.point.clone(),
@@ -456,7 +448,6 @@ impl<'a> Tokenizer<'a> {
 
     /// Apply the internal state.
     fn free(&mut self, previous: InternalState) {
-        self.index = previous.index;
         self.previous = previous.previous;
         self.current = previous.current;
         self.point = previous.point;
diff --git a/src/util/span.rs b/src/util/span.rs
index 72b451d..ca25924 100644
--- a/src/util/span.rs
+++ b/src/util/span.rs
@@ -24,7 +24,7 @@ pub struct Span {
 /// When `micromark` is used, this function never panics.
 pub fn from_exit_event(events: &[Event], index: usize) -> Span {
     let exit = &events[index];
-    let end_index = exit.index;
+    let end_index = exit.point.index;
     let token_type = exit.token_type.clone();
     assert_eq!(
         exit.event_type,
@@ -37,7 +37,7 @@ pub fn from_exit_event(events: &[Event], index: usize) -> Span {
         let enter = &events[enter_index];
         if enter.event_type == EventType::Enter && enter.token_type == token_type {
             return Span {
-                start_index: enter.index,
+                start_index: enter.point.index,
                 end_index,
             };
         }