1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
//! Deal with content in other content.
//!
//! To deal with content in content, *you* (a `micromark-rs` contributor) add
//! information on events.
//! Events are a flat list, but they can be connected to each other by setting
//! `previous` and `next` links.
//! These links:
//!
//! * …must occur on [`Enter`][EventType::Enter] events only
//! * …must occur on void events (they are followed by their corresponding
//! [`Exit`][EventType::Exit] event)
//! * …must have `content_type` field to define the kind of subcontent
//!
//! Links will then be passed through a tokenizer for the corresponding content
//! type by `subtokenize`.
//! The subevents they result in are split up into slots for each linked token
//! and replace those links.
//!
//! Subevents are not immediately subtokenized again because markdown prevents
//! us from doing so due to definitions, which can occur after references, and
//! thus the whole document needs to be parsed up to the level of definitions,
//! before any level that can include references can be parsed.
use crate::content::{string::start as string, text::start as text};
use crate::parser::ParseState;
use crate::tokenizer::{ContentType, Event, EventType, State, Tokenizer};
use crate::util::{edit_map::EditMap, span};
/// Create a link between two [`Event`][]s.
///
/// Arbitrary (void) events can be linked together.
/// This optimizes for the common case where the token at `index` is connected
/// to the previous void token.
pub fn link(events: &mut [Event], index: usize) {
link_to(events, index - 2, index);
}
/// Link two arbitrary [`Event`][]s together.
pub fn link_to(events: &mut [Event], pevious: usize, next: usize) {
assert_eq!(events[pevious].event_type, EventType::Enter);
assert_eq!(events[pevious + 1].event_type, EventType::Exit);
assert_eq!(events[pevious + 1].token_type, events[pevious].token_type);
assert_eq!(events[next].event_type, EventType::Enter);
// Note: the exit of this event may not exist, so don’t check for that.
let link_previous = events[pevious]
.link
.as_mut()
.expect("expected `link` on previous");
link_previous.next = Some(next);
let link_next = events[next].link.as_mut().expect("expected `link` on next");
link_next.previous = Some(pevious);
assert_eq!(
events[pevious].link.as_ref().unwrap().content_type,
events[next].link.as_ref().unwrap().content_type
);
}
/// Parse linked events.
///
/// Supposed to be called repeatedly, returns `1: true` when done.
pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
let mut map = EditMap::new();
let mut done = true;
let mut index = 0;
while index < events.len() {
let event = &events[index];
// Find each first opening chunk.
if let Some(ref link) = event.link {
assert_eq!(event.event_type, EventType::Enter);
// No need to enter linked events again.
if link.previous == None {
// Index into `events` pointing to a chunk.
let mut link_index = Some(index);
// Subtokenizer.
let mut tokenizer = Tokenizer::new(event.point.clone(), parse_state);
// Substate.
let mut state = State::Fn(Box::new(if link.content_type == ContentType::String {
string
} else {
text
}));
// Loop through links to pass them in order to the subtokenizer.
while let Some(index) = link_index {
let enter = &events[index];
let link_curr = enter.link.as_ref().expect("expected link");
assert_eq!(enter.event_type, EventType::Enter);
let span = span::Span {
start_index: enter.point.index,
end_index: events[index + 1].point.index,
};
if link_curr.previous != None {
tokenizer.define_skip(&enter.point);
}
let func = match state {
State::Fn(func) => func,
_ => unreachable!("cannot be ok/nok"),
};
state = tokenizer.push(
span::codes(&parse_state.codes, &span),
func,
link_curr.next == None,
);
link_index = link_curr.next;
}
// Now, loop through all subevents to figure out which parts
// belong where and fix deep links.
let mut subindex = 0;
let mut link_index = index;
let mut slices = vec![];
let mut slice_start = 0;
while subindex < tokenizer.events.len() {
let subevent = &mut tokenizer.events[subindex];
// Find the first event that starts after the end we’re looking
// for.
if subevent.event_type == EventType::Enter
&& subevent.point.index >= events[link_index + 1].point.index
{
slices.push((link_index, slice_start));
slice_start = subindex;
link_index = events[link_index].link.as_ref().unwrap().next.unwrap();
}
if subevent.link.is_some() {
// Need to call `subtokenize` again.
done = false;
}
// If there is a `next` link in the subevents, we have to change
// its index to account for the shifted events.
// If it points to a next event, we also change the next event’s
// reference back to *this* event.
if let Some(sublink_curr) = &mut subevent.link {
if let Some(next) = sublink_curr.next {
// The `index` in `events` where the current link is,
// minus 2 events (the enter and exit) for each removed
// link.
let shift = link_index - (slices.len() * 2);
sublink_curr.next = sublink_curr.next.map(|next| next + shift);
let next_ev = &mut tokenizer.events[next];
let sublink_next = next_ev.link.as_mut().unwrap();
sublink_next.previous =
sublink_next.previous.map(|previous| previous + shift);
}
}
subindex += 1;
}
slices.push((link_index, slice_start));
// Finally, inject the subevents.
let mut index = slices.len();
while index > 0 {
index -= 1;
map.add(
slices[index].0,
2,
tokenizer.events.split_off(slices[index].1),
);
}
}
}
index += 1;
}
map.consume(events);
done
}
|
