1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
|
use crate::content::string::string;
use crate::tokenizer::{Code, Event, EventType, TokenType};
use crate::util::{slice_codes, Span};
pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> Vec<Event> {
let mut events = events;
let mut index = 0;
// println!("before");
// while index < events.len() {
// let event = &events[index];
// println!(
// "ev1: {:?} {:?} {:?}",
// event.event_type, event.token_type, index
// );
// index += 1;
// }
//
// index = 0;
//
// println!("change");
while index < events.len() {
let event = &events[index];
// println!(
// "ev2: {:?} {:?} {:?}",
// event.event_type, event.token_type, index
// );
if event.event_type == EventType::Enter && event.token_type == TokenType::ChunkString {
let exit = &events[index + 1];
assert_eq!(
exit.event_type,
EventType::Exit,
"expected `enter` of `{:?}` to be follow by an `exit` event",
event.token_type
);
assert_eq!(
exit.token_type, event.token_type,
"expected `exit` of `{:?}` to follow its `enter` event",
event.token_type
);
let subevents = string(
slice_codes(
codes,
&Span {
start_index: event.index,
end_index: exit.index,
},
),
event.point.clone(),
event.index,
);
let len = subevents.len();
// To do: recursion needed?
events.splice(index..(index + 2), subevents);
index += len;
} else {
index += 1;
}
}
events
}
|
