aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/construct/attention.rs66
-rw-r--r--src/construct/heading_atx.rs2
-rw-r--r--src/construct/label_end.rs16
-rw-r--r--src/construct/list.rs3
-rw-r--r--src/construct/paragraph.rs2
-rw-r--r--src/construct/partial_data.rs6
-rw-r--r--src/content/document.rs11
-rw-r--r--src/parser.rs2
-rw-r--r--src/subtokenize.rs10
-rw-r--r--src/tokenizer.rs29
-rw-r--r--src/util/span.rs4
11 files changed, 49 insertions, 102 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs
index ac6aecf..38164a8 100644
--- a/src/construct/attention.rs
+++ b/src/construct/attention.rs
@@ -153,12 +153,8 @@ struct Sequence {
event_index: usize,
/// The (shifted) point where this sequence starts.
start_point: Point,
- /// The (shifted) index where this sequence starts.
- start_index: usize,
/// The (shifted) point where this sequence end.
end_point: Point,
- /// The (shifted) index where this sequence end.
- end_index: usize,
/// The number of markers we can still use.
size: usize,
/// Whether this sequence can open attention.
@@ -221,14 +217,14 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
if enter.token_type == Token::AttentionSequence {
let end = start + 1;
let exit = &tokenizer.events[end];
- let marker = MarkerKind::from_code(codes[enter.index]);
- let before = classify_character(if enter.index > 0 {
- codes[enter.index - 1]
+ let marker = MarkerKind::from_code(codes[enter.point.index]);
+ let before = classify_character(if enter.point.index > 0 {
+ codes[enter.point.index - 1]
} else {
Code::None
});
- let after = classify_character(if exit.index < codes.len() {
- codes[exit.index]
+ let after = classify_character(if exit.point.index < codes.len() {
+ codes[exit.point.index]
} else {
Code::None
});
@@ -245,10 +241,8 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_index: start,
balance,
start_point: enter.point.clone(),
- start_index: enter.index,
end_point: exit.point.clone(),
- end_index: exit.index,
- size: exit.index - enter.index,
+ size: exit.point.index - enter.point.index,
open: if marker == MarkerKind::Asterisk {
open
} else {
@@ -332,18 +326,12 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
let sequence_close = &mut sequences[close];
let close_event_index = sequence_close.event_index;
- let seq_close_enter = (
- sequence_close.start_point.clone(),
- sequence_close.start_index,
- );
+ let seq_close_enter = sequence_close.start_point.clone();
sequence_close.size -= take;
sequence_close.start_point.column += take;
sequence_close.start_point.offset += take;
- sequence_close.start_index += take;
- let seq_close_exit = (
- sequence_close.start_point.clone(),
- sequence_close.start_index,
- );
+ sequence_close.start_point.index += take;
+ let seq_close_exit = sequence_close.start_point.clone();
// Stay on this closing sequence for the next iteration: it
// might close more things.
@@ -359,18 +347,17 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
// other sequences, and the remainder can be on any side or
// somewhere in the middle.
let mut enter = &mut tokenizer.events[close_event_index];
- enter.point = seq_close_exit.0.clone();
- enter.index = seq_close_exit.1;
+ enter.point = seq_close_exit.clone();
}
let sequence_open = &mut sequences[open];
let open_event_index = sequence_open.event_index;
- let seq_open_exit = (sequence_open.end_point.clone(), sequence_open.end_index);
+ let seq_open_exit = sequence_open.end_point.clone();
sequence_open.size -= take;
sequence_open.end_point.column -= take;
sequence_open.end_point.offset -= take;
- sequence_open.end_index -= take;
- let seq_open_enter = (sequence_open.end_point.clone(), sequence_open.end_index);
+ sequence_open.end_point.index -= take;
+ let seq_open_enter = sequence_open.end_point.clone();
// Remove opening sequence if fully used.
if sequence_open.size == 0 {
@@ -381,8 +368,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
// Shift remaining opening sequence backwards.
// See note above for why that happens here.
let mut exit = &mut tokenizer.events[open_event_index + 1];
- exit.point = seq_open_enter.0.clone();
- exit.index = seq_open_enter.1;
+ exit.point = seq_open_enter.clone();
}
// Opening.
@@ -398,8 +384,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::Strong
},
- point: seq_open_enter.0.clone(),
- index: seq_open_enter.1,
+ point: seq_open_enter.clone(),
link: None,
},
Event {
@@ -409,8 +394,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::StrongSequence
},
- point: seq_open_enter.0.clone(),
- index: seq_open_enter.1,
+ point: seq_open_enter.clone(),
link: None,
},
Event {
@@ -420,8 +404,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::StrongSequence
},
- point: seq_open_exit.0.clone(),
- index: seq_open_exit.1,
+ point: seq_open_exit.clone(),
link: None,
},
Event {
@@ -431,8 +414,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::StrongText
},
- point: seq_open_exit.0.clone(),
- index: seq_open_exit.1,
+ point: seq_open_exit.clone(),
link: None,
},
],
@@ -449,8 +431,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::StrongText
},
- point: seq_close_enter.0.clone(),
- index: seq_close_enter.1,
+ point: seq_close_enter.clone(),
link: None,
},
Event {
@@ -460,8 +441,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::StrongSequence
},
- point: seq_close_enter.0.clone(),
- index: seq_close_enter.1,
+ point: seq_close_enter.clone(),
link: None,
},
Event {
@@ -471,8 +451,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::StrongSequence
},
- point: seq_close_exit.0.clone(),
- index: seq_close_exit.1,
+ point: seq_close_exit.clone(),
link: None,
},
Event {
@@ -482,8 +461,7 @@ fn resolve_attention(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
} else {
Token::Strong
},
- point: seq_close_exit.0.clone(),
- index: seq_close_exit.1,
+ point: seq_close_exit.clone(),
link: None,
},
],
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 0e4cda5..e544064 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -216,7 +216,6 @@ pub fn resolve(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Enter,
token_type: Token::HeadingAtxText,
point: tokenizer.events[start].point.clone(),
- index: tokenizer.events[start].index,
link: None,
}],
);
@@ -231,7 +230,6 @@ pub fn resolve(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Exit,
token_type: Token::HeadingAtxText,
point: tokenizer.events[end].point.clone(),
- index: tokenizer.events[end].index,
link: None,
}],
);
diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs
index 883e314..2dd8222 100644
--- a/src/construct/label_end.rs
+++ b/src/construct/label_end.rs
@@ -216,8 +216,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
id: normalize_identifier(&serialize(
&tokenizer.parse_state.codes,
&Span {
- start_index: tokenizer.events[label_start.start.1].index,
- end_index: tokenizer.events[label_end_start - 1].index,
+ start_index: tokenizer.events[label_start.start.1].point.index,
+ end_index: tokenizer.events[label_end_start - 1].point.index,
},
false,
)),
@@ -538,9 +538,9 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
let event = &events[index];
if event.token_type == Token::ReferenceString {
if event.event_type == EventType::Exit {
- end = Some(event.index);
+ end = Some(event.point.index);
} else {
- start = Some(event.index);
+ start = Some(event.point.index);
break;
}
}
@@ -635,14 +635,12 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Enter,
token_type: Token::Data,
point: events[data_enter_index].point.clone(),
- index: events[data_enter_index].index,
link: None,
},
Event {
event_type: EventType::Exit,
token_type: Token::Data,
point: events[data_exit_index].point.clone(),
- index: events[data_exit_index].index,
link: None,
},
],
@@ -685,14 +683,12 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
Token::Image
},
point: group_enter_event.point.clone(),
- index: group_enter_event.index,
link: None,
},
Event {
event_type: EventType::Enter,
token_type: Token::Label,
point: group_enter_event.point.clone(),
- index: group_enter_event.index,
link: None,
},
],
@@ -708,7 +704,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Enter,
token_type: Token::LabelText,
point: events[text_enter_index].point.clone(),
- index: events[text_enter_index].index,
link: None,
}],
);
@@ -721,7 +716,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Exit,
token_type: Token::LabelText,
point: events[text_exit_index].point.clone(),
- index: events[text_exit_index].index,
link: None,
}],
);
@@ -735,7 +729,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Exit,
token_type: Token::Label,
point: events[label_exit_index].point.clone(),
- index: events[label_exit_index].index,
link: None,
}],
);
@@ -748,7 +741,6 @@ pub fn resolve_media(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
event_type: EventType::Exit,
token_type: Token::Link,
point: events[group_end_index].point.clone(),
- index: events[group_end_index].index,
link: None,
}],
);
diff --git a/src/construct/list.rs b/src/construct/list.rs
index 12c666b..06be7ec 100644
--- a/src/construct/list.rs
+++ b/src/construct/list.rs
@@ -317,7 +317,8 @@ fn after(tokenizer: &mut Tokenizer, code: Code, blank: bool) -> StateFnResult {
tokenizer.events.len() - 1,
&[Token::ListItem],
);
- let prefix = tokenizer.index - tokenizer.events[start].index + (if blank { 1 } else { 0 });
+ let prefix = tokenizer.point.index - tokenizer.events[start].point.index
+ + (if blank { 1 } else { 0 });
let container = tokenizer.container.as_mut().unwrap();
container.blank_initial = blank;
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index 3932ad8..a2eb6ff 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -112,10 +112,8 @@ pub fn resolve(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
// Add Exit:LineEnding position info to Exit:Data.
let line_ending_exit = &tokenizer.events[exit_index + 2];
let line_ending_point = line_ending_exit.point.clone();
- let line_ending_index = line_ending_exit.index;
let data_exit = &mut tokenizer.events[exit_index - 1];
data_exit.point = line_ending_point;
- data_exit.index = line_ending_index;
// Link Enter:Data on the previous line to Enter:Data on this line.
if let Some(link) = &mut tokenizer.events[exit_index - 2].link {
diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs
index ea71bcf..ceeb89b 100644
--- a/src/construct/partial_data.rs
+++ b/src/construct/partial_data.rs
@@ -99,11 +99,7 @@ pub fn resolve_data(tokenizer: &mut Tokenizer, map: &mut EditMap) -> bool {
// Change positional info.
let exit_far = &tokenizer.events[exit_far_index];
- let point_end = exit_far.point.clone();
- let index_end = exit_far.index;
- let exit = &mut tokenizer.events[exit_index];
- exit.point = point_end;
- exit.index = index_end;
+ tokenizer.events[exit_index].point = exit_far.point.clone();
index = exit_far_index;
continue;
diff --git a/src/content/document.rs b/src/content/document.rs
index 46da830..1a0ffee 100644
--- a/src/content/document.rs
+++ b/src/content/document.rs
@@ -77,8 +77,8 @@ struct DocumentInfo {
}
/// Turn `codes` as the document content type into events.
-pub fn document(parse_state: &mut ParseState, point: Point, index: usize) -> Vec<Event> {
- let mut tokenizer = Tokenizer::new(point, index, parse_state);
+pub fn document(parse_state: &mut ParseState, point: Point) -> Vec<Event> {
+ let mut tokenizer = Tokenizer::new(point, parse_state);
tokenizer.push(&parse_state.codes, Box::new(start), true);
@@ -464,7 +464,6 @@ fn exit_containers(
token_type: token_type.clone(),
// Note: positions are fixed later.
point: tokenizer.point.clone(),
- index: tokenizer.index,
link: None,
});
}
@@ -514,18 +513,12 @@ fn resolve(tokenizer: &mut Tokenizer, info: &mut DocumentInfo) {
} else {
&tokenizer.point
};
- let index_rel = if let Some(index) = first_line_ending_in_run {
- tokenizer.events[index].index
- } else {
- tokenizer.index
- };
let close_index = first_line_ending_in_run.unwrap_or(index);
let mut subevent_index = 0;
while subevent_index < after.len() {
after[subevent_index].point = point_rel.clone();
- after[subevent_index].index = index_rel;
subevent_index += 1;
}
diff --git a/src/parser.rs b/src/parser.rs
index 1e689ee..0f71daf 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -34,8 +34,8 @@ pub fn parse<'a>(value: &str, options: &'a Options) -> (Vec<Event>, ParseState<'
line: 1,
column: 1,
offset: 0,
+ index: 0,
},
- 0,
);
(events, parse_state)
diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index ad13022..1f4c6e3 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -75,7 +75,7 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
// Index into `events` pointing to a chunk.
let mut link_index: Option<usize> = Some(index);
// Subtokenizer.
- let mut tokenizer = Tokenizer::new(event.point.clone(), event.index, parse_state);
+ let mut tokenizer = Tokenizer::new(event.point.clone(), parse_state);
// Substate.
let mut result: StateFnResult = (
State::Fn(Box::new(if link.content_type == ContentType::String {
@@ -92,12 +92,12 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
let link_curr = enter.link.as_ref().expect("expected link");
assert_eq!(enter.event_type, EventType::Enter);
let span = span::Span {
- start_index: enter.index,
- end_index: events[index + 1].index,
+ start_index: enter.point.index,
+ end_index: events[index + 1].point.index,
};
if link_curr.previous != None {
- tokenizer.define_skip(&enter.point, enter.index);
+ tokenizer.define_skip(&enter.point);
}
let func: Box<StateFn> = match result.0 {
@@ -127,7 +127,7 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool {
// Find the first event that starts after the end we’re looking
// for.
if subevent.event_type == EventType::Enter
- && subevent.index >= events[link_index + 1].index
+ && subevent.point.index >= events[link_index + 1].point.index
{
slices.push((link_index, slice_start));
slice_start = subindex;
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index b745ac8..f5ac1af 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -55,6 +55,8 @@ pub struct Point {
pub column: usize,
/// 0-indexed position in the document.
pub offset: usize,
+ /// Index into `codes`.
+ pub index: usize,
}
/// Possible event types.
@@ -80,7 +82,6 @@ pub struct Event {
pub event_type: EventType,
pub token_type: Token,
pub point: Point,
- pub index: usize,
pub link: Option<Link>,
}
@@ -168,8 +169,6 @@ struct InternalState {
previous: Code,
/// Current code.
current: Code,
- /// `index` in codes of the current code.
- index: usize,
/// Current relative and absolute position in the file.
point: Point,
}
@@ -198,8 +197,6 @@ pub struct Tokenizer<'a> {
pub previous: Code,
/// Current character code.
current: Code,
- /// `index` in codes of the current code.
- pub index: usize,
/// Current relative and absolute place in the file.
pub point: Point,
/// List of attached resolvers, which will be called when done feeding,
@@ -240,14 +237,13 @@ pub struct Tokenizer<'a> {
impl<'a> Tokenizer<'a> {
/// Create a new tokenizer.
- pub fn new(point: Point, index: usize, parse_state: &'a ParseState) -> Tokenizer<'a> {
+ pub fn new(point: Point, parse_state: &'a ParseState) -> Tokenizer<'a> {
Tokenizer {
previous: Code::None,
current: Code::None,
// To do: reserve size when feeding?
column_start: vec![],
line_start: point.line,
- index,
consumed: true,
drained: false,
point,
@@ -292,8 +288,8 @@ impl<'a> Tokenizer<'a> {
}
/// Define a jump between two places.
- pub fn define_skip(&mut self, point: &Point, index: usize) {
- define_skip_impl(self, point.line, (point.column, point.offset, index));
+ pub fn define_skip(&mut self, point: &Point) {
+ define_skip_impl(self, point.line, (point.column, point.offset, point.index));
}
/// Define the current place as a jump between two places.
@@ -301,7 +297,7 @@ impl<'a> Tokenizer<'a> {
define_skip_impl(
self,
self.point.line,
- (self.point.column, self.point.offset, self.index),
+ (self.point.column, self.point.offset, self.point.index),
);
}
@@ -316,7 +312,7 @@ impl<'a> Tokenizer<'a> {
Some((column, offset, index)) => {
self.point.column = *column;
self.point.offset = *offset;
- self.index = *index;
+ self.point.index = *index;
}
};
}
@@ -333,7 +329,7 @@ impl<'a> Tokenizer<'a> {
log::debug!("consume: `{:?}` ({:?})", code, self.point);
assert!(!self.consumed, "expected code to not have been consumed: this might be because `x(code)` instead of `x` was returned");
- self.index += 1;
+ self.point.index += 1;
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
@@ -383,7 +379,6 @@ impl<'a> Tokenizer<'a> {
event_type: EventType::Enter,
token_type: token_type.clone(),
point: self.point.clone(),
- index: self.index,
link,
});
self.stack.push(token_type);
@@ -399,11 +394,10 @@ impl<'a> Tokenizer<'a> {
);
let previous = self.events.last().expect("cannot close w/o open event");
- let mut index = self.index;
let mut point = self.point.clone();
assert!(
- current_token != previous.token_type || previous.index != index,
+ current_token != previous.token_type || previous.point.index != point.index,
"expected non-empty token"
);
@@ -429,7 +423,7 @@ impl<'a> Tokenizer<'a> {
} else {
1
};
- index = previous.index + 1;
+ point.index = previous.point.index + 1;
}
log::debug!("exit: `{:?}` ({:?})", token_type, point);
@@ -437,7 +431,6 @@ impl<'a> Tokenizer<'a> {
event_type: EventType::Exit,
token_type,
point,
- index,
link: None,
});
}
@@ -445,7 +438,6 @@ impl<'a> Tokenizer<'a> {
/// Capture the internal state.
fn capture(&mut self) -> InternalState {
InternalState {
- index: self.index,
previous: self.previous,
current: self.current,
point: self.point.clone(),
@@ -456,7 +448,6 @@ impl<'a> Tokenizer<'a> {
/// Apply the internal state.
fn free(&mut self, previous: InternalState) {
- self.index = previous.index;
self.previous = previous.previous;
self.current = previous.current;
self.point = previous.point;
diff --git a/src/util/span.rs b/src/util/span.rs
index 72b451d..ca25924 100644
--- a/src/util/span.rs
+++ b/src/util/span.rs
@@ -24,7 +24,7 @@ pub struct Span {
/// When `micromark` is used, this function never panics.
pub fn from_exit_event(events: &[Event], index: usize) -> Span {
let exit = &events[index];
- let end_index = exit.index;
+ let end_index = exit.point.index;
let token_type = exit.token_type.clone();
assert_eq!(
exit.event_type,
@@ -37,7 +37,7 @@ pub fn from_exit_event(events: &[Event], index: usize) -> Span {
let enter = &events[enter_index];
if enter.event_type == EventType::Enter && enter.token_type == token_type {
return Span {
- start_index: enter.index,
+ start_index: enter.point.index,
end_index,
};
}
generated by cgit (git 2.34.1) at 2024-11-22 02:20:16 +0000