diff options
Diffstat (limited to '')
| -rw-r--r-- | src/tokenizer.rs | 200 |
1 files changed, 77 insertions, 123 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 19e3392..ba18956 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -182,7 +182,7 @@ pub struct Tokenizer<'a> { /// Tracked to make sure everything’s valid. consumed: bool, /// Track whether this tokenizer is done. - drained: bool, + resolved: bool, /// Current character code. pub current: Code, /// Previous character code. @@ -204,8 +204,6 @@ pub struct Tokenizer<'a> { resolver_ids: Vec<String>, /// Shared parsing state across tokenizers. pub parse_state: &'a ParseState<'a>, - /// To do. - pub index: usize, /// Stack of label (start) that could form images and links. /// /// Used when tokenizing [text content][crate::content::text]. @@ -245,12 +243,11 @@ impl<'a> Tokenizer<'a> { column_start: vec![], line_start: point.line, consumed: true, - drained: false, + resolved: false, point, stack: vec![], events: vec![], parse_state, - index: 0, map: EditMap::new(), label_start_stack: vec![], label_start_list_loose: vec![], @@ -281,13 +278,6 @@ impl<'a> Tokenizer<'a> { } } - /// Prepare for a next code to get consumed. - pub fn expect(&mut self, code: Code) { - assert!(self.consumed, "expected previous character to be consumed"); - self.consumed = false; - self.current = code; - } - /// Define a jump between two places. pub fn define_skip(&mut self, point: &Point) { define_skip_impl(self, point.line, point.index); @@ -308,15 +298,20 @@ impl<'a> Tokenizer<'a> { } } + /// Prepare for a next code to get consumed. + pub fn expect(&mut self, code: Code) { + assert!(self.consumed, "expected previous character to be consumed"); + self.consumed = false; + self.current = code; + } + /// Consume the current character. /// Each [`StateFn`][] is expected to call this to signal that this code is /// used, or call a next `StateFn`. pub fn consume(&mut self) { log::debug!("consume: `{:?}` ({:?})", self.current, self.point); assert!(!self.consumed, "expected code to not have been consumed: this might be because `x(code)` instead of `x` was returned"); - - self.move_to(self.index + 1); - + self.move_to(self.point.index + 1); self.previous = self.current; // Mark as consumed. self.consumed = true; @@ -324,10 +319,9 @@ impl<'a> Tokenizer<'a> { /// To do. pub fn move_to(&mut self, to: usize) { - while self.index < to { - let code = &self.parse_state.codes[self.index]; + while self.point.index < to { + let code = &self.parse_state.codes[self.point.index]; self.point.index += 1; - self.index += 1; match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { @@ -478,13 +472,16 @@ impl<'a> Tokenizer<'a> { attempt_impl( state_fn, None, - self.index, - |end: usize, tokenizer: &mut Tokenizer, state| { + self.point.index, + |tokenizer: &mut Tokenizer, state| { if matches!(state, State::Ok) { - tokenizer.index = end; tokenizer.consumed = true; State::Fn(Box::new(after)) } else { + // Must be `Nok`. + // We don’t capture/free state because it is assumed that + // `go` itself is wrapped in another attempt that does that + // if it can occur. state } }, @@ -503,10 +500,12 @@ impl<'a> Tokenizer<'a> { attempt_impl( state_fn, Some(Box::new(until)), - self.index, - |end: usize, tokenizer: &mut Tokenizer, state| { - tokenizer.index = end; + self.point.index, + |tokenizer: &mut Tokenizer, state| { tokenizer.consumed = true; + // We don’t capture/free state because it is assumed that + // `go_until` itself is wrapped in another attempt that does + // that if it can occur. State::Fn(done(state)) }, ) @@ -527,15 +526,13 @@ impl<'a> Tokenizer<'a> { done: impl FnOnce(bool) -> Box<StateFn> + 'static, ) -> Box<StateFn> { let previous = self.capture(); - let start = self.index; attempt_impl( state_fn, None, - start, - move |_: usize, tokenizer: &mut Tokenizer, state| { + self.point.index, + |tokenizer: &mut Tokenizer, state| { tokenizer.free(previous); - tokenizer.index = start; tokenizer.consumed = true; State::Fn(done(matches!(state, State::Ok))) }, @@ -563,8 +560,8 @@ impl<'a> Tokenizer<'a> { attempt_impl( state_fn, None, - self.index, - |end: usize, tokenizer: &mut Tokenizer, state| { + self.point.index, + |tokenizer: &mut Tokenizer, state| { let ok = matches!(state, State::Ok); if !ok { @@ -573,7 +570,6 @@ impl<'a> Tokenizer<'a> { log::debug!("attempt: {:?}, at {:?}", ok, tokenizer.point); - tokenizer.index = end; tokenizer.consumed = true; State::Fn(done(ok)) }, @@ -619,21 +615,56 @@ impl<'a> Tokenizer<'a> { min: usize, max: usize, start: impl FnOnce(&mut Tokenizer) -> State + 'static, - drain: bool, ) -> State { - assert!(!self.drained, "cannot feed after drain"); + assert!(!self.resolved, "cannot feed after drain"); + assert!(min >= self.point.index, "cannot move backwards"); - let mut result = feed_impl(self, min, max, start); + self.move_to(min); - if drain { - let func = match result { - State::Fn(func) => func, - _ => unreachable!("expected next state"), - }; + let mut state = State::Fn(Box::new(start)); + + while self.point.index < max { + match state { + State::Ok | State::Nok => break, + State::Fn(func) => { + let code = self.parse_state.codes[self.point.index]; + log::debug!("main: passing: `{:?}` ({:?})", code, self.point); + self.expect(code); + state = func(self); + } + } + } - result = flush_impl(self, max, func); + state + } - self.drained = true; + /// Flush the tokenizer. + pub fn flush(&mut self, mut state: State, resolve: bool) { + let max = self.point.index; + + self.consumed = true; + + loop { + match state { + State::Ok | State::Nok => break, + State::Fn(func) => { + // We sometimes move back when flushing, so then we use those codes. + let code = if self.point.index < max { + self.parse_state.codes[self.point.index] + } else { + Code::None + }; + log::debug!("main: flushing {:?}", code); + self.expect(code); + state = func(self); + } + } + } + + assert!(matches!(state, State::Ok), "must be ok"); + + if resolve { + self.resolved = true; while !self.resolvers.is_empty() { let resolver = self.resolvers.remove(0); @@ -642,13 +673,6 @@ impl<'a> Tokenizer<'a> { self.map.consume(&mut self.events); } - - result - } - - /// Flush the tokenizer. - pub fn flush(&mut self, start: impl FnOnce(&mut Tokenizer) -> State + 'static) -> State { - flush_impl(self, self.index, start) } } @@ -660,98 +684,28 @@ fn attempt_impl( state: impl FnOnce(&mut Tokenizer) -> State + 'static, pause: Option<Box<dyn Fn(Code) -> bool + 'static>>, start: usize, - done: impl FnOnce(usize, &mut Tokenizer, State) -> State + 'static, + done: impl FnOnce(&mut Tokenizer, State) -> State + 'static, ) -> Box<StateFn> { Box::new(move |tokenizer| { if let Some(ref func) = pause { - if tokenizer.index > start && func(tokenizer.previous) { - return done(tokenizer.index, tokenizer, State::Fn(Box::new(state))); + if tokenizer.point.index > start && func(tokenizer.previous) { + return done(tokenizer, State::Fn(Box::new(state))); } } let state = state(tokenizer); match state { - State::Ok => { - assert!( - tokenizer.index >= start, - "`end` must not be smaller than `start`" - ); - done(tokenizer.index, tokenizer, state) - } - State::Nok => done(start, tokenizer, state), + State::Ok | State::Nok => done(tokenizer, state), State::Fn(func) => State::Fn(attempt_impl(func, pause, start, done)), } }) } -/// Feed a list of `codes` into `start`. -fn feed_impl( - tokenizer: &mut Tokenizer, - min: usize, - max: usize, - start: impl FnOnce(&mut Tokenizer) -> State + 'static, -) -> State { - let mut state = State::Fn(Box::new(start)); - - tokenizer.index = min; - tokenizer.consumed = true; - - while tokenizer.index < max { - let code = tokenizer.parse_state.codes[tokenizer.index]; - - match state { - State::Ok | State::Nok => { - break; - } - State::Fn(func) => { - log::debug!("main: passing: `{:?}` ({:?})", code, tokenizer.index); - tokenizer.expect(code); - state = func(tokenizer); - } - } - } - - state -} - /// Flush `start`: pass `eof`s to it until done. -fn flush_impl( - tokenizer: &mut Tokenizer, - max: usize, - start: impl FnOnce(&mut Tokenizer) -> State + 'static, -) -> State { - let mut state = State::Fn(Box::new(start)); - tokenizer.consumed = true; - - loop { - match state { - State::Ok | State::Nok => break, - State::Fn(func) => { - let code = if tokenizer.index < max { - tokenizer.parse_state.codes[tokenizer.index] - } else { - Code::None - }; - log::debug!("main: flushing {:?}", code); - tokenizer.expect(code); - state = func(tokenizer); - } - } - } - - match state { - State::Ok => {} - _ => unreachable!("expected final state to be `State::Ok`"), - } - - state -} - /// Define a jump between two places. /// -/// This defines how much columns, offsets, and the `index` are increased when -/// consuming a line ending. +/// This defines to which future index we move after a line ending. fn define_skip_impl(tokenizer: &mut Tokenizer, line: usize, index: usize) { log::debug!("position: define skip: {:?} -> ({:?})", line, index); let at = line - tokenizer.line_start; |