From f5b6e828d5f3b698b150ba4a0831b61574c6cc3b Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Fri, 8 Jul 2022 17:16:24 +0200 Subject: Fix positional info on line ending after shift --- src/tokenizer.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/tokenizer.rs b/src/tokenizer.rs index dcbcb09..c984a75 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -346,19 +346,32 @@ impl<'a> Tokenizer<'a> { ); let previous = self.events.last().expect("cannot close w/o open event"); - let point = self.point.clone(); + let mut index = self.index; + let mut point = self.point.clone(); assert!( - current_token != previous.token_type || previous.index != self.index, + current_token != previous.token_type || previous.index != index, "expected non-empty token" ); - log::debug!("exit `{:?}` ({:?})", token_type, self.point); + // A bit weird, but if we exit right after a line ending, we *don’t* want to consider\ + // potential skips. + if matches!( + self.previous, + Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') + ) { + let shift = point.column - 1; + point.column -= shift; + point.offset -= shift; + index -= shift; + } + + log::debug!("exit `{:?}` ({:?})", token_type, point); self.events.push(Event { event_type: EventType::Exit, token_type, point, - index: self.index, + index, previous: None, next: None, content_type: None, -- cgit