aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r--src/tokenizer.rs21
1 files changed, 20 insertions, 1 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 346e04e..e463b4b 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -13,7 +13,11 @@ use crate::parser::ParseState;
use crate::resolve::{call as call_resolve, Name as ResolveName};
use crate::state::{call, State};
use crate::subtokenize::Subresult;
-use crate::util::{char::format_byte_opt, constant::TAB_SIZE, edit_map::EditMap};
+
+#[cfg(feature = "log")]
+use crate::util::char::format_byte_opt;
+
+use crate::util::{constant::TAB_SIZE, edit_map::EditMap};
use alloc::{boxed::Box, string::String, vec, vec::Vec};
/// Containers.
@@ -405,7 +409,10 @@ impl<'a> Tokenizer<'a> {
move_point_back(self, &mut point);
let info = (point.index, point.vs);
+
+ #[cfg(feature = "log")]
log::debug!("position: define skip: {:?} -> ({:?})", point.line, info);
+
let at = point.line - self.first_line;
if at >= self.column_start.len() {
@@ -476,6 +483,8 @@ impl<'a> Tokenizer<'a> {
self.line_start = self.point.clone();
self.account_for_potential_skip();
+
+ #[cfg(feature = "log")]
log::debug!("position: after eol: `{:?}`", self.point);
} else {
self.point.column += 1;
@@ -533,7 +542,9 @@ impl<'a> Tokenizer<'a> {
move_point_back(self, &mut point);
}
+ #[cfg(feature = "log")]
log::debug!("exit: `{:?}`", name);
+
let event = Event {
kind: Kind::Exit,
name,
@@ -663,7 +674,9 @@ fn enter_impl(tokenizer: &mut Tokenizer, name: Name, link: Option<Link>) {
let mut point = tokenizer.point.clone();
move_point_back(tokenizer, &mut point);
+ #[cfg(feature = "log")]
log::debug!("enter: `{:?}`", name);
+
tokenizer.stack.push(name.clone());
tokenizer.events.push(Event {
kind: Kind::Enter,
@@ -708,7 +721,9 @@ fn push_impl(
attempt.nok
};
+ #[cfg(feature = "log")]
log::debug!("attempt: `{:?}` -> `{:?}`", state, next);
+
state = next;
} else {
break;
@@ -735,13 +750,17 @@ fn push_impl(
None
};
+ #[cfg(feature = "log")]
log::debug!("feed: {} to {:?}", format_byte_opt(byte), name);
+
tokenizer.expect(byte);
state = call(tokenizer, name);
};
}
State::Retry(name) => {
+ #[cfg(feature = "log")]
log::debug!("retry: `{:?}`", name);
+
state = call(tokenizer, name);
}
}