aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r--src/tokenizer.rs2
1 files changed, 2 insertions, 0 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 8a2f477..486bc75 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -513,6 +513,8 @@ impl Tokenizer {
let mut state = State::Fn(Box::new(start));
let mut index = 0;
+ println!("feed: {:?} {:?}", codes, drain);
+
self.consumed = true;
while index < codes.len() {