diff options
Diffstat (limited to 'src/subtokenize.rs')
-rw-r--r-- | src/subtokenize.rs | 9 |
1 files changed, 1 insertions, 8 deletions
diff --git a/src/subtokenize.rs b/src/subtokenize.rs index 0623a37..1188c61 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -66,14 +66,7 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) { }; result = tokenizer.feed(span::codes(codes, &span), func, enter.next == None); - - if let Some(ref x) = result.1 { - if !x.is_empty() { - // To do: handle? - unreachable!("subtokenize:remainder {:?}", x); - } - } - + assert!(result.1.is_none(), "expected no remainder"); index_opt = enter.next; } |