aboutsummaryrefslogtreecommitdiffstats
path: root/src/subtokenize.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/subtokenize.rs')
-rw-r--r--src/subtokenize.rs9
1 files changed, 1 insertions, 8 deletions
diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index 0623a37..1188c61 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -66,14 +66,7 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) {
};
result = tokenizer.feed(span::codes(codes, &span), func, enter.next == None);
-
- if let Some(ref x) = result.1 {
- if !x.is_empty() {
- // To do: handle?
- unreachable!("subtokenize:remainder {:?}", x);
- }
- }
-
+ assert!(result.1.is_none(), "expected no remainder");
index_opt = enter.next;
}