diff options
Diffstat (limited to '')
| -rw-r--r-- | src/subtokenize.rs | 1 |
1 files changed, 1 insertions, 0 deletions
diff --git a/src/subtokenize.rs b/src/subtokenize.rs index ee826b8..e004f7b 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -7,6 +7,7 @@ use crate::tokenizer::{ use crate::util::{slice_codes, Span}; use std::collections::HashMap; +/// To do. pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) { let mut events = events; let mut index = 0; |