From 82aca5003acba36a62b0032860af09f65c91ddae Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Tue, 14 Jun 2022 12:26:23 +0200 Subject: Add docs for html (text) --- src/subtokenize.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'src/subtokenize.rs') diff --git a/src/subtokenize.rs b/src/subtokenize.rs index ee826b8..e004f7b 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -7,6 +7,7 @@ use crate::tokenizer::{ use crate::util::{slice_codes, Span}; use std::collections::HashMap; +/// To do. pub fn subtokenize(events: Vec, codes: &[Code]) -> (Vec, bool) { let mut events = events; let mut index = 0; -- cgit