diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/subtokenize.rs | 9 | ||||
| -rw-r--r-- | src/tokenizer.rs | 39 |
2 files changed, 24 insertions, 24 deletions
diff --git a/src/subtokenize.rs b/src/subtokenize.rs index 0623a37..1188c61 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -66,14 +66,7 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) { }; result = tokenizer.feed(span::codes(codes, &span), func, enter.next == None); - - if let Some(ref x) = result.1 { - if !x.is_empty() { - // To do: handle? - unreachable!("subtokenize:remainder {:?}", x); - } - } - + assert!(result.1.is_none(), "expected no remainder"); index_opt = enter.next; } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index ba9bcbb..909a1d1 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -222,12 +222,14 @@ impl Tokenizer { self.current = code; } + /// To do. pub fn define_skip(&mut self, point: &Point, index: usize) { self.column_start.insert(point.line, point.column); self.account_for_potential_skip(); log::debug!("position: define skip: `{:?}` ({:?})", point, index); } + /// To do. fn account_for_potential_skip(&mut self) { match self.column_start.get(&self.point.line) { None => {} @@ -462,6 +464,7 @@ impl Tokenizer { } // To do: lifetimes, boxes, lmao. + /// To do. pub fn attempt_2( &mut self, a: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static, @@ -481,6 +484,7 @@ impl Tokenizer { ) } + /// To do. #[allow(clippy::too_many_arguments, clippy::many_single_char_names)] pub fn attempt_5( &mut self, @@ -504,6 +508,7 @@ impl Tokenizer { ) } + /// To do. #[allow(clippy::too_many_arguments, clippy::many_single_char_names)] pub fn attempt_7( &mut self, @@ -529,6 +534,7 @@ impl Tokenizer { ) } + /// To do. #[allow(clippy::too_many_arguments, clippy::many_single_char_names)] pub fn call_multiple( &mut self, @@ -606,7 +612,7 @@ impl Tokenizer { // Yield to a higher loop if we shouldn’t feed EOFs. if !drain { - return (state, Some(codes[index..].to_vec())); + return check_statefn_result((state, Some(codes[index..].to_vec()))); } loop { @@ -618,14 +624,7 @@ impl Tokenizer { log::debug!("main: passing eof"); self.expect(code); let (next, remainder) = check_statefn_result(func(self, code)); - - if let Some(ref x) = remainder { - if !x.is_empty() { - // To do: handle? - unreachable!("drain:remainder {:?}", x); - } - } - + assert!(remainder.is_none(), "expected no remainder"); state = next; } } @@ -661,8 +660,13 @@ fn attempt_impl( } } - // To do: `remainder` must never be bigger than codes I guess? - // To do: `remainder` probably has to be taken *from* `codes`, in a similar vain to the `Ok` handling below. + if let Some(ref list) = remainder { + assert!( + list.len() <= codes.len(), + "`remainder` must be less than or equal to `codes`" + ); + } + match next { State::Ok => { let remaining = if let Some(x) = remainder { x } else { vec![] }; @@ -670,6 +674,7 @@ fn attempt_impl( } State::Nok => check_statefn_result(done((codes, vec![]), false, tokenizer)), State::Fn(func) => { + assert!(remainder.is_none(), "expected no remainder"); check_statefn_result((State::Fn(attempt_impl(func, codes, done)), None)) } } @@ -712,20 +717,18 @@ pub fn as_codes(value: &str) -> Vec<Code> { } // Send a tab and virtual spaces. '\t' => { - // To do: is this correct? let remainder = column % TAB_SIZE; - let virtual_spaces = if remainder == 0 { + let mut virtual_spaces = if remainder == 0 { 0 } else { TAB_SIZE - remainder }; codes.push(Code::Char(char)); column += 1; - let mut index = 0; - while index < virtual_spaces { + while virtual_spaces > 0 { codes.push(Code::VirtualSpace); column += 1; - index += 1; + virtual_spaces -= 1; } } // Send an LF. @@ -770,6 +773,10 @@ fn check_statefn_result(result: StateFnResult) -> StateFnResult { if Some(&Code::None) == list.last() { list.pop(); } + + if list.is_empty() { + return (state, None); + } } (state, remainder) |