From 021d5f989ae41ae39a9b937b498141d9dc70d894 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Thu, 9 Jun 2022 15:01:46 +0200 Subject: Add basic subtokenization, string content in fenced code --- src/content/string.rs | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) (limited to 'src/content/string.rs') diff --git a/src/content/string.rs b/src/content/string.rs index 1239a36..64f544b 100644 --- a/src/content/string.rs +++ b/src/content/string.rs @@ -13,26 +13,14 @@ use crate::construct::{ character_escape::start as character_escape, character_reference::start as character_reference, }; -use crate::tokenizer::{Code, Event, State, StateFnResult, TokenType, Tokenizer}; +use crate::tokenizer::{Code, Event, Point, State, StateFnResult, TokenType, Tokenizer}; /// Turn `codes` as the string content type into events. // To do: remove this `allow` when all the content types are glued together. #[allow(dead_code)] -pub fn string(codes: &[Code]) -> Vec { - let mut tokenizer = Tokenizer::new(); - let (state, remainder) = tokenizer.feed(codes, Box::new(before), true); - - if let Some(ref x) = remainder { - if !x.is_empty() { - unreachable!("expected no final remainder {:?}", x); - } - } - - match state { - State::Ok => {} - _ => unreachable!("expected final state to be `State::Ok`"), - } - +pub fn string(codes: &[Code], point: Point, index: usize) -> Vec { + let mut tokenizer = Tokenizer::new(point, index); + tokenizer.feed(codes, Box::new(before), true); tokenizer.events } -- cgit