diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/construct/html_text.rs | 24 | ||||
| -rw-r--r-- | src/content/text.rs | 21 | ||||
| -rw-r--r-- | src/subtokenize.rs | 28 |
3 files changed, 47 insertions, 26 deletions
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index 95fb8c3..c118006 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -58,10 +58,14 @@ use crate::tokenizer::{Code, State, StateFn, StateFnResult, TokenType, Tokenizer /// a |<x> b /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::HtmlText); - tokenizer.enter(TokenType::HtmlTextData); - tokenizer.consume(code); - (State::Fn(Box::new(open)), None) + if Code::Char('<') == code { + tokenizer.enter(TokenType::HtmlText); + tokenizer.enter(TokenType::HtmlTextData); + tokenizer.consume(code); + (State::Fn(Box::new(open)), None) + } else { + (State::Nok, None) + } } /// After `<`, before a tag name or other stuff. @@ -582,9 +586,9 @@ pub fn tag_open_attribute_value_quoted( pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::Char('"' | '\'' | '<' | '=' | '`') => (State::Nok, None), - Code::CarriageReturnLineFeed | Code::VirtualSpace | Code::Char('\t' | ' ' | '/' | '>') => { - tag_open_between(tokenizer, code) - } + Code::CarriageReturnLineFeed + | Code::VirtualSpace + | Code::Char('\r' | '\n' | '\t' | ' ' | '/' | '>') => tag_open_between(tokenizer, code), Code::Char(_) => { tokenizer.consume(code); (State::Fn(Box::new(tag_open_attribute_value_unquoted)), None) @@ -603,9 +607,9 @@ pub fn tag_open_attribute_value_quoted_after( code: Code, ) -> StateFnResult { match code { - Code::CarriageReturnLineFeed | Code::VirtualSpace | Code::Char('\t' | ' ' | '>' | '/') => { - tag_open_between(tokenizer, code) - } + Code::CarriageReturnLineFeed + | Code::VirtualSpace + | Code::Char('\r' | '\n' | '\t' | ' ' | '>' | '/') => tag_open_between(tokenizer, code), _ => (State::Nok, None), } } diff --git a/src/content/text.rs b/src/content/text.rs index 73c2d55..433d030 100644 --- a/src/content/text.rs +++ b/src/content/text.rs @@ -52,12 +52,19 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// |qwe /// ``` fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - if let Code::None = code { - (State::Ok, None) - } else { - tokenizer.enter(TokenType::Data); - tokenizer.consume(code); - (State::Fn(Box::new(in_data)), None) + match code { + Code::None => (State::Ok, None), + Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => { + tokenizer.enter(TokenType::LineEnding); + tokenizer.consume(code); + tokenizer.exit(TokenType::LineEnding); + (State::Fn(Box::new(start)), None) + } + _ => { + tokenizer.enter(TokenType::Data); + tokenizer.consume(code); + (State::Fn(Box::new(in_data)), None) + } } } @@ -73,7 +80,7 @@ fn in_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { (State::Ok, None) } // To do: somehow get these markers from constructs. - Code::Char('&' | '\\' | '<') => { + Code::CarriageReturnLineFeed | Code::Char('\r' | '\n' | '&' | '\\' | '<') => { tokenizer.exit(TokenType::Data); start(tokenizer, code) } diff --git a/src/subtokenize.rs b/src/subtokenize.rs index 35d7672..71a84e1 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -82,12 +82,12 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) { let mut subindex = 0; // Index into subevents that starts the current slice. let mut last_start = 0; - // Counter into `ends`. + // Counter into `ends`: the linked token we are at. let mut end_index = 0; let mut index_opt: Option<usize> = Some(index); while subindex < tokenizer.events.len() { - let subevent = &tokenizer.events[subindex]; + let subevent = &mut tokenizer.events[subindex]; // Find the first event that starts after the end we’re looking // for. @@ -101,11 +101,26 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) { index_opt = events[link].next; } + // If there is a `next` link in the subevents, we have to change + // its index to account for the shifted events. + // If it points to a next event, we also change the next event’s + // reference back to *this* event. + if let Some(next) = subevent.next { + // The `index` in `events` where the current link is, + // minus 2 events (the enter and exit) for each removed + // link. + let shift = index_opt.unwrap() - (end_index * 2); + + subevent.next = Some(next + shift); + let next_ev = &mut tokenizer.events[next]; + let previous = next_ev.previous.unwrap(); + next_ev.previous = Some(previous + shift); + } + subindex += 1; } - let link = index_opt.unwrap(); - link_to_info.insert(link, (index, last_start, subindex)); + link_to_info.insert(index_opt.unwrap(), (index, last_start, subindex)); head_to_tokenizer.insert(index, tokenizer); } @@ -119,11 +134,6 @@ pub fn subtokenize(events: Vec<Event>, codes: &[Code]) -> (Vec<Event>, bool) { // from each slice and slices from events? let mut index = events.len() - 1; - // To do: this is broken, because it can inject linked events, which point - // to their links through indices, and this messes with all indices. - // We should try walking front to end instead, keep a count of the shifted - // index. - // It’s a bit complex but should work? while index > 0 { let slice_opt = link_to_info.get(&index); |