diff options
Diffstat (limited to '')
29 files changed, 878 insertions, 859 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs index 7e873ca..ac2ef25 100644 --- a/src/construct/attention.rs +++ b/src/construct/attention.rs @@ -51,9 +51,9 @@ //! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element //! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{Event, EventType, Point, Tokenizer}; +use crate::event::{Event, Kind, Name, Point}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; use crate::unicode::PUNCTUATION; use crate::util::slice::Slice; @@ -120,8 +120,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'*' | b'_') if tokenizer.parse_state.constructs.attention => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); - tokenizer.enter(Token::AttentionSequence); - State::Retry(Name::AttentionInside) + tokenizer.enter(Name::AttentionSequence); + State::Retry(StateName::AttentionInside) } _ => State::Nok, } @@ -137,10 +137,10 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'*' | b'_') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.consume(); - State::Next(Name::AttentionInside) + State::Next(StateName::AttentionInside) } _ => { - tokenizer.exit(Token::AttentionSequence); + tokenizer.exit(Name::AttentionSequence); tokenizer.register_resolver("attention".to_string(), Box::new(resolve_attention)); tokenizer.tokenize_state.marker = b'\0'; State::Ok @@ -159,10 +159,10 @@ fn resolve_attention(tokenizer: &mut Tokenizer) { while start < tokenizer.events.len() { let enter = &tokenizer.events[start]; - if enter.event_type == EventType::Enter { + if enter.kind == Kind::Enter { balance += 1; - if enter.token_type == Token::AttentionSequence { + if enter.name == Name::AttentionSequence { let end = start + 1; let exit = &tokenizer.events[end]; @@ -337,41 +337,41 @@ fn resolve_attention(tokenizer: &mut Tokenizer) { 0, vec![ Event { - event_type: EventType::Enter, - token_type: if take == 1 { - Token::Emphasis + kind: Kind::Enter, + name: if take == 1 { + Name::Emphasis } else { - Token::Strong + Name::Strong }, point: seq_open_enter.clone(), link: None, }, Event { - event_type: EventType::Enter, - token_type: if take == 1 { - Token::EmphasisSequence + kind: Kind::Enter, + name: if take == 1 { + Name::EmphasisSequence } else { - Token::StrongSequence + Name::StrongSequence }, point: seq_open_enter.clone(), link: None, }, Event { - event_type: EventType::Exit, - token_type: if take == 1 { - Token::EmphasisSequence + kind: Kind::Exit, + name: if take == 1 { + Name::EmphasisSequence } else { - Token::StrongSequence + Name::StrongSequence }, point: seq_open_exit.clone(), link: None, }, Event { - event_type: EventType::Enter, - token_type: if take == 1 { - Token::EmphasisText + kind: Kind::Enter, + name: if take == 1 { + Name::EmphasisText } else { - Token::StrongText + Name::StrongText }, point: seq_open_exit.clone(), link: None, @@ -384,41 +384,41 @@ fn resolve_attention(tokenizer: &mut Tokenizer) { 0, vec![ Event { - event_type: EventType::Exit, - token_type: if take == 1 { - Token::EmphasisText + kind: Kind::Exit, + name: if take == 1 { + Name::EmphasisText } else { - Token::StrongText + Name::StrongText }, point: seq_close_enter.clone(), link: None, }, Event { - event_type: EventType::Enter, - token_type: if take == 1 { - Token::EmphasisSequence + kind: Kind::Enter, + name: if take == 1 { + Name::EmphasisSequence } else { - Token::StrongSequence + Name::StrongSequence }, point: seq_close_enter.clone(), link: None, }, Event { - event_type: EventType::Exit, - token_type: if take == 1 { - Token::EmphasisSequence + kind: Kind::Exit, + name: if take == 1 { + Name::EmphasisSequence } else { - Token::StrongSequence + Name::StrongSequence }, point: seq_close_exit.clone(), link: None, }, Event { - event_type: EventType::Exit, - token_type: if take == 1 { - Token::Emphasis + kind: Kind::Exit, + name: if take == 1 { + Name::Emphasis } else { - Token::Strong + Name::Strong }, point: seq_close_exit.clone(), link: None, @@ -438,8 +438,8 @@ fn resolve_attention(tokenizer: &mut Tokenizer) { let mut index = 0; while index < sequences.len() { let sequence = &sequences[index]; - tokenizer.events[sequence.event_index].token_type = Token::Data; - tokenizer.events[sequence.event_index + 1].token_type = Token::Data; + tokenizer.events[sequence.event_index].name = Name::Data; + tokenizer.events[sequence.event_index + 1].name = Name::Data; index += 1; } diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index b635d96..c16b7c0 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -102,8 +102,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX}; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of an autolink. @@ -117,12 +117,12 @@ use crate::tokenizer::Tokenizer; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'<') if tokenizer.parse_state.constructs.autolink => { - tokenizer.enter(Token::Autolink); - tokenizer.enter(Token::AutolinkMarker); + tokenizer.enter(Name::Autolink); + tokenizer.enter(Name::AutolinkMarker); tokenizer.consume(); - tokenizer.exit(Token::AutolinkMarker); - tokenizer.enter(Token::AutolinkProtocol); - State::Next(Name::AutolinkOpen) + tokenizer.exit(Name::AutolinkMarker); + tokenizer.enter(Name::AutolinkProtocol); + State::Next(StateName::AutolinkOpen) } _ => State::Nok, } @@ -141,9 +141,9 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { // ASCII alphabetic. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::AutolinkSchemeOrEmailAtext) + State::Next(StateName::AutolinkSchemeOrEmailAtext) } - _ => State::Retry(Name::AutolinkEmailAtext), + _ => State::Retry(StateName::AutolinkEmailAtext), } } @@ -161,9 +161,9 @@ pub fn scheme_or_email_atext(tokenizer: &mut Tokenizer) -> State { Some(b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { // Count the previous alphabetical from `open` too. tokenizer.tokenize_state.size = 1; - State::Retry(Name::AutolinkSchemeInsideOrEmailAtext) + State::Retry(StateName::AutolinkSchemeInsideOrEmailAtext) } - _ => State::Retry(Name::AutolinkEmailAtext), + _ => State::Retry(StateName::AutolinkEmailAtext), } } @@ -180,7 +180,7 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State { Some(b':') => { tokenizer.consume(); tokenizer.tokenize_state.size = 0; - State::Next(Name::AutolinkUrlInside) + State::Next(StateName::AutolinkUrlInside) } // ASCII alphanumeric and `+`, `-`, and `.`. Some(b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') @@ -188,11 +188,11 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State { { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(Name::AutolinkSchemeInsideOrEmailAtext) + State::Next(StateName::AutolinkSchemeInsideOrEmailAtext) } _ => { tokenizer.tokenize_state.size = 0; - State::Retry(Name::AutolinkEmailAtext) + State::Retry(StateName::AutolinkEmailAtext) } } } @@ -206,18 +206,18 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State { pub fn url_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { - tokenizer.exit(Token::AutolinkProtocol); - tokenizer.enter(Token::AutolinkMarker); + tokenizer.exit(Name::AutolinkProtocol); + tokenizer.enter(Name::AutolinkMarker); tokenizer.consume(); - tokenizer.exit(Token::AutolinkMarker); - tokenizer.exit(Token::Autolink); + tokenizer.exit(Name::AutolinkMarker); + tokenizer.exit(Name::Autolink); State::Ok } // ASCII control, space, or `<`. None | Some(b'\0'..=0x1F | b' ' | b'<' | 0x7F) => State::Nok, Some(_) => { tokenizer.consume(); - State::Next(Name::AutolinkUrlInside) + State::Next(StateName::AutolinkUrlInside) } } } @@ -232,7 +232,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'@') => { tokenizer.consume(); - State::Next(Name::AutolinkEmailAtSignOrDot) + State::Next(StateName::AutolinkEmailAtSignOrDot) } // ASCII atext. // @@ -255,7 +255,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State { b'#'..=b'\'' | b'*' | b'+' | b'-'..=b'9' | b'=' | b'?' | b'A'..=b'Z' | b'^'..=b'~', ) => { tokenizer.consume(); - State::Next(Name::AutolinkEmailAtext) + State::Next(StateName::AutolinkEmailAtext) } _ => State::Nok, } @@ -270,7 +270,9 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State { pub fn email_at_sign_or_dot(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // ASCII alphanumeric. - Some(b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => State::Retry(Name::AutolinkEmailValue), + Some(b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { + State::Retry(StateName::AutolinkEmailValue) + } _ => State::Nok, } } @@ -286,22 +288,22 @@ pub fn email_label(tokenizer: &mut Tokenizer) -> State { Some(b'.') => { tokenizer.tokenize_state.size = 0; tokenizer.consume(); - State::Next(Name::AutolinkEmailAtSignOrDot) + State::Next(StateName::AutolinkEmailAtSignOrDot) } Some(b'>') => { tokenizer.tokenize_state.size = 0; let index = tokenizer.events.len(); - tokenizer.exit(Token::AutolinkProtocol); + tokenizer.exit(Name::AutolinkProtocol); // Change the token type. - tokenizer.events[index - 1].token_type = Token::AutolinkEmail; - tokenizer.events[index].token_type = Token::AutolinkEmail; - tokenizer.enter(Token::AutolinkMarker); + tokenizer.events[index - 1].name = Name::AutolinkEmail; + tokenizer.events[index].name = Name::AutolinkEmail; + tokenizer.enter(Name::AutolinkMarker); tokenizer.consume(); - tokenizer.exit(Token::AutolinkMarker); - tokenizer.exit(Token::Autolink); + tokenizer.exit(Name::AutolinkMarker); + tokenizer.exit(Name::Autolink); State::Ok } - _ => State::Retry(Name::AutolinkEmailValue), + _ => State::Retry(StateName::AutolinkEmailValue), } } @@ -320,9 +322,9 @@ pub fn email_value(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.size < AUTOLINK_DOMAIN_SIZE_MAX => { let name = if matches!(tokenizer.current, Some(b'-')) { - Name::AutolinkEmailValue + StateName::AutolinkEmailValue } else { - Name::AutolinkEmailLabel + StateName::AutolinkEmailLabel }; tokenizer.tokenize_state.size += 1; tokenizer.consume(); diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index e8a06e9..2ea986d 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -33,7 +33,7 @@ //! [flow]: crate::content::flow use crate::construct::partial_space_or_tab::space_or_tab; -use crate::state::{Name, State}; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of a blank line. @@ -50,8 +50,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::BlankLineAfter), - State::Next(Name::BlankLineAfter), + State::Next(StateName::BlankLineAfter), + State::Next(StateName::BlankLineAfter), ) } diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs index 7b8ce82..a32375e 100644 --- a/src/construct/block_quote.rs +++ b/src/construct/block_quote.rs @@ -35,8 +35,8 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::space_or_tab_min_max; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of block quote. @@ -56,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(Name::BlockQuoteBefore), State::Nok) + tokenizer.attempt(name, State::Next(StateName::BlockQuoteBefore), State::Nok) } else { State::Nok } @@ -71,10 +71,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { - tokenizer.enter(Token::BlockQuote); - State::Retry(Name::BlockQuoteContBefore) + tokenizer.enter(Name::BlockQuote); + State::Retry(StateName::BlockQuoteContBefore) } - _ => State::Retry(Name::BlockQuoteContBefore), + _ => State::Retry(StateName::BlockQuoteContBefore), } } @@ -95,7 +95,11 @@ pub fn cont_start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(Name::BlockQuoteContBefore), State::Nok) + tokenizer.attempt( + name, + State::Next(StateName::BlockQuoteContBefore), + State::Nok, + ) } /// After whitespace, before `>`. @@ -108,11 +112,11 @@ pub fn cont_start(tokenizer: &mut Tokenizer) -> State { pub fn cont_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { - tokenizer.enter(Token::BlockQuotePrefix); - tokenizer.enter(Token::BlockQuoteMarker); + tokenizer.enter(Name::BlockQuotePrefix); + tokenizer.enter(Name::BlockQuoteMarker); tokenizer.consume(); - tokenizer.exit(Token::BlockQuoteMarker); - State::Next(Name::BlockQuoteContAfter) + tokenizer.exit(Name::BlockQuoteMarker); + State::Next(StateName::BlockQuoteContAfter) } _ => State::Nok, } @@ -128,11 +132,11 @@ pub fn cont_before(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn cont_after(tokenizer: &mut Tokenizer) -> State { if let Some(b'\t' | b' ') = tokenizer.current { - tokenizer.enter(Token::SpaceOrTab); + tokenizer.enter(Name::SpaceOrTab); tokenizer.consume(); - tokenizer.exit(Token::SpaceOrTab); + tokenizer.exit(Name::SpaceOrTab); } - tokenizer.exit(Token::BlockQuotePrefix); + tokenizer.exit(Name::BlockQuotePrefix); State::Ok } diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index c3d5458..0cd7126 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -33,8 +33,8 @@ //! [character_reference]: crate::construct::character_reference //! [hard_break_escape]: crate::construct::hard_break_escape -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of a character escape. @@ -46,11 +46,11 @@ use crate::tokenizer::Tokenizer; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\\') if tokenizer.parse_state.constructs.character_escape => { - tokenizer.enter(Token::CharacterEscape); - tokenizer.enter(Token::CharacterEscapeMarker); + tokenizer.enter(Name::CharacterEscape); + tokenizer.enter(Name::CharacterEscapeMarker); tokenizer.consume(); - tokenizer.exit(Token::CharacterEscapeMarker); - State::Next(Name::CharacterEscapeInside) + tokenizer.exit(Name::CharacterEscapeMarker); + State::Next(StateName::CharacterEscapeInside) } _ => State::Nok, } @@ -62,15 +62,14 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// > | a\*b /// ^ /// ``` -// Name::CharacterEscapeInside pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // ASCII punctuation. Some(b'!'..=b'/' | b':'..=b'@' | b'['..=b'`' | b'{'..=b'~') => { - tokenizer.enter(Token::CharacterEscapeValue); + tokenizer.enter(Name::CharacterEscapeValue); tokenizer.consume(); - tokenizer.exit(Token::CharacterEscapeValue); - tokenizer.exit(Token::CharacterEscape); + tokenizer.exit(Name::CharacterEscapeValue); + tokenizer.exit(Name::CharacterEscape); State::Ok } _ => State::Nok, diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index 435c115..0158acf 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -65,8 +65,8 @@ use crate::constant::{ CHARACTER_REFERENCES, CHARACTER_REFERENCE_DECIMAL_SIZE_MAX, CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX, CHARACTER_REFERENCE_NAMED_SIZE_MAX, }; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; use crate::util::slice::Slice; @@ -83,11 +83,11 @@ use crate::util::slice::Slice; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'&') if tokenizer.parse_state.constructs.character_reference => { - tokenizer.enter(Token::CharacterReference); - tokenizer.enter(Token::CharacterReferenceMarker); + tokenizer.enter(Name::CharacterReference); + tokenizer.enter(Name::CharacterReferenceMarker); tokenizer.consume(); - tokenizer.exit(Token::CharacterReferenceMarker); - State::Next(Name::CharacterReferenceOpen) + tokenizer.exit(Name::CharacterReferenceMarker); + State::Next(StateName::CharacterReferenceOpen) } _ => State::Nok, } @@ -104,17 +104,16 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// > | a	b /// ^ /// ``` -// Name::CharacterReferenceOpen pub fn open(tokenizer: &mut Tokenizer) -> State { if let Some(b'#') = tokenizer.current { - tokenizer.enter(Token::CharacterReferenceMarkerNumeric); + tokenizer.enter(Name::CharacterReferenceMarkerNumeric); tokenizer.consume(); - tokenizer.exit(Token::CharacterReferenceMarkerNumeric); - State::Next(Name::CharacterReferenceNumeric) + tokenizer.exit(Name::CharacterReferenceMarkerNumeric); + State::Next(StateName::CharacterReferenceNumeric) } else { tokenizer.tokenize_state.marker = b'&'; - tokenizer.enter(Token::CharacterReferenceValue); - State::Retry(Name::CharacterReferenceValue) + tokenizer.enter(Name::CharacterReferenceValue); + State::Retry(StateName::CharacterReferenceValue) } } @@ -127,19 +126,18 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { /// > | a	b /// ^ /// ``` -// Name::CharacterReferenceNumeric pub fn numeric(tokenizer: &mut Tokenizer) -> State { if let Some(b'x' | b'X') = tokenizer.current { - tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(Name::CharacterReferenceMarkerHexadecimal); tokenizer.consume(); - tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal); - tokenizer.enter(Token::CharacterReferenceValue); + tokenizer.exit(Name::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(Name::CharacterReferenceValue); tokenizer.tokenize_state.marker = b'x'; - State::Next(Name::CharacterReferenceValue) + State::Next(StateName::CharacterReferenceValue) } else { - tokenizer.enter(Token::CharacterReferenceValue); + tokenizer.enter(Name::CharacterReferenceValue); tokenizer.tokenize_state.marker = b'#'; - State::Retry(Name::CharacterReferenceValue) + State::Retry(StateName::CharacterReferenceValue) } } @@ -176,11 +174,11 @@ pub fn value(tokenizer: &mut Tokenizer) -> State { } } - tokenizer.exit(Token::CharacterReferenceValue); - tokenizer.enter(Token::CharacterReferenceMarkerSemi); + tokenizer.exit(Name::CharacterReferenceValue); + tokenizer.enter(Name::CharacterReferenceMarkerSemi); tokenizer.consume(); - tokenizer.exit(Token::CharacterReferenceMarkerSemi); - tokenizer.exit(Token::CharacterReference); + tokenizer.exit(Name::CharacterReferenceMarkerSemi); + tokenizer.exit(Name::CharacterReference); tokenizer.tokenize_state.marker = 0; tokenizer.tokenize_state.size = 0; return State::Ok; @@ -203,7 +201,7 @@ pub fn value(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.size < max && test(&byte) { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - return State::Next(Name::CharacterReferenceValue); + return State::Next(StateName::CharacterReferenceValue); } } diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index 0ce8d02..9488f62 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -103,9 +103,9 @@ use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE}; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{ContentType, Tokenizer}; +use crate::event::{Content, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; use crate::util::slice::{Position, Slice}; /// Start of fenced code. @@ -118,8 +118,8 @@ use crate::util::slice::{Position, Slice}; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.parse_state.constructs.code_fenced { - tokenizer.enter(Token::CodeFenced); - tokenizer.enter(Token::CodeFencedFence); + tokenizer.enter(Name::CodeFenced); + tokenizer.enter(Name::CodeFencedFence); let name = space_or_tab_min_max( tokenizer, 0, @@ -131,7 +131,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(Name::CodeFencedBeforeSequenceOpen), + State::Next(StateName::CodeFencedBeforeSequenceOpen), State::Nok, ) } else { @@ -152,7 +152,7 @@ pub fn before_sequence_open(tokenizer: &mut Tokenizer) -> State { let mut prefix = 0; if let Some(event) = tail { - if event.token_type == Token::SpaceOrTab { + if event.name == Name::SpaceOrTab { prefix = Slice::from_position( tokenizer.parse_state.bytes, &Position::from_exit_event(&tokenizer.events, tokenizer.events.len() - 1), @@ -164,8 +164,8 @@ pub fn before_sequence_open(tokenizer: &mut Tokenizer) -> State { if let Some(b'`' | b'~') = tokenizer.current { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); tokenizer.tokenize_state.size_c = prefix; - tokenizer.enter(Token::CodeFencedFenceSequence); - State::Retry(Name::CodeFencedSequenceOpen) + tokenizer.enter(Name::CodeFencedFenceSequence); + State::Retry(StateName::CodeFencedSequenceOpen) } else { State::Nok } @@ -184,15 +184,15 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(Name::CodeFencedSequenceOpen) + State::Next(StateName::CodeFencedSequenceOpen) } _ if tokenizer.tokenize_state.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => { - tokenizer.exit(Token::CodeFencedFenceSequence); + tokenizer.exit(Name::CodeFencedFenceSequence); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::CodeFencedInfoBefore), - State::Next(Name::CodeFencedInfoBefore), + State::Next(StateName::CodeFencedInfoBefore), + State::Next(StateName::CodeFencedInfoBefore), ) } _ => { @@ -215,19 +215,19 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { pub fn info_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::CodeFencedFence); + tokenizer.exit(Name::CodeFencedFence); // Do not form containers. tokenizer.concrete = true; tokenizer.check( - Name::NonLazyContinuationStart, - State::Next(Name::CodeFencedAtNonLazyBreak), - State::Next(Name::CodeFencedAfter), + StateName::NonLazyContinuationStart, + State::Next(StateName::CodeFencedAtNonLazyBreak), + State::Next(StateName::CodeFencedAfter), ) } _ => { - tokenizer.enter(Token::CodeFencedFenceInfo); - tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(Name::CodeFencedInfo) + tokenizer.enter(Name::CodeFencedFenceInfo); + tokenizer.enter_with_content(Name::Data, Some(Content::String)); + State::Retry(StateName::CodeFencedInfo) } } } @@ -243,18 +243,18 @@ pub fn info_before(tokenizer: &mut Tokenizer) -> State { pub fn info(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::Data); - tokenizer.exit(Token::CodeFencedFenceInfo); - State::Retry(Name::CodeFencedInfoBefore) + tokenizer.exit(Name::Data); + tokenizer.exit(Name::CodeFencedFenceInfo); + State::Retry(StateName::CodeFencedInfoBefore) } Some(b'\t' | b' ') => { - tokenizer.exit(Token::Data); - tokenizer.exit(Token::CodeFencedFenceInfo); + tokenizer.exit(Name::Data); + tokenizer.exit(Name::CodeFencedFenceInfo); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::CodeFencedMetaBefore), - State::Next(Name::CodeFencedMetaBefore), + State::Next(StateName::CodeFencedMetaBefore), + State::Next(StateName::CodeFencedMetaBefore), ) } Some(b'`') if tokenizer.tokenize_state.marker == b'`' => { @@ -266,7 +266,7 @@ pub fn info(tokenizer: &mut Tokenizer) -> State { } Some(_) => { tokenizer.consume(); - State::Next(Name::CodeFencedInfo) + State::Next(StateName::CodeFencedInfo) } } } @@ -281,11 +281,11 @@ pub fn info(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn meta_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None | Some(b'\n') => State::Retry(Name::CodeFencedInfoBefore), + None | Some(b'\n') => State::Retry(StateName::CodeFencedInfoBefore), _ => { - tokenizer.enter(Token::CodeFencedFenceMeta); - tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(Name::CodeFencedMeta) + tokenizer.enter(Name::CodeFencedFenceMeta); + tokenizer.enter_with_content(Name::Data, Some(Content::String)); + State::Retry(StateName::CodeFencedMeta) } } } @@ -301,9 +301,9 @@ pub fn meta_before(tokenizer: &mut Tokenizer) -> State { pub fn meta(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::Data); - tokenizer.exit(Token::CodeFencedFenceMeta); - State::Retry(Name::CodeFencedInfoBefore) + tokenizer.exit(Name::Data); + tokenizer.exit(Name::CodeFencedFenceMeta); + State::Retry(StateName::CodeFencedInfoBefore) } Some(b'`') if tokenizer.tokenize_state.marker == b'`' => { tokenizer.concrete = false; @@ -314,7 +314,7 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.consume(); - State::Next(Name::CodeFencedMeta) + State::Next(StateName::CodeFencedMeta) } } } @@ -330,9 +330,9 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn at_non_lazy_break(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - Name::CodeFencedCloseBefore, - State::Next(Name::CodeFencedAfter), - State::Next(Name::CodeFencedContentBefore), + StateName::CodeFencedCloseBefore, + State::Next(StateName::CodeFencedAfter), + State::Next(StateName::CodeFencedContentBefore), ) } @@ -347,10 +347,10 @@ pub fn at_non_lazy_break(tokenizer: &mut Tokenizer) -> State { pub fn close_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::CodeFencedCloseStart) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::CodeFencedCloseStart) } _ => unreachable!("expected eol"), } @@ -365,7 +365,7 @@ pub fn close_before(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn close_start(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::CodeFencedFence); + tokenizer.enter(Name::CodeFencedFence); let name = space_or_tab_min_max( tokenizer, 0, @@ -377,7 +377,7 @@ pub fn close_start(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(Name::CodeFencedBeforeSequenceClose), + State::Next(StateName::CodeFencedBeforeSequenceClose), State::Nok, ) } @@ -393,8 +393,8 @@ pub fn close_start(tokenizer: &mut Tokenizer) -> State { pub fn before_sequence_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { - tokenizer.enter(Token::CodeFencedFenceSequence); - State::Retry(Name::CodeFencedSequenceClose) + tokenizer.enter(Name::CodeFencedFenceSequence); + State::Retry(StateName::CodeFencedSequenceClose) } _ => State::Nok, } @@ -413,18 +413,18 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.tokenize_state.size_b += 1; tokenizer.consume(); - State::Next(Name::CodeFencedSequenceClose) + State::Next(StateName::CodeFencedSequenceClose) } _ if tokenizer.tokenize_state.size_b >= CODE_FENCED_SEQUENCE_SIZE_MIN && tokenizer.tokenize_state.size_b >= tokenizer.tokenize_state.size => { tokenizer.tokenize_state.size_b = 0; - tokenizer.exit(Token::CodeFencedFenceSequence); + tokenizer.exit(Name::CodeFencedFenceSequence); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::CodeFencedAfterSequenceClose), - State::Next(Name::CodeFencedAfterSequenceClose), + State::Next(StateName::CodeFencedAfterSequenceClose), + State::Next(StateName::CodeFencedAfterSequenceClose), ) } _ => { @@ -445,7 +445,7 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { pub fn sequence_close_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::CodeFencedFence); + tokenizer.exit(Name::CodeFencedFence); State::Ok } _ => State::Nok, @@ -461,10 +461,10 @@ pub fn sequence_close_after(tokenizer: &mut Tokenizer) -> State { /// | ~~~ /// ``` pub fn content_before(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::CodeFencedContentStart) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::CodeFencedContentStart) } /// Before code content, definitely not before a closing fence. /// @@ -478,7 +478,7 @@ pub fn content_start(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_min_max(tokenizer, 0, tokenizer.tokenize_state.size_c); tokenizer.attempt( name, - State::Next(Name::CodeFencedBeforeContentChunk), + State::Next(StateName::CodeFencedBeforeContentChunk), State::Nok, ) } @@ -494,13 +494,13 @@ pub fn content_start(tokenizer: &mut Tokenizer) -> State { pub fn before_content_chunk(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => tokenizer.check( - Name::NonLazyContinuationStart, - State::Next(Name::CodeFencedAtNonLazyBreak), - State::Next(Name::CodeFencedAfter), + StateName::NonLazyContinuationStart, + State::Next(StateName::CodeFencedAtNonLazyBreak), + State::Next(StateName::CodeFencedAfter), ), _ => { - tokenizer.enter(Token::CodeFlowChunk); - State::Retry(Name::CodeFencedContentChunk) + tokenizer.enter(Name::CodeFlowChunk); + State::Retry(StateName::CodeFencedContentChunk) } } } @@ -516,12 +516,12 @@ pub fn before_content_chunk(tokenizer: &mut Tokenizer) -> State { pub fn content_chunk(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::CodeFlowChunk); - State::Retry(Name::CodeFencedBeforeContentChunk) + tokenizer.exit(Name::CodeFlowChunk); + State::Retry(StateName::CodeFencedBeforeContentChunk) } _ => { tokenizer.consume(); - State::Next(Name::CodeFencedContentChunk) + State::Next(StateName::CodeFencedContentChunk) } } } @@ -535,7 +535,7 @@ pub fn content_chunk(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn after(tokenizer: &mut Tokenizer) -> State { - tokenizer.exit(Token::CodeFenced); + tokenizer.exit(Name::CodeFenced); tokenizer.tokenize_state.marker = 0; tokenizer.tokenize_state.size_c = 0; tokenizer.tokenize_state.size = 0; diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index f442f27..7297759 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -47,8 +47,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::TAB_SIZE; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of code (indented). @@ -64,9 +64,13 @@ use crate::tokenizer::Tokenizer; pub fn start(tokenizer: &mut Tokenizer) -> State { // Do not interrupt paragraphs. if !tokenizer.interrupt && tokenizer.parse_state.constructs.code_indented { - tokenizer.enter(Token::CodeIndented); + tokenizer.enter(Name::CodeIndented); let name = space_or_tab_min_max(tokenizer, TAB_SIZE, TAB_SIZE); - tokenizer.attempt(name, State::Next(Name::CodeIndentedAtBreak), State::Nok) + tokenizer.attempt( + name, + State::Next(StateName::CodeIndentedAtBreak), + State::Nok, + ) } else { State::Nok } @@ -80,15 +84,15 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn at_break(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None => State::Retry(Name::CodeIndentedAfter), + None => State::Retry(StateName::CodeIndentedAfter), Some(b'\n') => tokenizer.attempt( - Name::CodeIndentedFurtherStart, - State::Next(Name::CodeIndentedAtBreak), - State::Next(Name::CodeIndentedAfter), + StateName::CodeIndentedFurtherStart, + State::Next(StateName::CodeIndentedAtBreak), + State::Next(StateName::CodeIndentedAfter), ), _ => { - tokenizer.enter(Token::CodeFlowChunk); - State::Retry(Name::CodeIndentedInside) + tokenizer.enter(Name::CodeFlowChunk); + State::Retry(StateName::CodeIndentedInside) } } } @@ -102,12 +106,12 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::CodeFlowChunk); - State::Retry(Name::CodeIndentedAtBreak) + tokenizer.exit(Name::CodeFlowChunk); + State::Retry(StateName::CodeIndentedAtBreak) } _ => { tokenizer.consume(); - State::Next(Name::CodeIndentedInside) + State::Next(StateName::CodeIndentedInside) } } } @@ -119,7 +123,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn after(tokenizer: &mut Tokenizer) -> State { - tokenizer.exit(Token::CodeIndented); + tokenizer.exit(Name::CodeIndented); // Feel free to interrupt. tokenizer.interrupt = false; State::Ok @@ -135,17 +139,17 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { pub fn further_start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') if !tokenizer.lazy => { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::CodeIndentedFurtherStart) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::CodeIndentedFurtherStart) } _ if !tokenizer.lazy => { let name = space_or_tab_min_max(tokenizer, TAB_SIZE, TAB_SIZE); tokenizer.attempt( name, - State::Next(Name::CodeIndentedFurtherEnd), - State::Next(Name::CodeIndentedFurtherBegin), + State::Next(StateName::CodeIndentedFurtherEnd), + State::Next(StateName::CodeIndentedFurtherBegin), ) } _ => State::Nok, @@ -174,8 +178,8 @@ pub fn further_begin(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::CodeIndentedFurtherAfter), - State::Next(Name::CodeIndentedFurtherAfter), + State::Next(StateName::CodeIndentedFurtherAfter), + State::Next(StateName::CodeIndentedFurtherAfter), ) } @@ -188,7 +192,7 @@ pub fn further_begin(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn further_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'\n') => State::Retry(Name::CodeIndentedFurtherStart), + Some(b'\n') => State::Retry(StateName::CodeIndentedFurtherStart), _ => State::Nok, } } diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index f626cfb..f48f63c 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -83,8 +83,8 @@ //! [code_fenced]: crate::construct::code_fenced //! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of code (text). @@ -101,12 +101,12 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.parse_state.constructs.code_text && (tokenizer.previous != Some(b'`') || (!tokenizer.events.is_empty() - && tokenizer.events[tokenizer.events.len() - 1].token_type - == Token::CharacterEscape)) => + && tokenizer.events[tokenizer.events.len() - 1].name + == Name::CharacterEscape)) => { - tokenizer.enter(Token::CodeText); - tokenizer.enter(Token::CodeTextSequence); - State::Retry(Name::CodeTextSequenceOpen) + tokenizer.enter(Name::CodeText); + tokenizer.enter(Name::CodeTextSequence); + State::Retry(StateName::CodeTextSequenceOpen) } _ => State::Nok, } @@ -122,10 +122,10 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { if let Some(b'`') = tokenizer.current { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(Name::CodeTextSequenceOpen) + State::Next(StateName::CodeTextSequenceOpen) } else { - tokenizer.exit(Token::CodeTextSequence); - State::Retry(Name::CodeTextBetween) + tokenizer.exit(Name::CodeTextSequence); + State::Retry(StateName::CodeTextBetween) } } @@ -142,18 +142,18 @@ pub fn between(tokenizer: &mut Tokenizer) -> State { State::Nok } Some(b'\n') => { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::CodeTextBetween) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::CodeTextBetween) } Some(b'`') => { - tokenizer.enter(Token::CodeTextSequence); - State::Retry(Name::CodeTextSequenceClose) + tokenizer.enter(Name::CodeTextSequence); + State::Retry(StateName::CodeTextSequenceClose) } _ => { - tokenizer.enter(Token::CodeTextData); - State::Retry(Name::CodeTextData) + tokenizer.enter(Name::CodeTextData); + State::Retry(StateName::CodeTextData) } } } @@ -167,12 +167,12 @@ pub fn between(tokenizer: &mut Tokenizer) -> State { pub fn data(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n' | b'`') => { - tokenizer.exit(Token::CodeTextData); - State::Retry(Name::CodeTextBetween) + tokenizer.exit(Name::CodeTextData); + State::Retry(StateName::CodeTextBetween) } _ => { tokenizer.consume(); - State::Next(Name::CodeTextData) + State::Next(StateName::CodeTextData) } } } @@ -188,23 +188,23 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { Some(b'`') => { tokenizer.tokenize_state.size_b += 1; tokenizer.consume(); - State::Next(Name::CodeTextSequenceClose) + State::Next(StateName::CodeTextSequenceClose) } _ => { if tokenizer.tokenize_state.size == tokenizer.tokenize_state.size_b { - tokenizer.exit(Token::CodeTextSequence); - tokenizer.exit(Token::CodeText); + tokenizer.exit(Name::CodeTextSequence); + tokenizer.exit(Name::CodeText); tokenizer.tokenize_state.size = 0; tokenizer.tokenize_state.size_b = 0; State::Ok } else { let index = tokenizer.events.len(); - tokenizer.exit(Token::CodeTextSequence); + tokenizer.exit(Name::CodeTextSequence); // More or less accents: mark as data. - tokenizer.events[index - 1].token_type = Token::CodeTextData; - tokenizer.events[index].token_type = Token::CodeTextData; + tokenizer.events[index - 1].name = Name::CodeTextData; + tokenizer.events[index].name = Name::CodeTextData; tokenizer.tokenize_state.size_b = 0; - State::Retry(Name::CodeTextBetween) + State::Retry(StateName::CodeTextBetween) } } } diff --git a/src/construct/definition.rs b/src/construct/definition.rs index 394375f..2533a1c 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -94,8 +94,8 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_eol}; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; use crate::util::skip::opt_back as skip_opt_back; @@ -112,19 +112,19 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { && tokenizer.events[skip_opt_back( &tokenizer.events, tokenizer.events.len() - 1, - &[Token::LineEnding, Token::SpaceOrTab], + &[Name::LineEnding, Name::SpaceOrTab], )] - .token_type - == Token::Definition); + .name + == Name::Definition); if possible && tokenizer.parse_state.constructs.definition { - tokenizer.enter(Token::Definition); + tokenizer.enter(Name::Definition); // Note: arbitrary whitespace allowed even if code (indented) is on. let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::DefinitionBefore), - State::Next(Name::DefinitionBefore), + State::Next(StateName::DefinitionBefore), + State::Next(StateName::DefinitionBefore), ) } else { State::Nok @@ -140,12 +140,12 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'[') => { - tokenizer.tokenize_state.token_1 = Token::DefinitionLabel; - tokenizer.tokenize_state.token_2 = Token::DefinitionLabelMarker; - tokenizer.tokenize_state.token_3 = Token::DefinitionLabelString; + tokenizer.tokenize_state.token_1 = Name::DefinitionLabel; + tokenizer.tokenize_state.token_2 = Name::DefinitionLabelMarker; + tokenizer.tokenize_state.token_3 = Name::DefinitionLabelString; tokenizer.attempt( - Name::LabelStart, - State::Next(Name::DefinitionLabelAfter), + StateName::LabelStart, + State::Next(StateName::DefinitionLabelAfter), State::Nok, ) } @@ -160,16 +160,16 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn label_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; match tokenizer.current { Some(b':') => { - tokenizer.enter(Token::DefinitionMarker); + tokenizer.enter(Name::DefinitionMarker); tokenizer.consume(); - tokenizer.exit(Token::DefinitionMarker); - State::Next(Name::DefinitionMarkerAfter) + tokenizer.exit(Name::DefinitionMarker); + State::Next(StateName::DefinitionMarkerAfter) } _ => State::Nok, } @@ -185,8 +185,8 @@ pub fn marker_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(Name::DefinitionDestinationBefore), - State::Next(Name::DefinitionDestinationBefore), + State::Next(StateName::DefinitionDestinationBefore), + State::Next(StateName::DefinitionDestinationBefore), ) } @@ -197,16 +197,16 @@ pub fn marker_after(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn destination_before(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::DefinitionDestination; - tokenizer.tokenize_state.token_2 = Token::DefinitionDestinationLiteral; - tokenizer.tokenize_state.token_3 = Token::DefinitionDestinationLiteralMarker; - tokenizer.tokenize_state.token_4 = Token::DefinitionDestinationRaw; - tokenizer.tokenize_state.token_5 = Token::DefinitionDestinationString; + tokenizer.tokenize_state.token_1 = Name::DefinitionDestination; + tokenizer.tokenize_state.token_2 = Name::DefinitionDestinationLiteral; + tokenizer.tokenize_state.token_3 = Name::DefinitionDestinationLiteralMarker; + tokenizer.tokenize_state.token_4 = Name::DefinitionDestinationRaw; + tokenizer.tokenize_state.token_5 = Name::DefinitionDestinationString; tokenizer.tokenize_state.size_b = usize::MAX; tokenizer.attempt( - Name::DestinationStart, - State::Next(Name::DefinitionDestinationAfter), - State::Next(Name::DefinitionDestinationMissing), + StateName::DestinationStart, + State::Next(StateName::DefinitionDestinationAfter), + State::Next(StateName::DefinitionDestinationMissing), ) } @@ -217,26 +217,26 @@ pub fn destination_before(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn destination_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; - tokenizer.tokenize_state.token_4 = Token::Data; - tokenizer.tokenize_state.token_5 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; + tokenizer.tokenize_state.token_4 = Name::Data; + tokenizer.tokenize_state.token_5 = Name::Data; tokenizer.tokenize_state.size_b = 0; tokenizer.attempt( - Name::DefinitionTitleBefore, - State::Next(Name::DefinitionAfter), - State::Next(Name::DefinitionAfter), + StateName::DefinitionTitleBefore, + State::Next(StateName::DefinitionAfter), + State::Next(StateName::DefinitionAfter), ) } /// Without destination. pub fn destination_missing(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; - tokenizer.tokenize_state.token_4 = Token::Data; - tokenizer.tokenize_state.token_5 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; + tokenizer.tokenize_state.token_4 = Name::Data; + tokenizer.tokenize_state.token_5 = Name::Data; tokenizer.tokenize_state.size_b = 0; State::Nok } @@ -253,8 +253,8 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::DefinitionAfterWhitespace), - State::Next(Name::DefinitionAfterWhitespace), + State::Next(StateName::DefinitionAfterWhitespace), + State::Next(StateName::DefinitionAfterWhitespace), ) } @@ -269,7 +269,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { pub fn after_whitespace(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::Definition); + tokenizer.exit(Name::Definition); // You’d be interrupting. tokenizer.interrupt = true; State::Ok @@ -290,7 +290,7 @@ pub fn title_before(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(Name::DefinitionTitleBeforeMarker), + State::Next(StateName::DefinitionTitleBeforeMarker), State::Nok, ) } @@ -303,12 +303,12 @@ pub fn title_before(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn title_before_marker(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::DefinitionTitle; - tokenizer.tokenize_state.token_2 = Token::DefinitionTitleMarker; - tokenizer.tokenize_state.token_3 = Token::DefinitionTitleString; + tokenizer.tokenize_state.token_1 = Name::DefinitionTitle; + tokenizer.tokenize_state.token_2 = Name::DefinitionTitleMarker; + tokenizer.tokenize_state.token_3 = Name::DefinitionTitleString; tokenizer.attempt( - Name::TitleStart, - State::Next(Name::DefinitionTitleAfter), + StateName::TitleStart, + State::Next(StateName::DefinitionTitleAfter), State::Nok, ) } @@ -320,14 +320,14 @@ pub fn title_before_marker(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn title_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::DefinitionTitleAfterOptionalWhitespace), - State::Next(Name::DefinitionTitleAfterOptionalWhitespace), + State::Next(StateName::DefinitionTitleAfterOptionalWhitespace), + State::Next(StateName::DefinitionTitleAfterOptionalWhitespace), ) } diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index 482648f..9af5b83 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -39,8 +39,8 @@ //! [hard_break_trailing]: crate::construct::partial_whitespace //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of a hard break (escape). @@ -53,9 +53,9 @@ use crate::tokenizer::Tokenizer; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\\') if tokenizer.parse_state.constructs.hard_break_escape => { - tokenizer.enter(Token::HardBreakEscape); + tokenizer.enter(Name::HardBreakEscape); tokenizer.consume(); - State::Next(Name::HardBreakEscapeAfter) + State::Next(StateName::HardBreakEscapeAfter) } _ => State::Nok, } @@ -71,7 +71,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => { - tokenizer.exit(Token::HardBreakEscape); + tokenizer.exit(Name::HardBreakEscape); State::Ok } _ => State::Nok, diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index 12bc5b1..e856ac3 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -56,9 +56,9 @@ use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE}; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{ContentType, Event, EventType, Tokenizer}; +use crate::event::{Content, Event, Kind, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; /// Start of a heading (atx). /// @@ -68,7 +68,7 @@ use crate::tokenizer::{ContentType, Event, EventType, Tokenizer}; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.parse_state.constructs.heading_atx { - tokenizer.enter(Token::HeadingAtx); + tokenizer.enter(Name::HeadingAtx); let name = space_or_tab_min_max( tokenizer, 0, @@ -78,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(Name::HeadingAtxBefore), State::Nok) + tokenizer.attempt(name, State::Next(StateName::HeadingAtxBefore), State::Nok) } else { State::Nok } @@ -92,8 +92,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn before(tokenizer: &mut Tokenizer) -> State { if Some(b'#') == tokenizer.current { - tokenizer.enter(Token::HeadingAtxSequence); - State::Retry(Name::HeadingAtxSequenceOpen) + tokenizer.enter(Name::HeadingAtxSequence); + State::Retry(StateName::HeadingAtxSequenceOpen) } else { State::Nok } @@ -109,19 +109,19 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') if tokenizer.tokenize_state.size > 0 => { tokenizer.tokenize_state.size = 0; - tokenizer.exit(Token::HeadingAtxSequence); - State::Retry(Name::HeadingAtxAtBreak) + tokenizer.exit(Name::HeadingAtxSequence); + State::Retry(StateName::HeadingAtxAtBreak) } Some(b'#') if tokenizer.tokenize_state.size < HEADING_ATX_OPENING_FENCE_SIZE_MAX => { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(Name::HeadingAtxSequenceOpen) + State::Next(StateName::HeadingAtxSequenceOpen) } _ if tokenizer.tokenize_state.size > 0 => { tokenizer.tokenize_state.size = 0; - tokenizer.exit(Token::HeadingAtxSequence); + tokenizer.exit(Name::HeadingAtxSequence); let name = space_or_tab(tokenizer); - tokenizer.attempt(name, State::Next(Name::HeadingAtxAtBreak), State::Nok) + tokenizer.attempt(name, State::Next(StateName::HeadingAtxAtBreak), State::Nok) } _ => { tokenizer.tokenize_state.size = 0; @@ -139,7 +139,7 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { pub fn at_break(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::HeadingAtx); + tokenizer.exit(Name::HeadingAtx); tokenizer.register_resolver("heading_atx".to_string(), Box::new(resolve)); // Feel free to interrupt. tokenizer.interrupt = false; @@ -147,15 +147,15 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { } Some(b'\t' | b' ') => { let name = space_or_tab(tokenizer); - tokenizer.attempt(name, State::Next(Name::HeadingAtxAtBreak), State::Nok) + tokenizer.attempt(name, State::Next(StateName::HeadingAtxAtBreak), State::Nok) } Some(b'#') => { - tokenizer.enter(Token::HeadingAtxSequence); - State::Retry(Name::HeadingAtxSequenceFurther) + tokenizer.enter(Name::HeadingAtxSequence); + State::Retry(StateName::HeadingAtxSequenceFurther) } Some(_) => { - tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); - State::Retry(Name::HeadingAtxData) + tokenizer.enter_with_content(Name::Data, Some(Content::Text)); + State::Retry(StateName::HeadingAtxData) } } } @@ -171,10 +171,10 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { pub fn sequence_further(tokenizer: &mut Tokenizer) -> State { if let Some(b'#') = tokenizer.current { tokenizer.consume(); - State::Next(Name::HeadingAtxSequenceFurther) + State::Next(StateName::HeadingAtxSequenceFurther) } else { - tokenizer.exit(Token::HeadingAtxSequence); - State::Retry(Name::HeadingAtxAtBreak) + tokenizer.exit(Name::HeadingAtxSequence); + State::Retry(StateName::HeadingAtxAtBreak) } } @@ -188,12 +188,12 @@ pub fn data(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text. None | Some(b'\t' | b'\n' | b' ') => { - tokenizer.exit(Token::Data); - State::Retry(Name::HeadingAtxAtBreak) + tokenizer.exit(Name::Data); + State::Retry(StateName::HeadingAtxAtBreak) } _ => { tokenizer.consume(); - State::Next(Name::HeadingAtxData) + State::Next(StateName::HeadingAtxData) } } } @@ -208,8 +208,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) { while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.token_type == Token::HeadingAtx { - if event.event_type == EventType::Enter { + if event.name == Name::HeadingAtx { + if event.kind == Kind::Enter { heading_inside = true; } else { if let Some(start) = data_start { @@ -220,8 +220,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) { start, 0, vec![Event { - event_type: EventType::Enter, - token_type: Token::HeadingAtxText, + kind: Kind::Enter, + name: Name::HeadingAtxText, point: tokenizer.events[start].point.clone(), link: None, }], @@ -234,8 +234,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) { end + 1, 0, vec![Event { - event_type: EventType::Exit, - token_type: Token::HeadingAtxText, + kind: Kind::Exit, + name: Name::HeadingAtxText, point: tokenizer.events[end].point.clone(), link: None, }], @@ -246,8 +246,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) { data_start = None; data_end = None; } - } else if heading_inside && event.token_type == Token::Data { - if event.event_type == EventType::Enter { + } else if heading_inside && event.name == Name::Data { + if event.kind == Kind::Enter { if data_start.is_none() { data_start = Some(index); } diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index 8b45fff..3a24f9f 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -59,9 +59,9 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{EventType, Tokenizer}; +use crate::event::{Kind, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; use crate::util::skip::opt_back as skip_opt_back; /// At a line ending, presumably an underline. @@ -79,10 +79,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { && tokenizer.events[skip_opt_back( &tokenizer.events, tokenizer.events.len() - 1, - &[Token::LineEnding, Token::SpaceOrTab], + &[Name::LineEnding, Name::SpaceOrTab], )] - .token_type - == Token::Paragraph) + .name + == Name::Paragraph) { let name = space_or_tab_min_max( tokenizer, @@ -94,7 +94,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { }, ); - tokenizer.attempt(name, State::Next(Name::HeadingSetextBefore), State::Nok) + tokenizer.attempt( + name, + State::Next(StateName::HeadingSetextBefore), + State::Nok, + ) } else { State::Nok } @@ -111,8 +115,8 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-' | b'=') => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); - tokenizer.enter(Token::HeadingSetextUnderline); - State::Retry(Name::HeadingSetextInside) + tokenizer.enter(Name::HeadingSetextUnderline); + State::Retry(StateName::HeadingSetextInside) } _ => State::Nok, } @@ -129,16 +133,16 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-' | b'=') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.consume(); - State::Next(Name::HeadingSetextInside) + State::Next(StateName::HeadingSetextInside) } _ => { tokenizer.tokenize_state.marker = 0; - tokenizer.exit(Token::HeadingSetextUnderline); + tokenizer.exit(Name::HeadingSetextUnderline); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::HeadingSetextAfter), - State::Next(Name::HeadingSetextAfter), + State::Next(StateName::HeadingSetextAfter), + State::Next(StateName::HeadingSetextAfter), ) } } @@ -173,28 +177,28 @@ pub fn resolve(tokenizer: &mut Tokenizer) { let event = &tokenizer.events[index]; // Find paragraphs. - if event.event_type == EventType::Enter { - if event.token_type == Token::Paragraph { + if event.kind == Kind::Enter { + if event.name == Name::Paragraph { paragraph_enter = Some(index); } - } else if event.token_type == Token::Paragraph { + } else if event.name == Name::Paragraph { paragraph_exit = Some(index); } // We know this is preceded by a paragraph. // Otherwise we don’t parse. - else if event.token_type == Token::HeadingSetextUnderline { + else if event.name == Name::HeadingSetextUnderline { let enter = paragraph_enter.take().unwrap(); let exit = paragraph_exit.take().unwrap(); // Change types of Enter:Paragraph, Exit:Paragraph. - tokenizer.events[enter].token_type = Token::HeadingSetextText; - tokenizer.events[exit].token_type = Token::HeadingSetextText; + tokenizer.events[enter].name = Name::HeadingSetextText; + tokenizer.events[exit].name = Name::HeadingSetextText; // Add Enter:HeadingSetext, Exit:HeadingSetext. let mut heading_enter = tokenizer.events[enter].clone(); - heading_enter.token_type = Token::HeadingSetext; + heading_enter.name = Name::HeadingSetext; let mut heading_exit = tokenizer.events[index].clone(); - heading_exit.token_type = Token::HeadingSetext; + heading_exit.name = Name::HeadingSetext; tokenizer.map.add(enter, 0, vec![heading_enter]); tokenizer.map.add(index + 1, 0, vec![heading_exit]); diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index c9f8610..8e5321f 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -104,8 +104,8 @@ use crate::constant::{ use crate::construct::partial_space_or_tab::{ space_or_tab_with_options, Options as SpaceOrTabOptions, }; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; use crate::util::slice::Slice; @@ -132,11 +132,11 @@ const COMPLETE: u8 = 7; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.parse_state.constructs.html_flow { - tokenizer.enter(Token::HtmlFlow); + tokenizer.enter(Name::HtmlFlow); let name = space_or_tab_with_options( tokenizer, SpaceOrTabOptions { - kind: Token::HtmlFlowData, + kind: Name::HtmlFlowData, min: 0, max: if tokenizer.parse_state.constructs.code_indented { TAB_SIZE - 1 @@ -148,7 +148,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { }, ); - tokenizer.attempt(name, State::Next(Name::HtmlFlowBefore), State::Nok) + tokenizer.attempt(name, State::Next(StateName::HtmlFlowBefore), State::Nok) } else { State::Nok } @@ -162,9 +162,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn before(tokenizer: &mut Tokenizer) -> State { if Some(b'<') == tokenizer.current { - tokenizer.enter(Token::HtmlFlowData); + tokenizer.enter(Name::HtmlFlowData); tokenizer.consume(); - State::Next(Name::HtmlFlowOpen) + State::Next(StateName::HtmlFlowOpen) } else { State::Nok } @@ -184,13 +184,13 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'!') => { tokenizer.consume(); - State::Next(Name::HtmlFlowDeclarationOpen) + State::Next(StateName::HtmlFlowDeclarationOpen) } Some(b'/') => { tokenizer.consume(); tokenizer.tokenize_state.seen = true; tokenizer.tokenize_state.start = tokenizer.point.index; - State::Next(Name::HtmlFlowTagCloseStart) + State::Next(StateName::HtmlFlowTagCloseStart) } Some(b'?') => { tokenizer.tokenize_state.marker = INSTRUCTION; @@ -199,12 +199,12 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { tokenizer.concrete = true; // While we’re in an instruction instead of a declaration, we’re on a `?` // right now, so we do need to search for `>`, similar to declarations. - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.tokenize_state.start = tokenizer.point.index; - State::Retry(Name::HtmlFlowTagName) + State::Retry(StateName::HtmlFlowTagName) } _ => State::Nok, } @@ -225,19 +225,19 @@ pub fn declaration_open(tokenizer: &mut Tokenizer) -> State { Some(b'-') => { tokenizer.consume(); tokenizer.tokenize_state.marker = COMMENT; - State::Next(Name::HtmlFlowCommentOpenInside) + State::Next(StateName::HtmlFlowCommentOpenInside) } Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); tokenizer.tokenize_state.marker = DECLARATION; // Do not form containers. tokenizer.concrete = true; - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } Some(b'[') => { tokenizer.consume(); tokenizer.tokenize_state.marker = CDATA; - State::Next(Name::HtmlFlowCdataOpenInside) + State::Next(StateName::HtmlFlowCdataOpenInside) } _ => State::Nok, } @@ -254,7 +254,7 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); // Do not form containers. tokenizer.concrete = true; - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -276,9 +276,9 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size = 0; // Do not form containers. tokenizer.concrete = true; - State::Next(Name::HtmlFlowContinuation) + State::Next(StateName::HtmlFlowContinuation) } else { - State::Next(Name::HtmlFlowCdataOpenInside) + State::Next(StateName::HtmlFlowCdataOpenInside) } } else { tokenizer.tokenize_state.marker = 0; @@ -296,7 +296,7 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State { pub fn tag_close_start(tokenizer: &mut Tokenizer) -> State { if let Some(b'A'..=b'Z' | b'a'..=b'z') = tokenizer.current { tokenizer.consume(); - State::Next(Name::HtmlFlowTagName) + State::Next(StateName::HtmlFlowTagName) } else { tokenizer.tokenize_state.seen = false; tokenizer.tokenize_state.start = 0; @@ -335,17 +335,17 @@ pub fn tag_name(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.marker = RAW; // Do not form containers. tokenizer.concrete = true; - State::Retry(Name::HtmlFlowContinuation) + State::Retry(StateName::HtmlFlowContinuation) } else if HTML_BLOCK_NAMES.contains(&name.as_str()) { tokenizer.tokenize_state.marker = BASIC; if slash { tokenizer.consume(); - State::Next(Name::HtmlFlowBasicSelfClosing) + State::Next(StateName::HtmlFlowBasicSelfClosing) } else { // Do not form containers. tokenizer.concrete = true; - State::Retry(Name::HtmlFlowContinuation) + State::Retry(StateName::HtmlFlowContinuation) } } else { tokenizer.tokenize_state.marker = COMPLETE; @@ -355,16 +355,16 @@ pub fn tag_name(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.marker = 0; State::Nok } else if closing_tag { - State::Retry(Name::HtmlFlowCompleteClosingTagAfter) + State::Retry(StateName::HtmlFlowCompleteClosingTagAfter) } else { - State::Retry(Name::HtmlFlowCompleteAttributeNameBefore) + State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore) } } } // ASCII alphanumerical and `-`. Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlFlowTagName) + State::Next(StateName::HtmlFlowTagName) } Some(_) => { tokenizer.tokenize_state.seen = false; @@ -384,7 +384,7 @@ pub fn basic_self_closing(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); // Do not form containers. tokenizer.concrete = true; - State::Next(Name::HtmlFlowContinuation) + State::Next(StateName::HtmlFlowContinuation) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -401,9 +401,9 @@ pub fn complete_closing_tag_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteClosingTagAfter) + State::Next(StateName::HtmlFlowCompleteClosingTagAfter) } - _ => State::Retry(Name::HtmlFlowCompleteEnd), + _ => State::Retry(StateName::HtmlFlowCompleteEnd), } } @@ -430,18 +430,18 @@ pub fn complete_attribute_name_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeNameBefore) + State::Next(StateName::HtmlFlowCompleteAttributeNameBefore) } Some(b'/') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteEnd) + State::Next(StateName::HtmlFlowCompleteEnd) } // ASCII alphanumerical and `:` and `_`. Some(b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeName) + State::Next(StateName::HtmlFlowCompleteAttributeName) } - _ => State::Retry(Name::HtmlFlowCompleteEnd), + _ => State::Retry(StateName::HtmlFlowCompleteEnd), } } @@ -460,9 +460,9 @@ pub fn complete_attribute_name(tokenizer: &mut Tokenizer) -> State { // ASCII alphanumerical and `-`, `.`, `:`, and `_`. Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeName) + State::Next(StateName::HtmlFlowCompleteAttributeName) } - _ => State::Retry(Name::HtmlFlowCompleteAttributeNameAfter), + _ => State::Retry(StateName::HtmlFlowCompleteAttributeNameAfter), } } @@ -479,13 +479,13 @@ pub fn complete_attribute_name_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeNameAfter) + State::Next(StateName::HtmlFlowCompleteAttributeNameAfter) } Some(b'=') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeValueBefore) + State::Next(StateName::HtmlFlowCompleteAttributeValueBefore) } - _ => State::Retry(Name::HtmlFlowCompleteAttributeNameBefore), + _ => State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore), } } @@ -506,14 +506,14 @@ pub fn complete_attribute_value_before(tokenizer: &mut Tokenizer) -> State { } Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeValueBefore) + State::Next(StateName::HtmlFlowCompleteAttributeValueBefore) } Some(b'"' | b'\'') => { tokenizer.tokenize_state.marker_b = tokenizer.current.unwrap(); tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeValueQuoted) + State::Next(StateName::HtmlFlowCompleteAttributeValueQuoted) } - _ => State::Retry(Name::HtmlFlowCompleteAttributeValueUnquoted), + _ => State::Retry(StateName::HtmlFlowCompleteAttributeValueUnquoted), } } @@ -535,11 +535,11 @@ pub fn complete_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { Some(b'"' | b'\'') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker_b => { tokenizer.tokenize_state.marker_b = 0; tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeValueQuotedAfter) + State::Next(StateName::HtmlFlowCompleteAttributeValueQuotedAfter) } _ => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeValueQuoted) + State::Next(StateName::HtmlFlowCompleteAttributeValueQuoted) } } } @@ -553,11 +553,11 @@ pub fn complete_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { pub fn complete_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\t' | b'\n' | b' ' | b'"' | b'\'' | b'/' | b'<' | b'=' | b'>' | b'`') => { - State::Retry(Name::HtmlFlowCompleteAttributeNameAfter) + State::Retry(StateName::HtmlFlowCompleteAttributeNameAfter) } Some(_) => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAttributeValueUnquoted) + State::Next(StateName::HtmlFlowCompleteAttributeValueUnquoted) } } } @@ -571,7 +571,7 @@ pub fn complete_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn complete_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State { if let Some(b'\t' | b' ' | b'/' | b'>') = tokenizer.current { - State::Retry(Name::HtmlFlowCompleteAttributeNameBefore) + State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -587,7 +587,7 @@ pub fn complete_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State pub fn complete_end(tokenizer: &mut Tokenizer) -> State { if let Some(b'>') = tokenizer.current { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAfter) + State::Next(StateName::HtmlFlowCompleteAfter) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -605,11 +605,11 @@ pub fn complete_after(tokenizer: &mut Tokenizer) -> State { None | Some(b'\n') => { // Do not form containers. tokenizer.concrete = true; - State::Retry(Name::HtmlFlowContinuation) + State::Retry(StateName::HtmlFlowContinuation) } Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlFlowCompleteAfter) + State::Next(StateName::HtmlFlowCompleteAfter) } Some(_) => { tokenizer.tokenize_state.marker = 0; @@ -630,41 +630,41 @@ pub fn continuation(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.marker == BASIC || tokenizer.tokenize_state.marker == COMPLETE => { - tokenizer.exit(Token::HtmlFlowData); + tokenizer.exit(Name::HtmlFlowData); tokenizer.check( - Name::HtmlFlowBlankLineBefore, - State::Next(Name::HtmlFlowContinuationAfter), - State::Next(Name::HtmlFlowContinuationStart), + StateName::HtmlFlowBlankLineBefore, + State::Next(StateName::HtmlFlowContinuationAfter), + State::Next(StateName::HtmlFlowContinuationStart), ) } // Note: important that this is after the basic/complete case. None | Some(b'\n') => { - tokenizer.exit(Token::HtmlFlowData); - State::Retry(Name::HtmlFlowContinuationStart) + tokenizer.exit(Name::HtmlFlowData); + State::Retry(StateName::HtmlFlowContinuationStart) } Some(b'-') if tokenizer.tokenize_state.marker == COMMENT => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationCommentInside) + State::Next(StateName::HtmlFlowContinuationCommentInside) } Some(b'<') if tokenizer.tokenize_state.marker == RAW => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationRawTagOpen) + State::Next(StateName::HtmlFlowContinuationRawTagOpen) } Some(b'>') if tokenizer.tokenize_state.marker == DECLARATION => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationClose) + State::Next(StateName::HtmlFlowContinuationClose) } Some(b'?') if tokenizer.tokenize_state.marker == INSTRUCTION => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } Some(b']') if tokenizer.tokenize_state.marker == CDATA => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationCdataInside) + State::Next(StateName::HtmlFlowContinuationCdataInside) } _ => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuation) + State::Next(StateName::HtmlFlowContinuation) } } } @@ -678,9 +678,9 @@ pub fn continuation(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn continuation_start(tokenizer: &mut Tokenizer) -> State { tokenizer.check( - Name::NonLazyContinuationStart, - State::Next(Name::HtmlFlowContinuationStartNonLazy), - State::Next(Name::HtmlFlowContinuationAfter), + StateName::NonLazyContinuationStart, + State::Next(StateName::HtmlFlowContinuationStartNonLazy), + State::Next(StateName::HtmlFlowContinuationAfter), ) } @@ -694,10 +694,10 @@ pub fn continuation_start(tokenizer: &mut Tokenizer) -> State { pub fn continuation_start_non_lazy(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::HtmlFlowContinuationBefore) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::HtmlFlowContinuationBefore) } _ => unreachable!("expected eol"), } @@ -712,10 +712,10 @@ pub fn continuation_start_non_lazy(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn continuation_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None | Some(b'\n') => State::Retry(Name::HtmlFlowContinuationStart), + None | Some(b'\n') => State::Retry(StateName::HtmlFlowContinuationStart), _ => { - tokenizer.enter(Token::HtmlFlowData); - State::Retry(Name::HtmlFlowContinuation) + tokenizer.enter(Name::HtmlFlowData); + State::Retry(StateName::HtmlFlowContinuation) } } } @@ -730,9 +730,9 @@ pub fn continuation_comment_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } - _ => State::Retry(Name::HtmlFlowContinuation), + _ => State::Retry(StateName::HtmlFlowContinuation), } } @@ -747,9 +747,9 @@ pub fn continuation_raw_tag_open(tokenizer: &mut Tokenizer) -> State { Some(b'/') => { tokenizer.consume(); tokenizer.tokenize_state.start = tokenizer.point.index; - State::Next(Name::HtmlFlowContinuationRawEndTag) + State::Next(StateName::HtmlFlowContinuationRawEndTag) } - _ => State::Retry(Name::HtmlFlowContinuation), + _ => State::Retry(StateName::HtmlFlowContinuation), } } @@ -774,20 +774,20 @@ pub fn continuation_raw_end_tag(tokenizer: &mut Tokenizer) -> State { if HTML_RAW_NAMES.contains(&name.as_str()) { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationClose) + State::Next(StateName::HtmlFlowContinuationClose) } else { - State::Retry(Name::HtmlFlowContinuation) + State::Retry(StateName::HtmlFlowContinuation) } } Some(b'A'..=b'Z' | b'a'..=b'z') if tokenizer.point.index - tokenizer.tokenize_state.start < HTML_RAW_SIZE_MAX => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationRawEndTag) + State::Next(StateName::HtmlFlowContinuationRawEndTag) } _ => { tokenizer.tokenize_state.start = 0; - State::Retry(Name::HtmlFlowContinuation) + State::Retry(StateName::HtmlFlowContinuation) } } } @@ -802,9 +802,9 @@ pub fn continuation_cdata_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b']') => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } - _ => State::Retry(Name::HtmlFlowContinuation), + _ => State::Retry(StateName::HtmlFlowContinuation), } } @@ -826,13 +826,13 @@ pub fn continuation_declaration_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationClose) + State::Next(StateName::HtmlFlowContinuationClose) } Some(b'-') if tokenizer.tokenize_state.marker == COMMENT => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationDeclarationInside) + State::Next(StateName::HtmlFlowContinuationDeclarationInside) } - _ => State::Retry(Name::HtmlFlowContinuation), + _ => State::Retry(StateName::HtmlFlowContinuation), } } @@ -845,12 +845,12 @@ pub fn continuation_declaration_inside(tokenizer: &mut Tokenizer) -> State { pub fn continuation_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::HtmlFlowData); - State::Retry(Name::HtmlFlowContinuationAfter) + tokenizer.exit(Name::HtmlFlowData); + State::Retry(StateName::HtmlFlowContinuationAfter) } _ => { tokenizer.consume(); - State::Next(Name::HtmlFlowContinuationClose) + State::Next(StateName::HtmlFlowContinuationClose) } } } @@ -862,7 +862,7 @@ pub fn continuation_close(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn continuation_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.exit(Token::HtmlFlow); + tokenizer.exit(Name::HtmlFlow); tokenizer.tokenize_state.marker = 0; // Feel free to interrupt. tokenizer.interrupt = false; @@ -879,8 +879,8 @@ pub fn continuation_after(tokenizer: &mut Tokenizer) -> State { /// | /// ``` pub fn blank_line_before(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::BlankLineStart) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::BlankLineStart) } diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index dd14e70..27e92f5 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -56,8 +56,8 @@ use crate::constant::HTML_CDATA_PREFIX; use crate::construct::partial_space_or_tab::space_or_tab; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of HTML (text) @@ -68,10 +68,10 @@ use crate::tokenizer::Tokenizer; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if Some(b'<') == tokenizer.current && tokenizer.parse_state.constructs.html_text { - tokenizer.enter(Token::HtmlText); - tokenizer.enter(Token::HtmlTextData); + tokenizer.enter(Name::HtmlText); + tokenizer.enter(Name::HtmlTextData); tokenizer.consume(); - State::Next(Name::HtmlTextOpen) + State::Next(StateName::HtmlTextOpen) } else { State::Nok } @@ -91,20 +91,20 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'!') => { tokenizer.consume(); - State::Next(Name::HtmlTextDeclarationOpen) + State::Next(StateName::HtmlTextDeclarationOpen) } Some(b'/') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagCloseStart) + State::Next(StateName::HtmlTextTagCloseStart) } Some(b'?') => { tokenizer.consume(); - State::Next(Name::HtmlTextInstruction) + State::Next(StateName::HtmlTextInstruction) } // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpen) + State::Next(StateName::HtmlTextTagOpen) } _ => State::Nok, } @@ -124,16 +124,16 @@ pub fn declaration_open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(Name::HtmlTextCommentOpenInside) + State::Next(StateName::HtmlTextCommentOpenInside) } // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextDeclaration) + State::Next(StateName::HtmlTextDeclaration) } Some(b'[') => { tokenizer.consume(); - State::Next(Name::HtmlTextCdataOpenInside) + State::Next(StateName::HtmlTextCdataOpenInside) } _ => State::Nok, } @@ -149,7 +149,7 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(Name::HtmlTextCommentStart) + State::Next(StateName::HtmlTextCommentStart) } _ => State::Nok, } @@ -173,9 +173,9 @@ pub fn comment_start(tokenizer: &mut Tokenizer) -> State { Some(b'>') => State::Nok, Some(b'-') => { tokenizer.consume(); - State::Next(Name::HtmlTextCommentStartDash) + State::Next(StateName::HtmlTextCommentStartDash) } - _ => State::Retry(Name::HtmlTextComment), + _ => State::Retry(StateName::HtmlTextComment), } } @@ -195,7 +195,7 @@ pub fn comment_start(tokenizer: &mut Tokenizer) -> State { pub fn comment_start_dash(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => State::Nok, - _ => State::Retry(Name::HtmlTextComment), + _ => State::Retry(StateName::HtmlTextComment), } } @@ -209,17 +209,17 @@ pub fn comment(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Nok, Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextComment), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextComment), State::Nok, ), Some(b'-') => { tokenizer.consume(); - State::Next(Name::HtmlTextCommentClose) + State::Next(StateName::HtmlTextCommentClose) } _ => { tokenizer.consume(); - State::Next(Name::HtmlTextComment) + State::Next(StateName::HtmlTextComment) } } } @@ -234,9 +234,9 @@ pub fn comment_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(Name::HtmlTextEnd) + State::Next(StateName::HtmlTextEnd) } - _ => State::Retry(Name::HtmlTextComment), + _ => State::Retry(StateName::HtmlTextComment), } } @@ -253,9 +253,9 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.size == HTML_CDATA_PREFIX.len() { tokenizer.tokenize_state.size = 0; - State::Next(Name::HtmlTextCdata) + State::Next(StateName::HtmlTextCdata) } else { - State::Next(Name::HtmlTextCdataOpenInside) + State::Next(StateName::HtmlTextCdataOpenInside) } } else { State::Nok @@ -272,17 +272,17 @@ pub fn cdata(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Nok, Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextCdata), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextCdata), State::Nok, ), Some(b']') => { tokenizer.consume(); - State::Next(Name::HtmlTextCdataClose) + State::Next(StateName::HtmlTextCdataClose) } _ => { tokenizer.consume(); - State::Next(Name::HtmlTextCdata) + State::Next(StateName::HtmlTextCdata) } } } @@ -297,9 +297,9 @@ pub fn cdata_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b']') => { tokenizer.consume(); - State::Next(Name::HtmlTextCdataEnd) + State::Next(StateName::HtmlTextCdataEnd) } - _ => State::Retry(Name::HtmlTextCdata), + _ => State::Retry(StateName::HtmlTextCdata), } } @@ -311,9 +311,9 @@ pub fn cdata_close(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn cdata_end(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'>') => State::Retry(Name::HtmlTextEnd), - Some(b']') => State::Retry(Name::HtmlTextCdataClose), - _ => State::Retry(Name::HtmlTextCdata), + Some(b'>') => State::Retry(StateName::HtmlTextEnd), + Some(b']') => State::Retry(StateName::HtmlTextCdataClose), + _ => State::Retry(StateName::HtmlTextCdata), } } @@ -325,15 +325,15 @@ pub fn cdata_end(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn declaration(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None | Some(b'>') => State::Retry(Name::HtmlTextEnd), + None | Some(b'>') => State::Retry(StateName::HtmlTextEnd), Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextDeclaration), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextDeclaration), State::Nok, ), _ => { tokenizer.consume(); - State::Next(Name::HtmlTextDeclaration) + State::Next(StateName::HtmlTextDeclaration) } } } @@ -348,17 +348,17 @@ pub fn instruction(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Nok, Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextInstruction), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextInstruction), State::Nok, ), Some(b'?') => { tokenizer.consume(); - State::Next(Name::HtmlTextInstructionClose) + State::Next(StateName::HtmlTextInstructionClose) } _ => { tokenizer.consume(); - State::Next(Name::HtmlTextInstruction) + State::Next(StateName::HtmlTextInstruction) } } } @@ -371,8 +371,8 @@ pub fn instruction(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn instruction_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'>') => State::Retry(Name::HtmlTextEnd), - _ => State::Retry(Name::HtmlTextInstruction), + Some(b'>') => State::Retry(StateName::HtmlTextEnd), + _ => State::Retry(StateName::HtmlTextInstruction), } } @@ -387,7 +387,7 @@ pub fn tag_close_start(tokenizer: &mut Tokenizer) -> State { // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagClose) + State::Next(StateName::HtmlTextTagClose) } _ => State::Nok, } @@ -404,9 +404,9 @@ pub fn tag_close(tokenizer: &mut Tokenizer) -> State { // ASCII alphanumerical and `-`. Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagClose) + State::Next(StateName::HtmlTextTagClose) } - _ => State::Retry(Name::HtmlTextTagCloseBetween), + _ => State::Retry(StateName::HtmlTextTagCloseBetween), } } @@ -419,15 +419,15 @@ pub fn tag_close(tokenizer: &mut Tokenizer) -> State { pub fn tag_close_between(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextTagCloseBetween), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextTagCloseBetween), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagCloseBetween) + State::Next(StateName::HtmlTextTagCloseBetween) } - _ => State::Retry(Name::HtmlTextEnd), + _ => State::Retry(StateName::HtmlTextEnd), } } @@ -442,9 +442,9 @@ pub fn tag_open(tokenizer: &mut Tokenizer) -> State { // ASCII alphanumerical and `-`. Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpen) + State::Next(StateName::HtmlTextTagOpen) } - Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(Name::HtmlTextTagOpenBetween), + Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(StateName::HtmlTextTagOpenBetween), _ => State::Nok, } } @@ -458,24 +458,24 @@ pub fn tag_open(tokenizer: &mut Tokenizer) -> State { pub fn tag_open_between(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextTagOpenBetween), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextTagOpenBetween), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenBetween) + State::Next(StateName::HtmlTextTagOpenBetween) } Some(b'/') => { tokenizer.consume(); - State::Next(Name::HtmlTextEnd) + State::Next(StateName::HtmlTextEnd) } // ASCII alphabetical and `:` and `_`. Some(b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeName) + State::Next(StateName::HtmlTextTagOpenAttributeName) } - _ => State::Retry(Name::HtmlTextEnd), + _ => State::Retry(StateName::HtmlTextEnd), } } @@ -490,9 +490,9 @@ pub fn tag_open_attribute_name(tokenizer: &mut Tokenizer) -> State { // ASCII alphabetical and `-`, `.`, `:`, and `_`. Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeName) + State::Next(StateName::HtmlTextTagOpenAttributeName) } - _ => State::Retry(Name::HtmlTextTagOpenAttributeNameAfter), + _ => State::Retry(StateName::HtmlTextTagOpenAttributeNameAfter), } } @@ -506,19 +506,19 @@ pub fn tag_open_attribute_name(tokenizer: &mut Tokenizer) -> State { pub fn tag_open_attribute_name_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextTagOpenAttributeNameAfter), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextTagOpenAttributeNameAfter), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeNameAfter) + State::Next(StateName::HtmlTextTagOpenAttributeNameAfter) } Some(b'=') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueBefore) + State::Next(StateName::HtmlTextTagOpenAttributeValueBefore) } - _ => State::Retry(Name::HtmlTextTagOpenBetween), + _ => State::Retry(StateName::HtmlTextTagOpenBetween), } } @@ -533,22 +533,22 @@ pub fn tag_open_attribute_value_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'<' | b'=' | b'>' | b'`') => State::Nok, Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextTagOpenAttributeValueBefore), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextTagOpenAttributeValueBefore), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueBefore) + State::Next(StateName::HtmlTextTagOpenAttributeValueBefore) } Some(b'"' | b'\'') => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueQuoted) + State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted) } Some(_) => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueUnquoted) + State::Next(StateName::HtmlTextTagOpenAttributeValueUnquoted) } } } @@ -566,18 +566,18 @@ pub fn tag_open_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { State::Nok } Some(b'\n') => tokenizer.attempt( - Name::HtmlTextLineEndingBefore, - State::Next(Name::HtmlTextTagOpenAttributeValueQuoted), + StateName::HtmlTextLineEndingBefore, + State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted), State::Nok, ), Some(b'"' | b'\'') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.tokenize_state.marker = 0; tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueQuotedAfter) + State::Next(StateName::HtmlTextTagOpenAttributeValueQuotedAfter) } _ => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueQuoted) + State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted) } } } @@ -591,10 +591,10 @@ pub fn tag_open_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'"' | b'\'' | b'<' | b'=' | b'`') => State::Nok, - Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(Name::HtmlTextTagOpenBetween), + Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(StateName::HtmlTextTagOpenBetween), Some(_) => { tokenizer.consume(); - State::Next(Name::HtmlTextTagOpenAttributeValueUnquoted) + State::Next(StateName::HtmlTextTagOpenAttributeValueUnquoted) } } } @@ -608,7 +608,7 @@ pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn tag_open_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'\t' | b'\n' | b' ' | b'>' | b'/') => State::Retry(Name::HtmlTextTagOpenBetween), + Some(b'\t' | b'\n' | b' ' | b'>' | b'/') => State::Retry(StateName::HtmlTextTagOpenBetween), _ => State::Nok, } } @@ -623,8 +623,8 @@ pub fn end(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { tokenizer.consume(); - tokenizer.exit(Token::HtmlTextData); - tokenizer.exit(Token::HtmlText); + tokenizer.exit(Name::HtmlTextData); + tokenizer.exit(Name::HtmlText); State::Ok } _ => State::Nok, @@ -644,11 +644,11 @@ pub fn end(tokenizer: &mut Tokenizer) -> State { pub fn line_ending_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => { - tokenizer.exit(Token::HtmlTextData); - tokenizer.enter(Token::LineEnding); + tokenizer.exit(Name::HtmlTextData); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::HtmlTextLineEndingAfter) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::HtmlTextLineEndingAfter) } _ => unreachable!("expected eol"), } @@ -668,8 +668,8 @@ pub fn line_ending_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::HtmlTextLineEndingAfterPrefix), - State::Next(Name::HtmlTextLineEndingAfterPrefix), + State::Next(StateName::HtmlTextLineEndingAfterPrefix), + State::Next(StateName::HtmlTextLineEndingAfterPrefix), ) } @@ -684,6 +684,6 @@ pub fn line_ending_after(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn line_ending_after_prefix(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::HtmlTextData); + tokenizer.enter(Name::HtmlTextData); State::Ok } diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs index 0607077..61f378d 100644 --- a/src/construct/label_end.rs +++ b/src/construct/label_end.rs @@ -148,9 +148,10 @@ use crate::constant::RESOURCE_DESTINATION_BALANCE_MAX; use crate::construct::partial_space_or_tab::space_or_tab_eol; -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{Event, EventType, Media, Tokenizer}; +use crate::event::{Event, Kind, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::{Media, Tokenizer}; + use crate::util::{ normalize_identifier::normalize_identifier, skip, @@ -195,15 +196,15 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { // Mark as balanced if the info is inactive. if label_start.inactive { - return State::Retry(Name::LabelEndNok); + return State::Retry(StateName::LabelEndNok); } - tokenizer.enter(Token::LabelEnd); - tokenizer.enter(Token::LabelMarker); + tokenizer.enter(Name::LabelEnd); + tokenizer.enter(Name::LabelMarker); tokenizer.consume(); - tokenizer.exit(Token::LabelMarker); - tokenizer.exit(Token::LabelEnd); - return State::Next(Name::LabelEndAfter); + tokenizer.exit(Name::LabelMarker); + tokenizer.exit(Name::LabelEnd); + return State::Next(StateName::LabelEndAfter); } } @@ -240,29 +241,29 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Resource (`[asd](fgh)`)? Some(b'(') => tokenizer.attempt( - Name::LabelEndResourceStart, - State::Next(Name::LabelEndOk), + StateName::LabelEndResourceStart, + State::Next(StateName::LabelEndOk), State::Next(if defined { - Name::LabelEndOk + StateName::LabelEndOk } else { - Name::LabelEndNok + StateName::LabelEndNok }), ), // Full (`[asd][fgh]`) or collapsed (`[asd][]`) reference? Some(b'[') => tokenizer.attempt( - Name::LabelEndReferenceFull, - State::Next(Name::LabelEndOk), + StateName::LabelEndReferenceFull, + State::Next(StateName::LabelEndOk), State::Next(if defined { - Name::LabelEndReferenceNotFull + StateName::LabelEndReferenceNotFull } else { - Name::LabelEndNok + StateName::LabelEndNok }), ), // Shortcut (`[asd]`) reference? _ => State::Retry(if defined { - Name::LabelEndOk + StateName::LabelEndOk } else { - Name::LabelEndNok + StateName::LabelEndNok }), } } @@ -279,9 +280,9 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn reference_not_full(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - Name::LabelEndReferenceCollapsed, - State::Next(Name::LabelEndOk), - State::Next(Name::LabelEndNok), + StateName::LabelEndReferenceCollapsed, + State::Next(StateName::LabelEndOk), + State::Next(StateName::LabelEndNok), ) } @@ -311,13 +312,13 @@ pub fn ok(tokenizer: &mut Tokenizer) -> State { .label_start_list_loose .append(&mut left); - let is_link = tokenizer.events[label_start.start.0].token_type == Token::LabelLink; + let is_link = tokenizer.events[label_start.start.0].name == Name::LabelLink; if is_link { let mut index = 0; while index < tokenizer.tokenize_state.label_start_stack.len() { let label_start = &mut tokenizer.tokenize_state.label_start_stack[index]; - if tokenizer.events[label_start.start.0].token_type == Token::LabelLink { + if tokenizer.events[label_start.start.0].name == Name::LabelLink { label_start.inactive = true; } index += 1; @@ -367,11 +368,11 @@ pub fn nok(tokenizer: &mut Tokenizer) -> State { pub fn resource_start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'(') => { - tokenizer.enter(Token::Resource); - tokenizer.enter(Token::ResourceMarker); + tokenizer.enter(Name::Resource); + tokenizer.enter(Name::ResourceMarker); tokenizer.consume(); - tokenizer.exit(Token::ResourceMarker); - State::Next(Name::LabelEndResourceBefore) + tokenizer.exit(Name::ResourceMarker); + State::Next(StateName::LabelEndResourceBefore) } _ => unreachable!("expected `(`"), } @@ -387,8 +388,8 @@ pub fn resource_before(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(Name::LabelEndResourceOpen), - State::Next(Name::LabelEndResourceOpen), + State::Next(StateName::LabelEndResourceOpen), + State::Next(StateName::LabelEndResourceOpen), ) } @@ -400,19 +401,19 @@ pub fn resource_before(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn resource_open(tokenizer: &mut Tokenizer) -> State { if let Some(b')') = tokenizer.current { - State::Retry(Name::LabelEndResourceEnd) + State::Retry(StateName::LabelEndResourceEnd) } else { - tokenizer.tokenize_state.token_1 = Token::ResourceDestination; - tokenizer.tokenize_state.token_2 = Token::ResourceDestinationLiteral; - tokenizer.tokenize_state.token_3 = Token::ResourceDestinationLiteralMarker; - tokenizer.tokenize_state.token_4 = Token::ResourceDestinationRaw; - tokenizer.tokenize_state.token_5 = Token::ResourceDestinationString; + tokenizer.tokenize_state.token_1 = Name::ResourceDestination; + tokenizer.tokenize_state.token_2 = Name::ResourceDestinationLiteral; + tokenizer.tokenize_state.token_3 = Name::ResourceDestinationLiteralMarker; + tokenizer.tokenize_state.token_4 = Name::ResourceDestinationRaw; + tokenizer.tokenize_state.token_5 = Name::ResourceDestinationString; tokenizer.tokenize_state.size_b = RESOURCE_DESTINATION_BALANCE_MAX; tokenizer.attempt( - Name::DestinationStart, - State::Next(Name::LabelEndResourceDestinationAfter), - State::Next(Name::LabelEndResourceDestinationMissing), + StateName::DestinationStart, + State::Next(StateName::LabelEndResourceDestinationAfter), + State::Next(StateName::LabelEndResourceDestinationMissing), ) } } @@ -424,27 +425,27 @@ pub fn resource_open(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn resource_destination_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; - tokenizer.tokenize_state.token_4 = Token::Data; - tokenizer.tokenize_state.token_5 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; + tokenizer.tokenize_state.token_4 = Name::Data; + tokenizer.tokenize_state.token_5 = Name::Data; tokenizer.tokenize_state.size_b = 0; let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(Name::LabelEndResourceBetween), - State::Next(Name::LabelEndResourceEnd), + State::Next(StateName::LabelEndResourceBetween), + State::Next(StateName::LabelEndResourceEnd), ) } /// Without destination. pub fn resource_destination_missing(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; - tokenizer.tokenize_state.token_4 = Token::Data; - tokenizer.tokenize_state.token_5 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; + tokenizer.tokenize_state.token_4 = Name::Data; + tokenizer.tokenize_state.token_5 = Name::Data; tokenizer.tokenize_state.size_b = 0; State::Nok } @@ -458,16 +459,16 @@ pub fn resource_destination_missing(tokenizer: &mut Tokenizer) -> State { pub fn resource_between(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'"' | b'\'' | b'(') => { - tokenizer.tokenize_state.token_1 = Token::ResourceTitle; - tokenizer.tokenize_state.token_2 = Token::ResourceTitleMarker; - tokenizer.tokenize_state.token_3 = Token::ResourceTitleString; + tokenizer.tokenize_state.token_1 = Name::ResourceTitle; + tokenizer.tokenize_state.token_2 = Name::ResourceTitleMarker; + tokenizer.tokenize_state.token_3 = Name::ResourceTitleString; tokenizer.attempt( - Name::TitleStart, - State::Next(Name::LabelEndResourceTitleAfter), + StateName::TitleStart, + State::Next(StateName::LabelEndResourceTitleAfter), State::Nok, ) } - _ => State::Retry(Name::LabelEndResourceEnd), + _ => State::Retry(StateName::LabelEndResourceEnd), } } @@ -478,14 +479,14 @@ pub fn resource_between(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn resource_title_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(Name::LabelEndResourceEnd), - State::Next(Name::LabelEndResourceEnd), + State::Next(StateName::LabelEndResourceEnd), + State::Next(StateName::LabelEndResourceEnd), ) } @@ -498,10 +499,10 @@ pub fn resource_title_after(tokenizer: &mut Tokenizer) -> State { pub fn resource_end(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b')') => { - tokenizer.enter(Token::ResourceMarker); + tokenizer.enter(Name::ResourceMarker); tokenizer.consume(); - tokenizer.exit(Token::ResourceMarker); - tokenizer.exit(Token::Resource); + tokenizer.exit(Name::ResourceMarker); + tokenizer.exit(Name::Resource); State::Ok } _ => State::Nok, @@ -517,12 +518,12 @@ pub fn resource_end(tokenizer: &mut Tokenizer) -> State { pub fn reference_full(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'[') => { - tokenizer.tokenize_state.token_1 = Token::Reference; - tokenizer.tokenize_state.token_2 = Token::ReferenceMarker; - tokenizer.tokenize_state.token_3 = Token::ReferenceString; + tokenizer.tokenize_state.token_1 = Name::Reference; + tokenizer.tokenize_state.token_2 = Name::ReferenceMarker; + tokenizer.tokenize_state.token_3 = Name::ReferenceString; tokenizer.attempt( - Name::LabelStart, - State::Next(Name::LabelEndReferenceFullAfter), + StateName::LabelStart, + State::Next(StateName::LabelEndReferenceFullAfter), State::Nok, ) } @@ -537,9 +538,9 @@ pub fn reference_full(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn reference_full_after(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.token_1 = Token::Data; - tokenizer.tokenize_state.token_2 = Token::Data; - tokenizer.tokenize_state.token_3 = Token::Data; + tokenizer.tokenize_state.token_1 = Name::Data; + tokenizer.tokenize_state.token_2 = Name::Data; + tokenizer.tokenize_state.token_3 = Name::Data; if tokenizer .parse_state @@ -553,7 +554,7 @@ pub fn reference_full_after(tokenizer: &mut Tokenizer) -> State { skip::to_back( &tokenizer.events, tokenizer.events.len() - 1, - &[Token::ReferenceString], + &[Name::ReferenceString], ), ), ) @@ -577,11 +578,11 @@ pub fn reference_full_after(tokenizer: &mut Tokenizer) -> State { pub fn reference_collapsed(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'[') => { - tokenizer.enter(Token::Reference); - tokenizer.enter(Token::ReferenceMarker); + tokenizer.enter(Name::Reference); + tokenizer.enter(Name::ReferenceMarker); tokenizer.consume(); - tokenizer.exit(Token::ReferenceMarker); - State::Next(Name::LabelEndReferenceCollapsedOpen) + tokenizer.exit(Name::ReferenceMarker); + State::Next(StateName::LabelEndReferenceCollapsedOpen) } _ => State::Nok, } @@ -598,10 +599,10 @@ pub fn reference_collapsed(tokenizer: &mut Tokenizer) -> State { pub fn reference_collapsed_open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b']') => { - tokenizer.enter(Token::ReferenceMarker); + tokenizer.enter(Name::ReferenceMarker); tokenizer.consume(); - tokenizer.exit(Token::ReferenceMarker); - tokenizer.exit(Token::Reference); + tokenizer.exit(Name::ReferenceMarker); + tokenizer.exit(Name::Reference); State::Ok } _ => State::Nok, @@ -633,14 +634,14 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { data_exit_index - data_enter_index + 1, vec![ Event { - event_type: EventType::Enter, - token_type: Token::Data, + kind: Kind::Enter, + name: Name::Data, point: events[data_enter_index].point.clone(), link: None, }, Event { - event_type: EventType::Exit, - token_type: Token::Data, + kind: Kind::Exit, + name: Name::Data, point: events[data_exit_index].point.clone(), link: None, }, @@ -659,7 +660,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { let group_enter_event = &events[group_enter_index]; // LabelLink:Exit or LabelImage:Exit. let text_enter_index = media.start.0 - + (if group_enter_event.token_type == Token::LabelLink { + + (if group_enter_event.name == Name::LabelLink { 4 } else { 6 @@ -677,18 +678,18 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { 0, vec![ Event { - event_type: EventType::Enter, - token_type: if group_enter_event.token_type == Token::LabelLink { - Token::Link + kind: Kind::Enter, + name: if group_enter_event.name == Name::LabelLink { + Name::Link } else { - Token::Image + Name::Image }, point: group_enter_event.point.clone(), link: None, }, Event { - event_type: EventType::Enter, - token_type: Token::Label, + kind: Kind::Enter, + name: Name::Label, point: group_enter_event.point.clone(), link: None, }, @@ -702,8 +703,8 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { text_enter_index, 0, vec![Event { - event_type: EventType::Enter, - token_type: Token::LabelText, + kind: Kind::Enter, + name: Name::LabelText, point: events[text_enter_index].point.clone(), link: None, }], @@ -714,8 +715,8 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { text_exit_index, 0, vec![Event { - event_type: EventType::Exit, - token_type: Token::LabelText, + kind: Kind::Exit, + name: Name::LabelText, point: events[text_exit_index].point.clone(), link: None, }], @@ -727,8 +728,8 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { label_exit_index + 1, 0, vec![Event { - event_type: EventType::Exit, - token_type: Token::Label, + kind: Kind::Exit, + name: Name::Label, point: events[label_exit_index].point.clone(), link: None, }], @@ -739,11 +740,11 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) { group_end_index + 1, 0, vec![Event { - event_type: EventType::Exit, - token_type: if group_enter_event.token_type == Token::LabelLink { - Token::Link + kind: Kind::Exit, + name: if group_enter_event.name == Name::LabelLink { + Name::Link } else { - Token::Image + Name::Image }, point: events[group_end_index].point.clone(), link: None, diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs index 7703ba4..2f7c0bf 100644 --- a/src/construct/label_start_image.rs +++ b/src/construct/label_start_image.rs @@ -29,8 +29,8 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use super::label_end::resolve_media; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::{LabelStart, Tokenizer}; /// Start of label (image) start. @@ -42,11 +42,11 @@ use crate::tokenizer::{LabelStart, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'!') if tokenizer.parse_state.constructs.label_start_image => { - tokenizer.enter(Token::LabelImage); - tokenizer.enter(Token::LabelImageMarker); + tokenizer.enter(Name::LabelImage); + tokenizer.enter(Name::LabelImageMarker); tokenizer.consume(); - tokenizer.exit(Token::LabelImageMarker); - State::Next(Name::LabelStartImageOpen) + tokenizer.exit(Name::LabelImageMarker); + State::Next(StateName::LabelStartImageOpen) } _ => State::Nok, } @@ -61,10 +61,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'[') => { - tokenizer.enter(Token::LabelMarker); + tokenizer.enter(Name::LabelMarker); tokenizer.consume(); - tokenizer.exit(Token::LabelMarker); - tokenizer.exit(Token::LabelImage); + tokenizer.exit(Name::LabelMarker); + tokenizer.exit(Name::LabelImage); tokenizer.tokenize_state.label_start_stack.push(LabelStart { start: (tokenizer.events.len() - 6, tokenizer.events.len() - 1), balanced: false, diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs index 3ca51bf..456a4e9 100644 --- a/src/construct/label_start_link.rs +++ b/src/construct/label_start_link.rs @@ -28,8 +28,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use super::label_end::resolve_media; +use crate::event::Name; use crate::state::State; -use crate::token::Token; use crate::tokenizer::{LabelStart, Tokenizer}; /// Start of label (link) start. @@ -42,11 +42,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'[') if tokenizer.parse_state.constructs.label_start_link => { let start = tokenizer.events.len(); - tokenizer.enter(Token::LabelLink); - tokenizer.enter(Token::LabelMarker); + tokenizer.enter(Name::LabelLink); + tokenizer.enter(Name::LabelMarker); tokenizer.consume(); - tokenizer.exit(Token::LabelMarker); - tokenizer.exit(Token::LabelLink); + tokenizer.exit(Name::LabelMarker); + tokenizer.exit(Name::LabelLink); tokenizer.tokenize_state.label_start_stack.push(LabelStart { start: (start, tokenizer.events.len() - 1), balanced: false, diff --git a/src/construct/list.rs b/src/construct/list.rs index 516cec7..ded77d0 100644 --- a/src/construct/list.rs +++ b/src/construct/list.rs @@ -46,9 +46,9 @@ use crate::constant::{LIST_ITEM_VALUE_SIZE_MAX, TAB_SIZE}; use crate::construct::partial_space_or_tab::space_or_tab_min_max; -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{EventType, Tokenizer}; +use crate::event::{Kind, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; use crate::util::{ skip, slice::{Position, Slice}, @@ -62,7 +62,7 @@ use crate::util::{ /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.parse_state.constructs.list { - tokenizer.enter(Token::ListItem); + tokenizer.enter(Name::ListItem); let name = space_or_tab_min_max( tokenizer, 0, @@ -72,7 +72,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(Name::ListBefore), State::Nok) + tokenizer.attempt(name, State::Next(StateName::ListBefore), State::Nok) } else { State::Nok } @@ -88,14 +88,14 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Unordered. Some(b'*' | b'-') => tokenizer.check( - Name::ThematicBreakStart, - State::Next(Name::ListNok), - State::Next(Name::ListBeforeUnordered), + StateName::ThematicBreakStart, + State::Next(StateName::ListNok), + State::Next(StateName::ListBeforeUnordered), ), - Some(b'+') => State::Retry(Name::ListBeforeUnordered), + Some(b'+') => State::Retry(StateName::ListBeforeUnordered), // Ordered. - Some(b'0'..=b'9') if !tokenizer.interrupt => State::Retry(Name::ListBeforeOrdered), - Some(b'1') => State::Retry(Name::ListBeforeOrdered), + Some(b'0'..=b'9') if !tokenizer.interrupt => State::Retry(StateName::ListBeforeOrdered), + Some(b'1') => State::Retry(StateName::ListBeforeOrdered), _ => State::Nok, } } @@ -109,8 +109,8 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn before_unordered(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::ListItemPrefix); - State::Retry(Name::ListMarker) + tokenizer.enter(Name::ListItemPrefix); + State::Retry(StateName::ListMarker) } /// Start of an ordered list item. @@ -120,9 +120,9 @@ pub fn before_unordered(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn before_ordered(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::ListItemPrefix); - tokenizer.enter(Token::ListItemValue); - State::Retry(Name::ListValue) + tokenizer.enter(Name::ListItemPrefix); + tokenizer.enter(Name::ListItemValue); + State::Retry(StateName::ListValue) } /// In an ordered list item value. @@ -134,13 +134,13 @@ pub fn before_ordered(tokenizer: &mut Tokenizer) -> State { pub fn value(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'.' | b')') if !tokenizer.interrupt || tokenizer.tokenize_state.size < 2 => { - tokenizer.exit(Token::ListItemValue); - State::Retry(Name::ListMarker) + tokenizer.exit(Name::ListItemValue); + State::Retry(StateName::ListMarker) } Some(b'0'..=b'9') if tokenizer.tokenize_state.size + 1 < LIST_ITEM_VALUE_SIZE_MAX => { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(Name::ListValue) + State::Next(StateName::ListValue) } _ => { tokenizer.tokenize_state.size = 0; @@ -158,10 +158,10 @@ pub fn value(tokenizer: &mut Tokenizer) -> State { /// ^ /// ``` pub fn marker(tokenizer: &mut Tokenizer) -> State { - tokenizer.enter(Token::ListItemMarker); + tokenizer.enter(Name::ListItemMarker); tokenizer.consume(); - tokenizer.exit(Token::ListItemMarker); - State::Next(Name::ListMarkerAfter) + tokenizer.exit(Name::ListItemMarker); + State::Next(StateName::ListMarkerAfter) } /// After a list item marker. @@ -175,9 +175,9 @@ pub fn marker(tokenizer: &mut Tokenizer) -> State { pub fn marker_after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size = 1; tokenizer.check( - Name::BlankLineStart, - State::Next(Name::ListAfter), - State::Next(Name::ListMarkerAfterFilled), + StateName::BlankLineStart, + State::Next(StateName::ListAfter), + State::Next(StateName::ListMarkerAfterFilled), ) } @@ -192,9 +192,9 @@ pub fn marker_after_filled(tokenizer: &mut Tokenizer) -> State { // Attempt to parse up to the largest allowed indent, `nok` if there is more whitespace. tokenizer.attempt( - Name::ListWhitespace, - State::Next(Name::ListAfter), - State::Next(Name::ListPrefixOther), + StateName::ListWhitespace, + State::Next(StateName::ListAfter), + State::Next(StateName::ListPrefixOther), ) } @@ -206,7 +206,11 @@ pub fn marker_after_filled(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn whitespace(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_min_max(tokenizer, 1, TAB_SIZE); - tokenizer.attempt(name, State::Next(Name::ListWhitespaceAfter), State::Nok) + tokenizer.attempt( + name, + State::Next(StateName::ListWhitespaceAfter), + State::Nok, + ) } /// After acceptable whitespace. @@ -232,10 +236,10 @@ pub fn whitespace_after(tokenizer: &mut Tokenizer) -> State { pub fn prefix_other(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { - tokenizer.enter(Token::SpaceOrTab); + tokenizer.enter(Name::SpaceOrTab); tokenizer.consume(); - tokenizer.exit(Token::SpaceOrTab); - State::Next(Name::ListAfter) + tokenizer.exit(Name::SpaceOrTab); + State::Next(StateName::ListAfter) } _ => State::Nok, } @@ -257,7 +261,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { let start = skip::to_back( &tokenizer.events, tokenizer.events.len() - 1, - &[Token::ListItem], + &[Name::ListItem], ); let mut prefix = Slice::from_position( tokenizer.parse_state.bytes, @@ -278,7 +282,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { container.blank_initial = blank; container.size = prefix; - tokenizer.exit(Token::ListItemPrefix); + tokenizer.exit(Name::ListItemPrefix); tokenizer.register_resolver_before("list_item".to_string(), Box::new(resolve_list_item)); State::Ok } @@ -293,9 +297,9 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn cont_start(tokenizer: &mut Tokenizer) -> State { tokenizer.check( - Name::BlankLineStart, - State::Next(Name::ListContBlank), - State::Next(Name::ListContFilled), + StateName::BlankLineStart, + State::Next(StateName::ListContBlank), + State::Next(StateName::ListContFilled), ) } @@ -317,7 +321,7 @@ pub fn cont_blank(tokenizer: &mut Tokenizer) -> State { } else { let name = space_or_tab_min_max(tokenizer, 0, size); // Consume, optionally, at most `size`. - tokenizer.attempt(name, State::Next(Name::ListOk), State::Nok) + tokenizer.attempt(name, State::Next(StateName::ListOk), State::Nok) } } @@ -337,7 +341,7 @@ pub fn cont_filled(tokenizer: &mut Tokenizer) -> State { // Consume exactly `size`. let name = space_or_tab_min_max(tokenizer, size, size); - tokenizer.attempt(name, State::Next(Name::ListOk), State::Nok) + tokenizer.attempt(name, State::Next(StateName::ListOk), State::Nok) } /// A state fn to yield [`State::Ok`]. @@ -361,10 +365,10 @@ pub fn resolve_list_item(tokenizer: &mut Tokenizer) { while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.token_type == Token::ListItem { - if event.event_type == EventType::Enter { - let end = skip::opt(&tokenizer.events, index, &[Token::ListItem]) - 1; - let marker = skip::to(&tokenizer.events, index, &[Token::ListItemMarker]); + if event.name == Name::ListItem { + if event.kind == Kind::Enter { + let end = skip::opt(&tokenizer.events, index, &[Name::ListItem]) - 1; + let marker = skip::to(&tokenizer.events, index, &[Name::ListItemMarker]); // Guaranteed to be a valid ASCII byte. let marker = Slice::from_index( tokenizer.parse_state.bytes, @@ -384,10 +388,10 @@ pub fn resolve_list_item(tokenizer: &mut Tokenizer) { &tokenizer.events, previous.3 + 1, &[ - Token::SpaceOrTab, - Token::LineEnding, - Token::BlankLineEnding, - Token::BlockQuotePrefix, + Name::SpaceOrTab, + Name::LineEnding, + Name::BlankLineEnding, + Name::BlockQuotePrefix, ], ); @@ -441,12 +445,12 @@ pub fn resolve_list_item(tokenizer: &mut Tokenizer) { let list_item = &lists[index]; let mut list_start = tokenizer.events[list_item.2].clone(); let mut list_end = tokenizer.events[list_item.3].clone(); - let token_type = match list_item.0 { - b'.' | b')' => Token::ListOrdered, - _ => Token::ListUnordered, + let name = match list_item.0 { + b'.' | b')' => Name::ListOrdered, + _ => Name::ListUnordered, }; - list_start.token_type = token_type.clone(); - list_end.token_type = token_type; + list_start.name = name.clone(); + list_end.name = name; tokenizer.map.add(list_item.2, 0, vec![list_start]); tokenizer.map.add(list_item.3 + 1, 0, vec![list_end]); diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs index dec25b8..b605c0f 100644 --- a/src/construct/paragraph.rs +++ b/src/construct/paragraph.rs @@ -32,9 +32,9 @@ //! [code_text]: crate::construct::code_text //! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{ContentType, EventType, Tokenizer}; +use crate::event::{Content, Kind, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; use crate::util::skip::opt as skip_opt; /// Before a paragraph. @@ -47,9 +47,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => unreachable!("unexpected eol/eof"), _ => { - tokenizer.enter(Token::Paragraph); - tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); - State::Retry(Name::ParagraphInside) + tokenizer.enter(Name::Paragraph); + tokenizer.enter_with_content(Name::Data, Some(Content::Text)); + State::Retry(StateName::ParagraphInside) } } } @@ -63,8 +63,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::Data); - tokenizer.exit(Token::Paragraph); + tokenizer.exit(Name::Data); + tokenizer.exit(Name::Paragraph); tokenizer.register_resolver_before("paragraph".to_string(), Box::new(resolve)); // You’d be interrupting. tokenizer.interrupt = true; @@ -72,7 +72,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.consume(); - State::Next(Name::ParagraphInside) + State::Next(StateName::ParagraphInside) } } } @@ -85,21 +85,21 @@ pub fn resolve(tokenizer: &mut Tokenizer) { while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.event_type == EventType::Enter && event.token_type == Token::Paragraph { + if event.kind == Kind::Enter && event.name == Name::Paragraph { // Exit:Paragraph let mut exit_index = index + 3; let mut enter_next_index = - skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]); + skip_opt(&tokenizer.events, exit_index + 1, &[Name::LineEnding]); // Enter:Paragraph enter_next_index = skip_opt( &tokenizer.events, enter_next_index, - &[Token::SpaceOrTab, Token::BlockQuotePrefix], + &[Name::SpaceOrTab, Name::BlockQuotePrefix], ); // Find future `Paragraphs`. while enter_next_index < tokenizer.events.len() - && tokenizer.events[enter_next_index].token_type == Token::Paragraph + && tokenizer.events[enter_next_index].name == Name::Paragraph { // Remove Exit:Paragraph, Enter:LineEnding, Exit:LineEnding, Enter:Paragraph. tokenizer.map.add(exit_index, 3, vec![]); @@ -123,12 +123,11 @@ pub fn resolve(tokenizer: &mut Tokenizer) { // Potential next start. exit_index = enter_next_index + 3; - enter_next_index = - skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]); + enter_next_index = skip_opt(&tokenizer.events, exit_index + 1, &[Name::LineEnding]); enter_next_index = skip_opt( &tokenizer.events, enter_next_index, - &[Token::SpaceOrTab, Token::BlockQuotePrefix], + &[Name::SpaceOrTab, Name::BlockQuotePrefix], ); } diff --git a/src/construct/partial_bom.rs b/src/construct/partial_bom.rs index cca0770..74ca04e 100644 --- a/src/construct/partial_bom.rs +++ b/src/construct/partial_bom.rs @@ -10,8 +10,8 @@ //! //! * [`micromark/lib/preprocess.js` in `micromark`](https://github.com/micromark/micromark/blob/ed23453/packages/micromark/dev/lib/preprocess.js#L54-L60) -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF]; @@ -24,8 +24,8 @@ const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF]; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.current == Some(BOM[0]) { - tokenizer.enter(Token::ByteOrderMark); - State::Retry(Name::BomInside) + tokenizer.enter(Name::ByteOrderMark); + State::Retry(StateName::BomInside) } else { State::Nok } @@ -42,11 +42,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size += 1; tokenizer.consume(); if tokenizer.tokenize_state.size == BOM.len() { - tokenizer.exit(Token::ByteOrderMark); + tokenizer.exit(Name::ByteOrderMark); tokenizer.tokenize_state.size = 0; State::Ok } else { - State::Next(Name::BomInside) + State::Next(StateName::BomInside) } } else { tokenizer.tokenize_state.size = 0; diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs index c05aaa5..fda021e 100644 --- a/src/construct/partial_data.rs +++ b/src/construct/partial_data.rs @@ -6,9 +6,9 @@ //! [string]: crate::content::string //! [text]: crate::content::text -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{EventType, Tokenizer}; +use crate::event::{Kind, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; /// At the beginning of data. /// @@ -20,11 +20,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Make sure to eat the first `markers`. Some(byte) if tokenizer.tokenize_state.markers.contains(&byte) => { - tokenizer.enter(Token::Data); + tokenizer.enter(Name::Data); tokenizer.consume(); - State::Next(Name::DataInside) + State::Next(StateName::DataInside) } - _ => State::Retry(Name::DataAtBreak), + _ => State::Retry(StateName::DataAtBreak), } } @@ -38,18 +38,18 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Ok, Some(b'\n') => { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::DataAtBreak) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::DataAtBreak) } Some(byte) if tokenizer.tokenize_state.markers.contains(&byte) => { tokenizer.register_resolver_before("data".to_string(), Box::new(resolve_data)); State::Ok } _ => { - tokenizer.enter(Token::Data); - State::Retry(Name::DataInside) + tokenizer.enter(Name::Data); + State::Retry(StateName::DataInside) } } } @@ -68,11 +68,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { }; if done { - tokenizer.exit(Token::Data); - State::Retry(Name::DataAtBreak) + tokenizer.exit(Name::Data); + State::Retry(StateName::DataAtBreak) } else { tokenizer.consume(); - State::Next(Name::DataInside) + State::Next(StateName::DataInside) } } @@ -85,13 +85,13 @@ pub fn resolve_data(tokenizer: &mut Tokenizer) { while index < len { let event = &tokenizer.events[index]; - if event.event_type == EventType::Enter && event.token_type == Token::Data { + if event.kind == Kind::Enter && event.name == Name::Data { let exit_index = index + 1; let mut exit_far_index = exit_index; // Find multiple `data` events. while exit_far_index + 1 < len - && tokenizer.events[exit_far_index + 1].token_type == Token::Data + && tokenizer.events[exit_far_index + 1].name == Name::Data { exit_far_index += 2; } diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs index 5aa0539..c1c1e10 100644 --- a/src/construct/partial_destination.rs +++ b/src/construct/partial_destination.rs @@ -71,9 +71,9 @@ //! [label_end]: crate::construct::label_end //! [sanitize_uri]: crate::util::sanitize_uri -use crate::state::{Name, State}; -use crate::token::Token; -use crate::tokenizer::{ContentType, Tokenizer}; +use crate::event::{Content, Name}; +use crate::state::{Name as StateName, State}; +use crate::tokenizer::Tokenizer; /// Before a destination. /// @@ -91,7 +91,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(tokenizer.tokenize_state.token_3.clone()); tokenizer.consume(); tokenizer.exit(tokenizer.tokenize_state.token_3.clone()); - State::Next(Name::DestinationEnclosedBefore) + State::Next(StateName::DestinationEnclosedBefore) } // ASCII control, space, closing paren, but *not* `\0`. None | Some(0x01..=0x1F | b' ' | b')' | 0x7F) => State::Nok, @@ -99,8 +99,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(tokenizer.tokenize_state.token_1.clone()); tokenizer.enter(tokenizer.tokenize_state.token_4.clone()); tokenizer.enter(tokenizer.tokenize_state.token_5.clone()); - tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(Name::DestinationRaw) + tokenizer.enter_with_content(Name::Data, Some(Content::String)); + State::Retry(StateName::DestinationRaw) } } } @@ -121,8 +121,8 @@ pub fn enclosed_before(tokenizer: &mut Tokenizer) -> State { State::Ok } else { tokenizer.enter(tokenizer.tokenize_state.token_5.clone()); - tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(Name::DestinationEnclosed) + tokenizer.enter_with_content(Name::Data, Some(Content::String)); + State::Retry(StateName::DestinationEnclosed) } } @@ -136,17 +136,17 @@ pub fn enclosed(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n' | b'<') => State::Nok, Some(b'>') => { - tokenizer.exit(Token::Data); + tokenizer.exit(Name::Data); tokenizer.exit(tokenizer.tokenize_state.token_5.clone()); - State::Retry(Name::DestinationEnclosedBefore) + State::Retry(StateName::DestinationEnclosedBefore) } Some(b'\\') => { tokenizer.consume(); - State::Next(Name::DestinationEnclosedEscape) + State::Next(StateName::DestinationEnclosedEscape) } _ => { tokenizer.consume(); - State::Next(Name::DestinationEnclosed) + State::Next(StateName::DestinationEnclosed) } } } @@ -161,9 +161,9 @@ pub fn enclosed_escape(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'<' | b'>' | b'\\') => { tokenizer.consume(); - State::Next(Name::DestinationEnclosed) + State::Next(StateName::DestinationEnclosed) } - _ => State::Retry(Name::DestinationEnclosed), + _ => State::Retry(StateName::DestinationEnclosed), } } @@ -176,7 +176,7 @@ pub fn enclosed_escape(tokenizer: &mut Tokenizer) -> State { pub fn raw(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\t' | b'\n' | b' ' | b')') if tokenizer.tokenize_state.size == 0 => { - tokenizer.exit(Token::Data); + tokenizer.exit(Name::Data); tokenizer.exit(tokenizer.tokenize_state.token_5.clone()); tokenizer.exit(tokenizer.tokenize_state.token_4.clone()); tokenizer.exit(tokenizer.tokenize_state.token_1.clone()); @@ -186,7 +186,7 @@ pub fn raw(tokenizer: &mut Tokenizer) -> State { Some(b'(') if tokenizer.tokenize_state.size < tokenizer.tokenize_state.size_b => { tokenizer.consume(); tokenizer.tokenize_state.size += 1; - State::Next(Name::DestinationRaw) + State::Next(StateName::DestinationRaw) } // ASCII control (but *not* `\0`) and space and `(`. None | Some(0x01..=0x1F | b' ' | b'(' | 0x7F) => { @@ -196,15 +196,15 @@ pub fn raw(tokenizer: &mut Tokenizer) -> State { Some(b')') => { tokenizer.consume(); tokenizer.tokenize_state.size -= 1; - State::Next(Name::DestinationRaw) + State::Next(StateName::DestinationRaw) } Some(b'\\') => { tokenizer.consume(); - State::Next(Name::DestinationRawEscape) + State::Next(StateName::DestinationRawEscape) } Some(_) => { tokenizer.consume(); - State::Next(Name::DestinationRaw) + State::Next(StateName::DestinationRaw) } } } @@ -219,8 +219,8 @@ pub fn raw_escape(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'(' | b')' | b'\\') => { tokenizer.consume(); - State::Next(Name::DestinationRaw) + State::Next(StateName::DestinationRaw) } - _ => State::Retry(Name::DestinationRaw), + _ => State::Retry(StateName::DestinationRaw), } } diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs index 6e7c843..85769eb 100644 --- a/src/construct/partial_label.rs +++ b/src/construct/partial_label.rs @@ -58,12 +58,12 @@ //! [code_text]: crate::construct::code_text //! [link_reference_size_max]: crate::constant::LINK_REFERENCE_SIZE_MAX -use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; use crate::constant::LINK_REFERENCE_SIZE_MAX; -use crate::state::{Name, State}; +use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; +use crate::event::{Content, Name}; +use crate::state::{Name as StateName, State}; use crate::subtokenize::link; -use crate::token::Token; -use crate::tokenizer::{ContentType, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Before a label. /// @@ -79,7 +79,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); tokenizer.exit(tokenizer.tokenize_state.token_2.clone()); tokenizer.enter(tokenizer.tokenize_state.token_3.clone()); - State::Next(Name::LabelAtBreak) + State::Next(StateName::LabelAtBreak) } _ => State::Nok, } @@ -106,14 +106,14 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol_with_options( tokenizer, EolOptions { - content_type: Some(ContentType::String), + content_type: Some(Content::String), connect: tokenizer.tokenize_state.connect, }, ); tokenizer.attempt( name, - State::Next(Name::LabelEolAfter), - State::Next(Name::LabelAtBlankLine), + State::Next(StateName::LabelEolAfter), + State::Next(StateName::LabelAtBlankLine), ) } Some(b']') => { @@ -128,7 +128,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { State::Ok } _ => { - tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Name::Data, Some(Content::String)); if tokenizer.tokenize_state.connect { let index = tokenizer.events.len() - 1; @@ -137,7 +137,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; } - State::Retry(Name::LabelInside) + State::Retry(StateName::LabelInside) } } } @@ -152,7 +152,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn eol_after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; - State::Retry(Name::LabelAtBreak) + State::Retry(StateName::LabelAtBreak) } /// In a label, at a blank line. @@ -178,13 +178,13 @@ pub fn at_blank_line(tokenizer: &mut Tokenizer) -> State { pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n' | b'[' | b']') => { - tokenizer.exit(Token::Data); - State::Retry(Name::LabelAtBreak) + tokenizer.exit(Name::Data); + State::Retry(StateName::LabelAtBreak) } Some(byte) => { if tokenizer.tokenize_state.size > LINK_REFERENCE_SIZE_MAX { - tokenizer.exit(Token::Data); - State::Retry(Name::LabelAtBreak) + tokenizer.exit(Name::Data); + State::Retry(StateName::LabelAtBreak) } else { tokenizer.consume(); tokenizer.tokenize_state.size += 1; @@ -192,9 +192,9 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.seen = true; } State::Next(if matches!(byte, b'\\') { - Name::LabelEscape + StateName::LabelEscape } else { - Name::LabelInside + StateName::LabelInside }) } } @@ -212,8 +212,8 @@ pub fn escape(tokenizer: &mut Tokenizer) -> State { Some(b'[' | b'\\' | b']') => { tokenizer.consume(); tokenizer.tokenize_state.size += 1; - State::Next(Name::LabelInside) + State::Next(StateName::LabelInside) } - _ => State::Retry(Name::LabelInside), + _ => State::Retry(StateName::LabelInside), } } diff --git a/src/construct/partial_non_lazy_continuation.rs b/src/construct/partial_non_lazy_continuation.rs index 497c81e..76854c8 100644 --- a/src/construct/partial_non_lazy_continuation.rs +++ b/src/construct/partial_non_lazy_continuation.rs @@ -10,8 +10,8 @@ //! [code_indented]: crate::construct::code_indented //! [html_flow]: crate::construct::html_flow -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of continuation. @@ -24,10 +24,10 @@ use crate::tokenizer::Tokenizer; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => { - tokenizer.enter(Token::LineEnding); + tokenizer.enter(Name::LineEnding); tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::NonLazyContinuationAfter) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::NonLazyContinuationAfter) } _ => State::Nok, } diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs index a8e8f06..3fc9484 100644 --- a/src/construct/partial_space_or_tab.rs +++ b/src/construct/partial_space_or_tab.rs @@ -4,10 +4,10 @@ //! //! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js) -use crate::state::{Name, State}; +use crate::event::{Content, Name}; +use crate::state::{Name as StateName, State}; use crate::subtokenize::link; -use crate::token::Token; -use crate::tokenizer::{ContentType, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Options to parse `space_or_tab`. #[derive(Debug)] @@ -17,11 +17,11 @@ pub struct Options { /// Maximum allowed bytes (inclusive). pub max: usize, /// Token type to use for whitespace events. - pub kind: Token, + pub kind: Name, /// Connect this whitespace to the previous. pub connect: bool, /// Embedded content type to use. - pub content_type: Option<ContentType>, + pub content_type: Option<Content>, } /// Options to parse `space_or_tab` and one optional eol, but no blank line. @@ -30,7 +30,7 @@ pub struct EolOptions { /// Connect this whitespace to the previous. pub connect: bool, /// Embedded content type to use. - pub content_type: Option<ContentType>, + pub content_type: Option<Content>, } /// One or more `space_or_tab`. @@ -38,7 +38,7 @@ pub struct EolOptions { /// ```bnf /// space_or_tab ::= 1*( ' ' '\t' ) /// ``` -pub fn space_or_tab(tokenizer: &mut Tokenizer) -> Name { +pub fn space_or_tab(tokenizer: &mut Tokenizer) -> StateName { space_or_tab_min_max(tokenizer, 1, usize::MAX) } @@ -47,11 +47,11 @@ pub fn space_or_tab(tokenizer: &mut Tokenizer) -> Name { /// ```bnf /// space_or_tab_min_max ::= x*y( ' ' '\t' ) /// ``` -pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -> Name { +pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -> StateName { space_or_tab_with_options( tokenizer, Options { - kind: Token::SpaceOrTab, + kind: Name::SpaceOrTab, min, max, content_type: None, @@ -61,13 +61,13 @@ pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) - } /// `space_or_tab`, with the given options. -pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> Name { +pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> StateName { tokenizer.tokenize_state.space_or_tab_connect = options.connect; tokenizer.tokenize_state.space_or_tab_content_type = options.content_type; tokenizer.tokenize_state.space_or_tab_min = options.min; tokenizer.tokenize_state.space_or_tab_max = options.max; tokenizer.tokenize_state.space_or_tab_token = options.kind; - Name::SpaceOrTabStart + StateName::SpaceOrTabStart } /// `space_or_tab`, or optionally `space_or_tab`, one `eol`, and @@ -76,7 +76,7 @@ pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> /// ```bnf /// space_or_tab_eol ::= 1*( ' ' '\t' ) | 0*( ' ' '\t' ) eol 0*( ' ' '\t' ) /// ``` -pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> Name { +pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> StateName { space_or_tab_eol_with_options( tokenizer, EolOptions { @@ -87,10 +87,10 @@ pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> Name { } /// `space_or_tab_eol`, with the given options. -pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> Name { +pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> StateName { tokenizer.tokenize_state.space_or_tab_eol_content_type = options.content_type; tokenizer.tokenize_state.space_or_tab_eol_connect = options.connect; - Name::SpaceOrTabEolStart + StateName::SpaceOrTabEolStart } /// Before `space_or_tab`. @@ -114,9 +114,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.space_or_tab_connect = true; } - State::Retry(Name::SpaceOrTabInside) + State::Retry(StateName::SpaceOrTabInside) } - _ => State::Retry(Name::SpaceOrTabAfter), + _ => State::Retry(StateName::SpaceOrTabAfter), } } @@ -134,11 +134,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { { tokenizer.consume(); tokenizer.tokenize_state.space_or_tab_size += 1; - State::Next(Name::SpaceOrTabInside) + State::Next(StateName::SpaceOrTabInside) } _ => { tokenizer.exit(tokenizer.tokenize_state.space_or_tab_token.clone()); - State::Retry(Name::SpaceOrTabAfter) + State::Retry(StateName::SpaceOrTabAfter) } } } @@ -162,7 +162,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.space_or_tab_size = 0; tokenizer.tokenize_state.space_or_tab_max = 0; tokenizer.tokenize_state.space_or_tab_min = 0; - tokenizer.tokenize_state.space_or_tab_token = Token::SpaceOrTab; + tokenizer.tokenize_state.space_or_tab_token = Name::SpaceOrTab; state } @@ -170,7 +170,7 @@ pub fn eol_start(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_with_options( tokenizer, Options { - kind: Token::SpaceOrTab, + kind: Name::SpaceOrTab, min: 1, max: usize::MAX, content_type: tokenizer @@ -183,8 +183,8 @@ pub fn eol_start(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( name, - State::Next(Name::SpaceOrTabEolAfterFirst), - State::Next(Name::SpaceOrTabEolAtEol), + State::Next(StateName::SpaceOrTabEolAfterFirst), + State::Next(StateName::SpaceOrTabEolAtEol), ) } @@ -199,7 +199,7 @@ pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.space_or_tab_eol_connect = true; } - State::Retry(Name::SpaceOrTabEolAtEol) + State::Retry(StateName::SpaceOrTabEolAtEol) } /// `space_or_tab_eol`: after optionally first `space_or_tab`. @@ -212,7 +212,7 @@ pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State { pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State { if let Some(b'\n') = tokenizer.current { tokenizer.enter_with_content( - Token::LineEnding, + Name::LineEnding, tokenizer .tokenize_state .space_or_tab_eol_content_type @@ -231,8 +231,8 @@ pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State { } tokenizer.consume(); - tokenizer.exit(Token::LineEnding); - State::Next(Name::SpaceOrTabEolAfterEol) + tokenizer.exit(Name::LineEnding); + State::Next(StateName::SpaceOrTabEolAfterEol) } else { let ok = tokenizer.tokenize_state.space_or_tab_eol_ok; tokenizer.tokenize_state.space_or_tab_eol_content_type = None; @@ -258,7 +258,7 @@ pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_with_options( tokenizer, Options { - kind: Token::SpaceOrTab, + kind: Name::SpaceOrTab, min: 1, max: usize::MAX, content_type: tokenizer @@ -270,8 +270,8 @@ pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(Name::SpaceOrTabEolAfterMore), - State::Next(Name::SpaceOrTabEolAfterMore), + State::Next(StateName::SpaceOrTabEolAfterMore), + State::Next(StateName::SpaceOrTabEolAfterMore), ) } diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs index 11c28bd..be06c02 100644 --- a/src/construct/partial_title.rs +++ b/src/construct/partial_title.rs @@ -31,10 +31,10 @@ //! [label_end]: crate::construct::label_end use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; -use crate::state::{Name, State}; +use crate::event::{Content, Name}; +use crate::state::{Name as StateName, State}; use crate::subtokenize::link; -use crate::token::Token; -use crate::tokenizer::{ContentType, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Before a title. /// @@ -51,7 +51,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(tokenizer.tokenize_state.token_2.clone()); tokenizer.consume(); tokenizer.exit(tokenizer.tokenize_state.token_2.clone()); - State::Next(Name::TitleBegin) + State::Next(StateName::TitleBegin) } _ => State::Nok, } @@ -80,7 +80,7 @@ pub fn begin(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.enter(tokenizer.tokenize_state.token_3.clone()); - State::Retry(Name::TitleAtBreak) + State::Retry(StateName::TitleAtBreak) } } } @@ -102,25 +102,25 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol_with_options( tokenizer, EolOptions { - content_type: Some(ContentType::String), + content_type: Some(Content::String), connect: tokenizer.tokenize_state.connect, }, ); tokenizer.attempt( name, - State::Next(Name::TitleAfterEol), - State::Next(Name::TitleAtBlankLine), + State::Next(StateName::TitleAfterEol), + State::Next(StateName::TitleAtBlankLine), ) } Some(b'"' | b'\'' | b')') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.exit(tokenizer.tokenize_state.token_3.clone()); - State::Retry(Name::TitleBegin) + State::Retry(StateName::TitleBegin) } Some(_) => { - tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Name::Data, Some(Content::String)); if tokenizer.tokenize_state.connect { let index = tokenizer.events.len() - 1; @@ -129,7 +129,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; } - State::Retry(Name::TitleInside) + State::Retry(StateName::TitleInside) } } } @@ -143,7 +143,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn after_eol(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; - State::Retry(Name::TitleAtBreak) + State::Retry(StateName::TitleAtBreak) } /// In a title, at a blank line. @@ -169,21 +169,21 @@ pub fn at_blank_line(tokenizer: &mut Tokenizer) -> State { pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { - tokenizer.exit(Token::Data); - State::Retry(Name::TitleAtBreak) + tokenizer.exit(Name::Data); + State::Retry(StateName::TitleAtBreak) } Some(b'"' | b'\'' | b')') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { - tokenizer.exit(Token::Data); - State::Retry(Name::TitleAtBreak) + tokenizer.exit(Name::Data); + State::Retry(StateName::TitleAtBreak) } Some(byte) => { tokenizer.consume(); State::Next(if matches!(byte, b'\\') { - Name::TitleEscape + StateName::TitleEscape } else { - Name::TitleInside + StateName::TitleInside }) } } @@ -199,8 +199,8 @@ pub fn escape(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'"' | b'\'' | b')') => { tokenizer.consume(); - State::Next(Name::TitleInside) + State::Next(StateName::TitleInside) } - _ => State::Retry(Name::TitleInside), + _ => State::Retry(StateName::TitleInside), } } diff --git a/src/construct/partial_whitespace.rs b/src/construct/partial_whitespace.rs index 0905e10..688e7fd 100644 --- a/src/construct/partial_whitespace.rs +++ b/src/construct/partial_whitespace.rs @@ -46,8 +46,8 @@ //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element use crate::constant::HARD_BREAK_PREFIX_SIZE_MIN; -use crate::token::Token; -use crate::tokenizer::{Event, EventType, Tokenizer}; +use crate::event::{Event, Kind, Name}; +use crate::tokenizer::Tokenizer; use crate::util::slice::{Position, Slice}; /// Resolve whitespace. @@ -57,12 +57,12 @@ pub fn resolve_whitespace(tokenizer: &mut Tokenizer, hard_break: bool, trim_whol while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.event_type == EventType::Exit && event.token_type == Token::Data { + if event.kind == Kind::Exit && event.name == Name::Data { let trim_start = (trim_whole && index == 1) - || (index > 1 && tokenizer.events[index - 2].token_type == Token::LineEnding); + || (index > 1 && tokenizer.events[index - 2].name == Name::LineEnding); let trim_end = (trim_whole && index == tokenizer.events.len() - 1) || (index + 1 < tokenizer.events.len() - && tokenizer.events[index + 1].token_type == Token::LineEnding); + && tokenizer.events[index + 1].name == Name::LineEnding); trim_data(tokenizer, index, trim_start, trim_end, hard_break); } @@ -98,21 +98,21 @@ fn trim_data( } let diff = slice.bytes.len() - index; - let token_type = if hard_break + let name = if hard_break && spaces_only && diff >= HARD_BREAK_PREFIX_SIZE_MIN && exit_index + 1 < tokenizer.events.len() { - Token::HardBreakTrailing + Name::HardBreakTrailing } else { - Token::SpaceOrTab + Name::SpaceOrTab }; // The whole data is whitespace. // We can be very fast: we only change the token types. if index == 0 { - tokenizer.events[exit_index - 1].token_type = token_type.clone(); - tokenizer.events[exit_index].token_type = token_type; + tokenizer.events[exit_index - 1].name = name.clone(); + tokenizer.events[exit_index].name = name; return; } @@ -128,14 +128,14 @@ fn trim_data( 0, vec![ Event { - event_type: EventType::Enter, - token_type: token_type.clone(), + kind: Kind::Enter, + name: name.clone(), point: enter_point.clone(), link: None, }, Event { - event_type: EventType::Exit, - token_type, + kind: Kind::Exit, + name, point: exit_point, link: None, }, @@ -159,8 +159,8 @@ fn trim_data( // The whole data is whitespace. // We can be very fast: we only change the token types. if index == slice.bytes.len() { - tokenizer.events[exit_index - 1].token_type = Token::SpaceOrTab; - tokenizer.events[exit_index].token_type = Token::SpaceOrTab; + tokenizer.events[exit_index - 1].name = Name::SpaceOrTab; + tokenizer.events[exit_index].name = Name::SpaceOrTab; return; } @@ -176,14 +176,14 @@ fn trim_data( 0, vec![ Event { - event_type: EventType::Enter, - token_type: Token::SpaceOrTab, + kind: Kind::Enter, + name: Name::SpaceOrTab, point: enter_point, link: None, }, Event { - event_type: EventType::Exit, - token_type: Token::SpaceOrTab, + kind: Kind::Exit, + name: Name::SpaceOrTab, point: exit_point.clone(), link: None, }, diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index fc71d73..8e3c4f4 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -50,8 +50,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN}; -use crate::state::{Name, State}; -use crate::token::Token; +use crate::event::Name; +use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; /// Start of a thematic break. @@ -62,7 +62,7 @@ use crate::tokenizer::Tokenizer; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.parse_state.constructs.thematic_break { - tokenizer.enter(Token::ThematicBreak); + tokenizer.enter(Name::ThematicBreak); let name = space_or_tab_min_max( tokenizer, 0, @@ -73,7 +73,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { }, ); - tokenizer.attempt(name, State::Next(Name::ThematicBreakBefore), State::Nok) + tokenizer.attempt( + name, + State::Next(StateName::ThematicBreakBefore), + State::Nok, + ) } else { State::Nok } @@ -89,7 +93,7 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'*' | b'-' | b'_') => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); - State::Retry(Name::ThematicBreakAtBreak) + State::Retry(StateName::ThematicBreakAtBreak) } _ => State::Nok, } @@ -106,7 +110,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { None | Some(b'\n') if tokenizer.tokenize_state.size >= THEMATIC_BREAK_MARKER_COUNT_MIN => { tokenizer.tokenize_state.marker = 0; tokenizer.tokenize_state.size = 0; - tokenizer.exit(Token::ThematicBreak); + tokenizer.exit(Name::ThematicBreak); // Feel free to interrupt. tokenizer.interrupt = false; State::Ok @@ -114,8 +118,8 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { Some(b'*' | b'-' | b'_') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { - tokenizer.enter(Token::ThematicBreakSequence); - State::Retry(Name::ThematicBreakSequence) + tokenizer.enter(Name::ThematicBreakSequence); + State::Retry(StateName::ThematicBreakSequence) } _ => { tokenizer.tokenize_state.marker = 0; @@ -138,15 +142,15 @@ pub fn sequence(tokenizer: &mut Tokenizer) -> State { { tokenizer.consume(); tokenizer.tokenize_state.size += 1; - State::Next(Name::ThematicBreakSequence) + State::Next(StateName::ThematicBreakSequence) } _ => { - tokenizer.exit(Token::ThematicBreakSequence); + tokenizer.exit(Name::ThematicBreakSequence); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(Name::ThematicBreakAtBreak), - State::Next(Name::ThematicBreakAtBreak), + State::Next(StateName::ThematicBreakAtBreak), + State::Next(StateName::ThematicBreakAtBreak), ) } } |