diff options
Diffstat (limited to '')
36 files changed, 1229 insertions, 1237 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs index d61813d..7e873ca 100644 --- a/src/construct/attention.rs +++ b/src/construct/attention.rs @@ -51,8 +51,9 @@ //! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element //! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{Event, EventType, Point, State, StateName, Tokenizer}; +use crate::tokenizer::{Event, EventType, Point, Tokenizer}; use crate::unicode::PUNCTUATION; use crate::util::slice::Slice; @@ -120,7 +121,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { Some(b'*' | b'_') if tokenizer.parse_state.constructs.attention => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); tokenizer.enter(Token::AttentionSequence); - State::Retry(StateName::AttentionInside) + State::Retry(Name::AttentionInside) } _ => State::Nok, } @@ -136,7 +137,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'*' | b'_') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.consume(); - State::Next(StateName::AttentionInside) + State::Next(Name::AttentionInside) } _ => { tokenizer.exit(Token::AttentionSequence); diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index eef3840..b635d96 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -102,8 +102,9 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX}; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of an autolink. /// @@ -121,7 +122,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); tokenizer.exit(Token::AutolinkMarker); tokenizer.enter(Token::AutolinkProtocol); - State::Next(StateName::AutolinkOpen) + State::Next(Name::AutolinkOpen) } _ => State::Nok, } @@ -140,9 +141,9 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { // ASCII alphabetic. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::AutolinkSchemeOrEmailAtext) + State::Next(Name::AutolinkSchemeOrEmailAtext) } - _ => State::Retry(StateName::AutolinkEmailAtext), + _ => State::Retry(Name::AutolinkEmailAtext), } } @@ -160,9 +161,9 @@ pub fn scheme_or_email_atext(tokenizer: &mut Tokenizer) -> State { Some(b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { // Count the previous alphabetical from `open` too. tokenizer.tokenize_state.size = 1; - State::Retry(StateName::AutolinkSchemeInsideOrEmailAtext) + State::Retry(Name::AutolinkSchemeInsideOrEmailAtext) } - _ => State::Retry(StateName::AutolinkEmailAtext), + _ => State::Retry(Name::AutolinkEmailAtext), } } @@ -179,7 +180,7 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State { Some(b':') => { tokenizer.consume(); tokenizer.tokenize_state.size = 0; - State::Next(StateName::AutolinkUrlInside) + State::Next(Name::AutolinkUrlInside) } // ASCII alphanumeric and `+`, `-`, and `.`. Some(b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') @@ -187,11 +188,11 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State { { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(StateName::AutolinkSchemeInsideOrEmailAtext) + State::Next(Name::AutolinkSchemeInsideOrEmailAtext) } _ => { tokenizer.tokenize_state.size = 0; - State::Retry(StateName::AutolinkEmailAtext) + State::Retry(Name::AutolinkEmailAtext) } } } @@ -216,7 +217,7 @@ pub fn url_inside(tokenizer: &mut Tokenizer) -> State { None | Some(b'\0'..=0x1F | b' ' | b'<' | 0x7F) => State::Nok, Some(_) => { tokenizer.consume(); - State::Next(StateName::AutolinkUrlInside) + State::Next(Name::AutolinkUrlInside) } } } @@ -231,7 +232,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'@') => { tokenizer.consume(); - State::Next(StateName::AutolinkEmailAtSignOrDot) + State::Next(Name::AutolinkEmailAtSignOrDot) } // ASCII atext. // @@ -254,7 +255,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State { b'#'..=b'\'' | b'*' | b'+' | b'-'..=b'9' | b'=' | b'?' | b'A'..=b'Z' | b'^'..=b'~', ) => { tokenizer.consume(); - State::Next(StateName::AutolinkEmailAtext) + State::Next(Name::AutolinkEmailAtext) } _ => State::Nok, } @@ -269,9 +270,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State { pub fn email_at_sign_or_dot(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // ASCII alphanumeric. - Some(b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { - State::Retry(StateName::AutolinkEmailValue) - } + Some(b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => State::Retry(Name::AutolinkEmailValue), _ => State::Nok, } } @@ -287,7 +286,7 @@ pub fn email_label(tokenizer: &mut Tokenizer) -> State { Some(b'.') => { tokenizer.tokenize_state.size = 0; tokenizer.consume(); - State::Next(StateName::AutolinkEmailAtSignOrDot) + State::Next(Name::AutolinkEmailAtSignOrDot) } Some(b'>') => { tokenizer.tokenize_state.size = 0; @@ -302,7 +301,7 @@ pub fn email_label(tokenizer: &mut Tokenizer) -> State { tokenizer.exit(Token::Autolink); State::Ok } - _ => State::Retry(StateName::AutolinkEmailValue), + _ => State::Retry(Name::AutolinkEmailValue), } } @@ -321,9 +320,9 @@ pub fn email_value(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.size < AUTOLINK_DOMAIN_SIZE_MAX => { let name = if matches!(tokenizer.current, Some(b'-')) { - StateName::AutolinkEmailValue + Name::AutolinkEmailValue } else { - StateName::AutolinkEmailLabel + Name::AutolinkEmailLabel }; tokenizer.tokenize_state.size += 1; tokenizer.consume(); diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index d7d4817..e8a06e9 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -33,7 +33,8 @@ //! [flow]: crate::content::flow use crate::construct::partial_space_or_tab::space_or_tab; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::state::{Name, State}; +use crate::tokenizer::Tokenizer; /// Start of a blank line. /// @@ -49,8 +50,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::BlankLineAfter), - State::Next(StateName::BlankLineAfter), + State::Next(Name::BlankLineAfter), + State::Next(Name::BlankLineAfter), ) } diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs index bbfad5b..7b8ce82 100644 --- a/src/construct/block_quote.rs +++ b/src/construct/block_quote.rs @@ -35,8 +35,9 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::space_or_tab_min_max; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of block quote. /// @@ -55,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(StateName::BlockQuoteBefore), State::Nok) + tokenizer.attempt(name, State::Next(Name::BlockQuoteBefore), State::Nok) } else { State::Nok } @@ -71,9 +72,9 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { tokenizer.enter(Token::BlockQuote); - State::Retry(StateName::BlockQuoteContBefore) + State::Retry(Name::BlockQuoteContBefore) } - _ => State::Retry(StateName::BlockQuoteContBefore), + _ => State::Retry(Name::BlockQuoteContBefore), } } @@ -94,11 +95,7 @@ pub fn cont_start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt( - name, - State::Next(StateName::BlockQuoteContBefore), - State::Nok, - ) + tokenizer.attempt(name, State::Next(Name::BlockQuoteContBefore), State::Nok) } /// After whitespace, before `>`. @@ -115,7 +112,7 @@ pub fn cont_before(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::BlockQuoteMarker); tokenizer.consume(); tokenizer.exit(Token::BlockQuoteMarker); - State::Next(StateName::BlockQuoteContAfter) + State::Next(Name::BlockQuoteContAfter) } _ => State::Nok, } diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index 52b2873..c3d5458 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -33,8 +33,9 @@ //! [character_reference]: crate::construct::character_reference //! [hard_break_escape]: crate::construct::hard_break_escape +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of a character escape. /// @@ -49,7 +50,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::CharacterEscapeMarker); tokenizer.consume(); tokenizer.exit(Token::CharacterEscapeMarker); - State::Next(StateName::CharacterEscapeInside) + State::Next(Name::CharacterEscapeInside) } _ => State::Nok, } @@ -61,7 +62,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// > | a\*b /// ^ /// ``` -// StateName::CharacterEscapeInside +// Name::CharacterEscapeInside pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // ASCII punctuation. diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index e1c7e79..435c115 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -65,8 +65,9 @@ use crate::constant::{ CHARACTER_REFERENCES, CHARACTER_REFERENCE_DECIMAL_SIZE_MAX, CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX, CHARACTER_REFERENCE_NAMED_SIZE_MAX, }; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; use crate::util::slice::Slice; /// Start of a character reference. @@ -86,7 +87,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::CharacterReferenceMarker); tokenizer.consume(); tokenizer.exit(Token::CharacterReferenceMarker); - State::Next(StateName::CharacterReferenceOpen) + State::Next(Name::CharacterReferenceOpen) } _ => State::Nok, } @@ -103,17 +104,17 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// > | a	b /// ^ /// ``` -// StateName::CharacterReferenceOpen +// Name::CharacterReferenceOpen pub fn open(tokenizer: &mut Tokenizer) -> State { if let Some(b'#') = tokenizer.current { tokenizer.enter(Token::CharacterReferenceMarkerNumeric); tokenizer.consume(); tokenizer.exit(Token::CharacterReferenceMarkerNumeric); - State::Next(StateName::CharacterReferenceNumeric) + State::Next(Name::CharacterReferenceNumeric) } else { tokenizer.tokenize_state.marker = b'&'; tokenizer.enter(Token::CharacterReferenceValue); - State::Retry(StateName::CharacterReferenceValue) + State::Retry(Name::CharacterReferenceValue) } } @@ -126,7 +127,7 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { /// > | a	b /// ^ /// ``` -// StateName::CharacterReferenceNumeric +// Name::CharacterReferenceNumeric pub fn numeric(tokenizer: &mut Tokenizer) -> State { if let Some(b'x' | b'X') = tokenizer.current { tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal); @@ -134,11 +135,11 @@ pub fn numeric(tokenizer: &mut Tokenizer) -> State { tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal); tokenizer.enter(Token::CharacterReferenceValue); tokenizer.tokenize_state.marker = b'x'; - State::Next(StateName::CharacterReferenceValue) + State::Next(Name::CharacterReferenceValue) } else { tokenizer.enter(Token::CharacterReferenceValue); tokenizer.tokenize_state.marker = b'#'; - State::Retry(StateName::CharacterReferenceValue) + State::Retry(Name::CharacterReferenceValue) } } @@ -202,7 +203,7 @@ pub fn value(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.size < max && test(&byte) { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - return State::Next(StateName::CharacterReferenceValue); + return State::Next(Name::CharacterReferenceValue); } } diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index 26e1148..0ce8d02 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -103,8 +103,9 @@ use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE}; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{ContentType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, Tokenizer}; use crate::util::slice::{Position, Slice}; /// Start of fenced code. @@ -130,7 +131,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(StateName::CodeFencedBeforeSequenceOpen), + State::Next(Name::CodeFencedBeforeSequenceOpen), State::Nok, ) } else { @@ -164,7 +165,7 @@ pub fn before_sequence_open(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); tokenizer.tokenize_state.size_c = prefix; tokenizer.enter(Token::CodeFencedFenceSequence); - State::Retry(StateName::CodeFencedSequenceOpen) + State::Retry(Name::CodeFencedSequenceOpen) } else { State::Nok } @@ -183,15 +184,15 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(StateName::CodeFencedSequenceOpen) + State::Next(Name::CodeFencedSequenceOpen) } _ if tokenizer.tokenize_state.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => { tokenizer.exit(Token::CodeFencedFenceSequence); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::CodeFencedInfoBefore), - State::Next(StateName::CodeFencedInfoBefore), + State::Next(Name::CodeFencedInfoBefore), + State::Next(Name::CodeFencedInfoBefore), ) } _ => { @@ -218,15 +219,15 @@ pub fn info_before(tokenizer: &mut Tokenizer) -> State { // Do not form containers. tokenizer.concrete = true; tokenizer.check( - StateName::NonLazyContinuationStart, - State::Next(StateName::CodeFencedAtNonLazyBreak), - State::Next(StateName::CodeFencedAfter), + Name::NonLazyContinuationStart, + State::Next(Name::CodeFencedAtNonLazyBreak), + State::Next(Name::CodeFencedAfter), ) } _ => { tokenizer.enter(Token::CodeFencedFenceInfo); tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(StateName::CodeFencedInfo) + State::Retry(Name::CodeFencedInfo) } } } @@ -244,7 +245,7 @@ pub fn info(tokenizer: &mut Tokenizer) -> State { None | Some(b'\n') => { tokenizer.exit(Token::Data); tokenizer.exit(Token::CodeFencedFenceInfo); - State::Retry(StateName::CodeFencedInfoBefore) + State::Retry(Name::CodeFencedInfoBefore) } Some(b'\t' | b' ') => { tokenizer.exit(Token::Data); @@ -252,8 +253,8 @@ pub fn info(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::CodeFencedMetaBefore), - State::Next(StateName::CodeFencedMetaBefore), + State::Next(Name::CodeFencedMetaBefore), + State::Next(Name::CodeFencedMetaBefore), ) } Some(b'`') if tokenizer.tokenize_state.marker == b'`' => { @@ -265,7 +266,7 @@ pub fn info(tokenizer: &mut Tokenizer) -> State { } Some(_) => { tokenizer.consume(); - State::Next(StateName::CodeFencedInfo) + State::Next(Name::CodeFencedInfo) } } } @@ -280,11 +281,11 @@ pub fn info(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn meta_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None | Some(b'\n') => State::Retry(StateName::CodeFencedInfoBefore), + None | Some(b'\n') => State::Retry(Name::CodeFencedInfoBefore), _ => { tokenizer.enter(Token::CodeFencedFenceMeta); tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(StateName::CodeFencedMeta) + State::Retry(Name::CodeFencedMeta) } } } @@ -302,7 +303,7 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State { None | Some(b'\n') => { tokenizer.exit(Token::Data); tokenizer.exit(Token::CodeFencedFenceMeta); - State::Retry(StateName::CodeFencedInfoBefore) + State::Retry(Name::CodeFencedInfoBefore) } Some(b'`') if tokenizer.tokenize_state.marker == b'`' => { tokenizer.concrete = false; @@ -313,7 +314,7 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.consume(); - State::Next(StateName::CodeFencedMeta) + State::Next(Name::CodeFencedMeta) } } } @@ -329,9 +330,9 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn at_non_lazy_break(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::CodeFencedCloseBefore, - State::Next(StateName::CodeFencedAfter), - State::Next(StateName::CodeFencedContentBefore), + Name::CodeFencedCloseBefore, + State::Next(Name::CodeFencedAfter), + State::Next(Name::CodeFencedContentBefore), ) } @@ -349,7 +350,7 @@ pub fn close_before(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::CodeFencedCloseStart) + State::Next(Name::CodeFencedCloseStart) } _ => unreachable!("expected eol"), } @@ -376,7 +377,7 @@ pub fn close_start(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(StateName::CodeFencedBeforeSequenceClose), + State::Next(Name::CodeFencedBeforeSequenceClose), State::Nok, ) } @@ -393,7 +394,7 @@ pub fn before_sequence_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.enter(Token::CodeFencedFenceSequence); - State::Retry(StateName::CodeFencedSequenceClose) + State::Retry(Name::CodeFencedSequenceClose) } _ => State::Nok, } @@ -412,7 +413,7 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.tokenize_state.size_b += 1; tokenizer.consume(); - State::Next(StateName::CodeFencedSequenceClose) + State::Next(Name::CodeFencedSequenceClose) } _ if tokenizer.tokenize_state.size_b >= CODE_FENCED_SEQUENCE_SIZE_MIN && tokenizer.tokenize_state.size_b >= tokenizer.tokenize_state.size => @@ -422,8 +423,8 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::CodeFencedAfterSequenceClose), - State::Next(StateName::CodeFencedAfterSequenceClose), + State::Next(Name::CodeFencedAfterSequenceClose), + State::Next(Name::CodeFencedAfterSequenceClose), ) } _ => { @@ -463,7 +464,7 @@ pub fn content_before(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::CodeFencedContentStart) + State::Next(Name::CodeFencedContentStart) } /// Before code content, definitely not before a closing fence. /// @@ -477,7 +478,7 @@ pub fn content_start(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_min_max(tokenizer, 0, tokenizer.tokenize_state.size_c); tokenizer.attempt( name, - State::Next(StateName::CodeFencedBeforeContentChunk), + State::Next(Name::CodeFencedBeforeContentChunk), State::Nok, ) } @@ -493,13 +494,13 @@ pub fn content_start(tokenizer: &mut Tokenizer) -> State { pub fn before_content_chunk(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => tokenizer.check( - StateName::NonLazyContinuationStart, - State::Next(StateName::CodeFencedAtNonLazyBreak), - State::Next(StateName::CodeFencedAfter), + Name::NonLazyContinuationStart, + State::Next(Name::CodeFencedAtNonLazyBreak), + State::Next(Name::CodeFencedAfter), ), _ => { tokenizer.enter(Token::CodeFlowChunk); - State::Retry(StateName::CodeFencedContentChunk) + State::Retry(Name::CodeFencedContentChunk) } } } @@ -516,11 +517,11 @@ pub fn content_chunk(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { tokenizer.exit(Token::CodeFlowChunk); - State::Retry(StateName::CodeFencedBeforeContentChunk) + State::Retry(Name::CodeFencedBeforeContentChunk) } _ => { tokenizer.consume(); - State::Next(StateName::CodeFencedContentChunk) + State::Next(Name::CodeFencedContentChunk) } } } diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index 36ae4c6..f442f27 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -47,8 +47,9 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::TAB_SIZE; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of code (indented). /// @@ -65,11 +66,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { if !tokenizer.interrupt && tokenizer.parse_state.constructs.code_indented { tokenizer.enter(Token::CodeIndented); let name = space_or_tab_min_max(tokenizer, TAB_SIZE, TAB_SIZE); - tokenizer.attempt( - name, - State::Next(StateName::CodeIndentedAtBreak), - State::Nok, - ) + tokenizer.attempt(name, State::Next(Name::CodeIndentedAtBreak), State::Nok) } else { State::Nok } @@ -83,15 +80,15 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn at_break(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None => State::Retry(StateName::CodeIndentedAfter), + None => State::Retry(Name::CodeIndentedAfter), Some(b'\n') => tokenizer.attempt( - StateName::CodeIndentedFurtherStart, - State::Next(StateName::CodeIndentedAtBreak), - State::Next(StateName::CodeIndentedAfter), + Name::CodeIndentedFurtherStart, + State::Next(Name::CodeIndentedAtBreak), + State::Next(Name::CodeIndentedAfter), ), _ => { tokenizer.enter(Token::CodeFlowChunk); - State::Retry(StateName::CodeIndentedInside) + State::Retry(Name::CodeIndentedInside) } } } @@ -106,11 +103,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { tokenizer.exit(Token::CodeFlowChunk); - State::Retry(StateName::CodeIndentedAtBreak) + State::Retry(Name::CodeIndentedAtBreak) } _ => { tokenizer.consume(); - State::Next(StateName::CodeIndentedInside) + State::Next(Name::CodeIndentedInside) } } } @@ -141,14 +138,14 @@ pub fn further_start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::CodeIndentedFurtherStart) + State::Next(Name::CodeIndentedFurtherStart) } _ if !tokenizer.lazy => { let name = space_or_tab_min_max(tokenizer, TAB_SIZE, TAB_SIZE); tokenizer.attempt( name, - State::Next(StateName::CodeIndentedFurtherEnd), - State::Next(StateName::CodeIndentedFurtherBegin), + State::Next(Name::CodeIndentedFurtherEnd), + State::Next(Name::CodeIndentedFurtherBegin), ) } _ => State::Nok, @@ -177,8 +174,8 @@ pub fn further_begin(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::CodeIndentedFurtherAfter), - State::Next(StateName::CodeIndentedFurtherAfter), + State::Next(Name::CodeIndentedFurtherAfter), + State::Next(Name::CodeIndentedFurtherAfter), ) } @@ -191,7 +188,7 @@ pub fn further_begin(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn further_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'\n') => State::Retry(StateName::CodeIndentedFurtherStart), + Some(b'\n') => State::Retry(Name::CodeIndentedFurtherStart), _ => State::Nok, } } diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index d7ada3d..f626cfb 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -83,8 +83,9 @@ //! [code_fenced]: crate::construct::code_fenced //! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of code (text). /// @@ -105,7 +106,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { { tokenizer.enter(Token::CodeText); tokenizer.enter(Token::CodeTextSequence); - State::Retry(StateName::CodeTextSequenceOpen) + State::Retry(Name::CodeTextSequenceOpen) } _ => State::Nok, } @@ -121,10 +122,10 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { if let Some(b'`') = tokenizer.current { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(StateName::CodeTextSequenceOpen) + State::Next(Name::CodeTextSequenceOpen) } else { tokenizer.exit(Token::CodeTextSequence); - State::Retry(StateName::CodeTextBetween) + State::Retry(Name::CodeTextBetween) } } @@ -144,15 +145,15 @@ pub fn between(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::CodeTextBetween) + State::Next(Name::CodeTextBetween) } Some(b'`') => { tokenizer.enter(Token::CodeTextSequence); - State::Retry(StateName::CodeTextSequenceClose) + State::Retry(Name::CodeTextSequenceClose) } _ => { tokenizer.enter(Token::CodeTextData); - State::Retry(StateName::CodeTextData) + State::Retry(Name::CodeTextData) } } } @@ -167,11 +168,11 @@ pub fn data(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n' | b'`') => { tokenizer.exit(Token::CodeTextData); - State::Retry(StateName::CodeTextBetween) + State::Retry(Name::CodeTextBetween) } _ => { tokenizer.consume(); - State::Next(StateName::CodeTextData) + State::Next(Name::CodeTextData) } } } @@ -187,7 +188,7 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { Some(b'`') => { tokenizer.tokenize_state.size_b += 1; tokenizer.consume(); - State::Next(StateName::CodeTextSequenceClose) + State::Next(Name::CodeTextSequenceClose) } _ => { if tokenizer.tokenize_state.size == tokenizer.tokenize_state.size_b { @@ -203,7 +204,7 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State { tokenizer.events[index - 1].token_type = Token::CodeTextData; tokenizer.events[index].token_type = Token::CodeTextData; tokenizer.tokenize_state.size_b = 0; - State::Retry(StateName::CodeTextBetween) + State::Retry(Name::CodeTextBetween) } } } diff --git a/src/construct/definition.rs b/src/construct/definition.rs index 5db611b..394375f 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -94,8 +94,9 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_eol}; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; use crate::util::skip::opt_back as skip_opt_back; /// At the start of a definition. @@ -122,8 +123,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::DefinitionBefore), - State::Next(StateName::DefinitionBefore), + State::Next(Name::DefinitionBefore), + State::Next(Name::DefinitionBefore), ) } else { State::Nok @@ -143,8 +144,8 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.token_2 = Token::DefinitionLabelMarker; tokenizer.tokenize_state.token_3 = Token::DefinitionLabelString; tokenizer.attempt( - StateName::LabelStart, - State::Next(StateName::DefinitionLabelAfter), + Name::LabelStart, + State::Next(Name::DefinitionLabelAfter), State::Nok, ) } @@ -168,7 +169,7 @@ pub fn label_after(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::DefinitionMarker); tokenizer.consume(); tokenizer.exit(Token::DefinitionMarker); - State::Next(StateName::DefinitionMarkerAfter) + State::Next(Name::DefinitionMarkerAfter) } _ => State::Nok, } @@ -184,8 +185,8 @@ pub fn marker_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(StateName::DefinitionDestinationBefore), - State::Next(StateName::DefinitionDestinationBefore), + State::Next(Name::DefinitionDestinationBefore), + State::Next(Name::DefinitionDestinationBefore), ) } @@ -203,9 +204,9 @@ pub fn destination_before(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.token_5 = Token::DefinitionDestinationString; tokenizer.tokenize_state.size_b = usize::MAX; tokenizer.attempt( - StateName::DestinationStart, - State::Next(StateName::DefinitionDestinationAfter), - State::Next(StateName::DefinitionDestinationMissing), + Name::DestinationStart, + State::Next(Name::DefinitionDestinationAfter), + State::Next(Name::DefinitionDestinationMissing), ) } @@ -223,9 +224,9 @@ pub fn destination_after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.token_5 = Token::Data; tokenizer.tokenize_state.size_b = 0; tokenizer.attempt( - StateName::DefinitionTitleBefore, - State::Next(StateName::DefinitionAfter), - State::Next(StateName::DefinitionAfter), + Name::DefinitionTitleBefore, + State::Next(Name::DefinitionAfter), + State::Next(Name::DefinitionAfter), ) } @@ -252,8 +253,8 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::DefinitionAfterWhitespace), - State::Next(StateName::DefinitionAfterWhitespace), + State::Next(Name::DefinitionAfterWhitespace), + State::Next(Name::DefinitionAfterWhitespace), ) } @@ -289,7 +290,7 @@ pub fn title_before(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(StateName::DefinitionTitleBeforeMarker), + State::Next(Name::DefinitionTitleBeforeMarker), State::Nok, ) } @@ -306,8 +307,8 @@ pub fn title_before_marker(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.token_2 = Token::DefinitionTitleMarker; tokenizer.tokenize_state.token_3 = Token::DefinitionTitleString; tokenizer.attempt( - StateName::TitleStart, - State::Next(StateName::DefinitionTitleAfter), + Name::TitleStart, + State::Next(Name::DefinitionTitleAfter), State::Nok, ) } @@ -325,8 +326,8 @@ pub fn title_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::DefinitionTitleAfterOptionalWhitespace), - State::Next(StateName::DefinitionTitleAfterOptionalWhitespace), + State::Next(Name::DefinitionTitleAfterOptionalWhitespace), + State::Next(Name::DefinitionTitleAfterOptionalWhitespace), ) } diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index fc2cbdf..482648f 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -39,8 +39,9 @@ //! [hard_break_trailing]: crate::construct::partial_whitespace //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of a hard break (escape). /// @@ -54,7 +55,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { Some(b'\\') if tokenizer.parse_state.constructs.hard_break_escape => { tokenizer.enter(Token::HardBreakEscape); tokenizer.consume(); - State::Next(StateName::HardBreakEscapeAfter) + State::Next(Name::HardBreakEscapeAfter) } _ => State::Nok, } diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index d70f7db..12bc5b1 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -54,10 +54,11 @@ //! [wiki-setext]: https://en.wikipedia.org/wiki/Setext //! [atx]: http://www.aaronsw.com/2002/atx/ -use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE}; +use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{ContentType, Event, EventType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, Event, EventType, Tokenizer}; /// Start of a heading (atx). /// @@ -77,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(StateName::HeadingAtxBefore), State::Nok) + tokenizer.attempt(name, State::Next(Name::HeadingAtxBefore), State::Nok) } else { State::Nok } @@ -92,7 +93,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn before(tokenizer: &mut Tokenizer) -> State { if Some(b'#') == tokenizer.current { tokenizer.enter(Token::HeadingAtxSequence); - State::Retry(StateName::HeadingAtxSequenceOpen) + State::Retry(Name::HeadingAtxSequenceOpen) } else { State::Nok } @@ -109,18 +110,18 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State { None | Some(b'\n') if tokenizer.tokenize_state.size > 0 => { tokenizer.tokenize_state.size = 0; tokenizer.exit(Token::HeadingAtxSequence); - State::Retry(StateName::HeadingAtxAtBreak) + State::Retry(Name::HeadingAtxAtBreak) } Some(b'#') if tokenizer.tokenize_state.size < HEADING_ATX_OPENING_FENCE_SIZE_MAX => { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(StateName::HeadingAtxSequenceOpen) + State::Next(Name::HeadingAtxSequenceOpen) } _ if tokenizer.tokenize_state.size > 0 => { tokenizer.tokenize_state.size = 0; tokenizer.exit(Token::HeadingAtxSequence); let name = space_or_tab(tokenizer); - tokenizer.attempt(name, State::Next(StateName::HeadingAtxAtBreak), State::Nok) + tokenizer.attempt(name, State::Next(Name::HeadingAtxAtBreak), State::Nok) } _ => { tokenizer.tokenize_state.size = 0; @@ -146,15 +147,15 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { } Some(b'\t' | b' ') => { let name = space_or_tab(tokenizer); - tokenizer.attempt(name, State::Next(StateName::HeadingAtxAtBreak), State::Nok) + tokenizer.attempt(name, State::Next(Name::HeadingAtxAtBreak), State::Nok) } Some(b'#') => { tokenizer.enter(Token::HeadingAtxSequence); - State::Retry(StateName::HeadingAtxSequenceFurther) + State::Retry(Name::HeadingAtxSequenceFurther) } Some(_) => { tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); - State::Retry(StateName::HeadingAtxData) + State::Retry(Name::HeadingAtxData) } } } @@ -170,10 +171,10 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { pub fn sequence_further(tokenizer: &mut Tokenizer) -> State { if let Some(b'#') = tokenizer.current { tokenizer.consume(); - State::Next(StateName::HeadingAtxSequenceFurther) + State::Next(Name::HeadingAtxSequenceFurther) } else { tokenizer.exit(Token::HeadingAtxSequence); - State::Retry(StateName::HeadingAtxAtBreak) + State::Retry(Name::HeadingAtxAtBreak) } } @@ -188,11 +189,11 @@ pub fn data(tokenizer: &mut Tokenizer) -> State { // Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text. None | Some(b'\t' | b'\n' | b' ') => { tokenizer.exit(Token::Data); - State::Retry(StateName::HeadingAtxAtBreak) + State::Retry(Name::HeadingAtxAtBreak) } _ => { tokenizer.consume(); - State::Next(StateName::HeadingAtxData) + State::Next(Name::HeadingAtxData) } } } diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index f653d75..8b45fff 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -59,8 +59,9 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{EventType, State, StateName, Tokenizer}; +use crate::tokenizer::{EventType, Tokenizer}; use crate::util::skip::opt_back as skip_opt_back; /// At a line ending, presumably an underline. @@ -93,11 +94,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { }, ); - tokenizer.attempt( - name, - State::Next(StateName::HeadingSetextBefore), - State::Nok, - ) + tokenizer.attempt(name, State::Next(Name::HeadingSetextBefore), State::Nok) } else { State::Nok } @@ -115,7 +112,7 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { Some(b'-' | b'=') => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); tokenizer.enter(Token::HeadingSetextUnderline); - State::Retry(StateName::HeadingSetextInside) + State::Retry(Name::HeadingSetextInside) } _ => State::Nok, } @@ -132,7 +129,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-' | b'=') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.consume(); - State::Next(StateName::HeadingSetextInside) + State::Next(Name::HeadingSetextInside) } _ => { tokenizer.tokenize_state.marker = 0; @@ -140,8 +137,8 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::HeadingSetextAfter), - State::Next(StateName::HeadingSetextAfter), + State::Next(Name::HeadingSetextAfter), + State::Next(Name::HeadingSetextAfter), ) } } diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index 7a346e9..c9f8610 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -104,8 +104,9 @@ use crate::constant::{ use crate::construct::partial_space_or_tab::{ space_or_tab_with_options, Options as SpaceOrTabOptions, }; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; use crate::util::slice::Slice; /// Symbol for `<script>` (condition 1). @@ -147,7 +148,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { }, ); - tokenizer.attempt(name, State::Next(StateName::HtmlFlowBefore), State::Nok) + tokenizer.attempt(name, State::Next(Name::HtmlFlowBefore), State::Nok) } else { State::Nok } @@ -163,7 +164,7 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { if Some(b'<') == tokenizer.current { tokenizer.enter(Token::HtmlFlowData); tokenizer.consume(); - State::Next(StateName::HtmlFlowOpen) + State::Next(Name::HtmlFlowOpen) } else { State::Nok } @@ -183,13 +184,13 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'!') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowDeclarationOpen) + State::Next(Name::HtmlFlowDeclarationOpen) } Some(b'/') => { tokenizer.consume(); tokenizer.tokenize_state.seen = true; tokenizer.tokenize_state.start = tokenizer.point.index; - State::Next(StateName::HtmlFlowTagCloseStart) + State::Next(Name::HtmlFlowTagCloseStart) } Some(b'?') => { tokenizer.tokenize_state.marker = INSTRUCTION; @@ -198,12 +199,12 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { tokenizer.concrete = true; // While we’re in an instruction instead of a declaration, we’re on a `?` // right now, so we do need to search for `>`, similar to declarations. - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.tokenize_state.start = tokenizer.point.index; - State::Retry(StateName::HtmlFlowTagName) + State::Retry(Name::HtmlFlowTagName) } _ => State::Nok, } @@ -224,19 +225,19 @@ pub fn declaration_open(tokenizer: &mut Tokenizer) -> State { Some(b'-') => { tokenizer.consume(); tokenizer.tokenize_state.marker = COMMENT; - State::Next(StateName::HtmlFlowCommentOpenInside) + State::Next(Name::HtmlFlowCommentOpenInside) } Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); tokenizer.tokenize_state.marker = DECLARATION; // Do not form containers. tokenizer.concrete = true; - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } Some(b'[') => { tokenizer.consume(); tokenizer.tokenize_state.marker = CDATA; - State::Next(StateName::HtmlFlowCdataOpenInside) + State::Next(Name::HtmlFlowCdataOpenInside) } _ => State::Nok, } @@ -253,7 +254,7 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); // Do not form containers. tokenizer.concrete = true; - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -275,9 +276,9 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size = 0; // Do not form containers. tokenizer.concrete = true; - State::Next(StateName::HtmlFlowContinuation) + State::Next(Name::HtmlFlowContinuation) } else { - State::Next(StateName::HtmlFlowCdataOpenInside) + State::Next(Name::HtmlFlowCdataOpenInside) } } else { tokenizer.tokenize_state.marker = 0; @@ -295,7 +296,7 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State { pub fn tag_close_start(tokenizer: &mut Tokenizer) -> State { if let Some(b'A'..=b'Z' | b'a'..=b'z') = tokenizer.current { tokenizer.consume(); - State::Next(StateName::HtmlFlowTagName) + State::Next(Name::HtmlFlowTagName) } else { tokenizer.tokenize_state.seen = false; tokenizer.tokenize_state.start = 0; @@ -334,17 +335,17 @@ pub fn tag_name(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.marker = RAW; // Do not form containers. tokenizer.concrete = true; - State::Retry(StateName::HtmlFlowContinuation) + State::Retry(Name::HtmlFlowContinuation) } else if HTML_BLOCK_NAMES.contains(&name.as_str()) { tokenizer.tokenize_state.marker = BASIC; if slash { tokenizer.consume(); - State::Next(StateName::HtmlFlowBasicSelfClosing) + State::Next(Name::HtmlFlowBasicSelfClosing) } else { // Do not form containers. tokenizer.concrete = true; - State::Retry(StateName::HtmlFlowContinuation) + State::Retry(Name::HtmlFlowContinuation) } } else { tokenizer.tokenize_state.marker = COMPLETE; @@ -354,16 +355,16 @@ pub fn tag_name(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.marker = 0; State::Nok } else if closing_tag { - State::Retry(StateName::HtmlFlowCompleteClosingTagAfter) + State::Retry(Name::HtmlFlowCompleteClosingTagAfter) } else { - State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore) + State::Retry(Name::HtmlFlowCompleteAttributeNameBefore) } } } // ASCII alphanumerical and `-`. Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowTagName) + State::Next(Name::HtmlFlowTagName) } Some(_) => { tokenizer.tokenize_state.seen = false; @@ -383,7 +384,7 @@ pub fn basic_self_closing(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); // Do not form containers. tokenizer.concrete = true; - State::Next(StateName::HtmlFlowContinuation) + State::Next(Name::HtmlFlowContinuation) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -400,9 +401,9 @@ pub fn complete_closing_tag_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteClosingTagAfter) + State::Next(Name::HtmlFlowCompleteClosingTagAfter) } - _ => State::Retry(StateName::HtmlFlowCompleteEnd), + _ => State::Retry(Name::HtmlFlowCompleteEnd), } } @@ -429,18 +430,18 @@ pub fn complete_attribute_name_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeNameBefore) + State::Next(Name::HtmlFlowCompleteAttributeNameBefore) } Some(b'/') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteEnd) + State::Next(Name::HtmlFlowCompleteEnd) } // ASCII alphanumerical and `:` and `_`. Some(b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeName) + State::Next(Name::HtmlFlowCompleteAttributeName) } - _ => State::Retry(StateName::HtmlFlowCompleteEnd), + _ => State::Retry(Name::HtmlFlowCompleteEnd), } } @@ -459,9 +460,9 @@ pub fn complete_attribute_name(tokenizer: &mut Tokenizer) -> State { // ASCII alphanumerical and `-`, `.`, `:`, and `_`. Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeName) + State::Next(Name::HtmlFlowCompleteAttributeName) } - _ => State::Retry(StateName::HtmlFlowCompleteAttributeNameAfter), + _ => State::Retry(Name::HtmlFlowCompleteAttributeNameAfter), } } @@ -478,13 +479,13 @@ pub fn complete_attribute_name_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeNameAfter) + State::Next(Name::HtmlFlowCompleteAttributeNameAfter) } Some(b'=') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeValueBefore) + State::Next(Name::HtmlFlowCompleteAttributeValueBefore) } - _ => State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore), + _ => State::Retry(Name::HtmlFlowCompleteAttributeNameBefore), } } @@ -505,14 +506,14 @@ pub fn complete_attribute_value_before(tokenizer: &mut Tokenizer) -> State { } Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeValueBefore) + State::Next(Name::HtmlFlowCompleteAttributeValueBefore) } Some(b'"' | b'\'') => { tokenizer.tokenize_state.marker_b = tokenizer.current.unwrap(); tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeValueQuoted) + State::Next(Name::HtmlFlowCompleteAttributeValueQuoted) } - _ => State::Retry(StateName::HtmlFlowCompleteAttributeValueUnquoted), + _ => State::Retry(Name::HtmlFlowCompleteAttributeValueUnquoted), } } @@ -534,11 +535,11 @@ pub fn complete_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { Some(b'"' | b'\'') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker_b => { tokenizer.tokenize_state.marker_b = 0; tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeValueQuotedAfter) + State::Next(Name::HtmlFlowCompleteAttributeValueQuotedAfter) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeValueQuoted) + State::Next(Name::HtmlFlowCompleteAttributeValueQuoted) } } } @@ -552,11 +553,11 @@ pub fn complete_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { pub fn complete_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\t' | b'\n' | b' ' | b'"' | b'\'' | b'/' | b'<' | b'=' | b'>' | b'`') => { - State::Retry(StateName::HtmlFlowCompleteAttributeNameAfter) + State::Retry(Name::HtmlFlowCompleteAttributeNameAfter) } Some(_) => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAttributeValueUnquoted) + State::Next(Name::HtmlFlowCompleteAttributeValueUnquoted) } } } @@ -570,7 +571,7 @@ pub fn complete_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn complete_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State { if let Some(b'\t' | b' ' | b'/' | b'>') = tokenizer.current { - State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore) + State::Retry(Name::HtmlFlowCompleteAttributeNameBefore) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -586,7 +587,7 @@ pub fn complete_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State pub fn complete_end(tokenizer: &mut Tokenizer) -> State { if let Some(b'>') = tokenizer.current { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAfter) + State::Next(Name::HtmlFlowCompleteAfter) } else { tokenizer.tokenize_state.marker = 0; State::Nok @@ -604,11 +605,11 @@ pub fn complete_after(tokenizer: &mut Tokenizer) -> State { None | Some(b'\n') => { // Do not form containers. tokenizer.concrete = true; - State::Retry(StateName::HtmlFlowContinuation) + State::Retry(Name::HtmlFlowContinuation) } Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowCompleteAfter) + State::Next(Name::HtmlFlowCompleteAfter) } Some(_) => { tokenizer.tokenize_state.marker = 0; @@ -631,39 +632,39 @@ pub fn continuation(tokenizer: &mut Tokenizer) -> State { { tokenizer.exit(Token::HtmlFlowData); tokenizer.check( - StateName::HtmlFlowBlankLineBefore, - State::Next(StateName::HtmlFlowContinuationAfter), - State::Next(StateName::HtmlFlowContinuationStart), + Name::HtmlFlowBlankLineBefore, + State::Next(Name::HtmlFlowContinuationAfter), + State::Next(Name::HtmlFlowContinuationStart), ) } // Note: important that this is after the basic/complete case. None | Some(b'\n') => { tokenizer.exit(Token::HtmlFlowData); - State::Retry(StateName::HtmlFlowContinuationStart) + State::Retry(Name::HtmlFlowContinuationStart) } Some(b'-') if tokenizer.tokenize_state.marker == COMMENT => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationCommentInside) + State::Next(Name::HtmlFlowContinuationCommentInside) } Some(b'<') if tokenizer.tokenize_state.marker == RAW => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationRawTagOpen) + State::Next(Name::HtmlFlowContinuationRawTagOpen) } Some(b'>') if tokenizer.tokenize_state.marker == DECLARATION => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationClose) + State::Next(Name::HtmlFlowContinuationClose) } Some(b'?') if tokenizer.tokenize_state.marker == INSTRUCTION => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } Some(b']') if tokenizer.tokenize_state.marker == CDATA => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationCdataInside) + State::Next(Name::HtmlFlowContinuationCdataInside) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuation) + State::Next(Name::HtmlFlowContinuation) } } } @@ -677,9 +678,9 @@ pub fn continuation(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn continuation_start(tokenizer: &mut Tokenizer) -> State { tokenizer.check( - StateName::NonLazyContinuationStart, - State::Next(StateName::HtmlFlowContinuationStartNonLazy), - State::Next(StateName::HtmlFlowContinuationAfter), + Name::NonLazyContinuationStart, + State::Next(Name::HtmlFlowContinuationStartNonLazy), + State::Next(Name::HtmlFlowContinuationAfter), ) } @@ -696,7 +697,7 @@ pub fn continuation_start_non_lazy(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::HtmlFlowContinuationBefore) + State::Next(Name::HtmlFlowContinuationBefore) } _ => unreachable!("expected eol"), } @@ -711,10 +712,10 @@ pub fn continuation_start_non_lazy(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn continuation_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None | Some(b'\n') => State::Retry(StateName::HtmlFlowContinuationStart), + None | Some(b'\n') => State::Retry(Name::HtmlFlowContinuationStart), _ => { tokenizer.enter(Token::HtmlFlowData); - State::Retry(StateName::HtmlFlowContinuation) + State::Retry(Name::HtmlFlowContinuation) } } } @@ -729,9 +730,9 @@ pub fn continuation_comment_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } - _ => State::Retry(StateName::HtmlFlowContinuation), + _ => State::Retry(Name::HtmlFlowContinuation), } } @@ -746,9 +747,9 @@ pub fn continuation_raw_tag_open(tokenizer: &mut Tokenizer) -> State { Some(b'/') => { tokenizer.consume(); tokenizer.tokenize_state.start = tokenizer.point.index; - State::Next(StateName::HtmlFlowContinuationRawEndTag) + State::Next(Name::HtmlFlowContinuationRawEndTag) } - _ => State::Retry(StateName::HtmlFlowContinuation), + _ => State::Retry(Name::HtmlFlowContinuation), } } @@ -773,20 +774,20 @@ pub fn continuation_raw_end_tag(tokenizer: &mut Tokenizer) -> State { if HTML_RAW_NAMES.contains(&name.as_str()) { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationClose) + State::Next(Name::HtmlFlowContinuationClose) } else { - State::Retry(StateName::HtmlFlowContinuation) + State::Retry(Name::HtmlFlowContinuation) } } Some(b'A'..=b'Z' | b'a'..=b'z') if tokenizer.point.index - tokenizer.tokenize_state.start < HTML_RAW_SIZE_MAX => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationRawEndTag) + State::Next(Name::HtmlFlowContinuationRawEndTag) } _ => { tokenizer.tokenize_state.start = 0; - State::Retry(StateName::HtmlFlowContinuation) + State::Retry(Name::HtmlFlowContinuation) } } } @@ -801,9 +802,9 @@ pub fn continuation_cdata_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b']') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } - _ => State::Retry(StateName::HtmlFlowContinuation), + _ => State::Retry(Name::HtmlFlowContinuation), } } @@ -825,13 +826,13 @@ pub fn continuation_declaration_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationClose) + State::Next(Name::HtmlFlowContinuationClose) } Some(b'-') if tokenizer.tokenize_state.marker == COMMENT => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationDeclarationInside) + State::Next(Name::HtmlFlowContinuationDeclarationInside) } - _ => State::Retry(StateName::HtmlFlowContinuation), + _ => State::Retry(Name::HtmlFlowContinuation), } } @@ -845,11 +846,11 @@ pub fn continuation_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { tokenizer.exit(Token::HtmlFlowData); - State::Retry(StateName::HtmlFlowContinuationAfter) + State::Retry(Name::HtmlFlowContinuationAfter) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlFlowContinuationClose) + State::Next(Name::HtmlFlowContinuationClose) } } } @@ -881,5 +882,5 @@ pub fn blank_line_before(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::BlankLineStart) + State::Next(Name::BlankLineStart) } diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index 7474dbf..dd14e70 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -56,8 +56,9 @@ use crate::constant::HTML_CDATA_PREFIX; use crate::construct::partial_space_or_tab::space_or_tab; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of HTML (text) /// @@ -70,7 +71,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::HtmlText); tokenizer.enter(Token::HtmlTextData); tokenizer.consume(); - State::Next(StateName::HtmlTextOpen) + State::Next(Name::HtmlTextOpen) } else { State::Nok } @@ -90,20 +91,20 @@ pub fn open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'!') => { tokenizer.consume(); - State::Next(StateName::HtmlTextDeclarationOpen) + State::Next(Name::HtmlTextDeclarationOpen) } Some(b'/') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagCloseStart) + State::Next(Name::HtmlTextTagCloseStart) } Some(b'?') => { tokenizer.consume(); - State::Next(StateName::HtmlTextInstruction) + State::Next(Name::HtmlTextInstruction) } // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpen) + State::Next(Name::HtmlTextTagOpen) } _ => State::Nok, } @@ -123,16 +124,16 @@ pub fn declaration_open(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCommentOpenInside) + State::Next(Name::HtmlTextCommentOpenInside) } // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextDeclaration) + State::Next(Name::HtmlTextDeclaration) } Some(b'[') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCdataOpenInside) + State::Next(Name::HtmlTextCdataOpenInside) } _ => State::Nok, } @@ -148,7 +149,7 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCommentStart) + State::Next(Name::HtmlTextCommentStart) } _ => State::Nok, } @@ -172,9 +173,9 @@ pub fn comment_start(tokenizer: &mut Tokenizer) -> State { Some(b'>') => State::Nok, Some(b'-') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCommentStartDash) + State::Next(Name::HtmlTextCommentStartDash) } - _ => State::Retry(StateName::HtmlTextComment), + _ => State::Retry(Name::HtmlTextComment), } } @@ -194,7 +195,7 @@ pub fn comment_start(tokenizer: &mut Tokenizer) -> State { pub fn comment_start_dash(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'>') => State::Nok, - _ => State::Retry(StateName::HtmlTextComment), + _ => State::Retry(Name::HtmlTextComment), } } @@ -208,17 +209,17 @@ pub fn comment(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Nok, Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextComment), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextComment), State::Nok, ), Some(b'-') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCommentClose) + State::Next(Name::HtmlTextCommentClose) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlTextComment) + State::Next(Name::HtmlTextComment) } } } @@ -233,9 +234,9 @@ pub fn comment_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'-') => { tokenizer.consume(); - State::Next(StateName::HtmlTextEnd) + State::Next(Name::HtmlTextEnd) } - _ => State::Retry(StateName::HtmlTextComment), + _ => State::Retry(Name::HtmlTextComment), } } @@ -252,9 +253,9 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State { if tokenizer.tokenize_state.size == HTML_CDATA_PREFIX.len() { tokenizer.tokenize_state.size = 0; - State::Next(StateName::HtmlTextCdata) + State::Next(Name::HtmlTextCdata) } else { - State::Next(StateName::HtmlTextCdataOpenInside) + State::Next(Name::HtmlTextCdataOpenInside) } } else { State::Nok @@ -271,17 +272,17 @@ pub fn cdata(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Nok, Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextCdata), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextCdata), State::Nok, ), Some(b']') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCdataClose) + State::Next(Name::HtmlTextCdataClose) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlTextCdata) + State::Next(Name::HtmlTextCdata) } } } @@ -296,9 +297,9 @@ pub fn cdata_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b']') => { tokenizer.consume(); - State::Next(StateName::HtmlTextCdataEnd) + State::Next(Name::HtmlTextCdataEnd) } - _ => State::Retry(StateName::HtmlTextCdata), + _ => State::Retry(Name::HtmlTextCdata), } } @@ -310,9 +311,9 @@ pub fn cdata_close(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn cdata_end(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'>') => State::Retry(StateName::HtmlTextEnd), - Some(b']') => State::Retry(StateName::HtmlTextCdataClose), - _ => State::Retry(StateName::HtmlTextCdata), + Some(b'>') => State::Retry(Name::HtmlTextEnd), + Some(b']') => State::Retry(Name::HtmlTextCdataClose), + _ => State::Retry(Name::HtmlTextCdata), } } @@ -324,15 +325,15 @@ pub fn cdata_end(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn declaration(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - None | Some(b'>') => State::Retry(StateName::HtmlTextEnd), + None | Some(b'>') => State::Retry(Name::HtmlTextEnd), Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextDeclaration), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextDeclaration), State::Nok, ), _ => { tokenizer.consume(); - State::Next(StateName::HtmlTextDeclaration) + State::Next(Name::HtmlTextDeclaration) } } } @@ -347,17 +348,17 @@ pub fn instruction(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Nok, Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextInstruction), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextInstruction), State::Nok, ), Some(b'?') => { tokenizer.consume(); - State::Next(StateName::HtmlTextInstructionClose) + State::Next(Name::HtmlTextInstructionClose) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlTextInstruction) + State::Next(Name::HtmlTextInstruction) } } } @@ -370,8 +371,8 @@ pub fn instruction(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn instruction_close(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'>') => State::Retry(StateName::HtmlTextEnd), - _ => State::Retry(StateName::HtmlTextInstruction), + Some(b'>') => State::Retry(Name::HtmlTextEnd), + _ => State::Retry(Name::HtmlTextInstruction), } } @@ -386,7 +387,7 @@ pub fn tag_close_start(tokenizer: &mut Tokenizer) -> State { // ASCII alphabetical. Some(b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagClose) + State::Next(Name::HtmlTextTagClose) } _ => State::Nok, } @@ -403,9 +404,9 @@ pub fn tag_close(tokenizer: &mut Tokenizer) -> State { // ASCII alphanumerical and `-`. Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagClose) + State::Next(Name::HtmlTextTagClose) } - _ => State::Retry(StateName::HtmlTextTagCloseBetween), + _ => State::Retry(Name::HtmlTextTagCloseBetween), } } @@ -418,15 +419,15 @@ pub fn tag_close(tokenizer: &mut Tokenizer) -> State { pub fn tag_close_between(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextTagCloseBetween), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextTagCloseBetween), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagCloseBetween) + State::Next(Name::HtmlTextTagCloseBetween) } - _ => State::Retry(StateName::HtmlTextEnd), + _ => State::Retry(Name::HtmlTextEnd), } } @@ -441,9 +442,9 @@ pub fn tag_open(tokenizer: &mut Tokenizer) -> State { // ASCII alphanumerical and `-`. Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpen) + State::Next(Name::HtmlTextTagOpen) } - Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(StateName::HtmlTextTagOpenBetween), + Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(Name::HtmlTextTagOpenBetween), _ => State::Nok, } } @@ -457,24 +458,24 @@ pub fn tag_open(tokenizer: &mut Tokenizer) -> State { pub fn tag_open_between(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextTagOpenBetween), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextTagOpenBetween), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenBetween) + State::Next(Name::HtmlTextTagOpenBetween) } Some(b'/') => { tokenizer.consume(); - State::Next(StateName::HtmlTextEnd) + State::Next(Name::HtmlTextEnd) } // ASCII alphabetical and `:` and `_`. Some(b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeName) + State::Next(Name::HtmlTextTagOpenAttributeName) } - _ => State::Retry(StateName::HtmlTextEnd), + _ => State::Retry(Name::HtmlTextEnd), } } @@ -489,9 +490,9 @@ pub fn tag_open_attribute_name(tokenizer: &mut Tokenizer) -> State { // ASCII alphabetical and `-`, `.`, `:`, and `_`. Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeName) + State::Next(Name::HtmlTextTagOpenAttributeName) } - _ => State::Retry(StateName::HtmlTextTagOpenAttributeNameAfter), + _ => State::Retry(Name::HtmlTextTagOpenAttributeNameAfter), } } @@ -505,19 +506,19 @@ pub fn tag_open_attribute_name(tokenizer: &mut Tokenizer) -> State { pub fn tag_open_attribute_name_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextTagOpenAttributeNameAfter), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextTagOpenAttributeNameAfter), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeNameAfter) + State::Next(Name::HtmlTextTagOpenAttributeNameAfter) } Some(b'=') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueBefore) + State::Next(Name::HtmlTextTagOpenAttributeValueBefore) } - _ => State::Retry(StateName::HtmlTextTagOpenBetween), + _ => State::Retry(Name::HtmlTextTagOpenBetween), } } @@ -532,22 +533,22 @@ pub fn tag_open_attribute_value_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'<' | b'=' | b'>' | b'`') => State::Nok, Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextTagOpenAttributeValueBefore), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextTagOpenAttributeValueBefore), State::Nok, ), Some(b'\t' | b' ') => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueBefore) + State::Next(Name::HtmlTextTagOpenAttributeValueBefore) } Some(b'"' | b'\'') => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted) + State::Next(Name::HtmlTextTagOpenAttributeValueQuoted) } Some(_) => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueUnquoted) + State::Next(Name::HtmlTextTagOpenAttributeValueUnquoted) } } } @@ -565,18 +566,18 @@ pub fn tag_open_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { State::Nok } Some(b'\n') => tokenizer.attempt( - StateName::HtmlTextLineEndingBefore, - State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted), + Name::HtmlTextLineEndingBefore, + State::Next(Name::HtmlTextTagOpenAttributeValueQuoted), State::Nok, ), Some(b'"' | b'\'') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.tokenize_state.marker = 0; tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueQuotedAfter) + State::Next(Name::HtmlTextTagOpenAttributeValueQuotedAfter) } _ => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted) + State::Next(Name::HtmlTextTagOpenAttributeValueQuoted) } } } @@ -590,10 +591,10 @@ pub fn tag_open_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State { pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'"' | b'\'' | b'<' | b'=' | b'`') => State::Nok, - Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(StateName::HtmlTextTagOpenBetween), + Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(Name::HtmlTextTagOpenBetween), Some(_) => { tokenizer.consume(); - State::Next(StateName::HtmlTextTagOpenAttributeValueUnquoted) + State::Next(Name::HtmlTextTagOpenAttributeValueUnquoted) } } } @@ -607,7 +608,7 @@ pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn tag_open_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Some(b'\t' | b'\n' | b' ' | b'>' | b'/') => State::Retry(StateName::HtmlTextTagOpenBetween), + Some(b'\t' | b'\n' | b' ' | b'>' | b'/') => State::Retry(Name::HtmlTextTagOpenBetween), _ => State::Nok, } } @@ -647,7 +648,7 @@ pub fn line_ending_before(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::HtmlTextLineEndingAfter) + State::Next(Name::HtmlTextLineEndingAfter) } _ => unreachable!("expected eol"), } @@ -667,8 +668,8 @@ pub fn line_ending_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::HtmlTextLineEndingAfterPrefix), - State::Next(StateName::HtmlTextLineEndingAfterPrefix), + State::Next(Name::HtmlTextLineEndingAfterPrefix), + State::Next(Name::HtmlTextLineEndingAfterPrefix), ) } diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs index a25f917..0607077 100644 --- a/src/construct/label_end.rs +++ b/src/construct/label_end.rs @@ -148,8 +148,9 @@ use crate::constant::RESOURCE_DESTINATION_BALANCE_MAX; use crate::construct::partial_space_or_tab::space_or_tab_eol; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{Event, EventType, Media, State, StateName, Tokenizer}; +use crate::tokenizer::{Event, EventType, Media, Tokenizer}; use crate::util::{ normalize_identifier::normalize_identifier, skip, @@ -194,7 +195,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { // Mark as balanced if the info is inactive. if label_start.inactive { - return State::Retry(StateName::LabelEndNok); + return State::Retry(Name::LabelEndNok); } tokenizer.enter(Token::LabelEnd); @@ -202,7 +203,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); tokenizer.exit(Token::LabelMarker); tokenizer.exit(Token::LabelEnd); - return State::Next(StateName::LabelEndAfter); + return State::Next(Name::LabelEndAfter); } } @@ -239,29 +240,29 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Resource (`[asd](fgh)`)? Some(b'(') => tokenizer.attempt( - StateName::LabelEndResourceStart, - State::Next(StateName::LabelEndOk), + Name::LabelEndResourceStart, + State::Next(Name::LabelEndOk), State::Next(if defined { - StateName::LabelEndOk + Name::LabelEndOk } else { - StateName::LabelEndNok + Name::LabelEndNok }), ), // Full (`[asd][fgh]`) or collapsed (`[asd][]`) reference? Some(b'[') => tokenizer.attempt( - StateName::LabelEndReferenceFull, - State::Next(StateName::LabelEndOk), + Name::LabelEndReferenceFull, + State::Next(Name::LabelEndOk), State::Next(if defined { - StateName::LabelEndReferenceNotFull + Name::LabelEndReferenceNotFull } else { - StateName::LabelEndNok + Name::LabelEndNok }), ), // Shortcut (`[asd]`) reference? _ => State::Retry(if defined { - StateName::LabelEndOk + Name::LabelEndOk } else { - StateName::LabelEndNok + Name::LabelEndNok }), } } @@ -278,9 +279,9 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn reference_not_full(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::LabelEndReferenceCollapsed, - State::Next(StateName::LabelEndOk), - State::Next(StateName::LabelEndNok), + Name::LabelEndReferenceCollapsed, + State::Next(Name::LabelEndOk), + State::Next(Name::LabelEndNok), ) } @@ -370,7 +371,7 @@ pub fn resource_start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::ResourceMarker); tokenizer.consume(); tokenizer.exit(Token::ResourceMarker); - State::Next(StateName::LabelEndResourceBefore) + State::Next(Name::LabelEndResourceBefore) } _ => unreachable!("expected `(`"), } @@ -386,8 +387,8 @@ pub fn resource_before(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(StateName::LabelEndResourceOpen), - State::Next(StateName::LabelEndResourceOpen), + State::Next(Name::LabelEndResourceOpen), + State::Next(Name::LabelEndResourceOpen), ) } @@ -399,7 +400,7 @@ pub fn resource_before(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn resource_open(tokenizer: &mut Tokenizer) -> State { if let Some(b')') = tokenizer.current { - State::Retry(StateName::LabelEndResourceEnd) + State::Retry(Name::LabelEndResourceEnd) } else { tokenizer.tokenize_state.token_1 = Token::ResourceDestination; tokenizer.tokenize_state.token_2 = Token::ResourceDestinationLiteral; @@ -409,9 +410,9 @@ pub fn resource_open(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size_b = RESOURCE_DESTINATION_BALANCE_MAX; tokenizer.attempt( - StateName::DestinationStart, - State::Next(StateName::LabelEndResourceDestinationAfter), - State::Next(StateName::LabelEndResourceDestinationMissing), + Name::DestinationStart, + State::Next(Name::LabelEndResourceDestinationAfter), + State::Next(Name::LabelEndResourceDestinationMissing), ) } } @@ -432,8 +433,8 @@ pub fn resource_destination_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(StateName::LabelEndResourceBetween), - State::Next(StateName::LabelEndResourceEnd), + State::Next(Name::LabelEndResourceBetween), + State::Next(Name::LabelEndResourceEnd), ) } @@ -461,12 +462,12 @@ pub fn resource_between(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.token_2 = Token::ResourceTitleMarker; tokenizer.tokenize_state.token_3 = Token::ResourceTitleString; tokenizer.attempt( - StateName::TitleStart, - State::Next(StateName::LabelEndResourceTitleAfter), + Name::TitleStart, + State::Next(Name::LabelEndResourceTitleAfter), State::Nok, ) } - _ => State::Retry(StateName::LabelEndResourceEnd), + _ => State::Retry(Name::LabelEndResourceEnd), } } @@ -483,8 +484,8 @@ pub fn resource_title_after(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_eol(tokenizer); tokenizer.attempt( name, - State::Next(StateName::LabelEndResourceEnd), - State::Next(StateName::LabelEndResourceEnd), + State::Next(Name::LabelEndResourceEnd), + State::Next(Name::LabelEndResourceEnd), ) } @@ -520,8 +521,8 @@ pub fn reference_full(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.token_2 = Token::ReferenceMarker; tokenizer.tokenize_state.token_3 = Token::ReferenceString; tokenizer.attempt( - StateName::LabelStart, - State::Next(StateName::LabelEndReferenceFullAfter), + Name::LabelStart, + State::Next(Name::LabelEndReferenceFullAfter), State::Nok, ) } @@ -580,7 +581,7 @@ pub fn reference_collapsed(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::ReferenceMarker); tokenizer.consume(); tokenizer.exit(Token::ReferenceMarker); - State::Next(StateName::LabelEndReferenceCollapsedOpen) + State::Next(Name::LabelEndReferenceCollapsedOpen) } _ => State::Nok, } diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs index 629e836..7703ba4 100644 --- a/src/construct/label_start_image.rs +++ b/src/construct/label_start_image.rs @@ -29,8 +29,9 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use super::label_end::resolve_media; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{LabelStart, State, StateName, Tokenizer}; +use crate::tokenizer::{LabelStart, Tokenizer}; /// Start of label (image) start. /// @@ -45,7 +46,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LabelImageMarker); tokenizer.consume(); tokenizer.exit(Token::LabelImageMarker); - State::Next(StateName::LabelStartImageOpen) + State::Next(Name::LabelStartImageOpen) } _ => State::Nok, } diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs index 6eb7b40..3ca51bf 100644 --- a/src/construct/label_start_link.rs +++ b/src/construct/label_start_link.rs @@ -28,8 +28,9 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use super::label_end::resolve_media; +use crate::state::State; use crate::token::Token; -use crate::tokenizer::{LabelStart, State, Tokenizer}; +use crate::tokenizer::{LabelStart, Tokenizer}; /// Start of label (link) start. /// diff --git a/src/construct/list.rs b/src/construct/list.rs index d726c73..516cec7 100644 --- a/src/construct/list.rs +++ b/src/construct/list.rs @@ -46,8 +46,9 @@ use crate::constant::{LIST_ITEM_VALUE_SIZE_MAX, TAB_SIZE}; use crate::construct::partial_space_or_tab::space_or_tab_min_max; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{EventType, State, StateName, Tokenizer}; +use crate::tokenizer::{EventType, Tokenizer}; use crate::util::{ skip, slice::{Position, Slice}, @@ -71,7 +72,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { usize::MAX }, ); - tokenizer.attempt(name, State::Next(StateName::ListBefore), State::Nok) + tokenizer.attempt(name, State::Next(Name::ListBefore), State::Nok) } else { State::Nok } @@ -87,14 +88,14 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Unordered. Some(b'*' | b'-') => tokenizer.check( - StateName::ThematicBreakStart, - State::Next(StateName::ListNok), - State::Next(StateName::ListBeforeUnordered), + Name::ThematicBreakStart, + State::Next(Name::ListNok), + State::Next(Name::ListBeforeUnordered), ), - Some(b'+') => State::Retry(StateName::ListBeforeUnordered), + Some(b'+') => State::Retry(Name::ListBeforeUnordered), // Ordered. - Some(b'0'..=b'9') if !tokenizer.interrupt => State::Retry(StateName::ListBeforeOrdered), - Some(b'1') => State::Retry(StateName::ListBeforeOrdered), + Some(b'0'..=b'9') if !tokenizer.interrupt => State::Retry(Name::ListBeforeOrdered), + Some(b'1') => State::Retry(Name::ListBeforeOrdered), _ => State::Nok, } } @@ -109,7 +110,7 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn before_unordered(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::ListItemPrefix); - State::Retry(StateName::ListMarker) + State::Retry(Name::ListMarker) } /// Start of an ordered list item. @@ -121,7 +122,7 @@ pub fn before_unordered(tokenizer: &mut Tokenizer) -> State { pub fn before_ordered(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::ListItemPrefix); tokenizer.enter(Token::ListItemValue); - State::Retry(StateName::ListValue) + State::Retry(Name::ListValue) } /// In an ordered list item value. @@ -134,12 +135,12 @@ pub fn value(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'.' | b')') if !tokenizer.interrupt || tokenizer.tokenize_state.size < 2 => { tokenizer.exit(Token::ListItemValue); - State::Retry(StateName::ListMarker) + State::Retry(Name::ListMarker) } Some(b'0'..=b'9') if tokenizer.tokenize_state.size + 1 < LIST_ITEM_VALUE_SIZE_MAX => { tokenizer.tokenize_state.size += 1; tokenizer.consume(); - State::Next(StateName::ListValue) + State::Next(Name::ListValue) } _ => { tokenizer.tokenize_state.size = 0; @@ -160,7 +161,7 @@ pub fn marker(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::ListItemMarker); tokenizer.consume(); tokenizer.exit(Token::ListItemMarker); - State::Next(StateName::ListMarkerAfter) + State::Next(Name::ListMarkerAfter) } /// After a list item marker. @@ -174,9 +175,9 @@ pub fn marker(tokenizer: &mut Tokenizer) -> State { pub fn marker_after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size = 1; tokenizer.check( - StateName::BlankLineStart, - State::Next(StateName::ListAfter), - State::Next(StateName::ListMarkerAfterFilled), + Name::BlankLineStart, + State::Next(Name::ListAfter), + State::Next(Name::ListMarkerAfterFilled), ) } @@ -191,9 +192,9 @@ pub fn marker_after_filled(tokenizer: &mut Tokenizer) -> State { // Attempt to parse up to the largest allowed indent, `nok` if there is more whitespace. tokenizer.attempt( - StateName::ListWhitespace, - State::Next(StateName::ListAfter), - State::Next(StateName::ListPrefixOther), + Name::ListWhitespace, + State::Next(Name::ListAfter), + State::Next(Name::ListPrefixOther), ) } @@ -205,11 +206,7 @@ pub fn marker_after_filled(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn whitespace(tokenizer: &mut Tokenizer) -> State { let name = space_or_tab_min_max(tokenizer, 1, TAB_SIZE); - tokenizer.attempt( - name, - State::Next(StateName::ListWhitespaceAfter), - State::Nok, - ) + tokenizer.attempt(name, State::Next(Name::ListWhitespaceAfter), State::Nok) } /// After acceptable whitespace. @@ -238,7 +235,7 @@ pub fn prefix_other(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::SpaceOrTab); tokenizer.consume(); tokenizer.exit(Token::SpaceOrTab); - State::Next(StateName::ListAfter) + State::Next(Name::ListAfter) } _ => State::Nok, } @@ -296,9 +293,9 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn cont_start(tokenizer: &mut Tokenizer) -> State { tokenizer.check( - StateName::BlankLineStart, - State::Next(StateName::ListContBlank), - State::Next(StateName::ListContFilled), + Name::BlankLineStart, + State::Next(Name::ListContBlank), + State::Next(Name::ListContFilled), ) } @@ -320,7 +317,7 @@ pub fn cont_blank(tokenizer: &mut Tokenizer) -> State { } else { let name = space_or_tab_min_max(tokenizer, 0, size); // Consume, optionally, at most `size`. - tokenizer.attempt(name, State::Next(StateName::ListOk), State::Nok) + tokenizer.attempt(name, State::Next(Name::ListOk), State::Nok) } } @@ -340,7 +337,7 @@ pub fn cont_filled(tokenizer: &mut Tokenizer) -> State { // Consume exactly `size`. let name = space_or_tab_min_max(tokenizer, size, size); - tokenizer.attempt(name, State::Next(StateName::ListOk), State::Nok) + tokenizer.attempt(name, State::Next(Name::ListOk), State::Nok) } /// A state fn to yield [`State::Ok`]. diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs index bac4369..dec25b8 100644 --- a/src/construct/paragraph.rs +++ b/src/construct/paragraph.rs @@ -32,8 +32,9 @@ //! [code_text]: crate::construct::code_text //! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{ContentType, EventType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, EventType, Tokenizer}; use crate::util::skip::opt as skip_opt; /// Before a paragraph. @@ -48,7 +49,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { _ => { tokenizer.enter(Token::Paragraph); tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); - State::Retry(StateName::ParagraphInside) + State::Retry(Name::ParagraphInside) } } } @@ -71,7 +72,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.consume(); - State::Next(StateName::ParagraphInside) + State::Next(Name::ParagraphInside) } } } diff --git a/src/construct/partial_bom.rs b/src/construct/partial_bom.rs index d20c2c7..cca0770 100644 --- a/src/construct/partial_bom.rs +++ b/src/construct/partial_bom.rs @@ -10,8 +10,9 @@ //! //! * [`micromark/lib/preprocess.js` in `micromark`](https://github.com/micromark/micromark/blob/ed23453/packages/micromark/dev/lib/preprocess.js#L54-L60) +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF]; @@ -24,7 +25,7 @@ const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF]; pub fn start(tokenizer: &mut Tokenizer) -> State { if tokenizer.current == Some(BOM[0]) { tokenizer.enter(Token::ByteOrderMark); - State::Retry(StateName::BomInside) + State::Retry(Name::BomInside) } else { State::Nok } @@ -45,7 +46,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.size = 0; State::Ok } else { - State::Next(StateName::BomInside) + State::Next(Name::BomInside) } } else { tokenizer.tokenize_state.size = 0; diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs index 0ad67c5..c05aaa5 100644 --- a/src/construct/partial_data.rs +++ b/src/construct/partial_data.rs @@ -6,8 +6,9 @@ //! [string]: crate::content::string //! [text]: crate::content::text +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{EventType, State, StateName, Tokenizer}; +use crate::tokenizer::{EventType, Tokenizer}; /// At the beginning of data. /// @@ -21,9 +22,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { Some(byte) if tokenizer.tokenize_state.markers.contains(&byte) => { tokenizer.enter(Token::Data); tokenizer.consume(); - State::Next(StateName::DataInside) + State::Next(Name::DataInside) } - _ => State::Retry(StateName::DataAtBreak), + _ => State::Retry(Name::DataAtBreak), } } @@ -40,7 +41,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::DataAtBreak) + State::Next(Name::DataAtBreak) } Some(byte) if tokenizer.tokenize_state.markers.contains(&byte) => { tokenizer.register_resolver_before("data".to_string(), Box::new(resolve_data)); @@ -48,7 +49,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.enter(Token::Data); - State::Retry(StateName::DataInside) + State::Retry(Name::DataInside) } } } @@ -68,10 +69,10 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { if done { tokenizer.exit(Token::Data); - State::Retry(StateName::DataAtBreak) + State::Retry(Name::DataAtBreak) } else { tokenizer.consume(); - State::Next(StateName::DataInside) + State::Next(Name::DataInside) } } diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs index 735fb38..5aa0539 100644 --- a/src/construct/partial_destination.rs +++ b/src/construct/partial_destination.rs @@ -71,8 +71,9 @@ //! [label_end]: crate::construct::label_end //! [sanitize_uri]: crate::util::sanitize_uri +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{ContentType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, Tokenizer}; /// Before a destination. /// @@ -90,7 +91,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(tokenizer.tokenize_state.token_3.clone()); tokenizer.consume(); tokenizer.exit(tokenizer.tokenize_state.token_3.clone()); - State::Next(StateName::DestinationEnclosedBefore) + State::Next(Name::DestinationEnclosedBefore) } // ASCII control, space, closing paren, but *not* `\0`. None | Some(0x01..=0x1F | b' ' | b')' | 0x7F) => State::Nok, @@ -99,7 +100,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(tokenizer.tokenize_state.token_4.clone()); tokenizer.enter(tokenizer.tokenize_state.token_5.clone()); tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(StateName::DestinationRaw) + State::Retry(Name::DestinationRaw) } } } @@ -121,7 +122,7 @@ pub fn enclosed_before(tokenizer: &mut Tokenizer) -> State { } else { tokenizer.enter(tokenizer.tokenize_state.token_5.clone()); tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); - State::Retry(StateName::DestinationEnclosed) + State::Retry(Name::DestinationEnclosed) } } @@ -137,15 +138,15 @@ pub fn enclosed(tokenizer: &mut Tokenizer) -> State { Some(b'>') => { tokenizer.exit(Token::Data); tokenizer.exit(tokenizer.tokenize_state.token_5.clone()); - State::Retry(StateName::DestinationEnclosedBefore) + State::Retry(Name::DestinationEnclosedBefore) } Some(b'\\') => { tokenizer.consume(); - State::Next(StateName::DestinationEnclosedEscape) + State::Next(Name::DestinationEnclosedEscape) } _ => { tokenizer.consume(); - State::Next(StateName::DestinationEnclosed) + State::Next(Name::DestinationEnclosed) } } } @@ -160,9 +161,9 @@ pub fn enclosed_escape(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'<' | b'>' | b'\\') => { tokenizer.consume(); - State::Next(StateName::DestinationEnclosed) + State::Next(Name::DestinationEnclosed) } - _ => State::Retry(StateName::DestinationEnclosed), + _ => State::Retry(Name::DestinationEnclosed), } } @@ -185,7 +186,7 @@ pub fn raw(tokenizer: &mut Tokenizer) -> State { Some(b'(') if tokenizer.tokenize_state.size < tokenizer.tokenize_state.size_b => { tokenizer.consume(); tokenizer.tokenize_state.size += 1; - State::Next(StateName::DestinationRaw) + State::Next(Name::DestinationRaw) } // ASCII control (but *not* `\0`) and space and `(`. None | Some(0x01..=0x1F | b' ' | b'(' | 0x7F) => { @@ -195,15 +196,15 @@ pub fn raw(tokenizer: &mut Tokenizer) -> State { Some(b')') => { tokenizer.consume(); tokenizer.tokenize_state.size -= 1; - State::Next(StateName::DestinationRaw) + State::Next(Name::DestinationRaw) } Some(b'\\') => { tokenizer.consume(); - State::Next(StateName::DestinationRawEscape) + State::Next(Name::DestinationRawEscape) } Some(_) => { tokenizer.consume(); - State::Next(StateName::DestinationRaw) + State::Next(Name::DestinationRaw) } } } @@ -218,8 +219,8 @@ pub fn raw_escape(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'(' | b')' | b'\\') => { tokenizer.consume(); - State::Next(StateName::DestinationRaw) + State::Next(Name::DestinationRaw) } - _ => State::Retry(StateName::DestinationRaw), + _ => State::Retry(Name::DestinationRaw), } } diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs index 6447961..6e7c843 100644 --- a/src/construct/partial_label.rs +++ b/src/construct/partial_label.rs @@ -60,9 +60,10 @@ use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; use crate::constant::LINK_REFERENCE_SIZE_MAX; +use crate::state::{Name, State}; use crate::subtokenize::link; use crate::token::Token; -use crate::tokenizer::{ContentType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, Tokenizer}; /// Before a label. /// @@ -78,7 +79,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); tokenizer.exit(tokenizer.tokenize_state.token_2.clone()); tokenizer.enter(tokenizer.tokenize_state.token_3.clone()); - State::Next(StateName::LabelAtBreak) + State::Next(Name::LabelAtBreak) } _ => State::Nok, } @@ -111,8 +112,8 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(StateName::LabelEolAfter), - State::Next(StateName::LabelAtBlankLine), + State::Next(Name::LabelEolAfter), + State::Next(Name::LabelAtBlankLine), ) } Some(b']') => { @@ -136,7 +137,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; } - State::Retry(StateName::LabelInside) + State::Retry(Name::LabelInside) } } } @@ -151,7 +152,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn eol_after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; - State::Retry(StateName::LabelAtBreak) + State::Retry(Name::LabelAtBreak) } /// In a label, at a blank line. @@ -178,12 +179,12 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n' | b'[' | b']') => { tokenizer.exit(Token::Data); - State::Retry(StateName::LabelAtBreak) + State::Retry(Name::LabelAtBreak) } Some(byte) => { if tokenizer.tokenize_state.size > LINK_REFERENCE_SIZE_MAX { tokenizer.exit(Token::Data); - State::Retry(StateName::LabelAtBreak) + State::Retry(Name::LabelAtBreak) } else { tokenizer.consume(); tokenizer.tokenize_state.size += 1; @@ -191,9 +192,9 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.seen = true; } State::Next(if matches!(byte, b'\\') { - StateName::LabelEscape + Name::LabelEscape } else { - StateName::LabelInside + Name::LabelInside }) } } @@ -211,8 +212,8 @@ pub fn escape(tokenizer: &mut Tokenizer) -> State { Some(b'[' | b'\\' | b']') => { tokenizer.consume(); tokenizer.tokenize_state.size += 1; - State::Next(StateName::LabelInside) + State::Next(Name::LabelInside) } - _ => State::Retry(StateName::LabelInside), + _ => State::Retry(Name::LabelInside), } } diff --git a/src/construct/partial_non_lazy_continuation.rs b/src/construct/partial_non_lazy_continuation.rs index 9d19860..497c81e 100644 --- a/src/construct/partial_non_lazy_continuation.rs +++ b/src/construct/partial_non_lazy_continuation.rs @@ -10,8 +10,9 @@ //! [code_indented]: crate::construct::code_indented //! [html_flow]: crate::construct::html_flow +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of continuation. /// @@ -26,7 +27,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::NonLazyContinuationAfter) + State::Next(Name::NonLazyContinuationAfter) } _ => State::Nok, } diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs index 5e8c212..a8e8f06 100644 --- a/src/construct/partial_space_or_tab.rs +++ b/src/construct/partial_space_or_tab.rs @@ -4,9 +4,10 @@ //! //! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js) +use crate::state::{Name, State}; use crate::subtokenize::link; use crate::token::Token; -use crate::tokenizer::{ContentType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, Tokenizer}; /// Options to parse `space_or_tab`. #[derive(Debug)] @@ -37,7 +38,7 @@ pub struct EolOptions { /// ```bnf /// space_or_tab ::= 1*( ' ' '\t' ) /// ``` -pub fn space_or_tab(tokenizer: &mut Tokenizer) -> StateName { +pub fn space_or_tab(tokenizer: &mut Tokenizer) -> Name { space_or_tab_min_max(tokenizer, 1, usize::MAX) } @@ -46,7 +47,7 @@ pub fn space_or_tab(tokenizer: &mut Tokenizer) -> StateName { /// ```bnf /// space_or_tab_min_max ::= x*y( ' ' '\t' ) /// ``` -pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -> StateName { +pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -> Name { space_or_tab_with_options( tokenizer, Options { @@ -60,13 +61,13 @@ pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) - } /// `space_or_tab`, with the given options. -pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> StateName { +pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> Name { tokenizer.tokenize_state.space_or_tab_connect = options.connect; tokenizer.tokenize_state.space_or_tab_content_type = options.content_type; tokenizer.tokenize_state.space_or_tab_min = options.min; tokenizer.tokenize_state.space_or_tab_max = options.max; tokenizer.tokenize_state.space_or_tab_token = options.kind; - StateName::SpaceOrTabStart + Name::SpaceOrTabStart } /// `space_or_tab`, or optionally `space_or_tab`, one `eol`, and @@ -75,7 +76,7 @@ pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> /// ```bnf /// space_or_tab_eol ::= 1*( ' ' '\t' ) | 0*( ' ' '\t' ) eol 0*( ' ' '\t' ) /// ``` -pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> StateName { +pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> Name { space_or_tab_eol_with_options( tokenizer, EolOptions { @@ -86,10 +87,10 @@ pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> StateName { } /// `space_or_tab_eol`, with the given options. -pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> StateName { +pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> Name { tokenizer.tokenize_state.space_or_tab_eol_content_type = options.content_type; tokenizer.tokenize_state.space_or_tab_eol_connect = options.connect; - StateName::SpaceOrTabEolStart + Name::SpaceOrTabEolStart } /// Before `space_or_tab`. @@ -113,9 +114,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.space_or_tab_connect = true; } - State::Retry(StateName::SpaceOrTabInside) + State::Retry(Name::SpaceOrTabInside) } - _ => State::Retry(StateName::SpaceOrTabAfter), + _ => State::Retry(Name::SpaceOrTabAfter), } } @@ -133,11 +134,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { { tokenizer.consume(); tokenizer.tokenize_state.space_or_tab_size += 1; - State::Next(StateName::SpaceOrTabInside) + State::Next(Name::SpaceOrTabInside) } _ => { tokenizer.exit(tokenizer.tokenize_state.space_or_tab_token.clone()); - State::Retry(StateName::SpaceOrTabAfter) + State::Retry(Name::SpaceOrTabAfter) } } } @@ -182,8 +183,8 @@ pub fn eol_start(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( name, - State::Next(StateName::SpaceOrTabEolAfterFirst), - State::Next(StateName::SpaceOrTabEolAtEol), + State::Next(Name::SpaceOrTabEolAfterFirst), + State::Next(Name::SpaceOrTabEolAtEol), ) } @@ -198,7 +199,7 @@ pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.space_or_tab_eol_connect = true; } - State::Retry(StateName::SpaceOrTabEolAtEol) + State::Retry(Name::SpaceOrTabEolAtEol) } /// `space_or_tab_eol`: after optionally first `space_or_tab`. @@ -231,7 +232,7 @@ pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State { tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::SpaceOrTabEolAfterEol) + State::Next(Name::SpaceOrTabEolAfterEol) } else { let ok = tokenizer.tokenize_state.space_or_tab_eol_ok; tokenizer.tokenize_state.space_or_tab_eol_content_type = None; @@ -269,8 +270,8 @@ pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State { ); tokenizer.attempt( name, - State::Next(StateName::SpaceOrTabEolAfterMore), - State::Next(StateName::SpaceOrTabEolAfterMore), + State::Next(Name::SpaceOrTabEolAfterMore), + State::Next(Name::SpaceOrTabEolAfterMore), ) } diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs index 209240e..11c28bd 100644 --- a/src/construct/partial_title.rs +++ b/src/construct/partial_title.rs @@ -31,9 +31,10 @@ //! [label_end]: crate::construct::label_end use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; +use crate::state::{Name, State}; use crate::subtokenize::link; use crate::token::Token; -use crate::tokenizer::{ContentType, State, StateName, Tokenizer}; +use crate::tokenizer::{ContentType, Tokenizer}; /// Before a title. /// @@ -50,7 +51,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(tokenizer.tokenize_state.token_2.clone()); tokenizer.consume(); tokenizer.exit(tokenizer.tokenize_state.token_2.clone()); - State::Next(StateName::TitleBegin) + State::Next(Name::TitleBegin) } _ => State::Nok, } @@ -79,7 +80,7 @@ pub fn begin(tokenizer: &mut Tokenizer) -> State { } _ => { tokenizer.enter(tokenizer.tokenize_state.token_3.clone()); - State::Retry(StateName::TitleAtBreak) + State::Retry(Name::TitleAtBreak) } } } @@ -108,15 +109,15 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( name, - State::Next(StateName::TitleAfterEol), - State::Next(StateName::TitleAtBlankLine), + State::Next(Name::TitleAfterEol), + State::Next(Name::TitleAtBlankLine), ) } Some(b'"' | b'\'' | b')') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.exit(tokenizer.tokenize_state.token_3.clone()); - State::Retry(StateName::TitleBegin) + State::Retry(Name::TitleBegin) } Some(_) => { tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); @@ -128,7 +129,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; } - State::Retry(StateName::TitleInside) + State::Retry(Name::TitleInside) } } } @@ -142,7 +143,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn after_eol(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.connect = true; - State::Retry(StateName::TitleAtBreak) + State::Retry(Name::TitleAtBreak) } /// In a title, at a blank line. @@ -169,20 +170,20 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None | Some(b'\n') => { tokenizer.exit(Token::Data); - State::Retry(StateName::TitleAtBreak) + State::Retry(Name::TitleAtBreak) } Some(b'"' | b'\'' | b')') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.exit(Token::Data); - State::Retry(StateName::TitleAtBreak) + State::Retry(Name::TitleAtBreak) } Some(byte) => { tokenizer.consume(); State::Next(if matches!(byte, b'\\') { - StateName::TitleEscape + Name::TitleEscape } else { - StateName::TitleInside + Name::TitleInside }) } } @@ -198,8 +199,8 @@ pub fn escape(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'"' | b'\'' | b')') => { tokenizer.consume(); - State::Next(StateName::TitleInside) + State::Next(Name::TitleInside) } - _ => State::Retry(StateName::TitleInside), + _ => State::Retry(Name::TitleInside), } } diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index 288d818..fc71d73 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -50,8 +50,9 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN}; +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Start of a thematic break. /// @@ -72,11 +73,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { }, ); - tokenizer.attempt( - name, - State::Next(StateName::ThematicBreakBefore), - State::Nok, - ) + tokenizer.attempt(name, State::Next(Name::ThematicBreakBefore), State::Nok) } else { State::Nok } @@ -92,7 +89,7 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'*' | b'-' | b'_') => { tokenizer.tokenize_state.marker = tokenizer.current.unwrap(); - State::Retry(StateName::ThematicBreakAtBreak) + State::Retry(Name::ThematicBreakAtBreak) } _ => State::Nok, } @@ -118,7 +115,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => { tokenizer.enter(Token::ThematicBreakSequence); - State::Retry(StateName::ThematicBreakSequence) + State::Retry(Name::ThematicBreakSequence) } _ => { tokenizer.tokenize_state.marker = 0; @@ -141,15 +138,15 @@ pub fn sequence(tokenizer: &mut Tokenizer) -> State { { tokenizer.consume(); tokenizer.tokenize_state.size += 1; - State::Next(StateName::ThematicBreakSequence) + State::Next(Name::ThematicBreakSequence) } _ => { tokenizer.exit(Token::ThematicBreakSequence); let name = space_or_tab(tokenizer); tokenizer.attempt( name, - State::Next(StateName::ThematicBreakAtBreak), - State::Next(StateName::ThematicBreakAtBreak), + State::Next(Name::ThematicBreakAtBreak), + State::Next(Name::ThematicBreakAtBreak), ) } } diff --git a/src/content/document.rs b/src/content/document.rs index 49ca919..b5ff532 100644 --- a/src/content/document.rs +++ b/src/content/document.rs @@ -9,11 +9,11 @@ //! * [List][crate::construct::list] use crate::parser::ParseState; +use crate::state::{Name, State}; use crate::subtokenize::{divide_events, subtokenize}; use crate::token::Token; use crate::tokenizer::{ - Container, ContainerState, ContentType, Event, EventType, Link, Point, State, StateName, - Tokenizer, + Container, ContainerState, ContentType, Event, EventType, Link, Point, Tokenizer, }; use crate::util::{ normalize_identifier::normalize_identifier, @@ -59,7 +59,7 @@ pub fn document(parse_state: &mut ParseState, point: Point) -> Vec<Event> { let state = tokenizer.push( (0, 0), (parse_state.bytes.len(), 0), - State::Next(StateName::DocumentStart), + State::Next(Name::DocumentStart), ); tokenizer.flush(state, true); @@ -111,9 +111,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { ))); tokenizer.attempt( - StateName::BomStart, - State::Next(StateName::DocumentContainerExistingBefore), - State::Next(StateName::DocumentContainerExistingBefore), + Name::BomStart, + State::Next(Name::DocumentContainerExistingBefore), + State::Next(Name::DocumentContainerExistingBefore), ) } @@ -134,16 +134,16 @@ pub fn container_existing_before(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( match container.kind { - Container::BlockQuote => StateName::BlockQuoteContStart, - Container::ListItem => StateName::ListContStart, + Container::BlockQuote => Name::BlockQuoteContStart, + Container::ListItem => Name::ListContStart, }, - State::Next(StateName::DocumentContainerExistingAfter), - State::Next(StateName::DocumentContainerNewBefore), + State::Next(Name::DocumentContainerExistingAfter), + State::Next(Name::DocumentContainerNewBefore), ) } // Otherwise, check new containers. else { - State::Retry(StateName::DocumentContainerNewBefore) + State::Retry(Name::DocumentContainerNewBefore) } } @@ -156,7 +156,7 @@ pub fn container_existing_before(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn container_existing_after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.document_continued += 1; - State::Retry(StateName::DocumentContainerExistingBefore) + State::Retry(Name::DocumentContainerExistingBefore) } /// Before a new container. @@ -180,7 +180,7 @@ pub fn container_new_before(tokenizer: &mut Tokenizer) -> State { // …and if we’re in a concrete construct, new containers can’t “pierce” // into them. if child.concrete { - return State::Retry(StateName::DocumentContainersAfter); + return State::Retry(Name::DocumentContainersAfter); } } @@ -203,9 +203,9 @@ pub fn container_new_before(tokenizer: &mut Tokenizer) -> State { .swap(tokenizer.tokenize_state.document_continued, tail); tokenizer.attempt( - StateName::BlockQuoteStart, - State::Next(StateName::DocumentContainerNewAfter), - State::Next(StateName::DocumentContainerNewBeforeNotBlockQuote), + Name::BlockQuoteStart, + State::Next(Name::DocumentContainerNewAfter), + State::Next(Name::DocumentContainerNewBeforeNotBlockQuote), ) } @@ -226,9 +226,9 @@ pub fn container_new_before_not_block_quote(tokenizer: &mut Tokenizer) -> State }; tokenizer.attempt( - StateName::ListStart, - State::Next(StateName::DocumentContainerNewAfter), - State::Next(StateName::DocumentContainerNewBeforeNotList), + Name::ListStart, + State::Next(Name::DocumentContainerNewAfter), + State::Next(Name::DocumentContainerNewBeforeNotList), ) } @@ -247,7 +247,7 @@ pub fn container_new_before_not_list(tokenizer: &mut Tokenizer) -> State { .document_container_stack .swap_remove(tokenizer.tokenize_state.document_continued); - State::Retry(StateName::DocumentContainersAfter) + State::Retry(Name::DocumentContainersAfter) } /// After a new container. @@ -281,7 +281,7 @@ pub fn container_new_after(tokenizer: &mut Tokenizer) -> State { .push(container); tokenizer.tokenize_state.document_continued += 1; tokenizer.interrupt = false; - State::Retry(StateName::DocumentContainerNewBefore) + State::Retry(Name::DocumentContainerNewBefore) } /// After containers, before flow. @@ -301,7 +301,7 @@ pub fn containers_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { // Note: EOL is part of data. - None => State::Retry(StateName::DocumentFlowEnd), + None => State::Retry(Name::DocumentFlowEnd), Some(_) => { let current = tokenizer.events.len(); let previous = tokenizer.tokenize_state.document_data_index; @@ -317,7 +317,7 @@ pub fn containers_after(tokenizer: &mut Tokenizer) -> State { content_type: ContentType::Flow, }), ); - State::Retry(StateName::DocumentFlowInside) + State::Retry(Name::DocumentFlowInside) } } } @@ -332,17 +332,17 @@ pub fn flow_inside(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => { tokenizer.exit(Token::Data); - State::Retry(StateName::DocumentFlowEnd) + State::Retry(Name::DocumentFlowEnd) } // Note: EOL is part of data. Some(b'\n') => { tokenizer.consume(); tokenizer.exit(Token::Data); - State::Next(StateName::DocumentFlowEnd) + State::Next(Name::DocumentFlowEnd) } Some(_) => { tokenizer.consume(); - State::Next(StateName::DocumentFlowInside) + State::Next(Name::DocumentFlowInside) } } } @@ -359,7 +359,7 @@ pub fn flow_end(tokenizer: &mut Tokenizer) -> State { let state = tokenizer .tokenize_state .document_child_state - .unwrap_or(State::Next(StateName::FlowStart)); + .unwrap_or(State::Next(Name::FlowStart)); tokenizer.tokenize_state.document_exits.push(None); @@ -369,7 +369,7 @@ pub fn flow_end(tokenizer: &mut Tokenizer) -> State { state, ); - let paragraph = matches!(state, State::Next(StateName::ParagraphInside)) + let paragraph = matches!(state, State::Next(Name::ParagraphInside)) || (!child.events.is_empty() && child.events [skip::opt_back(&child.events, child.events.len() - 1, &[Token::LineEnding])] @@ -401,7 +401,7 @@ pub fn flow_end(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.document_paragraph_before = paragraph; // Containers would only be interrupting if we’ve continued. tokenizer.interrupt = false; - State::Retry(StateName::DocumentContainerExistingBefore) + State::Retry(Name::DocumentContainerExistingBefore) } } } @@ -421,7 +421,7 @@ fn exit_containers(tokenizer: &mut Tokenizer, phase: &Phase) { .tokenize_state .document_child_state .take() - .unwrap_or(State::Next(StateName::FlowStart)); + .unwrap_or(State::Next(Name::FlowStart)); child.flush(state, false); } diff --git a/src/content/flow.rs b/src/content/flow.rs index 886b5f0..16a1cba 100644 --- a/src/content/flow.rs +++ b/src/content/flow.rs @@ -19,8 +19,9 @@ //! * [HTML (flow)][crate::construct::html_flow] //! * [Thematic break][crate::construct::thematic_break] +use crate::state::{Name, State}; use crate::token::Token; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::tokenizer::Tokenizer; /// Before flow. /// @@ -35,42 +36,42 @@ use crate::tokenizer::{State, StateName, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { Some(b'`' | b'~') => tokenizer.attempt( - StateName::CodeFencedStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::CodeFencedStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ), Some(b'<') => tokenizer.attempt( - StateName::HtmlFlowStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::HtmlFlowStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ), Some(b'#') => tokenizer.attempt( - StateName::HeadingAtxStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::HeadingAtxStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ), // Note: `-` is also used in thematic breaks, so it’s not included here. Some(b'=') => tokenizer.attempt( - StateName::HeadingSetextStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::HeadingSetextStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ), Some(b'*' | b'_') => tokenizer.attempt( - StateName::ThematicBreakStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::ThematicBreakStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ), Some(b'[') => tokenizer.attempt( - StateName::DefinitionStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::DefinitionStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ), // Actual parsing: blank line? Indented code? Indented anything? // Also includes `-` which can be a setext heading underline or a thematic break. - None | Some(b'\t' | b'\n' | b' ' | b'-') => State::Retry(StateName::FlowBlankLineBefore), + None | Some(b'\t' | b'\n' | b' ' | b'-') => State::Retry(Name::FlowBlankLineBefore), Some(_) => tokenizer.attempt( - StateName::ParagraphStart, - State::Next(StateName::FlowAfter), + Name::ParagraphStart, + State::Next(Name::FlowAfter), State::Nok, ), } @@ -78,9 +79,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { pub fn blank_line_before(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::BlankLineStart, - State::Next(StateName::FlowBlankLineAfter), - State::Next(StateName::FlowBeforeCodeIndented), + Name::BlankLineStart, + State::Next(Name::FlowBlankLineAfter), + State::Next(Name::FlowBeforeCodeIndented), ) } @@ -98,57 +99,57 @@ pub fn blank_line_before(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn before_code_indented(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::CodeIndentedStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeCodeFenced), + Name::CodeIndentedStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeCodeFenced), ) } pub fn before_code_fenced(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::CodeFencedStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeHtml), + Name::CodeFencedStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeHtml), ) } pub fn before_html(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::HtmlFlowStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeHeadingAtx), + Name::HtmlFlowStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeHeadingAtx), ) } pub fn before_heading_atx(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::HeadingAtxStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeHeadingSetext), + Name::HeadingAtxStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeHeadingSetext), ) } pub fn before_heading_setext(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::HeadingSetextStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeThematicBreak), + Name::HeadingSetextStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeThematicBreak), ) } pub fn before_thematic_break(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::ThematicBreakStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeDefinition), + Name::ThematicBreakStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeDefinition), ) } pub fn before_definition(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::DefinitionStart, - State::Next(StateName::FlowAfter), - State::Next(StateName::FlowBeforeParagraph), + Name::DefinitionStart, + State::Next(Name::FlowAfter), + State::Next(Name::FlowBeforeParagraph), ) } @@ -168,7 +169,7 @@ pub fn blank_line_after(tokenizer: &mut Tokenizer) -> State { tokenizer.exit(Token::BlankLineEnding); // Feel free to interrupt. tokenizer.interrupt = false; - State::Next(StateName::FlowStart) + State::Next(Name::FlowStart) } _ => unreachable!("expected eol/eof"), } @@ -190,7 +191,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); - State::Next(StateName::FlowStart) + State::Next(Name::FlowStart) } _ => unreachable!("expected eol/eof"), } @@ -203,8 +204,8 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { /// ``` pub fn before_paragraph(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::ParagraphStart, - State::Next(StateName::FlowAfter), + Name::ParagraphStart, + State::Next(Name::FlowAfter), State::Nok, ) } diff --git a/src/content/string.rs b/src/content/string.rs index 5dfceb0..927f582 100644 --- a/src/content/string.rs +++ b/src/content/string.rs @@ -13,7 +13,8 @@ //! [text]: crate::content::text use crate::construct::partial_whitespace::resolve_whitespace; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::state::{Name, State}; +use crate::tokenizer::Tokenizer; const MARKERS: [u8; 2] = [b'&', b'\\']; @@ -21,7 +22,7 @@ const MARKERS: [u8; 2] = [b'&', b'\\']; pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.register_resolver("whitespace".to_string(), Box::new(resolve)); tokenizer.tokenize_state.markers = &MARKERS; - State::Retry(StateName::StringBefore) + State::Retry(Name::StringBefore) } /// Before string. @@ -29,26 +30,22 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Ok, Some(b'&') => tokenizer.attempt( - StateName::CharacterReferenceStart, - State::Next(StateName::StringBefore), - State::Next(StateName::StringBeforeData), + Name::CharacterReferenceStart, + State::Next(Name::StringBefore), + State::Next(Name::StringBeforeData), ), Some(b'\\') => tokenizer.attempt( - StateName::CharacterEscapeStart, - State::Next(StateName::StringBefore), - State::Next(StateName::StringBeforeData), + Name::CharacterEscapeStart, + State::Next(Name::StringBefore), + State::Next(Name::StringBeforeData), ), - _ => State::Retry(StateName::StringBeforeData), + _ => State::Retry(Name::StringBeforeData), } } /// At data. pub fn before_data(tokenizer: &mut Tokenizer) -> State { - tokenizer.attempt( - StateName::DataStart, - State::Next(StateName::StringBefore), - State::Nok, - ) + tokenizer.attempt(Name::DataStart, State::Next(Name::StringBefore), State::Nok) } /// Resolve whitespace. diff --git a/src/content/text.rs b/src/content/text.rs index 4e93779..1b3890e 100644 --- a/src/content/text.rs +++ b/src/content/text.rs @@ -21,7 +21,8 @@ //! > [whitespace][crate::construct::partial_whitespace]. use crate::construct::partial_whitespace::resolve_whitespace; -use crate::tokenizer::{State, StateName, Tokenizer}; +use crate::state::{Name, State}; +use crate::tokenizer::Tokenizer; const MARKERS: [u8; 9] = [ b'!', // `label_start_image` @@ -39,7 +40,7 @@ const MARKERS: [u8; 9] = [ pub fn start(tokenizer: &mut Tokenizer) -> State { tokenizer.register_resolver("whitespace".to_string(), Box::new(resolve)); tokenizer.tokenize_state.markers = &MARKERS; - State::Retry(StateName::TextBefore) + State::Retry(Name::TextBefore) } /// Before text. @@ -47,75 +48,71 @@ pub fn before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { None => State::Ok, Some(b'!') => tokenizer.attempt( - StateName::LabelStartImageStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::LabelStartImageStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ), Some(b'&') => tokenizer.attempt( - StateName::CharacterReferenceStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::CharacterReferenceStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ), Some(b'*' | b'_') => tokenizer.attempt( - StateName::AttentionStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::AttentionStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ), // `autolink`, `html_text` (order does not matter) Some(b'<') => tokenizer.attempt( - StateName::AutolinkStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeHtml), + Name::AutolinkStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeHtml), ), Some(b'[') => tokenizer.attempt( - StateName::LabelStartLinkStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::LabelStartLinkStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ), Some(b'\\') => tokenizer.attempt( - StateName::CharacterEscapeStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeHardBreakEscape), + Name::CharacterEscapeStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeHardBreakEscape), ), Some(b']') => tokenizer.attempt( - StateName::LabelEndStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::LabelEndStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ), Some(b'`') => tokenizer.attempt( - StateName::CodeTextStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::CodeTextStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ), - _ => State::Retry(StateName::TextBeforeData), + _ => State::Retry(Name::TextBeforeData), } } /// At `<`, which wasn’t an autolink: before HTML? pub fn before_html(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::HtmlTextStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::HtmlTextStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ) } /// At `\`, which wasn’t a character escape: before a hard break? pub fn before_hard_break_escape(tokenizer: &mut Tokenizer) -> State { tokenizer.attempt( - StateName::HardBreakEscapeStart, - State::Next(StateName::TextBefore), - State::Next(StateName::TextBeforeData), + Name::HardBreakEscapeStart, + State::Next(Name::TextBefore), + State::Next(Name::TextBeforeData), ) } /// At data. pub fn before_data(tokenizer: &mut Tokenizer) -> State { - tokenizer.attempt( - StateName::DataStart, - State::Next(StateName::TextBefore), - State::Nok, - ) + tokenizer.attempt(Name::DataStart, State::Next(Name::TextBefore), State::Nok) } /// Resolve whitespace. @@ -9,6 +9,7 @@ mod constant; mod construct; mod content; mod parser; +mod state; mod subtokenize; mod token; mod tokenizer; diff --git a/src/state.rs b/src/state.rs new file mode 100644 index 0000000..7e28b0c --- /dev/null +++ b/src/state.rs @@ -0,0 +1,622 @@ +use crate::construct; +use crate::content; +use crate::tokenizer::Tokenizer; + +/// The result of a state. +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum State { + /// Move to [`Name`][] next. + Next(Name), + /// Retry in [`Name`][]. + Retry(Name), + /// The state is successful. + Ok, + /// The state is not successful. + Nok, +} + +/// Names of functions to move to. +#[derive(Debug, Clone, Copy, PartialEq)] +#[allow(clippy::enum_variant_names)] +pub enum Name { + AttentionStart, + AttentionInside, + + AutolinkStart, + AutolinkOpen, + AutolinkSchemeOrEmailAtext, + AutolinkSchemeInsideOrEmailAtext, + AutolinkUrlInside, + AutolinkEmailAtSignOrDot, + AutolinkEmailAtext, + AutolinkEmailValue, + AutolinkEmailLabel, + + BlankLineStart, + BlankLineAfter, + + BlockQuoteStart, + BlockQuoteBefore, + BlockQuoteContStart, + BlockQuoteContBefore, + BlockQuoteContAfter, + + BomStart, + BomInside, + + CharacterEscapeStart, + CharacterEscapeInside, + + CharacterReferenceStart, + CharacterReferenceOpen, + CharacterReferenceNumeric, + CharacterReferenceValue, + + CodeFencedStart, + CodeFencedBeforeSequenceOpen, + CodeFencedSequenceOpen, + CodeFencedInfoBefore, + CodeFencedInfo, + CodeFencedMetaBefore, + CodeFencedMeta, + CodeFencedAtNonLazyBreak, + CodeFencedCloseBefore, + CodeFencedCloseStart, + CodeFencedBeforeSequenceClose, + CodeFencedSequenceClose, + CodeFencedAfterSequenceClose, + CodeFencedContentBefore, + CodeFencedContentStart, + CodeFencedBeforeContentChunk, + CodeFencedContentChunk, + CodeFencedAfter, + + CodeIndentedStart, + CodeIndentedAtBreak, + CodeIndentedAfter, + CodeIndentedFurtherStart, + CodeIndentedInside, + CodeIndentedFurtherEnd, + CodeIndentedFurtherBegin, + CodeIndentedFurtherAfter, + + CodeTextStart, + CodeTextSequenceOpen, + CodeTextBetween, + CodeTextData, + CodeTextSequenceClose, + + DataStart, + DataInside, + DataAtBreak, + + DefinitionStart, + DefinitionBefore, + DefinitionLabelAfter, + DefinitionMarkerAfter, + DefinitionDestinationBefore, + DefinitionDestinationAfter, + DefinitionDestinationMissing, + DefinitionTitleBefore, + DefinitionAfter, + DefinitionAfterWhitespace, + DefinitionTitleBeforeMarker, + DefinitionTitleAfter, + DefinitionTitleAfterOptionalWhitespace, + + DestinationStart, + DestinationEnclosedBefore, + DestinationEnclosed, + DestinationEnclosedEscape, + DestinationRaw, + DestinationRawEscape, + + DocumentStart, + DocumentContainerExistingBefore, + DocumentContainerExistingAfter, + DocumentContainerNewBefore, + DocumentContainerNewBeforeNotBlockQuote, + DocumentContainerNewBeforeNotList, + DocumentContainerNewAfter, + DocumentContainersAfter, + DocumentFlowInside, + DocumentFlowEnd, + + FlowStart, + FlowBeforeCodeIndented, + FlowBeforeCodeFenced, + FlowBeforeHtml, + FlowBeforeHeadingAtx, + FlowBeforeHeadingSetext, + FlowBeforeThematicBreak, + FlowBeforeDefinition, + FlowAfter, + FlowBlankLineBefore, + FlowBlankLineAfter, + FlowBeforeParagraph, + + HardBreakEscapeStart, + HardBreakEscapeAfter, + + HeadingAtxStart, + HeadingAtxBefore, + HeadingAtxSequenceOpen, + HeadingAtxAtBreak, + HeadingAtxSequenceFurther, + HeadingAtxData, + + HeadingSetextStart, + HeadingSetextBefore, + HeadingSetextInside, + HeadingSetextAfter, + + HtmlFlowStart, + HtmlFlowBefore, + HtmlFlowOpen, + HtmlFlowDeclarationOpen, + HtmlFlowCommentOpenInside, + HtmlFlowCdataOpenInside, + HtmlFlowTagCloseStart, + HtmlFlowTagName, + HtmlFlowBasicSelfClosing, + HtmlFlowCompleteClosingTagAfter, + HtmlFlowCompleteEnd, + HtmlFlowCompleteAttributeNameBefore, + HtmlFlowCompleteAttributeName, + HtmlFlowCompleteAttributeNameAfter, + HtmlFlowCompleteAttributeValueBefore, + HtmlFlowCompleteAttributeValueQuoted, + HtmlFlowCompleteAttributeValueQuotedAfter, + HtmlFlowCompleteAttributeValueUnquoted, + HtmlFlowCompleteAfter, + HtmlFlowBlankLineBefore, + HtmlFlowContinuation, + HtmlFlowContinuationDeclarationInside, + HtmlFlowContinuationAfter, + HtmlFlowContinuationStart, + HtmlFlowContinuationBefore, + HtmlFlowContinuationCommentInside, + HtmlFlowContinuationRawTagOpen, + HtmlFlowContinuationRawEndTag, + HtmlFlowContinuationClose, + HtmlFlowContinuationCdataInside, + HtmlFlowContinuationStartNonLazy, + + HtmlTextStart, + HtmlTextOpen, + HtmlTextDeclarationOpen, + HtmlTextTagCloseStart, + HtmlTextTagClose, + HtmlTextTagCloseBetween, + HtmlTextTagOpen, + HtmlTextTagOpenBetween, + HtmlTextTagOpenAttributeName, + HtmlTextTagOpenAttributeNameAfter, + HtmlTextTagOpenAttributeValueBefore, + HtmlTextTagOpenAttributeValueQuoted, + HtmlTextTagOpenAttributeValueQuotedAfter, + HtmlTextTagOpenAttributeValueUnquoted, + HtmlTextCdata, + HtmlTextCdataOpenInside, + HtmlTextCdataClose, + HtmlTextCdataEnd, + HtmlTextCommentOpenInside, + HtmlTextCommentStart, + HtmlTextCommentStartDash, + HtmlTextComment, + HtmlTextCommentClose, + HtmlTextDeclaration, + HtmlTextEnd, + HtmlTextInstruction, + HtmlTextInstructionClose, + HtmlTextLineEndingBefore, + HtmlTextLineEndingAfter, + HtmlTextLineEndingAfterPrefix, + + LabelStart, + LabelAtBreak, + LabelEolAfter, + LabelAtBlankLine, + LabelEscape, + LabelInside, + + LabelEndStart, + LabelEndAfter, + LabelEndResourceStart, + LabelEndResourceBefore, + LabelEndResourceOpen, + LabelEndResourceDestinationAfter, + LabelEndResourceDestinationMissing, + LabelEndResourceBetween, + LabelEndResourceTitleAfter, + LabelEndResourceEnd, + LabelEndOk, + LabelEndNok, + LabelEndReferenceFull, + LabelEndReferenceFullAfter, + LabelEndReferenceNotFull, + LabelEndReferenceCollapsed, + LabelEndReferenceCollapsedOpen, + + LabelStartImageStart, + LabelStartImageOpen, + + LabelStartLinkStart, + + ListStart, + ListBefore, + ListNok, + ListBeforeOrdered, + ListBeforeUnordered, + ListValue, + ListMarker, + ListMarkerAfter, + ListAfter, + ListMarkerAfterFilled, + ListWhitespace, + ListPrefixOther, + ListWhitespaceAfter, + ListContStart, + ListContBlank, + ListContFilled, + ListOk, + + NonLazyContinuationStart, + NonLazyContinuationAfter, + + ParagraphStart, + ParagraphInside, + + SpaceOrTabStart, + SpaceOrTabInside, + SpaceOrTabAfter, + + SpaceOrTabEolStart, + SpaceOrTabEolAfterFirst, + SpaceOrTabEolAfterEol, + SpaceOrTabEolAtEol, + SpaceOrTabEolAfterMore, + + StringStart, + StringBefore, + StringBeforeData, + + TextStart, + TextBefore, + TextBeforeHtml, + TextBeforeHardBreakEscape, + TextBeforeData, + + ThematicBreakStart, + ThematicBreakBefore, + ThematicBreakSequence, + ThematicBreakAtBreak, + + TitleStart, + TitleBegin, + TitleAfterEol, + TitleAtBreak, + TitleAtBlankLine, + TitleEscape, + TitleInside, +} + +#[allow(clippy::too_many_lines)] +/// Call the corresponding function for a state name. +pub fn call(tokenizer: &mut Tokenizer, name: Name) -> State { + let func = match name { + Name::AttentionStart => construct::attention::start, + Name::AttentionInside => construct::attention::inside, + + Name::AutolinkStart => construct::autolink::start, + Name::AutolinkOpen => construct::autolink::open, + Name::AutolinkSchemeOrEmailAtext => construct::autolink::scheme_or_email_atext, + Name::AutolinkSchemeInsideOrEmailAtext => construct::autolink::scheme_inside_or_email_atext, + Name::AutolinkUrlInside => construct::autolink::url_inside, + Name::AutolinkEmailAtSignOrDot => construct::autolink::email_at_sign_or_dot, + Name::AutolinkEmailAtext => construct::autolink::email_atext, + Name::AutolinkEmailValue => construct::autolink::email_value, + Name::AutolinkEmailLabel => construct::autolink::email_label, + + Name::BlankLineStart => construct::blank_line::start, + Name::BlankLineAfter => construct::blank_line::after, + + Name::BlockQuoteStart => construct::block_quote::start, + Name::BlockQuoteBefore => construct::block_quote::before, + Name::BlockQuoteContStart => construct::block_quote::cont_start, + Name::BlockQuoteContBefore => construct::block_quote::cont_before, + Name::BlockQuoteContAfter => construct::block_quote::cont_after, + + Name::BomStart => construct::partial_bom::start, + Name::BomInside => construct::partial_bom::inside, + + Name::CharacterEscapeStart => construct::character_escape::start, + Name::CharacterEscapeInside => construct::character_escape::inside, + + Name::CharacterReferenceStart => construct::character_reference::start, + Name::CharacterReferenceOpen => construct::character_reference::open, + Name::CharacterReferenceNumeric => construct::character_reference::numeric, + Name::CharacterReferenceValue => construct::character_reference::value, + + Name::CodeFencedStart => construct::code_fenced::start, + Name::CodeFencedBeforeSequenceOpen => construct::code_fenced::before_sequence_open, + Name::CodeFencedSequenceOpen => construct::code_fenced::sequence_open, + Name::CodeFencedInfoBefore => construct::code_fenced::info_before, + Name::CodeFencedInfo => construct::code_fenced::info, + Name::CodeFencedMetaBefore => construct::code_fenced::meta_before, + Name::CodeFencedMeta => construct::code_fenced::meta, + Name::CodeFencedAtNonLazyBreak => construct::code_fenced::at_non_lazy_break, + Name::CodeFencedCloseBefore => construct::code_fenced::close_before, + Name::CodeFencedCloseStart => construct::code_fenced::close_start, + Name::CodeFencedBeforeSequenceClose => construct::code_fenced::before_sequence_close, + Name::CodeFencedSequenceClose => construct::code_fenced::sequence_close, + Name::CodeFencedAfterSequenceClose => construct::code_fenced::sequence_close_after, + Name::CodeFencedContentBefore => construct::code_fenced::content_before, + Name::CodeFencedContentStart => construct::code_fenced::content_start, + Name::CodeFencedBeforeContentChunk => construct::code_fenced::before_content_chunk, + Name::CodeFencedContentChunk => construct::code_fenced::content_chunk, + Name::CodeFencedAfter => construct::code_fenced::after, + + Name::CodeIndentedStart => construct::code_indented::start, + Name::CodeIndentedAtBreak => construct::code_indented::at_break, + Name::CodeIndentedAfter => construct::code_indented::after, + Name::CodeIndentedFurtherStart => construct::code_indented::further_start, + Name::CodeIndentedInside => construct::code_indented::inside, + Name::CodeIndentedFurtherEnd => construct::code_indented::further_end, + Name::CodeIndentedFurtherBegin => construct::code_indented::further_begin, + Name::CodeIndentedFurtherAfter => construct::code_indented::further_after, + + Name::CodeTextStart => construct::code_text::start, + Name::CodeTextSequenceOpen => construct::code_text::sequence_open, + Name::CodeTextBetween => construct::code_text::between, + Name::CodeTextData => construct::code_text::data, + Name::CodeTextSequenceClose => construct::code_text::sequence_close, + + Name::DataStart => construct::partial_data::start, + Name::DataInside => construct::partial_data::inside, + Name::DataAtBreak => construct::partial_data::at_break, + + Name::DefinitionStart => construct::definition::start, + Name::DefinitionBefore => construct::definition::before, + Name::DefinitionLabelAfter => construct::definition::label_after, + Name::DefinitionMarkerAfter => construct::definition::marker_after, + Name::DefinitionDestinationBefore => construct::definition::destination_before, + Name::DefinitionDestinationAfter => construct::definition::destination_after, + Name::DefinitionDestinationMissing => construct::definition::destination_missing, + Name::DefinitionTitleBefore => construct::definition::title_before, + Name::DefinitionAfter => construct::definition::after, + Name::DefinitionAfterWhitespace => construct::definition::after_whitespace, + Name::DefinitionTitleBeforeMarker => construct::definition::title_before_marker, + Name::DefinitionTitleAfter => construct::definition::title_after, + Name::DefinitionTitleAfterOptionalWhitespace => { + construct::definition::title_after_optional_whitespace + } + + Name::DestinationStart => construct::partial_destination::start, + Name::DestinationEnclosedBefore => construct::partial_destination::enclosed_before, + Name::DestinationEnclosed => construct::partial_destination::enclosed, + Name::DestinationEnclosedEscape => construct::partial_destination::enclosed_escape, + Name::DestinationRaw => construct::partial_destination::raw, + Name::DestinationRawEscape => construct::partial_destination::raw_escape, + + Name::DocumentStart => content::document::start, + Name::DocumentContainerExistingBefore => content::document::container_existing_before, + Name::DocumentContainerExistingAfter => content::document::container_existing_after, + Name::DocumentContainerNewBefore => content::document::container_new_before, + Name::DocumentContainerNewBeforeNotBlockQuote => { + content::document::container_new_before_not_block_quote + } + Name::DocumentContainerNewBeforeNotList => content::document::container_new_before_not_list, + Name::DocumentContainerNewAfter => content::document::container_new_after, + Name::DocumentContainersAfter => content::document::containers_after, + Name::DocumentFlowEnd => content::document::flow_end, + Name::DocumentFlowInside => content::document::flow_inside, + + Name::FlowStart => content::flow::start, + Name::FlowBeforeCodeIndented => content::flow::before_code_indented, + Name::FlowBeforeCodeFenced => content::flow::before_code_fenced, + Name::FlowBeforeHtml => content::flow::before_html, + Name::FlowBeforeHeadingAtx => content::flow::before_heading_atx, + Name::FlowBeforeHeadingSetext => content::flow::before_heading_setext, + Name::FlowBeforeThematicBreak => content::flow::before_thematic_break, + Name::FlowBeforeDefinition => content::flow::before_definition, + Name::FlowAfter => content::flow::after, + Name::FlowBlankLineBefore => content::flow::blank_line_before, + Name::FlowBlankLineAfter => content::flow::blank_line_after, + Name::FlowBeforeParagraph => content::flow::before_paragraph, + + Name::HardBreakEscapeStart => construct::hard_break_escape::start, + Name::HardBreakEscapeAfter => construct::hard_break_escape::after, + + Name::HeadingAtxStart => construct::heading_atx::start, + Name::HeadingAtxBefore => construct::heading_atx::before, + Name::HeadingAtxSequenceOpen => construct::heading_atx::sequence_open, + Name::HeadingAtxAtBreak => construct::heading_atx::at_break, + Name::HeadingAtxSequenceFurther => construct::heading_atx::sequence_further, + Name::HeadingAtxData => construct::heading_atx::data, + + Name::HeadingSetextStart => construct::heading_setext::start, + Name::HeadingSetextBefore => construct::heading_setext::before, + Name::HeadingSetextInside => construct::heading_setext::inside, + Name::HeadingSetextAfter => construct::heading_setext::after, + + Name::HtmlFlowStart => construct::html_flow::start, + Name::HtmlFlowBefore => construct::html_flow::before, + Name::HtmlFlowOpen => construct::html_flow::open, + Name::HtmlFlowDeclarationOpen => construct::html_flow::declaration_open, + Name::HtmlFlowCommentOpenInside => construct::html_flow::comment_open_inside, + Name::HtmlFlowCdataOpenInside => construct::html_flow::cdata_open_inside, + Name::HtmlFlowTagCloseStart => construct::html_flow::tag_close_start, + Name::HtmlFlowTagName => construct::html_flow::tag_name, + Name::HtmlFlowBasicSelfClosing => construct::html_flow::basic_self_closing, + Name::HtmlFlowCompleteClosingTagAfter => construct::html_flow::complete_closing_tag_after, + Name::HtmlFlowCompleteEnd => construct::html_flow::complete_end, + Name::HtmlFlowCompleteAttributeNameBefore => { + construct::html_flow::complete_attribute_name_before + } + Name::HtmlFlowCompleteAttributeName => construct::html_flow::complete_attribute_name, + Name::HtmlFlowCompleteAttributeNameAfter => { + construct::html_flow::complete_attribute_name_after + } + Name::HtmlFlowCompleteAttributeValueBefore => { + construct::html_flow::complete_attribute_value_before + } + Name::HtmlFlowCompleteAttributeValueQuoted => { + construct::html_flow::complete_attribute_value_quoted + } + Name::HtmlFlowCompleteAttributeValueQuotedAfter => { + construct::html_flow::complete_attribute_value_quoted_after + } + Name::HtmlFlowCompleteAttributeValueUnquoted => { + construct::html_flow::complete_attribute_value_unquoted + } + Name::HtmlFlowCompleteAfter => construct::html_flow::complete_after, + Name::HtmlFlowBlankLineBefore => construct::html_flow::blank_line_before, + Name::HtmlFlowContinuation => construct::html_flow::continuation, + Name::HtmlFlowContinuationDeclarationInside => { + construct::html_flow::continuation_declaration_inside + } + Name::HtmlFlowContinuationAfter => construct::html_flow::continuation_after, + Name::HtmlFlowContinuationStart => construct::html_flow::continuation_start, + Name::HtmlFlowContinuationBefore => construct::html_flow::continuation_before, + Name::HtmlFlowContinuationCommentInside => { + construct::html_flow::continuation_comment_inside + } + Name::HtmlFlowContinuationRawTagOpen => construct::html_flow::continuation_raw_tag_open, + Name::HtmlFlowContinuationRawEndTag => construct::html_flow::continuation_raw_end_tag, + Name::HtmlFlowContinuationClose => construct::html_flow::continuation_close, + Name::HtmlFlowContinuationCdataInside => construct::html_flow::continuation_cdata_inside, + Name::HtmlFlowContinuationStartNonLazy => construct::html_flow::continuation_start_non_lazy, + + Name::HtmlTextStart => construct::html_text::start, + Name::HtmlTextOpen => construct::html_text::open, + Name::HtmlTextDeclarationOpen => construct::html_text::declaration_open, + Name::HtmlTextTagCloseStart => construct::html_text::tag_close_start, + Name::HtmlTextTagClose => construct::html_text::tag_close, + Name::HtmlTextTagCloseBetween => construct::html_text::tag_close_between, + Name::HtmlTextTagOpen => construct::html_text::tag_open, + Name::HtmlTextTagOpenBetween => construct::html_text::tag_open_between, + Name::HtmlTextTagOpenAttributeName => construct::html_text::tag_open_attribute_name, + Name::HtmlTextTagOpenAttributeNameAfter => { + construct::html_text::tag_open_attribute_name_after + } + Name::HtmlTextTagOpenAttributeValueBefore => { + construct::html_text::tag_open_attribute_value_before + } + Name::HtmlTextTagOpenAttributeValueQuoted => { + construct::html_text::tag_open_attribute_value_quoted + } + Name::HtmlTextTagOpenAttributeValueQuotedAfter => { + construct::html_text::tag_open_attribute_value_quoted_after + } + Name::HtmlTextTagOpenAttributeValueUnquoted => { + construct::html_text::tag_open_attribute_value_unquoted + } + Name::HtmlTextCdata => construct::html_text::cdata, + Name::HtmlTextCdataOpenInside => construct::html_text::cdata_open_inside, + Name::HtmlTextCdataClose => construct::html_text::cdata_close, + Name::HtmlTextCdataEnd => construct::html_text::cdata_end, + Name::HtmlTextCommentOpenInside => construct::html_text::comment_open_inside, + Name::HtmlTextCommentStart => construct::html_text::comment_start, + Name::HtmlTextCommentStartDash => construct::html_text::comment_start_dash, + Name::HtmlTextComment => construct::html_text::comment, + Name::HtmlTextCommentClose => construct::html_text::comment_close, + Name::HtmlTextDeclaration => construct::html_text::declaration, + Name::HtmlTextEnd => construct::html_text::end, + Name::HtmlTextInstruction => construct::html_text::instruction, + Name::HtmlTextInstructionClose => construct::html_text::instruction_close, + Name::HtmlTextLineEndingBefore => construct::html_text::line_ending_before, + Name::HtmlTextLineEndingAfter => construct::html_text::line_ending_after, + Name::HtmlTextLineEndingAfterPrefix => construct::html_text::line_ending_after_prefix, + + Name::LabelStart => construct::partial_label::start, + Name::LabelAtBreak => construct::partial_label::at_break, + Name::LabelEolAfter => construct::partial_label::eol_after, + Name::LabelAtBlankLine => construct::partial_label::at_blank_line, + Name::LabelEscape => construct::partial_label::escape, + Name::LabelInside => construct::partial_label::inside, + + Name::LabelEndStart => construct::label_end::start, + Name::LabelEndAfter => construct::label_end::after, + Name::LabelEndResourceStart => construct::label_end::resource_start, + Name::LabelEndResourceBefore => construct::label_end::resource_before, + Name::LabelEndResourceOpen => construct::label_end::resource_open, + Name::LabelEndResourceDestinationAfter => construct::label_end::resource_destination_after, + Name::LabelEndResourceDestinationMissing => { + construct::label_end::resource_destination_missing + } + Name::LabelEndResourceBetween => construct::label_end::resource_between, + Name::LabelEndResourceTitleAfter => construct::label_end::resource_title_after, + Name::LabelEndResourceEnd => construct::label_end::resource_end, + Name::LabelEndOk => construct::label_end::ok, + Name::LabelEndNok => construct::label_end::nok, + Name::LabelEndReferenceFull => construct::label_end::reference_full, + Name::LabelEndReferenceFullAfter => construct::label_end::reference_full_after, + Name::LabelEndReferenceNotFull => construct::label_end::reference_not_full, + Name::LabelEndReferenceCollapsed => construct::label_end::reference_collapsed, + Name::LabelEndReferenceCollapsedOpen => construct::label_end::reference_collapsed_open, + + Name::LabelStartImageStart => construct::label_start_image::start, + Name::LabelStartImageOpen => construct::label_start_image::open, + Name::LabelStartLinkStart => construct::label_start_link::start, + + Name::ListStart => construct::list::start, + Name::ListBefore => construct::list::before, + Name::ListNok => construct::list::nok, + Name::ListBeforeOrdered => construct::list::before_ordered, + Name::ListBeforeUnordered => construct::list::before_unordered, + Name::ListValue => construct::list::value, + Name::ListMarker => construct::list::marker, + Name::ListMarkerAfter => construct::list::marker_after, + Name::ListAfter => construct::list::after, + Name::ListMarkerAfterFilled => construct::list::marker_after_filled, + Name::ListWhitespace => construct::list::whitespace, + Name::ListWhitespaceAfter => construct::list::whitespace_after, + Name::ListPrefixOther => construct::list::prefix_other, + Name::ListContStart => construct::list::cont_start, + Name::ListContBlank => construct::list::cont_blank, + Name::ListContFilled => construct::list::cont_filled, + Name::ListOk => construct::list::ok, + + Name::NonLazyContinuationStart => construct::partial_non_lazy_continuation::start, + Name::NonLazyContinuationAfter => construct::partial_non_lazy_continuation::after, + + Name::ParagraphStart => construct::paragraph::start, + Name::ParagraphInside => construct::paragraph::inside, + + Name::SpaceOrTabStart => construct::partial_space_or_tab::start, + Name::SpaceOrTabInside => construct::partial_space_or_tab::inside, + Name::SpaceOrTabAfter => construct::partial_space_or_tab::after, + + Name::SpaceOrTabEolStart => construct::partial_space_or_tab::eol_start, + Name::SpaceOrTabEolAfterFirst => construct::partial_space_or_tab::eol_after_first, + Name::SpaceOrTabEolAfterEol => construct::partial_space_or_tab::eol_after_eol, + Name::SpaceOrTabEolAtEol => construct::partial_space_or_tab::eol_at_eol, + Name::SpaceOrTabEolAfterMore => construct::partial_space_or_tab::eol_after_more, + + Name::StringStart => content::string::start, + Name::StringBefore => content::string::before, + Name::StringBeforeData => content::string::before_data, + + Name::TextStart => content::text::start, + Name::TextBefore => content::text::before, + Name::TextBeforeHtml => content::text::before_html, + Name::TextBeforeHardBreakEscape => content::text::before_hard_break_escape, + Name::TextBeforeData => content::text::before_data, + + Name::ThematicBreakStart => construct::thematic_break::start, + Name::ThematicBreakBefore => construct::thematic_break::before, + Name::ThematicBreakSequence => construct::thematic_break::sequence, + Name::ThematicBreakAtBreak => construct::thematic_break::at_break, + + Name::TitleStart => construct::partial_title::start, + Name::TitleBegin => construct::partial_title::begin, + Name::TitleAfterEol => construct::partial_title::after_eol, + Name::TitleAtBreak => construct::partial_title::at_break, + Name::TitleAtBlankLine => construct::partial_title::at_blank_line, + Name::TitleEscape => construct::partial_title::escape, + Name::TitleInside => construct::partial_title::inside, + }; + + func(tokenizer) +} diff --git a/src/subtokenize.rs b/src/subtokenize.rs index bf6a106..c545043 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -22,7 +22,8 @@ //! before any level that can include references can be parsed. use crate::parser::ParseState; -use crate::tokenizer::{ContentType, Event, EventType, State, StateName, Tokenizer}; +use crate::state::{Name, State}; +use crate::tokenizer::{ContentType, Event, EventType, Tokenizer}; use crate::util::edit_map::EditMap; /// Create a link between two [`Event`][]s. @@ -79,9 +80,9 @@ pub fn subtokenize(events: &mut Vec<Event>, parse_state: &ParseState) -> bool { let mut tokenizer = Tokenizer::new(event.point.clone(), parse_state); // Substate. let mut state = State::Next(if link.content_type == ContentType::String { - StateName::StringStart + Name::StringStart } else { - StateName::TextStart + Name::TextStart }); // Loop through links to pass them in order to the subtokenizer. diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 04a8cc3..baad6ed 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -12,9 +12,8 @@ //! [`check`]: Tokenizer::check use crate::constant::TAB_SIZE; -use crate::construct; -use crate::content; use crate::parser::ParseState; +use crate::state::{call, Name, State}; use crate::token::{Token, VOID_TOKENS}; use crate::util::edit_map::EditMap; @@ -96,304 +95,6 @@ pub struct Event { /// the compiler and other users. pub type Resolver = dyn FnOnce(&mut Tokenizer); -/// Names of functions to move to. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum StateName { - AttentionStart, - AttentionInside, - - AutolinkStart, - AutolinkOpen, - AutolinkSchemeOrEmailAtext, - AutolinkSchemeInsideOrEmailAtext, - AutolinkUrlInside, - AutolinkEmailAtSignOrDot, - AutolinkEmailAtext, - AutolinkEmailValue, - AutolinkEmailLabel, - - BlankLineStart, - BlankLineAfter, - - BlockQuoteStart, - BlockQuoteBefore, - BlockQuoteContStart, - BlockQuoteContBefore, - BlockQuoteContAfter, - - BomStart, - BomInside, - - CharacterEscapeStart, - CharacterEscapeInside, - - CharacterReferenceStart, - CharacterReferenceOpen, - CharacterReferenceNumeric, - CharacterReferenceValue, - - CodeFencedStart, - CodeFencedBeforeSequenceOpen, - CodeFencedSequenceOpen, - CodeFencedInfoBefore, - CodeFencedInfo, - CodeFencedMetaBefore, - CodeFencedMeta, - CodeFencedAtNonLazyBreak, - CodeFencedCloseBefore, - CodeFencedCloseStart, - CodeFencedBeforeSequenceClose, - CodeFencedSequenceClose, - CodeFencedAfterSequenceClose, - CodeFencedContentBefore, - CodeFencedContentStart, - CodeFencedBeforeContentChunk, - CodeFencedContentChunk, - CodeFencedAfter, - - CodeIndentedStart, - CodeIndentedAtBreak, - CodeIndentedAfter, - CodeIndentedFurtherStart, - CodeIndentedInside, - CodeIndentedFurtherEnd, - CodeIndentedFurtherBegin, - CodeIndentedFurtherAfter, - - CodeTextStart, - CodeTextSequenceOpen, - CodeTextBetween, - CodeTextData, - CodeTextSequenceClose, - - DataStart, - DataInside, - DataAtBreak, - - DefinitionStart, - DefinitionBefore, - DefinitionLabelAfter, - DefinitionMarkerAfter, - DefinitionDestinationBefore, - DefinitionDestinationAfter, - DefinitionDestinationMissing, - DefinitionTitleBefore, - DefinitionAfter, - DefinitionAfterWhitespace, - DefinitionTitleBeforeMarker, - DefinitionTitleAfter, - DefinitionTitleAfterOptionalWhitespace, - - DestinationStart, - DestinationEnclosedBefore, - DestinationEnclosed, - DestinationEnclosedEscape, - DestinationRaw, - DestinationRawEscape, - - DocumentStart, - DocumentContainerExistingBefore, - DocumentContainerExistingAfter, - DocumentContainerNewBefore, - DocumentContainerNewBeforeNotBlockQuote, - DocumentContainerNewBeforeNotList, - DocumentContainerNewAfter, - DocumentContainersAfter, - DocumentFlowInside, - DocumentFlowEnd, - - FlowStart, - FlowBeforeCodeIndented, - FlowBeforeCodeFenced, - FlowBeforeHtml, - FlowBeforeHeadingAtx, - FlowBeforeHeadingSetext, - FlowBeforeThematicBreak, - FlowBeforeDefinition, - FlowAfter, - FlowBlankLineBefore, - FlowBlankLineAfter, - FlowBeforeParagraph, - - HardBreakEscapeStart, - HardBreakEscapeAfter, - - HeadingAtxStart, - HeadingAtxBefore, - HeadingAtxSequenceOpen, - HeadingAtxAtBreak, - HeadingAtxSequenceFurther, - HeadingAtxData, - - HeadingSetextStart, - HeadingSetextBefore, - HeadingSetextInside, - HeadingSetextAfter, - - HtmlFlowStart, - HtmlFlowBefore, - HtmlFlowOpen, - HtmlFlowDeclarationOpen, - HtmlFlowCommentOpenInside, - HtmlFlowCdataOpenInside, - HtmlFlowTagCloseStart, - HtmlFlowTagName, - HtmlFlowBasicSelfClosing, - HtmlFlowCompleteClosingTagAfter, - HtmlFlowCompleteEnd, - HtmlFlowCompleteAttributeNameBefore, - HtmlFlowCompleteAttributeName, - HtmlFlowCompleteAttributeNameAfter, - HtmlFlowCompleteAttributeValueBefore, - HtmlFlowCompleteAttributeValueQuoted, - HtmlFlowCompleteAttributeValueQuotedAfter, - HtmlFlowCompleteAttributeValueUnquoted, - HtmlFlowCompleteAfter, - HtmlFlowBlankLineBefore, - HtmlFlowContinuation, - HtmlFlowContinuationDeclarationInside, - HtmlFlowContinuationAfter, - HtmlFlowContinuationStart, - HtmlFlowContinuationBefore, - HtmlFlowContinuationCommentInside, - HtmlFlowContinuationRawTagOpen, - HtmlFlowContinuationRawEndTag, - HtmlFlowContinuationClose, - HtmlFlowContinuationCdataInside, - HtmlFlowContinuationStartNonLazy, - - HtmlTextStart, - HtmlTextOpen, - HtmlTextDeclarationOpen, - HtmlTextTagCloseStart, - HtmlTextTagClose, - HtmlTextTagCloseBetween, - HtmlTextTagOpen, - HtmlTextTagOpenBetween, - HtmlTextTagOpenAttributeName, - HtmlTextTagOpenAttributeNameAfter, - HtmlTextTagOpenAttributeValueBefore, - HtmlTextTagOpenAttributeValueQuoted, - HtmlTextTagOpenAttributeValueQuotedAfter, - HtmlTextTagOpenAttributeValueUnquoted, - HtmlTextCdata, - HtmlTextCdataOpenInside, - HtmlTextCdataClose, - HtmlTextCdataEnd, - HtmlTextCommentOpenInside, - HtmlTextCommentStart, - HtmlTextCommentStartDash, - HtmlTextComment, - HtmlTextCommentClose, - HtmlTextDeclaration, - HtmlTextEnd, - HtmlTextInstruction, - HtmlTextInstructionClose, - HtmlTextLineEndingBefore, - HtmlTextLineEndingAfter, - HtmlTextLineEndingAfterPrefix, - - LabelStart, - LabelAtBreak, - LabelEolAfter, - LabelAtBlankLine, - LabelEscape, - LabelInside, - - LabelEndStart, - LabelEndAfter, - LabelEndResourceStart, - LabelEndResourceBefore, - LabelEndResourceOpen, - LabelEndResourceDestinationAfter, - LabelEndResourceDestinationMissing, - LabelEndResourceBetween, - LabelEndResourceTitleAfter, - LabelEndResourceEnd, - LabelEndOk, - LabelEndNok, - LabelEndReferenceFull, - LabelEndReferenceFullAfter, - LabelEndReferenceNotFull, - LabelEndReferenceCollapsed, - LabelEndReferenceCollapsedOpen, - - LabelStartImageStart, - LabelStartImageOpen, - - LabelStartLinkStart, - - ListStart, - ListBefore, - ListNok, - ListBeforeOrdered, - ListBeforeUnordered, - ListValue, - ListMarker, - ListMarkerAfter, - ListAfter, - ListMarkerAfterFilled, - ListWhitespace, - ListPrefixOther, - ListWhitespaceAfter, - ListContStart, - ListContBlank, - ListContFilled, - ListOk, - - NonLazyContinuationStart, - NonLazyContinuationAfter, - - ParagraphStart, - ParagraphInside, - - SpaceOrTabStart, - SpaceOrTabInside, - SpaceOrTabAfter, - - SpaceOrTabEolStart, - SpaceOrTabEolAfterFirst, - SpaceOrTabEolAfterEol, - SpaceOrTabEolAtEol, - SpaceOrTabEolAfterMore, - - StringStart, - StringBefore, - StringBeforeData, - - TextStart, - TextBefore, - TextBeforeHtml, - TextBeforeHardBreakEscape, - TextBeforeData, - - ThematicBreakStart, - ThematicBreakBefore, - ThematicBreakSequence, - ThematicBreakAtBreak, - - TitleStart, - TitleBegin, - TitleAfterEol, - TitleAtBreak, - TitleAtBlankLine, - TitleEscape, - TitleInside, -} - -/// The result of a state. -#[derive(Debug, PartialEq, Copy, Clone)] -pub enum State { - /// Move to [`StateName`][] next. - Next(StateName), - /// Retry in [`StateName`][]. - Retry(StateName), - /// The state is successful. - Ok, - /// The state is not successful. - Nok, -} - /// Loose label starts we found. #[derive(Debug)] pub struct LabelStart { @@ -894,7 +595,7 @@ impl<'a> Tokenizer<'a> { /// Parse with `name` and its future states, to see if that results in /// [`State::Ok`][] or [`State::Nok`][], then revert in both cases. - pub fn check(&mut self, name: StateName, ok: State, nok: State) -> State { + pub fn check(&mut self, name: Name, ok: State, nok: State) -> State { // Always capture (and restore) when checking. // No need to capture (and restore) when `nok` is `State::Nok`, because the // parent attempt will do it. @@ -907,13 +608,13 @@ impl<'a> Tokenizer<'a> { nok, }); - call_impl(self, name) + call(self, name) } /// Parse with `name` and its future states, to see if that results in /// [`State::Ok`][] or [`State::Nok`][], revert in the case of /// `State::Nok`. - pub fn attempt(&mut self, name: StateName, ok: State, nok: State) -> State { + pub fn attempt(&mut self, name: Name, ok: State, nok: State) -> State { // Always capture (and restore) when checking. // No need to capture (and restore) when `nok` is `State::Nok`, because the // parent attempt will do it. @@ -930,7 +631,7 @@ impl<'a> Tokenizer<'a> { nok, }); - call_impl(self, name) + call(self, name) } /// Tokenize. @@ -1032,12 +733,12 @@ fn push_impl( log::debug!("feed: `{:?}` to {:?}", byte, name); tokenizer.expect(byte); - state = call_impl(tokenizer, name); + state = call(tokenizer, name); }; } State::Retry(name) => { log::debug!("retry: {:?}", name); - state = call_impl(tokenizer, name); + state = call(tokenizer, name); } } } @@ -1094,337 +795,3 @@ fn byte_action(bytes: &[u8], point: &Point) -> ByteAction { unreachable!("out of bounds") } } - -#[allow(clippy::too_many_lines)] -/// Call the corresponding function for a state name. -fn call_impl(tokenizer: &mut Tokenizer, name: StateName) -> State { - let func = match name { - StateName::AttentionStart => construct::attention::start, - StateName::AttentionInside => construct::attention::inside, - - StateName::AutolinkStart => construct::autolink::start, - StateName::AutolinkOpen => construct::autolink::open, - StateName::AutolinkSchemeOrEmailAtext => construct::autolink::scheme_or_email_atext, - StateName::AutolinkSchemeInsideOrEmailAtext => { - construct::autolink::scheme_inside_or_email_atext - } - StateName::AutolinkUrlInside => construct::autolink::url_inside, - StateName::AutolinkEmailAtSignOrDot => construct::autolink::email_at_sign_or_dot, - StateName::AutolinkEmailAtext => construct::autolink::email_atext, - StateName::AutolinkEmailValue => construct::autolink::email_value, - StateName::AutolinkEmailLabel => construct::autolink::email_label, - - StateName::BlankLineStart => construct::blank_line::start, - StateName::BlankLineAfter => construct::blank_line::after, - - StateName::BlockQuoteStart => construct::block_quote::start, - StateName::BlockQuoteBefore => construct::block_quote::before, - StateName::BlockQuoteContStart => construct::block_quote::cont_start, - StateName::BlockQuoteContBefore => construct::block_quote::cont_before, - StateName::BlockQuoteContAfter => construct::block_quote::cont_after, - - StateName::BomStart => construct::partial_bom::start, - StateName::BomInside => construct::partial_bom::inside, - - StateName::CharacterEscapeStart => construct::character_escape::start, - StateName::CharacterEscapeInside => construct::character_escape::inside, - - StateName::CharacterReferenceStart => construct::character_reference::start, - StateName::CharacterReferenceOpen => construct::character_reference::open, - StateName::CharacterReferenceNumeric => construct::character_reference::numeric, - StateName::CharacterReferenceValue => construct::character_reference::value, - - StateName::CodeFencedStart => construct::code_fenced::start, - StateName::CodeFencedBeforeSequenceOpen => construct::code_fenced::before_sequence_open, - StateName::CodeFencedSequenceOpen => construct::code_fenced::sequence_open, - StateName::CodeFencedInfoBefore => construct::code_fenced::info_before, - StateName::CodeFencedInfo => construct::code_fenced::info, - StateName::CodeFencedMetaBefore => construct::code_fenced::meta_before, - StateName::CodeFencedMeta => construct::code_fenced::meta, - StateName::CodeFencedAtNonLazyBreak => construct::code_fenced::at_non_lazy_break, - StateName::CodeFencedCloseBefore => construct::code_fenced::close_before, - StateName::CodeFencedCloseStart => construct::code_fenced::close_start, - StateName::CodeFencedBeforeSequenceClose => construct::code_fenced::before_sequence_close, - StateName::CodeFencedSequenceClose => construct::code_fenced::sequence_close, - StateName::CodeFencedAfterSequenceClose => construct::code_fenced::sequence_close_after, - StateName::CodeFencedContentBefore => construct::code_fenced::content_before, - StateName::CodeFencedContentStart => construct::code_fenced::content_start, - StateName::CodeFencedBeforeContentChunk => construct::code_fenced::before_content_chunk, - StateName::CodeFencedContentChunk => construct::code_fenced::content_chunk, - StateName::CodeFencedAfter => construct::code_fenced::after, - - StateName::CodeIndentedStart => construct::code_indented::start, - StateName::CodeIndentedAtBreak => construct::code_indented::at_break, - StateName::CodeIndentedAfter => construct::code_indented::after, - StateName::CodeIndentedFurtherStart => construct::code_indented::further_start, - StateName::CodeIndentedInside => construct::code_indented::inside, - StateName::CodeIndentedFurtherEnd => construct::code_indented::further_end, - StateName::CodeIndentedFurtherBegin => construct::code_indented::further_begin, - StateName::CodeIndentedFurtherAfter => construct::code_indented::further_after, - - StateName::CodeTextStart => construct::code_text::start, - StateName::CodeTextSequenceOpen => construct::code_text::sequence_open, - StateName::CodeTextBetween => construct::code_text::between, - StateName::CodeTextData => construct::code_text::data, - StateName::CodeTextSequenceClose => construct::code_text::sequence_close, - - StateName::DataStart => construct::partial_data::start, - StateName::DataInside => construct::partial_data::inside, - StateName::DataAtBreak => construct::partial_data::at_break, - - StateName::DefinitionStart => construct::definition::start, - StateName::DefinitionBefore => construct::definition::before, - StateName::DefinitionLabelAfter => construct::definition::label_after, - StateName::DefinitionMarkerAfter => construct::definition::marker_after, - StateName::DefinitionDestinationBefore => construct::definition::destination_before, - StateName::DefinitionDestinationAfter => construct::definition::destination_after, - StateName::DefinitionDestinationMissing => construct::definition::destination_missing, - StateName::DefinitionTitleBefore => construct::definition::title_before, - StateName::DefinitionAfter => construct::definition::after, - StateName::DefinitionAfterWhitespace => construct::definition::after_whitespace, - StateName::DefinitionTitleBeforeMarker => construct::definition::title_before_marker, - StateName::DefinitionTitleAfter => construct::definition::title_after, - StateName::DefinitionTitleAfterOptionalWhitespace => { - construct::definition::title_after_optional_whitespace - } - - StateName::DestinationStart => construct::partial_destination::start, - StateName::DestinationEnclosedBefore => construct::partial_destination::enclosed_before, - StateName::DestinationEnclosed => construct::partial_destination::enclosed, - StateName::DestinationEnclosedEscape => construct::partial_destination::enclosed_escape, - StateName::DestinationRaw => construct::partial_destination::raw, - StateName::DestinationRawEscape => construct::partial_destination::raw_escape, - - StateName::DocumentStart => content::document::start, - StateName::DocumentContainerExistingBefore => content::document::container_existing_before, - StateName::DocumentContainerExistingAfter => content::document::container_existing_after, - StateName::DocumentContainerNewBefore => content::document::container_new_before, - StateName::DocumentContainerNewBeforeNotBlockQuote => { - content::document::container_new_before_not_block_quote - } - StateName::DocumentContainerNewBeforeNotList => { - content::document::container_new_before_not_list - } - StateName::DocumentContainerNewAfter => content::document::container_new_after, - StateName::DocumentContainersAfter => content::document::containers_after, - StateName::DocumentFlowEnd => content::document::flow_end, - StateName::DocumentFlowInside => content::document::flow_inside, - - StateName::FlowStart => content::flow::start, - StateName::FlowBeforeCodeIndented => content::flow::before_code_indented, - StateName::FlowBeforeCodeFenced => content::flow::before_code_fenced, - StateName::FlowBeforeHtml => content::flow::before_html, - StateName::FlowBeforeHeadingAtx => content::flow::before_heading_atx, - StateName::FlowBeforeHeadingSetext => content::flow::before_heading_setext, - StateName::FlowBeforeThematicBreak => content::flow::before_thematic_break, - StateName::FlowBeforeDefinition => content::flow::before_definition, - StateName::FlowAfter => content::flow::after, - StateName::FlowBlankLineBefore => content::flow::blank_line_before, - StateName::FlowBlankLineAfter => content::flow::blank_line_after, - StateName::FlowBeforeParagraph => content::flow::before_paragraph, - - StateName::HardBreakEscapeStart => construct::hard_break_escape::start, - StateName::HardBreakEscapeAfter => construct::hard_break_escape::after, - - StateName::HeadingAtxStart => construct::heading_atx::start, - StateName::HeadingAtxBefore => construct::heading_atx::before, - StateName::HeadingAtxSequenceOpen => construct::heading_atx::sequence_open, - StateName::HeadingAtxAtBreak => construct::heading_atx::at_break, - StateName::HeadingAtxSequenceFurther => construct::heading_atx::sequence_further, - StateName::HeadingAtxData => construct::heading_atx::data, - - StateName::HeadingSetextStart => construct::heading_setext::start, - StateName::HeadingSetextBefore => construct::heading_setext::before, - StateName::HeadingSetextInside => construct::heading_setext::inside, - StateName::HeadingSetextAfter => construct::heading_setext::after, - - StateName::HtmlFlowStart => construct::html_flow::start, - StateName::HtmlFlowBefore => construct::html_flow::before, - StateName::HtmlFlowOpen => construct::html_flow::open, - StateName::HtmlFlowDeclarationOpen => construct::html_flow::declaration_open, - StateName::HtmlFlowCommentOpenInside => construct::html_flow::comment_open_inside, - StateName::HtmlFlowCdataOpenInside => construct::html_flow::cdata_open_inside, - StateName::HtmlFlowTagCloseStart => construct::html_flow::tag_close_start, - StateName::HtmlFlowTagName => construct::html_flow::tag_name, - StateName::HtmlFlowBasicSelfClosing => construct::html_flow::basic_self_closing, - StateName::HtmlFlowCompleteClosingTagAfter => { - construct::html_flow::complete_closing_tag_after - } - StateName::HtmlFlowCompleteEnd => construct::html_flow::complete_end, - StateName::HtmlFlowCompleteAttributeNameBefore => { - construct::html_flow::complete_attribute_name_before - } - StateName::HtmlFlowCompleteAttributeName => construct::html_flow::complete_attribute_name, - StateName::HtmlFlowCompleteAttributeNameAfter => { - construct::html_flow::complete_attribute_name_after - } - StateName::HtmlFlowCompleteAttributeValueBefore => { - construct::html_flow::complete_attribute_value_before - } - StateName::HtmlFlowCompleteAttributeValueQuoted => { - construct::html_flow::complete_attribute_value_quoted - } - StateName::HtmlFlowCompleteAttributeValueQuotedAfter => { - construct::html_flow::complete_attribute_value_quoted_after - } - StateName::HtmlFlowCompleteAttributeValueUnquoted => { - construct::html_flow::complete_attribute_value_unquoted - } - StateName::HtmlFlowCompleteAfter => construct::html_flow::complete_after, - StateName::HtmlFlowBlankLineBefore => construct::html_flow::blank_line_before, - StateName::HtmlFlowContinuation => construct::html_flow::continuation, - StateName::HtmlFlowContinuationDeclarationInside => { - construct::html_flow::continuation_declaration_inside - } - StateName::HtmlFlowContinuationAfter => construct::html_flow::continuation_after, - StateName::HtmlFlowContinuationStart => construct::html_flow::continuation_start, - StateName::HtmlFlowContinuationBefore => construct::html_flow::continuation_before, - StateName::HtmlFlowContinuationCommentInside => { - construct::html_flow::continuation_comment_inside - } - StateName::HtmlFlowContinuationRawTagOpen => { - construct::html_flow::continuation_raw_tag_open - } - StateName::HtmlFlowContinuationRawEndTag => construct::html_flow::continuation_raw_end_tag, - StateName::HtmlFlowContinuationClose => construct::html_flow::continuation_close, - StateName::HtmlFlowContinuationCdataInside => { - construct::html_flow::continuation_cdata_inside - } - StateName::HtmlFlowContinuationStartNonLazy => { - construct::html_flow::continuation_start_non_lazy - } - - StateName::HtmlTextStart => construct::html_text::start, - StateName::HtmlTextOpen => construct::html_text::open, - StateName::HtmlTextDeclarationOpen => construct::html_text::declaration_open, - StateName::HtmlTextTagCloseStart => construct::html_text::tag_close_start, - StateName::HtmlTextTagClose => construct::html_text::tag_close, - StateName::HtmlTextTagCloseBetween => construct::html_text::tag_close_between, - StateName::HtmlTextTagOpen => construct::html_text::tag_open, - StateName::HtmlTextTagOpenBetween => construct::html_text::tag_open_between, - StateName::HtmlTextTagOpenAttributeName => construct::html_text::tag_open_attribute_name, - StateName::HtmlTextTagOpenAttributeNameAfter => { - construct::html_text::tag_open_attribute_name_after - } - StateName::HtmlTextTagOpenAttributeValueBefore => { - construct::html_text::tag_open_attribute_value_before - } - StateName::HtmlTextTagOpenAttributeValueQuoted => { - construct::html_text::tag_open_attribute_value_quoted - } - StateName::HtmlTextTagOpenAttributeValueQuotedAfter => { - construct::html_text::tag_open_attribute_value_quoted_after - } - StateName::HtmlTextTagOpenAttributeValueUnquoted => { - construct::html_text::tag_open_attribute_value_unquoted - } - StateName::HtmlTextCdata => construct::html_text::cdata, - StateName::HtmlTextCdataOpenInside => construct::html_text::cdata_open_inside, - StateName::HtmlTextCdataClose => construct::html_text::cdata_close, - StateName::HtmlTextCdataEnd => construct::html_text::cdata_end, - StateName::HtmlTextCommentOpenInside => construct::html_text::comment_open_inside, - StateName::HtmlTextCommentStart => construct::html_text::comment_start, - StateName::HtmlTextCommentStartDash => construct::html_text::comment_start_dash, - StateName::HtmlTextComment => construct::html_text::comment, - StateName::HtmlTextCommentClose => construct::html_text::comment_close, - StateName::HtmlTextDeclaration => construct::html_text::declaration, - StateName::HtmlTextEnd => construct::html_text::end, - StateName::HtmlTextInstruction => construct::html_text::instruction, - StateName::HtmlTextInstructionClose => construct::html_text::instruction_close, - StateName::HtmlTextLineEndingBefore => construct::html_text::line_ending_before, - StateName::HtmlTextLineEndingAfter => construct::html_text::line_ending_after, - StateName::HtmlTextLineEndingAfterPrefix => construct::html_text::line_ending_after_prefix, - - StateName::LabelStart => construct::partial_label::start, - StateName::LabelAtBreak => construct::partial_label::at_break, - StateName::LabelEolAfter => construct::partial_label::eol_after, - StateName::LabelAtBlankLine => construct::partial_label::at_blank_line, - StateName::LabelEscape => construct::partial_label::escape, - StateName::LabelInside => construct::partial_label::inside, - - StateName::LabelEndStart => construct::label_end::start, - StateName::LabelEndAfter => construct::label_end::after, - StateName::LabelEndResourceStart => construct::label_end::resource_start, - StateName::LabelEndResourceBefore => construct::label_end::resource_before, - StateName::LabelEndResourceOpen => construct::label_end::resource_open, - StateName::LabelEndResourceDestinationAfter => { - construct::label_end::resource_destination_after - } - StateName::LabelEndResourceDestinationMissing => { - construct::label_end::resource_destination_missing - } - StateName::LabelEndResourceBetween => construct::label_end::resource_between, - StateName::LabelEndResourceTitleAfter => construct::label_end::resource_title_after, - StateName::LabelEndResourceEnd => construct::label_end::resource_end, - StateName::LabelEndOk => construct::label_end::ok, - StateName::LabelEndNok => construct::label_end::nok, - StateName::LabelEndReferenceFull => construct::label_end::reference_full, - StateName::LabelEndReferenceFullAfter => construct::label_end::reference_full_after, - StateName::LabelEndReferenceNotFull => construct::label_end::reference_not_full, - StateName::LabelEndReferenceCollapsed => construct::label_end::reference_collapsed, - StateName::LabelEndReferenceCollapsedOpen => construct::label_end::reference_collapsed_open, - - StateName::LabelStartImageStart => construct::label_start_image::start, - StateName::LabelStartImageOpen => construct::label_start_image::open, - StateName::LabelStartLinkStart => construct::label_start_link::start, - - StateName::ListStart => construct::list::start, - StateName::ListBefore => construct::list::before, - StateName::ListNok => construct::list::nok, - StateName::ListBeforeOrdered => construct::list::before_ordered, - StateName::ListBeforeUnordered => construct::list::before_unordered, - StateName::ListValue => construct::list::value, - StateName::ListMarker => construct::list::marker, - StateName::ListMarkerAfter => construct::list::marker_after, - StateName::ListAfter => construct::list::after, - StateName::ListMarkerAfterFilled => construct::list::marker_after_filled, - StateName::ListWhitespace => construct::list::whitespace, - StateName::ListWhitespaceAfter => construct::list::whitespace_after, - StateName::ListPrefixOther => construct::list::prefix_other, - StateName::ListContStart => construct::list::cont_start, - StateName::ListContBlank => construct::list::cont_blank, - StateName::ListContFilled => construct::list::cont_filled, - StateName::ListOk => construct::list::ok, - - StateName::NonLazyContinuationStart => construct::partial_non_lazy_continuation::start, - StateName::NonLazyContinuationAfter => construct::partial_non_lazy_continuation::after, - - StateName::ParagraphStart => construct::paragraph::start, - StateName::ParagraphInside => construct::paragraph::inside, - - StateName::SpaceOrTabStart => construct::partial_space_or_tab::start, - StateName::SpaceOrTabInside => construct::partial_space_or_tab::inside, - StateName::SpaceOrTabAfter => construct::partial_space_or_tab::after, - - StateName::SpaceOrTabEolStart => construct::partial_space_or_tab::eol_start, - StateName::SpaceOrTabEolAfterFirst => construct::partial_space_or_tab::eol_after_first, - StateName::SpaceOrTabEolAfterEol => construct::partial_space_or_tab::eol_after_eol, - StateName::SpaceOrTabEolAtEol => construct::partial_space_or_tab::eol_at_eol, - StateName::SpaceOrTabEolAfterMore => construct::partial_space_or_tab::eol_after_more, - - StateName::StringStart => content::string::start, - StateName::StringBefore => content::string::before, - StateName::StringBeforeData => content::string::before_data, - - StateName::TextStart => content::text::start, - StateName::TextBefore => content::text::before, - StateName::TextBeforeHtml => content::text::before_html, - StateName::TextBeforeHardBreakEscape => content::text::before_hard_break_escape, - StateName::TextBeforeData => content::text::before_data, - - StateName::ThematicBreakStart => construct::thematic_break::start, - StateName::ThematicBreakBefore => construct::thematic_break::before, - StateName::ThematicBreakSequence => construct::thematic_break::sequence, - StateName::ThematicBreakAtBreak => construct::thematic_break::at_break, - - StateName::TitleStart => construct::partial_title::start, - StateName::TitleBegin => construct::partial_title::begin, - StateName::TitleAfterEol => construct::partial_title::after_eol, - StateName::TitleAtBreak => construct::partial_title::at_break, - StateName::TitleAtBlankLine => construct::partial_title::at_blank_line, - StateName::TitleEscape => construct::partial_title::escape, - StateName::TitleInside => construct::partial_title::inside, - }; - - func(tokenizer) -} |