diff options
Diffstat (limited to '')
26 files changed, 433 insertions, 413 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs index daf662e..1750692 100644 --- a/src/construct/attention.rs +++ b/src/construct/attention.rs @@ -32,14 +32,14 @@ //! //! ## Tokens //! -//! * [`Emphasis`][TokenType::Emphasis] -//! * [`EmphasisSequence`][TokenType::EmphasisSequence] -//! * [`EmphasisText`][TokenType::EmphasisText] -//! * [`Strong`][TokenType::Strong] -//! * [`StrongSequence`][TokenType::StrongSequence] -//! * [`StrongText`][TokenType::StrongText] +//! * [`Emphasis`][Token::Emphasis] +//! * [`EmphasisSequence`][Token::EmphasisSequence] +//! * [`EmphasisText`][Token::EmphasisText] +//! * [`Strong`][Token::Strong] +//! * [`StrongSequence`][Token::StrongSequence] +//! * [`StrongText`][Token::StrongText] //! -//! > 👉 **Note**: while parsing, [`AttentionSequence`][TokenType::AttentionSequence] +//! > 👉 **Note**: while parsing, [`AttentionSequence`][Token::AttentionSequence] //! > is used, which is later compiled away. //! //! ## References @@ -51,7 +51,8 @@ //! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element //! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element -use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, Tokenizer}; use crate::unicode::PUNCTUATION; use crate::util::edit_map::EditMap; @@ -174,7 +175,7 @@ struct Sequence { pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('*' | '_') => { - tokenizer.enter(TokenType::AttentionSequence); + tokenizer.enter(Token::AttentionSequence); inside(tokenizer, code, MarkerKind::from_code(code)) } _ => (State::Nok, None), @@ -193,7 +194,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, marker: MarkerKind) -> StateFnR (State::Fn(Box::new(move |t, c| inside(t, c, marker))), None) } _ => { - tokenizer.exit(TokenType::AttentionSequence); + tokenizer.exit(Token::AttentionSequence); tokenizer.register_resolver("attention".to_string(), Box::new(resolve)); (State::Ok, Some(vec![code])) } @@ -216,7 +217,7 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { if enter.event_type == EventType::Enter { balance += 1; - if enter.token_type == TokenType::AttentionSequence { + if enter.token_type == Token::AttentionSequence { let end = start + 1; let exit = &tokenizer.events[end]; let marker = MarkerKind::from_code(codes[enter.index]); @@ -392,9 +393,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::Emphasis + Token::Emphasis } else { - TokenType::Strong + Token::Strong }, point: seq_open_enter.0.clone(), index: seq_open_enter.1, @@ -405,9 +406,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_open_enter.0.clone(), index: seq_open_enter.1, @@ -418,9 +419,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_open_exit.0.clone(), index: seq_open_exit.1, @@ -431,9 +432,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::EmphasisText + Token::EmphasisText } else { - TokenType::StrongText + Token::StrongText }, point: seq_open_exit.0.clone(), index: seq_open_exit.1, @@ -451,9 +452,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::EmphasisText + Token::EmphasisText } else { - TokenType::StrongText + Token::StrongText }, point: seq_close_enter.0.clone(), index: seq_close_enter.1, @@ -464,9 +465,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_close_enter.0.clone(), index: seq_close_enter.1, @@ -477,9 +478,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_close_exit.0.clone(), index: seq_close_exit.1, @@ -490,9 +491,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::Emphasis + Token::Emphasis } else { - TokenType::Strong + Token::Strong }, point: seq_close_exit.0.clone(), index: seq_close_exit.1, @@ -515,8 +516,8 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { let mut index = 0; while index < sequences.len() { let sequence = &sequences[index]; - tokenizer.events[sequence.event_index].token_type = TokenType::Data; - tokenizer.events[sequence.event_index + 1].token_type = TokenType::Data; + tokenizer.events[sequence.event_index].token_type = Token::Data; + tokenizer.events[sequence.event_index + 1].token_type = Token::Data; index += 1; } diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index e29bf8b..db4365f 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -84,10 +84,10 @@ //! //! ## Tokens //! -//! * [`Autolink`][TokenType::Autolink] -//! * [`AutolinkEmail`][TokenType::AutolinkEmail] -//! * [`AutolinkMarker`][TokenType::AutolinkMarker] -//! * [`AutolinkProtocol`][TokenType::AutolinkProtocol] +//! * [`Autolink`][Token::Autolink] +//! * [`AutolinkEmail`][Token::AutolinkEmail] +//! * [`AutolinkMarker`][Token::AutolinkMarker] +//! * [`AutolinkProtocol`][Token::AutolinkProtocol] //! //! ## References //! @@ -102,7 +102,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX}; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of an autolink. /// @@ -113,11 +114,11 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('<') => { - tokenizer.enter(TokenType::Autolink); - tokenizer.enter(TokenType::AutolinkMarker); + tokenizer.enter(Token::Autolink); + tokenizer.enter(Token::AutolinkMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::AutolinkMarker); - tokenizer.enter(TokenType::AutolinkProtocol); + tokenizer.exit(Token::AutolinkMarker); + tokenizer.enter(Token::AutolinkProtocol); (State::Fn(Box::new(open)), None) } _ => (State::Nok, None), @@ -195,7 +196,7 @@ fn scheme_inside_or_email_atext( fn url_inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.exit(TokenType::AutolinkProtocol); + tokenizer.exit(Token::AutolinkProtocol); end(tokenizer, code) } Code::Char(char) if char.is_ascii_control() => (State::Nok, None), @@ -260,10 +261,10 @@ fn email_label(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes } Code::Char('>') => { let index = tokenizer.events.len(); - tokenizer.exit(TokenType::AutolinkProtocol); + tokenizer.exit(Token::AutolinkProtocol); // Change the token type. - tokenizer.events[index - 1].token_type = TokenType::AutolinkEmail; - tokenizer.events[index].token_type = TokenType::AutolinkEmail; + tokenizer.events[index - 1].token_type = Token::AutolinkEmail; + tokenizer.events[index].token_type = Token::AutolinkEmail; end(tokenizer, code) } _ => email_value(tokenizer, code, size), @@ -306,10 +307,10 @@ fn email_value(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.enter(TokenType::AutolinkMarker); + tokenizer.enter(Token::AutolinkMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::AutolinkMarker); - tokenizer.exit(TokenType::Autolink); + tokenizer.exit(Token::AutolinkMarker); + tokenizer.exit(Token::Autolink); (State::Ok, None) } _ => unreachable!("expected `>`"), diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index 3ca3266..69a4709 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -20,7 +20,7 @@ //! //! ## Tokens //! -//! * [`SpaceOrTab`][crate::tokenizer::TokenType::SpaceOrTab] +//! * [`SpaceOrTab`][crate::token::Token::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs index a3ffbc7..096ac0a 100644 --- a/src/construct/block_quote.rs +++ b/src/construct/block_quote.rs @@ -19,10 +19,10 @@ //! //! ## Tokens //! -//! * [`BlockQuote`][TokenType::BlockQuote] -//! * [`BlockQuoteMarker`][TokenType::BlockQuoteMarker] -//! * [`BlockQuotePrefix`][TokenType::BlockQuotePrefix] -//! * [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace] +//! * [`BlockQuote`][Token::BlockQuote] +//! * [`BlockQuoteMarker`][Token::BlockQuoteMarker] +//! * [`BlockQuotePrefix`][Token::BlockQuotePrefix] +//! * [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace] //! //! ## References //! @@ -35,7 +35,8 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::space_or_tab_min_max; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of block quote. /// @@ -55,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.enter(TokenType::BlockQuote); + tokenizer.enter(Token::BlockQuote); cont_before(tokenizer, code) } _ => cont_before(tokenizer, code), @@ -82,10 +83,10 @@ pub fn cont(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.enter(TokenType::BlockQuotePrefix); - tokenizer.enter(TokenType::BlockQuoteMarker); + tokenizer.enter(Token::BlockQuotePrefix); + tokenizer.enter(Token::BlockQuoteMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::BlockQuoteMarker); + tokenizer.exit(Token::BlockQuoteMarker); (State::Fn(Box::new(cont_after)), None) } _ => (State::Nok, None), @@ -101,20 +102,20 @@ fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn cont_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::VirtualSpace | Code::Char('\t' | ' ') => { - tokenizer.enter(TokenType::BlockQuoteWhitespace); + tokenizer.enter(Token::BlockQuoteWhitespace); tokenizer.consume(code); - tokenizer.exit(TokenType::BlockQuoteWhitespace); - tokenizer.exit(TokenType::BlockQuotePrefix); + tokenizer.exit(Token::BlockQuoteWhitespace); + tokenizer.exit(Token::BlockQuotePrefix); (State::Ok, None) } _ => { - tokenizer.exit(TokenType::BlockQuotePrefix); + tokenizer.exit(Token::BlockQuotePrefix); (State::Ok, Some(vec![code])) } } } /// End of a block quote. -pub fn end() -> Vec<TokenType> { - vec![TokenType::BlockQuote] +pub fn end() -> Vec<Token> { + vec![Token::BlockQuote] } diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index 0ccc574..55d321a 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -19,9 +19,9 @@ //! //! ## Tokens //! -//! * [`CharacterEscape`][TokenType::CharacterEscape] -//! * [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker] -//! * [`CharacterEscapeValue`][TokenType::CharacterEscapeValue] +//! * [`CharacterEscape`][Token::CharacterEscape] +//! * [`CharacterEscapeMarker`][Token::CharacterEscapeMarker] +//! * [`CharacterEscapeValue`][Token::CharacterEscapeValue] //! //! ## References //! @@ -33,7 +33,8 @@ //! [character_reference]: crate::construct::character_reference //! [hard_break_escape]: crate::construct::hard_break_escape -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of a character escape. /// @@ -45,10 +46,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('\\') => { - tokenizer.enter(TokenType::CharacterEscape); - tokenizer.enter(TokenType::CharacterEscapeMarker); + tokenizer.enter(Token::CharacterEscape); + tokenizer.enter(Token::CharacterEscapeMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterEscapeMarker); + tokenizer.exit(Token::CharacterEscapeMarker); (State::Fn(Box::new(inside)), None) } _ => (State::Nok, None), @@ -65,10 +66,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(char) if char.is_ascii_punctuation() => { - tokenizer.enter(TokenType::CharacterEscapeValue); + tokenizer.enter(Token::CharacterEscapeValue); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterEscapeValue); - tokenizer.exit(TokenType::CharacterEscape); + tokenizer.exit(Token::CharacterEscapeValue); + tokenizer.exit(Token::CharacterEscape); (State::Ok, None) } _ => (State::Nok, None), diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index 65e49ca..04e89d2 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -42,12 +42,12 @@ //! //! ## Tokens //! -//! * [`CharacterReference`][TokenType::CharacterReference] -//! * [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker] -//! * [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal] -//! * [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric] -//! * [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi] -//! * [`CharacterReferenceValue`][TokenType::CharacterReferenceValue] +//! * [`CharacterReference`][Token::CharacterReference] +//! * [`CharacterReferenceMarker`][Token::CharacterReferenceMarker] +//! * [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal] +//! * [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric] +//! * [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi] +//! * [`CharacterReferenceValue`][Token::CharacterReferenceValue] //! //! ## References //! @@ -65,7 +65,8 @@ use crate::constant::{ CHARACTER_REFERENCE_DECIMAL_SIZE_MAX, CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX, CHARACTER_REFERENCE_NAMED_SIZE_MAX, CHARACTER_REFERENCE_NAMES, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Kind of a character reference. #[derive(Debug, Clone, PartialEq)] @@ -119,10 +120,10 @@ struct Info { pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('&') => { - tokenizer.enter(TokenType::CharacterReference); - tokenizer.enter(TokenType::CharacterReferenceMarker); + tokenizer.enter(Token::CharacterReference); + tokenizer.enter(Token::CharacterReferenceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarker); + tokenizer.exit(Token::CharacterReferenceMarker); (State::Fn(Box::new(open)), None) } _ => (State::Nok, None), @@ -143,12 +144,12 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { kind: Kind::Named, }; if let Code::Char('#') = code { - tokenizer.enter(TokenType::CharacterReferenceMarkerNumeric); + tokenizer.enter(Token::CharacterReferenceMarkerNumeric); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerNumeric); + tokenizer.exit(Token::CharacterReferenceMarkerNumeric); (State::Fn(Box::new(|t, c| numeric(t, c, info))), None) } else { - tokenizer.enter(TokenType::CharacterReferenceValue); + tokenizer.enter(Token::CharacterReferenceValue); value(tokenizer, code, info) } } @@ -162,14 +163,14 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// ``` fn numeric(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { if let Code::Char('x' | 'X') = code { - tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal); - tokenizer.enter(TokenType::CharacterReferenceValue); + tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(Token::CharacterReferenceValue); info.kind = Kind::Hexadecimal; (State::Fn(Box::new(|t, c| value(t, c, info))), None) } else { - tokenizer.enter(TokenType::CharacterReferenceValue); + tokenizer.enter(Token::CharacterReferenceValue); info.kind = Kind::Decimal; value(tokenizer, code, info) } @@ -194,11 +195,11 @@ fn value(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { (State::Nok, None) } else { - tokenizer.exit(TokenType::CharacterReferenceValue); - tokenizer.enter(TokenType::CharacterReferenceMarkerSemi); + tokenizer.exit(Token::CharacterReferenceValue); + tokenizer.enter(Token::CharacterReferenceMarkerSemi); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerSemi); - tokenizer.exit(TokenType::CharacterReference); + tokenizer.exit(Token::CharacterReferenceMarkerSemi); + tokenizer.exit(Token::CharacterReference); (State::Ok, None) } } diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index 05266ba..617979f 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -77,14 +77,14 @@ //! //! ## Tokens //! -//! * [`CodeFenced`][TokenType::CodeFenced] -//! * [`CodeFencedFence`][TokenType::CodeFencedFence] -//! * [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo] -//! * [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta] -//! * [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence] -//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`CodeFenced`][Token::CodeFenced] +//! * [`CodeFencedFence`][Token::CodeFencedFence] +//! * [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo] +//! * [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta] +//! * [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence] +//! * [`CodeFlowChunk`][Token::CodeFlowChunk] +//! * [`LineEnding`][Token::LineEnding] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -103,7 +103,8 @@ use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE}; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; use crate::util::span::from_exit_event; /// Kind of fences. @@ -184,8 +185,8 @@ struct Info { /// ~~~ /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::CodeFenced); - tokenizer.enter(TokenType::CodeFencedFence); + tokenizer.enter(Token::CodeFenced); + tokenizer.enter(Token::CodeFencedFence); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before_sequence_open)(tokenizer, code) } @@ -202,7 +203,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult let mut prefix = 0; if let Some(event) = tail { - if event.token_type == TokenType::SpaceOrTab { + if event.token_type == Token::SpaceOrTab { let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1); prefix = span.end_index - span.start_index; } @@ -210,7 +211,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult match code { Code::Char('`' | '~') => { - tokenizer.enter(TokenType::CodeFencedFenceSequence); + tokenizer.enter(Token::CodeFencedFenceSequence); sequence_open( tokenizer, code, @@ -245,7 +246,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State ) } _ if info.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => { - tokenizer.exit(TokenType::CodeFencedFenceSequence); + tokenizer.exit(Token::CodeFencedFenceSequence); tokenizer.attempt_opt(space_or_tab(), |t, c| info_before(t, c, info))(tokenizer, code) } _ => (State::Nok, None), @@ -262,12 +263,12 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State fn info_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } _ => { - tokenizer.enter(TokenType::CodeFencedFenceInfo); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter(Token::CodeFencedFenceInfo); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); info_inside(tokenizer, code, info, vec![]) } } @@ -288,14 +289,14 @@ fn info_inside( ) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::CodeFencedFenceInfo); - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::CodeFencedFenceInfo); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } Code::VirtualSpace | Code::Char('\t' | ' ') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::CodeFencedFenceInfo); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::CodeFencedFenceInfo); tokenizer.attempt_opt(space_or_tab(), |t, c| meta_before(t, c, info))(tokenizer, code) } Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None), @@ -320,12 +321,12 @@ fn info_inside( fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } _ => { - tokenizer.enter(TokenType::CodeFencedFenceMeta); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter(Token::CodeFencedFenceMeta); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); meta(tokenizer, code, info) } } @@ -341,9 +342,9 @@ fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu fn meta(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::CodeFencedFenceMeta); - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::CodeFencedFenceMeta); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None), @@ -390,9 +391,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| close_start(t, c, info))), None) } _ => unreachable!("expected eol"), @@ -411,7 +412,7 @@ fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu /// | ~~~ /// ``` fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { - tokenizer.enter(TokenType::CodeFencedFence); + tokenizer.enter(Token::CodeFencedFence); tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), |t, c| { close_before(t, c, info) })(tokenizer, code) @@ -431,7 +432,7 @@ fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu fn close_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::Char(char) if char == info.kind.as_char() => { - tokenizer.enter(TokenType::CodeFencedFenceSequence); + tokenizer.enter(Token::CodeFencedFenceSequence); close_sequence(tokenizer, code, info, 0) } _ => (State::Nok, None), @@ -455,7 +456,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize ) } _ if size >= CODE_FENCED_SEQUENCE_SIZE_MIN && size >= info.size => { - tokenizer.exit(TokenType::CodeFencedFenceSequence); + tokenizer.exit(Token::CodeFencedFenceSequence); tokenizer.attempt_opt(space_or_tab(), close_sequence_after)(tokenizer, code) } _ => (State::Nok, None), @@ -472,7 +473,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::CodeFencedFence); (State::Ok, Some(vec![code])) } _ => (State::Nok, None), @@ -487,9 +488,9 @@ fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult /// ~~~ /// ``` fn content_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| content_start(t, c, info))), None) } /// Before code content, definitely not before a closing fence. @@ -518,7 +519,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe at_break(tokenizer, code, info) } _ => { - tokenizer.enter(TokenType::CodeFlowChunk); + tokenizer.enter(Token::CodeFlowChunk); content_continue(tokenizer, code, info) } } @@ -536,7 +537,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFlowChunk); + tokenizer.exit(Token::CodeFlowChunk); at_break(tokenizer, code, info) } _ => { @@ -557,7 +558,7 @@ fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateF /// ~~~| /// ``` fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.exit(TokenType::CodeFenced); + tokenizer.exit(Token::CodeFenced); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index 9bdfd71..8966249 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -28,10 +28,10 @@ //! //! ## Tokens //! -//! * [`CodeIndented`][TokenType::CodeIndented] -//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`CodeIndented`][Token::CodeIndented] +//! * [`CodeFlowChunk`][Token::CodeFlowChunk] +//! * [`LineEnding`][Token::LineEnding] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -47,7 +47,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::TAB_SIZE; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of code (indented). /// @@ -63,7 +64,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { if tokenizer.interrupt { (State::Nok, None) } else { - tokenizer.enter(TokenType::CodeIndented); + tokenizer.enter(Token::CodeIndented); tokenizer.go(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), at_break)(tokenizer, code) } } @@ -82,7 +83,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { Box::new(if ok { at_break } else { after }) })(tokenizer, code), _ => { - tokenizer.enter(TokenType::CodeFlowChunk); + tokenizer.enter(Token::CodeFlowChunk); content(tokenizer, code) } } @@ -98,7 +99,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFlowChunk); + tokenizer.exit(Token::CodeFlowChunk); at_break(tokenizer, code) } _ => { @@ -114,7 +115,7 @@ fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// ab| /// ``` fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.exit(TokenType::CodeIndented); + tokenizer.exit(Token::CodeIndented); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) @@ -130,9 +131,9 @@ fn further_start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { // To do: `nok` if lazy line. match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(further_start)), None) } _ => tokenizer.attempt(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), |ok| { diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index a6dc7eb..6df61b8 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -67,10 +67,10 @@ //! //! ## Tokens //! -//! * [`CodeText`][TokenType::CodeText] -//! * [`CodeTextData`][TokenType::CodeTextData] -//! * [`CodeTextLineEnding`][TokenType::CodeTextLineEnding] -//! * [`CodeTextSequence`][TokenType::CodeTextSequence] +//! * [`CodeText`][Token::CodeText] +//! * [`CodeTextData`][Token::CodeTextData] +//! * [`CodeTextLineEnding`][Token::CodeTextLineEnding] +//! * [`CodeTextSequence`][Token::CodeTextSequence] //! //! ## References //! @@ -83,7 +83,8 @@ //! [code_fenced]: crate::construct::code_fenced //! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of code (text). /// @@ -100,11 +101,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('`') if tokenizer.previous != Code::Char('`') - || (len > 0 - && tokenizer.events[len - 1].token_type == TokenType::CharacterEscape) => + || (len > 0 && tokenizer.events[len - 1].token_type == Token::CharacterEscape) => { - tokenizer.enter(TokenType::CodeText); - tokenizer.enter(TokenType::CodeTextSequence); + tokenizer.enter(Token::CodeText); + tokenizer.enter(Token::CodeTextSequence); sequence_open(tokenizer, code, 0) } _ => (State::Nok, None), @@ -124,7 +124,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnR None, ) } else { - tokenizer.exit(TokenType::CodeTextSequence); + tokenizer.exit(Token::CodeTextSequence); between(tokenizer, code, size) } } @@ -139,20 +139,20 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe match code { Code::None => (State::Nok, None), Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::CodeTextLineEnding); + tokenizer.enter(Token::CodeTextLineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::CodeTextLineEnding); + tokenizer.exit(Token::CodeTextLineEnding); ( State::Fn(Box::new(move |t, c| between(t, c, size_open))), None, ) } Code::Char('`') => { - tokenizer.enter(TokenType::CodeTextSequence); + tokenizer.enter(Token::CodeTextSequence); sequence_close(tokenizer, code, size_open, 0) } _ => { - tokenizer.enter(TokenType::CodeTextData); + tokenizer.enter(Token::CodeTextData); data(tokenizer, code, size_open) } } @@ -166,7 +166,7 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe fn data(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '`') => { - tokenizer.exit(TokenType::CodeTextData); + tokenizer.exit(Token::CodeTextData); between(tokenizer, code, size_open) } _ => { @@ -198,16 +198,16 @@ fn sequence_close( ) } _ if size_open == size => { - tokenizer.exit(TokenType::CodeTextSequence); - tokenizer.exit(TokenType::CodeText); + tokenizer.exit(Token::CodeTextSequence); + tokenizer.exit(Token::CodeText); (State::Ok, Some(vec![code])) } _ => { let index = tokenizer.events.len(); - tokenizer.exit(TokenType::CodeTextSequence); + tokenizer.exit(Token::CodeTextSequence); // Change the token type. - tokenizer.events[index - 1].token_type = TokenType::CodeTextData; - tokenizer.events[index].token_type = TokenType::CodeTextData; + tokenizer.events[index - 1].token_type = Token::CodeTextData; + tokenizer.events[index].token_type = Token::CodeTextData; between(tokenizer, code, size_open) } } diff --git a/src/construct/definition.rs b/src/construct/definition.rs index db4a009..4d14653 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -59,21 +59,21 @@ //! //! ## Tokens //! -//! * [`Definition`][TokenType::Definition] -//! * [`DefinitionDestination`][TokenType::DefinitionDestination] -//! * [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral] -//! * [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker] -//! * [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] -//! * [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] -//! * [`DefinitionLabel`][TokenType::DefinitionLabel] -//! * [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker] -//! * [`DefinitionLabelString`][TokenType::DefinitionLabelString] -//! * [`DefinitionMarker`][TokenType::DefinitionMarker] -//! * [`DefinitionTitle`][TokenType::DefinitionTitle] -//! * [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker] -//! * [`DefinitionTitleString`][TokenType::DefinitionTitleString] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`Definition`][Token::Definition] +//! * [`DefinitionDestination`][Token::DefinitionDestination] +//! * [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral] +//! * [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker] +//! * [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw] +//! * [`DefinitionDestinationString`][Token::DefinitionDestinationString] +//! * [`DefinitionLabel`][Token::DefinitionLabel] +//! * [`DefinitionLabelMarker`][Token::DefinitionLabelMarker] +//! * [`DefinitionLabelString`][Token::DefinitionLabelString] +//! * [`DefinitionMarker`][Token::DefinitionMarker] +//! * [`DefinitionTitle`][Token::DefinitionTitle] +//! * [`DefinitionTitleMarker`][Token::DefinitionTitleMarker] +//! * [`DefinitionTitleString`][Token::DefinitionTitleString] +//! * [`LineEnding`][Token::LineEnding] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -99,7 +99,8 @@ use crate::construct::{ partial_space_or_tab::{space_or_tab, space_or_tab_eol}, partial_title::{start as title, Options as TitleOptions}, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// At the start of a definition. /// @@ -109,14 +110,14 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { let index = tokenizer.events.len(); let definition_before = index > 3 - && tokenizer.events[index - 1].token_type == TokenType::LineEnding - && tokenizer.events[index - 3].token_type == TokenType::Definition; + && tokenizer.events[index - 1].token_type == Token::LineEnding + && tokenizer.events[index - 3].token_type == Token::Definition; // Do not interrupt paragraphs (but do follow definitions). if tokenizer.interrupt && !definition_before { (State::Nok, None) } else { - tokenizer.enter(TokenType::Definition); + tokenizer.enter(Token::Definition); // Note: arbitrary whitespace allowed even if code (indented) is on. tokenizer.attempt_opt(space_or_tab(), before)(tokenizer, code) } @@ -135,9 +136,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, LabelOptions { - label: TokenType::DefinitionLabel, - marker: TokenType::DefinitionLabelMarker, - string: TokenType::DefinitionLabelString, + label: Token::DefinitionLabel, + marker: Token::DefinitionLabelMarker, + string: Token::DefinitionLabelString, }, ) }, @@ -155,9 +156,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn label_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(':') => { - tokenizer.enter(TokenType::DefinitionMarker); + tokenizer.enter(Token::DefinitionMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::DefinitionMarker); + tokenizer.exit(Token::DefinitionMarker); ( State::Fn(Box::new( tokenizer.attempt_opt(space_or_tab_eol(), destination_before), @@ -185,11 +186,11 @@ fn destination_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { c, DestinationOptions { limit: usize::MAX, - destination: TokenType::DefinitionDestination, - literal: TokenType::DefinitionDestinationLiteral, - marker: TokenType::DefinitionDestinationLiteralMarker, - raw: TokenType::DefinitionDestinationRaw, - string: TokenType::DefinitionDestinationString, + destination: Token::DefinitionDestination, + literal: Token::DefinitionDestinationLiteral, + marker: Token::DefinitionDestinationLiteralMarker, + raw: Token::DefinitionDestinationRaw, + string: Token::DefinitionDestinationString, }, ) }, @@ -228,7 +229,7 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn after_whitespace(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Definition); + tokenizer.exit(Token::Definition); // You’d be interrupting. tokenizer.interrupt = true; (State::Ok, Some(vec![code])) @@ -262,9 +263,9 @@ fn title_before_marker(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, TitleOptions { - title: TokenType::DefinitionTitle, - marker: TokenType::DefinitionTitleMarker, - string: TokenType::DefinitionTitleString, + title: Token::DefinitionTitle, + marker: Token::DefinitionTitleMarker, + string: Token::DefinitionTitleString, }, ) }, diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index 212d276..4fb87bf 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -26,8 +26,8 @@ //! //! ## Tokens //! -//! * [`HardBreakEscape`][TokenType::HardBreakEscape] -//! * [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker] +//! * [`HardBreakEscape`][Token::HardBreakEscape] +//! * [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker] //! //! ## References //! @@ -40,7 +40,8 @@ //! [hard_break_trailing]: crate::construct::hard_break_trailing //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of a hard break (escape). /// @@ -50,10 +51,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('\\') => { - tokenizer.enter(TokenType::HardBreakEscape); - tokenizer.enter(TokenType::HardBreakEscapeMarker); + tokenizer.enter(Token::HardBreakEscape); + tokenizer.enter(Token::HardBreakEscapeMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::HardBreakEscapeMarker); + tokenizer.exit(Token::HardBreakEscapeMarker); (State::Fn(Box::new(inside)), None) } _ => (State::Nok, None), @@ -68,7 +69,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HardBreakEscape); + tokenizer.exit(Token::HardBreakEscape); (State::Ok, Some(vec![code])) } _ => (State::Nok, None), diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs index 35a7cab..6626675 100644 --- a/src/construct/hard_break_trailing.rs +++ b/src/construct/hard_break_trailing.rs @@ -26,8 +26,8 @@ //! //! ## Tokens //! -//! * [`HardBreakTrailing`][TokenType::HardBreakTrailing] -//! * [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace] +//! * [`HardBreakTrailing`][Token::HardBreakTrailing] +//! * [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace] //! //! ## References //! @@ -41,7 +41,8 @@ //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element use crate::constant::HARD_BREAK_PREFIX_SIZE_MIN; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of a hard break (trailing). /// @@ -52,8 +53,8 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(' ') => { - tokenizer.enter(TokenType::HardBreakTrailing); - tokenizer.enter(TokenType::HardBreakTrailingSpace); + tokenizer.enter(Token::HardBreakTrailing); + tokenizer.enter(Token::HardBreakTrailingSpace); tokenizer.consume(code); (State::Fn(Box::new(|t, c| inside(t, c, 1))), None) } @@ -79,8 +80,8 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnResult { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if size >= HARD_BREAK_PREFIX_SIZE_MIN => { - tokenizer.exit(TokenType::HardBreakTrailingSpace); - tokenizer.exit(TokenType::HardBreakTrailing); + tokenizer.exit(Token::HardBreakTrailingSpace); + tokenizer.exit(Token::HardBreakTrailing); (State::Ok, Some(vec![code])) } _ => (State::Nok, None), diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index 9fa2ace..4a4992a 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -37,10 +37,10 @@ //! //! ## Tokens //! -//! * [`HeadingAtx`][TokenType::HeadingAtx] -//! * [`HeadingAtxSequence`][TokenType::HeadingAtxSequence] -//! * [`HeadingAtxText`][TokenType::HeadingAtxText] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`HeadingAtx`][Token::HeadingAtx] +//! * [`HeadingAtxSequence`][Token::HeadingAtxSequence] +//! * [`HeadingAtxText`][Token::HeadingAtxText] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -56,9 +56,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE}; -use crate::tokenizer::{ - Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer, -}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::edit_map::EditMap; /// Start of a heading (atx). @@ -67,7 +66,7 @@ use crate::util::edit_map::EditMap; /// |## alpha /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::HeadingAtx); + tokenizer.enter(Token::HeadingAtx); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code) } @@ -79,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// ``` fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { if Code::Char('#') == code { - tokenizer.enter(TokenType::HeadingAtxSequence); + tokenizer.enter(Token::HeadingAtxSequence); sequence_open(tokenizer, code, 0) } else { (State::Nok, None) @@ -94,7 +93,7 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if rank > 0 => { - tokenizer.exit(TokenType::HeadingAtxSequence); + tokenizer.exit(Token::HeadingAtxSequence); at_break(tokenizer, code) } Code::Char('#') if rank < HEADING_ATX_OPENING_FENCE_SIZE_MAX => { @@ -107,7 +106,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR ) } _ if rank > 0 => { - tokenizer.exit(TokenType::HeadingAtxSequence); + tokenizer.exit(Token::HeadingAtxSequence); tokenizer.go(space_or_tab(), at_break)(tokenizer, code) } _ => (State::Nok, None), @@ -126,7 +125,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HeadingAtx); + tokenizer.exit(Token::HeadingAtx); tokenizer.register_resolver("heading_atx".to_string(), Box::new(resolve)); // Feel free to interrupt. tokenizer.interrupt = false; @@ -136,11 +135,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { tokenizer.go(space_or_tab(), at_break)(tokenizer, code) } Code::Char('#') => { - tokenizer.enter(TokenType::HeadingAtxSequence); + tokenizer.enter(Token::HeadingAtxSequence); further_sequence(tokenizer, code) } Code::Char(_) => { - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); data(tokenizer, code) } } @@ -157,7 +156,7 @@ fn further_sequence(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { tokenizer.consume(code); (State::Fn(Box::new(further_sequence)), None) } else { - tokenizer.exit(TokenType::HeadingAtxSequence); + tokenizer.exit(Token::HeadingAtxSequence); at_break(tokenizer, code) } } @@ -171,7 +170,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { // Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text. Code::None | Code::CarriageReturnLineFeed | Code::Char('\t' | '\n' | '\r' | ' ') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code) } _ => { @@ -192,7 +191,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.token_type == TokenType::HeadingAtx { + if event.token_type == Token::HeadingAtx { if event.event_type == EventType::Enter { heading_start = Some(index); } else if let Some(start) = data_start { @@ -204,7 +203,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Enter, - token_type: TokenType::HeadingAtxText, + token_type: Token::HeadingAtxText, point: tokenizer.events[start].point.clone(), index: tokenizer.events[start].index, previous: None, @@ -221,7 +220,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::HeadingAtxText, + token_type: Token::HeadingAtxText, point: tokenizer.events[end].point.clone(), index: tokenizer.events[end].index, previous: None, @@ -234,7 +233,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { data_start = None; data_end = None; } - } else if heading_start.is_some() && event.token_type == TokenType::Data { + } else if heading_start.is_some() && event.token_type == Token::Data { if event.event_type == EventType::Enter { if data_start.is_none() { data_start = Some(index); diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index 440baa8..633f7de 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -40,9 +40,9 @@ //! //! ## Tokens //! -//! * [`HeadingSetext`][TokenType::HeadingSetext] -//! * [`HeadingSetextText`][TokenType::HeadingSetextText] -//! * [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline] +//! * [`HeadingSetext`][Token::HeadingSetext] +//! * [`HeadingSetextText`][Token::HeadingSetextText] +//! * [`HeadingSetextUnderline`][Token::HeadingSetextUnderline] //! //! ## References //! @@ -59,7 +59,8 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::{edit_map::EditMap, skip::opt_back as skip_opt_back}; /// Kind of underline. @@ -120,14 +121,14 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { skip_opt_back( &tokenizer.events, index - 1, - &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix], + &[Token::SpaceOrTab, Token::BlockQuotePrefix], ) } else { 0 }; - let previous = skip_opt_back(&tokenizer.events, previous, &[TokenType::LineEnding]); + let previous = skip_opt_back(&tokenizer.events, previous, &[Token::LineEnding]); let paragraph_before = - previous > 1 && tokenizer.events[previous].token_type == TokenType::Paragraph; + previous > 1 && tokenizer.events[previous].token_type == Token::Paragraph; println!( "setext-start: {:?} {:?} {:?}", @@ -152,7 +153,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(char) if char == '-' || char == '=' => { - tokenizer.enter(TokenType::HeadingSetextUnderline); + tokenizer.enter(Token::HeadingSetextUnderline); inside(tokenizer, code, Kind::from_char(char)) } _ => (State::Nok, None), @@ -184,7 +185,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult { fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HeadingSetextUnderline); + tokenizer.exit(Token::HeadingSetextUnderline); // Feel free to interrupt. tokenizer.interrupt = false; tokenizer.register_resolver("heading_setext".to_string(), Box::new(resolve)); @@ -206,27 +207,27 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { // Find paragraphs. if event.event_type == EventType::Enter { - if event.token_type == TokenType::Paragraph { + if event.token_type == Token::Paragraph { paragraph_enter = Some(index); } - } else if event.token_type == TokenType::Paragraph { + } else if event.token_type == Token::Paragraph { paragraph_exit = Some(index); } // We know this is preceded by a paragraph. // Otherwise we don’t parse. - else if event.token_type == TokenType::HeadingSetextUnderline { + else if event.token_type == Token::HeadingSetextUnderline { let enter = paragraph_enter.take().unwrap(); let exit = paragraph_exit.take().unwrap(); // Change types of Enter:Paragraph, Exit:Paragraph. - tokenizer.events[enter].token_type = TokenType::HeadingSetextText; - tokenizer.events[exit].token_type = TokenType::HeadingSetextText; + tokenizer.events[enter].token_type = Token::HeadingSetextText; + tokenizer.events[exit].token_type = Token::HeadingSetextText; // Add Enter:HeadingSetext, Exit:HeadingSetext. let mut heading_enter = tokenizer.events[enter].clone(); - heading_enter.token_type = TokenType::HeadingSetext; + heading_enter.token_type = Token::HeadingSetext; let mut heading_exit = tokenizer.events[index].clone(); - heading_exit.token_type = TokenType::HeadingSetext; + heading_exit.token_type = Token::HeadingSetext; edit_map.add(enter, 0, vec![heading_enter]); edit_map.add(index + 1, 0, vec![heading_exit]); diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index be7a3a9..fde0a34 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -82,9 +82,9 @@ //! //! ## Tokens //! -//! * [`HtmlFlow`][TokenType::HtmlFlow] -//! * [`HtmlFlowData`][TokenType::HtmlFlowData] -//! * [`LineEnding`][TokenType::LineEnding] +//! * [`HtmlFlow`][Token::HtmlFlow] +//! * [`HtmlFlowData`][Token::HtmlFlowData] +//! * [`LineEnding`][Token::LineEnding] //! //! ## References //! @@ -102,7 +102,8 @@ use crate::constant::{HTML_BLOCK_NAMES, HTML_RAW_NAMES, HTML_RAW_SIZE_MAX, TAB_S use crate::construct::{ blank_line::start as blank_line, partial_space_or_tab::space_or_tab_min_max, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; use crate::util::codes::{parse, serialize}; // To do: mark as concrete (block quotes or lists can’t “pierce” into HTML). @@ -203,8 +204,8 @@ struct Info { /// ``` /// pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::HtmlFlow); - tokenizer.enter(TokenType::HtmlFlowData); + tokenizer.enter(Token::HtmlFlow); + tokenizer.enter(Token::HtmlFlowData); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code) } @@ -776,7 +777,7 @@ fn continuation(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRes /// <x>| /// ``` fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { - tokenizer.exit(TokenType::HtmlFlowData); + tokenizer.exit(Token::HtmlFlowData); html_continue_start(tokenizer, code, info) } @@ -789,23 +790,23 @@ fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info fn html_continue_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None => { - tokenizer.exit(TokenType::HtmlFlow); + tokenizer.exit(Token::HtmlFlow); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) } // To do: do not allow lazy lines. Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); ( State::Fn(Box::new(|t, c| html_continue_start(t, c, info))), None, ) } _ => { - tokenizer.enter(TokenType::HtmlFlowData); + tokenizer.enter(Token::HtmlFlowData); continuation(tokenizer, code, info) } } @@ -955,8 +956,8 @@ fn continuation_declaration_inside( fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HtmlFlowData); - tokenizer.exit(TokenType::HtmlFlow); + tokenizer.exit(Token::HtmlFlowData); + tokenizer.exit(Token::HtmlFlow); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) @@ -978,8 +979,8 @@ fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> Stat /// /// ``` fn blank_line_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(blank_line)), None) } diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index 0926f48..cdd7c69 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -42,8 +42,8 @@ //! //! ## Tokens //! -//! * [`HtmlText`][TokenType::HtmlText] -//! * [`HtmlTextData`][TokenType::HtmlTextData] +//! * [`HtmlText`][Token::HtmlText] +//! * [`HtmlTextData`][Token::HtmlTextData] //! //! ## References //! @@ -55,7 +55,8 @@ //! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing use crate::construct::partial_space_or_tab::space_or_tab; -use crate::tokenizer::{Code, State, StateFn, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFn, StateFnResult, Tokenizer}; use crate::util::codes::parse; /// Start of HTML (text) @@ -65,8 +66,8 @@ use crate::util::codes::parse; /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { if Code::Char('<') == code { - tokenizer.enter(TokenType::HtmlText); - tokenizer.enter(TokenType::HtmlTextData); + tokenizer.enter(Token::HtmlText); + tokenizer.enter(Token::HtmlTextData); tokenizer.consume(code); (State::Fn(Box::new(open)), None) } else { @@ -617,8 +618,8 @@ fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { tokenizer.consume(code); - tokenizer.exit(TokenType::HtmlTextData); - tokenizer.exit(TokenType::HtmlText); + tokenizer.exit(Token::HtmlTextData); + tokenizer.exit(Token::HtmlText); (State::Ok, None) } _ => (State::Nok, None), @@ -641,10 +642,10 @@ fn at_line_ending( ) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HtmlTextData); - tokenizer.enter(TokenType::LineEnding); + tokenizer.exit(Token::HtmlTextData); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); ( State::Fn(Box::new(|t, c| after_line_ending(t, c, return_state))), None, @@ -687,6 +688,6 @@ fn after_line_ending_prefix( code: Code, return_state: Box<StateFn>, ) -> StateFnResult { - tokenizer.enter(TokenType::HtmlTextData); + tokenizer.enter(Token::HtmlTextData); return_state(tokenizer, code) } diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs index 3a40cc2..29ac6f9 100644 --- a/src/construct/label_end.rs +++ b/src/construct/label_end.rs @@ -102,28 +102,28 @@ //! //! ## Tokens //! -//! * [`Data`][TokenType::Data] -//! * [`Image`][TokenType::Image] -//! * [`Label`][TokenType::Label] -//! * [`LabelEnd`][TokenType::LabelEnd] -//! * [`LabelMarker`][TokenType::LabelMarker] -//! * [`LabelText`][TokenType::LabelText] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`Link`][TokenType::Link] -//! * [`Reference`][TokenType::Reference] -//! * [`ReferenceMarker`][TokenType::ReferenceMarker] -//! * [`ReferenceString`][TokenType::ReferenceString] -//! * [`Resource`][TokenType::Resource] -//! * [`ResourceDestination`][TokenType::ResourceDestination] -//! * [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral] -//! * [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker] -//! * [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw] -//! * [`ResourceDestinationString`][TokenType::ResourceDestinationString] -//! * [`ResourceMarker`][TokenType::ResourceMarker] -//! * [`ResourceTitle`][TokenType::ResourceTitle] -//! * [`ResourceTitleMarker`][TokenType::ResourceTitleMarker] -//! * [`ResourceTitleString`][TokenType::ResourceTitleString] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`Data`][Token::Data] +//! * [`Image`][Token::Image] +//! * [`Label`][Token::Label] +//! * [`LabelEnd`][Token::LabelEnd] +//! * [`LabelMarker`][Token::LabelMarker] +//! * [`LabelText`][Token::LabelText] +//! * [`LineEnding`][Token::LineEnding] +//! * [`Link`][Token::Link] +//! * [`Reference`][Token::Reference] +//! * [`ReferenceMarker`][Token::ReferenceMarker] +//! * [`ReferenceString`][Token::ReferenceString] +//! * [`Resource`][Token::Resource] +//! * [`ResourceDestination`][Token::ResourceDestination] +//! * [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral] +//! * [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker] +//! * [`ResourceDestinationRaw`][Token::ResourceDestinationRaw] +//! * [`ResourceDestinationString`][Token::ResourceDestinationString] +//! * [`ResourceMarker`][Token::ResourceMarker] +//! * [`ResourceTitle`][Token::ResourceTitle] +//! * [`ResourceTitleMarker`][Token::ResourceTitleMarker] +//! * [`ResourceTitleString`][Token::ResourceTitleString] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -153,8 +153,9 @@ use crate::construct::{ partial_space_or_tab::space_or_tab_eol, partial_title::{start as title, Options as TitleOptions}, }; +use crate::token::Token; use crate::tokenizer::{ - Code, Event, EventType, LabelStart, Media, State, StateFnResult, TokenType, Tokenizer, + Code, Event, EventType, LabelStart, Media, State, StateFnResult, Tokenizer, }; use crate::util::{ edit_map::EditMap, @@ -198,7 +199,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { vec![ Event { event_type: EventType::Enter, - token_type: TokenType::Data, + token_type: Token::Data, point: events[data_enter_index].point.clone(), index: events[data_enter_index].index, previous: None, @@ -207,7 +208,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { }, Event { event_type: EventType::Exit, - token_type: TokenType::Data, + token_type: Token::Data, point: events[data_exit_index].point.clone(), index: events[data_exit_index].index, previous: None, @@ -229,7 +230,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { let group_enter_event = &events[group_enter_index]; // LabelLink:Exit or LabelImage:Exit. let text_enter_index = media.start.0 - + (if group_enter_event.token_type == TokenType::LabelLink { + + (if group_enter_event.token_type == Token::LabelLink { 4 } else { 6 @@ -248,10 +249,10 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { vec![ Event { event_type: EventType::Enter, - token_type: if group_enter_event.token_type == TokenType::LabelLink { - TokenType::Link + token_type: if group_enter_event.token_type == Token::LabelLink { + Token::Link } else { - TokenType::Image + Token::Image }, point: group_enter_event.point.clone(), index: group_enter_event.index, @@ -261,7 +262,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { }, Event { event_type: EventType::Enter, - token_type: TokenType::Label, + token_type: Token::Label, point: group_enter_event.point.clone(), index: group_enter_event.index, previous: None, @@ -279,7 +280,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Enter, - token_type: TokenType::LabelText, + token_type: Token::LabelText, point: events[text_enter_index].point.clone(), index: events[text_enter_index].index, previous: None, @@ -294,7 +295,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::LabelText, + token_type: Token::LabelText, point: events[text_exit_index].point.clone(), index: events[text_exit_index].index, previous: None, @@ -310,7 +311,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::Label, + token_type: Token::Label, point: events[label_exit_index].point.clone(), index: events[label_exit_index].index, previous: None, @@ -325,7 +326,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::Link, + token_type: Token::Link, point: events[group_end_index].point.clone(), index: events[group_end_index].index, previous: None, @@ -393,11 +394,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { }, }; - tokenizer.enter(TokenType::LabelEnd); - tokenizer.enter(TokenType::LabelMarker); + tokenizer.enter(Token::LabelEnd); + tokenizer.enter(Token::LabelMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelMarker); - tokenizer.exit(TokenType::LabelEnd); + tokenizer.exit(Token::LabelMarker); + tokenizer.exit(Token::LabelEnd); return (State::Fn(Box::new(move |t, c| after(t, c, info))), None); } @@ -495,13 +496,13 @@ fn ok(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { left.remove(0); tokenizer.label_start_list_loose.append(&mut left); - let is_link = tokenizer.events[info.media.start.0].token_type == TokenType::LabelLink; + let is_link = tokenizer.events[info.media.start.0].token_type == Token::LabelLink; if is_link { let mut index = 0; while index < tokenizer.label_start_stack.len() { let label_start = &mut tokenizer.label_start_stack[index]; - if tokenizer.events[label_start.start.0].token_type == TokenType::LabelLink { + if tokenizer.events[label_start.start.0].token_type == Token::LabelLink { label_start.inactive = true; } index += 1; @@ -543,10 +544,10 @@ fn nok(tokenizer: &mut Tokenizer, _code: Code, label_start_index: usize) -> Stat fn resource(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('(') => { - tokenizer.enter(TokenType::Resource); - tokenizer.enter(TokenType::ResourceMarker); + tokenizer.enter(Token::Resource); + tokenizer.enter(Token::ResourceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ResourceMarker); + tokenizer.exit(Token::ResourceMarker); (State::Fn(Box::new(resource_start)), None) } _ => unreachable!("expected `(`"), @@ -577,11 +578,11 @@ fn resource_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { c, DestinationOptions { limit: RESOURCE_DESTINATION_BALANCE_MAX, - destination: TokenType::ResourceDestination, - literal: TokenType::ResourceDestinationLiteral, - marker: TokenType::ResourceDestinationLiteralMarker, - raw: TokenType::ResourceDestinationRaw, - string: TokenType::ResourceDestinationString, + destination: Token::ResourceDestination, + literal: Token::ResourceDestinationLiteral, + marker: Token::ResourceDestinationLiteralMarker, + raw: Token::ResourceDestinationRaw, + string: Token::ResourceDestinationString, }, ) }, @@ -616,9 +617,9 @@ fn resource_between(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, TitleOptions { - title: TokenType::ResourceTitle, - marker: TokenType::ResourceTitleMarker, - string: TokenType::ResourceTitleString, + title: Token::ResourceTitle, + marker: Token::ResourceTitleMarker, + string: Token::ResourceTitleString, }, ) }, @@ -647,10 +648,10 @@ fn title_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn resource_end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(')') => { - tokenizer.enter(TokenType::ResourceMarker); + tokenizer.enter(Token::ResourceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ResourceMarker); - tokenizer.exit(TokenType::Resource); + tokenizer.exit(Token::ResourceMarker); + tokenizer.exit(Token::Resource); (State::Ok, None) } _ => (State::Nok, None), @@ -670,9 +671,9 @@ fn full_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, LabelOptions { - label: TokenType::Reference, - marker: TokenType::ReferenceMarker, - string: TokenType::ReferenceString, + label: Token::Reference, + marker: Token::ReferenceMarker, + string: Token::ReferenceString, }, ) }, @@ -696,7 +697,7 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult while index > 0 { index -= 1; let event = &events[index]; - if event.token_type == TokenType::ReferenceString { + if event.token_type == Token::ReferenceString { if event.event_type == EventType::Exit { end = Some(event.index); } else { @@ -735,10 +736,10 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('[') => { - tokenizer.enter(TokenType::Reference); - tokenizer.enter(TokenType::ReferenceMarker); + tokenizer.enter(Token::Reference); + tokenizer.enter(Token::ReferenceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ReferenceMarker); + tokenizer.exit(Token::ReferenceMarker); (State::Fn(Box::new(collapsed_reference_open)), None) } _ => (State::Nok, None), @@ -755,10 +756,10 @@ fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn collapsed_reference_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(']') => { - tokenizer.enter(TokenType::ReferenceMarker); + tokenizer.enter(Token::ReferenceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ReferenceMarker); - tokenizer.exit(TokenType::Reference); + tokenizer.exit(Token::ReferenceMarker); + tokenizer.exit(Token::Reference); (State::Ok, None) } _ => (State::Nok, None), diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs index a45205a..f9b8300 100644 --- a/src/construct/label_start_image.rs +++ b/src/construct/label_start_image.rs @@ -15,9 +15,9 @@ //! //! ## Tokens //! -//! * [`LabelImage`][TokenType::LabelImage] -//! * [`LabelImageMarker`][TokenType::LabelImageMarker] -//! * [`LabelMarker`][TokenType::LabelMarker] +//! * [`LabelImage`][Token::LabelImage] +//! * [`LabelImageMarker`][Token::LabelImageMarker] +//! * [`LabelMarker`][Token::LabelMarker] //! //! ## References //! @@ -29,7 +29,8 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use super::label_end::resolve_media; -use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer}; /// Start of label (image) start. /// @@ -39,10 +40,10 @@ use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokeni pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('!') => { - tokenizer.enter(TokenType::LabelImage); - tokenizer.enter(TokenType::LabelImageMarker); + tokenizer.enter(Token::LabelImage); + tokenizer.enter(Token::LabelImageMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelImageMarker); + tokenizer.exit(Token::LabelImageMarker); (State::Fn(Box::new(open)), None) } _ => (State::Nok, None), @@ -57,10 +58,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { pub fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('[') => { - tokenizer.enter(TokenType::LabelMarker); + tokenizer.enter(Token::LabelMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelMarker); - tokenizer.exit(TokenType::LabelImage); + tokenizer.exit(Token::LabelMarker); + tokenizer.exit(Token::LabelImage); let end = tokenizer.events.len() - 1; tokenizer.label_start_stack.push(LabelStart { start: (end - 5, end), diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs index 6c4d7ae..59729cc 100644 --- a/src/construct/label_start_link.rs +++ b/src/construct/label_start_link.rs @@ -15,8 +15,8 @@ //! //! ## Tokens //! -//! * [`LabelLink`][TokenType::LabelLink] -//! * [`LabelMarker`][TokenType::LabelMarker] +//! * [`LabelLink`][Token::LabelLink] +//! * [`LabelMarker`][Token::LabelMarker] //! //! ## References //! @@ -28,7 +28,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use super::label_end::resolve_media; -use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer}; /// Start of label (link) start. /// @@ -39,11 +40,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('[') => { let start = tokenizer.events.len(); - tokenizer.enter(TokenType::LabelLink); - tokenizer.enter(TokenType::LabelMarker); + tokenizer.enter(Token::LabelLink); + tokenizer.enter(Token::LabelMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelMarker); - tokenizer.exit(TokenType::LabelLink); + tokenizer.exit(Token::LabelMarker); + tokenizer.exit(Token::LabelLink); tokenizer.label_start_stack.push(LabelStart { start: (start, tokenizer.events.len() - 1), balanced: false, diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs index ace174f..967e009 100644 --- a/src/construct/paragraph.rs +++ b/src/construct/paragraph.rs @@ -19,7 +19,7 @@ //! //! ## Tokens //! -//! * [`Paragraph`][TokenType::Paragraph] +//! * [`Paragraph`][Token::Paragraph] //! //! ## References //! @@ -32,9 +32,8 @@ //! [code_text]: crate::construct::code_text //! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element -use crate::tokenizer::{ - Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer, -}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::{edit_map::EditMap, skip::opt as skip_opt}; /// Before a paragraph. @@ -48,8 +47,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { unreachable!("unexpected eol/eof") } _ => { - tokenizer.enter(TokenType::Paragraph); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text)); + tokenizer.enter(Token::Paragraph); + tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); inside(tokenizer, code) } } @@ -63,8 +62,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::Paragraph); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::Paragraph); tokenizer.register_resolver_before("paragraph".to_string(), Box::new(resolve)); // You’d be interrupting. tokenizer.interrupt = true; @@ -87,21 +86,21 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { while index < len { let event = &tokenizer.events[index]; - if event.event_type == EventType::Enter && event.token_type == TokenType::Paragraph { + if event.event_type == EventType::Enter && event.token_type == Token::Paragraph { // Exit:Paragraph let mut exit_index = index + 3; let mut enter_next_index = - skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]); + skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]); // Enter:Paragraph enter_next_index = skip_opt( &tokenizer.events, enter_next_index, - &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix], + &[Token::SpaceOrTab, Token::BlockQuotePrefix], ); // Find future `Paragraphs`. while enter_next_index < tokenizer.events.len() - && tokenizer.events[enter_next_index].token_type == TokenType::Paragraph + && tokenizer.events[enter_next_index].token_type == Token::Paragraph { // Remove Exit:Paragraph, Enter:LineEnding, Exit:LineEnding, Enter:Paragraph. edit_map.add(exit_index, 3, vec![]); @@ -126,11 +125,11 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { // Potential next start. exit_index = enter_next_index + 3; enter_next_index = - skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]); + skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]); enter_next_index = skip_opt( &tokenizer.events, enter_next_index, - &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix], + &[Token::SpaceOrTab, Token::BlockQuotePrefix], ); } diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs index 555ccaf..186665d 100644 --- a/src/construct/partial_data.rs +++ b/src/construct/partial_data.rs @@ -6,7 +6,8 @@ //! [string]: crate::content::string //! [text]: crate::content::text -use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::edit_map::EditMap; /// At the beginning of data. @@ -16,7 +17,7 @@ use crate::util::edit_map::EditMap; /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult { if stop.contains(&code) { - tokenizer.enter(TokenType::Data); + tokenizer.enter(Token::Data); tokenizer.consume(code); (State::Fn(Box::new(|t, c| data(t, c, stop))), None) } else { @@ -33,9 +34,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe match code { Code::None => (State::Ok, None), Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| at_break(t, c, stop))), None) } _ if stop.contains(&code) => { @@ -43,7 +44,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe (State::Ok, Some(vec![code])) } _ => { - tokenizer.enter(TokenType::Data); + tokenizer.enter(Token::Data); data(tokenizer, code, stop) } } @@ -62,7 +63,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult }; if done { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, stop) } else { tokenizer.consume(code); @@ -80,13 +81,13 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { while index < len { let event = &tokenizer.events[index]; - if event.event_type == EventType::Enter && event.token_type == TokenType::Data { + if event.event_type == EventType::Enter && event.token_type == Token::Data { let exit_index = index + 1; let mut exit_far_index = exit_index; // Find multiple `data` events. while exit_far_index + 1 < len - && tokenizer.events[exit_far_index + 1].token_type == TokenType::Data + && tokenizer.events[exit_far_index + 1].token_type == Token::Data { exit_far_index += 2; } diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs index 31c13ec..daa968a 100644 --- a/src/construct/partial_destination.rs +++ b/src/construct/partial_destination.rs @@ -71,7 +71,8 @@ //! [label_end]: crate::construct::label_end //! [sanitize_uri]: crate::util::sanitize_uri -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; /// Configuration. /// @@ -79,15 +80,15 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token #[derive(Debug)] pub struct Options { /// Token for the whole destination. - pub destination: TokenType, + pub destination: Token, /// Token for a literal (enclosed) destination. - pub literal: TokenType, + pub literal: Token, /// Token for a literal marker. - pub marker: TokenType, + pub marker: Token, /// Token for a raw destination. - pub raw: TokenType, + pub raw: Token, /// Token for a the string. - pub string: TokenType, + pub string: Token, /// Maximum unbalanced parens. pub limit: usize, } @@ -133,7 +134,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code, options: Options) -> StateFn tokenizer.enter(info.options.destination.clone()); tokenizer.enter(info.options.raw.clone()); tokenizer.enter(info.options.string.clone()); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); raw(tokenizer, code, info) } } @@ -154,7 +155,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn (State::Ok, None) } else { tokenizer.enter(info.options.string.clone()); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); enclosed(tokenizer, code, info) } } @@ -167,7 +168,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn fn enclosed(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); tokenizer.exit(info.options.string.clone()); enclosed_before(tokenizer, code, info) } @@ -221,7 +222,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { } Code::Char(')') => { if info.balance == 0 { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); tokenizer.exit(info.options.string.clone()); tokenizer.exit(info.options.raw.clone()); tokenizer.exit(info.options.destination); @@ -239,7 +240,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { if info.balance > 0 { (State::Nok, None) } else { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); tokenizer.exit(info.options.string.clone()); tokenizer.exit(info.options.raw.clone()); tokenizer.exit(info.options.destination); diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs index f201f60..f380c7d 100644 --- a/src/construct/partial_label.rs +++ b/src/construct/partial_label.rs @@ -61,7 +61,8 @@ use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; use crate::constant::LINK_REFERENCE_SIZE_MAX; use crate::subtokenize::link; -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; /// Configuration. /// @@ -69,11 +70,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token #[derive(Debug)] pub struct Options { /// Token for the whole label. - pub label: TokenType, + pub label: Token, /// Token for the markers. - pub marker: TokenType, + pub marker: Token, /// Token for the string (inside the markers). - pub string: TokenType, + pub string: Token, } /// State needed to parse labels. @@ -144,7 +145,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes }, )(tokenizer, code), _ => { - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); if info.connect { let index = tokenizer.events.len() - 1; @@ -166,11 +167,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes fn label(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '[' | ']') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } _ if info.size > LINK_REFERENCE_SIZE_MAX => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } Code::VirtualSpace | Code::Char('\t' | ' ') => { diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs index 5b1ec5e..78477de 100644 --- a/src/construct/partial_space_or_tab.rs +++ b/src/construct/partial_space_or_tab.rs @@ -5,7 +5,8 @@ //! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js) use crate::subtokenize::link; -use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, Tokenizer}; /// Options to parse `space_or_tab`. #[derive(Debug)] @@ -15,7 +16,7 @@ pub struct Options { /// Maximum allowed characters (inclusive). pub max: usize, /// Token type to use for whitespace events. - pub kind: TokenType, + pub kind: Token, /// Connect this whitespace to the previous. pub connect: bool, /// Embedded content type to use. @@ -67,7 +68,7 @@ pub fn space_or_tab() -> Box<StateFn> { /// ``` pub fn space_or_tab_min_max(min: usize, max: usize) -> Box<StateFn> { space_or_tab_with_options(Options { - kind: TokenType::SpaceOrTab, + kind: Token::SpaceOrTab, min, max, content_type: None, @@ -104,7 +105,7 @@ pub fn space_or_tab_eol_with_options(options: EolOptions) -> Box<StateFn> { tokenizer.attempt( space_or_tab_with_options(Options { - kind: TokenType::SpaceOrTab, + kind: Token::SpaceOrTab, min: 1, max: usize::MAX, content_type: info.options.content_type, @@ -196,7 +197,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResul fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter_with_content(TokenType::LineEnding, info.options.content_type); + tokenizer.enter_with_content(Token::LineEnding, info.options.content_type); if info.connect { let index = tokenizer.events.len() - 1; @@ -206,7 +207,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) } tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| after_eol(t, c, info))), None) } _ if info.ok => (State::Ok, Some(vec![code])), @@ -229,7 +230,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) fn after_eol(tokenizer: &mut Tokenizer, code: Code, info: EolInfo) -> StateFnResult { tokenizer.attempt_opt( space_or_tab_with_options(Options { - kind: TokenType::SpaceOrTab, + kind: Token::SpaceOrTab, min: 1, max: usize::MAX, content_type: info.options.content_type, diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs index 010f554..6303da8 100644 --- a/src/construct/partial_title.rs +++ b/src/construct/partial_title.rs @@ -32,7 +32,8 @@ use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; use crate::subtokenize::link; -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; /// Configuration. /// @@ -40,11 +41,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token #[derive(Debug)] pub struct Options { /// Token for the whole title. - pub title: TokenType, + pub title: Token, /// Token for the marker. - pub marker: TokenType, + pub marker: Token, /// Token for the string inside the quotes. - pub string: TokenType, + pub string: Token, } /// Type of title. @@ -204,7 +205,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes }, )(tokenizer, code), _ => { - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); if info.connect { let index = tokenizer.events.len() - 1; @@ -226,11 +227,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes fn title(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::Char(char) if char == info.kind.as_char() => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } Code::Char('\\') => { diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index 28aca34..4d92f8d 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -35,8 +35,8 @@ //! //! ## Tokens //! -//! * [`ThematicBreak`][TokenType::ThematicBreak] -//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence] +//! * [`ThematicBreak`][Token::ThematicBreak] +//! * [`ThematicBreakSequence`][Token::ThematicBreakSequence] //! //! ## References //! @@ -51,7 +51,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN}; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Type of thematic break. #[derive(Debug, PartialEq)] @@ -134,7 +135,7 @@ struct Info { /// |*** /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::ThematicBreak); + tokenizer.enter(Token::ThematicBreak); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code) } @@ -170,13 +171,13 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if info.size >= THEMATIC_BREAK_MARKER_COUNT_MIN => { - tokenizer.exit(TokenType::ThematicBreak); + tokenizer.exit(Token::ThematicBreak); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) } Code::Char(char) if char == info.kind.as_char() => { - tokenizer.enter(TokenType::ThematicBreakSequence); + tokenizer.enter(Token::ThematicBreakSequence); sequence(tokenizer, code, info) } _ => (State::Nok, None), @@ -198,7 +199,7 @@ fn sequence(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes (State::Fn(Box::new(|t, c| sequence(t, c, info))), None) } _ => { - tokenizer.exit(TokenType::ThematicBreakSequence); + tokenizer.exit(Token::ThematicBreakSequence); tokenizer.attempt_opt(space_or_tab(), |t, c| at_break(t, c, info))(tokenizer, code) } } |