From 3048b7aca0690691d25cb8409d543b2377e065e1 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Thu, 11 Aug 2022 15:54:13 +0200 Subject: Refactor to move `space_or_tab_eol` to own file --- src/construct/attention.rs | 14 +-- src/construct/autolink.rs | 8 +- src/construct/blank_line.rs | 2 +- src/construct/block_quote.rs | 8 +- src/construct/character_escape.rs | 6 +- src/construct/character_reference.rs | 12 +-- src/construct/code_fenced.rs | 16 +-- src/construct/code_indented.rs | 8 +- src/construct/code_text.rs | 8 +- src/construct/definition.rs | 33 +++--- src/construct/hard_break_escape.rs | 2 +- src/construct/heading_atx.rs | 8 +- src/construct/heading_setext.rs | 6 +- src/construct/html_flow.rs | 6 +- src/construct/html_text.rs | 4 +- src/construct/label_end.rs | 46 ++++---- src/construct/label_start_image.rs | 6 +- src/construct/label_start_link.rs | 4 +- src/construct/list.rs | 12 +-- src/construct/mod.rs | 2 + src/construct/paragraph.rs | 2 +- src/construct/partial_bom.rs | 2 +- src/construct/partial_label.rs | 4 +- src/construct/partial_space_or_tab.rs | 161 --------------------------- src/construct/partial_space_or_tab_eol.rs | 174 ++++++++++++++++++++++++++++++ src/construct/partial_title.rs | 4 +- src/construct/partial_whitespace.rs | 6 +- src/construct/thematic_break.rs | 4 +- 28 files changed, 292 insertions(+), 276 deletions(-) create mode 100644 src/construct/partial_space_or_tab_eol.rs (limited to 'src/construct') diff --git a/src/construct/attention.rs b/src/construct/attention.rs index 6f91370..e974fae 100644 --- a/src/construct/attention.rs +++ b/src/construct/attention.rs @@ -32,14 +32,14 @@ //! //! ## Tokens //! -//! * [`Emphasis`][Token::Emphasis] -//! * [`EmphasisSequence`][Token::EmphasisSequence] -//! * [`EmphasisText`][Token::EmphasisText] -//! * [`Strong`][Token::Strong] -//! * [`StrongSequence`][Token::StrongSequence] -//! * [`StrongText`][Token::StrongText] +//! * [`Emphasis`][Name::Emphasis] +//! * [`EmphasisSequence`][Name::EmphasisSequence] +//! * [`EmphasisText`][Name::EmphasisText] +//! * [`Strong`][Name::Strong] +//! * [`StrongSequence`][Name::StrongSequence] +//! * [`StrongText`][Name::StrongText] //! -//! > 👉 **Note**: while parsing, [`AttentionSequence`][Token::AttentionSequence] +//! > 👉 **Note**: while parsing, [`AttentionSequence`][Name::AttentionSequence] //! > is used, which is later compiled away. //! //! ## References diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index c16b7c0..c0d9ae3 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -84,10 +84,10 @@ //! //! ## Tokens //! -//! * [`Autolink`][Token::Autolink] -//! * [`AutolinkEmail`][Token::AutolinkEmail] -//! * [`AutolinkMarker`][Token::AutolinkMarker] -//! * [`AutolinkProtocol`][Token::AutolinkProtocol] +//! * [`Autolink`][Name::Autolink] +//! * [`AutolinkEmail`][Name::AutolinkEmail] +//! * [`AutolinkMarker`][Name::AutolinkMarker] +//! * [`AutolinkProtocol`][Name::AutolinkProtocol] //! //! ## References //! diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index 2ea986d..2adc7a4 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -20,7 +20,7 @@ //! //! ## Tokens //! -//! * [`SpaceOrTab`][crate::token::Token::SpaceOrTab] +//! * [`SpaceOrTab`][crate::event::Name::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs index a32375e..f2b0179 100644 --- a/src/construct/block_quote.rs +++ b/src/construct/block_quote.rs @@ -19,10 +19,10 @@ //! //! ## Tokens //! -//! * [`BlockQuote`][Token::BlockQuote] -//! * [`BlockQuoteMarker`][Token::BlockQuoteMarker] -//! * [`BlockQuotePrefix`][Token::BlockQuotePrefix] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`BlockQuote`][Name::BlockQuote] +//! * [`BlockQuoteMarker`][Name::BlockQuoteMarker] +//! * [`BlockQuotePrefix`][Name::BlockQuotePrefix] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index 0cd7126..e0f36c7 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -19,9 +19,9 @@ //! //! ## Tokens //! -//! * [`CharacterEscape`][Token::CharacterEscape] -//! * [`CharacterEscapeMarker`][Token::CharacterEscapeMarker] -//! * [`CharacterEscapeValue`][Token::CharacterEscapeValue] +//! * [`CharacterEscape`][Name::CharacterEscape] +//! * [`CharacterEscapeMarker`][Name::CharacterEscapeMarker] +//! * [`CharacterEscapeValue`][Name::CharacterEscapeValue] //! //! ## References //! diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index 0158acf..476ea14 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -42,12 +42,12 @@ //! //! ## Tokens //! -//! * [`CharacterReference`][Token::CharacterReference] -//! * [`CharacterReferenceMarker`][Token::CharacterReferenceMarker] -//! * [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal] -//! * [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric] -//! * [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi] -//! * [`CharacterReferenceValue`][Token::CharacterReferenceValue] +//! * [`CharacterReference`][Name::CharacterReference] +//! * [`CharacterReferenceMarker`][Name::CharacterReferenceMarker] +//! * [`CharacterReferenceMarkerHexadecimal`][Name::CharacterReferenceMarkerHexadecimal] +//! * [`CharacterReferenceMarkerNumeric`][Name::CharacterReferenceMarkerNumeric] +//! * [`CharacterReferenceMarkerSemi`][Name::CharacterReferenceMarkerSemi] +//! * [`CharacterReferenceValue`][Name::CharacterReferenceValue] //! //! ## References //! diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index 9488f62..6e29010 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -77,14 +77,14 @@ //! //! ## Tokens //! -//! * [`CodeFenced`][Token::CodeFenced] -//! * [`CodeFencedFence`][Token::CodeFencedFence] -//! * [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo] -//! * [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta] -//! * [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence] -//! * [`CodeFlowChunk`][Token::CodeFlowChunk] -//! * [`LineEnding`][Token::LineEnding] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`CodeFenced`][Name::CodeFenced] +//! * [`CodeFencedFence`][Name::CodeFencedFence] +//! * [`CodeFencedFenceInfo`][Name::CodeFencedFenceInfo] +//! * [`CodeFencedFenceMeta`][Name::CodeFencedFenceMeta] +//! * [`CodeFencedFenceSequence`][Name::CodeFencedFenceSequence] +//! * [`CodeFlowChunk`][Name::CodeFlowChunk] +//! * [`LineEnding`][Name::LineEnding] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index 7297759..598d2b0 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -28,10 +28,10 @@ //! //! ## Tokens //! -//! * [`CodeIndented`][Token::CodeIndented] -//! * [`CodeFlowChunk`][Token::CodeFlowChunk] -//! * [`LineEnding`][Token::LineEnding] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`CodeIndented`][Name::CodeIndented] +//! * [`CodeFlowChunk`][Name::CodeFlowChunk] +//! * [`LineEnding`][Name::LineEnding] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index f48f63c..d321f64 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -67,10 +67,10 @@ //! //! ## Tokens //! -//! * [`CodeText`][Token::CodeText] -//! * [`CodeTextData`][Token::CodeTextData] -//! * [`CodeTextSequence`][Token::CodeTextSequence] -//! * [`LineEnding`][Token::LineEnding] +//! * [`CodeText`][Name::CodeText] +//! * [`CodeTextData`][Name::CodeTextData] +//! * [`CodeTextSequence`][Name::CodeTextSequence] +//! * [`LineEnding`][Name::LineEnding] //! //! ## References //! diff --git a/src/construct/definition.rs b/src/construct/definition.rs index 2533a1c..6f63c79 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -59,21 +59,21 @@ //! //! ## Tokens //! -//! * [`Definition`][Token::Definition] -//! * [`DefinitionDestination`][Token::DefinitionDestination] -//! * [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral] -//! * [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker] -//! * [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw] -//! * [`DefinitionDestinationString`][Token::DefinitionDestinationString] -//! * [`DefinitionLabel`][Token::DefinitionLabel] -//! * [`DefinitionLabelMarker`][Token::DefinitionLabelMarker] -//! * [`DefinitionLabelString`][Token::DefinitionLabelString] -//! * [`DefinitionMarker`][Token::DefinitionMarker] -//! * [`DefinitionTitle`][Token::DefinitionTitle] -//! * [`DefinitionTitleMarker`][Token::DefinitionTitleMarker] -//! * [`DefinitionTitleString`][Token::DefinitionTitleString] -//! * [`LineEnding`][Token::LineEnding] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`Definition`][Name::Definition] +//! * [`DefinitionDestination`][Name::DefinitionDestination] +//! * [`DefinitionDestinationLiteral`][Name::DefinitionDestinationLiteral] +//! * [`DefinitionDestinationLiteralMarker`][Name::DefinitionDestinationLiteralMarker] +//! * [`DefinitionDestinationRaw`][Name::DefinitionDestinationRaw] +//! * [`DefinitionDestinationString`][Name::DefinitionDestinationString] +//! * [`DefinitionLabel`][Name::DefinitionLabel] +//! * [`DefinitionLabelMarker`][Name::DefinitionLabelMarker] +//! * [`DefinitionLabelString`][Name::DefinitionLabelString] +//! * [`DefinitionMarker`][Name::DefinitionMarker] +//! * [`DefinitionTitle`][Name::DefinitionTitle] +//! * [`DefinitionTitleMarker`][Name::DefinitionTitleMarker] +//! * [`DefinitionTitleString`][Name::DefinitionTitleString] +//! * [`LineEnding`][Name::LineEnding] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! @@ -93,7 +93,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element -use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_eol}; +use crate::construct::partial_space_or_tab::space_or_tab; +use crate::construct::partial_space_or_tab_eol::space_or_tab_eol; use crate::event::Name; use crate::state::{Name as StateName, State}; use crate::tokenizer::Tokenizer; diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index 9af5b83..f5030aa 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -26,7 +26,7 @@ //! //! ## Tokens //! -//! * [`HardBreakEscape`][Token::HardBreakEscape] +//! * [`HardBreakEscape`][Name::HardBreakEscape] //! //! ## References //! diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index a114051..4e656d4 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -37,10 +37,10 @@ //! //! ## Tokens //! -//! * [`HeadingAtx`][Token::HeadingAtx] -//! * [`HeadingAtxSequence`][Token::HeadingAtxSequence] -//! * [`HeadingAtxText`][Token::HeadingAtxText] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`HeadingAtx`][Name::HeadingAtx] +//! * [`HeadingAtxSequence`][Name::HeadingAtxSequence] +//! * [`HeadingAtxText`][Name::HeadingAtxText] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index a3c513b..91a40b3 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -40,9 +40,9 @@ //! //! ## Tokens //! -//! * [`HeadingSetext`][Token::HeadingSetext] -//! * [`HeadingSetextText`][Token::HeadingSetextText] -//! * [`HeadingSetextUnderline`][Token::HeadingSetextUnderline] +//! * [`HeadingSetext`][Name::HeadingSetext] +//! * [`HeadingSetextText`][Name::HeadingSetextText] +//! * [`HeadingSetextUnderline`][Name::HeadingSetextUnderline] //! //! ## References //! diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index 8e5321f..9998797 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -82,9 +82,9 @@ //! //! ## Tokens //! -//! * [`HtmlFlow`][Token::HtmlFlow] -//! * [`HtmlFlowData`][Token::HtmlFlowData] -//! * [`LineEnding`][Token::LineEnding] +//! * [`HtmlFlow`][Name::HtmlFlow] +//! * [`HtmlFlowData`][Name::HtmlFlowData] +//! * [`LineEnding`][Name::LineEnding] //! //! ## References //! diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index 27e92f5..1b15956 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -42,8 +42,8 @@ //! //! ## Tokens //! -//! * [`HtmlText`][Token::HtmlText] -//! * [`HtmlTextData`][Token::HtmlTextData] +//! * [`HtmlText`][Name::HtmlText] +//! * [`HtmlTextData`][Name::HtmlTextData] //! //! ## References //! diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs index f27d79f..26dcf6b 100644 --- a/src/construct/label_end.rs +++ b/src/construct/label_end.rs @@ -102,28 +102,28 @@ //! //! ## Tokens //! -//! * [`Data`][Token::Data] -//! * [`Image`][Token::Image] -//! * [`Label`][Token::Label] -//! * [`LabelEnd`][Token::LabelEnd] -//! * [`LabelMarker`][Token::LabelMarker] -//! * [`LabelText`][Token::LabelText] -//! * [`LineEnding`][Token::LineEnding] -//! * [`Link`][Token::Link] -//! * [`Reference`][Token::Reference] -//! * [`ReferenceMarker`][Token::ReferenceMarker] -//! * [`ReferenceString`][Token::ReferenceString] -//! * [`Resource`][Token::Resource] -//! * [`ResourceDestination`][Token::ResourceDestination] -//! * [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral] -//! * [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker] -//! * [`ResourceDestinationRaw`][Token::ResourceDestinationRaw] -//! * [`ResourceDestinationString`][Token::ResourceDestinationString] -//! * [`ResourceMarker`][Token::ResourceMarker] -//! * [`ResourceTitle`][Token::ResourceTitle] -//! * [`ResourceTitleMarker`][Token::ResourceTitleMarker] -//! * [`ResourceTitleString`][Token::ResourceTitleString] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`Data`][Name::Data] +//! * [`Image`][Name::Image] +//! * [`Label`][Name::Label] +//! * [`LabelEnd`][Name::LabelEnd] +//! * [`LabelMarker`][Name::LabelMarker] +//! * [`LabelText`][Name::LabelText] +//! * [`LineEnding`][Name::LineEnding] +//! * [`Link`][Name::Link] +//! * [`Reference`][Name::Reference] +//! * [`ReferenceMarker`][Name::ReferenceMarker] +//! * [`ReferenceString`][Name::ReferenceString] +//! * [`Resource`][Name::Resource] +//! * [`ResourceDestination`][Name::ResourceDestination] +//! * [`ResourceDestinationLiteral`][Name::ResourceDestinationLiteral] +//! * [`ResourceDestinationLiteralMarker`][Name::ResourceDestinationLiteralMarker] +//! * [`ResourceDestinationRaw`][Name::ResourceDestinationRaw] +//! * [`ResourceDestinationString`][Name::ResourceDestinationString] +//! * [`ResourceMarker`][Name::ResourceMarker] +//! * [`ResourceTitle`][Name::ResourceTitle] +//! * [`ResourceTitleMarker`][Name::ResourceTitleMarker] +//! * [`ResourceTitleString`][Name::ResourceTitleString] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! @@ -147,7 +147,7 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use crate::constant::RESOURCE_DESTINATION_BALANCE_MAX; -use crate::construct::partial_space_or_tab::space_or_tab_eol; +use crate::construct::partial_space_or_tab_eol::space_or_tab_eol; use crate::event::{Event, Kind, Name}; use crate::resolve::Name as ResolveName; use crate::state::{Name as StateName, State}; diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs index e8aec8b..c9f76b0 100644 --- a/src/construct/label_start_image.rs +++ b/src/construct/label_start_image.rs @@ -15,9 +15,9 @@ //! //! ## Tokens //! -//! * [`LabelImage`][Token::LabelImage] -//! * [`LabelImageMarker`][Token::LabelImageMarker] -//! * [`LabelMarker`][Token::LabelMarker] +//! * [`LabelImage`][Name::LabelImage] +//! * [`LabelImageMarker`][Name::LabelImageMarker] +//! * [`LabelMarker`][Name::LabelMarker] //! //! ## References //! diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs index 530d83e..6b022c5 100644 --- a/src/construct/label_start_link.rs +++ b/src/construct/label_start_link.rs @@ -15,8 +15,8 @@ //! //! ## Tokens //! -//! * [`LabelLink`][Token::LabelLink] -//! * [`LabelMarker`][Token::LabelMarker] +//! * [`LabelLink`][Name::LabelLink] +//! * [`LabelMarker`][Name::LabelMarker] //! //! ## References //! diff --git a/src/construct/list.rs b/src/construct/list.rs index 028e283..1da1f4e 100644 --- a/src/construct/list.rs +++ b/src/construct/list.rs @@ -25,12 +25,12 @@ //! //! ## Tokens //! -//! * [`ListItem`][Token::ListItem] -//! * [`ListItemMarker`][Token::ListItemMarker] -//! * [`ListItemPrefix`][Token::ListItemPrefix] -//! * [`ListItemValue`][Token::ListItemValue] -//! * [`ListOrdered`][Token::ListOrdered] -//! * [`ListUnordered`][Token::ListUnordered] +//! * [`ListItem`][Name::ListItem] +//! * [`ListItemMarker`][Name::ListItemMarker] +//! * [`ListItemPrefix`][Name::ListItemPrefix] +//! * [`ListItemValue`][Name::ListItemValue] +//! * [`ListOrdered`][Name::ListOrdered] +//! * [`ListUnordered`][Name::ListUnordered] //! //! ## References //! diff --git a/src/construct/mod.rs b/src/construct/mod.rs index cfaca0a..0adf611 100644 --- a/src/construct/mod.rs +++ b/src/construct/mod.rs @@ -47,6 +47,7 @@ //! * [label][partial_label] //! * [non lazy continuation][partial_non_lazy_continuation] //! * [space or tab][partial_space_or_tab] +//! * [space or tab, eol][partial_space_or_tab_eol] //! * [title][partial_title] //! * [whitespace][partial_whitespace] //! @@ -91,6 +92,7 @@ pub mod partial_destination; pub mod partial_label; pub mod partial_non_lazy_continuation; pub mod partial_space_or_tab; +pub mod partial_space_or_tab_eol; pub mod partial_title; pub mod partial_whitespace; pub mod thematic_break; diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs index acbee83..663b01b 100644 --- a/src/construct/paragraph.rs +++ b/src/construct/paragraph.rs @@ -19,7 +19,7 @@ //! //! ## Tokens //! -//! * [`Paragraph`][Token::Paragraph] +//! * [`Paragraph`][Name::Paragraph] //! //! ## References //! diff --git a/src/construct/partial_bom.rs b/src/construct/partial_bom.rs index 74ca04e..0175971 100644 --- a/src/construct/partial_bom.rs +++ b/src/construct/partial_bom.rs @@ -4,7 +4,7 @@ //! //! ## Tokens //! -//! * [`ByteOrderMark`][Token::ByteOrderMark] +//! * [`ByteOrderMark`][Name::ByteOrderMark] //! //! ## References //! diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs index 85769eb..762baaa 100644 --- a/src/construct/partial_label.rs +++ b/src/construct/partial_label.rs @@ -59,7 +59,7 @@ //! [link_reference_size_max]: crate::constant::LINK_REFERENCE_SIZE_MAX use crate::constant::LINK_REFERENCE_SIZE_MAX; -use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; +use crate::construct::partial_space_or_tab_eol::{space_or_tab_eol_with_options, Options}; use crate::event::{Content, Name}; use crate::state::{Name as StateName, State}; use crate::subtokenize::link; @@ -105,7 +105,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { Some(b'\n') => { let name = space_or_tab_eol_with_options( tokenizer, - EolOptions { + Options { content_type: Some(Content::String), connect: tokenizer.tokenize_state.connect, }, diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs index 3fc9484..f2d9a73 100644 --- a/src/construct/partial_space_or_tab.rs +++ b/src/construct/partial_space_or_tab.rs @@ -24,15 +24,6 @@ pub struct Options { pub content_type: Option, } -/// Options to parse `space_or_tab` and one optional eol, but no blank line. -#[derive(Debug)] -pub struct EolOptions { - /// Connect this whitespace to the previous. - pub connect: bool, - /// Embedded content type to use. - pub content_type: Option, -} - /// One or more `space_or_tab`. /// /// ```bnf @@ -70,29 +61,6 @@ pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> StateName::SpaceOrTabStart } -/// `space_or_tab`, or optionally `space_or_tab`, one `eol`, and -/// optionally `space_or_tab`. -/// -/// ```bnf -/// space_or_tab_eol ::= 1*( ' ' '\t' ) | 0*( ' ' '\t' ) eol 0*( ' ' '\t' ) -/// ``` -pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> StateName { - space_or_tab_eol_with_options( - tokenizer, - EolOptions { - content_type: None, - connect: false, - }, - ) -} - -/// `space_or_tab_eol`, with the given options. -pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> StateName { - tokenizer.tokenize_state.space_or_tab_eol_content_type = options.content_type; - tokenizer.tokenize_state.space_or_tab_eol_connect = options.connect; - StateName::SpaceOrTabEolStart -} - /// Before `space_or_tab`. /// /// ```markdown @@ -165,132 +133,3 @@ pub fn after(tokenizer: &mut Tokenizer) -> State { tokenizer.tokenize_state.space_or_tab_token = Name::SpaceOrTab; state } - -pub fn eol_start(tokenizer: &mut Tokenizer) -> State { - let name = space_or_tab_with_options( - tokenizer, - Options { - kind: Name::SpaceOrTab, - min: 1, - max: usize::MAX, - content_type: tokenizer - .tokenize_state - .space_or_tab_eol_content_type - .clone(), - connect: tokenizer.tokenize_state.space_or_tab_eol_connect, - }, - ); - - tokenizer.attempt( - name, - State::Next(StateName::SpaceOrTabEolAfterFirst), - State::Next(StateName::SpaceOrTabEolAtEol), - ) -} - -pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.space_or_tab_eol_ok = true; - - if tokenizer - .tokenize_state - .space_or_tab_eol_content_type - .is_some() - { - tokenizer.tokenize_state.space_or_tab_eol_connect = true; - } - - State::Retry(StateName::SpaceOrTabEolAtEol) -} - -/// `space_or_tab_eol`: after optionally first `space_or_tab`. -/// -/// ```markdown -/// > | a -/// ^ -/// | b -/// ``` -pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State { - if let Some(b'\n') = tokenizer.current { - tokenizer.enter_with_content( - Name::LineEnding, - tokenizer - .tokenize_state - .space_or_tab_eol_content_type - .clone(), - ); - - if tokenizer.tokenize_state.space_or_tab_eol_connect { - let index = tokenizer.events.len() - 1; - link(&mut tokenizer.events, index); - } else if tokenizer - .tokenize_state - .space_or_tab_eol_content_type - .is_some() - { - tokenizer.tokenize_state.space_or_tab_eol_connect = true; - } - - tokenizer.consume(); - tokenizer.exit(Name::LineEnding); - State::Next(StateName::SpaceOrTabEolAfterEol) - } else { - let ok = tokenizer.tokenize_state.space_or_tab_eol_ok; - tokenizer.tokenize_state.space_or_tab_eol_content_type = None; - tokenizer.tokenize_state.space_or_tab_eol_connect = false; - tokenizer.tokenize_state.space_or_tab_eol_ok = false; - if ok { - State::Ok - } else { - State::Nok - } - } -} - -/// `space_or_tab_eol`: after eol. -/// -/// ```markdown -/// | a -/// > | b -/// ^ -/// ``` -#[allow(clippy::needless_pass_by_value)] -pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State { - let name = space_or_tab_with_options( - tokenizer, - Options { - kind: Name::SpaceOrTab, - min: 1, - max: usize::MAX, - content_type: tokenizer - .tokenize_state - .space_or_tab_eol_content_type - .clone(), - connect: tokenizer.tokenize_state.space_or_tab_eol_connect, - }, - ); - tokenizer.attempt( - name, - State::Next(StateName::SpaceOrTabEolAfterMore), - State::Next(StateName::SpaceOrTabEolAfterMore), - ) -} - -/// `space_or_tab_eol`: after more (optional) `space_or_tab`. -/// -/// ```markdown -/// | a -/// > | b -/// ^ -/// ``` -pub fn eol_after_more(tokenizer: &mut Tokenizer) -> State { - tokenizer.tokenize_state.space_or_tab_eol_content_type = None; - tokenizer.tokenize_state.space_or_tab_eol_connect = false; - tokenizer.tokenize_state.space_or_tab_eol_ok = false; - - // Blank line not allowed. - if matches!(tokenizer.current, None | Some(b'\n')) { - State::Nok - } else { - State::Ok - } -} diff --git a/src/construct/partial_space_or_tab_eol.rs b/src/construct/partial_space_or_tab_eol.rs new file mode 100644 index 0000000..0807a5f --- /dev/null +++ b/src/construct/partial_space_or_tab_eol.rs @@ -0,0 +1,174 @@ +//! Several helpers to parse whitespace (`space_or_tab`, `space_or_tab_eol`). +//! +//! ## References +//! +//! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js) + +use crate::construct::partial_space_or_tab::{ + space_or_tab_with_options, Options as SpaceOrTabOptions, +}; +use crate::event::{Content, Name}; +use crate::state::{Name as StateName, State}; +use crate::subtokenize::link; +use crate::tokenizer::Tokenizer; + +/// Options to parse `space_or_tab` and one optional eol, but no blank line. +#[derive(Debug)] +pub struct Options { + /// Connect this whitespace to the previous. + pub connect: bool, + /// Embedded content type to use. + pub content_type: Option, +} + +/// `space_or_tab`, or optionally `space_or_tab`, one `eol`, and +/// optionally `space_or_tab`. +/// +/// ```bnf +/// space_or_tab_eol ::= 1*( ' ' '\t' ) | 0*( ' ' '\t' ) eol 0*( ' ' '\t' ) +/// ``` +pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> StateName { + space_or_tab_eol_with_options( + tokenizer, + Options { + content_type: None, + connect: false, + }, + ) +} + +/// `space_or_tab_eol`, with the given options. +pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: Options) -> StateName { + tokenizer.tokenize_state.space_or_tab_eol_content_type = options.content_type; + tokenizer.tokenize_state.space_or_tab_eol_connect = options.connect; + StateName::SpaceOrTabEolStart +} + +pub fn eol_start(tokenizer: &mut Tokenizer) -> State { + let name = space_or_tab_with_options( + tokenizer, + SpaceOrTabOptions { + kind: Name::SpaceOrTab, + min: 1, + max: usize::MAX, + content_type: tokenizer + .tokenize_state + .space_or_tab_eol_content_type + .clone(), + connect: tokenizer.tokenize_state.space_or_tab_eol_connect, + }, + ); + + tokenizer.attempt( + name, + State::Next(StateName::SpaceOrTabEolAfterFirst), + State::Next(StateName::SpaceOrTabEolAtEol), + ) +} + +pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State { + tokenizer.tokenize_state.space_or_tab_eol_ok = true; + + if tokenizer + .tokenize_state + .space_or_tab_eol_content_type + .is_some() + { + tokenizer.tokenize_state.space_or_tab_eol_connect = true; + } + + State::Retry(StateName::SpaceOrTabEolAtEol) +} + +/// `space_or_tab_eol`: after optionally first `space_or_tab`. +/// +/// ```markdown +/// > | a +/// ^ +/// | b +/// ``` +pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State { + if let Some(b'\n') = tokenizer.current { + tokenizer.enter_with_content( + Name::LineEnding, + tokenizer + .tokenize_state + .space_or_tab_eol_content_type + .clone(), + ); + + if tokenizer.tokenize_state.space_or_tab_eol_connect { + let index = tokenizer.events.len() - 1; + link(&mut tokenizer.events, index); + } else if tokenizer + .tokenize_state + .space_or_tab_eol_content_type + .is_some() + { + tokenizer.tokenize_state.space_or_tab_eol_connect = true; + } + + tokenizer.consume(); + tokenizer.exit(Name::LineEnding); + State::Next(StateName::SpaceOrTabEolAfterEol) + } else { + let ok = tokenizer.tokenize_state.space_or_tab_eol_ok; + tokenizer.tokenize_state.space_or_tab_eol_content_type = None; + tokenizer.tokenize_state.space_or_tab_eol_connect = false; + tokenizer.tokenize_state.space_or_tab_eol_ok = false; + if ok { + State::Ok + } else { + State::Nok + } + } +} + +/// `space_or_tab_eol`: after eol. +/// +/// ```markdown +/// | a +/// > | b +/// ^ +/// ``` +#[allow(clippy::needless_pass_by_value)] +pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State { + let name = space_or_tab_with_options( + tokenizer, + SpaceOrTabOptions { + kind: Name::SpaceOrTab, + min: 1, + max: usize::MAX, + content_type: tokenizer + .tokenize_state + .space_or_tab_eol_content_type + .clone(), + connect: tokenizer.tokenize_state.space_or_tab_eol_connect, + }, + ); + tokenizer.attempt( + name, + State::Next(StateName::SpaceOrTabEolAfterMore), + State::Next(StateName::SpaceOrTabEolAfterMore), + ) +} + +/// `space_or_tab_eol`: after more (optional) `space_or_tab`. +/// +/// ```markdown +/// | a +/// > | b +/// ^ +/// ``` +pub fn eol_after_more(tokenizer: &mut Tokenizer) -> State { + tokenizer.tokenize_state.space_or_tab_eol_content_type = None; + tokenizer.tokenize_state.space_or_tab_eol_connect = false; + tokenizer.tokenize_state.space_or_tab_eol_ok = false; + + // Blank line not allowed. + if matches!(tokenizer.current, None | Some(b'\n')) { + State::Nok + } else { + State::Ok + } +} diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs index be06c02..6421360 100644 --- a/src/construct/partial_title.rs +++ b/src/construct/partial_title.rs @@ -30,7 +30,7 @@ //! [character_reference]: crate::construct::character_reference //! [label_end]: crate::construct::label_end -use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; +use crate::construct::partial_space_or_tab_eol::{space_or_tab_eol_with_options, Options}; use crate::event::{Content, Name}; use crate::state::{Name as StateName, State}; use crate::subtokenize::link; @@ -101,7 +101,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State { Some(b'\n') => { let name = space_or_tab_eol_with_options( tokenizer, - EolOptions { + Options { content_type: Some(Content::String), connect: tokenizer.tokenize_state.connect, }, diff --git a/src/construct/partial_whitespace.rs b/src/construct/partial_whitespace.rs index 688e7fd..7152881 100644 --- a/src/construct/partial_whitespace.rs +++ b/src/construct/partial_whitespace.rs @@ -30,8 +30,8 @@ //! removed, making hard break (trailing) hard to use. //! ## Tokens //! -//! * [`HardBreakTrailing`][Token::HardBreakTrailing] -//! * [`SpaceOrTab`][Token::SpaceOrTab] +//! * [`HardBreakTrailing`][Name::HardBreakTrailing] +//! * [`SpaceOrTab`][Name::SpaceOrTab] //! //! ## References //! @@ -71,7 +71,7 @@ pub fn resolve_whitespace(tokenizer: &mut Tokenizer, hard_break: bool, trim_whol } } -/// Trim a [`Data`][Token::Data] token. +/// Trim a [`Data`][Name::Data] token. fn trim_data( tokenizer: &mut Tokenizer, exit_index: usize, diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index 8e3c4f4..5969e77 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -35,8 +35,8 @@ //! //! ## Tokens //! -//! * [`ThematicBreak`][Token::ThematicBreak] -//! * [`ThematicBreakSequence`][Token::ThematicBreakSequence] +//! * [`ThematicBreak`][Name::ThematicBreak] +//! * [`ThematicBreakSequence`][Name::ThematicBreakSequence] //! //! ## References //! -- cgit