diff options
| author |  Titus Wormer <tituswormer@gmail.com> | 2022-07-07 18:56:06 +0200 |
|---|---|---|
| committer | Titus Wormer <tituswormer@gmail.com> | 2022-07-07 18:56:06 +0200 |
| commit | 92b42e06f943338ce8b54b7e22cbb116ff598fa6 (patch) | |
| tree | ff51df093f52dc33bfac5e1c236b41cfbd21c220 /src/tokenizer.rs | |
| parent | fdb1f1694f44cfbc59d303a10371300b48d74627 (diff) | |
| download | markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.tar.gz markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.tar.bz2 markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.zip | |
Refactor to move token types to `token`
Diffstat (limited to 'src/tokenizer.rs')
| -rw-r--r-- | src/tokenizer.rs | 1780 |
1 files changed, 6 insertions, 1774 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6198776..64b66cc 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -12,1777 +12,9 @@ //! [`check`]: Tokenizer::check use crate::parser::ParseState; +use crate::token::Token; use std::collections::HashMap; -/// Semantic label of a span. -#[derive(Debug, Clone, PartialEq, Hash, Eq)] -pub enum TokenType { - /// Attention sequence. - /// - /// > 👉 **Note**: this is used while parsing but compiled away. - AttentionSequence, - /// Whole autolink. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`AutolinkEmail`][TokenType::AutolinkEmail], - /// [`AutolinkMarker`][TokenType::AutolinkMarker], - /// [`AutolinkProtocol`][TokenType::AutolinkProtocol] - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <https://example.com> and <admin@example.com> - /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ - /// ``` - Autolink, - /// Email autolink w/o markers. - /// - /// ## Info - /// - /// * **Context**: - /// [`Autolink`][TokenType::Autolink] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <admin@example.com> - /// ^^^^^^^^^^^^^^^^^ - /// ``` - AutolinkEmail, - /// Marker of an autolink. - /// - /// ## Info - /// - /// * **Context**: - /// [`Autolink`][TokenType::Autolink] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <https://example.com> - /// ^ ^ - /// ``` - AutolinkMarker, - /// Protocol autolink w/o markers. - /// - /// ## Info - /// - /// * **Context**: - /// [`Autolink`][TokenType::Autolink] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <https://example.com> - /// ^^^^^^^^^^^^^^^^^^^ - /// ``` - AutolinkProtocol, - /// Line ending preceded only by whitespace or nothing at all. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`blank_line`][crate::construct::blank_line] - /// - /// ## Example - /// - /// ```markdown - /// > | ␠␠␊ - /// ^ - /// ``` - BlankLineEnding, - /// Whole block quote. - /// - /// ## Info - /// - /// * **Context**: - /// [document content][crate::content::document] - /// * **Content model**: - /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix], - /// [flow content][crate::content::flow] - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^^^ - /// > | b - /// ^ - /// ``` - BlockQuote, - /// Block quote marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^ - /// | b - /// ``` - BlockQuoteMarker, - /// Block quote prefix. - /// - /// ## Info - /// - /// * **Context**: - /// [`BlockQuote`][TokenType::BlockQuote] - /// * **Content model**: - /// [`BlockQuoteMarker`][TokenType::BlockQuoteMarker], - /// [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace] - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^^ - /// | b - /// ``` - BlockQuotePrefix, - /// Block quote white space. - /// - /// ## Info - /// - /// * **Context**: - /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^ - /// | b - /// ``` - BlockQuoteWhitespace, - /// Whole character escape. - /// - /// ## Info - /// - /// * **Context**: - /// [string content][crate::content::string] or - /// [text content][crate::content::text] - /// * **Content model**: - /// [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker], - /// [`CharacterEscapeValue`][TokenType::CharacterEscapeValue] - /// * **Construct**: - /// [`character_escape`][crate::construct::character_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a \- b - /// ^^ - /// ``` - CharacterEscape, - /// Character escape marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterEscape`][TokenType::CharacterEscape] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_escape`][crate::construct::character_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a \- b - /// ^ - /// ``` - CharacterEscapeMarker, - /// Character escape value. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterEscape`][TokenType::CharacterEscape] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_escape`][crate::construct::character_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a \- b - /// ^ - /// ``` - CharacterEscapeValue, - /// Whole character reference. - /// - /// ## Info - /// - /// * **Context**: - /// [string content][crate::content::string] or - /// [text content][crate::content::text] - /// * **Content model**: - /// [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker], - /// [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal], - /// [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric], - /// [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi], - /// [`CharacterReferenceValue`][TokenType::CharacterReferenceValue] - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^^^^^ ^^^^^^^ ^^^^^^^^^ - /// ``` - CharacterReference, - /// Character reference opening marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ ^ ^ - /// ``` - CharacterReferenceMarker, - /// Character reference hexadecimal numeric marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ - /// ``` - CharacterReferenceMarkerHexadecimal, - /// Character reference numeric marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ ^ - /// ``` - CharacterReferenceMarkerNumeric, - /// Character reference closing marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ ^ ^ - /// ``` - CharacterReferenceMarkerSemi, - /// Character reference value. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^^^ ^^^^ ^^^^^ - /// ``` - CharacterReferenceValue, - /// Whole code (fenced). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`CodeFencedFence`][TokenType::CodeFencedFence], - /// [`CodeFlowChunk`][TokenType::CodeFlowChunk], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^^^^ - /// > | console.log(1) - /// ^^^^^^^^^^^^^^ - /// > | ``` - /// ^^^ - /// ```` - CodeFenced, - /// A code (fenced) fence. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFenced`][TokenType::CodeFenced] - /// * **Content model**: - /// [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo], - /// [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta], - /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^^^^ - /// | console.log(1) - /// > | ``` - /// ^^^ - /// ```` - CodeFencedFence, - /// A code (fenced) fence info word. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFencedFence`][TokenType::CodeFencedFence] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^ - /// | console.log(1) - /// | ``` - /// ```` - CodeFencedFenceInfo, - /// A code (fenced) fence meta string. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFencedFence`][TokenType::CodeFencedFence] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js highlight="1" - /// ^^^^^^^^^^^^^ - /// | console.log(1) - /// | ``` - /// ```` - CodeFencedFenceMeta, - /// A code (fenced) fence sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^^ - /// | console.log(1) - /// > | ``` - /// ^^^ - /// ```` - CodeFencedFenceSequence, - /// A code (fenced, indented) chunk. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFenced`][TokenType::CodeFenced], - /// [`CodeIndented`][TokenType::CodeIndented] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced], - /// [`code_indented`][crate::construct::code_indented] - /// - /// ## Example - /// - /// ````markdown - /// | ```js - /// > | console.log(1) - /// ^^^^^^^^^^^^^^ - /// | ``` - /// ```` - /// - /// ```markdown - /// > | ␠␠␠␠console.log(1) - /// ^^^^^^^^^^^^^^ - /// ``` - CodeFlowChunk, - /// Whole code (indented). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`CodeFlowChunk`][TokenType::CodeFlowChunk], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ```markdown - /// ␠␠␠␠console.log(1) - /// ^^^^^^^^^^^^^^^^^^ - /// ``` - CodeIndented, - /// Whole code (text). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`CodeTextData`][TokenType::CodeTextData], - /// [`CodeTextSequence`][TokenType::CodeTextSequence], - /// [`CodeTextLineEnding`][TokenType::CodeTextLineEnding] - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b` c - /// ^^^ - /// ``` - CodeText, - /// Code (text) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeText`][TokenType::CodeText], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b` c - /// ^ - /// ``` - CodeTextData, - /// Line ending in code (text). - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeText`][TokenType::CodeText], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b␊ - /// ^ - /// | c` d - /// ``` - CodeTextLineEnding, - /// Code (text) sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeText`][TokenType::CodeText], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b` c - /// ^ ^ - /// ``` - CodeTextSequence, - /// Data. - /// - /// ## Info - /// - /// * **Context**: - /// [string content][crate::content::string], - /// [text content][crate::content::text] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`data`][crate::construct::partial_data] - /// - /// ## Example - /// - /// ```markdown - /// > | aa *bb* cc - /// ^^^ ^^ ^^^ - /// ``` - Data, - /// Whole definition. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`DefinitionMarker`][TokenType::DefinitionMarker], - /// [`DefinitionLabel`][TokenType::DefinitionLabel], - /// [`DefinitionDestination`][TokenType::DefinitionDestination], - /// [`DefinitionTitle`][TokenType::DefinitionTitle], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`definition`][crate::construct::definition] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^^^^^^^^^^ - /// ``` - Definition, - /// Whole definition destination. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral], - /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// > | [a]: <b> "c" - /// ^^^ - /// ``` - DefinitionDestination, - /// Definition destination literal. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestination`][TokenType::DefinitionDestination] - /// * **Content model**: - /// [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker], - /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: <b> "c" - /// ^^^ - /// ``` - DefinitionDestinationLiteral, - /// Definition destination literal marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: <b> "c" - /// ^ ^ - /// ``` - DefinitionDestinationLiteralMarker, - /// Definition destination raw. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestination`][TokenType::DefinitionDestination] - /// * **Content model**: - /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionDestinationRaw, - /// Definition destination data. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral], - /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// > | [a]: <b> "c" - /// ^ - /// ``` - DefinitionDestinationString, - /// Whole definition label. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker], - /// [`DefinitionLabelString`][TokenType::DefinitionLabelString], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^^^ - /// ``` - DefinitionLabel, - /// Definition label marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionLabel`][TokenType::DefinitionLabel] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ ^ - /// ``` - DefinitionLabelMarker, - /// Definition label data. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionLabel`][TokenType::DefinitionLabel] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionLabelString, - /// Definition marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`definition`][crate::construct::definition] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionMarker, - /// Whole definition title. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker], - /// [`DefinitionTitleString`][TokenType::DefinitionTitleString], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^^^ - /// ``` - DefinitionTitle, - /// Definition title marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionTitle`][TokenType::DefinitionTitle] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ ^ - /// ``` - DefinitionTitleMarker, - /// Definition title data. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionTitle`][TokenType::DefinitionTitle] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionTitleString, - /// Emphasis. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`EmphasisSequence`][TokenType::EmphasisSequence], - /// [`EmphasisText`][TokenType::EmphasisText] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | *a* - /// ^^^ - /// ``` - Emphasis, - /// Emphasis sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`Emphasis`][TokenType::Emphasis] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | *a* - /// ^ ^ - /// ``` - EmphasisSequence, - /// Emphasis text. - /// - /// ## Info - /// - /// * **Context**: - /// [`Emphasis`][TokenType::Emphasis] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | *a* - /// ^ - /// ``` - EmphasisText, - /// Whole hard break (escape). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker] - /// * **Construct**: - /// [`hard_break_escape`][crate::construct::hard_break_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a\␊ - /// ^^ - /// > | b - /// ``` - HardBreakEscape, - /// Hard break (escape) marker. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`hard_break_escape`][crate::construct::hard_break_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a\␊ - /// ^ - /// > | b - /// ``` - HardBreakEscapeMarker, - /// Whole hard break (trailing). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace] - /// * **Construct**: - /// [`hard_break_trailing`][crate::construct::hard_break_trailing] - /// - /// ## Example - /// - /// ```markdown - /// > | a␠␠␊ - /// ^^^ - /// > | b - /// ``` - HardBreakTrailing, - /// Hard break (trailing) spaces. - /// - /// ## Info - /// - /// * **Context**: - /// [`HardBreakTrailing`][TokenType::HardBreakTrailing] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`hard_break_trailing`][crate::construct::hard_break_trailing] - /// - /// ## Example - /// - /// ```markdown - /// > | a␠␠␊ - /// ^^ - /// > | b - /// ``` - HardBreakTrailingSpace, - /// Whole heading (atx). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`HeadingAtxSequence`][TokenType::HeadingAtxSequence], - /// [`HeadingAtxText`][TokenType::HeadingAtxText], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`heading_atx`][crate::construct::heading_atx] - /// - /// ## Example - /// - /// ```markdown - /// > | # alpha - /// ^^^^^^^ - /// ``` - HeadingAtx, - /// Heading (atx) sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingAtx`][TokenType::HeadingAtx] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`heading_atx`][crate::construct::heading_atx] - /// - /// ## Example - /// - /// ```markdown - /// > | # alpha - /// ^ - /// ``` - HeadingAtxSequence, - /// Heading (atx) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingAtx`][TokenType::HeadingAtx], - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`heading_atx`][crate::construct::heading_atx] - /// - /// ## Example - /// - /// ```markdown - /// > | # alpha - /// ^^^^^ - /// ``` - HeadingAtxText, - /// Whole heading (setext). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`HeadingSetextText`][TokenType::HeadingSetextText], - /// [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`heading_setext`][crate::construct::heading_setext] - /// - /// ## Example - /// - /// ```markdown - /// > | alpha - /// ^^^^^ - /// > | ===== - /// ^^^^^ - /// ``` - HeadingSetext, - /// Heading (setext) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingSetext`][TokenType::HeadingSetext] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`heading_setext`][crate::construct::heading_setext] - /// - /// ## Example - /// - /// ```markdown - /// > | alpha - /// ^^^^^ - /// | ===== - /// ``` - HeadingSetextText, - /// Heading (setext) underline. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingSetext`][TokenType::HeadingSetext] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`heading_setext`][crate::construct::heading_setext] - /// - /// ## Example - /// - /// ```markdown - /// | alpha - /// > | ===== - /// ^^^^^ - /// ``` - HeadingSetextUnderline, - /// Whole html (flow). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`HtmlFlowData`][TokenType::HtmlFlowData], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`html_flow`][crate::construct::html_flow] - /// - /// ## Example - /// - /// ```markdown - /// > | <div> - /// ^^^^^ - /// ``` - HtmlFlow, - /// HTML (flow) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HtmlFlow`][TokenType::HtmlFlow], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`html_flow`][crate::construct::html_flow] - /// - /// ## Example - /// - /// ```markdown - /// > | <div> - /// ^^^^^ - /// ``` - HtmlFlowData, - /// Whole html (text). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`HtmlTextData`][TokenType::HtmlTextData], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`html_text`][crate::construct::html_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a <b> c - /// ^^^ - /// ``` - HtmlText, - /// HTML (text) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HtmlText`][TokenType::HtmlText] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`html_text`][crate::construct::html_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a <b> c - /// ^^^ - /// ``` - HtmlTextData, - /// Image. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`Label`][TokenType::Label], - /// [`Resource`][TokenType::Resource], - /// [`Reference`][TokenType::Reference] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b] c - /// ^^^^ - /// > | a ![b][c] d - /// ^^^^^^^ - /// > | a  d - /// ^^^^^^^ - /// ``` - Image, - /// Label. - /// - /// ## Info - /// - /// * **Context**: - /// [`Image`][TokenType::Image], - /// [`Link`][TokenType::Link] - /// * **Content model**: - /// [`LabelImage`][TokenType::LabelImage], - /// [`LabelLink`][TokenType::LabelLink], - /// [`LabelEnd`][TokenType::LabelEnd], - /// [`LabelText`][TokenType::LabelText] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b] c - /// ^^^ - /// > | a ![b][c] d - /// ^^^^ - /// > | a [b](c) d - /// ^^^ - /// ``` - Label, - /// Label end. - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [`LabelMarker`][TokenType::LabelMarker] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^ - /// > | a [b](c) d - /// ^ - /// ``` - LabelEnd, - /// Label start (image). - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [`LabelImageMarker`][TokenType::LabelImageMarker], - /// [`LabelMarker`][TokenType::LabelMarker] - /// * **Construct**: - /// [`label_start_image`][crate::construct::label_start_image] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^^ - /// ``` - LabelImage, - /// Label start (image) marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`LabelImage`][TokenType::LabelImage] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label_start_image`][crate::construct::label_start_image] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^ - /// ``` - LabelImageMarker, - /// Label start (link). - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [`LabelMarker`][TokenType::LabelMarker] - /// * **Construct**: - /// [`label_start_link`][crate::construct::label_start_link] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b](c) d - /// ^ - /// ``` - LabelLink, - /// Label marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`LabelImage`][TokenType::LabelImage], - /// [`LabelLink`][TokenType::LabelLink], - /// [`LabelEnd`][TokenType::LabelEnd] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label_start_image`][crate::construct::label_start_image], - /// [`label_start_link`][crate::construct::label_start_link], - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^ ^ - /// > | a [b](c) d - /// ^ ^ - /// ``` - LabelMarker, - /// Label text. - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b] c - /// ^ - /// > | a ![b][c] d - /// ^ - /// > | a [b](c) d - /// ^ - /// ``` - LabelText, - /// Line ending. - /// - /// ## Info - /// - /// * **Context**: - /// basically everywhere - /// * **Content model**: - /// void - /// * **Construct**: - /// n/a - /// - /// ## Example - /// - /// ```markdown - /// > | a␊ - /// ^ - /// | b - /// ``` - LineEnding, - /// Link. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`Label`][TokenType::Label], - /// [`Resource`][TokenType::Resource], - /// [`Reference`][TokenType::Reference] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b] c - /// ^^^ - /// > | a [b][c] d - /// ^^^^^^ - /// > | a [b](c) d - /// ^^^^^^ - /// ``` - Link, - /// Whole paragraph. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`paragraph`][crate::construct::paragraph] - /// - /// ## Example - /// - /// ```markdown - /// > | a b - /// ^^^ - /// > | c. - /// ^^ - /// ``` - Paragraph, - /// Reference. - /// - /// ## Info - /// - /// * **Context**: - /// [`Image`][TokenType::Image], - /// [`Link`][TokenType::Link] - /// * **Content model**: - /// [`ReferenceMarker`][TokenType::ReferenceMarker], - /// [`ReferenceString`][TokenType::ReferenceString] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b][c] d - /// ^^^ - /// ``` - Reference, - /// Reference marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`Reference`][TokenType::Reference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b][c] d - /// ^ ^ - /// ``` - ReferenceMarker, - /// Reference string. - /// - /// ## Info - /// - /// * **Context**: - /// [`Reference`][TokenType::Reference] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b][c] d - /// ^ - /// ``` - ReferenceString, - /// Resource. - /// - /// ## Info - /// - /// * **Context**: - /// [`Image`][TokenType::Image], - /// [`Link`][TokenType::Link] - /// * **Content model**: - /// [`ResourceMarker`][TokenType::ResourceMarker], - /// [`ResourceDestination`][TokenType::ResourceDestination], - /// [`ResourceTitle`][TokenType::ResourceTitle], - /// [`SpaceOrTab`][TokenType::SpaceOrTab], - /// [`LineEnding`][TokenType::LineEnding] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^^^^^^^ - /// > | a [b](c) d - /// ^^^ - /// ``` - Resource, - /// Resource destination. - /// - /// ## Info - /// - /// * **Context**: - /// [`Resource`][TokenType::Resource] - /// * **Content model**: - /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral], - /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// ``` - ResourceDestination, - /// Resource destination literal. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestination`][TokenType::ResourceDestination] - /// * **Content model**: - /// [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker], - /// [`ResourceDestinationString`][TokenType::ResourceDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^^^ - /// ``` - ResourceDestinationLiteral, - /// Resource destination literal marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ ^ - /// ``` - ResourceDestinationLiteralMarker, - /// Resource destination raw. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestination`][TokenType::ResourceDestination] - /// * **Content model**: - /// [`ResourceDestinationString`][TokenType::ResourceDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// ``` - ResourceDestinationRaw, - /// Resource destination raw. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral], - /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// > | a  e - /// ^ - /// ``` - ResourceDestinationString, - /// Resource marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`Resource`][TokenType::Resource] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ ^ - /// ``` - ResourceMarker, - /// Resource title. - /// - /// ## Info - /// - /// * **Context**: - /// [`Resource`][TokenType::Resource] - /// * **Content model**: - /// [`ResourceTitleMarker`][TokenType::ResourceTitleMarker], - /// [`ResourceTitleString`][TokenType::ResourceTitleString] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^^^ - /// ``` - ResourceTitle, - /// Resource title marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceTitle`][TokenType::ResourceTitle] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ ^ - /// ``` - ResourceTitleMarker, - /// Resource title string. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceTitle`][TokenType::ResourceTitle] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// ``` - ResourceTitleString, - /// SpaceOrTab. - /// - /// ## Info - /// - /// * **Context**: - /// basically everywhere - /// * **Content model**: - /// void - /// * **Construct**: - /// n/a - /// - /// ## Example - /// - /// ```markdown - /// > | ␠* * *␠ - /// ^ ^ ^ ^ - /// ``` - SpaceOrTab, - /// Strong. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`StrongSequence`][TokenType::StrongSequence], - /// [`StrongText`][TokenType::StrongText] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | **a** - /// ^^^^^ - /// ``` - Strong, - /// Strong sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`Strong`][TokenType::Strong] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | **a** - /// ^^ ^^ - /// ``` - StrongSequence, - /// Strong text. - /// - /// ## Info - /// - /// * **Context**: - /// [`Strong`][TokenType::Strong] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | **a** - /// ^ - /// ``` - StrongText, - /// Whole thematic break. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`ThematicBreakSequence`][TokenType::ThematicBreakSequence], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`thematic_break`][crate::construct::thematic_break] - /// - /// ## Example - /// - /// ```markdown - /// > | * * * - /// ^^^^^ - /// ``` - ThematicBreak, - /// Thematic break sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`ThematicBreak`][TokenType::ThematicBreak] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`thematic_break`][crate::construct::thematic_break] - /// - /// ## Example - /// - /// ```markdown - /// > | * * * - /// ^ ^ ^ - /// ``` - ThematicBreakSequence, -} - /// Embedded content type. #[derive(Debug, Clone, Copy, PartialEq)] pub enum ContentType { @@ -1840,7 +72,7 @@ pub enum EventType { #[derive(Debug, Clone)] pub struct Event { pub event_type: EventType, - pub token_type: TokenType, + pub token_type: Token, pub point: Point, pub index: usize, pub previous: Option<usize>, @@ -1935,7 +167,7 @@ pub struct Tokenizer<'a> { /// Hierarchy of semantic labels. /// /// Tracked to make sure everything’s valid. - pub stack: Vec<TokenType>, + pub stack: Vec<Token>, /// Previous character code. pub previous: Code, /// Current character code. @@ -2082,11 +314,11 @@ impl<'a> Tokenizer<'a> { } /// Mark the start of a semantic label. - pub fn enter(&mut self, token_type: TokenType) { + pub fn enter(&mut self, token_type: Token) { self.enter_with_content(token_type, None); } - pub fn enter_with_content(&mut self, token_type: TokenType, content_type: Option<ContentType>) { + pub fn enter_with_content(&mut self, token_type: Token, content_type: Option<ContentType>) { log::debug!("enter `{:?}` ({:?})", token_type, self.point); self.events.push(Event { event_type: EventType::Enter, @@ -2101,7 +333,7 @@ impl<'a> Tokenizer<'a> { } /// Mark the end of a semantic label. - pub fn exit(&mut self, token_type: TokenType) { + pub fn exit(&mut self, token_type: Token) { let current_token = self.stack.pop().expect("cannot close w/o open tokens"); assert_eq!( |