diff options
Diffstat (limited to '')
| -rw-r--r-- | readme.md | 2 | ||||
| -rw-r--r-- | src/construct/autolink.rs | 7 | ||||
| -rw-r--r-- | src/construct/blank_line.rs | 4 | ||||
| -rw-r--r-- | src/construct/character_escape.rs | 6 | ||||
| -rw-r--r-- | src/construct/character_reference.rs | 68 | ||||
| -rw-r--r-- | src/construct/code_fenced.rs | 11 | ||||
| -rw-r--r-- | src/construct/code_indented.rs | 7 | ||||
| -rw-r--r-- | src/construct/code_text.rs | 7 | ||||
| -rw-r--r-- | src/construct/definition.rs | 18 | ||||
| -rw-r--r-- | src/construct/hard_break_escape.rs | 5 | ||||
| -rw-r--r-- | src/construct/hard_break_trailing.rs | 5 | ||||
| -rw-r--r-- | src/construct/heading_atx.rs | 7 | ||||
| -rw-r--r-- | src/construct/heading_setext.rs | 6 | ||||
| -rw-r--r-- | src/construct/html_flow.rs | 6 | ||||
| -rw-r--r-- | src/construct/html_text.rs | 5 | ||||
| -rw-r--r-- | src/construct/paragraph.rs | 4 | ||||
| -rw-r--r-- | src/construct/thematic_break.rs | 5 | ||||
| -rw-r--r-- | src/subtokenize.rs | 4 | ||||
| -rw-r--r-- | src/tokenizer.rs | 1108 |
19 files changed, 1246 insertions, 39 deletions
@@ -68,7 +68,6 @@ cargo doc --document-private-items #### Docs -- [ ] (1) Add docs for token types - [ ] (1) Add docs for tokenizer (`go`, `define_skip`, `account_for_potential_skip`, `attempt_5`, `attempt_7`, `call_multiple`) - [ ] (1) Add docs for sanitation (autolink, definition, resource) @@ -234,6 +233,7 @@ cargo doc --document-private-items - [x] (1) Add docs for virtual spaces - [x] (1) Add docs to `subtokenize.rs` - [x] (1) Add docs for `link.rs` +- [x] (1) Add docs for token types ### Extensions diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index 8376b98..84c483d 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -82,6 +82,13 @@ //! <p><a href="#"></a><a href="https://example.com">https://example.com</a></p> //! ``` //! +//! ## Tokens +//! +//! * [`Autolink`][TokenType::Autolink] +//! * [`AutolinkEmail`][TokenType::AutolinkEmail] +//! * [`AutolinkMarker`][TokenType::AutolinkMarker] +//! * [`AutolinkProtocol`][TokenType::AutolinkProtocol] +//! //! ## References //! //! * [`autolink.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/autolink.js) diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index 86091d9..1fd22fd 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -18,6 +18,10 @@ //! Because blank lines can be empty (line endings are not considered part of //! it), and events cannot be empty, blank lines are not present as a token. //! +//! ## Tokens +//! +//! * [`Whitespace`][crate::tokenizer::TokenType::Whitespace] +//! //! ## References //! //! * [`blank-line.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/blank-line.js) diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index 743cbf8..0ccc574 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -17,6 +17,12 @@ //! construct: a [hard break (escape)][hard_break_escape] is a backslash followed //! by a line ending (that is part of the construct instead of ending it). //! +//! ## Tokens +//! +//! * [`CharacterEscape`][TokenType::CharacterEscape] +//! * [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker] +//! * [`CharacterEscapeValue`][TokenType::CharacterEscapeValue] +//! //! ## References //! //! * [`character-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-escape.js) diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index decf852..bc42d21 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -40,6 +40,15 @@ //! See [`CHARACTER_REFERENCE_NAMES`][character_reference_names] for which //! names match. //! +//! ## Tokens +//! +//! * [`CharacterReference`][TokenType::CharacterReference] +//! * [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker] +//! * [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal] +//! * [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric] +//! * [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi] +//! * [`CharacterReferenceValue`][TokenType::CharacterReferenceValue] +//! //! ## References //! //! * [`character-reference.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-reference.js) @@ -155,39 +164,36 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// a&#|x9;b /// ``` fn numeric(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - match code { - Code::Char(char) if char == 'x' || char == 'X' => { - tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal); - tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal); - tokenizer.enter(TokenType::CharacterReferenceValue); + if let Code::Char('x' | 'X') = code { + tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal); + tokenizer.consume(code); + tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(TokenType::CharacterReferenceValue); - ( - State::Fn(Box::new(|t, c| { - value( - t, - c, - Info { - buffer: vec![], - kind: Kind::Hexadecimal, - }, - ) - })), - None, - ) - } - _ => { - tokenizer.enter(TokenType::CharacterReferenceValue); + ( + State::Fn(Box::new(|t, c| { + value( + t, + c, + Info { + buffer: vec![], + kind: Kind::Hexadecimal, + }, + ) + })), + None, + ) + } else { + tokenizer.enter(TokenType::CharacterReferenceValue); - value( - tokenizer, - code, - Info { - buffer: vec![], - kind: Kind::Decimal, - }, - ) - } + value( + tokenizer, + code, + Info { + buffer: vec![], + kind: Kind::Decimal, + }, + ) } } diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index f79705c..724a0b3 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -75,6 +75,17 @@ //! support for specifying the programming language, so it is recommended to //! use code (fenced) instead of code (indented). //! +//! ## Tokens +//! +//! * [`CodeFenced`][TokenType::CodeFenced] +//! * [`CodeFencedFence`][TokenType::CodeFencedFence] +//! * [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence] +//! * [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo] +//! * [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta] +//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk] +//! * [`LineEnding`][TokenType::LineEnding] +//! * [`Whitespace`][TokenType::Whitespace] +//! //! ## References //! //! * [`code-fenced.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-fenced.js) diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index 64956be..a0b543a 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -26,6 +26,13 @@ //! support for specifying the programming language that the code is in, so it //! is recommended to use that instead of indented code. //! +//! ## Tokens +//! +//! * [`CodeIndented`][TokenType::CodeIndented] +//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk] +//! * [`LineEnding`][TokenType::LineEnding] +//! * [`Whitespace`][TokenType::Whitespace] +//! //! ## References //! //! * [`code-indented.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-indented.js) diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index 94e0106..70b3805 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -65,6 +65,13 @@ //! that the code is in, so it is recommended to use that instead of indented //! code. //! +//! ## Tokens +//! +//! * [`CodeText`][TokenType::CodeText] +//! * [`CodeTextData`][TokenType::CodeTextData] +//! * [`CodeTextSequence`][TokenType::CodeTextSequence] +//! * [`CodeTextLineEnding`][TokenType::CodeTextLineEnding] +//! //! ## References //! //! * [`code-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-text.js) diff --git a/src/construct/definition.rs b/src/construct/definition.rs index f8dc249..3dd1fbb 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -41,6 +41,24 @@ //! That means that [character escapes][character_escape] and //! [character references][character_reference] are allowed. //! +//! ## Tokens +//! +//! * [`Definition`][TokenType::Definition] +//! * [`DefinitionMarker`][TokenType::DefinitionMarker] +//! * [`DefinitionLabel`][TokenType::DefinitionLabel] +//! * [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker] +//! * [`DefinitionLabelString`][TokenType::DefinitionLabelString] +//! * [`DefinitionDestination`][TokenType::DefinitionDestination] +//! * [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral] +//! * [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker] +//! * [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] +//! * [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] +//! * [`DefinitionTitle`][TokenType::DefinitionTitle] +//! * [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker] +//! * [`DefinitionTitleString`][TokenType::DefinitionTitleString] +//! * [`LineEnding`][TokenType::LineEnding] +//! * [`Whitespace`][TokenType::Whitespace] +//! //! ## References //! //! * [`definition.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/definition.js) diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index 51da953..4458ccb 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -23,6 +23,11 @@ //! Arbitrary characters can be escaped with //! [character reference][character_reference]s. //! +//! ## Tokens +//! +//! * [`HardBreakEscape`][TokenType::HardBreakEscape] +//! * [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker] +//! //! ## References //! //! * [`hard-break-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/hard-break-escape.js) diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs index 46337c5..6709e51 100644 --- a/src/construct/hard_break_trailing.rs +++ b/src/construct/hard_break_trailing.rs @@ -24,6 +24,11 @@ //! Trailing spaces are typically invisible in editors, or even automatically //! removed, making hard break (trailing) hard to use. //! +//! ## Tokens +//! +//! * [`HardBreakTrailing`][TokenType::HardBreakTrailing] +//! * [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace] +//! //! ## References //! //! * [`lib/initialize/text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark/dev/lib/initialize/text.js) diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index 12d4193..6460235 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -35,6 +35,13 @@ //! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for //! > more info. //! +//! ## Tokens +//! +//! * [`HeadingAtx`][TokenType::HeadingAtx] +//! * [`HeadingAtxSequence`][TokenType::HeadingAtxSequence] +//! * [`HeadingAtxText`][TokenType::HeadingAtxText] +//! * [`HeadingAtxWhitespace`][TokenType::HeadingAtxWhitespace] +//! //! ## References //! //! * [`heading-atx.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/heading-atx.js) diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index d9ff96c..924f840 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -36,6 +36,12 @@ //! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for //! > more info. //! +//! ## Tokens +//! +//! * [`HeadingSetext`][TokenType::HeadingSetext] +//! * [`HeadingSetextText`][TokenType::HeadingSetextText] +//! * [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline] +//! //! ## References //! //! * [`setext-underline.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/setext-underline.js) diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index d5937c5..0af9e3c 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -80,6 +80,12 @@ //! We *can* interrupt and don’t have to care too much about it being //! well-formed. //! +//! ## Tokens +//! +//! * [`HtmlFlow`][TokenType::HtmlFlow] +//! * [`HtmlFlowData`][TokenType::HtmlFlowData] +//! * [`LineEnding`][TokenType::LineEnding] +//! //! ## References //! //! * [`html-flow.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-flow.js) diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index c4d2353..18c5f9c 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -40,6 +40,11 @@ //! This is equivalent to <code>*emphasised* code</code>. //! ``` //! +//! ## Tokens +//! +//! * [`HtmlText`][TokenType::HtmlText] +//! * [`HtmlTextData`][TokenType::HtmlTextData] +//! //! ## References //! //! * [`html-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-text.js) diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs index b7ab919..8cd8d36 100644 --- a/src/construct/paragraph.rs +++ b/src/construct/paragraph.rs @@ -17,6 +17,10 @@ //! The paragraph is interpreted as the [text][] content type. //! That means that [autolinks][autolink], [code (text)][code_text], etc are allowed. //! +//! ## Tokens +//! +//! * [`Paragraph`][TokenType::Paragraph] +//! //! ## References //! //! * [`content.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/content.js) diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index a9e5732..f33f8f3 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -33,6 +33,11 @@ //! can look for asterisks to find syntax while not worrying about other //! characters. //! +//! ## Tokens +//! +//! * [`ThematicBreak`][TokenType::ThematicBreak] +//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence] +//! //! ## References //! //! * [`thematic-break.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/thematic-break.js) diff --git a/src/subtokenize.rs b/src/subtokenize.rs index 690eb58..1702387 100644 --- a/src/subtokenize.rs +++ b/src/subtokenize.rs @@ -24,12 +24,14 @@ //! //! <!-- To do: `ChunkFlow` when it exists. --> +/// To do: could we do without `HashMap`, so we don’t need `std`? +use std::collections::HashMap; + use crate::content::{string::start as string, text::start as text}; use crate::tokenizer::{ Code, Event, EventType, State, StateFn, StateFnResult, TokenType, Tokenizer, }; use crate::util::span; -use std::collections::HashMap; /// Create a link between two [`Event`][]s. /// diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9fca299..d85ec45 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -11,77 +11,1173 @@ //! [`attempt`]: Tokenizer::attempt //! [`check`]: Tokenizer::check -use crate::constant::TAB_SIZE; +/// To do: could we do without `HashMap`, so we don’t need `std`? use std::collections::HashMap; +use crate::constant::TAB_SIZE; + /// Semantic label of a span. // To do: figure out how to share this so extensions can add their own stuff, // though perhaps that’s impossible and we should inline all extensions? -// To do: document each variant. #[derive(Debug, Clone, PartialEq)] pub enum TokenType { + /// Whole autolink. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`AutolinkEmail`][TokenType::AutolinkEmail], + /// [`AutolinkMarker`][TokenType::AutolinkMarker], + /// [`AutolinkProtocol`][TokenType::AutolinkProtocol] + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <https://example.com> and <admin@example.com> + /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ + /// ``` Autolink, + /// Email autolink w/o markers. + /// + /// ## Info + /// + /// * **Context**: + /// [`Autolink`][TokenType::Autolink] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <admin@example.com> + /// ^^^^^^^^^^^^^^^^^ + /// ``` + AutolinkEmail, + /// Marker of an autolink. + /// + /// ## Info + /// + /// * **Context**: + /// [`Autolink`][TokenType::Autolink] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <https://example.com> + /// ^ ^ + /// ``` AutolinkMarker, + /// Protocol autolink w/o markers. + /// + /// ## Info + /// + /// * **Context**: + /// [`Autolink`][TokenType::Autolink] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <https://example.com> + /// ^^^^^^^^^^^^^^^^^^^ + /// ``` AutolinkProtocol, - AutolinkEmail, + /// Line ending preceded only by whitespace or nothing at all. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`blank_line`][crate::construct::blank_line] + /// + /// ## Example + /// + /// ```markdown + /// > | ␠␠␊ + /// ^ + /// ``` BlankLineEnding, + /// Whole character escape. + /// + /// ## Info + /// + /// * **Context**: + /// [string content][crate::content::string] or + /// [text content][crate::content::text] + /// * **Content model**: + /// [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker], + /// [`CharacterEscapeValue`][TokenType::CharacterEscapeValue] + /// * **Construct**: + /// [`character_escape`][crate::construct::character_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a \- b + /// ^^ + /// ``` CharacterEscape, + /// Character escape marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterEscape`][TokenType::CharacterEscape] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_escape`][crate::construct::character_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a \- b + /// ^ + /// ``` CharacterEscapeMarker, + /// Character escape value. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterEscape`][TokenType::CharacterEscape] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_escape`][crate::construct::character_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a \- b + /// ^ + /// ``` CharacterEscapeValue, + /// Whole character reference. + /// + /// ## Info + /// + /// * **Context**: + /// [string content][crate::content::string] or + /// [text content][crate::content::text] + /// * **Content model**: + /// [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker], + /// [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal], + /// [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric], + /// [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi], + /// [`CharacterReferenceValue`][TokenType::CharacterReferenceValue] + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^^^^^ ^^^^^^^ ^^^^^^^^^ + /// ``` CharacterReference, + /// Character reference opening marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][TokenType::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ ^ ^ + /// ``` CharacterReferenceMarker, + /// Character reference numeric marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][TokenType::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ ^ + /// ``` CharacterReferenceMarkerNumeric, + /// Character reference hexadecimal numeric marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][TokenType::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ + /// ``` CharacterReferenceMarkerHexadecimal, + /// Character reference closing marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][TokenType::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ ^ ^ + /// ``` CharacterReferenceMarkerSemi, + /// Character reference value. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][TokenType::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^^^ ^^^^ ^^^^^ + /// ``` CharacterReferenceValue, + /// Whole code (fenced). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`CodeFencedFence`][TokenType::CodeFencedFence], + /// [`CodeFlowChunk`][TokenType::CodeFlowChunk], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^^^^ + /// > | console.log(1) + /// ^^^^^^^^^^^^^^ + /// > | ``` + /// ^^^ + /// ```` CodeFenced, + /// A code (fenced) fence. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFenced`][TokenType::CodeFenced] + /// * **Content model**: + /// [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo], + /// [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta], + /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^^^^ + /// | console.log(1) + /// > | ``` + /// ^^^ + /// ```` CodeFencedFence, + /// A code (fenced) fence sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^^ + /// | console.log(1) + /// > | ``` + /// ^^^ + /// ```` CodeFencedFenceSequence, + /// A code (fenced) fence info word. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFencedFence`][TokenType::CodeFencedFence] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^ + /// | console.log(1) + /// | ``` + /// ```` CodeFencedFenceInfo, + /// A code (fenced) fence meta string. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFencedFence`][TokenType::CodeFencedFence] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js highlight="1" + /// ^^^^^^^^^^^^^ + /// | console.log(1) + /// | ``` + /// ```` CodeFencedFenceMeta, + /// A code (fenced, indented) chunk. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFenced`][TokenType::CodeFenced], + /// [`CodeIndented`][TokenType::CodeIndented] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced], + /// [`code_indented`][crate::construct::code_indented] + /// + /// ## Example + /// + /// ````markdown + /// | ```js + /// > | console.log(1) + /// ^^^^^^^^^^^^^^ + /// | ``` + /// ```` + /// + /// ```markdown + /// > | ␠␠␠␠console.log(1) + /// ^^^^^^^^^^^^^^ + /// ``` CodeFlowChunk, + /// Whole code (indented). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`CodeFlowChunk`][TokenType::CodeFlowChunk], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ```markdown + /// ␠␠␠␠console.log(1) + /// ^^^^^^^^^^^^^^^^^^ + /// ``` CodeIndented, + /// Whole code (text). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`CodeTextData`][TokenType::CodeTextData], + /// [`CodeTextSequence`][TokenType::CodeTextSequence], + /// [`CodeTextLineEnding`][TokenType::CodeTextLineEnding] + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b` c + /// ^^^ + /// ``` CodeText, + /// Code (text) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeText`][TokenType::CodeText], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b` c + /// ^ + /// ``` + CodeTextData, + /// Code (text) sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeText`][TokenType::CodeText], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b` c + /// ^ ^ + /// ``` CodeTextSequence, + /// Line ending in code (text). + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeText`][TokenType::CodeText], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b␊ + /// ^ + /// | c` d + /// ``` CodeTextLineEnding, - CodeTextData, + /// Data. + /// + /// ## Info + /// + /// * **Context**: + /// [string content][crate::content::string], + /// [text content][crate::content::text] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`partial_data`][crate::construct::partial_data] + /// + /// ## Example + /// + /// ```markdown + /// > | aa *bb* cc + /// ^^^ ^^ ^^^ + /// ``` Data, + /// Whole definition. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`DefinitionMarker`][TokenType::DefinitionMarker], + /// [`DefinitionLabel`][TokenType::DefinitionLabel], + /// [`DefinitionDestination`][TokenType::DefinitionDestination], + /// [`DefinitionTitle`][TokenType::DefinitionTitle], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`definition`][crate::construct::definition] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^^^^^^^^^^ + /// ``` Definition, + /// Whole definition label. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][TokenType::Definition] + /// * **Content model**: + /// [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker], + /// [`DefinitionLabelString`][TokenType::DefinitionLabelString], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`partial_label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^^^ + /// ``` DefinitionLabel, + /// Definition label marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionLabel`][TokenType::DefinitionLabel] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`partial_label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ ^ + /// ``` DefinitionLabelMarker, + /// Definition label data. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionLabel`][TokenType::DefinitionLabel] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`partial_label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` DefinitionLabelString, + /// Definition marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][TokenType::Definition] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`definition`][crate::construct::definition] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` DefinitionMarker, + /// Whole definition destination. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][TokenType::Definition] + /// * **Content model**: + /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral], + /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] + /// * **Construct**: + /// [`partial_destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// > | [a]: <b> "c" + /// ^^^ + /// ``` DefinitionDestination, + /// Definition destination literal. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestination`][TokenType::DefinitionDestination] + /// * **Content model**: + /// [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker], + /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] + /// * **Construct**: + /// [`partial_destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: <b> "c" + /// ^^^ + /// ``` DefinitionDestinationLiteral, + /// Definition destination literal marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`partial_destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: <b> "c" + /// ^ ^ + /// ``` DefinitionDestinationLiteralMarker, + /// Definition destination raw. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestination`][TokenType::DefinitionDestination] + /// * **Content model**: + /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] + /// * **Construct**: + /// [`partial_destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` DefinitionDestinationRaw, + /// Definition destination data. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral], + /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`partial_destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// > | [a]: <b> "c" + /// ^ + /// ``` DefinitionDestinationString, + /// Whole definition title. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][TokenType::Definition] + /// * **Content model**: + /// [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker], + /// [`DefinitionTitleString`][TokenType::DefinitionTitleString], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`partial_title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^^^ + /// ``` DefinitionTitle, + /// Definition title marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionTitle`][TokenType::DefinitionTitle] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`partial_title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ ^ + /// ``` DefinitionTitleMarker, + /// Definition title data. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionTitle`][TokenType::DefinitionTitle] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`partial_title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` DefinitionTitleString, + /// Whole hard break (escape). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker] + /// * **Construct**: + /// [`hard_break_escape`][crate::construct::hard_break_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a\␊ + /// ^^ + /// > | b + /// ``` HardBreakEscape, + /// Hard break (escape) marker. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`hard_break_escape`][crate::construct::hard_break_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a\␊ + /// ^ + /// > | b + /// ``` HardBreakEscapeMarker, + /// Whole hard break (trailing). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace] + /// * **Construct**: + /// [`hard_break_trailing`][crate::construct::hard_break_trailing] + /// + /// ## Example + /// + /// ```markdown + /// > | a␠␠␊ + /// ^^^ + /// > | b + /// ``` HardBreakTrailing, + /// Hard break (trailing) spaces. + /// + /// ## Info + /// + /// * **Context**: + /// [`HardBreakTrailing`][TokenType::HardBreakTrailing] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`hard_break_trailing`][crate::construct::hard_break_trailing] + /// + /// ## Example + /// + /// ```markdown + /// > | a␠␠␊ + /// ^^ + /// > | b + /// ``` HardBreakTrailingSpace, + /// Whole heading (atx). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`HeadingAtxSequence`][TokenType::HeadingAtxSequence], + /// [`HeadingAtxText`][TokenType::HeadingAtxText], + /// [`HeadingAtxWhitespace`][TokenType::HeadingAtxWhitespace] + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^^^^^^^ + /// ``` HeadingAtx, + /// Heading (atx) sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingAtx`][TokenType::HeadingAtx], + /// [flow content][crate::content::flow] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^ + /// ``` HeadingAtxSequence, - HeadingAtxWhitespace, + /// Heading (atx) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingAtx`][TokenType::HeadingAtx], + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^^^^^ + /// ``` HeadingAtxText, + /// Heading (atx) whitespace. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingAtx`][TokenType::HeadingAtx], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^ + /// ``` + HeadingAtxWhitespace, + /// Whole heading (setext). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`HeadingSetextText`][TokenType::HeadingSetextText], + /// [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`heading_setext`][crate::construct::heading_setext] + /// + /// ## Example + /// + /// ```markdown + /// > | alpha + /// ^^^^^ + /// > | ===== + /// ^^^^^ + /// ``` HeadingSetext, + /// Heading (setext) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingSetext`][TokenType::HeadingSetext] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`heading_setext`][crate::construct::heading_setext] + /// + /// ## Example + /// + /// ```markdown + /// > | alpha + /// ^^^^^ + /// | ===== + /// ``` HeadingSetextText, + /// Heading (setext) underline. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingSetext`][TokenType::HeadingSetext] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`heading_setext`][crate::construct::heading_setext] + /// + /// ## Example + /// + /// ```markdown + /// | alpha + /// > | ===== + /// ^^^^^ + /// ``` HeadingSetextUnderline, + /// Whole html (flow). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`HtmlFlowData`][TokenType::HtmlFlowData], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`html_flow`][crate::construct::html_flow] + /// + /// ## Example + /// + /// ```markdown + /// > | <div> + /// ^^^^^ + /// ``` HtmlFlow, + /// HTML (flow) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HtmlFlow`][TokenType::HtmlFlow], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`html_flow`][crate::construct::html_flow] + /// + /// ## Example + /// + /// ```markdown + /// > | <div> + /// ^^^^^ + /// ``` HtmlFlowData, + /// Whole html (text). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`HtmlTextData`][TokenType::HtmlTextData], + /// [`LineEnding`][TokenType::LineEnding], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`html_text`][crate::construct::html_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a <b> c + /// ^^^ + /// ``` HtmlText, + /// HTML (text) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HtmlText`][TokenType::HtmlText] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`html_text`][crate::construct::html_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a <b> c + /// ^^^ + /// ``` HtmlTextData, + /// Line ending. + /// + /// ## Info + /// + /// * **Context**: + /// basically everywhere + /// * **Content model**: + /// void + /// * **Construct**: + /// n/a + /// + /// ## Example + /// + /// ```markdown + /// > | a␊ + /// ^ + /// | b + /// ``` LineEnding, + /// Whole paragraph. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`paragraph`][crate::construct::paragraph] + /// + /// ## Example + /// + /// ```markdown + /// > | a b + /// ^^^ + /// > | c. + /// ^^ + /// ``` Paragraph, + /// Whole thematic break. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`ThematicBreakSequence`][TokenType::ThematicBreakSequence], + /// [`Whitespace`][TokenType::Whitespace] + /// * **Construct**: + /// [`thematic_break`][crate::construct::thematic_break] + /// + /// ## Example + /// + /// ```markdown + /// > | * * * + /// ^^^^^ + /// ``` ThematicBreak, + /// Thematic break sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`ThematicBreak`][TokenType::ThematicBreak] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`thematic_break`][crate::construct::thematic_break] + /// + /// ## Example + /// + /// ```markdown + /// > | * * * + /// ^ ^ ^ + /// ``` ThematicBreakSequence, + /// Whitespace. + /// + /// ## Info + /// + /// * **Context**: + /// basically everywhere + /// * **Content model**: + /// void + /// * **Construct**: + /// n/a + /// + /// ## Example + /// + /// ```markdown + /// > | ␠* * *␠ + /// ^ ^ ^ ^ + /// ``` Whitespace, - // Chunks are tokenizer, but unraveled by `subtokenize`. + /// Chunk (string). + /// + /// Tokenized where [string content][crate::content::string] can exist and + /// unraveled by [`subtokenize`][crate::subtokenize]. ChunkString, + + /// Chunk (text). + /// + /// Tokenized where [text content][crate::content::text] can exist and + /// unraveled by [`subtokenize`][crate::subtokenize]. ChunkText, } |