aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLibravatar Titus Wormer <tituswormer@gmail.com>2022-06-22 12:34:23 +0200
committerLibravatar Titus Wormer <tituswormer@gmail.com>2022-06-22 12:34:23 +0200
commit0fcfeaf05a95ea17763a72d91b6aa1c01843d067 (patch)
tree616d82b6e1d5a1164c8d6db110bc73c6b3583c13
parent6a7462e031ccd5333628f232ba55ec27c8fda404 (diff)
downloadmarkdown-rs-0fcfeaf05a95ea17763a72d91b6aa1c01843d067.tar.gz
markdown-rs-0fcfeaf05a95ea17763a72d91b6aa1c01843d067.tar.bz2
markdown-rs-0fcfeaf05a95ea17763a72d91b6aa1c01843d067.zip
Add docs for token types
Diffstat (limited to '')
-rw-r--r--readme.md2
-rw-r--r--src/construct/autolink.rs7
-rw-r--r--src/construct/blank_line.rs4
-rw-r--r--src/construct/character_escape.rs6
-rw-r--r--src/construct/character_reference.rs68
-rw-r--r--src/construct/code_fenced.rs11
-rw-r--r--src/construct/code_indented.rs7
-rw-r--r--src/construct/code_text.rs7
-rw-r--r--src/construct/definition.rs18
-rw-r--r--src/construct/hard_break_escape.rs5
-rw-r--r--src/construct/hard_break_trailing.rs5
-rw-r--r--src/construct/heading_atx.rs7
-rw-r--r--src/construct/heading_setext.rs6
-rw-r--r--src/construct/html_flow.rs6
-rw-r--r--src/construct/html_text.rs5
-rw-r--r--src/construct/paragraph.rs4
-rw-r--r--src/construct/thematic_break.rs5
-rw-r--r--src/subtokenize.rs4
-rw-r--r--src/tokenizer.rs1108
19 files changed, 1246 insertions, 39 deletions
diff --git a/readme.md b/readme.md
index 94e127e..e4983a5 100644
--- a/readme.md
+++ b/readme.md
@@ -68,7 +68,6 @@ cargo doc --document-private-items
#### Docs
-- [ ] (1) Add docs for token types
- [ ] (1) Add docs for tokenizer (`go`, `define_skip`,
`account_for_potential_skip`, `attempt_5`, `attempt_7`, `call_multiple`)
- [ ] (1) Add docs for sanitation (autolink, definition, resource)
@@ -234,6 +233,7 @@ cargo doc --document-private-items
- [x] (1) Add docs for virtual spaces
- [x] (1) Add docs to `subtokenize.rs`
- [x] (1) Add docs for `link.rs`
+- [x] (1) Add docs for token types
### Extensions
diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs
index 8376b98..84c483d 100644
--- a/src/construct/autolink.rs
+++ b/src/construct/autolink.rs
@@ -82,6 +82,13 @@
//! <p><a href="#"></a><a href="https://example.com">https://example.com</a></p>
//! ```
//!
+//! ## Tokens
+//!
+//! * [`Autolink`][TokenType::Autolink]
+//! * [`AutolinkEmail`][TokenType::AutolinkEmail]
+//! * [`AutolinkMarker`][TokenType::AutolinkMarker]
+//! * [`AutolinkProtocol`][TokenType::AutolinkProtocol]
+//!
//! ## References
//!
//! * [`autolink.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/autolink.js)
diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs
index 86091d9..1fd22fd 100644
--- a/src/construct/blank_line.rs
+++ b/src/construct/blank_line.rs
@@ -18,6 +18,10 @@
//! Because blank lines can be empty (line endings are not considered part of
//! it), and events cannot be empty, blank lines are not present as a token.
//!
+//! ## Tokens
+//!
+//! * [`Whitespace`][crate::tokenizer::TokenType::Whitespace]
+//!
//! ## References
//!
//! * [`blank-line.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/blank-line.js)
diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs
index 743cbf8..0ccc574 100644
--- a/src/construct/character_escape.rs
+++ b/src/construct/character_escape.rs
@@ -17,6 +17,12 @@
//! construct: a [hard break (escape)][hard_break_escape] is a backslash followed
//! by a line ending (that is part of the construct instead of ending it).
//!
+//! ## Tokens
+//!
+//! * [`CharacterEscape`][TokenType::CharacterEscape]
+//! * [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker]
+//! * [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
+//!
//! ## References
//!
//! * [`character-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-escape.js)
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index decf852..bc42d21 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -40,6 +40,15 @@
//! See [`CHARACTER_REFERENCE_NAMES`][character_reference_names] for which
//! names match.
//!
+//! ## Tokens
+//!
+//! * [`CharacterReference`][TokenType::CharacterReference]
+//! * [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker]
+//! * [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal]
+//! * [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric]
+//! * [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi]
+//! * [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
+//!
//! ## References
//!
//! * [`character-reference.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-reference.js)
@@ -155,39 +164,36 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// a&#|x9;b
/// ```
fn numeric(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- match code {
- Code::Char(char) if char == 'x' || char == 'X' => {
- tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal);
- tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal);
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ if let Code::Char('x' | 'X') = code {
+ tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal);
+ tokenizer.consume(code);
+ tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(TokenType::CharacterReferenceValue);
- (
- State::Fn(Box::new(|t, c| {
- value(
- t,
- c,
- Info {
- buffer: vec![],
- kind: Kind::Hexadecimal,
- },
- )
- })),
- None,
- )
- }
- _ => {
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ (
+ State::Fn(Box::new(|t, c| {
+ value(
+ t,
+ c,
+ Info {
+ buffer: vec![],
+ kind: Kind::Hexadecimal,
+ },
+ )
+ })),
+ None,
+ )
+ } else {
+ tokenizer.enter(TokenType::CharacterReferenceValue);
- value(
- tokenizer,
- code,
- Info {
- buffer: vec![],
- kind: Kind::Decimal,
- },
- )
- }
+ value(
+ tokenizer,
+ code,
+ Info {
+ buffer: vec![],
+ kind: Kind::Decimal,
+ },
+ )
}
}
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index f79705c..724a0b3 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -75,6 +75,17 @@
//! support for specifying the programming language, so it is recommended to
//! use code (fenced) instead of code (indented).
//!
+//! ## Tokens
+//!
+//! * [`CodeFenced`][TokenType::CodeFenced]
+//! * [`CodeFencedFence`][TokenType::CodeFencedFence]
+//! * [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
+//! * [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo]
+//! * [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta]
+//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk]
+//! * [`LineEnding`][TokenType::LineEnding]
+//! * [`Whitespace`][TokenType::Whitespace]
+//!
//! ## References
//!
//! * [`code-fenced.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-fenced.js)
diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index 64956be..a0b543a 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -26,6 +26,13 @@
//! support for specifying the programming language that the code is in, so it
//! is recommended to use that instead of indented code.
//!
+//! ## Tokens
+//!
+//! * [`CodeIndented`][TokenType::CodeIndented]
+//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk]
+//! * [`LineEnding`][TokenType::LineEnding]
+//! * [`Whitespace`][TokenType::Whitespace]
+//!
//! ## References
//!
//! * [`code-indented.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-indented.js)
diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs
index 94e0106..70b3805 100644
--- a/src/construct/code_text.rs
+++ b/src/construct/code_text.rs
@@ -65,6 +65,13 @@
//! that the code is in, so it is recommended to use that instead of indented
//! code.
//!
+//! ## Tokens
+//!
+//! * [`CodeText`][TokenType::CodeText]
+//! * [`CodeTextData`][TokenType::CodeTextData]
+//! * [`CodeTextSequence`][TokenType::CodeTextSequence]
+//! * [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
+//!
//! ## References
//!
//! * [`code-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-text.js)
diff --git a/src/construct/definition.rs b/src/construct/definition.rs
index f8dc249..3dd1fbb 100644
--- a/src/construct/definition.rs
+++ b/src/construct/definition.rs
@@ -41,6 +41,24 @@
//! That means that [character escapes][character_escape] and
//! [character references][character_reference] are allowed.
//!
+//! ## Tokens
+//!
+//! * [`Definition`][TokenType::Definition]
+//! * [`DefinitionMarker`][TokenType::DefinitionMarker]
+//! * [`DefinitionLabel`][TokenType::DefinitionLabel]
+//! * [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker]
+//! * [`DefinitionLabelString`][TokenType::DefinitionLabelString]
+//! * [`DefinitionDestination`][TokenType::DefinitionDestination]
+//! * [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
+//! * [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker]
+//! * [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
+//! * [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
+//! * [`DefinitionTitle`][TokenType::DefinitionTitle]
+//! * [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker]
+//! * [`DefinitionTitleString`][TokenType::DefinitionTitleString]
+//! * [`LineEnding`][TokenType::LineEnding]
+//! * [`Whitespace`][TokenType::Whitespace]
+//!
//! ## References
//!
//! * [`definition.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/definition.js)
diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs
index 51da953..4458ccb 100644
--- a/src/construct/hard_break_escape.rs
+++ b/src/construct/hard_break_escape.rs
@@ -23,6 +23,11 @@
//! Arbitrary characters can be escaped with
//! [character reference][character_reference]s.
//!
+//! ## Tokens
+//!
+//! * [`HardBreakEscape`][TokenType::HardBreakEscape]
+//! * [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
+//!
//! ## References
//!
//! * [`hard-break-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/hard-break-escape.js)
diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs
index 46337c5..6709e51 100644
--- a/src/construct/hard_break_trailing.rs
+++ b/src/construct/hard_break_trailing.rs
@@ -24,6 +24,11 @@
//! Trailing spaces are typically invisible in editors, or even automatically
//! removed, making hard break (trailing) hard to use.
//!
+//! ## Tokens
+//!
+//! * [`HardBreakTrailing`][TokenType::HardBreakTrailing]
+//! * [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
+//!
//! ## References
//!
//! * [`lib/initialize/text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark/dev/lib/initialize/text.js)
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 12d4193..6460235 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -35,6 +35,13 @@
//! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for
//! > more info.
//!
+//! ## Tokens
+//!
+//! * [`HeadingAtx`][TokenType::HeadingAtx]
+//! * [`HeadingAtxSequence`][TokenType::HeadingAtxSequence]
+//! * [`HeadingAtxText`][TokenType::HeadingAtxText]
+//! * [`HeadingAtxWhitespace`][TokenType::HeadingAtxWhitespace]
+//!
//! ## References
//!
//! * [`heading-atx.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/heading-atx.js)
diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index d9ff96c..924f840 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -36,6 +36,12 @@
//! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for
//! > more info.
//!
+//! ## Tokens
+//!
+//! * [`HeadingSetext`][TokenType::HeadingSetext]
+//! * [`HeadingSetextText`][TokenType::HeadingSetextText]
+//! * [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline]
+//!
//! ## References
//!
//! * [`setext-underline.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/setext-underline.js)
diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs
index d5937c5..0af9e3c 100644
--- a/src/construct/html_flow.rs
+++ b/src/construct/html_flow.rs
@@ -80,6 +80,12 @@
//! We *can* interrupt and don’t have to care too much about it being
//! well-formed.
//!
+//! ## Tokens
+//!
+//! * [`HtmlFlow`][TokenType::HtmlFlow]
+//! * [`HtmlFlowData`][TokenType::HtmlFlowData]
+//! * [`LineEnding`][TokenType::LineEnding]
+//!
//! ## References
//!
//! * [`html-flow.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-flow.js)
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs
index c4d2353..18c5f9c 100644
--- a/src/construct/html_text.rs
+++ b/src/construct/html_text.rs
@@ -40,6 +40,11 @@
//! This is equivalent to <code>*emphasised* code</code>.
//! ```
//!
+//! ## Tokens
+//!
+//! * [`HtmlText`][TokenType::HtmlText]
+//! * [`HtmlTextData`][TokenType::HtmlTextData]
+//!
//! ## References
//!
//! * [`html-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-text.js)
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index b7ab919..8cd8d36 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -17,6 +17,10 @@
//! The paragraph is interpreted as the [text][] content type.
//! That means that [autolinks][autolink], [code (text)][code_text], etc are allowed.
//!
+//! ## Tokens
+//!
+//! * [`Paragraph`][TokenType::Paragraph]
+//!
//! ## References
//!
//! * [`content.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/content.js)
diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs
index a9e5732..f33f8f3 100644
--- a/src/construct/thematic_break.rs
+++ b/src/construct/thematic_break.rs
@@ -33,6 +33,11 @@
//! can look for asterisks to find syntax while not worrying about other
//! characters.
//!
+//! ## Tokens
+//!
+//! * [`ThematicBreak`][TokenType::ThematicBreak]
+//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence]
+//!
//! ## References
//!
//! * [`thematic-break.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/thematic-break.js)
diff --git a/src/subtokenize.rs b/src/subtokenize.rs
index 690eb58..1702387 100644
--- a/src/subtokenize.rs
+++ b/src/subtokenize.rs
@@ -24,12 +24,14 @@
//!
//! <!-- To do: `ChunkFlow` when it exists. -->
+/// To do: could we do without `HashMap`, so we don’t need `std`?
+use std::collections::HashMap;
+
use crate::content::{string::start as string, text::start as text};
use crate::tokenizer::{
Code, Event, EventType, State, StateFn, StateFnResult, TokenType, Tokenizer,
};
use crate::util::span;
-use std::collections::HashMap;
/// Create a link between two [`Event`][]s.
///
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 9fca299..d85ec45 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -11,77 +11,1173 @@
//! [`attempt`]: Tokenizer::attempt
//! [`check`]: Tokenizer::check
-use crate::constant::TAB_SIZE;
+/// To do: could we do without `HashMap`, so we don’t need `std`?
use std::collections::HashMap;
+use crate::constant::TAB_SIZE;
+
/// Semantic label of a span.
// To do: figure out how to share this so extensions can add their own stuff,
// though perhaps that’s impossible and we should inline all extensions?
-// To do: document each variant.
#[derive(Debug, Clone, PartialEq)]
pub enum TokenType {
+ /// Whole autolink.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`AutolinkEmail`][TokenType::AutolinkEmail],
+ /// [`AutolinkMarker`][TokenType::AutolinkMarker],
+ /// [`AutolinkProtocol`][TokenType::AutolinkProtocol]
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <https://example.com> and <admin@example.com>
+ /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
+ /// ```
Autolink,
+ /// Email autolink w/o markers.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][TokenType::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <admin@example.com>
+ /// ^^^^^^^^^^^^^^^^^
+ /// ```
+ AutolinkEmail,
+ /// Marker of an autolink.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][TokenType::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <https://example.com>
+ /// ^ ^
+ /// ```
AutolinkMarker,
+ /// Protocol autolink w/o markers.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][TokenType::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <https://example.com>
+ /// ^^^^^^^^^^^^^^^^^^^
+ /// ```
AutolinkProtocol,
- AutolinkEmail,
+ /// Line ending preceded only by whitespace or nothing at all.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`blank_line`][crate::construct::blank_line]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | ␠␠␊
+ /// ^
+ /// ```
BlankLineEnding,
+ /// Whole character escape.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string] or
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker],
+ /// [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^^
+ /// ```
CharacterEscape,
+ /// Character escape marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterEscape`][TokenType::CharacterEscape]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^
+ /// ```
CharacterEscapeMarker,
+ /// Character escape value.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterEscape`][TokenType::CharacterEscape]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^
+ /// ```
CharacterEscapeValue,
+ /// Whole character reference.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string] or
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker],
+ /// [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal],
+ /// [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric],
+ /// [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi],
+ /// [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^^^^^ ^^^^^^^ ^^^^^^^^^
+ /// ```
CharacterReference,
+ /// Character reference opening marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][TokenType::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^ ^ ^
+ /// ```
CharacterReferenceMarker,
+ /// Character reference numeric marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][TokenType::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^ ^
+ /// ```
CharacterReferenceMarkerNumeric,
+ /// Character reference hexadecimal numeric marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][TokenType::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^
+ /// ```
CharacterReferenceMarkerHexadecimal,
+ /// Character reference closing marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][TokenType::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^ ^ ^
+ /// ```
CharacterReferenceMarkerSemi,
+ /// Character reference value.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][TokenType::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^^^ ^^^^ ^^^^^
+ /// ```
CharacterReferenceValue,
+ /// Whole code (fenced).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`CodeFencedFence`][TokenType::CodeFencedFence],
+ /// [`CodeFlowChunk`][TokenType::CodeFlowChunk],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^^^
+ /// > | console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// > | ```
+ /// ^^^
+ /// ````
CodeFenced,
+ /// A code (fenced) fence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFenced`][TokenType::CodeFenced]
+ /// * **Content model**:
+ /// [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo],
+ /// [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta],
+ /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^^^
+ /// | console.log(1)
+ /// > | ```
+ /// ^^^
+ /// ````
CodeFencedFence,
+ /// A code (fenced) fence sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^
+ /// | console.log(1)
+ /// > | ```
+ /// ^^^
+ /// ````
CodeFencedFenceSequence,
+ /// A code (fenced) fence info word.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFence`][TokenType::CodeFencedFence]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^
+ /// | console.log(1)
+ /// | ```
+ /// ````
CodeFencedFenceInfo,
+ /// A code (fenced) fence meta string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFence`][TokenType::CodeFencedFence]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js highlight="1"
+ /// ^^^^^^^^^^^^^
+ /// | console.log(1)
+ /// | ```
+ /// ````
CodeFencedFenceMeta,
+ /// A code (fenced, indented) chunk.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFenced`][TokenType::CodeFenced],
+ /// [`CodeIndented`][TokenType::CodeIndented]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced],
+ /// [`code_indented`][crate::construct::code_indented]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// | ```js
+ /// > | console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// | ```
+ /// ````
+ ///
+ /// ```markdown
+ /// > | ␠␠␠␠console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// ```
CodeFlowChunk,
+ /// Whole code (indented).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`CodeFlowChunk`][TokenType::CodeFlowChunk],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// ␠␠␠␠console.log(1)
+ /// ^^^^^^^^^^^^^^^^^^
+ /// ```
CodeIndented,
+ /// Whole code (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CodeTextData`][TokenType::CodeTextData],
+ /// [`CodeTextSequence`][TokenType::CodeTextSequence],
+ /// [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^^^
+ /// ```
CodeText,
+ /// Code (text) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][TokenType::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^
+ /// ```
+ CodeTextData,
+ /// Code (text) sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][TokenType::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^ ^
+ /// ```
CodeTextSequence,
+ /// Line ending in code (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][TokenType::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b␊
+ /// ^
+ /// | c` d
+ /// ```
CodeTextLineEnding,
- CodeTextData,
+ /// Data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string],
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`partial_data`][crate::construct::partial_data]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | aa *bb* cc
+ /// ^^^ ^^ ^^^
+ /// ```
Data,
+ /// Whole definition.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`DefinitionMarker`][TokenType::DefinitionMarker],
+ /// [`DefinitionLabel`][TokenType::DefinitionLabel],
+ /// [`DefinitionDestination`][TokenType::DefinitionDestination],
+ /// [`DefinitionTitle`][TokenType::DefinitionTitle],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`definition`][crate::construct::definition]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^^^^^^^^
+ /// ```
Definition,
+ /// Whole definition label.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][TokenType::Definition]
+ /// * **Content model**:
+ /// [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker],
+ /// [`DefinitionLabelString`][TokenType::DefinitionLabelString],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`partial_label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^
+ /// ```
DefinitionLabel,
+ /// Definition label marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionLabel`][TokenType::DefinitionLabel]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`partial_label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^ ^
+ /// ```
DefinitionLabelMarker,
+ /// Definition label data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionLabel`][TokenType::DefinitionLabel]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`partial_label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
DefinitionLabelString,
+ /// Definition marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][TokenType::Definition]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`definition`][crate::construct::definition]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
DefinitionMarker,
+ /// Whole definition destination.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][TokenType::Definition]
+ /// * **Content model**:
+ /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral],
+ /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
+ /// * **Construct**:
+ /// [`partial_destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// > | [a]: <b> "c"
+ /// ^^^
+ /// ```
DefinitionDestination,
+ /// Definition destination literal.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestination`][TokenType::DefinitionDestination]
+ /// * **Content model**:
+ /// [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker],
+ /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
+ /// * **Construct**:
+ /// [`partial_destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: <b> "c"
+ /// ^^^
+ /// ```
DefinitionDestinationLiteral,
+ /// Definition destination literal marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`partial_destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: <b> "c"
+ /// ^ ^
+ /// ```
DefinitionDestinationLiteralMarker,
+ /// Definition destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestination`][TokenType::DefinitionDestination]
+ /// * **Content model**:
+ /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
+ /// * **Construct**:
+ /// [`partial_destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
DefinitionDestinationRaw,
+ /// Definition destination data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral],
+ /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`partial_destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// > | [a]: <b> "c"
+ /// ^
+ /// ```
DefinitionDestinationString,
+ /// Whole definition title.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][TokenType::Definition]
+ /// * **Content model**:
+ /// [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker],
+ /// [`DefinitionTitleString`][TokenType::DefinitionTitleString],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`partial_title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^
+ /// ```
DefinitionTitle,
+ /// Definition title marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionTitle`][TokenType::DefinitionTitle]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`partial_title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^ ^
+ /// ```
DefinitionTitleMarker,
+ /// Definition title data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionTitle`][TokenType::DefinitionTitle]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`partial_title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
DefinitionTitleString,
+ /// Whole hard break (escape).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
+ /// * **Construct**:
+ /// [`hard_break_escape`][crate::construct::hard_break_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a\␊
+ /// ^^
+ /// > | b
+ /// ```
HardBreakEscape,
+ /// Hard break (escape) marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`hard_break_escape`][crate::construct::hard_break_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a\␊
+ /// ^
+ /// > | b
+ /// ```
HardBreakEscapeMarker,
+ /// Whole hard break (trailing).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
+ /// * **Construct**:
+ /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␠␠␊
+ /// ^^^
+ /// > | b
+ /// ```
HardBreakTrailing,
+ /// Hard break (trailing) spaces.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HardBreakTrailing`][TokenType::HardBreakTrailing]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␠␠␊
+ /// ^^
+ /// > | b
+ /// ```
HardBreakTrailingSpace,
+ /// Whole heading (atx).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HeadingAtxSequence`][TokenType::HeadingAtxSequence],
+ /// [`HeadingAtxText`][TokenType::HeadingAtxText],
+ /// [`HeadingAtxWhitespace`][TokenType::HeadingAtxWhitespace]
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^^^^^^^
+ /// ```
HeadingAtx,
+ /// Heading (atx) sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][TokenType::HeadingAtx],
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^
+ /// ```
HeadingAtxSequence,
- HeadingAtxWhitespace,
+ /// Heading (atx) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][TokenType::HeadingAtx],
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^^^^^
+ /// ```
HeadingAtxText,
+ /// Heading (atx) whitespace.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][TokenType::HeadingAtx],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^
+ /// ```
+ HeadingAtxWhitespace,
+ /// Whole heading (setext).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HeadingSetextText`][TokenType::HeadingSetextText],
+ /// [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | alpha
+ /// ^^^^^
+ /// > | =====
+ /// ^^^^^
+ /// ```
HeadingSetext,
+ /// Heading (setext) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingSetext`][TokenType::HeadingSetext]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | alpha
+ /// ^^^^^
+ /// | =====
+ /// ```
HeadingSetextText,
+ /// Heading (setext) underline.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingSetext`][TokenType::HeadingSetext]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// | alpha
+ /// > | =====
+ /// ^^^^^
+ /// ```
HeadingSetextUnderline,
+ /// Whole html (flow).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HtmlFlowData`][TokenType::HtmlFlowData],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`html_flow`][crate::construct::html_flow]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <div>
+ /// ^^^^^
+ /// ```
HtmlFlow,
+ /// HTML (flow) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HtmlFlow`][TokenType::HtmlFlow],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`html_flow`][crate::construct::html_flow]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <div>
+ /// ^^^^^
+ /// ```
HtmlFlowData,
+ /// Whole html (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HtmlTextData`][TokenType::HtmlTextData],
+ /// [`LineEnding`][TokenType::LineEnding],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`html_text`][crate::construct::html_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a <b> c
+ /// ^^^
+ /// ```
HtmlText,
+ /// HTML (text) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HtmlText`][TokenType::HtmlText]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`html_text`][crate::construct::html_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a <b> c
+ /// ^^^
+ /// ```
HtmlTextData,
+ /// Line ending.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// basically everywhere
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// n/a
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␊
+ /// ^
+ /// | b
+ /// ```
LineEnding,
+ /// Whole paragraph.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`paragraph`][crate::construct::paragraph]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a b
+ /// ^^^
+ /// > | c.
+ /// ^^
+ /// ```
Paragraph,
+ /// Whole thematic break.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`ThematicBreakSequence`][TokenType::ThematicBreakSequence],
+ /// [`Whitespace`][TokenType::Whitespace]
+ /// * **Construct**:
+ /// [`thematic_break`][crate::construct::thematic_break]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | * * *
+ /// ^^^^^
+ /// ```
ThematicBreak,
+ /// Thematic break sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ThematicBreak`][TokenType::ThematicBreak]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`thematic_break`][crate::construct::thematic_break]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | * * *
+ /// ^ ^ ^
+ /// ```
ThematicBreakSequence,
+ /// Whitespace.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// basically everywhere
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// n/a
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | ␠* * *␠
+ /// ^ ^ ^ ^
+ /// ```
Whitespace,
- // Chunks are tokenizer, but unraveled by `subtokenize`.
+ /// Chunk (string).
+ ///
+ /// Tokenized where [string content][crate::content::string] can exist and
+ /// unraveled by [`subtokenize`][crate::subtokenize].
ChunkString,
+
+ /// Chunk (text).
+ ///
+ /// Tokenized where [text content][crate::content::text] can exist and
+ /// unraveled by [`subtokenize`][crate::subtokenize].
ChunkText,
}