Add docs for token types

16 files changed, 140 insertions, 31 deletions

diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs
index 8376b98..84c483d 100644
--- a/src/construct/autolink.rs
+++ b/src/construct/autolink.rs
@@ -82,6 +82,13 @@
 //! <p><a href="#"></a><a href="https://example.com">https://example.com</a></p>
 //! ```
 //!
+//! ## Tokens
+//!
+//! *   [`Autolink`][TokenType::Autolink]
+//! *   [`AutolinkEmail`][TokenType::AutolinkEmail]
+//! *   [`AutolinkMarker`][TokenType::AutolinkMarker]
+//! *   [`AutolinkProtocol`][TokenType::AutolinkProtocol]
+//!
 //! ## References
 //!
 //! *   [`autolink.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/autolink.js)
diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs
index 86091d9..1fd22fd 100644
--- a/src/construct/blank_line.rs
+++ b/src/construct/blank_line.rs
@@ -18,6 +18,10 @@
 //! Because blank lines can be empty (line endings are not considered part of
 //! it), and events cannot be empty, blank lines are not present as a token.
 //!
+//! ## Tokens
+//!
+//! *   [`Whitespace`][crate::tokenizer::TokenType::Whitespace]
+//!
 //! ## References
 //!
 //! *   [`blank-line.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/blank-line.js)
diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs
index 743cbf8..0ccc574 100644
--- a/src/construct/character_escape.rs
+++ b/src/construct/character_escape.rs
@@ -17,6 +17,12 @@
 //! construct: a [hard break (escape)][hard_break_escape] is a backslash followed
 //! by a line ending (that is part of the construct instead of ending it).
 //!
+//! ## Tokens
+//!
+//! *   [`CharacterEscape`][TokenType::CharacterEscape]
+//! *   [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker]
+//! *   [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
+//!
 //! ## References
 //!
 //! *   [`character-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-escape.js)
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index decf852..bc42d21 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -40,6 +40,15 @@
 //! See [`CHARACTER_REFERENCE_NAMES`][character_reference_names] for which
 //! names match.
 //!
+//! ## Tokens
+//!
+//! *   [`CharacterReference`][TokenType::CharacterReference]
+//! *   [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker]
+//! *   [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal]
+//! *   [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric]
+//! *   [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi]
+//! *   [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
+//!
 //! ## References
 //!
 //! *   [`character-reference.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/character-reference.js)
@@ -155,39 +164,36 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// a&#|x9;b
 /// ```
 fn numeric(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
-    match code {
-        Code::Char(char) if char == 'x' || char == 'X' => {
-            tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal);
-            tokenizer.consume(code);
-            tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal);
-            tokenizer.enter(TokenType::CharacterReferenceValue);
+    if let Code::Char('x' | 'X') = code {
+        tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal);
+        tokenizer.consume(code);
+        tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal);
+        tokenizer.enter(TokenType::CharacterReferenceValue);
 
-            (
-                State::Fn(Box::new(|t, c| {
-                    value(
-                        t,
-                        c,
-                        Info {
-                            buffer: vec![],
-                            kind: Kind::Hexadecimal,
-                        },
-                    )
-                })),
-                None,
-            )
-        }
-        _ => {
-            tokenizer.enter(TokenType::CharacterReferenceValue);
+        (
+            State::Fn(Box::new(|t, c| {
+                value(
+                    t,
+                    c,
+                    Info {
+                        buffer: vec![],
+                        kind: Kind::Hexadecimal,
+                    },
+                )
+            })),
+            None,
+        )
+    } else {
+        tokenizer.enter(TokenType::CharacterReferenceValue);
 
-            value(
-                tokenizer,
-                code,
-                Info {
-                    buffer: vec![],
-                    kind: Kind::Decimal,
-                },
-            )
-        }
+        value(
+            tokenizer,
+            code,
+            Info {
+                buffer: vec![],
+                kind: Kind::Decimal,
+            },
+        )
     }
 }
 
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index f79705c..724a0b3 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -75,6 +75,17 @@
 //! support for specifying the programming language, so it is recommended to
 //! use code (fenced) instead of code (indented).
 //!
+//! ## Tokens
+//!
+//! *   [`CodeFenced`][TokenType::CodeFenced]
+//! *   [`CodeFencedFence`][TokenType::CodeFencedFence]
+//! *   [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
+//! *   [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo]
+//! *   [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta]
+//! *   [`CodeFlowChunk`][TokenType::CodeFlowChunk]
+//! *   [`LineEnding`][TokenType::LineEnding]
+//! *   [`Whitespace`][TokenType::Whitespace]
+//!
 //! ## References
 //!
 //! *   [`code-fenced.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-fenced.js)
diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index 64956be..a0b543a 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -26,6 +26,13 @@
 //! support for specifying the programming language that the code is in, so it
 //! is recommended to use that instead of indented code.
 //!
+//! ## Tokens
+//!
+//! *   [`CodeIndented`][TokenType::CodeIndented]
+//! *   [`CodeFlowChunk`][TokenType::CodeFlowChunk]
+//! *   [`LineEnding`][TokenType::LineEnding]
+//! *   [`Whitespace`][TokenType::Whitespace]
+//!
 //! ## References
 //!
 //! *   [`code-indented.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-indented.js)
diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs
index 94e0106..70b3805 100644
--- a/src/construct/code_text.rs
+++ b/src/construct/code_text.rs
@@ -65,6 +65,13 @@
 //! that the code is in, so it is recommended to use that instead of indented
 //! code.
 //!
+//! ## Tokens
+//!
+//! *   [`CodeText`][TokenType::CodeText]
+//! *   [`CodeTextData`][TokenType::CodeTextData]
+//! *   [`CodeTextSequence`][TokenType::CodeTextSequence]
+//! *   [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
+//!
 //! ## References
 //!
 //! *   [`code-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/code-text.js)
diff --git a/src/construct/definition.rs b/src/construct/definition.rs
index f8dc249..3dd1fbb 100644
--- a/src/construct/definition.rs
+++ b/src/construct/definition.rs
@@ -41,6 +41,24 @@
 //! That means that [character escapes][character_escape] and
 //! [character references][character_reference] are allowed.
 //!
+//! ## Tokens
+//!
+//! *   [`Definition`][TokenType::Definition]
+//! *   [`DefinitionMarker`][TokenType::DefinitionMarker]
+//! *   [`DefinitionLabel`][TokenType::DefinitionLabel]
+//! *   [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker]
+//! *   [`DefinitionLabelString`][TokenType::DefinitionLabelString]
+//! *   [`DefinitionDestination`][TokenType::DefinitionDestination]
+//! *   [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
+//! *   [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker]
+//! *   [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
+//! *   [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
+//! *   [`DefinitionTitle`][TokenType::DefinitionTitle]
+//! *   [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker]
+//! *   [`DefinitionTitleString`][TokenType::DefinitionTitleString]
+//! *   [`LineEnding`][TokenType::LineEnding]
+//! *   [`Whitespace`][TokenType::Whitespace]
+//!
 //! ## References
 //!
 //! *   [`definition.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/definition.js)
diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs
index 51da953..4458ccb 100644
--- a/src/construct/hard_break_escape.rs
+++ b/src/construct/hard_break_escape.rs
@@ -23,6 +23,11 @@
 //! Arbitrary characters can be escaped with
 //! [character reference][character_reference]s.
 //!
+//! ## Tokens
+//!
+//! *   [`HardBreakEscape`][TokenType::HardBreakEscape]
+//! *   [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
+//!
 //! ## References
 //!
 //! *   [`hard-break-escape.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/hard-break-escape.js)
diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs
index 46337c5..6709e51 100644
--- a/src/construct/hard_break_trailing.rs
+++ b/src/construct/hard_break_trailing.rs
@@ -24,6 +24,11 @@
 //! Trailing spaces are typically invisible in editors, or even automatically
 //! removed, making hard break (trailing) hard to use.
 //!
+//! ## Tokens
+//!
+//! *   [`HardBreakTrailing`][TokenType::HardBreakTrailing]
+//! *   [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
+//!
 //! ## References
 //!
 //! *   [`lib/initialize/text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark/dev/lib/initialize/text.js)
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 12d4193..6460235 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -35,6 +35,13 @@
 //! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for
 //! > more info.
 //!
+//! ## Tokens
+//!
+//! *   [`HeadingAtx`][TokenType::HeadingAtx]
+//! *   [`HeadingAtxSequence`][TokenType::HeadingAtxSequence]
+//! *   [`HeadingAtxText`][TokenType::HeadingAtxText]
+//! *   [`HeadingAtxWhitespace`][TokenType::HeadingAtxWhitespace]
+//!
 //! ## References
 //!
 //! *   [`heading-atx.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/heading-atx.js)
diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index d9ff96c..924f840 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -36,6 +36,12 @@
 //! > See [*§ atx, the true structured text format* on `aaronsw.com`][atx] for
 //! > more info.
 //!
+//! ## Tokens
+//!
+//! *   [`HeadingSetext`][TokenType::HeadingSetext]
+//! *   [`HeadingSetextText`][TokenType::HeadingSetextText]
+//! *   [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline]
+//!
 //! ## References
 //!
 //! *   [`setext-underline.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/setext-underline.js)
diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs
index d5937c5..0af9e3c 100644
--- a/src/construct/html_flow.rs
+++ b/src/construct/html_flow.rs
@@ -80,6 +80,12 @@
 //! We *can* interrupt and don’t have to care too much about it being
 //! well-formed.
 //!
+//! ## Tokens
+//!
+//! *   [`HtmlFlow`][TokenType::HtmlFlow]
+//! *   [`HtmlFlowData`][TokenType::HtmlFlowData]
+//! *   [`LineEnding`][TokenType::LineEnding]
+//!
 //! ## References
 //!
 //! *   [`html-flow.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-flow.js)
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs
index c4d2353..18c5f9c 100644
--- a/src/construct/html_text.rs
+++ b/src/construct/html_text.rs
@@ -40,6 +40,11 @@
 //! This is equivalent to <code>*emphasised* code</code>.
 //! ```
 //!
+//! ## Tokens
+//!
+//! *   [`HtmlText`][TokenType::HtmlText]
+//! *   [`HtmlTextData`][TokenType::HtmlTextData]
+//!
 //! ## References
 //!
 //! *   [`html-text.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/html-text.js)
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index b7ab919..8cd8d36 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -17,6 +17,10 @@
 //! The paragraph is interpreted as the [text][] content type.
 //! That means that [autolinks][autolink], [code (text)][code_text], etc are allowed.
 //!
+//! ## Tokens
+//!
+//! *   [`Paragraph`][TokenType::Paragraph]
+//!
 //! ## References
 //!
 //! *   [`content.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/content.js)
diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs
index a9e5732..f33f8f3 100644
--- a/src/construct/thematic_break.rs
+++ b/src/construct/thematic_break.rs
@@ -33,6 +33,11 @@
 //! can look for asterisks to find syntax while not worrying about other
 //! characters.
 //!
+//! ## Tokens
+//!
+//! *   [`ThematicBreak`][TokenType::ThematicBreak]
+//! *   [`ThematicBreakSequence`][TokenType::ThematicBreakSequence]
+//!
 //! ## References
 //!
 //! *   [`thematic-break.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-core-commonmark/dev/lib/thematic-break.js)