From 92b42e06f943338ce8b54b7e22cbb116ff598fa6 Mon Sep 17 00:00:00 2001
From: Titus Wormer ".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`Resource`][TokenType::Resource].
+/// Handle [`Enter`][EventType::Enter]:[`Resource`][Token::Resource].
fn on_enter_resource(context: &mut CompileContext) {
context.buffer(); // We can have line endings in the resource, ignore them.
let media = context.media_stack.last_mut().unwrap();
media.destination = Some("".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`ResourceDestinationString`][TokenType::ResourceDestinationString].
+/// Handle [`Enter`][EventType::Enter]:[`ResourceDestinationString`][Token::ResourceDestinationString].
fn on_enter_resource_destination_string(context: &mut CompileContext) {
context.buffer();
// Ignore encoding the result, as we’ll first percent encode the url and
@@ -721,12 +716,12 @@ fn on_enter_resource_destination_string(context: &mut CompileContext) {
context.ignore_encode = true;
}
-/// Handle [`Enter`][EventType::Enter]:[`Strong`][TokenType::Strong].
+/// Handle [`Enter`][EventType::Enter]:[`Strong`][Token::Strong].
fn on_enter_strong(context: &mut CompileContext) {
context.tag("".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`AutolinkEmail`][TokenType::AutolinkEmail].
+/// Handle [`Exit`][EventType::Exit]:[`AutolinkEmail`][Token::AutolinkEmail].
fn on_exit_autolink_email(context: &mut CompileContext) {
let slice = serialize(
context.codes,
@@ -744,7 +739,7 @@ fn on_exit_autolink_email(context: &mut CompileContext) {
context.tag("".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`AutolinkProtocol`][TokenType::AutolinkProtocol].
+/// Handle [`Exit`][EventType::Exit]:[`AutolinkProtocol`][Token::AutolinkProtocol].
fn on_exit_autolink_protocol(context: &mut CompileContext) {
let slice = serialize(
context.codes,
@@ -759,12 +754,12 @@ fn on_exit_autolink_protocol(context: &mut CompileContext) {
context.tag("".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:{[`HardBreakEscape`][TokenType::HardBreakEscape],[`HardBreakTrailing`][TokenType::HardBreakTrailing]}.
+/// Handle [`Exit`][EventType::Exit]:{[`HardBreakEscape`][Token::HardBreakEscape],[`HardBreakTrailing`][Token::HardBreakTrailing]}.
fn on_exit_break(context: &mut CompileContext) {
context.tag("".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`CodeIndented`][TokenType::CodeIndented].
+/// Handle [`Enter`][EventType::Enter]:[`CodeIndented`][Token::CodeIndented].
fn on_enter_code_indented(context: &mut CompileContext) {
context.code_flow_seen_data = Some(false);
context.line_ending_if_needed();
context.tag("
".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`CodeFenced`][TokenType::CodeFenced].
+/// Handle [`Enter`][EventType::Enter]:[`CodeFenced`][Token::CodeFenced].
fn on_enter_code_fenced(context: &mut CompileContext) {
context.code_flow_seen_data = Some(false);
context.line_ending_if_needed();
@@ -630,13 +625,13 @@ fn on_enter_code_fenced(context: &mut CompileContext) {
context.code_fenced_fences_count = Some(0);
}
-/// Handle [`Enter`][EventType::Enter]:[`CodeText`][TokenType::CodeText].
+/// Handle [`Enter`][EventType::Enter]:[`CodeText`][Token::CodeText].
fn on_enter_code_text(context: &mut CompileContext) {
context.tag("
".to_string());
context.buffer();
}
-/// Handle [`Enter`][EventType::Enter]:[`Definition`][TokenType::Definition].
+/// Handle [`Enter`][EventType::Enter]:[`Definition`][Token::Definition].
fn on_enter_definition(context: &mut CompileContext) {
context.buffer();
context.media_stack.push(Media {
@@ -649,18 +644,18 @@ fn on_enter_definition(context: &mut CompileContext) {
});
}
-/// Handle [`Enter`][EventType::Enter]:[`DefinitionDestinationString`][TokenType::DefinitionDestinationString].
+/// Handle [`Enter`][EventType::Enter]:[`DefinitionDestinationString`][Token::DefinitionDestinationString].
fn on_enter_definition_destination_string(context: &mut CompileContext) {
context.buffer();
context.ignore_encode = true;
}
-/// Handle [`Enter`][EventType::Enter]:[`Emphasis`][TokenType::Emphasis].
+/// Handle [`Enter`][EventType::Enter]:[`Emphasis`][Token::Emphasis].
fn on_enter_emphasis(context: &mut CompileContext) {
context.tag("".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`HtmlFlow`][TokenType::HtmlFlow].
+/// Handle [`Enter`][EventType::Enter]:[`HtmlFlow`][Token::HtmlFlow].
fn on_enter_html_flow(context: &mut CompileContext) {
context.line_ending_if_needed();
if context.allow_dangerous_html {
@@ -668,14 +663,14 @@ fn on_enter_html_flow(context: &mut CompileContext) {
}
}
-/// Handle [`Enter`][EventType::Enter]:[`HtmlText`][TokenType::HtmlText].
+/// Handle [`Enter`][EventType::Enter]:[`HtmlText`][Token::HtmlText].
fn on_enter_html_text(context: &mut CompileContext) {
if context.allow_dangerous_html {
context.ignore_encode = true;
}
}
-/// Handle [`Enter`][EventType::Enter]:[`Image`][TokenType::Image].
+/// Handle [`Enter`][EventType::Enter]:[`Image`][Token::Image].
fn on_enter_image(context: &mut CompileContext) {
context.media_stack.push(Media {
image: true,
@@ -688,7 +683,7 @@ fn on_enter_image(context: &mut CompileContext) {
context.tags = false; // Disallow tags.
}
-/// Handle [`Enter`][EventType::Enter]:[`Link`][TokenType::Link].
+/// Handle [`Enter`][EventType::Enter]:[`Link`][Token::Link].
fn on_enter_link(context: &mut CompileContext) {
context.media_stack.push(Media {
image: false,
@@ -700,20 +695,20 @@ fn on_enter_link(context: &mut CompileContext) {
});
}
-/// Handle [`Enter`][EventType::Enter]:[`Paragraph`][TokenType::Paragraph].
+/// Handle [`Enter`][EventType::Enter]:[`Paragraph`][Token::Paragraph].
fn on_enter_paragraph(context: &mut CompileContext) {
context.line_ending_if_needed();
context.tag("
".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeTextLineEnding`][TokenType::CodeTextLineEnding].
+/// Handle [`Exit`][EventType::Exit]:[`CodeTextLineEnding`][Token::CodeTextLineEnding].
fn on_exit_code_text_line_ending(context: &mut CompileContext) {
context.push(" ".to_string());
}
@@ -910,7 +905,7 @@ fn on_exit_drop(context: &mut CompileContext) {
context.resume();
}
-/// Handle [`Exit`][EventType::Exit]:{[`CodeTextData`][TokenType::CodeTextData],[`Data`][TokenType::Data],[`CharacterEscapeValue`][TokenType::CharacterEscapeValue]}.
+/// Handle [`Exit`][EventType::Exit]:{[`CodeTextData`][Token::CodeTextData],[`Data`][Token::Data],[`CharacterEscapeValue`][Token::CharacterEscapeValue]}.
fn on_exit_data(context: &mut CompileContext) {
// Just output it.
context.push(context.encode_opt(&serialize(
@@ -920,7 +915,7 @@ fn on_exit_data(context: &mut CompileContext) {
)));
}
-/// Handle [`Exit`][EventType::Exit]:[`Definition`][TokenType::Definition].
+/// Handle [`Exit`][EventType::Exit]:[`Definition`][Token::Definition].
fn on_exit_definition(context: &mut CompileContext) {
let definition = context.media_stack.pop().unwrap();
let reference_id = normalize_identifier(&definition.reference_id.unwrap());
@@ -935,7 +930,7 @@ fn on_exit_definition(context: &mut CompileContext) {
.or_insert(Definition { destination, title });
}
-/// Handle [`Exit`][EventType::Exit]:[`DefinitionDestinationString`][TokenType::DefinitionDestinationString].
+/// Handle [`Exit`][EventType::Exit]:[`DefinitionDestinationString`][Token::DefinitionDestinationString].
fn on_exit_definition_destination_string(context: &mut CompileContext) {
let buf = context.resume();
let definition = context.media_stack.last_mut().unwrap();
@@ -943,7 +938,7 @@ fn on_exit_definition_destination_string(context: &mut CompileContext) {
context.ignore_encode = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`DefinitionLabelString`][TokenType::DefinitionLabelString].
+/// Handle [`Exit`][EventType::Exit]:[`DefinitionLabelString`][Token::DefinitionLabelString].
fn on_exit_definition_label_string(context: &mut CompileContext) {
// Discard label, use the source content instead.
context.resume();
@@ -955,19 +950,19 @@ fn on_exit_definition_label_string(context: &mut CompileContext) {
));
}
-/// Handle [`Exit`][EventType::Exit]:[`DefinitionTitleString`][TokenType::DefinitionTitleString].
+/// Handle [`Exit`][EventType::Exit]:[`DefinitionTitleString`][Token::DefinitionTitleString].
fn on_exit_definition_title_string(context: &mut CompileContext) {
let buf = context.resume();
let definition = context.media_stack.last_mut().unwrap();
definition.title = Some(buf);
}
-/// Handle [`Exit`][EventType::Exit]:[`Strong`][TokenType::Emphasis].
+/// Handle [`Exit`][EventType::Exit]:[`Strong`][Token::Emphasis].
fn on_exit_emphasis(context: &mut CompileContext) {
context.tag("".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingAtx`][TokenType::HeadingAtx].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingAtx`][Token::HeadingAtx].
fn on_exit_heading_atx(context: &mut CompileContext) {
let rank = context
.atx_opening_sequence_size
@@ -977,7 +972,7 @@ fn on_exit_heading_atx(context: &mut CompileContext) {
context.tag(format!("", rank));
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxSequence`][TokenType::HeadingAtxSequence].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxSequence`][Token::HeadingAtxSequence].
fn on_exit_heading_atx_sequence(context: &mut CompileContext) {
// First fence we see.
if context.atx_opening_sequence_size.is_none() {
@@ -993,20 +988,20 @@ fn on_exit_heading_atx_sequence(context: &mut CompileContext) {
}
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxText`][TokenType::HeadingAtxText].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxText`][Token::HeadingAtxText].
fn on_exit_heading_atx_text(context: &mut CompileContext) {
let value = context.resume();
context.push(value);
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextText`][TokenType::HeadingSetextText].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextText`][Token::HeadingSetextText].
fn on_exit_heading_setext_text(context: &mut CompileContext) {
let buf = context.resume();
context.heading_setext_buffer = Some(buf);
context.slurp_one_line_ending = true;
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextUnderline`][Token::HeadingSetextUnderline].
fn on_exit_heading_setext_underline(context: &mut CompileContext) {
let text = context
.heading_setext_buffer
@@ -1024,12 +1019,12 @@ fn on_exit_heading_setext_underline(context: &mut CompileContext) {
context.tag(format!("", level));
}
-/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlow`][TokenType::HtmlFlow],[`HtmlText`][TokenType::HtmlText]}.
+/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlow`][Token::HtmlFlow],[`HtmlText`][Token::HtmlText]}.
fn on_exit_html(context: &mut CompileContext) {
context.ignore_encode = false;
}
-/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlowData`][TokenType::HtmlFlowData],[`HtmlTextData`][TokenType::HtmlTextData]}.
+/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlowData`][Token::HtmlFlowData],[`HtmlTextData`][Token::HtmlTextData]}.
fn on_exit_html_data(context: &mut CompileContext) {
let slice = serialize(
context.codes,
@@ -1039,14 +1034,14 @@ fn on_exit_html_data(context: &mut CompileContext) {
context.push(context.encode_opt(&slice));
}
-/// Handle [`Exit`][EventType::Exit]:[`Label`][TokenType::Label].
+/// Handle [`Exit`][EventType::Exit]:[`Label`][Token::Label].
fn on_exit_label(context: &mut CompileContext) {
let buf = context.resume();
let media = context.media_stack.last_mut().unwrap();
media.label = Some(buf);
}
-/// Handle [`Exit`][EventType::Exit]:[`LabelText`][TokenType::LabelText].
+/// Handle [`Exit`][EventType::Exit]:[`LabelText`][Token::LabelText].
fn on_exit_label_text(context: &mut CompileContext) {
let media = context.media_stack.last_mut().unwrap();
media.label_id = Some(serialize(
@@ -1056,7 +1051,7 @@ fn on_exit_label_text(context: &mut CompileContext) {
));
}
-/// Handle [`Exit`][EventType::Exit]:[`LineEnding`][TokenType::LineEnding].
+/// Handle [`Exit`][EventType::Exit]:[`LineEnding`][Token::LineEnding].
fn on_exit_line_ending(context: &mut CompileContext) {
// if slurp_all_line_endings {
// // Empty.
@@ -1072,7 +1067,7 @@ fn on_exit_line_ending(context: &mut CompileContext) {
}
}
-/// Handle [`Exit`][EventType::Exit]:{[`Image`][TokenType::Image],[`Link`][TokenType::Link]}.
+/// Handle [`Exit`][EventType::Exit]:{[`Image`][Token::Image],[`Link`][Token::Link]}.
fn on_exit_media(context: &mut CompileContext) {
let mut is_in_image = false;
let mut index = 0;
@@ -1136,12 +1131,12 @@ fn on_exit_media(context: &mut CompileContext) {
};
}
-/// Handle [`Exit`][EventType::Exit]:[`Paragraph`][TokenType::Paragraph].
+/// Handle [`Exit`][EventType::Exit]:[`Paragraph`][Token::Paragraph].
fn on_exit_paragraph(context: &mut CompileContext) {
context.tag("
".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`BlockQuote`][TokenType::BlockQuote].
+/// Handle [`Exit`][EventType::Exit]:[`BlockQuote`][Token::BlockQuote].
fn on_exit_block_quote(context: &mut CompileContext) {
// tightStack.pop()
context.line_ending_if_needed();
@@ -772,22 +767,22 @@ fn on_exit_block_quote(context: &mut CompileContext) {
// let mut slurp_all_line_endings = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarker`][Token::CharacterReferenceMarker].
fn on_exit_character_reference_marker(context: &mut CompileContext) {
context.character_reference_kind = Some(CharacterReferenceKind::Named);
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal].
fn on_exit_character_reference_marker_hexadecimal(context: &mut CompileContext) {
context.character_reference_kind = Some(CharacterReferenceKind::Hexadecimal);
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric].
fn on_exit_character_reference_marker_numeric(context: &mut CompileContext) {
context.character_reference_kind = Some(CharacterReferenceKind::Decimal);
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceValue`][TokenType::CharacterReferenceValue].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceValue`][Token::CharacterReferenceValue].
fn on_exit_character_reference_value(context: &mut CompileContext) {
let kind = context
.character_reference_kind
@@ -808,7 +803,7 @@ fn on_exit_character_reference_value(context: &mut CompileContext) {
context.push(context.encode_opt(&value));
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeFlowChunk`][TokenType::CodeFlowChunk].
+/// Handle [`Exit`][EventType::Exit]:[`CodeFlowChunk`][Token::CodeFlowChunk].
fn on_exit_code_flow_chunk(context: &mut CompileContext) {
context.code_flow_seen_data = Some(true);
context.push(context.encode_opt(&serialize(
@@ -818,7 +813,7 @@ fn on_exit_code_flow_chunk(context: &mut CompileContext) {
)));
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFence`][TokenType::CodeFencedFence].
+/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFence`][Token::CodeFencedFence].
fn on_exit_code_fenced_fence(context: &mut CompileContext) {
let count = if let Some(count) = context.code_fenced_fences_count {
count
@@ -834,13 +829,13 @@ fn on_exit_code_fenced_fence(context: &mut CompileContext) {
context.code_fenced_fences_count = Some(count + 1);
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo].
+/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo].
fn on_exit_code_fenced_fence_info(context: &mut CompileContext) {
let value = context.resume();
context.tag(format!(" class=\"language-{}\"", value));
}
-/// Handle [`Exit`][EventType::Exit]:{[`CodeFenced`][TokenType::CodeFenced],[`CodeIndented`][TokenType::CodeIndented]}.
+/// Handle [`Exit`][EventType::Exit]:{[`CodeFenced`][Token::CodeFenced],[`CodeIndented`][Token::CodeIndented]}.
fn on_exit_code_flow(context: &mut CompileContext) {
let seen_data = context
.code_flow_seen_data
@@ -874,7 +869,7 @@ fn on_exit_code_flow(context: &mut CompileContext) {
context.slurp_one_line_ending = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeText`][TokenType::CodeText].
+/// Handle [`Exit`][EventType::Exit]:[`CodeText`][Token::CodeText].
fn on_exit_code_text(context: &mut CompileContext) {
let result = context.resume();
let mut chars = result.chars();
@@ -898,7 +893,7 @@ fn on_exit_code_text(context: &mut CompileContext) {
context.tag("
) -> StateFnResult {
if stop.contains(&code) {
- tokenizer.enter(TokenType::Data);
+ tokenizer.enter(Token::Data);
tokenizer.consume(code);
(State::Fn(Box::new(|t, c| data(t, c, stop))), None)
} else {
@@ -33,9 +34,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec) -> StateFnRe
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| at_break(t, c, stop))), None)
}
_ if stop.contains(&code) => {
@@ -43,7 +44,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec) -> StateFnRe
(State::Ok, Some(vec![code]))
}
_ => {
- tokenizer.enter(TokenType::Data);
+ tokenizer.enter(Token::Data);
data(tokenizer, code, stop)
}
}
@@ -62,7 +63,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code, stop: Vec) -> StateFnResult
};
if done {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, stop)
} else {
tokenizer.consume(code);
@@ -80,13 +81,13 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec {
while index < len {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == TokenType::Data {
+ if event.event_type == EventType::Enter && event.token_type == Token::Data {
let exit_index = index + 1;
let mut exit_far_index = exit_index;
// Find multiple `data` events.
while exit_far_index + 1 < len
- && tokenizer.events[exit_far_index + 1].token_type == TokenType::Data
+ && tokenizer.events[exit_far_index + 1].token_type == Token::Data
{
exit_far_index += 2;
}
diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs
index 31c13ec..daa968a 100644
--- a/src/construct/partial_destination.rs
+++ b/src/construct/partial_destination.rs
@@ -71,7 +71,8 @@
//! [label_end]: crate::construct::label_end
//! [sanitize_uri]: crate::util::sanitize_uri
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -79,15 +80,15 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole destination.
- pub destination: TokenType,
+ pub destination: Token,
/// Token for a literal (enclosed) destination.
- pub literal: TokenType,
+ pub literal: Token,
/// Token for a literal marker.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for a raw destination.
- pub raw: TokenType,
+ pub raw: Token,
/// Token for a the string.
- pub string: TokenType,
+ pub string: Token,
/// Maximum unbalanced parens.
pub limit: usize,
}
@@ -133,7 +134,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code, options: Options) -> StateFn
tokenizer.enter(info.options.destination.clone());
tokenizer.enter(info.options.raw.clone());
tokenizer.enter(info.options.string.clone());
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
raw(tokenizer, code, info)
}
}
@@ -154,7 +155,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn
(State::Ok, None)
} else {
tokenizer.enter(info.options.string.clone());
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
enclosed(tokenizer, code, info)
}
}
@@ -167,7 +168,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn
fn enclosed(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
enclosed_before(tokenizer, code, info)
}
@@ -221,7 +222,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
}
Code::Char(')') => {
if info.balance == 0 {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
tokenizer.exit(info.options.raw.clone());
tokenizer.exit(info.options.destination);
@@ -239,7 +240,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
if info.balance > 0 {
(State::Nok, None)
} else {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
tokenizer.exit(info.options.raw.clone());
tokenizer.exit(info.options.destination);
diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs
index f201f60..f380c7d 100644
--- a/src/construct/partial_label.rs
+++ b/src/construct/partial_label.rs
@@ -61,7 +61,8 @@
use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::constant::LINK_REFERENCE_SIZE_MAX;
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -69,11 +70,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole label.
- pub label: TokenType,
+ pub label: Token,
/// Token for the markers.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for the string (inside the markers).
- pub string: TokenType,
+ pub string: Token,
}
/// State needed to parse labels.
@@ -144,7 +145,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
},
)(tokenizer, code),
_ => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -166,11 +167,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
fn label(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '[' | ']') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
_ if info.size > LINK_REFERENCE_SIZE_MAX => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::VirtualSpace | Code::Char('\t' | ' ') => {
diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs
index 5b1ec5e..78477de 100644
--- a/src/construct/partial_space_or_tab.rs
+++ b/src/construct/partial_space_or_tab.rs
@@ -5,7 +5,8 @@
//! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js)
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, Tokenizer};
/// Options to parse `space_or_tab`.
#[derive(Debug)]
@@ -15,7 +16,7 @@ pub struct Options {
/// Maximum allowed characters (inclusive).
pub max: usize,
/// Token type to use for whitespace events.
- pub kind: TokenType,
+ pub kind: Token,
/// Connect this whitespace to the previous.
pub connect: bool,
/// Embedded content type to use.
@@ -67,7 +68,7 @@ pub fn space_or_tab() -> Box {
/// ```
pub fn space_or_tab_min_max(min: usize, max: usize) -> Box {
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min,
max,
content_type: None,
@@ -104,7 +105,7 @@ pub fn space_or_tab_eol_with_options(options: EolOptions) -> Box {
tokenizer.attempt(
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: info.options.content_type,
@@ -196,7 +197,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResul
fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter_with_content(TokenType::LineEnding, info.options.content_type);
+ tokenizer.enter_with_content(Token::LineEnding, info.options.content_type);
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -206,7 +207,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo)
}
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| after_eol(t, c, info))), None)
}
_ if info.ok => (State::Ok, Some(vec![code])),
@@ -229,7 +230,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo)
fn after_eol(tokenizer: &mut Tokenizer, code: Code, info: EolInfo) -> StateFnResult {
tokenizer.attempt_opt(
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: info.options.content_type,
diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs
index 010f554..6303da8 100644
--- a/src/construct/partial_title.rs
+++ b/src/construct/partial_title.rs
@@ -32,7 +32,8 @@
use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -40,11 +41,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole title.
- pub title: TokenType,
+ pub title: Token,
/// Token for the marker.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for the string inside the quotes.
- pub string: TokenType,
+ pub string: Token,
}
/// Type of title.
@@ -204,7 +205,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
},
)(tokenizer, code),
_ => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -226,11 +227,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
fn title(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::Char('\\') => {
diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs
index 28aca34..4d92f8d 100644
--- a/src/construct/thematic_break.rs
+++ b/src/construct/thematic_break.rs
@@ -35,8 +35,8 @@
//!
//! ## Tokens
//!
-//! * [`ThematicBreak`][TokenType::ThematicBreak]
-//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence]
+//! * [`ThematicBreak`][Token::ThematicBreak]
+//! * [`ThematicBreakSequence`][Token::ThematicBreakSequence]
//!
//! ## References
//!
@@ -51,7 +51,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Type of thematic break.
#[derive(Debug, PartialEq)]
@@ -134,7 +135,7 @@ struct Info {
/// |***
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::ThematicBreak);
+ tokenizer.enter(Token::ThematicBreak);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -170,13 +171,13 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')
if info.size >= THEMATIC_BREAK_MARKER_COUNT_MIN =>
{
- tokenizer.exit(TokenType::ThematicBreak);
+ tokenizer.exit(Token::ThematicBreak);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
}
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.enter(TokenType::ThematicBreakSequence);
+ tokenizer.enter(Token::ThematicBreakSequence);
sequence(tokenizer, code, info)
}
_ => (State::Nok, None),
@@ -198,7 +199,7 @@ fn sequence(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
(State::Fn(Box::new(|t, c| sequence(t, c, info))), None)
}
_ => {
- tokenizer.exit(TokenType::ThematicBreakSequence);
+ tokenizer.exit(Token::ThematicBreakSequence);
tokenizer.attempt_opt(space_or_tab(), |t, c| at_break(t, c, info))(tokenizer, code)
}
}
diff --git a/src/content/document.rs b/src/content/document.rs
index dd5038f..b1f3083 100644
--- a/src/content/document.rs
+++ b/src/content/document.rs
@@ -14,9 +14,8 @@ use crate::construct::block_quote::{
use crate::content::flow::start as flow;
use crate::parser::ParseState;
use crate::subtokenize::subtokenize;
-use crate::tokenizer::{
- Code, Event, EventType, Point, State, StateFn, StateFnResult, TokenType, Tokenizer,
-};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, Point, State, StateFn, StateFnResult, Tokenizer};
use crate::util::edit_map::EditMap;
use crate::util::{
normalize_identifier::normalize_identifier,
@@ -44,9 +43,7 @@ pub fn document(parse_state: &mut ParseState, point: Point, index: usize) -> Vec
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Exit
- && event.token_type == TokenType::DefinitionLabelString
- {
+ if event.event_type == EventType::Exit && event.token_type == Token::DefinitionLabelString {
next_definitions.insert(normalize_identifier(
serialize(
&parse_state.codes,
@@ -409,7 +406,7 @@ fn flow_end(
// To do: blank lines? Other things?
if tokenizer.events.len() > 2
- && tokenizer.events[tokenizer.events.len() - 1].token_type == TokenType::LineEnding
+ && tokenizer.events[tokenizer.events.len() - 1].token_type == Token::LineEnding
{
info.last_line_ending_index = Some(tokenizer.events.len() - 2);
} else {
diff --git a/src/content/flow.rs b/src/content/flow.rs
index f406685..e52f113 100644
--- a/src/content/flow.rs
+++ b/src/content/flow.rs
@@ -26,7 +26,8 @@ use crate::construct::{
html_flow::start as html_flow, paragraph::start as paragraph,
thematic_break::start as thematic_break,
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Before flow.
///
@@ -88,9 +89,9 @@ fn blank_line_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::BlankLineEnding);
+ tokenizer.enter(Token::BlankLineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::BlankLineEnding);
+ tokenizer.exit(Token::BlankLineEnding);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Fn(Box::new(start)), None)
@@ -112,9 +113,9 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(start)), None)
}
_ => unreachable!("expected eol/eof"),
diff --git a/src/lib.rs b/src/lib.rs
index e1f26b4..1f49303 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,6 +10,7 @@ mod construct;
mod content;
mod parser;
mod subtokenize;
+mod token;
mod tokenizer;
mod unicode;
mod util;
diff --git a/src/token.rs b/src/token.rs
new file mode 100644
index 0000000..9b59719
--- /dev/null
+++ b/src/token.rs
@@ -0,0 +1,1768 @@
+/// Semantic label of a span.
+#[derive(Debug, Clone, PartialEq, Hash, Eq)]
+pub enum Token {
+ /// Attention sequence.
+ ///
+ /// > 👉 **Note**: this is used while parsing but compiled away.
+ AttentionSequence,
+ /// Whole autolink.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`AutolinkEmail`][Token::AutolinkEmail],
+ /// [`AutolinkMarker`][Token::AutolinkMarker],
+ /// [`AutolinkProtocol`][Token::AutolinkProtocol]
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | and
+ /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
+ /// ```
+ Autolink,
+ /// Email autolink w/o markers.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][Token::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > |
+ /// ^^^^^^^^^^^^^^^^^
+ /// ```
+ AutolinkEmail,
+ /// Marker of an autolink.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][Token::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > |
+ /// ^ ^
+ /// ```
+ AutolinkMarker,
+ /// Protocol autolink w/o markers.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][Token::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > |
+ /// ^^^^^^^^^^^^^^^^^^^
+ /// ```
+ AutolinkProtocol,
+ /// Line ending preceded only by whitespace or nothing at all.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`blank_line`][crate::construct::blank_line]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | ␠␠␊
+ /// ^
+ /// ```
+ BlankLineEnding,
+ /// Whole block quote.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [document content][crate::content::document]
+ /// * **Content model**:
+ /// [`BlockQuotePrefix`][Token::BlockQuotePrefix],
+ /// [flow content][crate::content::flow]
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^^^
+ /// > | b
+ /// ^
+ /// ```
+ BlockQuote,
+ /// Block quote marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`BlockQuotePrefix`][Token::BlockQuotePrefix]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^
+ /// | b
+ /// ```
+ BlockQuoteMarker,
+ /// Block quote prefix.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`BlockQuote`][Token::BlockQuote]
+ /// * **Content model**:
+ /// [`BlockQuoteMarker`][Token::BlockQuoteMarker],
+ /// [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace]
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^^
+ /// | b
+ /// ```
+ BlockQuotePrefix,
+ /// Block quote white space.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`BlockQuotePrefix`][Token::BlockQuotePrefix]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^
+ /// | b
+ /// ```
+ BlockQuoteWhitespace,
+ /// Whole character escape.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string] or
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CharacterEscapeMarker`][Token::CharacterEscapeMarker],
+ /// [`CharacterEscapeValue`][Token::CharacterEscapeValue]
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^^
+ /// ```
+ CharacterEscape,
+ /// Character escape marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterEscape`][Token::CharacterEscape]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^
+ /// ```
+ CharacterEscapeMarker,
+ /// Character escape value.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterEscape`][Token::CharacterEscape]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^
+ /// ```
+ CharacterEscapeValue,
+ /// Whole character reference.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string] or
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CharacterReferenceMarker`][Token::CharacterReferenceMarker],
+ /// [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal],
+ /// [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric],
+ /// [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi],
+ /// [`CharacterReferenceValue`][Token::CharacterReferenceValue]
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a & b ≠ c 𝌆 d
+ /// ^^^^^ ^^^^^^^ ^^^^^^^^^
+ /// ```
+ CharacterReference,
+ /// Character reference opening marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a & b ≠ c 𝌆 d
+ /// ^ ^ ^
+ /// ```
+ CharacterReferenceMarker,
+ /// Character reference hexadecimal numeric marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a & b ≠ c 𝌆 d
+ /// ^
+ /// ```
+ CharacterReferenceMarkerHexadecimal,
+ /// Character reference numeric marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a & b ≠ c 𝌆 d
+ /// ^ ^
+ /// ```
+ CharacterReferenceMarkerNumeric,
+ /// Character reference closing marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a & b ≠ c 𝌆 d
+ /// ^ ^ ^
+ /// ```
+ CharacterReferenceMarkerSemi,
+ /// Character reference value.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a & b ≠ c 𝌆 d
+ /// ^^^ ^^^^ ^^^^^
+ /// ```
+ CharacterReferenceValue,
+ /// Whole code (fenced).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`CodeFencedFence`][Token::CodeFencedFence],
+ /// [`CodeFlowChunk`][Token::CodeFlowChunk],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^^^
+ /// > | console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// > | ```
+ /// ^^^
+ /// ````
+ CodeFenced,
+ /// A code (fenced) fence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFenced`][Token::CodeFenced]
+ /// * **Content model**:
+ /// [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo],
+ /// [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta],
+ /// [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^^^
+ /// | console.log(1)
+ /// > | ```
+ /// ^^^
+ /// ````
+ CodeFencedFence,
+ /// A code (fenced) fence info word.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFence`][Token::CodeFencedFence]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^
+ /// | console.log(1)
+ /// | ```
+ /// ````
+ CodeFencedFenceInfo,
+ /// A code (fenced) fence meta string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFence`][Token::CodeFencedFence]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js highlight="1"
+ /// ^^^^^^^^^^^^^
+ /// | console.log(1)
+ /// | ```
+ /// ````
+ CodeFencedFenceMeta,
+ /// A code (fenced) fence sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^
+ /// | console.log(1)
+ /// > | ```
+ /// ^^^
+ /// ````
+ CodeFencedFenceSequence,
+ /// A code (fenced, indented) chunk.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFenced`][Token::CodeFenced],
+ /// [`CodeIndented`][Token::CodeIndented]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced],
+ /// [`code_indented`][crate::construct::code_indented]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// | ```js
+ /// > | console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// | ```
+ /// ````
+ ///
+ /// ```markdown
+ /// > | ␠␠␠␠console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// ```
+ CodeFlowChunk,
+ /// Whole code (indented).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`CodeFlowChunk`][Token::CodeFlowChunk],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// ␠␠␠␠console.log(1)
+ /// ^^^^^^^^^^^^^^^^^^
+ /// ```
+ CodeIndented,
+ /// Whole code (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CodeTextData`][Token::CodeTextData],
+ /// [`CodeTextSequence`][Token::CodeTextSequence],
+ /// [`CodeTextLineEnding`][Token::CodeTextLineEnding]
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^^^
+ /// ```
+ CodeText,
+ /// Code (text) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][Token::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^
+ /// ```
+ CodeTextData,
+ /// Line ending in code (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][Token::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b␊
+ /// ^
+ /// | c` d
+ /// ```
+ CodeTextLineEnding,
+ /// Code (text) sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][Token::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^ ^
+ /// ```
+ CodeTextSequence,
+ /// Data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string],
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`data`][crate::construct::partial_data]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | aa *bb* cc
+ /// ^^^ ^^ ^^^
+ /// ```
+ Data,
+ /// Whole definition.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`DefinitionMarker`][Token::DefinitionMarker],
+ /// [`DefinitionLabel`][Token::DefinitionLabel],
+ /// [`DefinitionDestination`][Token::DefinitionDestination],
+ /// [`DefinitionTitle`][Token::DefinitionTitle],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`definition`][crate::construct::definition]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^^^^^^^^
+ /// ```
+ Definition,
+ /// Whole definition destination.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral],
+ /// [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// > | [a]: "c"
+ /// ^^^
+ /// ```
+ DefinitionDestination,
+ /// Definition destination literal.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestination`][Token::DefinitionDestination]
+ /// * **Content model**:
+ /// [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker],
+ /// [`DefinitionDestinationString`][Token::DefinitionDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: "c"
+ /// ^^^
+ /// ```
+ DefinitionDestinationLiteral,
+ /// Definition destination literal marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: "c"
+ /// ^ ^
+ /// ```
+ DefinitionDestinationLiteralMarker,
+ /// Definition destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestination`][Token::DefinitionDestination]
+ /// * **Content model**:
+ /// [`DefinitionDestinationString`][Token::DefinitionDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionDestinationRaw,
+ /// Definition destination data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral],
+ /// [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// > | [a]: "c"
+ /// ^
+ /// ```
+ DefinitionDestinationString,
+ /// Whole definition label.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// [`DefinitionLabelMarker`][Token::DefinitionLabelMarker],
+ /// [`DefinitionLabelString`][Token::DefinitionLabelString],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^
+ /// ```
+ DefinitionLabel,
+ /// Definition label marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionLabel`][Token::DefinitionLabel]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^ ^
+ /// ```
+ DefinitionLabelMarker,
+ /// Definition label data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionLabel`][Token::DefinitionLabel]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionLabelString,
+ /// Definition marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`definition`][crate::construct::definition]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionMarker,
+ /// Whole definition title.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// [`DefinitionTitleMarker`][Token::DefinitionTitleMarker],
+ /// [`DefinitionTitleString`][Token::DefinitionTitleString],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^
+ /// ```
+ DefinitionTitle,
+ /// Definition title marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionTitle`][Token::DefinitionTitle]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^ ^
+ /// ```
+ DefinitionTitleMarker,
+ /// Definition title data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionTitle`][Token::DefinitionTitle]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionTitleString,
+ /// Emphasis.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`EmphasisSequence`][Token::EmphasisSequence],
+ /// [`EmphasisText`][Token::EmphasisText]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | *a*
+ /// ^^^
+ /// ```
+ Emphasis,
+ /// Emphasis sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Emphasis`][Token::Emphasis]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | *a*
+ /// ^ ^
+ /// ```
+ EmphasisSequence,
+ /// Emphasis text.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Emphasis`][Token::Emphasis]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | *a*
+ /// ^
+ /// ```
+ EmphasisText,
+ /// Whole hard break (escape).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker]
+ /// * **Construct**:
+ /// [`hard_break_escape`][crate::construct::hard_break_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a\␊
+ /// ^^
+ /// > | b
+ /// ```
+ HardBreakEscape,
+ /// Hard break (escape) marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`hard_break_escape`][crate::construct::hard_break_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a\␊
+ /// ^
+ /// > | b
+ /// ```
+ HardBreakEscapeMarker,
+ /// Whole hard break (trailing).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace]
+ /// * **Construct**:
+ /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␠␠␊
+ /// ^^^
+ /// > | b
+ /// ```
+ HardBreakTrailing,
+ /// Hard break (trailing) spaces.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HardBreakTrailing`][Token::HardBreakTrailing]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␠␠␊
+ /// ^^
+ /// > | b
+ /// ```
+ HardBreakTrailingSpace,
+ /// Whole heading (atx).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HeadingAtxSequence`][Token::HeadingAtxSequence],
+ /// [`HeadingAtxText`][Token::HeadingAtxText],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^^^^^^^
+ /// ```
+ HeadingAtx,
+ /// Heading (atx) sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][Token::HeadingAtx]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^
+ /// ```
+ HeadingAtxSequence,
+ /// Heading (atx) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][Token::HeadingAtx],
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^^^^^
+ /// ```
+ HeadingAtxText,
+ /// Whole heading (setext).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HeadingSetextText`][Token::HeadingSetextText],
+ /// [`HeadingSetextUnderline`][Token::HeadingSetextUnderline],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | alpha
+ /// ^^^^^
+ /// > | =====
+ /// ^^^^^
+ /// ```
+ HeadingSetext,
+ /// Heading (setext) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingSetext`][Token::HeadingSetext]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | alpha
+ /// ^^^^^
+ /// | =====
+ /// ```
+ HeadingSetextText,
+ /// Heading (setext) underline.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingSetext`][Token::HeadingSetext]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// | alpha
+ /// > | =====
+ /// ^^^^^
+ /// ```
+ HeadingSetextUnderline,
+ /// Whole html (flow).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HtmlFlowData`][Token::HtmlFlowData],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`html_flow`][crate::construct::html_flow]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > |
+ /// ^^^^^
+ /// ```
+ HtmlFlow,
+ /// HTML (flow) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HtmlFlow`][Token::HtmlFlow],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`html_flow`][crate::construct::html_flow]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > |
+ /// ^^^^^
+ /// ```
+ HtmlFlowData,
+ /// Whole html (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HtmlTextData`][Token::HtmlTextData],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`html_text`][crate::construct::html_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a c
+ /// ^^^
+ /// ```
+ HtmlText,
+ /// HTML (text) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HtmlText`][Token::HtmlText]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`html_text`][crate::construct::html_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a c
+ /// ^^^
+ /// ```
+ HtmlTextData,
+ /// Image.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`Label`][Token::Label],
+ /// [`Resource`][Token::Resource],
+ /// [`Reference`][Token::Reference]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b] c
+ /// ^^^^
+ /// > | a ![b][c] d
+ /// ^^^^^^^
+ /// > | a ![b](c) d
+ /// ^^^^^^^
+ /// ```
+ Image,
+ /// Label.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Image`][Token::Image],
+ /// [`Link`][Token::Link]
+ /// * **Content model**:
+ /// [`LabelImage`][Token::LabelImage],
+ /// [`LabelLink`][Token::LabelLink],
+ /// [`LabelEnd`][Token::LabelEnd],
+ /// [`LabelText`][Token::LabelText]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b] c
+ /// ^^^
+ /// > | a ![b][c] d
+ /// ^^^^
+ /// > | a [b](c) d
+ /// ^^^
+ /// ```
+ Label,
+ /// Label end.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [`LabelMarker`][Token::LabelMarker]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^
+ /// > | a [b](c) d
+ /// ^
+ /// ```
+ LabelEnd,
+ /// Label start (image).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [`LabelImageMarker`][Token::LabelImageMarker],
+ /// [`LabelMarker`][Token::LabelMarker]
+ /// * **Construct**:
+ /// [`label_start_image`][crate::construct::label_start_image]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^^
+ /// ```
+ LabelImage,
+ /// Label start (image) marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`LabelImage`][Token::LabelImage]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label_start_image`][crate::construct::label_start_image]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^
+ /// ```
+ LabelImageMarker,
+ /// Label start (link).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [`LabelMarker`][Token::LabelMarker]
+ /// * **Construct**:
+ /// [`label_start_link`][crate::construct::label_start_link]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b](c) d
+ /// ^
+ /// ```
+ LabelLink,
+ /// Label marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`LabelImage`][Token::LabelImage],
+ /// [`LabelLink`][Token::LabelLink],
+ /// [`LabelEnd`][Token::LabelEnd]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label_start_image`][crate::construct::label_start_image],
+ /// [`label_start_link`][crate::construct::label_start_link],
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^ ^
+ /// > | a [b](c) d
+ /// ^ ^
+ /// ```
+ LabelMarker,
+ /// Label text.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b] c
+ /// ^
+ /// > | a ![b][c] d
+ /// ^
+ /// > | a [b](c) d
+ /// ^
+ /// ```
+ LabelText,
+ /// Line ending.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// basically everywhere
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// n/a
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␊
+ /// ^
+ /// | b
+ /// ```
+ LineEnding,
+ /// Link.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`Label`][Token::Label],
+ /// [`Resource`][Token::Resource],
+ /// [`Reference`][Token::Reference]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b] c
+ /// ^^^
+ /// > | a [b][c] d
+ /// ^^^^^^
+ /// > | a [b](c) d
+ /// ^^^^^^
+ /// ```
+ Link,
+ /// Whole paragraph.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`paragraph`][crate::construct::paragraph]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a b
+ /// ^^^
+ /// > | c.
+ /// ^^
+ /// ```
+ Paragraph,
+ /// Reference.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Image`][Token::Image],
+ /// [`Link`][Token::Link]
+ /// * **Content model**:
+ /// [`ReferenceMarker`][Token::ReferenceMarker],
+ /// [`ReferenceString`][Token::ReferenceString]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b][c] d
+ /// ^^^
+ /// ```
+ Reference,
+ /// Reference marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Reference`][Token::Reference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b][c] d
+ /// ^ ^
+ /// ```
+ ReferenceMarker,
+ /// Reference string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Reference`][Token::Reference]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b][c] d
+ /// ^
+ /// ```
+ ReferenceString,
+ /// Resource.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Image`][Token::Image],
+ /// [`Link`][Token::Link]
+ /// * **Content model**:
+ /// [`ResourceMarker`][Token::ResourceMarker],
+ /// [`ResourceDestination`][Token::ResourceDestination],
+ /// [`ResourceTitle`][Token::ResourceTitle],
+ /// [`SpaceOrTab`][Token::SpaceOrTab],
+ /// [`LineEnding`][Token::LineEnding]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^^^^^^^
+ /// > | a [b](c) d
+ /// ^^^
+ /// ```
+ Resource,
+ /// Resource destination.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Resource`][Token::Resource]
+ /// * **Content model**:
+ /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral],
+ /// [`ResourceDestinationRaw`][Token::ResourceDestinationRaw]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^
+ /// ```
+ ResourceDestination,
+ /// Resource destination literal.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestination`][Token::ResourceDestination]
+ /// * **Content model**:
+ /// [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker],
+ /// [`ResourceDestinationString`][Token::ResourceDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b]( "d") e
+ /// ^^^
+ /// ```
+ ResourceDestinationLiteral,
+ /// Resource destination literal marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b]( "d") e
+ /// ^ ^
+ /// ```
+ ResourceDestinationLiteralMarker,
+ /// Resource destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestination`][Token::ResourceDestination]
+ /// * **Content model**:
+ /// [`ResourceDestinationString`][Token::ResourceDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^
+ /// ```
+ ResourceDestinationRaw,
+ /// Resource destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral],
+ /// [`ResourceDestinationRaw`][Token::ResourceDestinationRaw]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b]( "d") e
+ /// ^
+ /// > | a ![b](c "d") e
+ /// ^
+ /// ```
+ ResourceDestinationString,
+ /// Resource marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Resource`][Token::Resource]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^ ^
+ /// ```
+ ResourceMarker,
+ /// Resource title.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Resource`][Token::Resource]
+ /// * **Content model**:
+ /// [`ResourceTitleMarker`][Token::ResourceTitleMarker],
+ /// [`ResourceTitleString`][Token::ResourceTitleString]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b]( "d") e
+ /// ^^^
+ /// ```
+ ResourceTitle,
+ /// Resource title marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceTitle`][Token::ResourceTitle]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b]( "d") e
+ /// ^ ^
+ /// ```
+ ResourceTitleMarker,
+ /// Resource title string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceTitle`][Token::ResourceTitle]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b]( "d") e
+ /// ^
+ /// ```
+ ResourceTitleString,
+ /// SpaceOrTab.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// basically everywhere
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// n/a
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | ␠* * *␠
+ /// ^ ^ ^ ^
+ /// ```
+ SpaceOrTab,
+ /// Strong.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`StrongSequence`][Token::StrongSequence],
+ /// [`StrongText`][Token::StrongText]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | **a**
+ /// ^^^^^
+ /// ```
+ Strong,
+ /// Strong sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Strong`][Token::Strong]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | **a**
+ /// ^^ ^^
+ /// ```
+ StrongSequence,
+ /// Strong text.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Strong`][Token::Strong]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | **a**
+ /// ^
+ /// ```
+ StrongText,
+ /// Whole thematic break.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`ThematicBreakSequence`][Token::ThematicBreakSequence],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`thematic_break`][crate::construct::thematic_break]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | * * *
+ /// ^^^^^
+ /// ```
+ ThematicBreak,
+ /// Thematic break sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ThematicBreak`][Token::ThematicBreak]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`thematic_break`][crate::construct::thematic_break]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | * * *
+ /// ^ ^ ^
+ /// ```
+ ThematicBreakSequence,
+}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 6198776..64b66cc 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -12,1777 +12,9 @@
//! [`check`]: Tokenizer::check
use crate::parser::ParseState;
+use crate::token::Token;
use std::collections::HashMap;
-/// Semantic label of a span.
-#[derive(Debug, Clone, PartialEq, Hash, Eq)]
-pub enum TokenType {
- /// Attention sequence.
- ///
- /// > 👉 **Note**: this is used while parsing but compiled away.
- AttentionSequence,
- /// Whole autolink.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`AutolinkEmail`][TokenType::AutolinkEmail],
- /// [`AutolinkMarker`][TokenType::AutolinkMarker],
- /// [`AutolinkProtocol`][TokenType::AutolinkProtocol]
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | and
- /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
- /// ```
- Autolink,
- /// Email autolink w/o markers.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Autolink`][TokenType::Autolink]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > |
- /// ^^^^^^^^^^^^^^^^^
- /// ```
- AutolinkEmail,
- /// Marker of an autolink.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Autolink`][TokenType::Autolink]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > |
- /// ^ ^
- /// ```
- AutolinkMarker,
- /// Protocol autolink w/o markers.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Autolink`][TokenType::Autolink]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > |
- /// ^^^^^^^^^^^^^^^^^^^
- /// ```
- AutolinkProtocol,
- /// Line ending preceded only by whitespace or nothing at all.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`blank_line`][crate::construct::blank_line]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | ␠␠␊
- /// ^
- /// ```
- BlankLineEnding,
- /// Whole block quote.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [document content][crate::content::document]
- /// * **Content model**:
- /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix],
- /// [flow content][crate::content::flow]
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^^^
- /// > | b
- /// ^
- /// ```
- BlockQuote,
- /// Block quote marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^
- /// | b
- /// ```
- BlockQuoteMarker,
- /// Block quote prefix.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`BlockQuote`][TokenType::BlockQuote]
- /// * **Content model**:
- /// [`BlockQuoteMarker`][TokenType::BlockQuoteMarker],
- /// [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace]
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^^
- /// | b
- /// ```
- BlockQuotePrefix,
- /// Block quote white space.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^
- /// | b
- /// ```
- BlockQuoteWhitespace,
- /// Whole character escape.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [string content][crate::content::string] or
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker],
- /// [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
- /// * **Construct**:
- /// [`character_escape`][crate::construct::character_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a \- b
- /// ^^
- /// ```
- CharacterEscape,
- /// Character escape marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterEscape`][TokenType::CharacterEscape]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_escape`][crate::construct::character_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a \- b
- /// ^
- /// ```
- CharacterEscapeMarker,
- /// Character escape value.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterEscape`][TokenType::CharacterEscape]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_escape`][crate::construct::character_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a \- b
- /// ^
- /// ```
- CharacterEscapeValue,
- /// Whole character reference.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [string content][crate::content::string] or
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker],
- /// [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal],
- /// [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric],
- /// [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi],
- /// [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a & b ≠ c 𝌆 d
- /// ^^^^^ ^^^^^^^ ^^^^^^^^^
- /// ```
- CharacterReference,
- /// Character reference opening marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a & b ≠ c 𝌆 d
- /// ^ ^ ^
- /// ```
- CharacterReferenceMarker,
- /// Character reference hexadecimal numeric marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a & b ≠ c 𝌆 d
- /// ^
- /// ```
- CharacterReferenceMarkerHexadecimal,
- /// Character reference numeric marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a & b ≠ c 𝌆 d
- /// ^ ^
- /// ```
- CharacterReferenceMarkerNumeric,
- /// Character reference closing marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a & b ≠ c 𝌆 d
- /// ^ ^ ^
- /// ```
- CharacterReferenceMarkerSemi,
- /// Character reference value.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a & b ≠ c 𝌆 d
- /// ^^^ ^^^^ ^^^^^
- /// ```
- CharacterReferenceValue,
- /// Whole code (fenced).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`CodeFencedFence`][TokenType::CodeFencedFence],
- /// [`CodeFlowChunk`][TokenType::CodeFlowChunk],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^^^^
- /// > | console.log(1)
- /// ^^^^^^^^^^^^^^
- /// > | ```
- /// ^^^
- /// ````
- CodeFenced,
- /// A code (fenced) fence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFenced`][TokenType::CodeFenced]
- /// * **Content model**:
- /// [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo],
- /// [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta],
- /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^^^^
- /// | console.log(1)
- /// > | ```
- /// ^^^
- /// ````
- CodeFencedFence,
- /// A code (fenced) fence info word.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFencedFence`][TokenType::CodeFencedFence]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^
- /// | console.log(1)
- /// | ```
- /// ````
- CodeFencedFenceInfo,
- /// A code (fenced) fence meta string.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFencedFence`][TokenType::CodeFencedFence]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js highlight="1"
- /// ^^^^^^^^^^^^^
- /// | console.log(1)
- /// | ```
- /// ````
- CodeFencedFenceMeta,
- /// A code (fenced) fence sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^^
- /// | console.log(1)
- /// > | ```
- /// ^^^
- /// ````
- CodeFencedFenceSequence,
- /// A code (fenced, indented) chunk.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFenced`][TokenType::CodeFenced],
- /// [`CodeIndented`][TokenType::CodeIndented]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced],
- /// [`code_indented`][crate::construct::code_indented]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// | ```js
- /// > | console.log(1)
- /// ^^^^^^^^^^^^^^
- /// | ```
- /// ````
- ///
- /// ```markdown
- /// > | ␠␠␠␠console.log(1)
- /// ^^^^^^^^^^^^^^
- /// ```
- CodeFlowChunk,
- /// Whole code (indented).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`CodeFlowChunk`][TokenType::CodeFlowChunk],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// ␠␠␠␠console.log(1)
- /// ^^^^^^^^^^^^^^^^^^
- /// ```
- CodeIndented,
- /// Whole code (text).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`CodeTextData`][TokenType::CodeTextData],
- /// [`CodeTextSequence`][TokenType::CodeTextSequence],
- /// [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b` c
- /// ^^^
- /// ```
- CodeText,
- /// Code (text) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeText`][TokenType::CodeText],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b` c
- /// ^
- /// ```
- CodeTextData,
- /// Line ending in code (text).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeText`][TokenType::CodeText],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b␊
- /// ^
- /// | c` d
- /// ```
- CodeTextLineEnding,
- /// Code (text) sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeText`][TokenType::CodeText],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b` c
- /// ^ ^
- /// ```
- CodeTextSequence,
- /// Data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [string content][crate::content::string],
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`data`][crate::construct::partial_data]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | aa *bb* cc
- /// ^^^ ^^ ^^^
- /// ```
- Data,
- /// Whole definition.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`DefinitionMarker`][TokenType::DefinitionMarker],
- /// [`DefinitionLabel`][TokenType::DefinitionLabel],
- /// [`DefinitionDestination`][TokenType::DefinitionDestination],
- /// [`DefinitionTitle`][TokenType::DefinitionTitle],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`definition`][crate::construct::definition]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^^^^^^^^^^
- /// ```
- Definition,
- /// Whole definition destination.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral],
- /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// > | [a]: "c"
- /// ^^^
- /// ```
- DefinitionDestination,
- /// Definition destination literal.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestination`][TokenType::DefinitionDestination]
- /// * **Content model**:
- /// [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker],
- /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: "c"
- /// ^^^
- /// ```
- DefinitionDestinationLiteral,
- /// Definition destination literal marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: "c"
- /// ^ ^
- /// ```
- DefinitionDestinationLiteralMarker,
- /// Definition destination raw.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestination`][TokenType::DefinitionDestination]
- /// * **Content model**:
- /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionDestinationRaw,
- /// Definition destination data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral],
- /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// > | [a]: "c"
- /// ^
- /// ```
- DefinitionDestinationString,
- /// Whole definition label.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker],
- /// [`DefinitionLabelString`][TokenType::DefinitionLabelString],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^^^
- /// ```
- DefinitionLabel,
- /// Definition label marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionLabel`][TokenType::DefinitionLabel]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^ ^
- /// ```
- DefinitionLabelMarker,
- /// Definition label data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionLabel`][TokenType::DefinitionLabel]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionLabelString,
- /// Definition marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`definition`][crate::construct::definition]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionMarker,
- /// Whole definition title.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker],
- /// [`DefinitionTitleString`][TokenType::DefinitionTitleString],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^^^
- /// ```
- DefinitionTitle,
- /// Definition title marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionTitle`][TokenType::DefinitionTitle]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^ ^
- /// ```
- DefinitionTitleMarker,
- /// Definition title data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionTitle`][TokenType::DefinitionTitle]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionTitleString,
- /// Emphasis.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`EmphasisSequence`][TokenType::EmphasisSequence],
- /// [`EmphasisText`][TokenType::EmphasisText]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | *a*
- /// ^^^
- /// ```
- Emphasis,
- /// Emphasis sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Emphasis`][TokenType::Emphasis]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | *a*
- /// ^ ^
- /// ```
- EmphasisSequence,
- /// Emphasis text.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Emphasis`][TokenType::Emphasis]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | *a*
- /// ^
- /// ```
- EmphasisText,
- /// Whole hard break (escape).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
- /// * **Construct**:
- /// [`hard_break_escape`][crate::construct::hard_break_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a\␊
- /// ^^
- /// > | b
- /// ```
- HardBreakEscape,
- /// Hard break (escape) marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`hard_break_escape`][crate::construct::hard_break_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a\␊
- /// ^
- /// > | b
- /// ```
- HardBreakEscapeMarker,
- /// Whole hard break (trailing).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
- /// * **Construct**:
- /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a␠␠␊
- /// ^^^
- /// > | b
- /// ```
- HardBreakTrailing,
- /// Hard break (trailing) spaces.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HardBreakTrailing`][TokenType::HardBreakTrailing]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a␠␠␊
- /// ^^
- /// > | b
- /// ```
- HardBreakTrailingSpace,
- /// Whole heading (atx).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`HeadingAtxSequence`][TokenType::HeadingAtxSequence],
- /// [`HeadingAtxText`][TokenType::HeadingAtxText],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`heading_atx`][crate::construct::heading_atx]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | # alpha
- /// ^^^^^^^
- /// ```
- HeadingAtx,
- /// Heading (atx) sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingAtx`][TokenType::HeadingAtx]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`heading_atx`][crate::construct::heading_atx]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | # alpha
- /// ^
- /// ```
- HeadingAtxSequence,
- /// Heading (atx) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingAtx`][TokenType::HeadingAtx],
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`heading_atx`][crate::construct::heading_atx]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | # alpha
- /// ^^^^^
- /// ```
- HeadingAtxText,
- /// Whole heading (setext).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`HeadingSetextText`][TokenType::HeadingSetextText],
- /// [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`heading_setext`][crate::construct::heading_setext]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | alpha
- /// ^^^^^
- /// > | =====
- /// ^^^^^
- /// ```
- HeadingSetext,
- /// Heading (setext) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingSetext`][TokenType::HeadingSetext]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`heading_setext`][crate::construct::heading_setext]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | alpha
- /// ^^^^^
- /// | =====
- /// ```
- HeadingSetextText,
- /// Heading (setext) underline.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingSetext`][TokenType::HeadingSetext]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`heading_setext`][crate::construct::heading_setext]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// | alpha
- /// > | =====
- /// ^^^^^
- /// ```
- HeadingSetextUnderline,
- /// Whole html (flow).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`HtmlFlowData`][TokenType::HtmlFlowData],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`html_flow`][crate::construct::html_flow]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > |
- /// ^^^^^
- /// ```
- HtmlFlow,
- /// HTML (flow) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HtmlFlow`][TokenType::HtmlFlow],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`html_flow`][crate::construct::html_flow]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > |
- /// ^^^^^
- /// ```
- HtmlFlowData,
- /// Whole html (text).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`HtmlTextData`][TokenType::HtmlTextData],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`html_text`][crate::construct::html_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a c
- /// ^^^
- /// ```
- HtmlText,
- /// HTML (text) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HtmlText`][TokenType::HtmlText]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`html_text`][crate::construct::html_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a c
- /// ^^^
- /// ```
- HtmlTextData,
- /// Image.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`Label`][TokenType::Label],
- /// [`Resource`][TokenType::Resource],
- /// [`Reference`][TokenType::Reference]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b] c
- /// ^^^^
- /// > | a ![b][c] d
- /// ^^^^^^^
- /// > | a ![b](c) d
- /// ^^^^^^^
- /// ```
- Image,
- /// Label.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Image`][TokenType::Image],
- /// [`Link`][TokenType::Link]
- /// * **Content model**:
- /// [`LabelImage`][TokenType::LabelImage],
- /// [`LabelLink`][TokenType::LabelLink],
- /// [`LabelEnd`][TokenType::LabelEnd],
- /// [`LabelText`][TokenType::LabelText]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b] c
- /// ^^^
- /// > | a ![b][c] d
- /// ^^^^
- /// > | a [b](c) d
- /// ^^^
- /// ```
- Label,
- /// Label end.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [`LabelMarker`][TokenType::LabelMarker]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^
- /// > | a [b](c) d
- /// ^
- /// ```
- LabelEnd,
- /// Label start (image).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [`LabelImageMarker`][TokenType::LabelImageMarker],
- /// [`LabelMarker`][TokenType::LabelMarker]
- /// * **Construct**:
- /// [`label_start_image`][crate::construct::label_start_image]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^^
- /// ```
- LabelImage,
- /// Label start (image) marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`LabelImage`][TokenType::LabelImage]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label_start_image`][crate::construct::label_start_image]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^
- /// ```
- LabelImageMarker,
- /// Label start (link).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [`LabelMarker`][TokenType::LabelMarker]
- /// * **Construct**:
- /// [`label_start_link`][crate::construct::label_start_link]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b](c) d
- /// ^
- /// ```
- LabelLink,
- /// Label marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`LabelImage`][TokenType::LabelImage],
- /// [`LabelLink`][TokenType::LabelLink],
- /// [`LabelEnd`][TokenType::LabelEnd]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label_start_image`][crate::construct::label_start_image],
- /// [`label_start_link`][crate::construct::label_start_link],
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^ ^
- /// > | a [b](c) d
- /// ^ ^
- /// ```
- LabelMarker,
- /// Label text.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b] c
- /// ^
- /// > | a ![b][c] d
- /// ^
- /// > | a [b](c) d
- /// ^
- /// ```
- LabelText,
- /// Line ending.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// basically everywhere
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// n/a
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a␊
- /// ^
- /// | b
- /// ```
- LineEnding,
- /// Link.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`Label`][TokenType::Label],
- /// [`Resource`][TokenType::Resource],
- /// [`Reference`][TokenType::Reference]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b] c
- /// ^^^
- /// > | a [b][c] d
- /// ^^^^^^
- /// > | a [b](c) d
- /// ^^^^^^
- /// ```
- Link,
- /// Whole paragraph.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`paragraph`][crate::construct::paragraph]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a b
- /// ^^^
- /// > | c.
- /// ^^
- /// ```
- Paragraph,
- /// Reference.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Image`][TokenType::Image],
- /// [`Link`][TokenType::Link]
- /// * **Content model**:
- /// [`ReferenceMarker`][TokenType::ReferenceMarker],
- /// [`ReferenceString`][TokenType::ReferenceString]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b][c] d
- /// ^^^
- /// ```
- Reference,
- /// Reference marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Reference`][TokenType::Reference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b][c] d
- /// ^ ^
- /// ```
- ReferenceMarker,
- /// Reference string.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Reference`][TokenType::Reference]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b][c] d
- /// ^
- /// ```
- ReferenceString,
- /// Resource.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Image`][TokenType::Image],
- /// [`Link`][TokenType::Link]
- /// * **Content model**:
- /// [`ResourceMarker`][TokenType::ResourceMarker],
- /// [`ResourceDestination`][TokenType::ResourceDestination],
- /// [`ResourceTitle`][TokenType::ResourceTitle],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab],
- /// [`LineEnding`][TokenType::LineEnding]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^^^^^^^
- /// > | a [b](c) d
- /// ^^^
- /// ```
- Resource,
- /// Resource destination.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Resource`][TokenType::Resource]
- /// * **Content model**:
- /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral],
- /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^
- /// ```
- ResourceDestination,
- /// Resource destination literal.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestination`][TokenType::ResourceDestination]
- /// * **Content model**:
- /// [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker],
- /// [`ResourceDestinationString`][TokenType::ResourceDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b]( "d") e
- /// ^^^
- /// ```
- ResourceDestinationLiteral,
- /// Resource destination literal marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b]( "d") e
- /// ^ ^
- /// ```
- ResourceDestinationLiteralMarker,
- /// Resource destination raw.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestination`][TokenType::ResourceDestination]
- /// * **Content model**:
- /// [`ResourceDestinationString`][TokenType::ResourceDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^
- /// ```
- ResourceDestinationRaw,
- /// Resource destination raw.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral],
- /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b]( "d") e
- /// ^
- /// > | a ![b](c "d") e
- /// ^
- /// ```
- ResourceDestinationString,
- /// Resource marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Resource`][TokenType::Resource]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^ ^
- /// ```
- ResourceMarker,
- /// Resource title.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Resource`][TokenType::Resource]
- /// * **Content model**:
- /// [`ResourceTitleMarker`][TokenType::ResourceTitleMarker],
- /// [`ResourceTitleString`][TokenType::ResourceTitleString]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b]( "d") e
- /// ^^^
- /// ```
- ResourceTitle,
- /// Resource title marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceTitle`][TokenType::ResourceTitle]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b]( "d") e
- /// ^ ^
- /// ```
- ResourceTitleMarker,
- /// Resource title string.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceTitle`][TokenType::ResourceTitle]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b]( "d") e
- /// ^
- /// ```
- ResourceTitleString,
- /// SpaceOrTab.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// basically everywhere
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// n/a
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | ␠* * *␠
- /// ^ ^ ^ ^
- /// ```
- SpaceOrTab,
- /// Strong.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`StrongSequence`][TokenType::StrongSequence],
- /// [`StrongText`][TokenType::StrongText]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | **a**
- /// ^^^^^
- /// ```
- Strong,
- /// Strong sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Strong`][TokenType::Strong]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | **a**
- /// ^^ ^^
- /// ```
- StrongSequence,
- /// Strong text.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Strong`][TokenType::Strong]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | **a**
- /// ^
- /// ```
- StrongText,
- /// Whole thematic break.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`ThematicBreakSequence`][TokenType::ThematicBreakSequence],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`thematic_break`][crate::construct::thematic_break]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | * * *
- /// ^^^^^
- /// ```
- ThematicBreak,
- /// Thematic break sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ThematicBreak`][TokenType::ThematicBreak]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`thematic_break`][crate::construct::thematic_break]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | * * *
- /// ^ ^ ^
- /// ```
- ThematicBreakSequence,
-}
-
/// Embedded content type.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ContentType {
@@ -1840,7 +72,7 @@ pub enum EventType {
#[derive(Debug, Clone)]
pub struct Event {
pub event_type: EventType,
- pub token_type: TokenType,
+ pub token_type: Token,
pub point: Point,
pub index: usize,
pub previous: Option,
@@ -1935,7 +167,7 @@ pub struct Tokenizer<'a> {
/// Hierarchy of semantic labels.
///
/// Tracked to make sure everything’s valid.
- pub stack: Vec,
+ pub stack: Vec,
/// Previous character code.
pub previous: Code,
/// Current character code.
@@ -2082,11 +314,11 @@ impl<'a> Tokenizer<'a> {
}
/// Mark the start of a semantic label.
- pub fn enter(&mut self, token_type: TokenType) {
+ pub fn enter(&mut self, token_type: Token) {
self.enter_with_content(token_type, None);
}
- pub fn enter_with_content(&mut self, token_type: TokenType, content_type: Option) {
+ pub fn enter_with_content(&mut self, token_type: Token, content_type: Option) {
log::debug!("enter `{:?}` ({:?})", token_type, self.point);
self.events.push(Event {
event_type: EventType::Enter,
@@ -2101,7 +333,7 @@ impl<'a> Tokenizer<'a> {
}
/// Mark the end of a semantic label.
- pub fn exit(&mut self, token_type: TokenType) {
+ pub fn exit(&mut self, token_type: Token) {
let current_token = self.stack.pop().expect("cannot close w/o open tokens");
assert_eq!(
diff --git a/src/util/skip.rs b/src/util/skip.rs
index a8e4cfe..971beb6 100644
--- a/src/util/skip.rs
+++ b/src/util/skip.rs
@@ -1,14 +1,15 @@
//! Utilities to deal with lists of events.
-use crate::tokenizer::{Event, TokenType};
+use crate::token::Token;
+use crate::tokenizer::Event;
/// Skip from `index`, optionally past `token_types`.
-pub fn opt(events: &[Event], index: usize, token_types: &[TokenType]) -> usize {
+pub fn opt(events: &[Event], index: usize, token_types: &[Token]) -> usize {
skip_opt_with_direction(events, index, token_types, true)
}
/// Skip from `index`, optionally past `token_types`, backwards.
-pub fn opt_back(events: &[Event], index: usize, token_types: &[TokenType]) -> usize {
+pub fn opt_back(events: &[Event], index: usize, token_types: &[Token]) -> usize {
skip_opt_with_direction(events, index, token_types, false)
}
@@ -16,7 +17,7 @@ pub fn opt_back(events: &[Event], index: usize, token_types: &[TokenType]) -> us
fn skip_opt_with_direction(
events: &[Event],
index: usize,
- token_types: &[TokenType],
+ token_types: &[Token],
forward: bool,
) -> usize {
let mut index = index;
--
cgit