aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLibravatar Titus Wormer <tituswormer@gmail.com>2022-07-07 18:56:06 +0200
committerLibravatar Titus Wormer <tituswormer@gmail.com>2022-07-07 18:56:06 +0200
commit92b42e06f943338ce8b54b7e22cbb116ff598fa6 (patch)
treeff51df093f52dc33bfac5e1c236b41cfbd21c220
parentfdb1f1694f44cfbc59d303a10371300b48d74627 (diff)
downloadmarkdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.tar.gz
markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.tar.bz2
markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.zip
Refactor to move token types to `token`
-rw-r--r--src/compiler.rs267
-rw-r--r--src/construct/attention.rs59
-rw-r--r--src/construct/autolink.rs33
-rw-r--r--src/construct/blank_line.rs2
-rw-r--r--src/construct/block_quote.rs31
-rw-r--r--src/construct/character_escape.rs21
-rw-r--r--src/construct/character_reference.rs43
-rw-r--r--src/construct/code_fenced.rs79
-rw-r--r--src/construct/code_indented.rs23
-rw-r--r--src/construct/code_text.rs40
-rw-r--r--src/construct/definition.rs67
-rw-r--r--src/construct/hard_break_escape.rs15
-rw-r--r--src/construct/hard_break_trailing.rs15
-rw-r--r--src/construct/heading_atx.rs39
-rw-r--r--src/construct/heading_setext.rs33
-rw-r--r--src/construct/html_flow.rs31
-rw-r--r--src/construct/html_text.rs23
-rw-r--r--src/construct/label_end.rs129
-rw-r--r--src/construct/label_start_image.rs21
-rw-r--r--src/construct/label_start_link.rs15
-rw-r--r--src/construct/paragraph.rs27
-rw-r--r--src/construct/partial_data.rs17
-rw-r--r--src/construct/partial_destination.rs23
-rw-r--r--src/construct/partial_label.rs15
-rw-r--r--src/construct/partial_space_or_tab.rs15
-rw-r--r--src/construct/partial_title.rs15
-rw-r--r--src/construct/thematic_break.rs15
-rw-r--r--src/content/document.rs11
-rw-r--r--src/content/flow.rs11
-rw-r--r--src/lib.rs1
-rw-r--r--src/token.rs1768
-rw-r--r--src/tokenizer.rs1780
-rw-r--r--src/util/skip.rs9
33 files changed, 2354 insertions, 2339 deletions
diff --git a/src/compiler.rs b/src/compiler.rs
index 2b6eafa..d675c48 100644
--- a/src/compiler.rs
+++ b/src/compiler.rs
@@ -1,7 +1,8 @@
//! Turn events into a string of HTML.
use crate::constant::{SAFE_PROTOCOL_HREF, SAFE_PROTOCOL_SRC};
use crate::construct::character_reference::Kind as CharacterReferenceKind;
-use crate::tokenizer::{Code, Event, EventType, TokenType};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType};
use crate::util::normalize_identifier::normalize_identifier;
use crate::util::{
decode_character_reference::{decode_named, decode_numeric},
@@ -218,8 +219,8 @@ pub struct Options {
/// The current event is available at `context.events[context.index]`.
type Handle = fn(&mut CompileContext);
-/// Map of [`TokenType`][] to [`Handle`][].
-type Map = HashMap<TokenType, Handle>;
+/// Map of [`Token`][] to [`Handle`][].
+type Map = HashMap<Token, Handle>;
/// Context used to compile markdown.
#[allow(clippy::struct_excessive_bools)]
@@ -386,9 +387,9 @@ pub fn compile(events: &[Event], codes: &[Code], options: &Options) -> String {
let event = &events[index];
if event.event_type == EventType::Exit
- && (event.token_type == TokenType::BlankLineEnding
- || event.token_type == TokenType::CodeTextLineEnding
- || event.token_type == TokenType::LineEnding)
+ && (event.token_type == Token::BlankLineEnding
+ || event.token_type == Token::CodeTextLineEnding
+ || event.token_type == Token::LineEnding)
{
let codes = codes_from_span(codes, &from_exit_event(events, index));
line_ending_inferred = Some(LineEnding::from_code(*codes.first().unwrap()));
@@ -409,118 +410,112 @@ pub fn compile(events: &[Event], codes: &[Code], options: &Options) -> String {
let mut enter_map: Map = HashMap::new();
- enter_map.insert(TokenType::BlockQuote, on_enter_block_quote);
- enter_map.insert(TokenType::CodeIndented, on_enter_code_indented);
- enter_map.insert(TokenType::CodeFenced, on_enter_code_fenced);
- enter_map.insert(TokenType::CodeFencedFenceInfo, on_enter_buffer);
- enter_map.insert(TokenType::CodeFencedFenceMeta, on_enter_buffer);
- enter_map.insert(TokenType::CodeText, on_enter_code_text);
- enter_map.insert(TokenType::Definition, on_enter_definition);
+ enter_map.insert(Token::BlockQuote, on_enter_block_quote);
+ enter_map.insert(Token::CodeIndented, on_enter_code_indented);
+ enter_map.insert(Token::CodeFenced, on_enter_code_fenced);
+ enter_map.insert(Token::CodeFencedFenceInfo, on_enter_buffer);
+ enter_map.insert(Token::CodeFencedFenceMeta, on_enter_buffer);
+ enter_map.insert(Token::CodeText, on_enter_code_text);
+ enter_map.insert(Token::Definition, on_enter_definition);
enter_map.insert(
- TokenType::DefinitionDestinationString,
+ Token::DefinitionDestinationString,
on_enter_definition_destination_string,
);
- enter_map.insert(TokenType::DefinitionLabelString, on_enter_buffer);
- enter_map.insert(TokenType::DefinitionTitleString, on_enter_buffer);
- enter_map.insert(TokenType::Emphasis, on_enter_emphasis);
- enter_map.insert(TokenType::HeadingAtxText, on_enter_buffer);
- enter_map.insert(TokenType::HeadingSetextText, on_enter_buffer);
- enter_map.insert(TokenType::HtmlFlow, on_enter_html_flow);
- enter_map.insert(TokenType::HtmlText, on_enter_html_text);
- enter_map.insert(TokenType::Image, on_enter_image);
- enter_map.insert(TokenType::Label, on_enter_buffer);
- enter_map.insert(TokenType::Link, on_enter_link);
- enter_map.insert(TokenType::Paragraph, on_enter_paragraph);
- enter_map.insert(TokenType::ReferenceString, on_enter_buffer);
- enter_map.insert(TokenType::Resource, on_enter_resource);
+ enter_map.insert(Token::DefinitionLabelString, on_enter_buffer);
+ enter_map.insert(Token::DefinitionTitleString, on_enter_buffer);
+ enter_map.insert(Token::Emphasis, on_enter_emphasis);
+ enter_map.insert(Token::HeadingAtxText, on_enter_buffer);
+ enter_map.insert(Token::HeadingSetextText, on_enter_buffer);
+ enter_map.insert(Token::HtmlFlow, on_enter_html_flow);
+ enter_map.insert(Token::HtmlText, on_enter_html_text);
+ enter_map.insert(Token::Image, on_enter_image);
+ enter_map.insert(Token::Label, on_enter_buffer);
+ enter_map.insert(Token::Link, on_enter_link);
+ enter_map.insert(Token::Paragraph, on_enter_paragraph);
+ enter_map.insert(Token::ReferenceString, on_enter_buffer);
+ enter_map.insert(Token::Resource, on_enter_resource);
enter_map.insert(
- TokenType::ResourceDestinationString,
+ Token::ResourceDestinationString,
on_enter_resource_destination_string,
);
- enter_map.insert(TokenType::ResourceTitleString, on_enter_buffer);
- enter_map.insert(TokenType::Strong, on_enter_strong);
+ enter_map.insert(Token::ResourceTitleString, on_enter_buffer);
+ enter_map.insert(Token::Strong, on_enter_strong);
let mut exit_map: Map = HashMap::new();
- exit_map.insert(TokenType::AutolinkEmail, on_exit_autolink_email);
- exit_map.insert(TokenType::AutolinkProtocol, on_exit_autolink_protocol);
- exit_map.insert(TokenType::BlockQuote, on_exit_block_quote);
- exit_map.insert(TokenType::CharacterEscapeValue, on_exit_data);
+ exit_map.insert(Token::AutolinkEmail, on_exit_autolink_email);
+ exit_map.insert(Token::AutolinkProtocol, on_exit_autolink_protocol);
+ exit_map.insert(Token::BlockQuote, on_exit_block_quote);
+ exit_map.insert(Token::CharacterEscapeValue, on_exit_data);
exit_map.insert(
- TokenType::CharacterReferenceMarker,
+ Token::CharacterReferenceMarker,
on_exit_character_reference_marker,
);
exit_map.insert(
- TokenType::CharacterReferenceMarkerNumeric,
+ Token::CharacterReferenceMarkerNumeric,
on_exit_character_reference_marker_numeric,
);
exit_map.insert(
- TokenType::CharacterReferenceMarkerHexadecimal,
+ Token::CharacterReferenceMarkerHexadecimal,
on_exit_character_reference_marker_hexadecimal,
);
exit_map.insert(
- TokenType::CharacterReferenceValue,
+ Token::CharacterReferenceValue,
on_exit_character_reference_value,
);
- exit_map.insert(TokenType::CodeFenced, on_exit_code_flow);
- exit_map.insert(TokenType::CodeFencedFence, on_exit_code_fenced_fence);
+ exit_map.insert(Token::CodeFenced, on_exit_code_flow);
+ exit_map.insert(Token::CodeFencedFence, on_exit_code_fenced_fence);
+ exit_map.insert(Token::CodeFencedFenceInfo, on_exit_code_fenced_fence_info);
+ exit_map.insert(Token::CodeFencedFenceMeta, on_exit_drop);
+ exit_map.insert(Token::CodeFlowChunk, on_exit_code_flow_chunk);
+ exit_map.insert(Token::CodeIndented, on_exit_code_flow);
+ exit_map.insert(Token::CodeText, on_exit_code_text);
+ exit_map.insert(Token::CodeTextData, on_exit_data);
+ exit_map.insert(Token::CodeTextLineEnding, on_exit_code_text_line_ending);
+ exit_map.insert(Token::Data, on_exit_data);
+ exit_map.insert(Token::Definition, on_exit_definition);
exit_map.insert(
- TokenType::CodeFencedFenceInfo,
- on_exit_code_fenced_fence_info,
- );
- exit_map.insert(TokenType::CodeFencedFenceMeta, on_exit_drop);
- exit_map.insert(TokenType::CodeFlowChunk, on_exit_code_flow_chunk);
- exit_map.insert(TokenType::CodeIndented, on_exit_code_flow);
- exit_map.insert(TokenType::CodeText, on_exit_code_text);
- exit_map.insert(TokenType::CodeTextData, on_exit_data);
- exit_map.insert(TokenType::CodeTextLineEnding, on_exit_code_text_line_ending);
- exit_map.insert(TokenType::Data, on_exit_data);
- exit_map.insert(TokenType::Definition, on_exit_definition);
- exit_map.insert(
- TokenType::DefinitionDestinationString,
+ Token::DefinitionDestinationString,
on_exit_definition_destination_string,
);
exit_map.insert(
- TokenType::DefinitionLabelString,
+ Token::DefinitionLabelString,
on_exit_definition_label_string,
);
exit_map.insert(
- TokenType::DefinitionTitleString,
+ Token::DefinitionTitleString,
on_exit_definition_title_string,
);
- exit_map.insert(TokenType::Emphasis, on_exit_emphasis);
-
- exit_map.insert(TokenType::HardBreakEscape, on_exit_break);
- exit_map.insert(TokenType::HardBreakTrailing, on_exit_break);
- exit_map.insert(TokenType::HeadingAtx, on_exit_heading_atx);
- exit_map.insert(TokenType::HeadingAtxSequence, on_exit_heading_atx_sequence);
- exit_map.insert(TokenType::HeadingAtxText, on_exit_heading_atx_text);
- exit_map.insert(TokenType::HeadingSetextText, on_exit_heading_setext_text);
+ exit_map.insert(Token::Emphasis, on_exit_emphasis);
+
+ exit_map.insert(Token::HardBreakEscape, on_exit_break);
+ exit_map.insert(Token::HardBreakTrailing, on_exit_break);
+ exit_map.insert(Token::HeadingAtx, on_exit_heading_atx);
+ exit_map.insert(Token::HeadingAtxSequence, on_exit_heading_atx_sequence);
+ exit_map.insert(Token::HeadingAtxText, on_exit_heading_atx_text);
+ exit_map.insert(Token::HeadingSetextText, on_exit_heading_setext_text);
exit_map.insert(
- TokenType::HeadingSetextUnderline,
+ Token::HeadingSetextUnderline,
on_exit_heading_setext_underline,
);
- exit_map.insert(TokenType::HtmlFlow, on_exit_html);
- exit_map.insert(TokenType::HtmlText, on_exit_html);
- exit_map.insert(TokenType::HtmlFlowData, on_exit_html_data);
- exit_map.insert(TokenType::HtmlTextData, on_exit_html_data);
- exit_map.insert(TokenType::Image, on_exit_media);
- exit_map.insert(TokenType::Label, on_exit_label);
- exit_map.insert(TokenType::LabelText, on_exit_label_text);
- exit_map.insert(TokenType::LineEnding, on_exit_line_ending);
- exit_map.insert(TokenType::Link, on_exit_media);
- exit_map.insert(TokenType::Paragraph, on_exit_paragraph);
- exit_map.insert(TokenType::ReferenceString, on_exit_reference_string);
- exit_map.insert(TokenType::Resource, on_exit_drop);
+ exit_map.insert(Token::HtmlFlow, on_exit_html);
+ exit_map.insert(Token::HtmlText, on_exit_html);
+ exit_map.insert(Token::HtmlFlowData, on_exit_html_data);
+ exit_map.insert(Token::HtmlTextData, on_exit_html_data);
+ exit_map.insert(Token::Image, on_exit_media);
+ exit_map.insert(Token::Label, on_exit_label);
+ exit_map.insert(Token::LabelText, on_exit_label_text);
+ exit_map.insert(Token::LineEnding, on_exit_line_ending);
+ exit_map.insert(Token::Link, on_exit_media);
+ exit_map.insert(Token::Paragraph, on_exit_paragraph);
+ exit_map.insert(Token::ReferenceString, on_exit_reference_string);
+ exit_map.insert(Token::Resource, on_exit_drop);
exit_map.insert(
- TokenType::ResourceDestinationString,
+ Token::ResourceDestinationString,
on_exit_resource_destination_string,
);
- exit_map.insert(
- TokenType::ResourceTitleString,
- on_exit_resource_title_string,
- );
- exit_map.insert(TokenType::Strong, on_exit_strong);
- exit_map.insert(TokenType::ThematicBreak, on_exit_thematic_break);
+ exit_map.insert(Token::ResourceTitleString, on_exit_resource_title_string);
+ exit_map.insert(Token::Strong, on_exit_strong);
+ exit_map.insert(Token::ThematicBreak, on_exit_thematic_break);
// Handle one event.
let handle = |context: &mut CompileContext, index: usize| {
@@ -556,7 +551,7 @@ pub fn compile(events: &[Event], codes: &[Code], options: &Options) -> String {
handle(&mut context, index);
}
- if event.token_type == TokenType::Definition {
+ if event.token_type == Token::Definition {
if event.event_type == EventType::Enter {
handle(&mut context, index); // Also handle start.
definition_inside = true;
@@ -607,21 +602,21 @@ fn on_enter_buffer(context: &mut CompileContext) {
context.buffer();
}
-/// Handle [`Enter`][EventType::Enter]:[`BlockQuote`][TokenType::BlockQuote].
+/// Handle [`Enter`][EventType::Enter]:[`BlockQuote`][Token::BlockQuote].
fn on_enter_block_quote(context: &mut CompileContext) {
// tightStack.push(false)
context.line_ending_if_needed();
context.tag("<blockquote>".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`CodeIndented`][TokenType::CodeIndented].
+/// Handle [`Enter`][EventType::Enter]:[`CodeIndented`][Token::CodeIndented].
fn on_enter_code_indented(context: &mut CompileContext) {
context.code_flow_seen_data = Some(false);
context.line_ending_if_needed();
context.tag("<pre><code>".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`CodeFenced`][TokenType::CodeFenced].
+/// Handle [`Enter`][EventType::Enter]:[`CodeFenced`][Token::CodeFenced].
fn on_enter_code_fenced(context: &mut CompileContext) {
context.code_flow_seen_data = Some(false);
context.line_ending_if_needed();
@@ -630,13 +625,13 @@ fn on_enter_code_fenced(context: &mut CompileContext) {
context.code_fenced_fences_count = Some(0);
}
-/// Handle [`Enter`][EventType::Enter]:[`CodeText`][TokenType::CodeText].
+/// Handle [`Enter`][EventType::Enter]:[`CodeText`][Token::CodeText].
fn on_enter_code_text(context: &mut CompileContext) {
context.tag("<code>".to_string());
context.buffer();
}
-/// Handle [`Enter`][EventType::Enter]:[`Definition`][TokenType::Definition].
+/// Handle [`Enter`][EventType::Enter]:[`Definition`][Token::Definition].
fn on_enter_definition(context: &mut CompileContext) {
context.buffer();
context.media_stack.push(Media {
@@ -649,18 +644,18 @@ fn on_enter_definition(context: &mut CompileContext) {
});
}
-/// Handle [`Enter`][EventType::Enter]:[`DefinitionDestinationString`][TokenType::DefinitionDestinationString].
+/// Handle [`Enter`][EventType::Enter]:[`DefinitionDestinationString`][Token::DefinitionDestinationString].
fn on_enter_definition_destination_string(context: &mut CompileContext) {
context.buffer();
context.ignore_encode = true;
}
-/// Handle [`Enter`][EventType::Enter]:[`Emphasis`][TokenType::Emphasis].
+/// Handle [`Enter`][EventType::Enter]:[`Emphasis`][Token::Emphasis].
fn on_enter_emphasis(context: &mut CompileContext) {
context.tag("<em>".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`HtmlFlow`][TokenType::HtmlFlow].
+/// Handle [`Enter`][EventType::Enter]:[`HtmlFlow`][Token::HtmlFlow].
fn on_enter_html_flow(context: &mut CompileContext) {
context.line_ending_if_needed();
if context.allow_dangerous_html {
@@ -668,14 +663,14 @@ fn on_enter_html_flow(context: &mut CompileContext) {
}
}
-/// Handle [`Enter`][EventType::Enter]:[`HtmlText`][TokenType::HtmlText].
+/// Handle [`Enter`][EventType::Enter]:[`HtmlText`][Token::HtmlText].
fn on_enter_html_text(context: &mut CompileContext) {
if context.allow_dangerous_html {
context.ignore_encode = true;
}
}
-/// Handle [`Enter`][EventType::Enter]:[`Image`][TokenType::Image].
+/// Handle [`Enter`][EventType::Enter]:[`Image`][Token::Image].
fn on_enter_image(context: &mut CompileContext) {
context.media_stack.push(Media {
image: true,
@@ -688,7 +683,7 @@ fn on_enter_image(context: &mut CompileContext) {
context.tags = false; // Disallow tags.
}
-/// Handle [`Enter`][EventType::Enter]:[`Link`][TokenType::Link].
+/// Handle [`Enter`][EventType::Enter]:[`Link`][Token::Link].
fn on_enter_link(context: &mut CompileContext) {
context.media_stack.push(Media {
image: false,
@@ -700,20 +695,20 @@ fn on_enter_link(context: &mut CompileContext) {
});
}
-/// Handle [`Enter`][EventType::Enter]:[`Paragraph`][TokenType::Paragraph].
+/// Handle [`Enter`][EventType::Enter]:[`Paragraph`][Token::Paragraph].
fn on_enter_paragraph(context: &mut CompileContext) {
context.line_ending_if_needed();
context.tag("<p>".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`Resource`][TokenType::Resource].
+/// Handle [`Enter`][EventType::Enter]:[`Resource`][Token::Resource].
fn on_enter_resource(context: &mut CompileContext) {
context.buffer(); // We can have line endings in the resource, ignore them.
let media = context.media_stack.last_mut().unwrap();
media.destination = Some("".to_string());
}
-/// Handle [`Enter`][EventType::Enter]:[`ResourceDestinationString`][TokenType::ResourceDestinationString].
+/// Handle [`Enter`][EventType::Enter]:[`ResourceDestinationString`][Token::ResourceDestinationString].
fn on_enter_resource_destination_string(context: &mut CompileContext) {
context.buffer();
// Ignore encoding the result, as we’ll first percent encode the url and
@@ -721,12 +716,12 @@ fn on_enter_resource_destination_string(context: &mut CompileContext) {
context.ignore_encode = true;
}
-/// Handle [`Enter`][EventType::Enter]:[`Strong`][TokenType::Strong].
+/// Handle [`Enter`][EventType::Enter]:[`Strong`][Token::Strong].
fn on_enter_strong(context: &mut CompileContext) {
context.tag("<strong>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`AutolinkEmail`][TokenType::AutolinkEmail].
+/// Handle [`Exit`][EventType::Exit]:[`AutolinkEmail`][Token::AutolinkEmail].
fn on_exit_autolink_email(context: &mut CompileContext) {
let slice = serialize(
context.codes,
@@ -744,7 +739,7 @@ fn on_exit_autolink_email(context: &mut CompileContext) {
context.tag("</a>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`AutolinkProtocol`][TokenType::AutolinkProtocol].
+/// Handle [`Exit`][EventType::Exit]:[`AutolinkProtocol`][Token::AutolinkProtocol].
fn on_exit_autolink_protocol(context: &mut CompileContext) {
let slice = serialize(
context.codes,
@@ -759,12 +754,12 @@ fn on_exit_autolink_protocol(context: &mut CompileContext) {
context.tag("</a>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:{[`HardBreakEscape`][TokenType::HardBreakEscape],[`HardBreakTrailing`][TokenType::HardBreakTrailing]}.
+/// Handle [`Exit`][EventType::Exit]:{[`HardBreakEscape`][Token::HardBreakEscape],[`HardBreakTrailing`][Token::HardBreakTrailing]}.
fn on_exit_break(context: &mut CompileContext) {
context.tag("<br />".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`BlockQuote`][TokenType::BlockQuote].
+/// Handle [`Exit`][EventType::Exit]:[`BlockQuote`][Token::BlockQuote].
fn on_exit_block_quote(context: &mut CompileContext) {
// tightStack.pop()
context.line_ending_if_needed();
@@ -772,22 +767,22 @@ fn on_exit_block_quote(context: &mut CompileContext) {
// let mut slurp_all_line_endings = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarker`][Token::CharacterReferenceMarker].
fn on_exit_character_reference_marker(context: &mut CompileContext) {
context.character_reference_kind = Some(CharacterReferenceKind::Named);
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal].
fn on_exit_character_reference_marker_hexadecimal(context: &mut CompileContext) {
context.character_reference_kind = Some(CharacterReferenceKind::Hexadecimal);
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric].
fn on_exit_character_reference_marker_numeric(context: &mut CompileContext) {
context.character_reference_kind = Some(CharacterReferenceKind::Decimal);
}
-/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceValue`][TokenType::CharacterReferenceValue].
+/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceValue`][Token::CharacterReferenceValue].
fn on_exit_character_reference_value(context: &mut CompileContext) {
let kind = context
.character_reference_kind
@@ -808,7 +803,7 @@ fn on_exit_character_reference_value(context: &mut CompileContext) {
context.push(context.encode_opt(&value));
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeFlowChunk`][TokenType::CodeFlowChunk].
+/// Handle [`Exit`][EventType::Exit]:[`CodeFlowChunk`][Token::CodeFlowChunk].
fn on_exit_code_flow_chunk(context: &mut CompileContext) {
context.code_flow_seen_data = Some(true);
context.push(context.encode_opt(&serialize(
@@ -818,7 +813,7 @@ fn on_exit_code_flow_chunk(context: &mut CompileContext) {
)));
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFence`][TokenType::CodeFencedFence].
+/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFence`][Token::CodeFencedFence].
fn on_exit_code_fenced_fence(context: &mut CompileContext) {
let count = if let Some(count) = context.code_fenced_fences_count {
count
@@ -834,13 +829,13 @@ fn on_exit_code_fenced_fence(context: &mut CompileContext) {
context.code_fenced_fences_count = Some(count + 1);
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo].
+/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo].
fn on_exit_code_fenced_fence_info(context: &mut CompileContext) {
let value = context.resume();
context.tag(format!(" class=\"language-{}\"", value));
}
-/// Handle [`Exit`][EventType::Exit]:{[`CodeFenced`][TokenType::CodeFenced],[`CodeIndented`][TokenType::CodeIndented]}.
+/// Handle [`Exit`][EventType::Exit]:{[`CodeFenced`][Token::CodeFenced],[`CodeIndented`][Token::CodeIndented]}.
fn on_exit_code_flow(context: &mut CompileContext) {
let seen_data = context
.code_flow_seen_data
@@ -874,7 +869,7 @@ fn on_exit_code_flow(context: &mut CompileContext) {
context.slurp_one_line_ending = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeText`][TokenType::CodeText].
+/// Handle [`Exit`][EventType::Exit]:[`CodeText`][Token::CodeText].
fn on_exit_code_text(context: &mut CompileContext) {
let result = context.resume();
let mut chars = result.chars();
@@ -898,7 +893,7 @@ fn on_exit_code_text(context: &mut CompileContext) {
context.tag("</code>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`CodeTextLineEnding`][TokenType::CodeTextLineEnding].
+/// Handle [`Exit`][EventType::Exit]:[`CodeTextLineEnding`][Token::CodeTextLineEnding].
fn on_exit_code_text_line_ending(context: &mut CompileContext) {
context.push(" ".to_string());
}
@@ -910,7 +905,7 @@ fn on_exit_drop(context: &mut CompileContext) {
context.resume();
}
-/// Handle [`Exit`][EventType::Exit]:{[`CodeTextData`][TokenType::CodeTextData],[`Data`][TokenType::Data],[`CharacterEscapeValue`][TokenType::CharacterEscapeValue]}.
+/// Handle [`Exit`][EventType::Exit]:{[`CodeTextData`][Token::CodeTextData],[`Data`][Token::Data],[`CharacterEscapeValue`][Token::CharacterEscapeValue]}.
fn on_exit_data(context: &mut CompileContext) {
// Just output it.
context.push(context.encode_opt(&serialize(
@@ -920,7 +915,7 @@ fn on_exit_data(context: &mut CompileContext) {
)));
}
-/// Handle [`Exit`][EventType::Exit]:[`Definition`][TokenType::Definition].
+/// Handle [`Exit`][EventType::Exit]:[`Definition`][Token::Definition].
fn on_exit_definition(context: &mut CompileContext) {
let definition = context.media_stack.pop().unwrap();
let reference_id = normalize_identifier(&definition.reference_id.unwrap());
@@ -935,7 +930,7 @@ fn on_exit_definition(context: &mut CompileContext) {
.or_insert(Definition { destination, title });
}
-/// Handle [`Exit`][EventType::Exit]:[`DefinitionDestinationString`][TokenType::DefinitionDestinationString].
+/// Handle [`Exit`][EventType::Exit]:[`DefinitionDestinationString`][Token::DefinitionDestinationString].
fn on_exit_definition_destination_string(context: &mut CompileContext) {
let buf = context.resume();
let definition = context.media_stack.last_mut().unwrap();
@@ -943,7 +938,7 @@ fn on_exit_definition_destination_string(context: &mut CompileContext) {
context.ignore_encode = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`DefinitionLabelString`][TokenType::DefinitionLabelString].
+/// Handle [`Exit`][EventType::Exit]:[`DefinitionLabelString`][Token::DefinitionLabelString].
fn on_exit_definition_label_string(context: &mut CompileContext) {
// Discard label, use the source content instead.
context.resume();
@@ -955,19 +950,19 @@ fn on_exit_definition_label_string(context: &mut CompileContext) {
));
}
-/// Handle [`Exit`][EventType::Exit]:[`DefinitionTitleString`][TokenType::DefinitionTitleString].
+/// Handle [`Exit`][EventType::Exit]:[`DefinitionTitleString`][Token::DefinitionTitleString].
fn on_exit_definition_title_string(context: &mut CompileContext) {
let buf = context.resume();
let definition = context.media_stack.last_mut().unwrap();
definition.title = Some(buf);
}
-/// Handle [`Exit`][EventType::Exit]:[`Strong`][TokenType::Emphasis].
+/// Handle [`Exit`][EventType::Exit]:[`Strong`][Token::Emphasis].
fn on_exit_emphasis(context: &mut CompileContext) {
context.tag("</em>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingAtx`][TokenType::HeadingAtx].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingAtx`][Token::HeadingAtx].
fn on_exit_heading_atx(context: &mut CompileContext) {
let rank = context
.atx_opening_sequence_size
@@ -977,7 +972,7 @@ fn on_exit_heading_atx(context: &mut CompileContext) {
context.tag(format!("</h{}>", rank));
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxSequence`][TokenType::HeadingAtxSequence].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxSequence`][Token::HeadingAtxSequence].
fn on_exit_heading_atx_sequence(context: &mut CompileContext) {
// First fence we see.
if context.atx_opening_sequence_size.is_none() {
@@ -993,20 +988,20 @@ fn on_exit_heading_atx_sequence(context: &mut CompileContext) {
}
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxText`][TokenType::HeadingAtxText].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxText`][Token::HeadingAtxText].
fn on_exit_heading_atx_text(context: &mut CompileContext) {
let value = context.resume();
context.push(value);
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextText`][TokenType::HeadingSetextText].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextText`][Token::HeadingSetextText].
fn on_exit_heading_setext_text(context: &mut CompileContext) {
let buf = context.resume();
context.heading_setext_buffer = Some(buf);
context.slurp_one_line_ending = true;
}
-/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline].
+/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextUnderline`][Token::HeadingSetextUnderline].
fn on_exit_heading_setext_underline(context: &mut CompileContext) {
let text = context
.heading_setext_buffer
@@ -1024,12 +1019,12 @@ fn on_exit_heading_setext_underline(context: &mut CompileContext) {
context.tag(format!("</h{}>", level));
}
-/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlow`][TokenType::HtmlFlow],[`HtmlText`][TokenType::HtmlText]}.
+/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlow`][Token::HtmlFlow],[`HtmlText`][Token::HtmlText]}.
fn on_exit_html(context: &mut CompileContext) {
context.ignore_encode = false;
}
-/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlowData`][TokenType::HtmlFlowData],[`HtmlTextData`][TokenType::HtmlTextData]}.
+/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlowData`][Token::HtmlFlowData],[`HtmlTextData`][Token::HtmlTextData]}.
fn on_exit_html_data(context: &mut CompileContext) {
let slice = serialize(
context.codes,
@@ -1039,14 +1034,14 @@ fn on_exit_html_data(context: &mut CompileContext) {
context.push(context.encode_opt(&slice));
}
-/// Handle [`Exit`][EventType::Exit]:[`Label`][TokenType::Label].
+/// Handle [`Exit`][EventType::Exit]:[`Label`][Token::Label].
fn on_exit_label(context: &mut CompileContext) {
let buf = context.resume();
let media = context.media_stack.last_mut().unwrap();
media.label = Some(buf);
}
-/// Handle [`Exit`][EventType::Exit]:[`LabelText`][TokenType::LabelText].
+/// Handle [`Exit`][EventType::Exit]:[`LabelText`][Token::LabelText].
fn on_exit_label_text(context: &mut CompileContext) {
let media = context.media_stack.last_mut().unwrap();
media.label_id = Some(serialize(
@@ -1056,7 +1051,7 @@ fn on_exit_label_text(context: &mut CompileContext) {
));
}
-/// Handle [`Exit`][EventType::Exit]:[`LineEnding`][TokenType::LineEnding].
+/// Handle [`Exit`][EventType::Exit]:[`LineEnding`][Token::LineEnding].
fn on_exit_line_ending(context: &mut CompileContext) {
// if slurp_all_line_endings {
// // Empty.
@@ -1072,7 +1067,7 @@ fn on_exit_line_ending(context: &mut CompileContext) {
}
}
-/// Handle [`Exit`][EventType::Exit]:{[`Image`][TokenType::Image],[`Link`][TokenType::Link]}.
+/// Handle [`Exit`][EventType::Exit]:{[`Image`][Token::Image],[`Link`][Token::Link]}.
fn on_exit_media(context: &mut CompileContext) {
let mut is_in_image = false;
let mut index = 0;
@@ -1136,12 +1131,12 @@ fn on_exit_media(context: &mut CompileContext) {
};
}
-/// Handle [`Exit`][EventType::Exit]:[`Paragraph`][TokenType::Paragraph].
+/// Handle [`Exit`][EventType::Exit]:[`Paragraph`][Token::Paragraph].
fn on_exit_paragraph(context: &mut CompileContext) {
context.tag("</p>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`ReferenceString`][TokenType::ReferenceString].
+/// Handle [`Exit`][EventType::Exit]:[`ReferenceString`][Token::ReferenceString].
fn on_exit_reference_string(context: &mut CompileContext) {
// Drop stuff.
context.resume();
@@ -1153,7 +1148,7 @@ fn on_exit_reference_string(context: &mut CompileContext) {
));
}
-/// Handle [`Exit`][EventType::Exit]:[`ResourceDestinationString`][TokenType::ResourceDestinationString].
+/// Handle [`Exit`][EventType::Exit]:[`ResourceDestinationString`][Token::ResourceDestinationString].
fn on_exit_resource_destination_string(context: &mut CompileContext) {
let buf = context.resume();
let media = context.media_stack.last_mut().unwrap();
@@ -1161,19 +1156,19 @@ fn on_exit_resource_destination_string(context: &mut CompileContext) {
context.ignore_encode = false;
}
-/// Handle [`Exit`][EventType::Exit]:[`ResourceTitleString`][TokenType::ResourceTitleString].
+/// Handle [`Exit`][EventType::Exit]:[`ResourceTitleString`][Token::ResourceTitleString].
fn on_exit_resource_title_string(context: &mut CompileContext) {
let buf = context.resume();
let media = context.media_stack.last_mut().unwrap();
media.title = Some(buf);
}
-/// Handle [`Exit`][EventType::Exit]:[`Strong`][TokenType::Strong].
+/// Handle [`Exit`][EventType::Exit]:[`Strong`][Token::Strong].
fn on_exit_strong(context: &mut CompileContext) {
context.tag("</strong>".to_string());
}
-/// Handle [`Exit`][EventType::Exit]:[`ThematicBreak`][TokenType::ThematicBreak].
+/// Handle [`Exit`][EventType::Exit]:[`ThematicBreak`][Token::ThematicBreak].
fn on_exit_thematic_break(context: &mut CompileContext) {
context.line_ending_if_needed();
context.tag("<hr />".to_string());
diff --git a/src/construct/attention.rs b/src/construct/attention.rs
index daf662e..1750692 100644
--- a/src/construct/attention.rs
+++ b/src/construct/attention.rs
@@ -32,14 +32,14 @@
//!
//! ## Tokens
//!
-//! * [`Emphasis`][TokenType::Emphasis]
-//! * [`EmphasisSequence`][TokenType::EmphasisSequence]
-//! * [`EmphasisText`][TokenType::EmphasisText]
-//! * [`Strong`][TokenType::Strong]
-//! * [`StrongSequence`][TokenType::StrongSequence]
-//! * [`StrongText`][TokenType::StrongText]
+//! * [`Emphasis`][Token::Emphasis]
+//! * [`EmphasisSequence`][Token::EmphasisSequence]
+//! * [`EmphasisText`][Token::EmphasisText]
+//! * [`Strong`][Token::Strong]
+//! * [`StrongSequence`][Token::StrongSequence]
+//! * [`StrongText`][Token::StrongText]
//!
-//! > 👉 **Note**: while parsing, [`AttentionSequence`][TokenType::AttentionSequence]
+//! > 👉 **Note**: while parsing, [`AttentionSequence`][Token::AttentionSequence]
//! > is used, which is later compiled away.
//!
//! ## References
@@ -51,7 +51,8 @@
//! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element
//! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element
-use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, Tokenizer};
use crate::unicode::PUNCTUATION;
use crate::util::edit_map::EditMap;
@@ -174,7 +175,7 @@ struct Sequence {
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('*' | '_') => {
- tokenizer.enter(TokenType::AttentionSequence);
+ tokenizer.enter(Token::AttentionSequence);
inside(tokenizer, code, MarkerKind::from_code(code))
}
_ => (State::Nok, None),
@@ -193,7 +194,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, marker: MarkerKind) -> StateFnR
(State::Fn(Box::new(move |t, c| inside(t, c, marker))), None)
}
_ => {
- tokenizer.exit(TokenType::AttentionSequence);
+ tokenizer.exit(Token::AttentionSequence);
tokenizer.register_resolver("attention".to_string(), Box::new(resolve));
(State::Ok, Some(vec![code]))
}
@@ -216,7 +217,7 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
if enter.event_type == EventType::Enter {
balance += 1;
- if enter.token_type == TokenType::AttentionSequence {
+ if enter.token_type == Token::AttentionSequence {
let end = start + 1;
let exit = &tokenizer.events[end];
let marker = MarkerKind::from_code(codes[enter.index]);
@@ -392,9 +393,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::Emphasis
+ Token::Emphasis
} else {
- TokenType::Strong
+ Token::Strong
},
point: seq_open_enter.0.clone(),
index: seq_open_enter.1,
@@ -405,9 +406,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_open_enter.0.clone(),
index: seq_open_enter.1,
@@ -418,9 +419,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_open_exit.0.clone(),
index: seq_open_exit.1,
@@ -431,9 +432,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::EmphasisText
+ Token::EmphasisText
} else {
- TokenType::StrongText
+ Token::StrongText
},
point: seq_open_exit.0.clone(),
index: seq_open_exit.1,
@@ -451,9 +452,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::EmphasisText
+ Token::EmphasisText
} else {
- TokenType::StrongText
+ Token::StrongText
},
point: seq_close_enter.0.clone(),
index: seq_close_enter.1,
@@ -464,9 +465,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_close_enter.0.clone(),
index: seq_close_enter.1,
@@ -477,9 +478,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_close_exit.0.clone(),
index: seq_close_exit.1,
@@ -490,9 +491,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::Emphasis
+ Token::Emphasis
} else {
- TokenType::Strong
+ Token::Strong
},
point: seq_close_exit.0.clone(),
index: seq_close_exit.1,
@@ -515,8 +516,8 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
let mut index = 0;
while index < sequences.len() {
let sequence = &sequences[index];
- tokenizer.events[sequence.event_index].token_type = TokenType::Data;
- tokenizer.events[sequence.event_index + 1].token_type = TokenType::Data;
+ tokenizer.events[sequence.event_index].token_type = Token::Data;
+ tokenizer.events[sequence.event_index + 1].token_type = Token::Data;
index += 1;
}
diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs
index e29bf8b..db4365f 100644
--- a/src/construct/autolink.rs
+++ b/src/construct/autolink.rs
@@ -84,10 +84,10 @@
//!
//! ## Tokens
//!
-//! * [`Autolink`][TokenType::Autolink]
-//! * [`AutolinkEmail`][TokenType::AutolinkEmail]
-//! * [`AutolinkMarker`][TokenType::AutolinkMarker]
-//! * [`AutolinkProtocol`][TokenType::AutolinkProtocol]
+//! * [`Autolink`][Token::Autolink]
+//! * [`AutolinkEmail`][Token::AutolinkEmail]
+//! * [`AutolinkMarker`][Token::AutolinkMarker]
+//! * [`AutolinkProtocol`][Token::AutolinkProtocol]
//!
//! ## References
//!
@@ -102,7 +102,8 @@
//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of an autolink.
///
@@ -113,11 +114,11 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('<') => {
- tokenizer.enter(TokenType::Autolink);
- tokenizer.enter(TokenType::AutolinkMarker);
+ tokenizer.enter(Token::Autolink);
+ tokenizer.enter(Token::AutolinkMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::AutolinkMarker);
- tokenizer.enter(TokenType::AutolinkProtocol);
+ tokenizer.exit(Token::AutolinkMarker);
+ tokenizer.enter(Token::AutolinkProtocol);
(State::Fn(Box::new(open)), None)
}
_ => (State::Nok, None),
@@ -195,7 +196,7 @@ fn scheme_inside_or_email_atext(
fn url_inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.exit(TokenType::AutolinkProtocol);
+ tokenizer.exit(Token::AutolinkProtocol);
end(tokenizer, code)
}
Code::Char(char) if char.is_ascii_control() => (State::Nok, None),
@@ -260,10 +261,10 @@ fn email_label(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes
}
Code::Char('>') => {
let index = tokenizer.events.len();
- tokenizer.exit(TokenType::AutolinkProtocol);
+ tokenizer.exit(Token::AutolinkProtocol);
// Change the token type.
- tokenizer.events[index - 1].token_type = TokenType::AutolinkEmail;
- tokenizer.events[index].token_type = TokenType::AutolinkEmail;
+ tokenizer.events[index - 1].token_type = Token::AutolinkEmail;
+ tokenizer.events[index].token_type = Token::AutolinkEmail;
end(tokenizer, code)
}
_ => email_value(tokenizer, code, size),
@@ -306,10 +307,10 @@ fn email_value(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes
fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.enter(TokenType::AutolinkMarker);
+ tokenizer.enter(Token::AutolinkMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::AutolinkMarker);
- tokenizer.exit(TokenType::Autolink);
+ tokenizer.exit(Token::AutolinkMarker);
+ tokenizer.exit(Token::Autolink);
(State::Ok, None)
}
_ => unreachable!("expected `>`"),
diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs
index 3ca3266..69a4709 100644
--- a/src/construct/blank_line.rs
+++ b/src/construct/blank_line.rs
@@ -20,7 +20,7 @@
//!
//! ## Tokens
//!
-//! * [`SpaceOrTab`][crate::tokenizer::TokenType::SpaceOrTab]
+//! * [`SpaceOrTab`][crate::token::Token::SpaceOrTab]
//!
//! ## References
//!
diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs
index a3ffbc7..096ac0a 100644
--- a/src/construct/block_quote.rs
+++ b/src/construct/block_quote.rs
@@ -19,10 +19,10 @@
//!
//! ## Tokens
//!
-//! * [`BlockQuote`][TokenType::BlockQuote]
-//! * [`BlockQuoteMarker`][TokenType::BlockQuoteMarker]
-//! * [`BlockQuotePrefix`][TokenType::BlockQuotePrefix]
-//! * [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace]
+//! * [`BlockQuote`][Token::BlockQuote]
+//! * [`BlockQuoteMarker`][Token::BlockQuoteMarker]
+//! * [`BlockQuotePrefix`][Token::BlockQuotePrefix]
+//! * [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace]
//!
//! ## References
//!
@@ -35,7 +35,8 @@
use crate::constant::TAB_SIZE;
use crate::construct::partial_space_or_tab::space_or_tab_min_max;
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of block quote.
///
@@ -55,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.enter(TokenType::BlockQuote);
+ tokenizer.enter(Token::BlockQuote);
cont_before(tokenizer, code)
}
_ => cont_before(tokenizer, code),
@@ -82,10 +83,10 @@ pub fn cont(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.enter(TokenType::BlockQuotePrefix);
- tokenizer.enter(TokenType::BlockQuoteMarker);
+ tokenizer.enter(Token::BlockQuotePrefix);
+ tokenizer.enter(Token::BlockQuoteMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::BlockQuoteMarker);
+ tokenizer.exit(Token::BlockQuoteMarker);
(State::Fn(Box::new(cont_after)), None)
}
_ => (State::Nok, None),
@@ -101,20 +102,20 @@ fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn cont_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::VirtualSpace | Code::Char('\t' | ' ') => {
- tokenizer.enter(TokenType::BlockQuoteWhitespace);
+ tokenizer.enter(Token::BlockQuoteWhitespace);
tokenizer.consume(code);
- tokenizer.exit(TokenType::BlockQuoteWhitespace);
- tokenizer.exit(TokenType::BlockQuotePrefix);
+ tokenizer.exit(Token::BlockQuoteWhitespace);
+ tokenizer.exit(Token::BlockQuotePrefix);
(State::Ok, None)
}
_ => {
- tokenizer.exit(TokenType::BlockQuotePrefix);
+ tokenizer.exit(Token::BlockQuotePrefix);
(State::Ok, Some(vec![code]))
}
}
}
/// End of a block quote.
-pub fn end() -> Vec<TokenType> {
- vec![TokenType::BlockQuote]
+pub fn end() -> Vec<Token> {
+ vec![Token::BlockQuote]
}
diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs
index 0ccc574..55d321a 100644
--- a/src/construct/character_escape.rs
+++ b/src/construct/character_escape.rs
@@ -19,9 +19,9 @@
//!
//! ## Tokens
//!
-//! * [`CharacterEscape`][TokenType::CharacterEscape]
-//! * [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker]
-//! * [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
+//! * [`CharacterEscape`][Token::CharacterEscape]
+//! * [`CharacterEscapeMarker`][Token::CharacterEscapeMarker]
+//! * [`CharacterEscapeValue`][Token::CharacterEscapeValue]
//!
//! ## References
//!
@@ -33,7 +33,8 @@
//! [character_reference]: crate::construct::character_reference
//! [hard_break_escape]: crate::construct::hard_break_escape
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of a character escape.
///
@@ -45,10 +46,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('\\') => {
- tokenizer.enter(TokenType::CharacterEscape);
- tokenizer.enter(TokenType::CharacterEscapeMarker);
+ tokenizer.enter(Token::CharacterEscape);
+ tokenizer.enter(Token::CharacterEscapeMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterEscapeMarker);
+ tokenizer.exit(Token::CharacterEscapeMarker);
(State::Fn(Box::new(inside)), None)
}
_ => (State::Nok, None),
@@ -65,10 +66,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(char) if char.is_ascii_punctuation() => {
- tokenizer.enter(TokenType::CharacterEscapeValue);
+ tokenizer.enter(Token::CharacterEscapeValue);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterEscapeValue);
- tokenizer.exit(TokenType::CharacterEscape);
+ tokenizer.exit(Token::CharacterEscapeValue);
+ tokenizer.exit(Token::CharacterEscape);
(State::Ok, None)
}
_ => (State::Nok, None),
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index 65e49ca..04e89d2 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -42,12 +42,12 @@
//!
//! ## Tokens
//!
-//! * [`CharacterReference`][TokenType::CharacterReference]
-//! * [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker]
-//! * [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal]
-//! * [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric]
-//! * [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi]
-//! * [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
+//! * [`CharacterReference`][Token::CharacterReference]
+//! * [`CharacterReferenceMarker`][Token::CharacterReferenceMarker]
+//! * [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal]
+//! * [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric]
+//! * [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi]
+//! * [`CharacterReferenceValue`][Token::CharacterReferenceValue]
//!
//! ## References
//!
@@ -65,7 +65,8 @@ use crate::constant::{
CHARACTER_REFERENCE_DECIMAL_SIZE_MAX, CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX,
CHARACTER_REFERENCE_NAMED_SIZE_MAX, CHARACTER_REFERENCE_NAMES,
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Kind of a character reference.
#[derive(Debug, Clone, PartialEq)]
@@ -119,10 +120,10 @@ struct Info {
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('&') => {
- tokenizer.enter(TokenType::CharacterReference);
- tokenizer.enter(TokenType::CharacterReferenceMarker);
+ tokenizer.enter(Token::CharacterReference);
+ tokenizer.enter(Token::CharacterReferenceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarker);
+ tokenizer.exit(Token::CharacterReferenceMarker);
(State::Fn(Box::new(open)), None)
}
_ => (State::Nok, None),
@@ -143,12 +144,12 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
kind: Kind::Named,
};
if let Code::Char('#') = code {
- tokenizer.enter(TokenType::CharacterReferenceMarkerNumeric);
+ tokenizer.enter(Token::CharacterReferenceMarkerNumeric);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerNumeric);
+ tokenizer.exit(Token::CharacterReferenceMarkerNumeric);
(State::Fn(Box::new(|t, c| numeric(t, c, info))), None)
} else {
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ tokenizer.enter(Token::CharacterReferenceValue);
value(tokenizer, code, info)
}
}
@@ -162,14 +163,14 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// ```
fn numeric(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
if let Code::Char('x' | 'X') = code {
- tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal);
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(Token::CharacterReferenceValue);
info.kind = Kind::Hexadecimal;
(State::Fn(Box::new(|t, c| value(t, c, info))), None)
} else {
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ tokenizer.enter(Token::CharacterReferenceValue);
info.kind = Kind::Decimal;
value(tokenizer, code, info)
}
@@ -194,11 +195,11 @@ fn value(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult
{
(State::Nok, None)
} else {
- tokenizer.exit(TokenType::CharacterReferenceValue);
- tokenizer.enter(TokenType::CharacterReferenceMarkerSemi);
+ tokenizer.exit(Token::CharacterReferenceValue);
+ tokenizer.enter(Token::CharacterReferenceMarkerSemi);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerSemi);
- tokenizer.exit(TokenType::CharacterReference);
+ tokenizer.exit(Token::CharacterReferenceMarkerSemi);
+ tokenizer.exit(Token::CharacterReference);
(State::Ok, None)
}
}
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index 05266ba..617979f 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -77,14 +77,14 @@
//!
//! ## Tokens
//!
-//! * [`CodeFenced`][TokenType::CodeFenced]
-//! * [`CodeFencedFence`][TokenType::CodeFencedFence]
-//! * [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo]
-//! * [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta]
-//! * [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
-//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`CodeFenced`][Token::CodeFenced]
+//! * [`CodeFencedFence`][Token::CodeFencedFence]
+//! * [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo]
+//! * [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta]
+//! * [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence]
+//! * [`CodeFlowChunk`][Token::CodeFlowChunk]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -103,7 +103,8 @@
use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE};
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
use crate::util::span::from_exit_event;
/// Kind of fences.
@@ -184,8 +185,8 @@ struct Info {
/// ~~~
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::CodeFenced);
- tokenizer.enter(TokenType::CodeFencedFence);
+ tokenizer.enter(Token::CodeFenced);
+ tokenizer.enter(Token::CodeFencedFence);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before_sequence_open)(tokenizer, code)
}
@@ -202,7 +203,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
let mut prefix = 0;
if let Some(event) = tail {
- if event.token_type == TokenType::SpaceOrTab {
+ if event.token_type == Token::SpaceOrTab {
let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1);
prefix = span.end_index - span.start_index;
}
@@ -210,7 +211,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
match code {
Code::Char('`' | '~') => {
- tokenizer.enter(TokenType::CodeFencedFenceSequence);
+ tokenizer.enter(Token::CodeFencedFenceSequence);
sequence_open(
tokenizer,
code,
@@ -245,7 +246,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State
)
}
_ if info.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => {
- tokenizer.exit(TokenType::CodeFencedFenceSequence);
+ tokenizer.exit(Token::CodeFencedFenceSequence);
tokenizer.attempt_opt(space_or_tab(), |t, c| info_before(t, c, info))(tokenizer, code)
}
_ => (State::Nok, None),
@@ -262,12 +263,12 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State
fn info_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
_ => {
- tokenizer.enter(TokenType::CodeFencedFenceInfo);
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter(Token::CodeFencedFenceInfo);
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
info_inside(tokenizer, code, info, vec![])
}
}
@@ -288,14 +289,14 @@ fn info_inside(
) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::CodeFencedFenceInfo);
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::CodeFencedFenceInfo);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
Code::VirtualSpace | Code::Char('\t' | ' ') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::CodeFencedFenceInfo);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::CodeFencedFenceInfo);
tokenizer.attempt_opt(space_or_tab(), |t, c| meta_before(t, c, info))(tokenizer, code)
}
Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None),
@@ -320,12 +321,12 @@ fn info_inside(
fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
_ => {
- tokenizer.enter(TokenType::CodeFencedFenceMeta);
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter(Token::CodeFencedFenceMeta);
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
meta(tokenizer, code, info)
}
}
@@ -341,9 +342,9 @@ fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu
fn meta(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::CodeFencedFenceMeta);
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::CodeFencedFenceMeta);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None),
@@ -390,9 +391,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult
fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| close_start(t, c, info))), None)
}
_ => unreachable!("expected eol"),
@@ -411,7 +412,7 @@ fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu
/// | ~~~
/// ```
fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
- tokenizer.enter(TokenType::CodeFencedFence);
+ tokenizer.enter(Token::CodeFencedFence);
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), |t, c| {
close_before(t, c, info)
})(tokenizer, code)
@@ -431,7 +432,7 @@ fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu
fn close_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.enter(TokenType::CodeFencedFenceSequence);
+ tokenizer.enter(Token::CodeFencedFenceSequence);
close_sequence(tokenizer, code, info, 0)
}
_ => (State::Nok, None),
@@ -455,7 +456,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize
)
}
_ if size >= CODE_FENCED_SEQUENCE_SIZE_MIN && size >= info.size => {
- tokenizer.exit(TokenType::CodeFencedFenceSequence);
+ tokenizer.exit(Token::CodeFencedFenceSequence);
tokenizer.attempt_opt(space_or_tab(), close_sequence_after)(tokenizer, code)
}
_ => (State::Nok, None),
@@ -472,7 +473,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize
fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::CodeFencedFence);
(State::Ok, Some(vec![code]))
}
_ => (State::Nok, None),
@@ -487,9 +488,9 @@ fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
/// ~~~
/// ```
fn content_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| content_start(t, c, info))), None)
}
/// Before code content, definitely not before a closing fence.
@@ -518,7 +519,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe
at_break(tokenizer, code, info)
}
_ => {
- tokenizer.enter(TokenType::CodeFlowChunk);
+ tokenizer.enter(Token::CodeFlowChunk);
content_continue(tokenizer, code, info)
}
}
@@ -536,7 +537,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe
fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFlowChunk);
+ tokenizer.exit(Token::CodeFlowChunk);
at_break(tokenizer, code, info)
}
_ => {
@@ -557,7 +558,7 @@ fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateF
/// ~~~|
/// ```
fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.exit(TokenType::CodeFenced);
+ tokenizer.exit(Token::CodeFenced);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index 9bdfd71..8966249 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -28,10 +28,10 @@
//!
//! ## Tokens
//!
-//! * [`CodeIndented`][TokenType::CodeIndented]
-//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`CodeIndented`][Token::CodeIndented]
+//! * [`CodeFlowChunk`][Token::CodeFlowChunk]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -47,7 +47,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::TAB_SIZE;
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of code (indented).
///
@@ -63,7 +64,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if tokenizer.interrupt {
(State::Nok, None)
} else {
- tokenizer.enter(TokenType::CodeIndented);
+ tokenizer.enter(Token::CodeIndented);
tokenizer.go(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), at_break)(tokenizer, code)
}
}
@@ -82,7 +83,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
Box::new(if ok { at_break } else { after })
})(tokenizer, code),
_ => {
- tokenizer.enter(TokenType::CodeFlowChunk);
+ tokenizer.enter(Token::CodeFlowChunk);
content(tokenizer, code)
}
}
@@ -98,7 +99,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFlowChunk);
+ tokenizer.exit(Token::CodeFlowChunk);
at_break(tokenizer, code)
}
_ => {
@@ -114,7 +115,7 @@ fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// ab|
/// ```
fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.exit(TokenType::CodeIndented);
+ tokenizer.exit(Token::CodeIndented);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
@@ -130,9 +131,9 @@ fn further_start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
// To do: `nok` if lazy line.
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(further_start)), None)
}
_ => tokenizer.attempt(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), |ok| {
diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs
index a6dc7eb..6df61b8 100644
--- a/src/construct/code_text.rs
+++ b/src/construct/code_text.rs
@@ -67,10 +67,10 @@
//!
//! ## Tokens
//!
-//! * [`CodeText`][TokenType::CodeText]
-//! * [`CodeTextData`][TokenType::CodeTextData]
-//! * [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
-//! * [`CodeTextSequence`][TokenType::CodeTextSequence]
+//! * [`CodeText`][Token::CodeText]
+//! * [`CodeTextData`][Token::CodeTextData]
+//! * [`CodeTextLineEnding`][Token::CodeTextLineEnding]
+//! * [`CodeTextSequence`][Token::CodeTextSequence]
//!
//! ## References
//!
@@ -83,7 +83,8 @@
//! [code_fenced]: crate::construct::code_fenced
//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of code (text).
///
@@ -100,11 +101,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('`')
if tokenizer.previous != Code::Char('`')
- || (len > 0
- && tokenizer.events[len - 1].token_type == TokenType::CharacterEscape) =>
+ || (len > 0 && tokenizer.events[len - 1].token_type == Token::CharacterEscape) =>
{
- tokenizer.enter(TokenType::CodeText);
- tokenizer.enter(TokenType::CodeTextSequence);
+ tokenizer.enter(Token::CodeText);
+ tokenizer.enter(Token::CodeTextSequence);
sequence_open(tokenizer, code, 0)
}
_ => (State::Nok, None),
@@ -124,7 +124,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnR
None,
)
} else {
- tokenizer.exit(TokenType::CodeTextSequence);
+ tokenizer.exit(Token::CodeTextSequence);
between(tokenizer, code, size)
}
}
@@ -139,20 +139,20 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe
match code {
Code::None => (State::Nok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::CodeTextLineEnding);
+ tokenizer.enter(Token::CodeTextLineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CodeTextLineEnding);
+ tokenizer.exit(Token::CodeTextLineEnding);
(
State::Fn(Box::new(move |t, c| between(t, c, size_open))),
None,
)
}
Code::Char('`') => {
- tokenizer.enter(TokenType::CodeTextSequence);
+ tokenizer.enter(Token::CodeTextSequence);
sequence_close(tokenizer, code, size_open, 0)
}
_ => {
- tokenizer.enter(TokenType::CodeTextData);
+ tokenizer.enter(Token::CodeTextData);
data(tokenizer, code, size_open)
}
}
@@ -166,7 +166,7 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe
fn data(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '`') => {
- tokenizer.exit(TokenType::CodeTextData);
+ tokenizer.exit(Token::CodeTextData);
between(tokenizer, code, size_open)
}
_ => {
@@ -198,16 +198,16 @@ fn sequence_close(
)
}
_ if size_open == size => {
- tokenizer.exit(TokenType::CodeTextSequence);
- tokenizer.exit(TokenType::CodeText);
+ tokenizer.exit(Token::CodeTextSequence);
+ tokenizer.exit(Token::CodeText);
(State::Ok, Some(vec![code]))
}
_ => {
let index = tokenizer.events.len();
- tokenizer.exit(TokenType::CodeTextSequence);
+ tokenizer.exit(Token::CodeTextSequence);
// Change the token type.
- tokenizer.events[index - 1].token_type = TokenType::CodeTextData;
- tokenizer.events[index].token_type = TokenType::CodeTextData;
+ tokenizer.events[index - 1].token_type = Token::CodeTextData;
+ tokenizer.events[index].token_type = Token::CodeTextData;
between(tokenizer, code, size_open)
}
}
diff --git a/src/construct/definition.rs b/src/construct/definition.rs
index db4a009..4d14653 100644
--- a/src/construct/definition.rs
+++ b/src/construct/definition.rs
@@ -59,21 +59,21 @@
//!
//! ## Tokens
//!
-//! * [`Definition`][TokenType::Definition]
-//! * [`DefinitionDestination`][TokenType::DefinitionDestination]
-//! * [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
-//! * [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker]
-//! * [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
-//! * [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
-//! * [`DefinitionLabel`][TokenType::DefinitionLabel]
-//! * [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker]
-//! * [`DefinitionLabelString`][TokenType::DefinitionLabelString]
-//! * [`DefinitionMarker`][TokenType::DefinitionMarker]
-//! * [`DefinitionTitle`][TokenType::DefinitionTitle]
-//! * [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker]
-//! * [`DefinitionTitleString`][TokenType::DefinitionTitleString]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`Definition`][Token::Definition]
+//! * [`DefinitionDestination`][Token::DefinitionDestination]
+//! * [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral]
+//! * [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker]
+//! * [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw]
+//! * [`DefinitionDestinationString`][Token::DefinitionDestinationString]
+//! * [`DefinitionLabel`][Token::DefinitionLabel]
+//! * [`DefinitionLabelMarker`][Token::DefinitionLabelMarker]
+//! * [`DefinitionLabelString`][Token::DefinitionLabelString]
+//! * [`DefinitionMarker`][Token::DefinitionMarker]
+//! * [`DefinitionTitle`][Token::DefinitionTitle]
+//! * [`DefinitionTitleMarker`][Token::DefinitionTitleMarker]
+//! * [`DefinitionTitleString`][Token::DefinitionTitleString]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -99,7 +99,8 @@ use crate::construct::{
partial_space_or_tab::{space_or_tab, space_or_tab_eol},
partial_title::{start as title, Options as TitleOptions},
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// At the start of a definition.
///
@@ -109,14 +110,14 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
let index = tokenizer.events.len();
let definition_before = index > 3
- && tokenizer.events[index - 1].token_type == TokenType::LineEnding
- && tokenizer.events[index - 3].token_type == TokenType::Definition;
+ && tokenizer.events[index - 1].token_type == Token::LineEnding
+ && tokenizer.events[index - 3].token_type == Token::Definition;
// Do not interrupt paragraphs (but do follow definitions).
if tokenizer.interrupt && !definition_before {
(State::Nok, None)
} else {
- tokenizer.enter(TokenType::Definition);
+ tokenizer.enter(Token::Definition);
// Note: arbitrary whitespace allowed even if code (indented) is on.
tokenizer.attempt_opt(space_or_tab(), before)(tokenizer, code)
}
@@ -135,9 +136,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
LabelOptions {
- label: TokenType::DefinitionLabel,
- marker: TokenType::DefinitionLabelMarker,
- string: TokenType::DefinitionLabelString,
+ label: Token::DefinitionLabel,
+ marker: Token::DefinitionLabelMarker,
+ string: Token::DefinitionLabelString,
},
)
},
@@ -155,9 +156,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn label_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(':') => {
- tokenizer.enter(TokenType::DefinitionMarker);
+ tokenizer.enter(Token::DefinitionMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::DefinitionMarker);
+ tokenizer.exit(Token::DefinitionMarker);
(
State::Fn(Box::new(
tokenizer.attempt_opt(space_or_tab_eol(), destination_before),
@@ -185,11 +186,11 @@ fn destination_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
c,
DestinationOptions {
limit: usize::MAX,
- destination: TokenType::DefinitionDestination,
- literal: TokenType::DefinitionDestinationLiteral,
- marker: TokenType::DefinitionDestinationLiteralMarker,
- raw: TokenType::DefinitionDestinationRaw,
- string: TokenType::DefinitionDestinationString,
+ destination: Token::DefinitionDestination,
+ literal: Token::DefinitionDestinationLiteral,
+ marker: Token::DefinitionDestinationLiteralMarker,
+ raw: Token::DefinitionDestinationRaw,
+ string: Token::DefinitionDestinationString,
},
)
},
@@ -228,7 +229,7 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn after_whitespace(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Definition);
+ tokenizer.exit(Token::Definition);
// You’d be interrupting.
tokenizer.interrupt = true;
(State::Ok, Some(vec![code]))
@@ -262,9 +263,9 @@ fn title_before_marker(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
TitleOptions {
- title: TokenType::DefinitionTitle,
- marker: TokenType::DefinitionTitleMarker,
- string: TokenType::DefinitionTitleString,
+ title: Token::DefinitionTitle,
+ marker: Token::DefinitionTitleMarker,
+ string: Token::DefinitionTitleString,
},
)
},
diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs
index 212d276..4fb87bf 100644
--- a/src/construct/hard_break_escape.rs
+++ b/src/construct/hard_break_escape.rs
@@ -26,8 +26,8 @@
//!
//! ## Tokens
//!
-//! * [`HardBreakEscape`][TokenType::HardBreakEscape]
-//! * [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
+//! * [`HardBreakEscape`][Token::HardBreakEscape]
+//! * [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker]
//!
//! ## References
//!
@@ -40,7 +40,8 @@
//! [hard_break_trailing]: crate::construct::hard_break_trailing
//! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of a hard break (escape).
///
@@ -50,10 +51,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('\\') => {
- tokenizer.enter(TokenType::HardBreakEscape);
- tokenizer.enter(TokenType::HardBreakEscapeMarker);
+ tokenizer.enter(Token::HardBreakEscape);
+ tokenizer.enter(Token::HardBreakEscapeMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::HardBreakEscapeMarker);
+ tokenizer.exit(Token::HardBreakEscapeMarker);
(State::Fn(Box::new(inside)), None)
}
_ => (State::Nok, None),
@@ -68,7 +69,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HardBreakEscape);
+ tokenizer.exit(Token::HardBreakEscape);
(State::Ok, Some(vec![code]))
}
_ => (State::Nok, None),
diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs
index 35a7cab..6626675 100644
--- a/src/construct/hard_break_trailing.rs
+++ b/src/construct/hard_break_trailing.rs
@@ -26,8 +26,8 @@
//!
//! ## Tokens
//!
-//! * [`HardBreakTrailing`][TokenType::HardBreakTrailing]
-//! * [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
+//! * [`HardBreakTrailing`][Token::HardBreakTrailing]
+//! * [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace]
//!
//! ## References
//!
@@ -41,7 +41,8 @@
//! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element
use crate::constant::HARD_BREAK_PREFIX_SIZE_MIN;
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of a hard break (trailing).
///
@@ -52,8 +53,8 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(' ') => {
- tokenizer.enter(TokenType::HardBreakTrailing);
- tokenizer.enter(TokenType::HardBreakTrailingSpace);
+ tokenizer.enter(Token::HardBreakTrailing);
+ tokenizer.enter(Token::HardBreakTrailingSpace);
tokenizer.consume(code);
(State::Fn(Box::new(|t, c| inside(t, c, 1))), None)
}
@@ -79,8 +80,8 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnResult {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')
if size >= HARD_BREAK_PREFIX_SIZE_MIN =>
{
- tokenizer.exit(TokenType::HardBreakTrailingSpace);
- tokenizer.exit(TokenType::HardBreakTrailing);
+ tokenizer.exit(Token::HardBreakTrailingSpace);
+ tokenizer.exit(Token::HardBreakTrailing);
(State::Ok, Some(vec![code]))
}
_ => (State::Nok, None),
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 9fa2ace..4a4992a 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -37,10 +37,10 @@
//!
//! ## Tokens
//!
-//! * [`HeadingAtx`][TokenType::HeadingAtx]
-//! * [`HeadingAtxSequence`][TokenType::HeadingAtxSequence]
-//! * [`HeadingAtxText`][TokenType::HeadingAtxText]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`HeadingAtx`][Token::HeadingAtx]
+//! * [`HeadingAtxSequence`][Token::HeadingAtxSequence]
+//! * [`HeadingAtxText`][Token::HeadingAtxText]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -56,9 +56,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE};
-use crate::tokenizer::{
- Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer,
-};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::edit_map::EditMap;
/// Start of a heading (atx).
@@ -67,7 +66,7 @@ use crate::util::edit_map::EditMap;
/// |## alpha
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::HeadingAtx);
+ tokenizer.enter(Token::HeadingAtx);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -79,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// ```
fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if Code::Char('#') == code {
- tokenizer.enter(TokenType::HeadingAtxSequence);
+ tokenizer.enter(Token::HeadingAtxSequence);
sequence_open(tokenizer, code, 0)
} else {
(State::Nok, None)
@@ -94,7 +93,7 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if rank > 0 => {
- tokenizer.exit(TokenType::HeadingAtxSequence);
+ tokenizer.exit(Token::HeadingAtxSequence);
at_break(tokenizer, code)
}
Code::Char('#') if rank < HEADING_ATX_OPENING_FENCE_SIZE_MAX => {
@@ -107,7 +106,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR
)
}
_ if rank > 0 => {
- tokenizer.exit(TokenType::HeadingAtxSequence);
+ tokenizer.exit(Token::HeadingAtxSequence);
tokenizer.go(space_or_tab(), at_break)(tokenizer, code)
}
_ => (State::Nok, None),
@@ -126,7 +125,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR
fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HeadingAtx);
+ tokenizer.exit(Token::HeadingAtx);
tokenizer.register_resolver("heading_atx".to_string(), Box::new(resolve));
// Feel free to interrupt.
tokenizer.interrupt = false;
@@ -136,11 +135,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
tokenizer.go(space_or_tab(), at_break)(tokenizer, code)
}
Code::Char('#') => {
- tokenizer.enter(TokenType::HeadingAtxSequence);
+ tokenizer.enter(Token::HeadingAtxSequence);
further_sequence(tokenizer, code)
}
Code::Char(_) => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::Text));
data(tokenizer, code)
}
}
@@ -157,7 +156,7 @@ fn further_sequence(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
tokenizer.consume(code);
(State::Fn(Box::new(further_sequence)), None)
} else {
- tokenizer.exit(TokenType::HeadingAtxSequence);
+ tokenizer.exit(Token::HeadingAtxSequence);
at_break(tokenizer, code)
}
}
@@ -171,7 +170,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
// Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text.
Code::None | Code::CarriageReturnLineFeed | Code::Char('\t' | '\n' | '\r' | ' ') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code)
}
_ => {
@@ -192,7 +191,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.token_type == TokenType::HeadingAtx {
+ if event.token_type == Token::HeadingAtx {
if event.event_type == EventType::Enter {
heading_start = Some(index);
} else if let Some(start) = data_start {
@@ -204,7 +203,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Enter,
- token_type: TokenType::HeadingAtxText,
+ token_type: Token::HeadingAtxText,
point: tokenizer.events[start].point.clone(),
index: tokenizer.events[start].index,
previous: None,
@@ -221,7 +220,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::HeadingAtxText,
+ token_type: Token::HeadingAtxText,
point: tokenizer.events[end].point.clone(),
index: tokenizer.events[end].index,
previous: None,
@@ -234,7 +233,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
data_start = None;
data_end = None;
}
- } else if heading_start.is_some() && event.token_type == TokenType::Data {
+ } else if heading_start.is_some() && event.token_type == Token::Data {
if event.event_type == EventType::Enter {
if data_start.is_none() {
data_start = Some(index);
diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index 440baa8..633f7de 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -40,9 +40,9 @@
//!
//! ## Tokens
//!
-//! * [`HeadingSetext`][TokenType::HeadingSetext]
-//! * [`HeadingSetextText`][TokenType::HeadingSetextText]
-//! * [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline]
+//! * [`HeadingSetext`][Token::HeadingSetext]
+//! * [`HeadingSetextText`][Token::HeadingSetextText]
+//! * [`HeadingSetextUnderline`][Token::HeadingSetextUnderline]
//!
//! ## References
//!
@@ -59,7 +59,8 @@
use crate::constant::TAB_SIZE;
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::{edit_map::EditMap, skip::opt_back as skip_opt_back};
/// Kind of underline.
@@ -120,14 +121,14 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
skip_opt_back(
&tokenizer.events,
index - 1,
- &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix],
+ &[Token::SpaceOrTab, Token::BlockQuotePrefix],
)
} else {
0
};
- let previous = skip_opt_back(&tokenizer.events, previous, &[TokenType::LineEnding]);
+ let previous = skip_opt_back(&tokenizer.events, previous, &[Token::LineEnding]);
let paragraph_before =
- previous > 1 && tokenizer.events[previous].token_type == TokenType::Paragraph;
+ previous > 1 && tokenizer.events[previous].token_type == Token::Paragraph;
println!(
"setext-start: {:?} {:?} {:?}",
@@ -152,7 +153,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(char) if char == '-' || char == '=' => {
- tokenizer.enter(TokenType::HeadingSetextUnderline);
+ tokenizer.enter(Token::HeadingSetextUnderline);
inside(tokenizer, code, Kind::from_char(char))
}
_ => (State::Nok, None),
@@ -184,7 +185,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult {
fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HeadingSetextUnderline);
+ tokenizer.exit(Token::HeadingSetextUnderline);
// Feel free to interrupt.
tokenizer.interrupt = false;
tokenizer.register_resolver("heading_setext".to_string(), Box::new(resolve));
@@ -206,27 +207,27 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
// Find paragraphs.
if event.event_type == EventType::Enter {
- if event.token_type == TokenType::Paragraph {
+ if event.token_type == Token::Paragraph {
paragraph_enter = Some(index);
}
- } else if event.token_type == TokenType::Paragraph {
+ } else if event.token_type == Token::Paragraph {
paragraph_exit = Some(index);
}
// We know this is preceded by a paragraph.
// Otherwise we don’t parse.
- else if event.token_type == TokenType::HeadingSetextUnderline {
+ else if event.token_type == Token::HeadingSetextUnderline {
let enter = paragraph_enter.take().unwrap();
let exit = paragraph_exit.take().unwrap();
// Change types of Enter:Paragraph, Exit:Paragraph.
- tokenizer.events[enter].token_type = TokenType::HeadingSetextText;
- tokenizer.events[exit].token_type = TokenType::HeadingSetextText;
+ tokenizer.events[enter].token_type = Token::HeadingSetextText;
+ tokenizer.events[exit].token_type = Token::HeadingSetextText;
// Add Enter:HeadingSetext, Exit:HeadingSetext.
let mut heading_enter = tokenizer.events[enter].clone();
- heading_enter.token_type = TokenType::HeadingSetext;
+ heading_enter.token_type = Token::HeadingSetext;
let mut heading_exit = tokenizer.events[index].clone();
- heading_exit.token_type = TokenType::HeadingSetext;
+ heading_exit.token_type = Token::HeadingSetext;
edit_map.add(enter, 0, vec![heading_enter]);
edit_map.add(index + 1, 0, vec![heading_exit]);
diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs
index be7a3a9..fde0a34 100644
--- a/src/construct/html_flow.rs
+++ b/src/construct/html_flow.rs
@@ -82,9 +82,9 @@
//!
//! ## Tokens
//!
-//! * [`HtmlFlow`][TokenType::HtmlFlow]
-//! * [`HtmlFlowData`][TokenType::HtmlFlowData]
-//! * [`LineEnding`][TokenType::LineEnding]
+//! * [`HtmlFlow`][Token::HtmlFlow]
+//! * [`HtmlFlowData`][Token::HtmlFlowData]
+//! * [`LineEnding`][Token::LineEnding]
//!
//! ## References
//!
@@ -102,7 +102,8 @@ use crate::constant::{HTML_BLOCK_NAMES, HTML_RAW_NAMES, HTML_RAW_SIZE_MAX, TAB_S
use crate::construct::{
blank_line::start as blank_line, partial_space_or_tab::space_or_tab_min_max,
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
use crate::util::codes::{parse, serialize};
// To do: mark as concrete (block quotes or lists can’t “pierce” into HTML).
@@ -203,8 +204,8 @@ struct Info {
/// ```
///
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::HtmlFlow);
- tokenizer.enter(TokenType::HtmlFlowData);
+ tokenizer.enter(Token::HtmlFlow);
+ tokenizer.enter(Token::HtmlFlowData);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -776,7 +777,7 @@ fn continuation(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRes
/// <x>|
/// ```
fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
- tokenizer.exit(TokenType::HtmlFlowData);
+ tokenizer.exit(Token::HtmlFlowData);
html_continue_start(tokenizer, code, info)
}
@@ -789,23 +790,23 @@ fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info
fn html_continue_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None => {
- tokenizer.exit(TokenType::HtmlFlow);
+ tokenizer.exit(Token::HtmlFlow);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
}
// To do: do not allow lazy lines.
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(
State::Fn(Box::new(|t, c| html_continue_start(t, c, info))),
None,
)
}
_ => {
- tokenizer.enter(TokenType::HtmlFlowData);
+ tokenizer.enter(Token::HtmlFlowData);
continuation(tokenizer, code, info)
}
}
@@ -955,8 +956,8 @@ fn continuation_declaration_inside(
fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HtmlFlowData);
- tokenizer.exit(TokenType::HtmlFlow);
+ tokenizer.exit(Token::HtmlFlowData);
+ tokenizer.exit(Token::HtmlFlow);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
@@ -978,8 +979,8 @@ fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> Stat
///
/// ```
fn blank_line_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(blank_line)), None)
}
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs
index 0926f48..cdd7c69 100644
--- a/src/construct/html_text.rs
+++ b/src/construct/html_text.rs
@@ -42,8 +42,8 @@
//!
//! ## Tokens
//!
-//! * [`HtmlText`][TokenType::HtmlText]
-//! * [`HtmlTextData`][TokenType::HtmlTextData]
+//! * [`HtmlText`][Token::HtmlText]
+//! * [`HtmlTextData`][Token::HtmlTextData]
//!
//! ## References
//!
@@ -55,7 +55,8 @@
//! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing
use crate::construct::partial_space_or_tab::space_or_tab;
-use crate::tokenizer::{Code, State, StateFn, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFn, StateFnResult, Tokenizer};
use crate::util::codes::parse;
/// Start of HTML (text)
@@ -65,8 +66,8 @@ use crate::util::codes::parse;
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if Code::Char('<') == code {
- tokenizer.enter(TokenType::HtmlText);
- tokenizer.enter(TokenType::HtmlTextData);
+ tokenizer.enter(Token::HtmlText);
+ tokenizer.enter(Token::HtmlTextData);
tokenizer.consume(code);
(State::Fn(Box::new(open)), None)
} else {
@@ -617,8 +618,8 @@ fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
tokenizer.consume(code);
- tokenizer.exit(TokenType::HtmlTextData);
- tokenizer.exit(TokenType::HtmlText);
+ tokenizer.exit(Token::HtmlTextData);
+ tokenizer.exit(Token::HtmlText);
(State::Ok, None)
}
_ => (State::Nok, None),
@@ -641,10 +642,10 @@ fn at_line_ending(
) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HtmlTextData);
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.exit(Token::HtmlTextData);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(
State::Fn(Box::new(|t, c| after_line_ending(t, c, return_state))),
None,
@@ -687,6 +688,6 @@ fn after_line_ending_prefix(
code: Code,
return_state: Box<StateFn>,
) -> StateFnResult {
- tokenizer.enter(TokenType::HtmlTextData);
+ tokenizer.enter(Token::HtmlTextData);
return_state(tokenizer, code)
}
diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs
index 3a40cc2..29ac6f9 100644
--- a/src/construct/label_end.rs
+++ b/src/construct/label_end.rs
@@ -102,28 +102,28 @@
//!
//! ## Tokens
//!
-//! * [`Data`][TokenType::Data]
-//! * [`Image`][TokenType::Image]
-//! * [`Label`][TokenType::Label]
-//! * [`LabelEnd`][TokenType::LabelEnd]
-//! * [`LabelMarker`][TokenType::LabelMarker]
-//! * [`LabelText`][TokenType::LabelText]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`Link`][TokenType::Link]
-//! * [`Reference`][TokenType::Reference]
-//! * [`ReferenceMarker`][TokenType::ReferenceMarker]
-//! * [`ReferenceString`][TokenType::ReferenceString]
-//! * [`Resource`][TokenType::Resource]
-//! * [`ResourceDestination`][TokenType::ResourceDestination]
-//! * [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral]
-//! * [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker]
-//! * [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw]
-//! * [`ResourceDestinationString`][TokenType::ResourceDestinationString]
-//! * [`ResourceMarker`][TokenType::ResourceMarker]
-//! * [`ResourceTitle`][TokenType::ResourceTitle]
-//! * [`ResourceTitleMarker`][TokenType::ResourceTitleMarker]
-//! * [`ResourceTitleString`][TokenType::ResourceTitleString]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`Data`][Token::Data]
+//! * [`Image`][Token::Image]
+//! * [`Label`][Token::Label]
+//! * [`LabelEnd`][Token::LabelEnd]
+//! * [`LabelMarker`][Token::LabelMarker]
+//! * [`LabelText`][Token::LabelText]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`Link`][Token::Link]
+//! * [`Reference`][Token::Reference]
+//! * [`ReferenceMarker`][Token::ReferenceMarker]
+//! * [`ReferenceString`][Token::ReferenceString]
+//! * [`Resource`][Token::Resource]
+//! * [`ResourceDestination`][Token::ResourceDestination]
+//! * [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral]
+//! * [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker]
+//! * [`ResourceDestinationRaw`][Token::ResourceDestinationRaw]
+//! * [`ResourceDestinationString`][Token::ResourceDestinationString]
+//! * [`ResourceMarker`][Token::ResourceMarker]
+//! * [`ResourceTitle`][Token::ResourceTitle]
+//! * [`ResourceTitleMarker`][Token::ResourceTitleMarker]
+//! * [`ResourceTitleString`][Token::ResourceTitleString]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -153,8 +153,9 @@ use crate::construct::{
partial_space_or_tab::space_or_tab_eol,
partial_title::{start as title, Options as TitleOptions},
};
+use crate::token::Token;
use crate::tokenizer::{
- Code, Event, EventType, LabelStart, Media, State, StateFnResult, TokenType, Tokenizer,
+ Code, Event, EventType, LabelStart, Media, State, StateFnResult, Tokenizer,
};
use crate::util::{
edit_map::EditMap,
@@ -198,7 +199,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
vec![
Event {
event_type: EventType::Enter,
- token_type: TokenType::Data,
+ token_type: Token::Data,
point: events[data_enter_index].point.clone(),
index: events[data_enter_index].index,
previous: None,
@@ -207,7 +208,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
},
Event {
event_type: EventType::Exit,
- token_type: TokenType::Data,
+ token_type: Token::Data,
point: events[data_exit_index].point.clone(),
index: events[data_exit_index].index,
previous: None,
@@ -229,7 +230,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
let group_enter_event = &events[group_enter_index];
// LabelLink:Exit or LabelImage:Exit.
let text_enter_index = media.start.0
- + (if group_enter_event.token_type == TokenType::LabelLink {
+ + (if group_enter_event.token_type == Token::LabelLink {
4
} else {
6
@@ -248,10 +249,10 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
vec![
Event {
event_type: EventType::Enter,
- token_type: if group_enter_event.token_type == TokenType::LabelLink {
- TokenType::Link
+ token_type: if group_enter_event.token_type == Token::LabelLink {
+ Token::Link
} else {
- TokenType::Image
+ Token::Image
},
point: group_enter_event.point.clone(),
index: group_enter_event.index,
@@ -261,7 +262,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
},
Event {
event_type: EventType::Enter,
- token_type: TokenType::Label,
+ token_type: Token::Label,
point: group_enter_event.point.clone(),
index: group_enter_event.index,
previous: None,
@@ -279,7 +280,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Enter,
- token_type: TokenType::LabelText,
+ token_type: Token::LabelText,
point: events[text_enter_index].point.clone(),
index: events[text_enter_index].index,
previous: None,
@@ -294,7 +295,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::LabelText,
+ token_type: Token::LabelText,
point: events[text_exit_index].point.clone(),
index: events[text_exit_index].index,
previous: None,
@@ -310,7 +311,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::Label,
+ token_type: Token::Label,
point: events[label_exit_index].point.clone(),
index: events[label_exit_index].index,
previous: None,
@@ -325,7 +326,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::Link,
+ token_type: Token::Link,
point: events[group_end_index].point.clone(),
index: events[group_end_index].index,
previous: None,
@@ -393,11 +394,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
},
};
- tokenizer.enter(TokenType::LabelEnd);
- tokenizer.enter(TokenType::LabelMarker);
+ tokenizer.enter(Token::LabelEnd);
+ tokenizer.enter(Token::LabelMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelMarker);
- tokenizer.exit(TokenType::LabelEnd);
+ tokenizer.exit(Token::LabelMarker);
+ tokenizer.exit(Token::LabelEnd);
return (State::Fn(Box::new(move |t, c| after(t, c, info))), None);
}
@@ -495,13 +496,13 @@ fn ok(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
left.remove(0);
tokenizer.label_start_list_loose.append(&mut left);
- let is_link = tokenizer.events[info.media.start.0].token_type == TokenType::LabelLink;
+ let is_link = tokenizer.events[info.media.start.0].token_type == Token::LabelLink;
if is_link {
let mut index = 0;
while index < tokenizer.label_start_stack.len() {
let label_start = &mut tokenizer.label_start_stack[index];
- if tokenizer.events[label_start.start.0].token_type == TokenType::LabelLink {
+ if tokenizer.events[label_start.start.0].token_type == Token::LabelLink {
label_start.inactive = true;
}
index += 1;
@@ -543,10 +544,10 @@ fn nok(tokenizer: &mut Tokenizer, _code: Code, label_start_index: usize) -> Stat
fn resource(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('(') => {
- tokenizer.enter(TokenType::Resource);
- tokenizer.enter(TokenType::ResourceMarker);
+ tokenizer.enter(Token::Resource);
+ tokenizer.enter(Token::ResourceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ResourceMarker);
+ tokenizer.exit(Token::ResourceMarker);
(State::Fn(Box::new(resource_start)), None)
}
_ => unreachable!("expected `(`"),
@@ -577,11 +578,11 @@ fn resource_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
c,
DestinationOptions {
limit: RESOURCE_DESTINATION_BALANCE_MAX,
- destination: TokenType::ResourceDestination,
- literal: TokenType::ResourceDestinationLiteral,
- marker: TokenType::ResourceDestinationLiteralMarker,
- raw: TokenType::ResourceDestinationRaw,
- string: TokenType::ResourceDestinationString,
+ destination: Token::ResourceDestination,
+ literal: Token::ResourceDestinationLiteral,
+ marker: Token::ResourceDestinationLiteralMarker,
+ raw: Token::ResourceDestinationRaw,
+ string: Token::ResourceDestinationString,
},
)
},
@@ -616,9 +617,9 @@ fn resource_between(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
TitleOptions {
- title: TokenType::ResourceTitle,
- marker: TokenType::ResourceTitleMarker,
- string: TokenType::ResourceTitleString,
+ title: Token::ResourceTitle,
+ marker: Token::ResourceTitleMarker,
+ string: Token::ResourceTitleString,
},
)
},
@@ -647,10 +648,10 @@ fn title_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn resource_end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(')') => {
- tokenizer.enter(TokenType::ResourceMarker);
+ tokenizer.enter(Token::ResourceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ResourceMarker);
- tokenizer.exit(TokenType::Resource);
+ tokenizer.exit(Token::ResourceMarker);
+ tokenizer.exit(Token::Resource);
(State::Ok, None)
}
_ => (State::Nok, None),
@@ -670,9 +671,9 @@ fn full_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
LabelOptions {
- label: TokenType::Reference,
- marker: TokenType::ReferenceMarker,
- string: TokenType::ReferenceString,
+ label: Token::Reference,
+ marker: Token::ReferenceMarker,
+ string: Token::ReferenceString,
},
)
},
@@ -696,7 +697,7 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
while index > 0 {
index -= 1;
let event = &events[index];
- if event.token_type == TokenType::ReferenceString {
+ if event.token_type == Token::ReferenceString {
if event.event_type == EventType::Exit {
end = Some(event.index);
} else {
@@ -735,10 +736,10 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('[') => {
- tokenizer.enter(TokenType::Reference);
- tokenizer.enter(TokenType::ReferenceMarker);
+ tokenizer.enter(Token::Reference);
+ tokenizer.enter(Token::ReferenceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ReferenceMarker);
+ tokenizer.exit(Token::ReferenceMarker);
(State::Fn(Box::new(collapsed_reference_open)), None)
}
_ => (State::Nok, None),
@@ -755,10 +756,10 @@ fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn collapsed_reference_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(']') => {
- tokenizer.enter(TokenType::ReferenceMarker);
+ tokenizer.enter(Token::ReferenceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ReferenceMarker);
- tokenizer.exit(TokenType::Reference);
+ tokenizer.exit(Token::ReferenceMarker);
+ tokenizer.exit(Token::Reference);
(State::Ok, None)
}
_ => (State::Nok, None),
diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs
index a45205a..f9b8300 100644
--- a/src/construct/label_start_image.rs
+++ b/src/construct/label_start_image.rs
@@ -15,9 +15,9 @@
//!
//! ## Tokens
//!
-//! * [`LabelImage`][TokenType::LabelImage]
-//! * [`LabelImageMarker`][TokenType::LabelImageMarker]
-//! * [`LabelMarker`][TokenType::LabelMarker]
+//! * [`LabelImage`][Token::LabelImage]
+//! * [`LabelImageMarker`][Token::LabelImageMarker]
+//! * [`LabelMarker`][Token::LabelMarker]
//!
//! ## References
//!
@@ -29,7 +29,8 @@
//! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element
use super::label_end::resolve_media;
-use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer};
/// Start of label (image) start.
///
@@ -39,10 +40,10 @@ use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokeni
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('!') => {
- tokenizer.enter(TokenType::LabelImage);
- tokenizer.enter(TokenType::LabelImageMarker);
+ tokenizer.enter(Token::LabelImage);
+ tokenizer.enter(Token::LabelImageMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelImageMarker);
+ tokenizer.exit(Token::LabelImageMarker);
(State::Fn(Box::new(open)), None)
}
_ => (State::Nok, None),
@@ -57,10 +58,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
pub fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('[') => {
- tokenizer.enter(TokenType::LabelMarker);
+ tokenizer.enter(Token::LabelMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelMarker);
- tokenizer.exit(TokenType::LabelImage);
+ tokenizer.exit(Token::LabelMarker);
+ tokenizer.exit(Token::LabelImage);
let end = tokenizer.events.len() - 1;
tokenizer.label_start_stack.push(LabelStart {
start: (end - 5, end),
diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs
index 6c4d7ae..59729cc 100644
--- a/src/construct/label_start_link.rs
+++ b/src/construct/label_start_link.rs
@@ -15,8 +15,8 @@
//!
//! ## Tokens
//!
-//! * [`LabelLink`][TokenType::LabelLink]
-//! * [`LabelMarker`][TokenType::LabelMarker]
+//! * [`LabelLink`][Token::LabelLink]
+//! * [`LabelMarker`][Token::LabelMarker]
//!
//! ## References
//!
@@ -28,7 +28,8 @@
//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use super::label_end::resolve_media;
-use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer};
/// Start of label (link) start.
///
@@ -39,11 +40,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('[') => {
let start = tokenizer.events.len();
- tokenizer.enter(TokenType::LabelLink);
- tokenizer.enter(TokenType::LabelMarker);
+ tokenizer.enter(Token::LabelLink);
+ tokenizer.enter(Token::LabelMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelMarker);
- tokenizer.exit(TokenType::LabelLink);
+ tokenizer.exit(Token::LabelMarker);
+ tokenizer.exit(Token::LabelLink);
tokenizer.label_start_stack.push(LabelStart {
start: (start, tokenizer.events.len() - 1),
balanced: false,
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index ace174f..967e009 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -19,7 +19,7 @@
//!
//! ## Tokens
//!
-//! * [`Paragraph`][TokenType::Paragraph]
+//! * [`Paragraph`][Token::Paragraph]
//!
//! ## References
//!
@@ -32,9 +32,8 @@
//! [code_text]: crate::construct::code_text
//! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element
-use crate::tokenizer::{
- Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer,
-};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::{edit_map::EditMap, skip::opt as skip_opt};
/// Before a paragraph.
@@ -48,8 +47,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
unreachable!("unexpected eol/eof")
}
_ => {
- tokenizer.enter(TokenType::Paragraph);
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text));
+ tokenizer.enter(Token::Paragraph);
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::Text));
inside(tokenizer, code)
}
}
@@ -63,8 +62,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::Paragraph);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::Paragraph);
tokenizer.register_resolver_before("paragraph".to_string(), Box::new(resolve));
// You’d be interrupting.
tokenizer.interrupt = true;
@@ -87,21 +86,21 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
while index < len {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == TokenType::Paragraph {
+ if event.event_type == EventType::Enter && event.token_type == Token::Paragraph {
// Exit:Paragraph
let mut exit_index = index + 3;
let mut enter_next_index =
- skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]);
+ skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]);
// Enter:Paragraph
enter_next_index = skip_opt(
&tokenizer.events,
enter_next_index,
- &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix],
+ &[Token::SpaceOrTab, Token::BlockQuotePrefix],
);
// Find future `Paragraphs`.
while enter_next_index < tokenizer.events.len()
- && tokenizer.events[enter_next_index].token_type == TokenType::Paragraph
+ && tokenizer.events[enter_next_index].token_type == Token::Paragraph
{
// Remove Exit:Paragraph, Enter:LineEnding, Exit:LineEnding, Enter:Paragraph.
edit_map.add(exit_index, 3, vec![]);
@@ -126,11 +125,11 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
// Potential next start.
exit_index = enter_next_index + 3;
enter_next_index =
- skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]);
+ skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]);
enter_next_index = skip_opt(
&tokenizer.events,
enter_next_index,
- &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix],
+ &[Token::SpaceOrTab, Token::BlockQuotePrefix],
);
}
diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs
index 555ccaf..186665d 100644
--- a/src/construct/partial_data.rs
+++ b/src/construct/partial_data.rs
@@ -6,7 +6,8 @@
//! [string]: crate::content::string
//! [text]: crate::content::text
-use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::edit_map::EditMap;
/// At the beginning of data.
@@ -16,7 +17,7 @@ use crate::util::edit_map::EditMap;
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult {
if stop.contains(&code) {
- tokenizer.enter(TokenType::Data);
+ tokenizer.enter(Token::Data);
tokenizer.consume(code);
(State::Fn(Box::new(|t, c| data(t, c, stop))), None)
} else {
@@ -33,9 +34,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| at_break(t, c, stop))), None)
}
_ if stop.contains(&code) => {
@@ -43,7 +44,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe
(State::Ok, Some(vec![code]))
}
_ => {
- tokenizer.enter(TokenType::Data);
+ tokenizer.enter(Token::Data);
data(tokenizer, code, stop)
}
}
@@ -62,7 +63,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult
};
if done {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, stop)
} else {
tokenizer.consume(code);
@@ -80,13 +81,13 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
while index < len {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == TokenType::Data {
+ if event.event_type == EventType::Enter && event.token_type == Token::Data {
let exit_index = index + 1;
let mut exit_far_index = exit_index;
// Find multiple `data` events.
while exit_far_index + 1 < len
- && tokenizer.events[exit_far_index + 1].token_type == TokenType::Data
+ && tokenizer.events[exit_far_index + 1].token_type == Token::Data
{
exit_far_index += 2;
}
diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs
index 31c13ec..daa968a 100644
--- a/src/construct/partial_destination.rs
+++ b/src/construct/partial_destination.rs
@@ -71,7 +71,8 @@
//! [label_end]: crate::construct::label_end
//! [sanitize_uri]: crate::util::sanitize_uri
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -79,15 +80,15 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole destination.
- pub destination: TokenType,
+ pub destination: Token,
/// Token for a literal (enclosed) destination.
- pub literal: TokenType,
+ pub literal: Token,
/// Token for a literal marker.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for a raw destination.
- pub raw: TokenType,
+ pub raw: Token,
/// Token for a the string.
- pub string: TokenType,
+ pub string: Token,
/// Maximum unbalanced parens.
pub limit: usize,
}
@@ -133,7 +134,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code, options: Options) -> StateFn
tokenizer.enter(info.options.destination.clone());
tokenizer.enter(info.options.raw.clone());
tokenizer.enter(info.options.string.clone());
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
raw(tokenizer, code, info)
}
}
@@ -154,7 +155,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn
(State::Ok, None)
} else {
tokenizer.enter(info.options.string.clone());
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
enclosed(tokenizer, code, info)
}
}
@@ -167,7 +168,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn
fn enclosed(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
enclosed_before(tokenizer, code, info)
}
@@ -221,7 +222,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
}
Code::Char(')') => {
if info.balance == 0 {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
tokenizer.exit(info.options.raw.clone());
tokenizer.exit(info.options.destination);
@@ -239,7 +240,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
if info.balance > 0 {
(State::Nok, None)
} else {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
tokenizer.exit(info.options.raw.clone());
tokenizer.exit(info.options.destination);
diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs
index f201f60..f380c7d 100644
--- a/src/construct/partial_label.rs
+++ b/src/construct/partial_label.rs
@@ -61,7 +61,8 @@
use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::constant::LINK_REFERENCE_SIZE_MAX;
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -69,11 +70,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole label.
- pub label: TokenType,
+ pub label: Token,
/// Token for the markers.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for the string (inside the markers).
- pub string: TokenType,
+ pub string: Token,
}
/// State needed to parse labels.
@@ -144,7 +145,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
},
)(tokenizer, code),
_ => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -166,11 +167,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
fn label(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '[' | ']') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
_ if info.size > LINK_REFERENCE_SIZE_MAX => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::VirtualSpace | Code::Char('\t' | ' ') => {
diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs
index 5b1ec5e..78477de 100644
--- a/src/construct/partial_space_or_tab.rs
+++ b/src/construct/partial_space_or_tab.rs
@@ -5,7 +5,8 @@
//! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js)
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, Tokenizer};
/// Options to parse `space_or_tab`.
#[derive(Debug)]
@@ -15,7 +16,7 @@ pub struct Options {
/// Maximum allowed characters (inclusive).
pub max: usize,
/// Token type to use for whitespace events.
- pub kind: TokenType,
+ pub kind: Token,
/// Connect this whitespace to the previous.
pub connect: bool,
/// Embedded content type to use.
@@ -67,7 +68,7 @@ pub fn space_or_tab() -> Box<StateFn> {
/// ```
pub fn space_or_tab_min_max(min: usize, max: usize) -> Box<StateFn> {
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min,
max,
content_type: None,
@@ -104,7 +105,7 @@ pub fn space_or_tab_eol_with_options(options: EolOptions) -> Box<StateFn> {
tokenizer.attempt(
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: info.options.content_type,
@@ -196,7 +197,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResul
fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter_with_content(TokenType::LineEnding, info.options.content_type);
+ tokenizer.enter_with_content(Token::LineEnding, info.options.content_type);
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -206,7 +207,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo)
}
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| after_eol(t, c, info))), None)
}
_ if info.ok => (State::Ok, Some(vec![code])),
@@ -229,7 +230,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo)
fn after_eol(tokenizer: &mut Tokenizer, code: Code, info: EolInfo) -> StateFnResult {
tokenizer.attempt_opt(
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: info.options.content_type,
diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs
index 010f554..6303da8 100644
--- a/src/construct/partial_title.rs
+++ b/src/construct/partial_title.rs
@@ -32,7 +32,8 @@
use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -40,11 +41,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole title.
- pub title: TokenType,
+ pub title: Token,
/// Token for the marker.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for the string inside the quotes.
- pub string: TokenType,
+ pub string: Token,
}
/// Type of title.
@@ -204,7 +205,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
},
)(tokenizer, code),
_ => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -226,11 +227,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
fn title(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::Char('\\') => {
diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs
index 28aca34..4d92f8d 100644
--- a/src/construct/thematic_break.rs
+++ b/src/construct/thematic_break.rs
@@ -35,8 +35,8 @@
//!
//! ## Tokens
//!
-//! * [`ThematicBreak`][TokenType::ThematicBreak]
-//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence]
+//! * [`ThematicBreak`][Token::ThematicBreak]
+//! * [`ThematicBreakSequence`][Token::ThematicBreakSequence]
//!
//! ## References
//!
@@ -51,7 +51,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Type of thematic break.
#[derive(Debug, PartialEq)]
@@ -134,7 +135,7 @@ struct Info {
/// |***
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::ThematicBreak);
+ tokenizer.enter(Token::ThematicBreak);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -170,13 +171,13 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')
if info.size >= THEMATIC_BREAK_MARKER_COUNT_MIN =>
{
- tokenizer.exit(TokenType::ThematicBreak);
+ tokenizer.exit(Token::ThematicBreak);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
}
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.enter(TokenType::ThematicBreakSequence);
+ tokenizer.enter(Token::ThematicBreakSequence);
sequence(tokenizer, code, info)
}
_ => (State::Nok, None),
@@ -198,7 +199,7 @@ fn sequence(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
(State::Fn(Box::new(|t, c| sequence(t, c, info))), None)
}
_ => {
- tokenizer.exit(TokenType::ThematicBreakSequence);
+ tokenizer.exit(Token::ThematicBreakSequence);
tokenizer.attempt_opt(space_or_tab(), |t, c| at_break(t, c, info))(tokenizer, code)
}
}
diff --git a/src/content/document.rs b/src/content/document.rs
index dd5038f..b1f3083 100644
--- a/src/content/document.rs
+++ b/src/content/document.rs
@@ -14,9 +14,8 @@ use crate::construct::block_quote::{
use crate::content::flow::start as flow;
use crate::parser::ParseState;
use crate::subtokenize::subtokenize;
-use crate::tokenizer::{
- Code, Event, EventType, Point, State, StateFn, StateFnResult, TokenType, Tokenizer,
-};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, Point, State, StateFn, StateFnResult, Tokenizer};
use crate::util::edit_map::EditMap;
use crate::util::{
normalize_identifier::normalize_identifier,
@@ -44,9 +43,7 @@ pub fn document(parse_state: &mut ParseState, point: Point, index: usize) -> Vec
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Exit
- && event.token_type == TokenType::DefinitionLabelString
- {
+ if event.event_type == EventType::Exit && event.token_type == Token::DefinitionLabelString {
next_definitions.insert(normalize_identifier(
serialize(
&parse_state.codes,
@@ -409,7 +406,7 @@ fn flow_end(
// To do: blank lines? Other things?
if tokenizer.events.len() > 2
- && tokenizer.events[tokenizer.events.len() - 1].token_type == TokenType::LineEnding
+ && tokenizer.events[tokenizer.events.len() - 1].token_type == Token::LineEnding
{
info.last_line_ending_index = Some(tokenizer.events.len() - 2);
} else {
diff --git a/src/content/flow.rs b/src/content/flow.rs
index f406685..e52f113 100644
--- a/src/content/flow.rs
+++ b/src/content/flow.rs
@@ -26,7 +26,8 @@ use crate::construct::{
html_flow::start as html_flow, paragraph::start as paragraph,
thematic_break::start as thematic_break,
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Before flow.
///
@@ -88,9 +89,9 @@ fn blank_line_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::BlankLineEnding);
+ tokenizer.enter(Token::BlankLineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::BlankLineEnding);
+ tokenizer.exit(Token::BlankLineEnding);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Fn(Box::new(start)), None)
@@ -112,9 +113,9 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(start)), None)
}
_ => unreachable!("expected eol/eof"),
diff --git a/src/lib.rs b/src/lib.rs
index e1f26b4..1f49303 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -10,6 +10,7 @@ mod construct;
mod content;
mod parser;
mod subtokenize;
+mod token;
mod tokenizer;
mod unicode;
mod util;
diff --git a/src/token.rs b/src/token.rs
new file mode 100644
index 0000000..9b59719
--- /dev/null
+++ b/src/token.rs
@@ -0,0 +1,1768 @@
+/// Semantic label of a span.
+#[derive(Debug, Clone, PartialEq, Hash, Eq)]
+pub enum Token {
+ /// Attention sequence.
+ ///
+ /// > 👉 **Note**: this is used while parsing but compiled away.
+ AttentionSequence,
+ /// Whole autolink.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`AutolinkEmail`][Token::AutolinkEmail],
+ /// [`AutolinkMarker`][Token::AutolinkMarker],
+ /// [`AutolinkProtocol`][Token::AutolinkProtocol]
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <https://example.com> and <admin@example.com>
+ /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
+ /// ```
+ Autolink,
+ /// Email autolink w/o markers.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][Token::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <admin@example.com>
+ /// ^^^^^^^^^^^^^^^^^
+ /// ```
+ AutolinkEmail,
+ /// Marker of an autolink.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][Token::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <https://example.com>
+ /// ^ ^
+ /// ```
+ AutolinkMarker,
+ /// Protocol autolink w/o markers.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Autolink`][Token::Autolink]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`autolink`][crate::construct::autolink]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <https://example.com>
+ /// ^^^^^^^^^^^^^^^^^^^
+ /// ```
+ AutolinkProtocol,
+ /// Line ending preceded only by whitespace or nothing at all.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`blank_line`][crate::construct::blank_line]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | ␠␠␊
+ /// ^
+ /// ```
+ BlankLineEnding,
+ /// Whole block quote.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [document content][crate::content::document]
+ /// * **Content model**:
+ /// [`BlockQuotePrefix`][Token::BlockQuotePrefix],
+ /// [flow content][crate::content::flow]
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^^^
+ /// > | b
+ /// ^
+ /// ```
+ BlockQuote,
+ /// Block quote marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`BlockQuotePrefix`][Token::BlockQuotePrefix]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^
+ /// | b
+ /// ```
+ BlockQuoteMarker,
+ /// Block quote prefix.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`BlockQuote`][Token::BlockQuote]
+ /// * **Content model**:
+ /// [`BlockQuoteMarker`][Token::BlockQuoteMarker],
+ /// [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace]
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^^
+ /// | b
+ /// ```
+ BlockQuotePrefix,
+ /// Block quote white space.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`BlockQuotePrefix`][Token::BlockQuotePrefix]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`block_quote`][crate::construct::block_quote]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | > a
+ /// ^
+ /// | b
+ /// ```
+ BlockQuoteWhitespace,
+ /// Whole character escape.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string] or
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CharacterEscapeMarker`][Token::CharacterEscapeMarker],
+ /// [`CharacterEscapeValue`][Token::CharacterEscapeValue]
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^^
+ /// ```
+ CharacterEscape,
+ /// Character escape marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterEscape`][Token::CharacterEscape]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^
+ /// ```
+ CharacterEscapeMarker,
+ /// Character escape value.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterEscape`][Token::CharacterEscape]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_escape`][crate::construct::character_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a \- b
+ /// ^
+ /// ```
+ CharacterEscapeValue,
+ /// Whole character reference.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string] or
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CharacterReferenceMarker`][Token::CharacterReferenceMarker],
+ /// [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal],
+ /// [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric],
+ /// [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi],
+ /// [`CharacterReferenceValue`][Token::CharacterReferenceValue]
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^^^^^ ^^^^^^^ ^^^^^^^^^
+ /// ```
+ CharacterReference,
+ /// Character reference opening marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^ ^ ^
+ /// ```
+ CharacterReferenceMarker,
+ /// Character reference hexadecimal numeric marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^
+ /// ```
+ CharacterReferenceMarkerHexadecimal,
+ /// Character reference numeric marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^ ^
+ /// ```
+ CharacterReferenceMarkerNumeric,
+ /// Character reference closing marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^ ^ ^
+ /// ```
+ CharacterReferenceMarkerSemi,
+ /// Character reference value.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CharacterReference`][Token::CharacterReference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`character_reference`][crate::construct::character_reference]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a &amp; b &#8800; c &#x1D306; d
+ /// ^^^ ^^^^ ^^^^^
+ /// ```
+ CharacterReferenceValue,
+ /// Whole code (fenced).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`CodeFencedFence`][Token::CodeFencedFence],
+ /// [`CodeFlowChunk`][Token::CodeFlowChunk],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^^^
+ /// > | console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// > | ```
+ /// ^^^
+ /// ````
+ CodeFenced,
+ /// A code (fenced) fence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFenced`][Token::CodeFenced]
+ /// * **Content model**:
+ /// [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo],
+ /// [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta],
+ /// [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^^^
+ /// | console.log(1)
+ /// > | ```
+ /// ^^^
+ /// ````
+ CodeFencedFence,
+ /// A code (fenced) fence info word.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFence`][Token::CodeFencedFence]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^
+ /// | console.log(1)
+ /// | ```
+ /// ````
+ CodeFencedFenceInfo,
+ /// A code (fenced) fence meta string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFence`][Token::CodeFencedFence]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js highlight="1"
+ /// ^^^^^^^^^^^^^
+ /// | console.log(1)
+ /// | ```
+ /// ````
+ CodeFencedFenceMeta,
+ /// A code (fenced) fence sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// > | ```js
+ /// ^^^
+ /// | console.log(1)
+ /// > | ```
+ /// ^^^
+ /// ````
+ CodeFencedFenceSequence,
+ /// A code (fenced, indented) chunk.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeFenced`][Token::CodeFenced],
+ /// [`CodeIndented`][Token::CodeIndented]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced],
+ /// [`code_indented`][crate::construct::code_indented]
+ ///
+ /// ## Example
+ ///
+ /// ````markdown
+ /// | ```js
+ /// > | console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// | ```
+ /// ````
+ ///
+ /// ```markdown
+ /// > | ␠␠␠␠console.log(1)
+ /// ^^^^^^^^^^^^^^
+ /// ```
+ CodeFlowChunk,
+ /// Whole code (indented).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`CodeFlowChunk`][Token::CodeFlowChunk],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`code_fenced`][crate::construct::code_fenced]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// ␠␠␠␠console.log(1)
+ /// ^^^^^^^^^^^^^^^^^^
+ /// ```
+ CodeIndented,
+ /// Whole code (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`CodeTextData`][Token::CodeTextData],
+ /// [`CodeTextSequence`][Token::CodeTextSequence],
+ /// [`CodeTextLineEnding`][Token::CodeTextLineEnding]
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^^^
+ /// ```
+ CodeText,
+ /// Code (text) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][Token::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^
+ /// ```
+ CodeTextData,
+ /// Line ending in code (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][Token::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b␊
+ /// ^
+ /// | c` d
+ /// ```
+ CodeTextLineEnding,
+ /// Code (text) sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`CodeText`][Token::CodeText],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`code_text`][crate::construct::code_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a `b` c
+ /// ^ ^
+ /// ```
+ CodeTextSequence,
+ /// Data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [string content][crate::content::string],
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`data`][crate::construct::partial_data]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | aa *bb* cc
+ /// ^^^ ^^ ^^^
+ /// ```
+ Data,
+ /// Whole definition.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`DefinitionMarker`][Token::DefinitionMarker],
+ /// [`DefinitionLabel`][Token::DefinitionLabel],
+ /// [`DefinitionDestination`][Token::DefinitionDestination],
+ /// [`DefinitionTitle`][Token::DefinitionTitle],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`definition`][crate::construct::definition]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^^^^^^^^
+ /// ```
+ Definition,
+ /// Whole definition destination.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral],
+ /// [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// > | [a]: <b> "c"
+ /// ^^^
+ /// ```
+ DefinitionDestination,
+ /// Definition destination literal.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestination`][Token::DefinitionDestination]
+ /// * **Content model**:
+ /// [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker],
+ /// [`DefinitionDestinationString`][Token::DefinitionDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: <b> "c"
+ /// ^^^
+ /// ```
+ DefinitionDestinationLiteral,
+ /// Definition destination literal marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: <b> "c"
+ /// ^ ^
+ /// ```
+ DefinitionDestinationLiteralMarker,
+ /// Definition destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestination`][Token::DefinitionDestination]
+ /// * **Content model**:
+ /// [`DefinitionDestinationString`][Token::DefinitionDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionDestinationRaw,
+ /// Definition destination data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral],
+ /// [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// > | [a]: <b> "c"
+ /// ^
+ /// ```
+ DefinitionDestinationString,
+ /// Whole definition label.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// [`DefinitionLabelMarker`][Token::DefinitionLabelMarker],
+ /// [`DefinitionLabelString`][Token::DefinitionLabelString],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^
+ /// ```
+ DefinitionLabel,
+ /// Definition label marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionLabel`][Token::DefinitionLabel]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^ ^
+ /// ```
+ DefinitionLabelMarker,
+ /// Definition label data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionLabel`][Token::DefinitionLabel]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionLabelString,
+ /// Definition marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`definition`][crate::construct::definition]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionMarker,
+ /// Whole definition title.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Definition`][Token::Definition]
+ /// * **Content model**:
+ /// [`DefinitionTitleMarker`][Token::DefinitionTitleMarker],
+ /// [`DefinitionTitleString`][Token::DefinitionTitleString],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^^^
+ /// ```
+ DefinitionTitle,
+ /// Definition title marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionTitle`][Token::DefinitionTitle]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^ ^
+ /// ```
+ DefinitionTitleMarker,
+ /// Definition title data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`DefinitionTitle`][Token::DefinitionTitle]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | [a]: b "c"
+ /// ^
+ /// ```
+ DefinitionTitleString,
+ /// Emphasis.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`EmphasisSequence`][Token::EmphasisSequence],
+ /// [`EmphasisText`][Token::EmphasisText]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | *a*
+ /// ^^^
+ /// ```
+ Emphasis,
+ /// Emphasis sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Emphasis`][Token::Emphasis]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | *a*
+ /// ^ ^
+ /// ```
+ EmphasisSequence,
+ /// Emphasis text.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Emphasis`][Token::Emphasis]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | *a*
+ /// ^
+ /// ```
+ EmphasisText,
+ /// Whole hard break (escape).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker]
+ /// * **Construct**:
+ /// [`hard_break_escape`][crate::construct::hard_break_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a\␊
+ /// ^^
+ /// > | b
+ /// ```
+ HardBreakEscape,
+ /// Hard break (escape) marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`hard_break_escape`][crate::construct::hard_break_escape]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a\␊
+ /// ^
+ /// > | b
+ /// ```
+ HardBreakEscapeMarker,
+ /// Whole hard break (trailing).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace]
+ /// * **Construct**:
+ /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␠␠␊
+ /// ^^^
+ /// > | b
+ /// ```
+ HardBreakTrailing,
+ /// Hard break (trailing) spaces.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HardBreakTrailing`][Token::HardBreakTrailing]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␠␠␊
+ /// ^^
+ /// > | b
+ /// ```
+ HardBreakTrailingSpace,
+ /// Whole heading (atx).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HeadingAtxSequence`][Token::HeadingAtxSequence],
+ /// [`HeadingAtxText`][Token::HeadingAtxText],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^^^^^^^
+ /// ```
+ HeadingAtx,
+ /// Heading (atx) sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][Token::HeadingAtx]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^
+ /// ```
+ HeadingAtxSequence,
+ /// Heading (atx) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingAtx`][Token::HeadingAtx],
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`heading_atx`][crate::construct::heading_atx]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | # alpha
+ /// ^^^^^
+ /// ```
+ HeadingAtxText,
+ /// Whole heading (setext).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HeadingSetextText`][Token::HeadingSetextText],
+ /// [`HeadingSetextUnderline`][Token::HeadingSetextUnderline],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | alpha
+ /// ^^^^^
+ /// > | =====
+ /// ^^^^^
+ /// ```
+ HeadingSetext,
+ /// Heading (setext) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingSetext`][Token::HeadingSetext]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | alpha
+ /// ^^^^^
+ /// | =====
+ /// ```
+ HeadingSetextText,
+ /// Heading (setext) underline.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HeadingSetext`][Token::HeadingSetext]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`heading_setext`][crate::construct::heading_setext]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// | alpha
+ /// > | =====
+ /// ^^^^^
+ /// ```
+ HeadingSetextUnderline,
+ /// Whole html (flow).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`HtmlFlowData`][Token::HtmlFlowData],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`html_flow`][crate::construct::html_flow]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <div>
+ /// ^^^^^
+ /// ```
+ HtmlFlow,
+ /// HTML (flow) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HtmlFlow`][Token::HtmlFlow],
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`html_flow`][crate::construct::html_flow]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | <div>
+ /// ^^^^^
+ /// ```
+ HtmlFlowData,
+ /// Whole html (text).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`HtmlTextData`][Token::HtmlTextData],
+ /// [`LineEnding`][Token::LineEnding],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`html_text`][crate::construct::html_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a <b> c
+ /// ^^^
+ /// ```
+ HtmlText,
+ /// HTML (text) data.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`HtmlText`][Token::HtmlText]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`html_text`][crate::construct::html_text]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a <b> c
+ /// ^^^
+ /// ```
+ HtmlTextData,
+ /// Image.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`Label`][Token::Label],
+ /// [`Resource`][Token::Resource],
+ /// [`Reference`][Token::Reference]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b] c
+ /// ^^^^
+ /// > | a ![b][c] d
+ /// ^^^^^^^
+ /// > | a ![b](c) d
+ /// ^^^^^^^
+ /// ```
+ Image,
+ /// Label.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Image`][Token::Image],
+ /// [`Link`][Token::Link]
+ /// * **Content model**:
+ /// [`LabelImage`][Token::LabelImage],
+ /// [`LabelLink`][Token::LabelLink],
+ /// [`LabelEnd`][Token::LabelEnd],
+ /// [`LabelText`][Token::LabelText]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b] c
+ /// ^^^
+ /// > | a ![b][c] d
+ /// ^^^^
+ /// > | a [b](c) d
+ /// ^^^
+ /// ```
+ Label,
+ /// Label end.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [`LabelMarker`][Token::LabelMarker]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^
+ /// > | a [b](c) d
+ /// ^
+ /// ```
+ LabelEnd,
+ /// Label start (image).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [`LabelImageMarker`][Token::LabelImageMarker],
+ /// [`LabelMarker`][Token::LabelMarker]
+ /// * **Construct**:
+ /// [`label_start_image`][crate::construct::label_start_image]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^^
+ /// ```
+ LabelImage,
+ /// Label start (image) marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`LabelImage`][Token::LabelImage]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label_start_image`][crate::construct::label_start_image]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^
+ /// ```
+ LabelImageMarker,
+ /// Label start (link).
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [`LabelMarker`][Token::LabelMarker]
+ /// * **Construct**:
+ /// [`label_start_link`][crate::construct::label_start_link]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b](c) d
+ /// ^
+ /// ```
+ LabelLink,
+ /// Label marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`LabelImage`][Token::LabelImage],
+ /// [`LabelLink`][Token::LabelLink],
+ /// [`LabelEnd`][Token::LabelEnd]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label_start_image`][crate::construct::label_start_image],
+ /// [`label_start_link`][crate::construct::label_start_link],
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c) d
+ /// ^ ^
+ /// > | a [b](c) d
+ /// ^ ^
+ /// ```
+ LabelMarker,
+ /// Label text.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Label`][Token::Label]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b] c
+ /// ^
+ /// > | a ![b][c] d
+ /// ^
+ /// > | a [b](c) d
+ /// ^
+ /// ```
+ LabelText,
+ /// Line ending.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// basically everywhere
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// n/a
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a␊
+ /// ^
+ /// | b
+ /// ```
+ LineEnding,
+ /// Link.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`Label`][Token::Label],
+ /// [`Resource`][Token::Resource],
+ /// [`Reference`][Token::Reference]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a [b] c
+ /// ^^^
+ /// > | a [b][c] d
+ /// ^^^^^^
+ /// > | a [b](c) d
+ /// ^^^^^^
+ /// ```
+ Link,
+ /// Whole paragraph.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`paragraph`][crate::construct::paragraph]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a b
+ /// ^^^
+ /// > | c.
+ /// ^^
+ /// ```
+ Paragraph,
+ /// Reference.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Image`][Token::Image],
+ /// [`Link`][Token::Link]
+ /// * **Content model**:
+ /// [`ReferenceMarker`][Token::ReferenceMarker],
+ /// [`ReferenceString`][Token::ReferenceString]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b][c] d
+ /// ^^^
+ /// ```
+ Reference,
+ /// Reference marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Reference`][Token::Reference]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b][c] d
+ /// ^ ^
+ /// ```
+ ReferenceMarker,
+ /// Reference string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Reference`][Token::Reference]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`label`][crate::construct::partial_label]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b][c] d
+ /// ^
+ /// ```
+ ReferenceString,
+ /// Resource.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Image`][Token::Image],
+ /// [`Link`][Token::Link]
+ /// * **Content model**:
+ /// [`ResourceMarker`][Token::ResourceMarker],
+ /// [`ResourceDestination`][Token::ResourceDestination],
+ /// [`ResourceTitle`][Token::ResourceTitle],
+ /// [`SpaceOrTab`][Token::SpaceOrTab],
+ /// [`LineEnding`][Token::LineEnding]
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^^^^^^^
+ /// > | a [b](c) d
+ /// ^^^
+ /// ```
+ Resource,
+ /// Resource destination.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Resource`][Token::Resource]
+ /// * **Content model**:
+ /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral],
+ /// [`ResourceDestinationRaw`][Token::ResourceDestinationRaw]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^
+ /// ```
+ ResourceDestination,
+ /// Resource destination literal.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestination`][Token::ResourceDestination]
+ /// * **Content model**:
+ /// [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker],
+ /// [`ResourceDestinationString`][Token::ResourceDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](<c> "d") e
+ /// ^^^
+ /// ```
+ ResourceDestinationLiteral,
+ /// Resource destination literal marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](<c> "d") e
+ /// ^ ^
+ /// ```
+ ResourceDestinationLiteralMarker,
+ /// Resource destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestination`][Token::ResourceDestination]
+ /// * **Content model**:
+ /// [`ResourceDestinationString`][Token::ResourceDestinationString]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^
+ /// ```
+ ResourceDestinationRaw,
+ /// Resource destination raw.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral],
+ /// [`ResourceDestinationRaw`][Token::ResourceDestinationRaw]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`destination`][crate::construct::partial_destination]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](<c> "d") e
+ /// ^
+ /// > | a ![b](c "d") e
+ /// ^
+ /// ```
+ ResourceDestinationString,
+ /// Resource marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Resource`][Token::Resource]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`label_end`][crate::construct::label_end]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](c "d") e
+ /// ^ ^
+ /// ```
+ ResourceMarker,
+ /// Resource title.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Resource`][Token::Resource]
+ /// * **Content model**:
+ /// [`ResourceTitleMarker`][Token::ResourceTitleMarker],
+ /// [`ResourceTitleString`][Token::ResourceTitleString]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](<c> "d") e
+ /// ^^^
+ /// ```
+ ResourceTitle,
+ /// Resource title marker.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceTitle`][Token::ResourceTitle]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](<c> "d") e
+ /// ^ ^
+ /// ```
+ ResourceTitleMarker,
+ /// Resource title string.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ResourceTitle`][Token::ResourceTitle]
+ /// * **Content model**:
+ /// [string content][crate::content::string]
+ /// * **Construct**:
+ /// [`title`][crate::construct::partial_title]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | a ![b](<c> "d") e
+ /// ^
+ /// ```
+ ResourceTitleString,
+ /// SpaceOrTab.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// basically everywhere
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// n/a
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | ␠* * *␠
+ /// ^ ^ ^ ^
+ /// ```
+ SpaceOrTab,
+ /// Strong.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [text content][crate::content::text]
+ /// * **Content model**:
+ /// [`StrongSequence`][Token::StrongSequence],
+ /// [`StrongText`][Token::StrongText]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | **a**
+ /// ^^^^^
+ /// ```
+ Strong,
+ /// Strong sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Strong`][Token::Strong]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | **a**
+ /// ^^ ^^
+ /// ```
+ StrongSequence,
+ /// Strong text.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`Strong`][Token::Strong]
+ /// * **Content model**:
+ /// [text content][crate::content::text]
+ /// * **Construct**:
+ /// [`attention`][crate::construct::attention]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | **a**
+ /// ^
+ /// ```
+ StrongText,
+ /// Whole thematic break.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [flow content][crate::content::flow]
+ /// * **Content model**:
+ /// [`ThematicBreakSequence`][Token::ThematicBreakSequence],
+ /// [`SpaceOrTab`][Token::SpaceOrTab]
+ /// * **Construct**:
+ /// [`thematic_break`][crate::construct::thematic_break]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | * * *
+ /// ^^^^^
+ /// ```
+ ThematicBreak,
+ /// Thematic break sequence.
+ ///
+ /// ## Info
+ ///
+ /// * **Context**:
+ /// [`ThematicBreak`][Token::ThematicBreak]
+ /// * **Content model**:
+ /// void
+ /// * **Construct**:
+ /// [`thematic_break`][crate::construct::thematic_break]
+ ///
+ /// ## Example
+ ///
+ /// ```markdown
+ /// > | * * *
+ /// ^ ^ ^
+ /// ```
+ ThematicBreakSequence,
+}
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 6198776..64b66cc 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -12,1777 +12,9 @@
//! [`check`]: Tokenizer::check
use crate::parser::ParseState;
+use crate::token::Token;
use std::collections::HashMap;
-/// Semantic label of a span.
-#[derive(Debug, Clone, PartialEq, Hash, Eq)]
-pub enum TokenType {
- /// Attention sequence.
- ///
- /// > 👉 **Note**: this is used while parsing but compiled away.
- AttentionSequence,
- /// Whole autolink.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`AutolinkEmail`][TokenType::AutolinkEmail],
- /// [`AutolinkMarker`][TokenType::AutolinkMarker],
- /// [`AutolinkProtocol`][TokenType::AutolinkProtocol]
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | <https://example.com> and <admin@example.com>
- /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^
- /// ```
- Autolink,
- /// Email autolink w/o markers.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Autolink`][TokenType::Autolink]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | <admin@example.com>
- /// ^^^^^^^^^^^^^^^^^
- /// ```
- AutolinkEmail,
- /// Marker of an autolink.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Autolink`][TokenType::Autolink]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | <https://example.com>
- /// ^ ^
- /// ```
- AutolinkMarker,
- /// Protocol autolink w/o markers.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Autolink`][TokenType::Autolink]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`autolink`][crate::construct::autolink]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | <https://example.com>
- /// ^^^^^^^^^^^^^^^^^^^
- /// ```
- AutolinkProtocol,
- /// Line ending preceded only by whitespace or nothing at all.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`blank_line`][crate::construct::blank_line]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | ␠␠␊
- /// ^
- /// ```
- BlankLineEnding,
- /// Whole block quote.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [document content][crate::content::document]
- /// * **Content model**:
- /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix],
- /// [flow content][crate::content::flow]
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^^^
- /// > | b
- /// ^
- /// ```
- BlockQuote,
- /// Block quote marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^
- /// | b
- /// ```
- BlockQuoteMarker,
- /// Block quote prefix.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`BlockQuote`][TokenType::BlockQuote]
- /// * **Content model**:
- /// [`BlockQuoteMarker`][TokenType::BlockQuoteMarker],
- /// [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace]
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^^
- /// | b
- /// ```
- BlockQuotePrefix,
- /// Block quote white space.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`block_quote`][crate::construct::block_quote]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | > a
- /// ^
- /// | b
- /// ```
- BlockQuoteWhitespace,
- /// Whole character escape.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [string content][crate::content::string] or
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker],
- /// [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
- /// * **Construct**:
- /// [`character_escape`][crate::construct::character_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a \- b
- /// ^^
- /// ```
- CharacterEscape,
- /// Character escape marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterEscape`][TokenType::CharacterEscape]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_escape`][crate::construct::character_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a \- b
- /// ^
- /// ```
- CharacterEscapeMarker,
- /// Character escape value.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterEscape`][TokenType::CharacterEscape]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_escape`][crate::construct::character_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a \- b
- /// ^
- /// ```
- CharacterEscapeValue,
- /// Whole character reference.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [string content][crate::content::string] or
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker],
- /// [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal],
- /// [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric],
- /// [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi],
- /// [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a &amp; b &#8800; c &#x1D306; d
- /// ^^^^^ ^^^^^^^ ^^^^^^^^^
- /// ```
- CharacterReference,
- /// Character reference opening marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a &amp; b &#8800; c &#x1D306; d
- /// ^ ^ ^
- /// ```
- CharacterReferenceMarker,
- /// Character reference hexadecimal numeric marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a &amp; b &#8800; c &#x1D306; d
- /// ^
- /// ```
- CharacterReferenceMarkerHexadecimal,
- /// Character reference numeric marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a &amp; b &#8800; c &#x1D306; d
- /// ^ ^
- /// ```
- CharacterReferenceMarkerNumeric,
- /// Character reference closing marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a &amp; b &#8800; c &#x1D306; d
- /// ^ ^ ^
- /// ```
- CharacterReferenceMarkerSemi,
- /// Character reference value.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CharacterReference`][TokenType::CharacterReference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`character_reference`][crate::construct::character_reference]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a &amp; b &#8800; c &#x1D306; d
- /// ^^^ ^^^^ ^^^^^
- /// ```
- CharacterReferenceValue,
- /// Whole code (fenced).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`CodeFencedFence`][TokenType::CodeFencedFence],
- /// [`CodeFlowChunk`][TokenType::CodeFlowChunk],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^^^^
- /// > | console.log(1)
- /// ^^^^^^^^^^^^^^
- /// > | ```
- /// ^^^
- /// ````
- CodeFenced,
- /// A code (fenced) fence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFenced`][TokenType::CodeFenced]
- /// * **Content model**:
- /// [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo],
- /// [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta],
- /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^^^^
- /// | console.log(1)
- /// > | ```
- /// ^^^
- /// ````
- CodeFencedFence,
- /// A code (fenced) fence info word.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFencedFence`][TokenType::CodeFencedFence]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^
- /// | console.log(1)
- /// | ```
- /// ````
- CodeFencedFenceInfo,
- /// A code (fenced) fence meta string.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFencedFence`][TokenType::CodeFencedFence]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js highlight="1"
- /// ^^^^^^^^^^^^^
- /// | console.log(1)
- /// | ```
- /// ````
- CodeFencedFenceMeta,
- /// A code (fenced) fence sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// > | ```js
- /// ^^^
- /// | console.log(1)
- /// > | ```
- /// ^^^
- /// ````
- CodeFencedFenceSequence,
- /// A code (fenced, indented) chunk.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeFenced`][TokenType::CodeFenced],
- /// [`CodeIndented`][TokenType::CodeIndented]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced],
- /// [`code_indented`][crate::construct::code_indented]
- ///
- /// ## Example
- ///
- /// ````markdown
- /// | ```js
- /// > | console.log(1)
- /// ^^^^^^^^^^^^^^
- /// | ```
- /// ````
- ///
- /// ```markdown
- /// > | ␠␠␠␠console.log(1)
- /// ^^^^^^^^^^^^^^
- /// ```
- CodeFlowChunk,
- /// Whole code (indented).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`CodeFlowChunk`][TokenType::CodeFlowChunk],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`code_fenced`][crate::construct::code_fenced]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// ␠␠␠␠console.log(1)
- /// ^^^^^^^^^^^^^^^^^^
- /// ```
- CodeIndented,
- /// Whole code (text).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`CodeTextData`][TokenType::CodeTextData],
- /// [`CodeTextSequence`][TokenType::CodeTextSequence],
- /// [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b` c
- /// ^^^
- /// ```
- CodeText,
- /// Code (text) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeText`][TokenType::CodeText],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b` c
- /// ^
- /// ```
- CodeTextData,
- /// Line ending in code (text).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeText`][TokenType::CodeText],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b␊
- /// ^
- /// | c` d
- /// ```
- CodeTextLineEnding,
- /// Code (text) sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`CodeText`][TokenType::CodeText],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`code_text`][crate::construct::code_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a `b` c
- /// ^ ^
- /// ```
- CodeTextSequence,
- /// Data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [string content][crate::content::string],
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`data`][crate::construct::partial_data]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | aa *bb* cc
- /// ^^^ ^^ ^^^
- /// ```
- Data,
- /// Whole definition.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`DefinitionMarker`][TokenType::DefinitionMarker],
- /// [`DefinitionLabel`][TokenType::DefinitionLabel],
- /// [`DefinitionDestination`][TokenType::DefinitionDestination],
- /// [`DefinitionTitle`][TokenType::DefinitionTitle],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`definition`][crate::construct::definition]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^^^^^^^^^^
- /// ```
- Definition,
- /// Whole definition destination.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral],
- /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// > | [a]: <b> "c"
- /// ^^^
- /// ```
- DefinitionDestination,
- /// Definition destination literal.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestination`][TokenType::DefinitionDestination]
- /// * **Content model**:
- /// [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker],
- /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: <b> "c"
- /// ^^^
- /// ```
- DefinitionDestinationLiteral,
- /// Definition destination literal marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: <b> "c"
- /// ^ ^
- /// ```
- DefinitionDestinationLiteralMarker,
- /// Definition destination raw.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestination`][TokenType::DefinitionDestination]
- /// * **Content model**:
- /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionDestinationRaw,
- /// Definition destination data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral],
- /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// > | [a]: <b> "c"
- /// ^
- /// ```
- DefinitionDestinationString,
- /// Whole definition label.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker],
- /// [`DefinitionLabelString`][TokenType::DefinitionLabelString],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^^^
- /// ```
- DefinitionLabel,
- /// Definition label marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionLabel`][TokenType::DefinitionLabel]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^ ^
- /// ```
- DefinitionLabelMarker,
- /// Definition label data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionLabel`][TokenType::DefinitionLabel]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionLabelString,
- /// Definition marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`definition`][crate::construct::definition]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionMarker,
- /// Whole definition title.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Definition`][TokenType::Definition]
- /// * **Content model**:
- /// [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker],
- /// [`DefinitionTitleString`][TokenType::DefinitionTitleString],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^^^
- /// ```
- DefinitionTitle,
- /// Definition title marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionTitle`][TokenType::DefinitionTitle]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^ ^
- /// ```
- DefinitionTitleMarker,
- /// Definition title data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`DefinitionTitle`][TokenType::DefinitionTitle]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | [a]: b "c"
- /// ^
- /// ```
- DefinitionTitleString,
- /// Emphasis.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`EmphasisSequence`][TokenType::EmphasisSequence],
- /// [`EmphasisText`][TokenType::EmphasisText]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | *a*
- /// ^^^
- /// ```
- Emphasis,
- /// Emphasis sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Emphasis`][TokenType::Emphasis]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | *a*
- /// ^ ^
- /// ```
- EmphasisSequence,
- /// Emphasis text.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Emphasis`][TokenType::Emphasis]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | *a*
- /// ^
- /// ```
- EmphasisText,
- /// Whole hard break (escape).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
- /// * **Construct**:
- /// [`hard_break_escape`][crate::construct::hard_break_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a\␊
- /// ^^
- /// > | b
- /// ```
- HardBreakEscape,
- /// Hard break (escape) marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`hard_break_escape`][crate::construct::hard_break_escape]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a\␊
- /// ^
- /// > | b
- /// ```
- HardBreakEscapeMarker,
- /// Whole hard break (trailing).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
- /// * **Construct**:
- /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a␠␠␊
- /// ^^^
- /// > | b
- /// ```
- HardBreakTrailing,
- /// Hard break (trailing) spaces.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HardBreakTrailing`][TokenType::HardBreakTrailing]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`hard_break_trailing`][crate::construct::hard_break_trailing]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a␠␠␊
- /// ^^
- /// > | b
- /// ```
- HardBreakTrailingSpace,
- /// Whole heading (atx).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`HeadingAtxSequence`][TokenType::HeadingAtxSequence],
- /// [`HeadingAtxText`][TokenType::HeadingAtxText],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`heading_atx`][crate::construct::heading_atx]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | # alpha
- /// ^^^^^^^
- /// ```
- HeadingAtx,
- /// Heading (atx) sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingAtx`][TokenType::HeadingAtx]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`heading_atx`][crate::construct::heading_atx]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | # alpha
- /// ^
- /// ```
- HeadingAtxSequence,
- /// Heading (atx) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingAtx`][TokenType::HeadingAtx],
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`heading_atx`][crate::construct::heading_atx]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | # alpha
- /// ^^^^^
- /// ```
- HeadingAtxText,
- /// Whole heading (setext).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`HeadingSetextText`][TokenType::HeadingSetextText],
- /// [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`heading_setext`][crate::construct::heading_setext]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | alpha
- /// ^^^^^
- /// > | =====
- /// ^^^^^
- /// ```
- HeadingSetext,
- /// Heading (setext) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingSetext`][TokenType::HeadingSetext]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`heading_setext`][crate::construct::heading_setext]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | alpha
- /// ^^^^^
- /// | =====
- /// ```
- HeadingSetextText,
- /// Heading (setext) underline.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HeadingSetext`][TokenType::HeadingSetext]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`heading_setext`][crate::construct::heading_setext]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// | alpha
- /// > | =====
- /// ^^^^^
- /// ```
- HeadingSetextUnderline,
- /// Whole html (flow).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`HtmlFlowData`][TokenType::HtmlFlowData],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`html_flow`][crate::construct::html_flow]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | <div>
- /// ^^^^^
- /// ```
- HtmlFlow,
- /// HTML (flow) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HtmlFlow`][TokenType::HtmlFlow],
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`html_flow`][crate::construct::html_flow]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | <div>
- /// ^^^^^
- /// ```
- HtmlFlowData,
- /// Whole html (text).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`HtmlTextData`][TokenType::HtmlTextData],
- /// [`LineEnding`][TokenType::LineEnding],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`html_text`][crate::construct::html_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a <b> c
- /// ^^^
- /// ```
- HtmlText,
- /// HTML (text) data.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`HtmlText`][TokenType::HtmlText]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`html_text`][crate::construct::html_text]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a <b> c
- /// ^^^
- /// ```
- HtmlTextData,
- /// Image.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`Label`][TokenType::Label],
- /// [`Resource`][TokenType::Resource],
- /// [`Reference`][TokenType::Reference]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b] c
- /// ^^^^
- /// > | a ![b][c] d
- /// ^^^^^^^
- /// > | a ![b](c) d
- /// ^^^^^^^
- /// ```
- Image,
- /// Label.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Image`][TokenType::Image],
- /// [`Link`][TokenType::Link]
- /// * **Content model**:
- /// [`LabelImage`][TokenType::LabelImage],
- /// [`LabelLink`][TokenType::LabelLink],
- /// [`LabelEnd`][TokenType::LabelEnd],
- /// [`LabelText`][TokenType::LabelText]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b] c
- /// ^^^
- /// > | a ![b][c] d
- /// ^^^^
- /// > | a [b](c) d
- /// ^^^
- /// ```
- Label,
- /// Label end.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [`LabelMarker`][TokenType::LabelMarker]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^
- /// > | a [b](c) d
- /// ^
- /// ```
- LabelEnd,
- /// Label start (image).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [`LabelImageMarker`][TokenType::LabelImageMarker],
- /// [`LabelMarker`][TokenType::LabelMarker]
- /// * **Construct**:
- /// [`label_start_image`][crate::construct::label_start_image]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^^
- /// ```
- LabelImage,
- /// Label start (image) marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`LabelImage`][TokenType::LabelImage]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label_start_image`][crate::construct::label_start_image]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^
- /// ```
- LabelImageMarker,
- /// Label start (link).
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [`LabelMarker`][TokenType::LabelMarker]
- /// * **Construct**:
- /// [`label_start_link`][crate::construct::label_start_link]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b](c) d
- /// ^
- /// ```
- LabelLink,
- /// Label marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`LabelImage`][TokenType::LabelImage],
- /// [`LabelLink`][TokenType::LabelLink],
- /// [`LabelEnd`][TokenType::LabelEnd]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label_start_image`][crate::construct::label_start_image],
- /// [`label_start_link`][crate::construct::label_start_link],
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c) d
- /// ^ ^
- /// > | a [b](c) d
- /// ^ ^
- /// ```
- LabelMarker,
- /// Label text.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Label`][TokenType::Label]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b] c
- /// ^
- /// > | a ![b][c] d
- /// ^
- /// > | a [b](c) d
- /// ^
- /// ```
- LabelText,
- /// Line ending.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// basically everywhere
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// n/a
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a␊
- /// ^
- /// | b
- /// ```
- LineEnding,
- /// Link.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`Label`][TokenType::Label],
- /// [`Resource`][TokenType::Resource],
- /// [`Reference`][TokenType::Reference]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a [b] c
- /// ^^^
- /// > | a [b][c] d
- /// ^^^^^^
- /// > | a [b](c) d
- /// ^^^^^^
- /// ```
- Link,
- /// Whole paragraph.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`paragraph`][crate::construct::paragraph]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a b
- /// ^^^
- /// > | c.
- /// ^^
- /// ```
- Paragraph,
- /// Reference.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Image`][TokenType::Image],
- /// [`Link`][TokenType::Link]
- /// * **Content model**:
- /// [`ReferenceMarker`][TokenType::ReferenceMarker],
- /// [`ReferenceString`][TokenType::ReferenceString]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b][c] d
- /// ^^^
- /// ```
- Reference,
- /// Reference marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Reference`][TokenType::Reference]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b][c] d
- /// ^ ^
- /// ```
- ReferenceMarker,
- /// Reference string.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Reference`][TokenType::Reference]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`label`][crate::construct::partial_label]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b][c] d
- /// ^
- /// ```
- ReferenceString,
- /// Resource.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Image`][TokenType::Image],
- /// [`Link`][TokenType::Link]
- /// * **Content model**:
- /// [`ResourceMarker`][TokenType::ResourceMarker],
- /// [`ResourceDestination`][TokenType::ResourceDestination],
- /// [`ResourceTitle`][TokenType::ResourceTitle],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab],
- /// [`LineEnding`][TokenType::LineEnding]
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^^^^^^^
- /// > | a [b](c) d
- /// ^^^
- /// ```
- Resource,
- /// Resource destination.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Resource`][TokenType::Resource]
- /// * **Content model**:
- /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral],
- /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^
- /// ```
- ResourceDestination,
- /// Resource destination literal.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestination`][TokenType::ResourceDestination]
- /// * **Content model**:
- /// [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker],
- /// [`ResourceDestinationString`][TokenType::ResourceDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](<c> "d") e
- /// ^^^
- /// ```
- ResourceDestinationLiteral,
- /// Resource destination literal marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](<c> "d") e
- /// ^ ^
- /// ```
- ResourceDestinationLiteralMarker,
- /// Resource destination raw.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestination`][TokenType::ResourceDestination]
- /// * **Content model**:
- /// [`ResourceDestinationString`][TokenType::ResourceDestinationString]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^
- /// ```
- ResourceDestinationRaw,
- /// Resource destination raw.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral],
- /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`destination`][crate::construct::partial_destination]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](<c> "d") e
- /// ^
- /// > | a ![b](c "d") e
- /// ^
- /// ```
- ResourceDestinationString,
- /// Resource marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Resource`][TokenType::Resource]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`label_end`][crate::construct::label_end]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](c "d") e
- /// ^ ^
- /// ```
- ResourceMarker,
- /// Resource title.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Resource`][TokenType::Resource]
- /// * **Content model**:
- /// [`ResourceTitleMarker`][TokenType::ResourceTitleMarker],
- /// [`ResourceTitleString`][TokenType::ResourceTitleString]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](<c> "d") e
- /// ^^^
- /// ```
- ResourceTitle,
- /// Resource title marker.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceTitle`][TokenType::ResourceTitle]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](<c> "d") e
- /// ^ ^
- /// ```
- ResourceTitleMarker,
- /// Resource title string.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ResourceTitle`][TokenType::ResourceTitle]
- /// * **Content model**:
- /// [string content][crate::content::string]
- /// * **Construct**:
- /// [`title`][crate::construct::partial_title]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | a ![b](<c> "d") e
- /// ^
- /// ```
- ResourceTitleString,
- /// SpaceOrTab.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// basically everywhere
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// n/a
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | ␠* * *␠
- /// ^ ^ ^ ^
- /// ```
- SpaceOrTab,
- /// Strong.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [text content][crate::content::text]
- /// * **Content model**:
- /// [`StrongSequence`][TokenType::StrongSequence],
- /// [`StrongText`][TokenType::StrongText]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | **a**
- /// ^^^^^
- /// ```
- Strong,
- /// Strong sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Strong`][TokenType::Strong]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | **a**
- /// ^^ ^^
- /// ```
- StrongSequence,
- /// Strong text.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`Strong`][TokenType::Strong]
- /// * **Content model**:
- /// [text content][crate::content::text]
- /// * **Construct**:
- /// [`attention`][crate::construct::attention]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | **a**
- /// ^
- /// ```
- StrongText,
- /// Whole thematic break.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [flow content][crate::content::flow]
- /// * **Content model**:
- /// [`ThematicBreakSequence`][TokenType::ThematicBreakSequence],
- /// [`SpaceOrTab`][TokenType::SpaceOrTab]
- /// * **Construct**:
- /// [`thematic_break`][crate::construct::thematic_break]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | * * *
- /// ^^^^^
- /// ```
- ThematicBreak,
- /// Thematic break sequence.
- ///
- /// ## Info
- ///
- /// * **Context**:
- /// [`ThematicBreak`][TokenType::ThematicBreak]
- /// * **Content model**:
- /// void
- /// * **Construct**:
- /// [`thematic_break`][crate::construct::thematic_break]
- ///
- /// ## Example
- ///
- /// ```markdown
- /// > | * * *
- /// ^ ^ ^
- /// ```
- ThematicBreakSequence,
-}
-
/// Embedded content type.
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ContentType {
@@ -1840,7 +72,7 @@ pub enum EventType {
#[derive(Debug, Clone)]
pub struct Event {
pub event_type: EventType,
- pub token_type: TokenType,
+ pub token_type: Token,
pub point: Point,
pub index: usize,
pub previous: Option<usize>,
@@ -1935,7 +167,7 @@ pub struct Tokenizer<'a> {
/// Hierarchy of semantic labels.
///
/// Tracked to make sure everything’s valid.
- pub stack: Vec<TokenType>,
+ pub stack: Vec<Token>,
/// Previous character code.
pub previous: Code,
/// Current character code.
@@ -2082,11 +314,11 @@ impl<'a> Tokenizer<'a> {
}
/// Mark the start of a semantic label.
- pub fn enter(&mut self, token_type: TokenType) {
+ pub fn enter(&mut self, token_type: Token) {
self.enter_with_content(token_type, None);
}
- pub fn enter_with_content(&mut self, token_type: TokenType, content_type: Option<ContentType>) {
+ pub fn enter_with_content(&mut self, token_type: Token, content_type: Option<ContentType>) {
log::debug!("enter `{:?}` ({:?})", token_type, self.point);
self.events.push(Event {
event_type: EventType::Enter,
@@ -2101,7 +333,7 @@ impl<'a> Tokenizer<'a> {
}
/// Mark the end of a semantic label.
- pub fn exit(&mut self, token_type: TokenType) {
+ pub fn exit(&mut self, token_type: Token) {
let current_token = self.stack.pop().expect("cannot close w/o open tokens");
assert_eq!(
diff --git a/src/util/skip.rs b/src/util/skip.rs
index a8e4cfe..971beb6 100644
--- a/src/util/skip.rs
+++ b/src/util/skip.rs
@@ -1,14 +1,15 @@
//! Utilities to deal with lists of events.
-use crate::tokenizer::{Event, TokenType};
+use crate::token::Token;
+use crate::tokenizer::Event;
/// Skip from `index`, optionally past `token_types`.
-pub fn opt(events: &[Event], index: usize, token_types: &[TokenType]) -> usize {
+pub fn opt(events: &[Event], index: usize, token_types: &[Token]) -> usize {
skip_opt_with_direction(events, index, token_types, true)
}
/// Skip from `index`, optionally past `token_types`, backwards.
-pub fn opt_back(events: &[Event], index: usize, token_types: &[TokenType]) -> usize {
+pub fn opt_back(events: &[Event], index: usize, token_types: &[Token]) -> usize {
skip_opt_with_direction(events, index, token_types, false)
}
@@ -16,7 +17,7 @@ pub fn opt_back(events: &[Event], index: usize, token_types: &[TokenType]) -> us
fn skip_opt_with_direction(
events: &[Event],
index: usize,
- token_types: &[TokenType],
+ token_types: &[Token],
forward: bool,
) -> usize {
let mut index = index;