diff options
Diffstat (limited to '')
33 files changed, 2354 insertions, 2339 deletions
diff --git a/src/compiler.rs b/src/compiler.rs index 2b6eafa..d675c48 100644 --- a/src/compiler.rs +++ b/src/compiler.rs @@ -1,7 +1,8 @@ //! Turn events into a string of HTML. use crate::constant::{SAFE_PROTOCOL_HREF, SAFE_PROTOCOL_SRC}; use crate::construct::character_reference::Kind as CharacterReferenceKind; -use crate::tokenizer::{Code, Event, EventType, TokenType}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType}; use crate::util::normalize_identifier::normalize_identifier; use crate::util::{ decode_character_reference::{decode_named, decode_numeric}, @@ -218,8 +219,8 @@ pub struct Options { /// The current event is available at `context.events[context.index]`. type Handle = fn(&mut CompileContext); -/// Map of [`TokenType`][] to [`Handle`][]. -type Map = HashMap<TokenType, Handle>; +/// Map of [`Token`][] to [`Handle`][]. +type Map = HashMap<Token, Handle>; /// Context used to compile markdown. #[allow(clippy::struct_excessive_bools)] @@ -386,9 +387,9 @@ pub fn compile(events: &[Event], codes: &[Code], options: &Options) -> String { let event = &events[index]; if event.event_type == EventType::Exit - && (event.token_type == TokenType::BlankLineEnding - || event.token_type == TokenType::CodeTextLineEnding - || event.token_type == TokenType::LineEnding) + && (event.token_type == Token::BlankLineEnding + || event.token_type == Token::CodeTextLineEnding + || event.token_type == Token::LineEnding) { let codes = codes_from_span(codes, &from_exit_event(events, index)); line_ending_inferred = Some(LineEnding::from_code(*codes.first().unwrap())); @@ -409,118 +410,112 @@ pub fn compile(events: &[Event], codes: &[Code], options: &Options) -> String { let mut enter_map: Map = HashMap::new(); - enter_map.insert(TokenType::BlockQuote, on_enter_block_quote); - enter_map.insert(TokenType::CodeIndented, on_enter_code_indented); - enter_map.insert(TokenType::CodeFenced, on_enter_code_fenced); - enter_map.insert(TokenType::CodeFencedFenceInfo, on_enter_buffer); - enter_map.insert(TokenType::CodeFencedFenceMeta, on_enter_buffer); - enter_map.insert(TokenType::CodeText, on_enter_code_text); - enter_map.insert(TokenType::Definition, on_enter_definition); + enter_map.insert(Token::BlockQuote, on_enter_block_quote); + enter_map.insert(Token::CodeIndented, on_enter_code_indented); + enter_map.insert(Token::CodeFenced, on_enter_code_fenced); + enter_map.insert(Token::CodeFencedFenceInfo, on_enter_buffer); + enter_map.insert(Token::CodeFencedFenceMeta, on_enter_buffer); + enter_map.insert(Token::CodeText, on_enter_code_text); + enter_map.insert(Token::Definition, on_enter_definition); enter_map.insert( - TokenType::DefinitionDestinationString, + Token::DefinitionDestinationString, on_enter_definition_destination_string, ); - enter_map.insert(TokenType::DefinitionLabelString, on_enter_buffer); - enter_map.insert(TokenType::DefinitionTitleString, on_enter_buffer); - enter_map.insert(TokenType::Emphasis, on_enter_emphasis); - enter_map.insert(TokenType::HeadingAtxText, on_enter_buffer); - enter_map.insert(TokenType::HeadingSetextText, on_enter_buffer); - enter_map.insert(TokenType::HtmlFlow, on_enter_html_flow); - enter_map.insert(TokenType::HtmlText, on_enter_html_text); - enter_map.insert(TokenType::Image, on_enter_image); - enter_map.insert(TokenType::Label, on_enter_buffer); - enter_map.insert(TokenType::Link, on_enter_link); - enter_map.insert(TokenType::Paragraph, on_enter_paragraph); - enter_map.insert(TokenType::ReferenceString, on_enter_buffer); - enter_map.insert(TokenType::Resource, on_enter_resource); + enter_map.insert(Token::DefinitionLabelString, on_enter_buffer); + enter_map.insert(Token::DefinitionTitleString, on_enter_buffer); + enter_map.insert(Token::Emphasis, on_enter_emphasis); + enter_map.insert(Token::HeadingAtxText, on_enter_buffer); + enter_map.insert(Token::HeadingSetextText, on_enter_buffer); + enter_map.insert(Token::HtmlFlow, on_enter_html_flow); + enter_map.insert(Token::HtmlText, on_enter_html_text); + enter_map.insert(Token::Image, on_enter_image); + enter_map.insert(Token::Label, on_enter_buffer); + enter_map.insert(Token::Link, on_enter_link); + enter_map.insert(Token::Paragraph, on_enter_paragraph); + enter_map.insert(Token::ReferenceString, on_enter_buffer); + enter_map.insert(Token::Resource, on_enter_resource); enter_map.insert( - TokenType::ResourceDestinationString, + Token::ResourceDestinationString, on_enter_resource_destination_string, ); - enter_map.insert(TokenType::ResourceTitleString, on_enter_buffer); - enter_map.insert(TokenType::Strong, on_enter_strong); + enter_map.insert(Token::ResourceTitleString, on_enter_buffer); + enter_map.insert(Token::Strong, on_enter_strong); let mut exit_map: Map = HashMap::new(); - exit_map.insert(TokenType::AutolinkEmail, on_exit_autolink_email); - exit_map.insert(TokenType::AutolinkProtocol, on_exit_autolink_protocol); - exit_map.insert(TokenType::BlockQuote, on_exit_block_quote); - exit_map.insert(TokenType::CharacterEscapeValue, on_exit_data); + exit_map.insert(Token::AutolinkEmail, on_exit_autolink_email); + exit_map.insert(Token::AutolinkProtocol, on_exit_autolink_protocol); + exit_map.insert(Token::BlockQuote, on_exit_block_quote); + exit_map.insert(Token::CharacterEscapeValue, on_exit_data); exit_map.insert( - TokenType::CharacterReferenceMarker, + Token::CharacterReferenceMarker, on_exit_character_reference_marker, ); exit_map.insert( - TokenType::CharacterReferenceMarkerNumeric, + Token::CharacterReferenceMarkerNumeric, on_exit_character_reference_marker_numeric, ); exit_map.insert( - TokenType::CharacterReferenceMarkerHexadecimal, + Token::CharacterReferenceMarkerHexadecimal, on_exit_character_reference_marker_hexadecimal, ); exit_map.insert( - TokenType::CharacterReferenceValue, + Token::CharacterReferenceValue, on_exit_character_reference_value, ); - exit_map.insert(TokenType::CodeFenced, on_exit_code_flow); - exit_map.insert(TokenType::CodeFencedFence, on_exit_code_fenced_fence); + exit_map.insert(Token::CodeFenced, on_exit_code_flow); + exit_map.insert(Token::CodeFencedFence, on_exit_code_fenced_fence); + exit_map.insert(Token::CodeFencedFenceInfo, on_exit_code_fenced_fence_info); + exit_map.insert(Token::CodeFencedFenceMeta, on_exit_drop); + exit_map.insert(Token::CodeFlowChunk, on_exit_code_flow_chunk); + exit_map.insert(Token::CodeIndented, on_exit_code_flow); + exit_map.insert(Token::CodeText, on_exit_code_text); + exit_map.insert(Token::CodeTextData, on_exit_data); + exit_map.insert(Token::CodeTextLineEnding, on_exit_code_text_line_ending); + exit_map.insert(Token::Data, on_exit_data); + exit_map.insert(Token::Definition, on_exit_definition); exit_map.insert( - TokenType::CodeFencedFenceInfo, - on_exit_code_fenced_fence_info, - ); - exit_map.insert(TokenType::CodeFencedFenceMeta, on_exit_drop); - exit_map.insert(TokenType::CodeFlowChunk, on_exit_code_flow_chunk); - exit_map.insert(TokenType::CodeIndented, on_exit_code_flow); - exit_map.insert(TokenType::CodeText, on_exit_code_text); - exit_map.insert(TokenType::CodeTextData, on_exit_data); - exit_map.insert(TokenType::CodeTextLineEnding, on_exit_code_text_line_ending); - exit_map.insert(TokenType::Data, on_exit_data); - exit_map.insert(TokenType::Definition, on_exit_definition); - exit_map.insert( - TokenType::DefinitionDestinationString, + Token::DefinitionDestinationString, on_exit_definition_destination_string, ); exit_map.insert( - TokenType::DefinitionLabelString, + Token::DefinitionLabelString, on_exit_definition_label_string, ); exit_map.insert( - TokenType::DefinitionTitleString, + Token::DefinitionTitleString, on_exit_definition_title_string, ); - exit_map.insert(TokenType::Emphasis, on_exit_emphasis); - - exit_map.insert(TokenType::HardBreakEscape, on_exit_break); - exit_map.insert(TokenType::HardBreakTrailing, on_exit_break); - exit_map.insert(TokenType::HeadingAtx, on_exit_heading_atx); - exit_map.insert(TokenType::HeadingAtxSequence, on_exit_heading_atx_sequence); - exit_map.insert(TokenType::HeadingAtxText, on_exit_heading_atx_text); - exit_map.insert(TokenType::HeadingSetextText, on_exit_heading_setext_text); + exit_map.insert(Token::Emphasis, on_exit_emphasis); + + exit_map.insert(Token::HardBreakEscape, on_exit_break); + exit_map.insert(Token::HardBreakTrailing, on_exit_break); + exit_map.insert(Token::HeadingAtx, on_exit_heading_atx); + exit_map.insert(Token::HeadingAtxSequence, on_exit_heading_atx_sequence); + exit_map.insert(Token::HeadingAtxText, on_exit_heading_atx_text); + exit_map.insert(Token::HeadingSetextText, on_exit_heading_setext_text); exit_map.insert( - TokenType::HeadingSetextUnderline, + Token::HeadingSetextUnderline, on_exit_heading_setext_underline, ); - exit_map.insert(TokenType::HtmlFlow, on_exit_html); - exit_map.insert(TokenType::HtmlText, on_exit_html); - exit_map.insert(TokenType::HtmlFlowData, on_exit_html_data); - exit_map.insert(TokenType::HtmlTextData, on_exit_html_data); - exit_map.insert(TokenType::Image, on_exit_media); - exit_map.insert(TokenType::Label, on_exit_label); - exit_map.insert(TokenType::LabelText, on_exit_label_text); - exit_map.insert(TokenType::LineEnding, on_exit_line_ending); - exit_map.insert(TokenType::Link, on_exit_media); - exit_map.insert(TokenType::Paragraph, on_exit_paragraph); - exit_map.insert(TokenType::ReferenceString, on_exit_reference_string); - exit_map.insert(TokenType::Resource, on_exit_drop); + exit_map.insert(Token::HtmlFlow, on_exit_html); + exit_map.insert(Token::HtmlText, on_exit_html); + exit_map.insert(Token::HtmlFlowData, on_exit_html_data); + exit_map.insert(Token::HtmlTextData, on_exit_html_data); + exit_map.insert(Token::Image, on_exit_media); + exit_map.insert(Token::Label, on_exit_label); + exit_map.insert(Token::LabelText, on_exit_label_text); + exit_map.insert(Token::LineEnding, on_exit_line_ending); + exit_map.insert(Token::Link, on_exit_media); + exit_map.insert(Token::Paragraph, on_exit_paragraph); + exit_map.insert(Token::ReferenceString, on_exit_reference_string); + exit_map.insert(Token::Resource, on_exit_drop); exit_map.insert( - TokenType::ResourceDestinationString, + Token::ResourceDestinationString, on_exit_resource_destination_string, ); - exit_map.insert( - TokenType::ResourceTitleString, - on_exit_resource_title_string, - ); - exit_map.insert(TokenType::Strong, on_exit_strong); - exit_map.insert(TokenType::ThematicBreak, on_exit_thematic_break); + exit_map.insert(Token::ResourceTitleString, on_exit_resource_title_string); + exit_map.insert(Token::Strong, on_exit_strong); + exit_map.insert(Token::ThematicBreak, on_exit_thematic_break); // Handle one event. let handle = |context: &mut CompileContext, index: usize| { @@ -556,7 +551,7 @@ pub fn compile(events: &[Event], codes: &[Code], options: &Options) -> String { handle(&mut context, index); } - if event.token_type == TokenType::Definition { + if event.token_type == Token::Definition { if event.event_type == EventType::Enter { handle(&mut context, index); // Also handle start. definition_inside = true; @@ -607,21 +602,21 @@ fn on_enter_buffer(context: &mut CompileContext) { context.buffer(); } -/// Handle [`Enter`][EventType::Enter]:[`BlockQuote`][TokenType::BlockQuote]. +/// Handle [`Enter`][EventType::Enter]:[`BlockQuote`][Token::BlockQuote]. fn on_enter_block_quote(context: &mut CompileContext) { // tightStack.push(false) context.line_ending_if_needed(); context.tag("<blockquote>".to_string()); } -/// Handle [`Enter`][EventType::Enter]:[`CodeIndented`][TokenType::CodeIndented]. +/// Handle [`Enter`][EventType::Enter]:[`CodeIndented`][Token::CodeIndented]. fn on_enter_code_indented(context: &mut CompileContext) { context.code_flow_seen_data = Some(false); context.line_ending_if_needed(); context.tag("<pre><code>".to_string()); } -/// Handle [`Enter`][EventType::Enter]:[`CodeFenced`][TokenType::CodeFenced]. +/// Handle [`Enter`][EventType::Enter]:[`CodeFenced`][Token::CodeFenced]. fn on_enter_code_fenced(context: &mut CompileContext) { context.code_flow_seen_data = Some(false); context.line_ending_if_needed(); @@ -630,13 +625,13 @@ fn on_enter_code_fenced(context: &mut CompileContext) { context.code_fenced_fences_count = Some(0); } -/// Handle [`Enter`][EventType::Enter]:[`CodeText`][TokenType::CodeText]. +/// Handle [`Enter`][EventType::Enter]:[`CodeText`][Token::CodeText]. fn on_enter_code_text(context: &mut CompileContext) { context.tag("<code>".to_string()); context.buffer(); } -/// Handle [`Enter`][EventType::Enter]:[`Definition`][TokenType::Definition]. +/// Handle [`Enter`][EventType::Enter]:[`Definition`][Token::Definition]. fn on_enter_definition(context: &mut CompileContext) { context.buffer(); context.media_stack.push(Media { @@ -649,18 +644,18 @@ fn on_enter_definition(context: &mut CompileContext) { }); } -/// Handle [`Enter`][EventType::Enter]:[`DefinitionDestinationString`][TokenType::DefinitionDestinationString]. +/// Handle [`Enter`][EventType::Enter]:[`DefinitionDestinationString`][Token::DefinitionDestinationString]. fn on_enter_definition_destination_string(context: &mut CompileContext) { context.buffer(); context.ignore_encode = true; } -/// Handle [`Enter`][EventType::Enter]:[`Emphasis`][TokenType::Emphasis]. +/// Handle [`Enter`][EventType::Enter]:[`Emphasis`][Token::Emphasis]. fn on_enter_emphasis(context: &mut CompileContext) { context.tag("<em>".to_string()); } -/// Handle [`Enter`][EventType::Enter]:[`HtmlFlow`][TokenType::HtmlFlow]. +/// Handle [`Enter`][EventType::Enter]:[`HtmlFlow`][Token::HtmlFlow]. fn on_enter_html_flow(context: &mut CompileContext) { context.line_ending_if_needed(); if context.allow_dangerous_html { @@ -668,14 +663,14 @@ fn on_enter_html_flow(context: &mut CompileContext) { } } -/// Handle [`Enter`][EventType::Enter]:[`HtmlText`][TokenType::HtmlText]. +/// Handle [`Enter`][EventType::Enter]:[`HtmlText`][Token::HtmlText]. fn on_enter_html_text(context: &mut CompileContext) { if context.allow_dangerous_html { context.ignore_encode = true; } } -/// Handle [`Enter`][EventType::Enter]:[`Image`][TokenType::Image]. +/// Handle [`Enter`][EventType::Enter]:[`Image`][Token::Image]. fn on_enter_image(context: &mut CompileContext) { context.media_stack.push(Media { image: true, @@ -688,7 +683,7 @@ fn on_enter_image(context: &mut CompileContext) { context.tags = false; // Disallow tags. } -/// Handle [`Enter`][EventType::Enter]:[`Link`][TokenType::Link]. +/// Handle [`Enter`][EventType::Enter]:[`Link`][Token::Link]. fn on_enter_link(context: &mut CompileContext) { context.media_stack.push(Media { image: false, @@ -700,20 +695,20 @@ fn on_enter_link(context: &mut CompileContext) { }); } -/// Handle [`Enter`][EventType::Enter]:[`Paragraph`][TokenType::Paragraph]. +/// Handle [`Enter`][EventType::Enter]:[`Paragraph`][Token::Paragraph]. fn on_enter_paragraph(context: &mut CompileContext) { context.line_ending_if_needed(); context.tag("<p>".to_string()); } -/// Handle [`Enter`][EventType::Enter]:[`Resource`][TokenType::Resource]. +/// Handle [`Enter`][EventType::Enter]:[`Resource`][Token::Resource]. fn on_enter_resource(context: &mut CompileContext) { context.buffer(); // We can have line endings in the resource, ignore them. let media = context.media_stack.last_mut().unwrap(); media.destination = Some("".to_string()); } -/// Handle [`Enter`][EventType::Enter]:[`ResourceDestinationString`][TokenType::ResourceDestinationString]. +/// Handle [`Enter`][EventType::Enter]:[`ResourceDestinationString`][Token::ResourceDestinationString]. fn on_enter_resource_destination_string(context: &mut CompileContext) { context.buffer(); // Ignore encoding the result, as we’ll first percent encode the url and @@ -721,12 +716,12 @@ fn on_enter_resource_destination_string(context: &mut CompileContext) { context.ignore_encode = true; } -/// Handle [`Enter`][EventType::Enter]:[`Strong`][TokenType::Strong]. +/// Handle [`Enter`][EventType::Enter]:[`Strong`][Token::Strong]. fn on_enter_strong(context: &mut CompileContext) { context.tag("<strong>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`AutolinkEmail`][TokenType::AutolinkEmail]. +/// Handle [`Exit`][EventType::Exit]:[`AutolinkEmail`][Token::AutolinkEmail]. fn on_exit_autolink_email(context: &mut CompileContext) { let slice = serialize( context.codes, @@ -744,7 +739,7 @@ fn on_exit_autolink_email(context: &mut CompileContext) { context.tag("</a>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`AutolinkProtocol`][TokenType::AutolinkProtocol]. +/// Handle [`Exit`][EventType::Exit]:[`AutolinkProtocol`][Token::AutolinkProtocol]. fn on_exit_autolink_protocol(context: &mut CompileContext) { let slice = serialize( context.codes, @@ -759,12 +754,12 @@ fn on_exit_autolink_protocol(context: &mut CompileContext) { context.tag("</a>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:{[`HardBreakEscape`][TokenType::HardBreakEscape],[`HardBreakTrailing`][TokenType::HardBreakTrailing]}. +/// Handle [`Exit`][EventType::Exit]:{[`HardBreakEscape`][Token::HardBreakEscape],[`HardBreakTrailing`][Token::HardBreakTrailing]}. fn on_exit_break(context: &mut CompileContext) { context.tag("<br />".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`BlockQuote`][TokenType::BlockQuote]. +/// Handle [`Exit`][EventType::Exit]:[`BlockQuote`][Token::BlockQuote]. fn on_exit_block_quote(context: &mut CompileContext) { // tightStack.pop() context.line_ending_if_needed(); @@ -772,22 +767,22 @@ fn on_exit_block_quote(context: &mut CompileContext) { // let mut slurp_all_line_endings = false; } -/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker]. +/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarker`][Token::CharacterReferenceMarker]. fn on_exit_character_reference_marker(context: &mut CompileContext) { context.character_reference_kind = Some(CharacterReferenceKind::Named); } -/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal]. +/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal]. fn on_exit_character_reference_marker_hexadecimal(context: &mut CompileContext) { context.character_reference_kind = Some(CharacterReferenceKind::Hexadecimal); } -/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric]. +/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric]. fn on_exit_character_reference_marker_numeric(context: &mut CompileContext) { context.character_reference_kind = Some(CharacterReferenceKind::Decimal); } -/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceValue`][TokenType::CharacterReferenceValue]. +/// Handle [`Exit`][EventType::Exit]:[`CharacterReferenceValue`][Token::CharacterReferenceValue]. fn on_exit_character_reference_value(context: &mut CompileContext) { let kind = context .character_reference_kind @@ -808,7 +803,7 @@ fn on_exit_character_reference_value(context: &mut CompileContext) { context.push(context.encode_opt(&value)); } -/// Handle [`Exit`][EventType::Exit]:[`CodeFlowChunk`][TokenType::CodeFlowChunk]. +/// Handle [`Exit`][EventType::Exit]:[`CodeFlowChunk`][Token::CodeFlowChunk]. fn on_exit_code_flow_chunk(context: &mut CompileContext) { context.code_flow_seen_data = Some(true); context.push(context.encode_opt(&serialize( @@ -818,7 +813,7 @@ fn on_exit_code_flow_chunk(context: &mut CompileContext) { ))); } -/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFence`][TokenType::CodeFencedFence]. +/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFence`][Token::CodeFencedFence]. fn on_exit_code_fenced_fence(context: &mut CompileContext) { let count = if let Some(count) = context.code_fenced_fences_count { count @@ -834,13 +829,13 @@ fn on_exit_code_fenced_fence(context: &mut CompileContext) { context.code_fenced_fences_count = Some(count + 1); } -/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo]. +/// Handle [`Exit`][EventType::Exit]:[`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo]. fn on_exit_code_fenced_fence_info(context: &mut CompileContext) { let value = context.resume(); context.tag(format!(" class=\"language-{}\"", value)); } -/// Handle [`Exit`][EventType::Exit]:{[`CodeFenced`][TokenType::CodeFenced],[`CodeIndented`][TokenType::CodeIndented]}. +/// Handle [`Exit`][EventType::Exit]:{[`CodeFenced`][Token::CodeFenced],[`CodeIndented`][Token::CodeIndented]}. fn on_exit_code_flow(context: &mut CompileContext) { let seen_data = context .code_flow_seen_data @@ -874,7 +869,7 @@ fn on_exit_code_flow(context: &mut CompileContext) { context.slurp_one_line_ending = false; } -/// Handle [`Exit`][EventType::Exit]:[`CodeText`][TokenType::CodeText]. +/// Handle [`Exit`][EventType::Exit]:[`CodeText`][Token::CodeText]. fn on_exit_code_text(context: &mut CompileContext) { let result = context.resume(); let mut chars = result.chars(); @@ -898,7 +893,7 @@ fn on_exit_code_text(context: &mut CompileContext) { context.tag("</code>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`CodeTextLineEnding`][TokenType::CodeTextLineEnding]. +/// Handle [`Exit`][EventType::Exit]:[`CodeTextLineEnding`][Token::CodeTextLineEnding]. fn on_exit_code_text_line_ending(context: &mut CompileContext) { context.push(" ".to_string()); } @@ -910,7 +905,7 @@ fn on_exit_drop(context: &mut CompileContext) { context.resume(); } -/// Handle [`Exit`][EventType::Exit]:{[`CodeTextData`][TokenType::CodeTextData],[`Data`][TokenType::Data],[`CharacterEscapeValue`][TokenType::CharacterEscapeValue]}. +/// Handle [`Exit`][EventType::Exit]:{[`CodeTextData`][Token::CodeTextData],[`Data`][Token::Data],[`CharacterEscapeValue`][Token::CharacterEscapeValue]}. fn on_exit_data(context: &mut CompileContext) { // Just output it. context.push(context.encode_opt(&serialize( @@ -920,7 +915,7 @@ fn on_exit_data(context: &mut CompileContext) { ))); } -/// Handle [`Exit`][EventType::Exit]:[`Definition`][TokenType::Definition]. +/// Handle [`Exit`][EventType::Exit]:[`Definition`][Token::Definition]. fn on_exit_definition(context: &mut CompileContext) { let definition = context.media_stack.pop().unwrap(); let reference_id = normalize_identifier(&definition.reference_id.unwrap()); @@ -935,7 +930,7 @@ fn on_exit_definition(context: &mut CompileContext) { .or_insert(Definition { destination, title }); } -/// Handle [`Exit`][EventType::Exit]:[`DefinitionDestinationString`][TokenType::DefinitionDestinationString]. +/// Handle [`Exit`][EventType::Exit]:[`DefinitionDestinationString`][Token::DefinitionDestinationString]. fn on_exit_definition_destination_string(context: &mut CompileContext) { let buf = context.resume(); let definition = context.media_stack.last_mut().unwrap(); @@ -943,7 +938,7 @@ fn on_exit_definition_destination_string(context: &mut CompileContext) { context.ignore_encode = false; } -/// Handle [`Exit`][EventType::Exit]:[`DefinitionLabelString`][TokenType::DefinitionLabelString]. +/// Handle [`Exit`][EventType::Exit]:[`DefinitionLabelString`][Token::DefinitionLabelString]. fn on_exit_definition_label_string(context: &mut CompileContext) { // Discard label, use the source content instead. context.resume(); @@ -955,19 +950,19 @@ fn on_exit_definition_label_string(context: &mut CompileContext) { )); } -/// Handle [`Exit`][EventType::Exit]:[`DefinitionTitleString`][TokenType::DefinitionTitleString]. +/// Handle [`Exit`][EventType::Exit]:[`DefinitionTitleString`][Token::DefinitionTitleString]. fn on_exit_definition_title_string(context: &mut CompileContext) { let buf = context.resume(); let definition = context.media_stack.last_mut().unwrap(); definition.title = Some(buf); } -/// Handle [`Exit`][EventType::Exit]:[`Strong`][TokenType::Emphasis]. +/// Handle [`Exit`][EventType::Exit]:[`Strong`][Token::Emphasis]. fn on_exit_emphasis(context: &mut CompileContext) { context.tag("</em>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`HeadingAtx`][TokenType::HeadingAtx]. +/// Handle [`Exit`][EventType::Exit]:[`HeadingAtx`][Token::HeadingAtx]. fn on_exit_heading_atx(context: &mut CompileContext) { let rank = context .atx_opening_sequence_size @@ -977,7 +972,7 @@ fn on_exit_heading_atx(context: &mut CompileContext) { context.tag(format!("</h{}>", rank)); } -/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxSequence`][TokenType::HeadingAtxSequence]. +/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxSequence`][Token::HeadingAtxSequence]. fn on_exit_heading_atx_sequence(context: &mut CompileContext) { // First fence we see. if context.atx_opening_sequence_size.is_none() { @@ -993,20 +988,20 @@ fn on_exit_heading_atx_sequence(context: &mut CompileContext) { } } -/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxText`][TokenType::HeadingAtxText]. +/// Handle [`Exit`][EventType::Exit]:[`HeadingAtxText`][Token::HeadingAtxText]. fn on_exit_heading_atx_text(context: &mut CompileContext) { let value = context.resume(); context.push(value); } -/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextText`][TokenType::HeadingSetextText]. +/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextText`][Token::HeadingSetextText]. fn on_exit_heading_setext_text(context: &mut CompileContext) { let buf = context.resume(); context.heading_setext_buffer = Some(buf); context.slurp_one_line_ending = true; } -/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline]. +/// Handle [`Exit`][EventType::Exit]:[`HeadingSetextUnderline`][Token::HeadingSetextUnderline]. fn on_exit_heading_setext_underline(context: &mut CompileContext) { let text = context .heading_setext_buffer @@ -1024,12 +1019,12 @@ fn on_exit_heading_setext_underline(context: &mut CompileContext) { context.tag(format!("</h{}>", level)); } -/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlow`][TokenType::HtmlFlow],[`HtmlText`][TokenType::HtmlText]}. +/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlow`][Token::HtmlFlow],[`HtmlText`][Token::HtmlText]}. fn on_exit_html(context: &mut CompileContext) { context.ignore_encode = false; } -/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlowData`][TokenType::HtmlFlowData],[`HtmlTextData`][TokenType::HtmlTextData]}. +/// Handle [`Exit`][EventType::Exit]:{[`HtmlFlowData`][Token::HtmlFlowData],[`HtmlTextData`][Token::HtmlTextData]}. fn on_exit_html_data(context: &mut CompileContext) { let slice = serialize( context.codes, @@ -1039,14 +1034,14 @@ fn on_exit_html_data(context: &mut CompileContext) { context.push(context.encode_opt(&slice)); } -/// Handle [`Exit`][EventType::Exit]:[`Label`][TokenType::Label]. +/// Handle [`Exit`][EventType::Exit]:[`Label`][Token::Label]. fn on_exit_label(context: &mut CompileContext) { let buf = context.resume(); let media = context.media_stack.last_mut().unwrap(); media.label = Some(buf); } -/// Handle [`Exit`][EventType::Exit]:[`LabelText`][TokenType::LabelText]. +/// Handle [`Exit`][EventType::Exit]:[`LabelText`][Token::LabelText]. fn on_exit_label_text(context: &mut CompileContext) { let media = context.media_stack.last_mut().unwrap(); media.label_id = Some(serialize( @@ -1056,7 +1051,7 @@ fn on_exit_label_text(context: &mut CompileContext) { )); } -/// Handle [`Exit`][EventType::Exit]:[`LineEnding`][TokenType::LineEnding]. +/// Handle [`Exit`][EventType::Exit]:[`LineEnding`][Token::LineEnding]. fn on_exit_line_ending(context: &mut CompileContext) { // if slurp_all_line_endings { // // Empty. @@ -1072,7 +1067,7 @@ fn on_exit_line_ending(context: &mut CompileContext) { } } -/// Handle [`Exit`][EventType::Exit]:{[`Image`][TokenType::Image],[`Link`][TokenType::Link]}. +/// Handle [`Exit`][EventType::Exit]:{[`Image`][Token::Image],[`Link`][Token::Link]}. fn on_exit_media(context: &mut CompileContext) { let mut is_in_image = false; let mut index = 0; @@ -1136,12 +1131,12 @@ fn on_exit_media(context: &mut CompileContext) { }; } -/// Handle [`Exit`][EventType::Exit]:[`Paragraph`][TokenType::Paragraph]. +/// Handle [`Exit`][EventType::Exit]:[`Paragraph`][Token::Paragraph]. fn on_exit_paragraph(context: &mut CompileContext) { context.tag("</p>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`ReferenceString`][TokenType::ReferenceString]. +/// Handle [`Exit`][EventType::Exit]:[`ReferenceString`][Token::ReferenceString]. fn on_exit_reference_string(context: &mut CompileContext) { // Drop stuff. context.resume(); @@ -1153,7 +1148,7 @@ fn on_exit_reference_string(context: &mut CompileContext) { )); } -/// Handle [`Exit`][EventType::Exit]:[`ResourceDestinationString`][TokenType::ResourceDestinationString]. +/// Handle [`Exit`][EventType::Exit]:[`ResourceDestinationString`][Token::ResourceDestinationString]. fn on_exit_resource_destination_string(context: &mut CompileContext) { let buf = context.resume(); let media = context.media_stack.last_mut().unwrap(); @@ -1161,19 +1156,19 @@ fn on_exit_resource_destination_string(context: &mut CompileContext) { context.ignore_encode = false; } -/// Handle [`Exit`][EventType::Exit]:[`ResourceTitleString`][TokenType::ResourceTitleString]. +/// Handle [`Exit`][EventType::Exit]:[`ResourceTitleString`][Token::ResourceTitleString]. fn on_exit_resource_title_string(context: &mut CompileContext) { let buf = context.resume(); let media = context.media_stack.last_mut().unwrap(); media.title = Some(buf); } -/// Handle [`Exit`][EventType::Exit]:[`Strong`][TokenType::Strong]. +/// Handle [`Exit`][EventType::Exit]:[`Strong`][Token::Strong]. fn on_exit_strong(context: &mut CompileContext) { context.tag("</strong>".to_string()); } -/// Handle [`Exit`][EventType::Exit]:[`ThematicBreak`][TokenType::ThematicBreak]. +/// Handle [`Exit`][EventType::Exit]:[`ThematicBreak`][Token::ThematicBreak]. fn on_exit_thematic_break(context: &mut CompileContext) { context.line_ending_if_needed(); context.tag("<hr />".to_string()); diff --git a/src/construct/attention.rs b/src/construct/attention.rs index daf662e..1750692 100644 --- a/src/construct/attention.rs +++ b/src/construct/attention.rs @@ -32,14 +32,14 @@ //! //! ## Tokens //! -//! * [`Emphasis`][TokenType::Emphasis] -//! * [`EmphasisSequence`][TokenType::EmphasisSequence] -//! * [`EmphasisText`][TokenType::EmphasisText] -//! * [`Strong`][TokenType::Strong] -//! * [`StrongSequence`][TokenType::StrongSequence] -//! * [`StrongText`][TokenType::StrongText] +//! * [`Emphasis`][Token::Emphasis] +//! * [`EmphasisSequence`][Token::EmphasisSequence] +//! * [`EmphasisText`][Token::EmphasisText] +//! * [`Strong`][Token::Strong] +//! * [`StrongSequence`][Token::StrongSequence] +//! * [`StrongText`][Token::StrongText] //! -//! > 👉 **Note**: while parsing, [`AttentionSequence`][TokenType::AttentionSequence] +//! > 👉 **Note**: while parsing, [`AttentionSequence`][Token::AttentionSequence] //! > is used, which is later compiled away. //! //! ## References @@ -51,7 +51,8 @@ //! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element //! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element -use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, Tokenizer}; use crate::unicode::PUNCTUATION; use crate::util::edit_map::EditMap; @@ -174,7 +175,7 @@ struct Sequence { pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('*' | '_') => { - tokenizer.enter(TokenType::AttentionSequence); + tokenizer.enter(Token::AttentionSequence); inside(tokenizer, code, MarkerKind::from_code(code)) } _ => (State::Nok, None), @@ -193,7 +194,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, marker: MarkerKind) -> StateFnR (State::Fn(Box::new(move |t, c| inside(t, c, marker))), None) } _ => { - tokenizer.exit(TokenType::AttentionSequence); + tokenizer.exit(Token::AttentionSequence); tokenizer.register_resolver("attention".to_string(), Box::new(resolve)); (State::Ok, Some(vec![code])) } @@ -216,7 +217,7 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { if enter.event_type == EventType::Enter { balance += 1; - if enter.token_type == TokenType::AttentionSequence { + if enter.token_type == Token::AttentionSequence { let end = start + 1; let exit = &tokenizer.events[end]; let marker = MarkerKind::from_code(codes[enter.index]); @@ -392,9 +393,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::Emphasis + Token::Emphasis } else { - TokenType::Strong + Token::Strong }, point: seq_open_enter.0.clone(), index: seq_open_enter.1, @@ -405,9 +406,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_open_enter.0.clone(), index: seq_open_enter.1, @@ -418,9 +419,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_open_exit.0.clone(), index: seq_open_exit.1, @@ -431,9 +432,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::EmphasisText + Token::EmphasisText } else { - TokenType::StrongText + Token::StrongText }, point: seq_open_exit.0.clone(), index: seq_open_exit.1, @@ -451,9 +452,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::EmphasisText + Token::EmphasisText } else { - TokenType::StrongText + Token::StrongText }, point: seq_close_enter.0.clone(), index: seq_close_enter.1, @@ -464,9 +465,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Enter, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_close_enter.0.clone(), index: seq_close_enter.1, @@ -477,9 +478,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::EmphasisSequence + Token::EmphasisSequence } else { - TokenType::StrongSequence + Token::StrongSequence }, point: seq_close_exit.0.clone(), index: seq_close_exit.1, @@ -490,9 +491,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { Event { event_type: EventType::Exit, token_type: if take == 1 { - TokenType::Emphasis + Token::Emphasis } else { - TokenType::Strong + Token::Strong }, point: seq_close_exit.0.clone(), index: seq_close_exit.1, @@ -515,8 +516,8 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { let mut index = 0; while index < sequences.len() { let sequence = &sequences[index]; - tokenizer.events[sequence.event_index].token_type = TokenType::Data; - tokenizer.events[sequence.event_index + 1].token_type = TokenType::Data; + tokenizer.events[sequence.event_index].token_type = Token::Data; + tokenizer.events[sequence.event_index + 1].token_type = Token::Data; index += 1; } diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs index e29bf8b..db4365f 100644 --- a/src/construct/autolink.rs +++ b/src/construct/autolink.rs @@ -84,10 +84,10 @@ //! //! ## Tokens //! -//! * [`Autolink`][TokenType::Autolink] -//! * [`AutolinkEmail`][TokenType::AutolinkEmail] -//! * [`AutolinkMarker`][TokenType::AutolinkMarker] -//! * [`AutolinkProtocol`][TokenType::AutolinkProtocol] +//! * [`Autolink`][Token::Autolink] +//! * [`AutolinkEmail`][Token::AutolinkEmail] +//! * [`AutolinkMarker`][Token::AutolinkMarker] +//! * [`AutolinkProtocol`][Token::AutolinkProtocol] //! //! ## References //! @@ -102,7 +102,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX}; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of an autolink. /// @@ -113,11 +114,11 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('<') => { - tokenizer.enter(TokenType::Autolink); - tokenizer.enter(TokenType::AutolinkMarker); + tokenizer.enter(Token::Autolink); + tokenizer.enter(Token::AutolinkMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::AutolinkMarker); - tokenizer.enter(TokenType::AutolinkProtocol); + tokenizer.exit(Token::AutolinkMarker); + tokenizer.enter(Token::AutolinkProtocol); (State::Fn(Box::new(open)), None) } _ => (State::Nok, None), @@ -195,7 +196,7 @@ fn scheme_inside_or_email_atext( fn url_inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.exit(TokenType::AutolinkProtocol); + tokenizer.exit(Token::AutolinkProtocol); end(tokenizer, code) } Code::Char(char) if char.is_ascii_control() => (State::Nok, None), @@ -260,10 +261,10 @@ fn email_label(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes } Code::Char('>') => { let index = tokenizer.events.len(); - tokenizer.exit(TokenType::AutolinkProtocol); + tokenizer.exit(Token::AutolinkProtocol); // Change the token type. - tokenizer.events[index - 1].token_type = TokenType::AutolinkEmail; - tokenizer.events[index].token_type = TokenType::AutolinkEmail; + tokenizer.events[index - 1].token_type = Token::AutolinkEmail; + tokenizer.events[index].token_type = Token::AutolinkEmail; end(tokenizer, code) } _ => email_value(tokenizer, code, size), @@ -306,10 +307,10 @@ fn email_value(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.enter(TokenType::AutolinkMarker); + tokenizer.enter(Token::AutolinkMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::AutolinkMarker); - tokenizer.exit(TokenType::Autolink); + tokenizer.exit(Token::AutolinkMarker); + tokenizer.exit(Token::Autolink); (State::Ok, None) } _ => unreachable!("expected `>`"), diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs index 3ca3266..69a4709 100644 --- a/src/construct/blank_line.rs +++ b/src/construct/blank_line.rs @@ -20,7 +20,7 @@ //! //! ## Tokens //! -//! * [`SpaceOrTab`][crate::tokenizer::TokenType::SpaceOrTab] +//! * [`SpaceOrTab`][crate::token::Token::SpaceOrTab] //! //! ## References //! diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs index a3ffbc7..096ac0a 100644 --- a/src/construct/block_quote.rs +++ b/src/construct/block_quote.rs @@ -19,10 +19,10 @@ //! //! ## Tokens //! -//! * [`BlockQuote`][TokenType::BlockQuote] -//! * [`BlockQuoteMarker`][TokenType::BlockQuoteMarker] -//! * [`BlockQuotePrefix`][TokenType::BlockQuotePrefix] -//! * [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace] +//! * [`BlockQuote`][Token::BlockQuote] +//! * [`BlockQuoteMarker`][Token::BlockQuoteMarker] +//! * [`BlockQuotePrefix`][Token::BlockQuotePrefix] +//! * [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace] //! //! ## References //! @@ -35,7 +35,8 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::space_or_tab_min_max; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of block quote. /// @@ -55,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.enter(TokenType::BlockQuote); + tokenizer.enter(Token::BlockQuote); cont_before(tokenizer, code) } _ => cont_before(tokenizer, code), @@ -82,10 +83,10 @@ pub fn cont(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.enter(TokenType::BlockQuotePrefix); - tokenizer.enter(TokenType::BlockQuoteMarker); + tokenizer.enter(Token::BlockQuotePrefix); + tokenizer.enter(Token::BlockQuoteMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::BlockQuoteMarker); + tokenizer.exit(Token::BlockQuoteMarker); (State::Fn(Box::new(cont_after)), None) } _ => (State::Nok, None), @@ -101,20 +102,20 @@ fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn cont_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::VirtualSpace | Code::Char('\t' | ' ') => { - tokenizer.enter(TokenType::BlockQuoteWhitespace); + tokenizer.enter(Token::BlockQuoteWhitespace); tokenizer.consume(code); - tokenizer.exit(TokenType::BlockQuoteWhitespace); - tokenizer.exit(TokenType::BlockQuotePrefix); + tokenizer.exit(Token::BlockQuoteWhitespace); + tokenizer.exit(Token::BlockQuotePrefix); (State::Ok, None) } _ => { - tokenizer.exit(TokenType::BlockQuotePrefix); + tokenizer.exit(Token::BlockQuotePrefix); (State::Ok, Some(vec![code])) } } } /// End of a block quote. -pub fn end() -> Vec<TokenType> { - vec![TokenType::BlockQuote] +pub fn end() -> Vec<Token> { + vec![Token::BlockQuote] } diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs index 0ccc574..55d321a 100644 --- a/src/construct/character_escape.rs +++ b/src/construct/character_escape.rs @@ -19,9 +19,9 @@ //! //! ## Tokens //! -//! * [`CharacterEscape`][TokenType::CharacterEscape] -//! * [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker] -//! * [`CharacterEscapeValue`][TokenType::CharacterEscapeValue] +//! * [`CharacterEscape`][Token::CharacterEscape] +//! * [`CharacterEscapeMarker`][Token::CharacterEscapeMarker] +//! * [`CharacterEscapeValue`][Token::CharacterEscapeValue] //! //! ## References //! @@ -33,7 +33,8 @@ //! [character_reference]: crate::construct::character_reference //! [hard_break_escape]: crate::construct::hard_break_escape -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of a character escape. /// @@ -45,10 +46,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('\\') => { - tokenizer.enter(TokenType::CharacterEscape); - tokenizer.enter(TokenType::CharacterEscapeMarker); + tokenizer.enter(Token::CharacterEscape); + tokenizer.enter(Token::CharacterEscapeMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterEscapeMarker); + tokenizer.exit(Token::CharacterEscapeMarker); (State::Fn(Box::new(inside)), None) } _ => (State::Nok, None), @@ -65,10 +66,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(char) if char.is_ascii_punctuation() => { - tokenizer.enter(TokenType::CharacterEscapeValue); + tokenizer.enter(Token::CharacterEscapeValue); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterEscapeValue); - tokenizer.exit(TokenType::CharacterEscape); + tokenizer.exit(Token::CharacterEscapeValue); + tokenizer.exit(Token::CharacterEscape); (State::Ok, None) } _ => (State::Nok, None), diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs index 65e49ca..04e89d2 100644 --- a/src/construct/character_reference.rs +++ b/src/construct/character_reference.rs @@ -42,12 +42,12 @@ //! //! ## Tokens //! -//! * [`CharacterReference`][TokenType::CharacterReference] -//! * [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker] -//! * [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal] -//! * [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric] -//! * [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi] -//! * [`CharacterReferenceValue`][TokenType::CharacterReferenceValue] +//! * [`CharacterReference`][Token::CharacterReference] +//! * [`CharacterReferenceMarker`][Token::CharacterReferenceMarker] +//! * [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal] +//! * [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric] +//! * [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi] +//! * [`CharacterReferenceValue`][Token::CharacterReferenceValue] //! //! ## References //! @@ -65,7 +65,8 @@ use crate::constant::{ CHARACTER_REFERENCE_DECIMAL_SIZE_MAX, CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX, CHARACTER_REFERENCE_NAMED_SIZE_MAX, CHARACTER_REFERENCE_NAMES, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Kind of a character reference. #[derive(Debug, Clone, PartialEq)] @@ -119,10 +120,10 @@ struct Info { pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('&') => { - tokenizer.enter(TokenType::CharacterReference); - tokenizer.enter(TokenType::CharacterReferenceMarker); + tokenizer.enter(Token::CharacterReference); + tokenizer.enter(Token::CharacterReferenceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarker); + tokenizer.exit(Token::CharacterReferenceMarker); (State::Fn(Box::new(open)), None) } _ => (State::Nok, None), @@ -143,12 +144,12 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { kind: Kind::Named, }; if let Code::Char('#') = code { - tokenizer.enter(TokenType::CharacterReferenceMarkerNumeric); + tokenizer.enter(Token::CharacterReferenceMarkerNumeric); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerNumeric); + tokenizer.exit(Token::CharacterReferenceMarkerNumeric); (State::Fn(Box::new(|t, c| numeric(t, c, info))), None) } else { - tokenizer.enter(TokenType::CharacterReferenceValue); + tokenizer.enter(Token::CharacterReferenceValue); value(tokenizer, code, info) } } @@ -162,14 +163,14 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// ``` fn numeric(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { if let Code::Char('x' | 'X') = code { - tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal); - tokenizer.enter(TokenType::CharacterReferenceValue); + tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal); + tokenizer.enter(Token::CharacterReferenceValue); info.kind = Kind::Hexadecimal; (State::Fn(Box::new(|t, c| value(t, c, info))), None) } else { - tokenizer.enter(TokenType::CharacterReferenceValue); + tokenizer.enter(Token::CharacterReferenceValue); info.kind = Kind::Decimal; value(tokenizer, code, info) } @@ -194,11 +195,11 @@ fn value(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { (State::Nok, None) } else { - tokenizer.exit(TokenType::CharacterReferenceValue); - tokenizer.enter(TokenType::CharacterReferenceMarkerSemi); + tokenizer.exit(Token::CharacterReferenceValue); + tokenizer.enter(Token::CharacterReferenceMarkerSemi); tokenizer.consume(code); - tokenizer.exit(TokenType::CharacterReferenceMarkerSemi); - tokenizer.exit(TokenType::CharacterReference); + tokenizer.exit(Token::CharacterReferenceMarkerSemi); + tokenizer.exit(Token::CharacterReference); (State::Ok, None) } } diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs index 05266ba..617979f 100644 --- a/src/construct/code_fenced.rs +++ b/src/construct/code_fenced.rs @@ -77,14 +77,14 @@ //! //! ## Tokens //! -//! * [`CodeFenced`][TokenType::CodeFenced] -//! * [`CodeFencedFence`][TokenType::CodeFencedFence] -//! * [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo] -//! * [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta] -//! * [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence] -//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`CodeFenced`][Token::CodeFenced] +//! * [`CodeFencedFence`][Token::CodeFencedFence] +//! * [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo] +//! * [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta] +//! * [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence] +//! * [`CodeFlowChunk`][Token::CodeFlowChunk] +//! * [`LineEnding`][Token::LineEnding] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -103,7 +103,8 @@ use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE}; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; use crate::util::span::from_exit_event; /// Kind of fences. @@ -184,8 +185,8 @@ struct Info { /// ~~~ /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::CodeFenced); - tokenizer.enter(TokenType::CodeFencedFence); + tokenizer.enter(Token::CodeFenced); + tokenizer.enter(Token::CodeFencedFence); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before_sequence_open)(tokenizer, code) } @@ -202,7 +203,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult let mut prefix = 0; if let Some(event) = tail { - if event.token_type == TokenType::SpaceOrTab { + if event.token_type == Token::SpaceOrTab { let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1); prefix = span.end_index - span.start_index; } @@ -210,7 +211,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult match code { Code::Char('`' | '~') => { - tokenizer.enter(TokenType::CodeFencedFenceSequence); + tokenizer.enter(Token::CodeFencedFenceSequence); sequence_open( tokenizer, code, @@ -245,7 +246,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State ) } _ if info.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => { - tokenizer.exit(TokenType::CodeFencedFenceSequence); + tokenizer.exit(Token::CodeFencedFenceSequence); tokenizer.attempt_opt(space_or_tab(), |t, c| info_before(t, c, info))(tokenizer, code) } _ => (State::Nok, None), @@ -262,12 +263,12 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State fn info_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } _ => { - tokenizer.enter(TokenType::CodeFencedFenceInfo); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter(Token::CodeFencedFenceInfo); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); info_inside(tokenizer, code, info, vec![]) } } @@ -288,14 +289,14 @@ fn info_inside( ) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::CodeFencedFenceInfo); - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::CodeFencedFenceInfo); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } Code::VirtualSpace | Code::Char('\t' | ' ') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::CodeFencedFenceInfo); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::CodeFencedFenceInfo); tokenizer.attempt_opt(space_or_tab(), |t, c| meta_before(t, c, info))(tokenizer, code) } Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None), @@ -320,12 +321,12 @@ fn info_inside( fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } _ => { - tokenizer.enter(TokenType::CodeFencedFenceMeta); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter(Token::CodeFencedFenceMeta); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); meta(tokenizer, code, info) } } @@ -341,9 +342,9 @@ fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu fn meta(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::CodeFencedFenceMeta); - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::CodeFencedFenceMeta); + tokenizer.exit(Token::CodeFencedFence); at_break(tokenizer, code, info) } Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None), @@ -390,9 +391,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| close_start(t, c, info))), None) } _ => unreachable!("expected eol"), @@ -411,7 +412,7 @@ fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu /// | ~~~ /// ``` fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { - tokenizer.enter(TokenType::CodeFencedFence); + tokenizer.enter(Token::CodeFencedFence); tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), |t, c| { close_before(t, c, info) })(tokenizer, code) @@ -431,7 +432,7 @@ fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu fn close_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::Char(char) if char == info.kind.as_char() => { - tokenizer.enter(TokenType::CodeFencedFenceSequence); + tokenizer.enter(Token::CodeFencedFenceSequence); close_sequence(tokenizer, code, info, 0) } _ => (State::Nok, None), @@ -455,7 +456,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize ) } _ if size >= CODE_FENCED_SEQUENCE_SIZE_MIN && size >= info.size => { - tokenizer.exit(TokenType::CodeFencedFenceSequence); + tokenizer.exit(Token::CodeFencedFenceSequence); tokenizer.attempt_opt(space_or_tab(), close_sequence_after)(tokenizer, code) } _ => (State::Nok, None), @@ -472,7 +473,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFencedFence); + tokenizer.exit(Token::CodeFencedFence); (State::Ok, Some(vec![code])) } _ => (State::Nok, None), @@ -487,9 +488,9 @@ fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult /// ~~~ /// ``` fn content_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| content_start(t, c, info))), None) } /// Before code content, definitely not before a closing fence. @@ -518,7 +519,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe at_break(tokenizer, code, info) } _ => { - tokenizer.enter(TokenType::CodeFlowChunk); + tokenizer.enter(Token::CodeFlowChunk); content_continue(tokenizer, code, info) } } @@ -536,7 +537,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFlowChunk); + tokenizer.exit(Token::CodeFlowChunk); at_break(tokenizer, code, info) } _ => { @@ -557,7 +558,7 @@ fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateF /// ~~~| /// ``` fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.exit(TokenType::CodeFenced); + tokenizer.exit(Token::CodeFenced); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs index 9bdfd71..8966249 100644 --- a/src/construct/code_indented.rs +++ b/src/construct/code_indented.rs @@ -28,10 +28,10 @@ //! //! ## Tokens //! -//! * [`CodeIndented`][TokenType::CodeIndented] -//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`CodeIndented`][Token::CodeIndented] +//! * [`CodeFlowChunk`][Token::CodeFlowChunk] +//! * [`LineEnding`][Token::LineEnding] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -47,7 +47,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::TAB_SIZE; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of code (indented). /// @@ -63,7 +64,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { if tokenizer.interrupt { (State::Nok, None) } else { - tokenizer.enter(TokenType::CodeIndented); + tokenizer.enter(Token::CodeIndented); tokenizer.go(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), at_break)(tokenizer, code) } } @@ -82,7 +83,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { Box::new(if ok { at_break } else { after }) })(tokenizer, code), _ => { - tokenizer.enter(TokenType::CodeFlowChunk); + tokenizer.enter(Token::CodeFlowChunk); content(tokenizer, code) } } @@ -98,7 +99,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::CodeFlowChunk); + tokenizer.exit(Token::CodeFlowChunk); at_break(tokenizer, code) } _ => { @@ -114,7 +115,7 @@ fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// ab| /// ``` fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.exit(TokenType::CodeIndented); + tokenizer.exit(Token::CodeIndented); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) @@ -130,9 +131,9 @@ fn further_start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { // To do: `nok` if lazy line. match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(further_start)), None) } _ => tokenizer.attempt(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), |ok| { diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs index a6dc7eb..6df61b8 100644 --- a/src/construct/code_text.rs +++ b/src/construct/code_text.rs @@ -67,10 +67,10 @@ //! //! ## Tokens //! -//! * [`CodeText`][TokenType::CodeText] -//! * [`CodeTextData`][TokenType::CodeTextData] -//! * [`CodeTextLineEnding`][TokenType::CodeTextLineEnding] -//! * [`CodeTextSequence`][TokenType::CodeTextSequence] +//! * [`CodeText`][Token::CodeText] +//! * [`CodeTextData`][Token::CodeTextData] +//! * [`CodeTextLineEnding`][Token::CodeTextLineEnding] +//! * [`CodeTextSequence`][Token::CodeTextSequence] //! //! ## References //! @@ -83,7 +83,8 @@ //! [code_fenced]: crate::construct::code_fenced //! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of code (text). /// @@ -100,11 +101,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('`') if tokenizer.previous != Code::Char('`') - || (len > 0 - && tokenizer.events[len - 1].token_type == TokenType::CharacterEscape) => + || (len > 0 && tokenizer.events[len - 1].token_type == Token::CharacterEscape) => { - tokenizer.enter(TokenType::CodeText); - tokenizer.enter(TokenType::CodeTextSequence); + tokenizer.enter(Token::CodeText); + tokenizer.enter(Token::CodeTextSequence); sequence_open(tokenizer, code, 0) } _ => (State::Nok, None), @@ -124,7 +124,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnR None, ) } else { - tokenizer.exit(TokenType::CodeTextSequence); + tokenizer.exit(Token::CodeTextSequence); between(tokenizer, code, size) } } @@ -139,20 +139,20 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe match code { Code::None => (State::Nok, None), Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::CodeTextLineEnding); + tokenizer.enter(Token::CodeTextLineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::CodeTextLineEnding); + tokenizer.exit(Token::CodeTextLineEnding); ( State::Fn(Box::new(move |t, c| between(t, c, size_open))), None, ) } Code::Char('`') => { - tokenizer.enter(TokenType::CodeTextSequence); + tokenizer.enter(Token::CodeTextSequence); sequence_close(tokenizer, code, size_open, 0) } _ => { - tokenizer.enter(TokenType::CodeTextData); + tokenizer.enter(Token::CodeTextData); data(tokenizer, code, size_open) } } @@ -166,7 +166,7 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe fn data(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '`') => { - tokenizer.exit(TokenType::CodeTextData); + tokenizer.exit(Token::CodeTextData); between(tokenizer, code, size_open) } _ => { @@ -198,16 +198,16 @@ fn sequence_close( ) } _ if size_open == size => { - tokenizer.exit(TokenType::CodeTextSequence); - tokenizer.exit(TokenType::CodeText); + tokenizer.exit(Token::CodeTextSequence); + tokenizer.exit(Token::CodeText); (State::Ok, Some(vec![code])) } _ => { let index = tokenizer.events.len(); - tokenizer.exit(TokenType::CodeTextSequence); + tokenizer.exit(Token::CodeTextSequence); // Change the token type. - tokenizer.events[index - 1].token_type = TokenType::CodeTextData; - tokenizer.events[index].token_type = TokenType::CodeTextData; + tokenizer.events[index - 1].token_type = Token::CodeTextData; + tokenizer.events[index].token_type = Token::CodeTextData; between(tokenizer, code, size_open) } } diff --git a/src/construct/definition.rs b/src/construct/definition.rs index db4a009..4d14653 100644 --- a/src/construct/definition.rs +++ b/src/construct/definition.rs @@ -59,21 +59,21 @@ //! //! ## Tokens //! -//! * [`Definition`][TokenType::Definition] -//! * [`DefinitionDestination`][TokenType::DefinitionDestination] -//! * [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral] -//! * [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker] -//! * [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] -//! * [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] -//! * [`DefinitionLabel`][TokenType::DefinitionLabel] -//! * [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker] -//! * [`DefinitionLabelString`][TokenType::DefinitionLabelString] -//! * [`DefinitionMarker`][TokenType::DefinitionMarker] -//! * [`DefinitionTitle`][TokenType::DefinitionTitle] -//! * [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker] -//! * [`DefinitionTitleString`][TokenType::DefinitionTitleString] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`Definition`][Token::Definition] +//! * [`DefinitionDestination`][Token::DefinitionDestination] +//! * [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral] +//! * [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker] +//! * [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw] +//! * [`DefinitionDestinationString`][Token::DefinitionDestinationString] +//! * [`DefinitionLabel`][Token::DefinitionLabel] +//! * [`DefinitionLabelMarker`][Token::DefinitionLabelMarker] +//! * [`DefinitionLabelString`][Token::DefinitionLabelString] +//! * [`DefinitionMarker`][Token::DefinitionMarker] +//! * [`DefinitionTitle`][Token::DefinitionTitle] +//! * [`DefinitionTitleMarker`][Token::DefinitionTitleMarker] +//! * [`DefinitionTitleString`][Token::DefinitionTitleString] +//! * [`LineEnding`][Token::LineEnding] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -99,7 +99,8 @@ use crate::construct::{ partial_space_or_tab::{space_or_tab, space_or_tab_eol}, partial_title::{start as title, Options as TitleOptions}, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// At the start of a definition. /// @@ -109,14 +110,14 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { let index = tokenizer.events.len(); let definition_before = index > 3 - && tokenizer.events[index - 1].token_type == TokenType::LineEnding - && tokenizer.events[index - 3].token_type == TokenType::Definition; + && tokenizer.events[index - 1].token_type == Token::LineEnding + && tokenizer.events[index - 3].token_type == Token::Definition; // Do not interrupt paragraphs (but do follow definitions). if tokenizer.interrupt && !definition_before { (State::Nok, None) } else { - tokenizer.enter(TokenType::Definition); + tokenizer.enter(Token::Definition); // Note: arbitrary whitespace allowed even if code (indented) is on. tokenizer.attempt_opt(space_or_tab(), before)(tokenizer, code) } @@ -135,9 +136,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, LabelOptions { - label: TokenType::DefinitionLabel, - marker: TokenType::DefinitionLabelMarker, - string: TokenType::DefinitionLabelString, + label: Token::DefinitionLabel, + marker: Token::DefinitionLabelMarker, + string: Token::DefinitionLabelString, }, ) }, @@ -155,9 +156,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn label_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(':') => { - tokenizer.enter(TokenType::DefinitionMarker); + tokenizer.enter(Token::DefinitionMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::DefinitionMarker); + tokenizer.exit(Token::DefinitionMarker); ( State::Fn(Box::new( tokenizer.attempt_opt(space_or_tab_eol(), destination_before), @@ -185,11 +186,11 @@ fn destination_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { c, DestinationOptions { limit: usize::MAX, - destination: TokenType::DefinitionDestination, - literal: TokenType::DefinitionDestinationLiteral, - marker: TokenType::DefinitionDestinationLiteralMarker, - raw: TokenType::DefinitionDestinationRaw, - string: TokenType::DefinitionDestinationString, + destination: Token::DefinitionDestination, + literal: Token::DefinitionDestinationLiteral, + marker: Token::DefinitionDestinationLiteralMarker, + raw: Token::DefinitionDestinationRaw, + string: Token::DefinitionDestinationString, }, ) }, @@ -228,7 +229,7 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn after_whitespace(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Definition); + tokenizer.exit(Token::Definition); // You’d be interrupting. tokenizer.interrupt = true; (State::Ok, Some(vec![code])) @@ -262,9 +263,9 @@ fn title_before_marker(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, TitleOptions { - title: TokenType::DefinitionTitle, - marker: TokenType::DefinitionTitleMarker, - string: TokenType::DefinitionTitleString, + title: Token::DefinitionTitle, + marker: Token::DefinitionTitleMarker, + string: Token::DefinitionTitleString, }, ) }, diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs index 212d276..4fb87bf 100644 --- a/src/construct/hard_break_escape.rs +++ b/src/construct/hard_break_escape.rs @@ -26,8 +26,8 @@ //! //! ## Tokens //! -//! * [`HardBreakEscape`][TokenType::HardBreakEscape] -//! * [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker] +//! * [`HardBreakEscape`][Token::HardBreakEscape] +//! * [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker] //! //! ## References //! @@ -40,7 +40,8 @@ //! [hard_break_trailing]: crate::construct::hard_break_trailing //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of a hard break (escape). /// @@ -50,10 +51,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('\\') => { - tokenizer.enter(TokenType::HardBreakEscape); - tokenizer.enter(TokenType::HardBreakEscapeMarker); + tokenizer.enter(Token::HardBreakEscape); + tokenizer.enter(Token::HardBreakEscapeMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::HardBreakEscapeMarker); + tokenizer.exit(Token::HardBreakEscapeMarker); (State::Fn(Box::new(inside)), None) } _ => (State::Nok, None), @@ -68,7 +69,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HardBreakEscape); + tokenizer.exit(Token::HardBreakEscape); (State::Ok, Some(vec![code])) } _ => (State::Nok, None), diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs index 35a7cab..6626675 100644 --- a/src/construct/hard_break_trailing.rs +++ b/src/construct/hard_break_trailing.rs @@ -26,8 +26,8 @@ //! //! ## Tokens //! -//! * [`HardBreakTrailing`][TokenType::HardBreakTrailing] -//! * [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace] +//! * [`HardBreakTrailing`][Token::HardBreakTrailing] +//! * [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace] //! //! ## References //! @@ -41,7 +41,8 @@ //! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element use crate::constant::HARD_BREAK_PREFIX_SIZE_MIN; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Start of a hard break (trailing). /// @@ -52,8 +53,8 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(' ') => { - tokenizer.enter(TokenType::HardBreakTrailing); - tokenizer.enter(TokenType::HardBreakTrailingSpace); + tokenizer.enter(Token::HardBreakTrailing); + tokenizer.enter(Token::HardBreakTrailingSpace); tokenizer.consume(code); (State::Fn(Box::new(|t, c| inside(t, c, 1))), None) } @@ -79,8 +80,8 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnResult { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if size >= HARD_BREAK_PREFIX_SIZE_MIN => { - tokenizer.exit(TokenType::HardBreakTrailingSpace); - tokenizer.exit(TokenType::HardBreakTrailing); + tokenizer.exit(Token::HardBreakTrailingSpace); + tokenizer.exit(Token::HardBreakTrailing); (State::Ok, Some(vec![code])) } _ => (State::Nok, None), diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs index 9fa2ace..4a4992a 100644 --- a/src/construct/heading_atx.rs +++ b/src/construct/heading_atx.rs @@ -37,10 +37,10 @@ //! //! ## Tokens //! -//! * [`HeadingAtx`][TokenType::HeadingAtx] -//! * [`HeadingAtxSequence`][TokenType::HeadingAtxSequence] -//! * [`HeadingAtxText`][TokenType::HeadingAtxText] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`HeadingAtx`][Token::HeadingAtx] +//! * [`HeadingAtxSequence`][Token::HeadingAtxSequence] +//! * [`HeadingAtxText`][Token::HeadingAtxText] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -56,9 +56,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE}; -use crate::tokenizer::{ - Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer, -}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::edit_map::EditMap; /// Start of a heading (atx). @@ -67,7 +66,7 @@ use crate::util::edit_map::EditMap; /// |## alpha /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::HeadingAtx); + tokenizer.enter(Token::HeadingAtx); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code) } @@ -79,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { /// ``` fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { if Code::Char('#') == code { - tokenizer.enter(TokenType::HeadingAtxSequence); + tokenizer.enter(Token::HeadingAtxSequence); sequence_open(tokenizer, code, 0) } else { (State::Nok, None) @@ -94,7 +93,7 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if rank > 0 => { - tokenizer.exit(TokenType::HeadingAtxSequence); + tokenizer.exit(Token::HeadingAtxSequence); at_break(tokenizer, code) } Code::Char('#') if rank < HEADING_ATX_OPENING_FENCE_SIZE_MAX => { @@ -107,7 +106,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR ) } _ if rank > 0 => { - tokenizer.exit(TokenType::HeadingAtxSequence); + tokenizer.exit(Token::HeadingAtxSequence); tokenizer.go(space_or_tab(), at_break)(tokenizer, code) } _ => (State::Nok, None), @@ -126,7 +125,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HeadingAtx); + tokenizer.exit(Token::HeadingAtx); tokenizer.register_resolver("heading_atx".to_string(), Box::new(resolve)); // Feel free to interrupt. tokenizer.interrupt = false; @@ -136,11 +135,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { tokenizer.go(space_or_tab(), at_break)(tokenizer, code) } Code::Char('#') => { - tokenizer.enter(TokenType::HeadingAtxSequence); + tokenizer.enter(Token::HeadingAtxSequence); further_sequence(tokenizer, code) } Code::Char(_) => { - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); data(tokenizer, code) } } @@ -157,7 +156,7 @@ fn further_sequence(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { tokenizer.consume(code); (State::Fn(Box::new(further_sequence)), None) } else { - tokenizer.exit(TokenType::HeadingAtxSequence); + tokenizer.exit(Token::HeadingAtxSequence); at_break(tokenizer, code) } } @@ -171,7 +170,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { // Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text. Code::None | Code::CarriageReturnLineFeed | Code::Char('\t' | '\n' | '\r' | ' ') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code) } _ => { @@ -192,7 +191,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.token_type == TokenType::HeadingAtx { + if event.token_type == Token::HeadingAtx { if event.event_type == EventType::Enter { heading_start = Some(index); } else if let Some(start) = data_start { @@ -204,7 +203,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Enter, - token_type: TokenType::HeadingAtxText, + token_type: Token::HeadingAtxText, point: tokenizer.events[start].point.clone(), index: tokenizer.events[start].index, previous: None, @@ -221,7 +220,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::HeadingAtxText, + token_type: Token::HeadingAtxText, point: tokenizer.events[end].point.clone(), index: tokenizer.events[end].index, previous: None, @@ -234,7 +233,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { data_start = None; data_end = None; } - } else if heading_start.is_some() && event.token_type == TokenType::Data { + } else if heading_start.is_some() && event.token_type == Token::Data { if event.event_type == EventType::Enter { if data_start.is_none() { data_start = Some(index); diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs index 440baa8..633f7de 100644 --- a/src/construct/heading_setext.rs +++ b/src/construct/heading_setext.rs @@ -40,9 +40,9 @@ //! //! ## Tokens //! -//! * [`HeadingSetext`][TokenType::HeadingSetext] -//! * [`HeadingSetextText`][TokenType::HeadingSetextText] -//! * [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline] +//! * [`HeadingSetext`][Token::HeadingSetext] +//! * [`HeadingSetextText`][Token::HeadingSetextText] +//! * [`HeadingSetextUnderline`][Token::HeadingSetextUnderline] //! //! ## References //! @@ -59,7 +59,8 @@ use crate::constant::TAB_SIZE; use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; -use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::{edit_map::EditMap, skip::opt_back as skip_opt_back}; /// Kind of underline. @@ -120,14 +121,14 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { skip_opt_back( &tokenizer.events, index - 1, - &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix], + &[Token::SpaceOrTab, Token::BlockQuotePrefix], ) } else { 0 }; - let previous = skip_opt_back(&tokenizer.events, previous, &[TokenType::LineEnding]); + let previous = skip_opt_back(&tokenizer.events, previous, &[Token::LineEnding]); let paragraph_before = - previous > 1 && tokenizer.events[previous].token_type == TokenType::Paragraph; + previous > 1 && tokenizer.events[previous].token_type == Token::Paragraph; println!( "setext-start: {:?} {:?} {:?}", @@ -152,7 +153,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(char) if char == '-' || char == '=' => { - tokenizer.enter(TokenType::HeadingSetextUnderline); + tokenizer.enter(Token::HeadingSetextUnderline); inside(tokenizer, code, Kind::from_char(char)) } _ => (State::Nok, None), @@ -184,7 +185,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult { fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HeadingSetextUnderline); + tokenizer.exit(Token::HeadingSetextUnderline); // Feel free to interrupt. tokenizer.interrupt = false; tokenizer.register_resolver("heading_setext".to_string(), Box::new(resolve)); @@ -206,27 +207,27 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { // Find paragraphs. if event.event_type == EventType::Enter { - if event.token_type == TokenType::Paragraph { + if event.token_type == Token::Paragraph { paragraph_enter = Some(index); } - } else if event.token_type == TokenType::Paragraph { + } else if event.token_type == Token::Paragraph { paragraph_exit = Some(index); } // We know this is preceded by a paragraph. // Otherwise we don’t parse. - else if event.token_type == TokenType::HeadingSetextUnderline { + else if event.token_type == Token::HeadingSetextUnderline { let enter = paragraph_enter.take().unwrap(); let exit = paragraph_exit.take().unwrap(); // Change types of Enter:Paragraph, Exit:Paragraph. - tokenizer.events[enter].token_type = TokenType::HeadingSetextText; - tokenizer.events[exit].token_type = TokenType::HeadingSetextText; + tokenizer.events[enter].token_type = Token::HeadingSetextText; + tokenizer.events[exit].token_type = Token::HeadingSetextText; // Add Enter:HeadingSetext, Exit:HeadingSetext. let mut heading_enter = tokenizer.events[enter].clone(); - heading_enter.token_type = TokenType::HeadingSetext; + heading_enter.token_type = Token::HeadingSetext; let mut heading_exit = tokenizer.events[index].clone(); - heading_exit.token_type = TokenType::HeadingSetext; + heading_exit.token_type = Token::HeadingSetext; edit_map.add(enter, 0, vec![heading_enter]); edit_map.add(index + 1, 0, vec![heading_exit]); diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs index be7a3a9..fde0a34 100644 --- a/src/construct/html_flow.rs +++ b/src/construct/html_flow.rs @@ -82,9 +82,9 @@ //! //! ## Tokens //! -//! * [`HtmlFlow`][TokenType::HtmlFlow] -//! * [`HtmlFlowData`][TokenType::HtmlFlowData] -//! * [`LineEnding`][TokenType::LineEnding] +//! * [`HtmlFlow`][Token::HtmlFlow] +//! * [`HtmlFlowData`][Token::HtmlFlowData] +//! * [`LineEnding`][Token::LineEnding] //! //! ## References //! @@ -102,7 +102,8 @@ use crate::constant::{HTML_BLOCK_NAMES, HTML_RAW_NAMES, HTML_RAW_SIZE_MAX, TAB_S use crate::construct::{ blank_line::start as blank_line, partial_space_or_tab::space_or_tab_min_max, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; use crate::util::codes::{parse, serialize}; // To do: mark as concrete (block quotes or lists can’t “pierce” into HTML). @@ -203,8 +204,8 @@ struct Info { /// ``` /// pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::HtmlFlow); - tokenizer.enter(TokenType::HtmlFlowData); + tokenizer.enter(Token::HtmlFlow); + tokenizer.enter(Token::HtmlFlowData); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code) } @@ -776,7 +777,7 @@ fn continuation(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRes /// <x>| /// ``` fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { - tokenizer.exit(TokenType::HtmlFlowData); + tokenizer.exit(Token::HtmlFlowData); html_continue_start(tokenizer, code, info) } @@ -789,23 +790,23 @@ fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info fn html_continue_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None => { - tokenizer.exit(TokenType::HtmlFlow); + tokenizer.exit(Token::HtmlFlow); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) } // To do: do not allow lazy lines. Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); ( State::Fn(Box::new(|t, c| html_continue_start(t, c, info))), None, ) } _ => { - tokenizer.enter(TokenType::HtmlFlowData); + tokenizer.enter(Token::HtmlFlowData); continuation(tokenizer, code, info) } } @@ -955,8 +956,8 @@ fn continuation_declaration_inside( fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HtmlFlowData); - tokenizer.exit(TokenType::HtmlFlow); + tokenizer.exit(Token::HtmlFlowData); + tokenizer.exit(Token::HtmlFlow); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) @@ -978,8 +979,8 @@ fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> Stat /// /// ``` fn blank_line_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(blank_line)), None) } diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs index 0926f48..cdd7c69 100644 --- a/src/construct/html_text.rs +++ b/src/construct/html_text.rs @@ -42,8 +42,8 @@ //! //! ## Tokens //! -//! * [`HtmlText`][TokenType::HtmlText] -//! * [`HtmlTextData`][TokenType::HtmlTextData] +//! * [`HtmlText`][Token::HtmlText] +//! * [`HtmlTextData`][Token::HtmlTextData] //! //! ## References //! @@ -55,7 +55,8 @@ //! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing use crate::construct::partial_space_or_tab::space_or_tab; -use crate::tokenizer::{Code, State, StateFn, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFn, StateFnResult, Tokenizer}; use crate::util::codes::parse; /// Start of HTML (text) @@ -65,8 +66,8 @@ use crate::util::codes::parse; /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { if Code::Char('<') == code { - tokenizer.enter(TokenType::HtmlText); - tokenizer.enter(TokenType::HtmlTextData); + tokenizer.enter(Token::HtmlText); + tokenizer.enter(Token::HtmlTextData); tokenizer.consume(code); (State::Fn(Box::new(open)), None) } else { @@ -617,8 +618,8 @@ fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('>') => { tokenizer.consume(code); - tokenizer.exit(TokenType::HtmlTextData); - tokenizer.exit(TokenType::HtmlText); + tokenizer.exit(Token::HtmlTextData); + tokenizer.exit(Token::HtmlText); (State::Ok, None) } _ => (State::Nok, None), @@ -641,10 +642,10 @@ fn at_line_ending( ) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::HtmlTextData); - tokenizer.enter(TokenType::LineEnding); + tokenizer.exit(Token::HtmlTextData); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); ( State::Fn(Box::new(|t, c| after_line_ending(t, c, return_state))), None, @@ -687,6 +688,6 @@ fn after_line_ending_prefix( code: Code, return_state: Box<StateFn>, ) -> StateFnResult { - tokenizer.enter(TokenType::HtmlTextData); + tokenizer.enter(Token::HtmlTextData); return_state(tokenizer, code) } diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs index 3a40cc2..29ac6f9 100644 --- a/src/construct/label_end.rs +++ b/src/construct/label_end.rs @@ -102,28 +102,28 @@ //! //! ## Tokens //! -//! * [`Data`][TokenType::Data] -//! * [`Image`][TokenType::Image] -//! * [`Label`][TokenType::Label] -//! * [`LabelEnd`][TokenType::LabelEnd] -//! * [`LabelMarker`][TokenType::LabelMarker] -//! * [`LabelText`][TokenType::LabelText] -//! * [`LineEnding`][TokenType::LineEnding] -//! * [`Link`][TokenType::Link] -//! * [`Reference`][TokenType::Reference] -//! * [`ReferenceMarker`][TokenType::ReferenceMarker] -//! * [`ReferenceString`][TokenType::ReferenceString] -//! * [`Resource`][TokenType::Resource] -//! * [`ResourceDestination`][TokenType::ResourceDestination] -//! * [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral] -//! * [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker] -//! * [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw] -//! * [`ResourceDestinationString`][TokenType::ResourceDestinationString] -//! * [`ResourceMarker`][TokenType::ResourceMarker] -//! * [`ResourceTitle`][TokenType::ResourceTitle] -//! * [`ResourceTitleMarker`][TokenType::ResourceTitleMarker] -//! * [`ResourceTitleString`][TokenType::ResourceTitleString] -//! * [`SpaceOrTab`][TokenType::SpaceOrTab] +//! * [`Data`][Token::Data] +//! * [`Image`][Token::Image] +//! * [`Label`][Token::Label] +//! * [`LabelEnd`][Token::LabelEnd] +//! * [`LabelMarker`][Token::LabelMarker] +//! * [`LabelText`][Token::LabelText] +//! * [`LineEnding`][Token::LineEnding] +//! * [`Link`][Token::Link] +//! * [`Reference`][Token::Reference] +//! * [`ReferenceMarker`][Token::ReferenceMarker] +//! * [`ReferenceString`][Token::ReferenceString] +//! * [`Resource`][Token::Resource] +//! * [`ResourceDestination`][Token::ResourceDestination] +//! * [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral] +//! * [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker] +//! * [`ResourceDestinationRaw`][Token::ResourceDestinationRaw] +//! * [`ResourceDestinationString`][Token::ResourceDestinationString] +//! * [`ResourceMarker`][Token::ResourceMarker] +//! * [`ResourceTitle`][Token::ResourceTitle] +//! * [`ResourceTitleMarker`][Token::ResourceTitleMarker] +//! * [`ResourceTitleString`][Token::ResourceTitleString] +//! * [`SpaceOrTab`][Token::SpaceOrTab] //! //! ## References //! @@ -153,8 +153,9 @@ use crate::construct::{ partial_space_or_tab::space_or_tab_eol, partial_title::{start as title, Options as TitleOptions}, }; +use crate::token::Token; use crate::tokenizer::{ - Code, Event, EventType, LabelStart, Media, State, StateFnResult, TokenType, Tokenizer, + Code, Event, EventType, LabelStart, Media, State, StateFnResult, Tokenizer, }; use crate::util::{ edit_map::EditMap, @@ -198,7 +199,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { vec![ Event { event_type: EventType::Enter, - token_type: TokenType::Data, + token_type: Token::Data, point: events[data_enter_index].point.clone(), index: events[data_enter_index].index, previous: None, @@ -207,7 +208,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { }, Event { event_type: EventType::Exit, - token_type: TokenType::Data, + token_type: Token::Data, point: events[data_exit_index].point.clone(), index: events[data_exit_index].index, previous: None, @@ -229,7 +230,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { let group_enter_event = &events[group_enter_index]; // LabelLink:Exit or LabelImage:Exit. let text_enter_index = media.start.0 - + (if group_enter_event.token_type == TokenType::LabelLink { + + (if group_enter_event.token_type == Token::LabelLink { 4 } else { 6 @@ -248,10 +249,10 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { vec![ Event { event_type: EventType::Enter, - token_type: if group_enter_event.token_type == TokenType::LabelLink { - TokenType::Link + token_type: if group_enter_event.token_type == Token::LabelLink { + Token::Link } else { - TokenType::Image + Token::Image }, point: group_enter_event.point.clone(), index: group_enter_event.index, @@ -261,7 +262,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { }, Event { event_type: EventType::Enter, - token_type: TokenType::Label, + token_type: Token::Label, point: group_enter_event.point.clone(), index: group_enter_event.index, previous: None, @@ -279,7 +280,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Enter, - token_type: TokenType::LabelText, + token_type: Token::LabelText, point: events[text_enter_index].point.clone(), index: events[text_enter_index].index, previous: None, @@ -294,7 +295,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::LabelText, + token_type: Token::LabelText, point: events[text_exit_index].point.clone(), index: events[text_exit_index].index, previous: None, @@ -310,7 +311,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::Label, + token_type: Token::Label, point: events[label_exit_index].point.clone(), index: events[label_exit_index].index, previous: None, @@ -325,7 +326,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> { 0, vec![Event { event_type: EventType::Exit, - token_type: TokenType::Link, + token_type: Token::Link, point: events[group_end_index].point.clone(), index: events[group_end_index].index, previous: None, @@ -393,11 +394,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { }, }; - tokenizer.enter(TokenType::LabelEnd); - tokenizer.enter(TokenType::LabelMarker); + tokenizer.enter(Token::LabelEnd); + tokenizer.enter(Token::LabelMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelMarker); - tokenizer.exit(TokenType::LabelEnd); + tokenizer.exit(Token::LabelMarker); + tokenizer.exit(Token::LabelEnd); return (State::Fn(Box::new(move |t, c| after(t, c, info))), None); } @@ -495,13 +496,13 @@ fn ok(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { left.remove(0); tokenizer.label_start_list_loose.append(&mut left); - let is_link = tokenizer.events[info.media.start.0].token_type == TokenType::LabelLink; + let is_link = tokenizer.events[info.media.start.0].token_type == Token::LabelLink; if is_link { let mut index = 0; while index < tokenizer.label_start_stack.len() { let label_start = &mut tokenizer.label_start_stack[index]; - if tokenizer.events[label_start.start.0].token_type == TokenType::LabelLink { + if tokenizer.events[label_start.start.0].token_type == Token::LabelLink { label_start.inactive = true; } index += 1; @@ -543,10 +544,10 @@ fn nok(tokenizer: &mut Tokenizer, _code: Code, label_start_index: usize) -> Stat fn resource(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('(') => { - tokenizer.enter(TokenType::Resource); - tokenizer.enter(TokenType::ResourceMarker); + tokenizer.enter(Token::Resource); + tokenizer.enter(Token::ResourceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ResourceMarker); + tokenizer.exit(Token::ResourceMarker); (State::Fn(Box::new(resource_start)), None) } _ => unreachable!("expected `(`"), @@ -577,11 +578,11 @@ fn resource_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { c, DestinationOptions { limit: RESOURCE_DESTINATION_BALANCE_MAX, - destination: TokenType::ResourceDestination, - literal: TokenType::ResourceDestinationLiteral, - marker: TokenType::ResourceDestinationLiteralMarker, - raw: TokenType::ResourceDestinationRaw, - string: TokenType::ResourceDestinationString, + destination: Token::ResourceDestination, + literal: Token::ResourceDestinationLiteral, + marker: Token::ResourceDestinationLiteralMarker, + raw: Token::ResourceDestinationRaw, + string: Token::ResourceDestinationString, }, ) }, @@ -616,9 +617,9 @@ fn resource_between(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, TitleOptions { - title: TokenType::ResourceTitle, - marker: TokenType::ResourceTitleMarker, - string: TokenType::ResourceTitleString, + title: Token::ResourceTitle, + marker: Token::ResourceTitleMarker, + string: Token::ResourceTitleString, }, ) }, @@ -647,10 +648,10 @@ fn title_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn resource_end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(')') => { - tokenizer.enter(TokenType::ResourceMarker); + tokenizer.enter(Token::ResourceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ResourceMarker); - tokenizer.exit(TokenType::Resource); + tokenizer.exit(Token::ResourceMarker); + tokenizer.exit(Token::Resource); (State::Ok, None) } _ => (State::Nok, None), @@ -670,9 +671,9 @@ fn full_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { t, c, LabelOptions { - label: TokenType::Reference, - marker: TokenType::ReferenceMarker, - string: TokenType::ReferenceString, + label: Token::Reference, + marker: Token::ReferenceMarker, + string: Token::ReferenceString, }, ) }, @@ -696,7 +697,7 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult while index > 0 { index -= 1; let event = &events[index]; - if event.token_type == TokenType::ReferenceString { + if event.token_type == Token::ReferenceString { if event.event_type == EventType::Exit { end = Some(event.index); } else { @@ -735,10 +736,10 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('[') => { - tokenizer.enter(TokenType::Reference); - tokenizer.enter(TokenType::ReferenceMarker); + tokenizer.enter(Token::Reference); + tokenizer.enter(Token::ReferenceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ReferenceMarker); + tokenizer.exit(Token::ReferenceMarker); (State::Fn(Box::new(collapsed_reference_open)), None) } _ => (State::Nok, None), @@ -755,10 +756,10 @@ fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn collapsed_reference_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char(']') => { - tokenizer.enter(TokenType::ReferenceMarker); + tokenizer.enter(Token::ReferenceMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::ReferenceMarker); - tokenizer.exit(TokenType::Reference); + tokenizer.exit(Token::ReferenceMarker); + tokenizer.exit(Token::Reference); (State::Ok, None) } _ => (State::Nok, None), diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs index a45205a..f9b8300 100644 --- a/src/construct/label_start_image.rs +++ b/src/construct/label_start_image.rs @@ -15,9 +15,9 @@ //! //! ## Tokens //! -//! * [`LabelImage`][TokenType::LabelImage] -//! * [`LabelImageMarker`][TokenType::LabelImageMarker] -//! * [`LabelMarker`][TokenType::LabelMarker] +//! * [`LabelImage`][Token::LabelImage] +//! * [`LabelImageMarker`][Token::LabelImageMarker] +//! * [`LabelMarker`][Token::LabelMarker] //! //! ## References //! @@ -29,7 +29,8 @@ //! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element use super::label_end::resolve_media; -use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer}; /// Start of label (image) start. /// @@ -39,10 +40,10 @@ use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokeni pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('!') => { - tokenizer.enter(TokenType::LabelImage); - tokenizer.enter(TokenType::LabelImageMarker); + tokenizer.enter(Token::LabelImage); + tokenizer.enter(Token::LabelImageMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelImageMarker); + tokenizer.exit(Token::LabelImageMarker); (State::Fn(Box::new(open)), None) } _ => (State::Nok, None), @@ -57,10 +58,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { pub fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('[') => { - tokenizer.enter(TokenType::LabelMarker); + tokenizer.enter(Token::LabelMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelMarker); - tokenizer.exit(TokenType::LabelImage); + tokenizer.exit(Token::LabelMarker); + tokenizer.exit(Token::LabelImage); let end = tokenizer.events.len() - 1; tokenizer.label_start_stack.push(LabelStart { start: (end - 5, end), diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs index 6c4d7ae..59729cc 100644 --- a/src/construct/label_start_link.rs +++ b/src/construct/label_start_link.rs @@ -15,8 +15,8 @@ //! //! ## Tokens //! -//! * [`LabelLink`][TokenType::LabelLink] -//! * [`LabelMarker`][TokenType::LabelMarker] +//! * [`LabelLink`][Token::LabelLink] +//! * [`LabelMarker`][Token::LabelMarker] //! //! ## References //! @@ -28,7 +28,8 @@ //! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element use super::label_end::resolve_media; -use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer}; /// Start of label (link) start. /// @@ -39,11 +40,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::Char('[') => { let start = tokenizer.events.len(); - tokenizer.enter(TokenType::LabelLink); - tokenizer.enter(TokenType::LabelMarker); + tokenizer.enter(Token::LabelLink); + tokenizer.enter(Token::LabelMarker); tokenizer.consume(code); - tokenizer.exit(TokenType::LabelMarker); - tokenizer.exit(TokenType::LabelLink); + tokenizer.exit(Token::LabelMarker); + tokenizer.exit(Token::LabelLink); tokenizer.label_start_stack.push(LabelStart { start: (start, tokenizer.events.len() - 1), balanced: false, diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs index ace174f..967e009 100644 --- a/src/construct/paragraph.rs +++ b/src/construct/paragraph.rs @@ -19,7 +19,7 @@ //! //! ## Tokens //! -//! * [`Paragraph`][TokenType::Paragraph] +//! * [`Paragraph`][Token::Paragraph] //! //! ## References //! @@ -32,9 +32,8 @@ //! [code_text]: crate::construct::code_text //! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element -use crate::tokenizer::{ - Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer, -}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::{edit_map::EditMap, skip::opt as skip_opt}; /// Before a paragraph. @@ -48,8 +47,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { unreachable!("unexpected eol/eof") } _ => { - tokenizer.enter(TokenType::Paragraph); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text)); + tokenizer.enter(Token::Paragraph); + tokenizer.enter_with_content(Token::Data, Some(ContentType::Text)); inside(tokenizer, code) } } @@ -63,8 +62,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); - tokenizer.exit(TokenType::Paragraph); + tokenizer.exit(Token::Data); + tokenizer.exit(Token::Paragraph); tokenizer.register_resolver_before("paragraph".to_string(), Box::new(resolve)); // You’d be interrupting. tokenizer.interrupt = true; @@ -87,21 +86,21 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { while index < len { let event = &tokenizer.events[index]; - if event.event_type == EventType::Enter && event.token_type == TokenType::Paragraph { + if event.event_type == EventType::Enter && event.token_type == Token::Paragraph { // Exit:Paragraph let mut exit_index = index + 3; let mut enter_next_index = - skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]); + skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]); // Enter:Paragraph enter_next_index = skip_opt( &tokenizer.events, enter_next_index, - &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix], + &[Token::SpaceOrTab, Token::BlockQuotePrefix], ); // Find future `Paragraphs`. while enter_next_index < tokenizer.events.len() - && tokenizer.events[enter_next_index].token_type == TokenType::Paragraph + && tokenizer.events[enter_next_index].token_type == Token::Paragraph { // Remove Exit:Paragraph, Enter:LineEnding, Exit:LineEnding, Enter:Paragraph. edit_map.add(exit_index, 3, vec![]); @@ -126,11 +125,11 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { // Potential next start. exit_index = enter_next_index + 3; enter_next_index = - skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]); + skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]); enter_next_index = skip_opt( &tokenizer.events, enter_next_index, - &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix], + &[Token::SpaceOrTab, Token::BlockQuotePrefix], ); } diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs index 555ccaf..186665d 100644 --- a/src/construct/partial_data.rs +++ b/src/construct/partial_data.rs @@ -6,7 +6,8 @@ //! [string]: crate::content::string //! [text]: crate::content::text -use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer}; use crate::util::edit_map::EditMap; /// At the beginning of data. @@ -16,7 +17,7 @@ use crate::util::edit_map::EditMap; /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult { if stop.contains(&code) { - tokenizer.enter(TokenType::Data); + tokenizer.enter(Token::Data); tokenizer.consume(code); (State::Fn(Box::new(|t, c| data(t, c, stop))), None) } else { @@ -33,9 +34,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe match code { Code::None => (State::Ok, None), Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| at_break(t, c, stop))), None) } _ if stop.contains(&code) => { @@ -43,7 +44,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe (State::Ok, Some(vec![code])) } _ => { - tokenizer.enter(TokenType::Data); + tokenizer.enter(Token::Data); data(tokenizer, code, stop) } } @@ -62,7 +63,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult }; if done { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, stop) } else { tokenizer.consume(code); @@ -80,13 +81,13 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> { while index < len { let event = &tokenizer.events[index]; - if event.event_type == EventType::Enter && event.token_type == TokenType::Data { + if event.event_type == EventType::Enter && event.token_type == Token::Data { let exit_index = index + 1; let mut exit_far_index = exit_index; // Find multiple `data` events. while exit_far_index + 1 < len - && tokenizer.events[exit_far_index + 1].token_type == TokenType::Data + && tokenizer.events[exit_far_index + 1].token_type == Token::Data { exit_far_index += 2; } diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs index 31c13ec..daa968a 100644 --- a/src/construct/partial_destination.rs +++ b/src/construct/partial_destination.rs @@ -71,7 +71,8 @@ //! [label_end]: crate::construct::label_end //! [sanitize_uri]: crate::util::sanitize_uri -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; /// Configuration. /// @@ -79,15 +80,15 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token #[derive(Debug)] pub struct Options { /// Token for the whole destination. - pub destination: TokenType, + pub destination: Token, /// Token for a literal (enclosed) destination. - pub literal: TokenType, + pub literal: Token, /// Token for a literal marker. - pub marker: TokenType, + pub marker: Token, /// Token for a raw destination. - pub raw: TokenType, + pub raw: Token, /// Token for a the string. - pub string: TokenType, + pub string: Token, /// Maximum unbalanced parens. pub limit: usize, } @@ -133,7 +134,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code, options: Options) -> StateFn tokenizer.enter(info.options.destination.clone()); tokenizer.enter(info.options.raw.clone()); tokenizer.enter(info.options.string.clone()); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); raw(tokenizer, code, info) } } @@ -154,7 +155,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn (State::Ok, None) } else { tokenizer.enter(info.options.string.clone()); - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); enclosed(tokenizer, code, info) } } @@ -167,7 +168,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn fn enclosed(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::Char('>') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); tokenizer.exit(info.options.string.clone()); enclosed_before(tokenizer, code, info) } @@ -221,7 +222,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { } Code::Char(')') => { if info.balance == 0 { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); tokenizer.exit(info.options.string.clone()); tokenizer.exit(info.options.raw.clone()); tokenizer.exit(info.options.destination); @@ -239,7 +240,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { if info.balance > 0 { (State::Nok, None) } else { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); tokenizer.exit(info.options.string.clone()); tokenizer.exit(info.options.raw.clone()); tokenizer.exit(info.options.destination); diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs index f201f60..f380c7d 100644 --- a/src/construct/partial_label.rs +++ b/src/construct/partial_label.rs @@ -61,7 +61,8 @@ use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; use crate::constant::LINK_REFERENCE_SIZE_MAX; use crate::subtokenize::link; -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; /// Configuration. /// @@ -69,11 +70,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token #[derive(Debug)] pub struct Options { /// Token for the whole label. - pub label: TokenType, + pub label: Token, /// Token for the markers. - pub marker: TokenType, + pub marker: Token, /// Token for the string (inside the markers). - pub string: TokenType, + pub string: Token, } /// State needed to parse labels. @@ -144,7 +145,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes }, )(tokenizer, code), _ => { - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); if info.connect { let index = tokenizer.events.len() - 1; @@ -166,11 +167,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes fn label(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult { match code { Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '[' | ']') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } _ if info.size > LINK_REFERENCE_SIZE_MAX => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } Code::VirtualSpace | Code::Char('\t' | ' ') => { diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs index 5b1ec5e..78477de 100644 --- a/src/construct/partial_space_or_tab.rs +++ b/src/construct/partial_space_or_tab.rs @@ -5,7 +5,8 @@ //! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js) use crate::subtokenize::link; -use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, Tokenizer}; /// Options to parse `space_or_tab`. #[derive(Debug)] @@ -15,7 +16,7 @@ pub struct Options { /// Maximum allowed characters (inclusive). pub max: usize, /// Token type to use for whitespace events. - pub kind: TokenType, + pub kind: Token, /// Connect this whitespace to the previous. pub connect: bool, /// Embedded content type to use. @@ -67,7 +68,7 @@ pub fn space_or_tab() -> Box<StateFn> { /// ``` pub fn space_or_tab_min_max(min: usize, max: usize) -> Box<StateFn> { space_or_tab_with_options(Options { - kind: TokenType::SpaceOrTab, + kind: Token::SpaceOrTab, min, max, content_type: None, @@ -104,7 +105,7 @@ pub fn space_or_tab_eol_with_options(options: EolOptions) -> Box<StateFn> { tokenizer.attempt( space_or_tab_with_options(Options { - kind: TokenType::SpaceOrTab, + kind: Token::SpaceOrTab, min: 1, max: usize::MAX, content_type: info.options.content_type, @@ -196,7 +197,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResul fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) -> StateFnResult { match code { Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter_with_content(TokenType::LineEnding, info.options.content_type); + tokenizer.enter_with_content(Token::LineEnding, info.options.content_type); if info.connect { let index = tokenizer.events.len() - 1; @@ -206,7 +207,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) } tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(|t, c| after_eol(t, c, info))), None) } _ if info.ok => (State::Ok, Some(vec![code])), @@ -229,7 +230,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) fn after_eol(tokenizer: &mut Tokenizer, code: Code, info: EolInfo) -> StateFnResult { tokenizer.attempt_opt( space_or_tab_with_options(Options { - kind: TokenType::SpaceOrTab, + kind: Token::SpaceOrTab, min: 1, max: usize::MAX, content_type: info.options.content_type, diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs index 010f554..6303da8 100644 --- a/src/construct/partial_title.rs +++ b/src/construct/partial_title.rs @@ -32,7 +32,8 @@ use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions}; use crate::subtokenize::link; -use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer}; /// Configuration. /// @@ -40,11 +41,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token #[derive(Debug)] pub struct Options { /// Token for the whole title. - pub title: TokenType, + pub title: Token, /// Token for the marker. - pub marker: TokenType, + pub marker: Token, /// Token for the string inside the quotes. - pub string: TokenType, + pub string: Token, } /// Type of title. @@ -204,7 +205,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes }, )(tokenizer, code), _ => { - tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String)); + tokenizer.enter_with_content(Token::Data, Some(ContentType::String)); if info.connect { let index = tokenizer.events.len() - 1; @@ -226,11 +227,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes fn title(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult { match code { Code::Char(char) if char == info.kind.as_char() => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.exit(TokenType::Data); + tokenizer.exit(Token::Data); at_break(tokenizer, code, info) } Code::Char('\\') => { diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs index 28aca34..4d92f8d 100644 --- a/src/construct/thematic_break.rs +++ b/src/construct/thematic_break.rs @@ -35,8 +35,8 @@ //! //! ## Tokens //! -//! * [`ThematicBreak`][TokenType::ThematicBreak] -//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence] +//! * [`ThematicBreak`][Token::ThematicBreak] +//! * [`ThematicBreakSequence`][Token::ThematicBreakSequence] //! //! ## References //! @@ -51,7 +51,8 @@ use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max}; use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN}; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Type of thematic break. #[derive(Debug, PartialEq)] @@ -134,7 +135,7 @@ struct Info { /// |*** /// ``` pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { - tokenizer.enter(TokenType::ThematicBreak); + tokenizer.enter(Token::ThematicBreak); // To do: allow arbitrary when code (indented) is turned off. tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code) } @@ -170,13 +171,13 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if info.size >= THEMATIC_BREAK_MARKER_COUNT_MIN => { - tokenizer.exit(TokenType::ThematicBreak); + tokenizer.exit(Token::ThematicBreak); // Feel free to interrupt. tokenizer.interrupt = false; (State::Ok, Some(vec![code])) } Code::Char(char) if char == info.kind.as_char() => { - tokenizer.enter(TokenType::ThematicBreakSequence); + tokenizer.enter(Token::ThematicBreakSequence); sequence(tokenizer, code, info) } _ => (State::Nok, None), @@ -198,7 +199,7 @@ fn sequence(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes (State::Fn(Box::new(|t, c| sequence(t, c, info))), None) } _ => { - tokenizer.exit(TokenType::ThematicBreakSequence); + tokenizer.exit(Token::ThematicBreakSequence); tokenizer.attempt_opt(space_or_tab(), |t, c| at_break(t, c, info))(tokenizer, code) } } diff --git a/src/content/document.rs b/src/content/document.rs index dd5038f..b1f3083 100644 --- a/src/content/document.rs +++ b/src/content/document.rs @@ -14,9 +14,8 @@ use crate::construct::block_quote::{ use crate::content::flow::start as flow; use crate::parser::ParseState; use crate::subtokenize::subtokenize; -use crate::tokenizer::{ - Code, Event, EventType, Point, State, StateFn, StateFnResult, TokenType, Tokenizer, -}; +use crate::token::Token; +use crate::tokenizer::{Code, Event, EventType, Point, State, StateFn, StateFnResult, Tokenizer}; use crate::util::edit_map::EditMap; use crate::util::{ normalize_identifier::normalize_identifier, @@ -44,9 +43,7 @@ pub fn document(parse_state: &mut ParseState, point: Point, index: usize) -> Vec while index < tokenizer.events.len() { let event = &tokenizer.events[index]; - if event.event_type == EventType::Exit - && event.token_type == TokenType::DefinitionLabelString - { + if event.event_type == EventType::Exit && event.token_type == Token::DefinitionLabelString { next_definitions.insert(normalize_identifier( serialize( &parse_state.codes, @@ -409,7 +406,7 @@ fn flow_end( // To do: blank lines? Other things? if tokenizer.events.len() > 2 - && tokenizer.events[tokenizer.events.len() - 1].token_type == TokenType::LineEnding + && tokenizer.events[tokenizer.events.len() - 1].token_type == Token::LineEnding { info.last_line_ending_index = Some(tokenizer.events.len() - 2); } else { diff --git a/src/content/flow.rs b/src/content/flow.rs index f406685..e52f113 100644 --- a/src/content/flow.rs +++ b/src/content/flow.rs @@ -26,7 +26,8 @@ use crate::construct::{ html_flow::start as html_flow, paragraph::start as paragraph, thematic_break::start as thematic_break, }; -use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer}; +use crate::token::Token; +use crate::tokenizer::{Code, State, StateFnResult, Tokenizer}; /// Before flow. /// @@ -88,9 +89,9 @@ fn blank_line_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None => (State::Ok, None), Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::BlankLineEnding); + tokenizer.enter(Token::BlankLineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::BlankLineEnding); + tokenizer.exit(Token::BlankLineEnding); // Feel free to interrupt. tokenizer.interrupt = false; (State::Fn(Box::new(start)), None) @@ -112,9 +113,9 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult { match code { Code::None => (State::Ok, None), Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { - tokenizer.enter(TokenType::LineEnding); + tokenizer.enter(Token::LineEnding); tokenizer.consume(code); - tokenizer.exit(TokenType::LineEnding); + tokenizer.exit(Token::LineEnding); (State::Fn(Box::new(start)), None) } _ => unreachable!("expected eol/eof"), @@ -10,6 +10,7 @@ mod construct; mod content; mod parser; mod subtokenize; +mod token; mod tokenizer; mod unicode; mod util; diff --git a/src/token.rs b/src/token.rs new file mode 100644 index 0000000..9b59719 --- /dev/null +++ b/src/token.rs @@ -0,0 +1,1768 @@ +/// Semantic label of a span. +#[derive(Debug, Clone, PartialEq, Hash, Eq)] +pub enum Token { + /// Attention sequence. + /// + /// > 👉 **Note**: this is used while parsing but compiled away. + AttentionSequence, + /// Whole autolink. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`AutolinkEmail`][Token::AutolinkEmail], + /// [`AutolinkMarker`][Token::AutolinkMarker], + /// [`AutolinkProtocol`][Token::AutolinkProtocol] + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <https://example.com> and <admin@example.com> + /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ + /// ``` + Autolink, + /// Email autolink w/o markers. + /// + /// ## Info + /// + /// * **Context**: + /// [`Autolink`][Token::Autolink] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <admin@example.com> + /// ^^^^^^^^^^^^^^^^^ + /// ``` + AutolinkEmail, + /// Marker of an autolink. + /// + /// ## Info + /// + /// * **Context**: + /// [`Autolink`][Token::Autolink] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <https://example.com> + /// ^ ^ + /// ``` + AutolinkMarker, + /// Protocol autolink w/o markers. + /// + /// ## Info + /// + /// * **Context**: + /// [`Autolink`][Token::Autolink] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`autolink`][crate::construct::autolink] + /// + /// ## Example + /// + /// ```markdown + /// > | <https://example.com> + /// ^^^^^^^^^^^^^^^^^^^ + /// ``` + AutolinkProtocol, + /// Line ending preceded only by whitespace or nothing at all. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`blank_line`][crate::construct::blank_line] + /// + /// ## Example + /// + /// ```markdown + /// > | ␠␠␊ + /// ^ + /// ``` + BlankLineEnding, + /// Whole block quote. + /// + /// ## Info + /// + /// * **Context**: + /// [document content][crate::content::document] + /// * **Content model**: + /// [`BlockQuotePrefix`][Token::BlockQuotePrefix], + /// [flow content][crate::content::flow] + /// * **Construct**: + /// [`block_quote`][crate::construct::block_quote] + /// + /// ## Example + /// + /// ```markdown + /// > | > a + /// ^^^ + /// > | b + /// ^ + /// ``` + BlockQuote, + /// Block quote marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`BlockQuotePrefix`][Token::BlockQuotePrefix] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`block_quote`][crate::construct::block_quote] + /// + /// ## Example + /// + /// ```markdown + /// > | > a + /// ^ + /// | b + /// ``` + BlockQuoteMarker, + /// Block quote prefix. + /// + /// ## Info + /// + /// * **Context**: + /// [`BlockQuote`][Token::BlockQuote] + /// * **Content model**: + /// [`BlockQuoteMarker`][Token::BlockQuoteMarker], + /// [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace] + /// * **Construct**: + /// [`block_quote`][crate::construct::block_quote] + /// + /// ## Example + /// + /// ```markdown + /// > | > a + /// ^^ + /// | b + /// ``` + BlockQuotePrefix, + /// Block quote white space. + /// + /// ## Info + /// + /// * **Context**: + /// [`BlockQuotePrefix`][Token::BlockQuotePrefix] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`block_quote`][crate::construct::block_quote] + /// + /// ## Example + /// + /// ```markdown + /// > | > a + /// ^ + /// | b + /// ``` + BlockQuoteWhitespace, + /// Whole character escape. + /// + /// ## Info + /// + /// * **Context**: + /// [string content][crate::content::string] or + /// [text content][crate::content::text] + /// * **Content model**: + /// [`CharacterEscapeMarker`][Token::CharacterEscapeMarker], + /// [`CharacterEscapeValue`][Token::CharacterEscapeValue] + /// * **Construct**: + /// [`character_escape`][crate::construct::character_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a \- b + /// ^^ + /// ``` + CharacterEscape, + /// Character escape marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterEscape`][Token::CharacterEscape] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_escape`][crate::construct::character_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a \- b + /// ^ + /// ``` + CharacterEscapeMarker, + /// Character escape value. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterEscape`][Token::CharacterEscape] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_escape`][crate::construct::character_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a \- b + /// ^ + /// ``` + CharacterEscapeValue, + /// Whole character reference. + /// + /// ## Info + /// + /// * **Context**: + /// [string content][crate::content::string] or + /// [text content][crate::content::text] + /// * **Content model**: + /// [`CharacterReferenceMarker`][Token::CharacterReferenceMarker], + /// [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal], + /// [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric], + /// [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi], + /// [`CharacterReferenceValue`][Token::CharacterReferenceValue] + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^^^^^ ^^^^^^^ ^^^^^^^^^ + /// ``` + CharacterReference, + /// Character reference opening marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][Token::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ ^ ^ + /// ``` + CharacterReferenceMarker, + /// Character reference hexadecimal numeric marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][Token::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ + /// ``` + CharacterReferenceMarkerHexadecimal, + /// Character reference numeric marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][Token::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ ^ + /// ``` + CharacterReferenceMarkerNumeric, + /// Character reference closing marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][Token::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^ ^ ^ + /// ``` + CharacterReferenceMarkerSemi, + /// Character reference value. + /// + /// ## Info + /// + /// * **Context**: + /// [`CharacterReference`][Token::CharacterReference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`character_reference`][crate::construct::character_reference] + /// + /// ## Example + /// + /// ```markdown + /// > | a & b ≠ c 𝌆 d + /// ^^^ ^^^^ ^^^^^ + /// ``` + CharacterReferenceValue, + /// Whole code (fenced). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`CodeFencedFence`][Token::CodeFencedFence], + /// [`CodeFlowChunk`][Token::CodeFlowChunk], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^^^^ + /// > | console.log(1) + /// ^^^^^^^^^^^^^^ + /// > | ``` + /// ^^^ + /// ```` + CodeFenced, + /// A code (fenced) fence. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFenced`][Token::CodeFenced] + /// * **Content model**: + /// [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo], + /// [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta], + /// [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^^^^ + /// | console.log(1) + /// > | ``` + /// ^^^ + /// ```` + CodeFencedFence, + /// A code (fenced) fence info word. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFencedFence`][Token::CodeFencedFence] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^ + /// | console.log(1) + /// | ``` + /// ```` + CodeFencedFenceInfo, + /// A code (fenced) fence meta string. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFencedFence`][Token::CodeFencedFence] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js highlight="1" + /// ^^^^^^^^^^^^^ + /// | console.log(1) + /// | ``` + /// ```` + CodeFencedFenceMeta, + /// A code (fenced) fence sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ````markdown + /// > | ```js + /// ^^^ + /// | console.log(1) + /// > | ``` + /// ^^^ + /// ```` + CodeFencedFenceSequence, + /// A code (fenced, indented) chunk. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeFenced`][Token::CodeFenced], + /// [`CodeIndented`][Token::CodeIndented] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced], + /// [`code_indented`][crate::construct::code_indented] + /// + /// ## Example + /// + /// ````markdown + /// | ```js + /// > | console.log(1) + /// ^^^^^^^^^^^^^^ + /// | ``` + /// ```` + /// + /// ```markdown + /// > | ␠␠␠␠console.log(1) + /// ^^^^^^^^^^^^^^ + /// ``` + CodeFlowChunk, + /// Whole code (indented). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`CodeFlowChunk`][Token::CodeFlowChunk], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`code_fenced`][crate::construct::code_fenced] + /// + /// ## Example + /// + /// ```markdown + /// ␠␠␠␠console.log(1) + /// ^^^^^^^^^^^^^^^^^^ + /// ``` + CodeIndented, + /// Whole code (text). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`CodeTextData`][Token::CodeTextData], + /// [`CodeTextSequence`][Token::CodeTextSequence], + /// [`CodeTextLineEnding`][Token::CodeTextLineEnding] + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b` c + /// ^^^ + /// ``` + CodeText, + /// Code (text) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeText`][Token::CodeText], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b` c + /// ^ + /// ``` + CodeTextData, + /// Line ending in code (text). + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeText`][Token::CodeText], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b␊ + /// ^ + /// | c` d + /// ``` + CodeTextLineEnding, + /// Code (text) sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`CodeText`][Token::CodeText], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`code_text`][crate::construct::code_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a `b` c + /// ^ ^ + /// ``` + CodeTextSequence, + /// Data. + /// + /// ## Info + /// + /// * **Context**: + /// [string content][crate::content::string], + /// [text content][crate::content::text] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`data`][crate::construct::partial_data] + /// + /// ## Example + /// + /// ```markdown + /// > | aa *bb* cc + /// ^^^ ^^ ^^^ + /// ``` + Data, + /// Whole definition. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`DefinitionMarker`][Token::DefinitionMarker], + /// [`DefinitionLabel`][Token::DefinitionLabel], + /// [`DefinitionDestination`][Token::DefinitionDestination], + /// [`DefinitionTitle`][Token::DefinitionTitle], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`definition`][crate::construct::definition] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^^^^^^^^^^ + /// ``` + Definition, + /// Whole definition destination. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][Token::Definition] + /// * **Content model**: + /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral], + /// [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// > | [a]: <b> "c" + /// ^^^ + /// ``` + DefinitionDestination, + /// Definition destination literal. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestination`][Token::DefinitionDestination] + /// * **Content model**: + /// [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker], + /// [`DefinitionDestinationString`][Token::DefinitionDestinationString] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: <b> "c" + /// ^^^ + /// ``` + DefinitionDestinationLiteral, + /// Definition destination literal marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: <b> "c" + /// ^ ^ + /// ``` + DefinitionDestinationLiteralMarker, + /// Definition destination raw. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestination`][Token::DefinitionDestination] + /// * **Content model**: + /// [`DefinitionDestinationString`][Token::DefinitionDestinationString] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` + DefinitionDestinationRaw, + /// Definition destination data. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral], + /// [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// > | [a]: <b> "c" + /// ^ + /// ``` + DefinitionDestinationString, + /// Whole definition label. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][Token::Definition] + /// * **Content model**: + /// [`DefinitionLabelMarker`][Token::DefinitionLabelMarker], + /// [`DefinitionLabelString`][Token::DefinitionLabelString], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^^^ + /// ``` + DefinitionLabel, + /// Definition label marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionLabel`][Token::DefinitionLabel] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ ^ + /// ``` + DefinitionLabelMarker, + /// Definition label data. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionLabel`][Token::DefinitionLabel] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` + DefinitionLabelString, + /// Definition marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][Token::Definition] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`definition`][crate::construct::definition] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` + DefinitionMarker, + /// Whole definition title. + /// + /// ## Info + /// + /// * **Context**: + /// [`Definition`][Token::Definition] + /// * **Content model**: + /// [`DefinitionTitleMarker`][Token::DefinitionTitleMarker], + /// [`DefinitionTitleString`][Token::DefinitionTitleString], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^^^ + /// ``` + DefinitionTitle, + /// Definition title marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionTitle`][Token::DefinitionTitle] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ ^ + /// ``` + DefinitionTitleMarker, + /// Definition title data. + /// + /// ## Info + /// + /// * **Context**: + /// [`DefinitionTitle`][Token::DefinitionTitle] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | [a]: b "c" + /// ^ + /// ``` + DefinitionTitleString, + /// Emphasis. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`EmphasisSequence`][Token::EmphasisSequence], + /// [`EmphasisText`][Token::EmphasisText] + /// * **Construct**: + /// [`attention`][crate::construct::attention] + /// + /// ## Example + /// + /// ```markdown + /// > | *a* + /// ^^^ + /// ``` + Emphasis, + /// Emphasis sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`Emphasis`][Token::Emphasis] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`attention`][crate::construct::attention] + /// + /// ## Example + /// + /// ```markdown + /// > | *a* + /// ^ ^ + /// ``` + EmphasisSequence, + /// Emphasis text. + /// + /// ## Info + /// + /// * **Context**: + /// [`Emphasis`][Token::Emphasis] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`attention`][crate::construct::attention] + /// + /// ## Example + /// + /// ```markdown + /// > | *a* + /// ^ + /// ``` + EmphasisText, + /// Whole hard break (escape). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker] + /// * **Construct**: + /// [`hard_break_escape`][crate::construct::hard_break_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a\␊ + /// ^^ + /// > | b + /// ``` + HardBreakEscape, + /// Hard break (escape) marker. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`hard_break_escape`][crate::construct::hard_break_escape] + /// + /// ## Example + /// + /// ```markdown + /// > | a\␊ + /// ^ + /// > | b + /// ``` + HardBreakEscapeMarker, + /// Whole hard break (trailing). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace] + /// * **Construct**: + /// [`hard_break_trailing`][crate::construct::hard_break_trailing] + /// + /// ## Example + /// + /// ```markdown + /// > | a␠␠␊ + /// ^^^ + /// > | b + /// ``` + HardBreakTrailing, + /// Hard break (trailing) spaces. + /// + /// ## Info + /// + /// * **Context**: + /// [`HardBreakTrailing`][Token::HardBreakTrailing] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`hard_break_trailing`][crate::construct::hard_break_trailing] + /// + /// ## Example + /// + /// ```markdown + /// > | a␠␠␊ + /// ^^ + /// > | b + /// ``` + HardBreakTrailingSpace, + /// Whole heading (atx). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`HeadingAtxSequence`][Token::HeadingAtxSequence], + /// [`HeadingAtxText`][Token::HeadingAtxText], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^^^^^^^ + /// ``` + HeadingAtx, + /// Heading (atx) sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingAtx`][Token::HeadingAtx] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^ + /// ``` + HeadingAtxSequence, + /// Heading (atx) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingAtx`][Token::HeadingAtx], + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`heading_atx`][crate::construct::heading_atx] + /// + /// ## Example + /// + /// ```markdown + /// > | # alpha + /// ^^^^^ + /// ``` + HeadingAtxText, + /// Whole heading (setext). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`HeadingSetextText`][Token::HeadingSetextText], + /// [`HeadingSetextUnderline`][Token::HeadingSetextUnderline], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`heading_setext`][crate::construct::heading_setext] + /// + /// ## Example + /// + /// ```markdown + /// > | alpha + /// ^^^^^ + /// > | ===== + /// ^^^^^ + /// ``` + HeadingSetext, + /// Heading (setext) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingSetext`][Token::HeadingSetext] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`heading_setext`][crate::construct::heading_setext] + /// + /// ## Example + /// + /// ```markdown + /// > | alpha + /// ^^^^^ + /// | ===== + /// ``` + HeadingSetextText, + /// Heading (setext) underline. + /// + /// ## Info + /// + /// * **Context**: + /// [`HeadingSetext`][Token::HeadingSetext] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`heading_setext`][crate::construct::heading_setext] + /// + /// ## Example + /// + /// ```markdown + /// | alpha + /// > | ===== + /// ^^^^^ + /// ``` + HeadingSetextUnderline, + /// Whole html (flow). + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`HtmlFlowData`][Token::HtmlFlowData], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`html_flow`][crate::construct::html_flow] + /// + /// ## Example + /// + /// ```markdown + /// > | <div> + /// ^^^^^ + /// ``` + HtmlFlow, + /// HTML (flow) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HtmlFlow`][Token::HtmlFlow], + /// * **Content model**: + /// void + /// * **Construct**: + /// [`html_flow`][crate::construct::html_flow] + /// + /// ## Example + /// + /// ```markdown + /// > | <div> + /// ^^^^^ + /// ``` + HtmlFlowData, + /// Whole html (text). + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`HtmlTextData`][Token::HtmlTextData], + /// [`LineEnding`][Token::LineEnding], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`html_text`][crate::construct::html_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a <b> c + /// ^^^ + /// ``` + HtmlText, + /// HTML (text) data. + /// + /// ## Info + /// + /// * **Context**: + /// [`HtmlText`][Token::HtmlText] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`html_text`][crate::construct::html_text] + /// + /// ## Example + /// + /// ```markdown + /// > | a <b> c + /// ^^^ + /// ``` + HtmlTextData, + /// Image. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`Label`][Token::Label], + /// [`Resource`][Token::Resource], + /// [`Reference`][Token::Reference] + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a ![b] c + /// ^^^^ + /// > | a ![b][c] d + /// ^^^^^^^ + /// > | a  d + /// ^^^^^^^ + /// ``` + Image, + /// Label. + /// + /// ## Info + /// + /// * **Context**: + /// [`Image`][Token::Image], + /// [`Link`][Token::Link] + /// * **Content model**: + /// [`LabelImage`][Token::LabelImage], + /// [`LabelLink`][Token::LabelLink], + /// [`LabelEnd`][Token::LabelEnd], + /// [`LabelText`][Token::LabelText] + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a [b] c + /// ^^^ + /// > | a ![b][c] d + /// ^^^^ + /// > | a [b](c) d + /// ^^^ + /// ``` + Label, + /// Label end. + /// + /// ## Info + /// + /// * **Context**: + /// [`Label`][Token::Label] + /// * **Content model**: + /// [`LabelMarker`][Token::LabelMarker] + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a  d + /// ^ + /// > | a [b](c) d + /// ^ + /// ``` + LabelEnd, + /// Label start (image). + /// + /// ## Info + /// + /// * **Context**: + /// [`Label`][Token::Label] + /// * **Content model**: + /// [`LabelImageMarker`][Token::LabelImageMarker], + /// [`LabelMarker`][Token::LabelMarker] + /// * **Construct**: + /// [`label_start_image`][crate::construct::label_start_image] + /// + /// ## Example + /// + /// ```markdown + /// > | a  d + /// ^^ + /// ``` + LabelImage, + /// Label start (image) marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`LabelImage`][Token::LabelImage] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`label_start_image`][crate::construct::label_start_image] + /// + /// ## Example + /// + /// ```markdown + /// > | a  d + /// ^ + /// ``` + LabelImageMarker, + /// Label start (link). + /// + /// ## Info + /// + /// * **Context**: + /// [`Label`][Token::Label] + /// * **Content model**: + /// [`LabelMarker`][Token::LabelMarker] + /// * **Construct**: + /// [`label_start_link`][crate::construct::label_start_link] + /// + /// ## Example + /// + /// ```markdown + /// > | a [b](c) d + /// ^ + /// ``` + LabelLink, + /// Label marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`LabelImage`][Token::LabelImage], + /// [`LabelLink`][Token::LabelLink], + /// [`LabelEnd`][Token::LabelEnd] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`label_start_image`][crate::construct::label_start_image], + /// [`label_start_link`][crate::construct::label_start_link], + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a  d + /// ^ ^ + /// > | a [b](c) d + /// ^ ^ + /// ``` + LabelMarker, + /// Label text. + /// + /// ## Info + /// + /// * **Context**: + /// [`Label`][Token::Label] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a [b] c + /// ^ + /// > | a ![b][c] d + /// ^ + /// > | a [b](c) d + /// ^ + /// ``` + LabelText, + /// Line ending. + /// + /// ## Info + /// + /// * **Context**: + /// basically everywhere + /// * **Content model**: + /// void + /// * **Construct**: + /// n/a + /// + /// ## Example + /// + /// ```markdown + /// > | a␊ + /// ^ + /// | b + /// ``` + LineEnding, + /// Link. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`Label`][Token::Label], + /// [`Resource`][Token::Resource], + /// [`Reference`][Token::Reference] + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a [b] c + /// ^^^ + /// > | a [b][c] d + /// ^^^^^^ + /// > | a [b](c) d + /// ^^^^^^ + /// ``` + Link, + /// Whole paragraph. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`paragraph`][crate::construct::paragraph] + /// + /// ## Example + /// + /// ```markdown + /// > | a b + /// ^^^ + /// > | c. + /// ^^ + /// ``` + Paragraph, + /// Reference. + /// + /// ## Info + /// + /// * **Context**: + /// [`Image`][Token::Image], + /// [`Link`][Token::Link] + /// * **Content model**: + /// [`ReferenceMarker`][Token::ReferenceMarker], + /// [`ReferenceString`][Token::ReferenceString] + /// * **Construct**: + /// [`label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | a ![b][c] d + /// ^^^ + /// ``` + Reference, + /// Reference marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`Reference`][Token::Reference] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | a ![b][c] d + /// ^ ^ + /// ``` + ReferenceMarker, + /// Reference string. + /// + /// ## Info + /// + /// * **Context**: + /// [`Reference`][Token::Reference] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`label`][crate::construct::partial_label] + /// + /// ## Example + /// + /// ```markdown + /// > | a ![b][c] d + /// ^ + /// ``` + ReferenceString, + /// Resource. + /// + /// ## Info + /// + /// * **Context**: + /// [`Image`][Token::Image], + /// [`Link`][Token::Link] + /// * **Content model**: + /// [`ResourceMarker`][Token::ResourceMarker], + /// [`ResourceDestination`][Token::ResourceDestination], + /// [`ResourceTitle`][Token::ResourceTitle], + /// [`SpaceOrTab`][Token::SpaceOrTab], + /// [`LineEnding`][Token::LineEnding] + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^^^^^^^ + /// > | a [b](c) d + /// ^^^ + /// ``` + Resource, + /// Resource destination. + /// + /// ## Info + /// + /// * **Context**: + /// [`Resource`][Token::Resource] + /// * **Content model**: + /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral], + /// [`ResourceDestinationRaw`][Token::ResourceDestinationRaw] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ + /// ``` + ResourceDestination, + /// Resource destination literal. + /// + /// ## Info + /// + /// * **Context**: + /// [`ResourceDestination`][Token::ResourceDestination] + /// * **Content model**: + /// [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker], + /// [`ResourceDestinationString`][Token::ResourceDestinationString] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^^^ + /// ``` + ResourceDestinationLiteral, + /// Resource destination literal marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ ^ + /// ``` + ResourceDestinationLiteralMarker, + /// Resource destination raw. + /// + /// ## Info + /// + /// * **Context**: + /// [`ResourceDestination`][Token::ResourceDestination] + /// * **Content model**: + /// [`ResourceDestinationString`][Token::ResourceDestinationString] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ + /// ``` + ResourceDestinationRaw, + /// Resource destination raw. + /// + /// ## Info + /// + /// * **Context**: + /// [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral], + /// [`ResourceDestinationRaw`][Token::ResourceDestinationRaw] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`destination`][crate::construct::partial_destination] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ + /// > | a  e + /// ^ + /// ``` + ResourceDestinationString, + /// Resource marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`Resource`][Token::Resource] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`label_end`][crate::construct::label_end] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ ^ + /// ``` + ResourceMarker, + /// Resource title. + /// + /// ## Info + /// + /// * **Context**: + /// [`Resource`][Token::Resource] + /// * **Content model**: + /// [`ResourceTitleMarker`][Token::ResourceTitleMarker], + /// [`ResourceTitleString`][Token::ResourceTitleString] + /// * **Construct**: + /// [`title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^^^ + /// ``` + ResourceTitle, + /// Resource title marker. + /// + /// ## Info + /// + /// * **Context**: + /// [`ResourceTitle`][Token::ResourceTitle] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ ^ + /// ``` + ResourceTitleMarker, + /// Resource title string. + /// + /// ## Info + /// + /// * **Context**: + /// [`ResourceTitle`][Token::ResourceTitle] + /// * **Content model**: + /// [string content][crate::content::string] + /// * **Construct**: + /// [`title`][crate::construct::partial_title] + /// + /// ## Example + /// + /// ```markdown + /// > | a  e + /// ^ + /// ``` + ResourceTitleString, + /// SpaceOrTab. + /// + /// ## Info + /// + /// * **Context**: + /// basically everywhere + /// * **Content model**: + /// void + /// * **Construct**: + /// n/a + /// + /// ## Example + /// + /// ```markdown + /// > | ␠* * *␠ + /// ^ ^ ^ ^ + /// ``` + SpaceOrTab, + /// Strong. + /// + /// ## Info + /// + /// * **Context**: + /// [text content][crate::content::text] + /// * **Content model**: + /// [`StrongSequence`][Token::StrongSequence], + /// [`StrongText`][Token::StrongText] + /// * **Construct**: + /// [`attention`][crate::construct::attention] + /// + /// ## Example + /// + /// ```markdown + /// > | **a** + /// ^^^^^ + /// ``` + Strong, + /// Strong sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`Strong`][Token::Strong] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`attention`][crate::construct::attention] + /// + /// ## Example + /// + /// ```markdown + /// > | **a** + /// ^^ ^^ + /// ``` + StrongSequence, + /// Strong text. + /// + /// ## Info + /// + /// * **Context**: + /// [`Strong`][Token::Strong] + /// * **Content model**: + /// [text content][crate::content::text] + /// * **Construct**: + /// [`attention`][crate::construct::attention] + /// + /// ## Example + /// + /// ```markdown + /// > | **a** + /// ^ + /// ``` + StrongText, + /// Whole thematic break. + /// + /// ## Info + /// + /// * **Context**: + /// [flow content][crate::content::flow] + /// * **Content model**: + /// [`ThematicBreakSequence`][Token::ThematicBreakSequence], + /// [`SpaceOrTab`][Token::SpaceOrTab] + /// * **Construct**: + /// [`thematic_break`][crate::construct::thematic_break] + /// + /// ## Example + /// + /// ```markdown + /// > | * * * + /// ^^^^^ + /// ``` + ThematicBreak, + /// Thematic break sequence. + /// + /// ## Info + /// + /// * **Context**: + /// [`ThematicBreak`][Token::ThematicBreak] + /// * **Content model**: + /// void + /// * **Construct**: + /// [`thematic_break`][crate::construct::thematic_break] + /// + /// ## Example + /// + /// ```markdown + /// > | * * * + /// ^ ^ ^ + /// ``` + ThematicBreakSequence, +} diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 6198776..64b66cc 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -12,1777 +12,9 @@ //! [`check`]: Tokenizer::check use crate::parser::ParseState; +use crate::token::Token; use std::collections::HashMap; -/// Semantic label of a span. -#[derive(Debug, Clone, PartialEq, Hash, Eq)] -pub enum TokenType { - /// Attention sequence. - /// - /// > 👉 **Note**: this is used while parsing but compiled away. - AttentionSequence, - /// Whole autolink. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`AutolinkEmail`][TokenType::AutolinkEmail], - /// [`AutolinkMarker`][TokenType::AutolinkMarker], - /// [`AutolinkProtocol`][TokenType::AutolinkProtocol] - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <https://example.com> and <admin@example.com> - /// ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ - /// ``` - Autolink, - /// Email autolink w/o markers. - /// - /// ## Info - /// - /// * **Context**: - /// [`Autolink`][TokenType::Autolink] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <admin@example.com> - /// ^^^^^^^^^^^^^^^^^ - /// ``` - AutolinkEmail, - /// Marker of an autolink. - /// - /// ## Info - /// - /// * **Context**: - /// [`Autolink`][TokenType::Autolink] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <https://example.com> - /// ^ ^ - /// ``` - AutolinkMarker, - /// Protocol autolink w/o markers. - /// - /// ## Info - /// - /// * **Context**: - /// [`Autolink`][TokenType::Autolink] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`autolink`][crate::construct::autolink] - /// - /// ## Example - /// - /// ```markdown - /// > | <https://example.com> - /// ^^^^^^^^^^^^^^^^^^^ - /// ``` - AutolinkProtocol, - /// Line ending preceded only by whitespace or nothing at all. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`blank_line`][crate::construct::blank_line] - /// - /// ## Example - /// - /// ```markdown - /// > | ␠␠␊ - /// ^ - /// ``` - BlankLineEnding, - /// Whole block quote. - /// - /// ## Info - /// - /// * **Context**: - /// [document content][crate::content::document] - /// * **Content model**: - /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix], - /// [flow content][crate::content::flow] - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^^^ - /// > | b - /// ^ - /// ``` - BlockQuote, - /// Block quote marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^ - /// | b - /// ``` - BlockQuoteMarker, - /// Block quote prefix. - /// - /// ## Info - /// - /// * **Context**: - /// [`BlockQuote`][TokenType::BlockQuote] - /// * **Content model**: - /// [`BlockQuoteMarker`][TokenType::BlockQuoteMarker], - /// [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace] - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^^ - /// | b - /// ``` - BlockQuotePrefix, - /// Block quote white space. - /// - /// ## Info - /// - /// * **Context**: - /// [`BlockQuotePrefix`][TokenType::BlockQuotePrefix] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`block_quote`][crate::construct::block_quote] - /// - /// ## Example - /// - /// ```markdown - /// > | > a - /// ^ - /// | b - /// ``` - BlockQuoteWhitespace, - /// Whole character escape. - /// - /// ## Info - /// - /// * **Context**: - /// [string content][crate::content::string] or - /// [text content][crate::content::text] - /// * **Content model**: - /// [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker], - /// [`CharacterEscapeValue`][TokenType::CharacterEscapeValue] - /// * **Construct**: - /// [`character_escape`][crate::construct::character_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a \- b - /// ^^ - /// ``` - CharacterEscape, - /// Character escape marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterEscape`][TokenType::CharacterEscape] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_escape`][crate::construct::character_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a \- b - /// ^ - /// ``` - CharacterEscapeMarker, - /// Character escape value. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterEscape`][TokenType::CharacterEscape] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_escape`][crate::construct::character_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a \- b - /// ^ - /// ``` - CharacterEscapeValue, - /// Whole character reference. - /// - /// ## Info - /// - /// * **Context**: - /// [string content][crate::content::string] or - /// [text content][crate::content::text] - /// * **Content model**: - /// [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker], - /// [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal], - /// [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric], - /// [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi], - /// [`CharacterReferenceValue`][TokenType::CharacterReferenceValue] - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^^^^^ ^^^^^^^ ^^^^^^^^^ - /// ``` - CharacterReference, - /// Character reference opening marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ ^ ^ - /// ``` - CharacterReferenceMarker, - /// Character reference hexadecimal numeric marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ - /// ``` - CharacterReferenceMarkerHexadecimal, - /// Character reference numeric marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ ^ - /// ``` - CharacterReferenceMarkerNumeric, - /// Character reference closing marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^ ^ ^ - /// ``` - CharacterReferenceMarkerSemi, - /// Character reference value. - /// - /// ## Info - /// - /// * **Context**: - /// [`CharacterReference`][TokenType::CharacterReference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`character_reference`][crate::construct::character_reference] - /// - /// ## Example - /// - /// ```markdown - /// > | a & b ≠ c 𝌆 d - /// ^^^ ^^^^ ^^^^^ - /// ``` - CharacterReferenceValue, - /// Whole code (fenced). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`CodeFencedFence`][TokenType::CodeFencedFence], - /// [`CodeFlowChunk`][TokenType::CodeFlowChunk], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^^^^ - /// > | console.log(1) - /// ^^^^^^^^^^^^^^ - /// > | ``` - /// ^^^ - /// ```` - CodeFenced, - /// A code (fenced) fence. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFenced`][TokenType::CodeFenced] - /// * **Content model**: - /// [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo], - /// [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta], - /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^^^^ - /// | console.log(1) - /// > | ``` - /// ^^^ - /// ```` - CodeFencedFence, - /// A code (fenced) fence info word. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFencedFence`][TokenType::CodeFencedFence] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^ - /// | console.log(1) - /// | ``` - /// ```` - CodeFencedFenceInfo, - /// A code (fenced) fence meta string. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFencedFence`][TokenType::CodeFencedFence] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js highlight="1" - /// ^^^^^^^^^^^^^ - /// | console.log(1) - /// | ``` - /// ```` - CodeFencedFenceMeta, - /// A code (fenced) fence sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ````markdown - /// > | ```js - /// ^^^ - /// | console.log(1) - /// > | ``` - /// ^^^ - /// ```` - CodeFencedFenceSequence, - /// A code (fenced, indented) chunk. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeFenced`][TokenType::CodeFenced], - /// [`CodeIndented`][TokenType::CodeIndented] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced], - /// [`code_indented`][crate::construct::code_indented] - /// - /// ## Example - /// - /// ````markdown - /// | ```js - /// > | console.log(1) - /// ^^^^^^^^^^^^^^ - /// | ``` - /// ```` - /// - /// ```markdown - /// > | ␠␠␠␠console.log(1) - /// ^^^^^^^^^^^^^^ - /// ``` - CodeFlowChunk, - /// Whole code (indented). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`CodeFlowChunk`][TokenType::CodeFlowChunk], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`code_fenced`][crate::construct::code_fenced] - /// - /// ## Example - /// - /// ```markdown - /// ␠␠␠␠console.log(1) - /// ^^^^^^^^^^^^^^^^^^ - /// ``` - CodeIndented, - /// Whole code (text). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`CodeTextData`][TokenType::CodeTextData], - /// [`CodeTextSequence`][TokenType::CodeTextSequence], - /// [`CodeTextLineEnding`][TokenType::CodeTextLineEnding] - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b` c - /// ^^^ - /// ``` - CodeText, - /// Code (text) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeText`][TokenType::CodeText], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b` c - /// ^ - /// ``` - CodeTextData, - /// Line ending in code (text). - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeText`][TokenType::CodeText], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b␊ - /// ^ - /// | c` d - /// ``` - CodeTextLineEnding, - /// Code (text) sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`CodeText`][TokenType::CodeText], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`code_text`][crate::construct::code_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a `b` c - /// ^ ^ - /// ``` - CodeTextSequence, - /// Data. - /// - /// ## Info - /// - /// * **Context**: - /// [string content][crate::content::string], - /// [text content][crate::content::text] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`data`][crate::construct::partial_data] - /// - /// ## Example - /// - /// ```markdown - /// > | aa *bb* cc - /// ^^^ ^^ ^^^ - /// ``` - Data, - /// Whole definition. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`DefinitionMarker`][TokenType::DefinitionMarker], - /// [`DefinitionLabel`][TokenType::DefinitionLabel], - /// [`DefinitionDestination`][TokenType::DefinitionDestination], - /// [`DefinitionTitle`][TokenType::DefinitionTitle], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`definition`][crate::construct::definition] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^^^^^^^^^^ - /// ``` - Definition, - /// Whole definition destination. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral], - /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// > | [a]: <b> "c" - /// ^^^ - /// ``` - DefinitionDestination, - /// Definition destination literal. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestination`][TokenType::DefinitionDestination] - /// * **Content model**: - /// [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker], - /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: <b> "c" - /// ^^^ - /// ``` - DefinitionDestinationLiteral, - /// Definition destination literal marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: <b> "c" - /// ^ ^ - /// ``` - DefinitionDestinationLiteralMarker, - /// Definition destination raw. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestination`][TokenType::DefinitionDestination] - /// * **Content model**: - /// [`DefinitionDestinationString`][TokenType::DefinitionDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionDestinationRaw, - /// Definition destination data. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral], - /// [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// > | [a]: <b> "c" - /// ^ - /// ``` - DefinitionDestinationString, - /// Whole definition label. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker], - /// [`DefinitionLabelString`][TokenType::DefinitionLabelString], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^^^ - /// ``` - DefinitionLabel, - /// Definition label marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionLabel`][TokenType::DefinitionLabel] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ ^ - /// ``` - DefinitionLabelMarker, - /// Definition label data. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionLabel`][TokenType::DefinitionLabel] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionLabelString, - /// Definition marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`definition`][crate::construct::definition] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionMarker, - /// Whole definition title. - /// - /// ## Info - /// - /// * **Context**: - /// [`Definition`][TokenType::Definition] - /// * **Content model**: - /// [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker], - /// [`DefinitionTitleString`][TokenType::DefinitionTitleString], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^^^ - /// ``` - DefinitionTitle, - /// Definition title marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionTitle`][TokenType::DefinitionTitle] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ ^ - /// ``` - DefinitionTitleMarker, - /// Definition title data. - /// - /// ## Info - /// - /// * **Context**: - /// [`DefinitionTitle`][TokenType::DefinitionTitle] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | [a]: b "c" - /// ^ - /// ``` - DefinitionTitleString, - /// Emphasis. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`EmphasisSequence`][TokenType::EmphasisSequence], - /// [`EmphasisText`][TokenType::EmphasisText] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | *a* - /// ^^^ - /// ``` - Emphasis, - /// Emphasis sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`Emphasis`][TokenType::Emphasis] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | *a* - /// ^ ^ - /// ``` - EmphasisSequence, - /// Emphasis text. - /// - /// ## Info - /// - /// * **Context**: - /// [`Emphasis`][TokenType::Emphasis] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | *a* - /// ^ - /// ``` - EmphasisText, - /// Whole hard break (escape). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker] - /// * **Construct**: - /// [`hard_break_escape`][crate::construct::hard_break_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a\␊ - /// ^^ - /// > | b - /// ``` - HardBreakEscape, - /// Hard break (escape) marker. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`hard_break_escape`][crate::construct::hard_break_escape] - /// - /// ## Example - /// - /// ```markdown - /// > | a\␊ - /// ^ - /// > | b - /// ``` - HardBreakEscapeMarker, - /// Whole hard break (trailing). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace] - /// * **Construct**: - /// [`hard_break_trailing`][crate::construct::hard_break_trailing] - /// - /// ## Example - /// - /// ```markdown - /// > | a␠␠␊ - /// ^^^ - /// > | b - /// ``` - HardBreakTrailing, - /// Hard break (trailing) spaces. - /// - /// ## Info - /// - /// * **Context**: - /// [`HardBreakTrailing`][TokenType::HardBreakTrailing] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`hard_break_trailing`][crate::construct::hard_break_trailing] - /// - /// ## Example - /// - /// ```markdown - /// > | a␠␠␊ - /// ^^ - /// > | b - /// ``` - HardBreakTrailingSpace, - /// Whole heading (atx). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`HeadingAtxSequence`][TokenType::HeadingAtxSequence], - /// [`HeadingAtxText`][TokenType::HeadingAtxText], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`heading_atx`][crate::construct::heading_atx] - /// - /// ## Example - /// - /// ```markdown - /// > | # alpha - /// ^^^^^^^ - /// ``` - HeadingAtx, - /// Heading (atx) sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingAtx`][TokenType::HeadingAtx] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`heading_atx`][crate::construct::heading_atx] - /// - /// ## Example - /// - /// ```markdown - /// > | # alpha - /// ^ - /// ``` - HeadingAtxSequence, - /// Heading (atx) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingAtx`][TokenType::HeadingAtx], - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`heading_atx`][crate::construct::heading_atx] - /// - /// ## Example - /// - /// ```markdown - /// > | # alpha - /// ^^^^^ - /// ``` - HeadingAtxText, - /// Whole heading (setext). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`HeadingSetextText`][TokenType::HeadingSetextText], - /// [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`heading_setext`][crate::construct::heading_setext] - /// - /// ## Example - /// - /// ```markdown - /// > | alpha - /// ^^^^^ - /// > | ===== - /// ^^^^^ - /// ``` - HeadingSetext, - /// Heading (setext) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingSetext`][TokenType::HeadingSetext] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`heading_setext`][crate::construct::heading_setext] - /// - /// ## Example - /// - /// ```markdown - /// > | alpha - /// ^^^^^ - /// | ===== - /// ``` - HeadingSetextText, - /// Heading (setext) underline. - /// - /// ## Info - /// - /// * **Context**: - /// [`HeadingSetext`][TokenType::HeadingSetext] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`heading_setext`][crate::construct::heading_setext] - /// - /// ## Example - /// - /// ```markdown - /// | alpha - /// > | ===== - /// ^^^^^ - /// ``` - HeadingSetextUnderline, - /// Whole html (flow). - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`HtmlFlowData`][TokenType::HtmlFlowData], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`html_flow`][crate::construct::html_flow] - /// - /// ## Example - /// - /// ```markdown - /// > | <div> - /// ^^^^^ - /// ``` - HtmlFlow, - /// HTML (flow) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HtmlFlow`][TokenType::HtmlFlow], - /// * **Content model**: - /// void - /// * **Construct**: - /// [`html_flow`][crate::construct::html_flow] - /// - /// ## Example - /// - /// ```markdown - /// > | <div> - /// ^^^^^ - /// ``` - HtmlFlowData, - /// Whole html (text). - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`HtmlTextData`][TokenType::HtmlTextData], - /// [`LineEnding`][TokenType::LineEnding], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`html_text`][crate::construct::html_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a <b> c - /// ^^^ - /// ``` - HtmlText, - /// HTML (text) data. - /// - /// ## Info - /// - /// * **Context**: - /// [`HtmlText`][TokenType::HtmlText] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`html_text`][crate::construct::html_text] - /// - /// ## Example - /// - /// ```markdown - /// > | a <b> c - /// ^^^ - /// ``` - HtmlTextData, - /// Image. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`Label`][TokenType::Label], - /// [`Resource`][TokenType::Resource], - /// [`Reference`][TokenType::Reference] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b] c - /// ^^^^ - /// > | a ![b][c] d - /// ^^^^^^^ - /// > | a  d - /// ^^^^^^^ - /// ``` - Image, - /// Label. - /// - /// ## Info - /// - /// * **Context**: - /// [`Image`][TokenType::Image], - /// [`Link`][TokenType::Link] - /// * **Content model**: - /// [`LabelImage`][TokenType::LabelImage], - /// [`LabelLink`][TokenType::LabelLink], - /// [`LabelEnd`][TokenType::LabelEnd], - /// [`LabelText`][TokenType::LabelText] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b] c - /// ^^^ - /// > | a ![b][c] d - /// ^^^^ - /// > | a [b](c) d - /// ^^^ - /// ``` - Label, - /// Label end. - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [`LabelMarker`][TokenType::LabelMarker] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^ - /// > | a [b](c) d - /// ^ - /// ``` - LabelEnd, - /// Label start (image). - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [`LabelImageMarker`][TokenType::LabelImageMarker], - /// [`LabelMarker`][TokenType::LabelMarker] - /// * **Construct**: - /// [`label_start_image`][crate::construct::label_start_image] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^^ - /// ``` - LabelImage, - /// Label start (image) marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`LabelImage`][TokenType::LabelImage] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label_start_image`][crate::construct::label_start_image] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^ - /// ``` - LabelImageMarker, - /// Label start (link). - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [`LabelMarker`][TokenType::LabelMarker] - /// * **Construct**: - /// [`label_start_link`][crate::construct::label_start_link] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b](c) d - /// ^ - /// ``` - LabelLink, - /// Label marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`LabelImage`][TokenType::LabelImage], - /// [`LabelLink`][TokenType::LabelLink], - /// [`LabelEnd`][TokenType::LabelEnd] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label_start_image`][crate::construct::label_start_image], - /// [`label_start_link`][crate::construct::label_start_link], - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  d - /// ^ ^ - /// > | a [b](c) d - /// ^ ^ - /// ``` - LabelMarker, - /// Label text. - /// - /// ## Info - /// - /// * **Context**: - /// [`Label`][TokenType::Label] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b] c - /// ^ - /// > | a ![b][c] d - /// ^ - /// > | a [b](c) d - /// ^ - /// ``` - LabelText, - /// Line ending. - /// - /// ## Info - /// - /// * **Context**: - /// basically everywhere - /// * **Content model**: - /// void - /// * **Construct**: - /// n/a - /// - /// ## Example - /// - /// ```markdown - /// > | a␊ - /// ^ - /// | b - /// ``` - LineEnding, - /// Link. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`Label`][TokenType::Label], - /// [`Resource`][TokenType::Resource], - /// [`Reference`][TokenType::Reference] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a [b] c - /// ^^^ - /// > | a [b][c] d - /// ^^^^^^ - /// > | a [b](c) d - /// ^^^^^^ - /// ``` - Link, - /// Whole paragraph. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`paragraph`][crate::construct::paragraph] - /// - /// ## Example - /// - /// ```markdown - /// > | a b - /// ^^^ - /// > | c. - /// ^^ - /// ``` - Paragraph, - /// Reference. - /// - /// ## Info - /// - /// * **Context**: - /// [`Image`][TokenType::Image], - /// [`Link`][TokenType::Link] - /// * **Content model**: - /// [`ReferenceMarker`][TokenType::ReferenceMarker], - /// [`ReferenceString`][TokenType::ReferenceString] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b][c] d - /// ^^^ - /// ``` - Reference, - /// Reference marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`Reference`][TokenType::Reference] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b][c] d - /// ^ ^ - /// ``` - ReferenceMarker, - /// Reference string. - /// - /// ## Info - /// - /// * **Context**: - /// [`Reference`][TokenType::Reference] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`label`][crate::construct::partial_label] - /// - /// ## Example - /// - /// ```markdown - /// > | a ![b][c] d - /// ^ - /// ``` - ReferenceString, - /// Resource. - /// - /// ## Info - /// - /// * **Context**: - /// [`Image`][TokenType::Image], - /// [`Link`][TokenType::Link] - /// * **Content model**: - /// [`ResourceMarker`][TokenType::ResourceMarker], - /// [`ResourceDestination`][TokenType::ResourceDestination], - /// [`ResourceTitle`][TokenType::ResourceTitle], - /// [`SpaceOrTab`][TokenType::SpaceOrTab], - /// [`LineEnding`][TokenType::LineEnding] - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^^^^^^^ - /// > | a [b](c) d - /// ^^^ - /// ``` - Resource, - /// Resource destination. - /// - /// ## Info - /// - /// * **Context**: - /// [`Resource`][TokenType::Resource] - /// * **Content model**: - /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral], - /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// ``` - ResourceDestination, - /// Resource destination literal. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestination`][TokenType::ResourceDestination] - /// * **Content model**: - /// [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker], - /// [`ResourceDestinationString`][TokenType::ResourceDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^^^ - /// ``` - ResourceDestinationLiteral, - /// Resource destination literal marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ ^ - /// ``` - ResourceDestinationLiteralMarker, - /// Resource destination raw. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestination`][TokenType::ResourceDestination] - /// * **Content model**: - /// [`ResourceDestinationString`][TokenType::ResourceDestinationString] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// ``` - ResourceDestinationRaw, - /// Resource destination raw. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral], - /// [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`destination`][crate::construct::partial_destination] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// > | a  e - /// ^ - /// ``` - ResourceDestinationString, - /// Resource marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`Resource`][TokenType::Resource] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`label_end`][crate::construct::label_end] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ ^ - /// ``` - ResourceMarker, - /// Resource title. - /// - /// ## Info - /// - /// * **Context**: - /// [`Resource`][TokenType::Resource] - /// * **Content model**: - /// [`ResourceTitleMarker`][TokenType::ResourceTitleMarker], - /// [`ResourceTitleString`][TokenType::ResourceTitleString] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^^^ - /// ``` - ResourceTitle, - /// Resource title marker. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceTitle`][TokenType::ResourceTitle] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ ^ - /// ``` - ResourceTitleMarker, - /// Resource title string. - /// - /// ## Info - /// - /// * **Context**: - /// [`ResourceTitle`][TokenType::ResourceTitle] - /// * **Content model**: - /// [string content][crate::content::string] - /// * **Construct**: - /// [`title`][crate::construct::partial_title] - /// - /// ## Example - /// - /// ```markdown - /// > | a  e - /// ^ - /// ``` - ResourceTitleString, - /// SpaceOrTab. - /// - /// ## Info - /// - /// * **Context**: - /// basically everywhere - /// * **Content model**: - /// void - /// * **Construct**: - /// n/a - /// - /// ## Example - /// - /// ```markdown - /// > | ␠* * *␠ - /// ^ ^ ^ ^ - /// ``` - SpaceOrTab, - /// Strong. - /// - /// ## Info - /// - /// * **Context**: - /// [text content][crate::content::text] - /// * **Content model**: - /// [`StrongSequence`][TokenType::StrongSequence], - /// [`StrongText`][TokenType::StrongText] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | **a** - /// ^^^^^ - /// ``` - Strong, - /// Strong sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`Strong`][TokenType::Strong] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | **a** - /// ^^ ^^ - /// ``` - StrongSequence, - /// Strong text. - /// - /// ## Info - /// - /// * **Context**: - /// [`Strong`][TokenType::Strong] - /// * **Content model**: - /// [text content][crate::content::text] - /// * **Construct**: - /// [`attention`][crate::construct::attention] - /// - /// ## Example - /// - /// ```markdown - /// > | **a** - /// ^ - /// ``` - StrongText, - /// Whole thematic break. - /// - /// ## Info - /// - /// * **Context**: - /// [flow content][crate::content::flow] - /// * **Content model**: - /// [`ThematicBreakSequence`][TokenType::ThematicBreakSequence], - /// [`SpaceOrTab`][TokenType::SpaceOrTab] - /// * **Construct**: - /// [`thematic_break`][crate::construct::thematic_break] - /// - /// ## Example - /// - /// ```markdown - /// > | * * * - /// ^^^^^ - /// ``` - ThematicBreak, - /// Thematic break sequence. - /// - /// ## Info - /// - /// * **Context**: - /// [`ThematicBreak`][TokenType::ThematicBreak] - /// * **Content model**: - /// void - /// * **Construct**: - /// [`thematic_break`][crate::construct::thematic_break] - /// - /// ## Example - /// - /// ```markdown - /// > | * * * - /// ^ ^ ^ - /// ``` - ThematicBreakSequence, -} - /// Embedded content type. #[derive(Debug, Clone, Copy, PartialEq)] pub enum ContentType { @@ -1840,7 +72,7 @@ pub enum EventType { #[derive(Debug, Clone)] pub struct Event { pub event_type: EventType, - pub token_type: TokenType, + pub token_type: Token, pub point: Point, pub index: usize, pub previous: Option<usize>, @@ -1935,7 +167,7 @@ pub struct Tokenizer<'a> { /// Hierarchy of semantic labels. /// /// Tracked to make sure everything’s valid. - pub stack: Vec<TokenType>, + pub stack: Vec<Token>, /// Previous character code. pub previous: Code, /// Current character code. @@ -2082,11 +314,11 @@ impl<'a> Tokenizer<'a> { } /// Mark the start of a semantic label. - pub fn enter(&mut self, token_type: TokenType) { + pub fn enter(&mut self, token_type: Token) { self.enter_with_content(token_type, None); } - pub fn enter_with_content(&mut self, token_type: TokenType, content_type: Option<ContentType>) { + pub fn enter_with_content(&mut self, token_type: Token, content_type: Option<ContentType>) { log::debug!("enter `{:?}` ({:?})", token_type, self.point); self.events.push(Event { event_type: EventType::Enter, @@ -2101,7 +333,7 @@ impl<'a> Tokenizer<'a> { } /// Mark the end of a semantic label. - pub fn exit(&mut self, token_type: TokenType) { + pub fn exit(&mut self, token_type: Token) { let current_token = self.stack.pop().expect("cannot close w/o open tokens"); assert_eq!( diff --git a/src/util/skip.rs b/src/util/skip.rs index a8e4cfe..971beb6 100644 --- a/src/util/skip.rs +++ b/src/util/skip.rs @@ -1,14 +1,15 @@ //! Utilities to deal with lists of events. -use crate::tokenizer::{Event, TokenType}; +use crate::token::Token; +use crate::tokenizer::Event; /// Skip from `index`, optionally past `token_types`. -pub fn opt(events: &[Event], index: usize, token_types: &[TokenType]) -> usize { +pub fn opt(events: &[Event], index: usize, token_types: &[Token]) -> usize { skip_opt_with_direction(events, index, token_types, true) } /// Skip from `index`, optionally past `token_types`, backwards. -pub fn opt_back(events: &[Event], index: usize, token_types: &[TokenType]) -> usize { +pub fn opt_back(events: &[Event], index: usize, token_types: &[Token]) -> usize { skip_opt_with_direction(events, index, token_types, false) } @@ -16,7 +17,7 @@ pub fn opt_back(events: &[Event], index: usize, token_types: &[TokenType]) -> us fn skip_opt_with_direction( events: &[Event], index: usize, - token_types: &[TokenType], + token_types: &[Token], forward: bool, ) -> usize { let mut index = index; |