aboutsummaryrefslogtreecommitdiffstats
path: root/src/construct
diff options
context:
space:
mode:
authorLibravatar Titus Wormer <tituswormer@gmail.com>2022-07-07 18:56:06 +0200
committerLibravatar Titus Wormer <tituswormer@gmail.com>2022-07-07 18:56:06 +0200
commit92b42e06f943338ce8b54b7e22cbb116ff598fa6 (patch)
treeff51df093f52dc33bfac5e1c236b41cfbd21c220 /src/construct
parentfdb1f1694f44cfbc59d303a10371300b48d74627 (diff)
downloadmarkdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.tar.gz
markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.tar.bz2
markdown-rs-92b42e06f943338ce8b54b7e22cbb116ff598fa6.zip
Refactor to move token types to `token`
Diffstat (limited to '')
-rw-r--r--src/construct/attention.rs59
-rw-r--r--src/construct/autolink.rs33
-rw-r--r--src/construct/blank_line.rs2
-rw-r--r--src/construct/block_quote.rs31
-rw-r--r--src/construct/character_escape.rs21
-rw-r--r--src/construct/character_reference.rs43
-rw-r--r--src/construct/code_fenced.rs79
-rw-r--r--src/construct/code_indented.rs23
-rw-r--r--src/construct/code_text.rs40
-rw-r--r--src/construct/definition.rs67
-rw-r--r--src/construct/hard_break_escape.rs15
-rw-r--r--src/construct/hard_break_trailing.rs15
-rw-r--r--src/construct/heading_atx.rs39
-rw-r--r--src/construct/heading_setext.rs33
-rw-r--r--src/construct/html_flow.rs31
-rw-r--r--src/construct/html_text.rs23
-rw-r--r--src/construct/label_end.rs129
-rw-r--r--src/construct/label_start_image.rs21
-rw-r--r--src/construct/label_start_link.rs15
-rw-r--r--src/construct/paragraph.rs27
-rw-r--r--src/construct/partial_data.rs17
-rw-r--r--src/construct/partial_destination.rs23
-rw-r--r--src/construct/partial_label.rs15
-rw-r--r--src/construct/partial_space_or_tab.rs15
-rw-r--r--src/construct/partial_title.rs15
-rw-r--r--src/construct/thematic_break.rs15
26 files changed, 433 insertions, 413 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs
index daf662e..1750692 100644
--- a/src/construct/attention.rs
+++ b/src/construct/attention.rs
@@ -32,14 +32,14 @@
//!
//! ## Tokens
//!
-//! * [`Emphasis`][TokenType::Emphasis]
-//! * [`EmphasisSequence`][TokenType::EmphasisSequence]
-//! * [`EmphasisText`][TokenType::EmphasisText]
-//! * [`Strong`][TokenType::Strong]
-//! * [`StrongSequence`][TokenType::StrongSequence]
-//! * [`StrongText`][TokenType::StrongText]
+//! * [`Emphasis`][Token::Emphasis]
+//! * [`EmphasisSequence`][Token::EmphasisSequence]
+//! * [`EmphasisText`][Token::EmphasisText]
+//! * [`Strong`][Token::Strong]
+//! * [`StrongSequence`][Token::StrongSequence]
+//! * [`StrongText`][Token::StrongText]
//!
-//! > 👉 **Note**: while parsing, [`AttentionSequence`][TokenType::AttentionSequence]
+//! > 👉 **Note**: while parsing, [`AttentionSequence`][Token::AttentionSequence]
//! > is used, which is later compiled away.
//!
//! ## References
@@ -51,7 +51,8 @@
//! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element
//! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element
-use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, Point, State, StateFnResult, Tokenizer};
use crate::unicode::PUNCTUATION;
use crate::util::edit_map::EditMap;
@@ -174,7 +175,7 @@ struct Sequence {
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('*' | '_') => {
- tokenizer.enter(TokenType::AttentionSequence);
+ tokenizer.enter(Token::AttentionSequence);
inside(tokenizer, code, MarkerKind::from_code(code))
}
_ => (State::Nok, None),
@@ -193,7 +194,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, marker: MarkerKind) -> StateFnR
(State::Fn(Box::new(move |t, c| inside(t, c, marker))), None)
}
_ => {
- tokenizer.exit(TokenType::AttentionSequence);
+ tokenizer.exit(Token::AttentionSequence);
tokenizer.register_resolver("attention".to_string(), Box::new(resolve));
(State::Ok, Some(vec![code]))
}
@@ -216,7 +217,7 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
if enter.event_type == EventType::Enter {
balance += 1;
- if enter.token_type == TokenType::AttentionSequence {
+ if enter.token_type == Token::AttentionSequence {
let end = start + 1;
let exit = &tokenizer.events[end];
let marker = MarkerKind::from_code(codes[enter.index]);
@@ -392,9 +393,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::Emphasis
+ Token::Emphasis
} else {
- TokenType::Strong
+ Token::Strong
},
point: seq_open_enter.0.clone(),
index: seq_open_enter.1,
@@ -405,9 +406,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_open_enter.0.clone(),
index: seq_open_enter.1,
@@ -418,9 +419,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_open_exit.0.clone(),
index: seq_open_exit.1,
@@ -431,9 +432,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::EmphasisText
+ Token::EmphasisText
} else {
- TokenType::StrongText
+ Token::StrongText
},
point: seq_open_exit.0.clone(),
index: seq_open_exit.1,
@@ -451,9 +452,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::EmphasisText
+ Token::EmphasisText
} else {
- TokenType::StrongText
+ Token::StrongText
},
point: seq_close_enter.0.clone(),
index: seq_close_enter.1,
@@ -464,9 +465,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Enter,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_close_enter.0.clone(),
index: seq_close_enter.1,
@@ -477,9 +478,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::EmphasisSequence
+ Token::EmphasisSequence
} else {
- TokenType::StrongSequence
+ Token::StrongSequence
},
point: seq_close_exit.0.clone(),
index: seq_close_exit.1,
@@ -490,9 +491,9 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
Event {
event_type: EventType::Exit,
token_type: if take == 1 {
- TokenType::Emphasis
+ Token::Emphasis
} else {
- TokenType::Strong
+ Token::Strong
},
point: seq_close_exit.0.clone(),
index: seq_close_exit.1,
@@ -515,8 +516,8 @@ fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
let mut index = 0;
while index < sequences.len() {
let sequence = &sequences[index];
- tokenizer.events[sequence.event_index].token_type = TokenType::Data;
- tokenizer.events[sequence.event_index + 1].token_type = TokenType::Data;
+ tokenizer.events[sequence.event_index].token_type = Token::Data;
+ tokenizer.events[sequence.event_index + 1].token_type = Token::Data;
index += 1;
}
diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs
index e29bf8b..db4365f 100644
--- a/src/construct/autolink.rs
+++ b/src/construct/autolink.rs
@@ -84,10 +84,10 @@
//!
//! ## Tokens
//!
-//! * [`Autolink`][TokenType::Autolink]
-//! * [`AutolinkEmail`][TokenType::AutolinkEmail]
-//! * [`AutolinkMarker`][TokenType::AutolinkMarker]
-//! * [`AutolinkProtocol`][TokenType::AutolinkProtocol]
+//! * [`Autolink`][Token::Autolink]
+//! * [`AutolinkEmail`][Token::AutolinkEmail]
+//! * [`AutolinkMarker`][Token::AutolinkMarker]
+//! * [`AutolinkProtocol`][Token::AutolinkProtocol]
//!
//! ## References
//!
@@ -102,7 +102,8 @@
//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of an autolink.
///
@@ -113,11 +114,11 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('<') => {
- tokenizer.enter(TokenType::Autolink);
- tokenizer.enter(TokenType::AutolinkMarker);
+ tokenizer.enter(Token::Autolink);
+ tokenizer.enter(Token::AutolinkMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::AutolinkMarker);
- tokenizer.enter(TokenType::AutolinkProtocol);
+ tokenizer.exit(Token::AutolinkMarker);
+ tokenizer.enter(Token::AutolinkProtocol);
(State::Fn(Box::new(open)), None)
}
_ => (State::Nok, None),
@@ -195,7 +196,7 @@ fn scheme_inside_or_email_atext(
fn url_inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.exit(TokenType::AutolinkProtocol);
+ tokenizer.exit(Token::AutolinkProtocol);
end(tokenizer, code)
}
Code::Char(char) if char.is_ascii_control() => (State::Nok, None),
@@ -260,10 +261,10 @@ fn email_label(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes
}
Code::Char('>') => {
let index = tokenizer.events.len();
- tokenizer.exit(TokenType::AutolinkProtocol);
+ tokenizer.exit(Token::AutolinkProtocol);
// Change the token type.
- tokenizer.events[index - 1].token_type = TokenType::AutolinkEmail;
- tokenizer.events[index].token_type = TokenType::AutolinkEmail;
+ tokenizer.events[index - 1].token_type = Token::AutolinkEmail;
+ tokenizer.events[index].token_type = Token::AutolinkEmail;
end(tokenizer, code)
}
_ => email_value(tokenizer, code, size),
@@ -306,10 +307,10 @@ fn email_value(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnRes
fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.enter(TokenType::AutolinkMarker);
+ tokenizer.enter(Token::AutolinkMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::AutolinkMarker);
- tokenizer.exit(TokenType::Autolink);
+ tokenizer.exit(Token::AutolinkMarker);
+ tokenizer.exit(Token::Autolink);
(State::Ok, None)
}
_ => unreachable!("expected `>`"),
diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs
index 3ca3266..69a4709 100644
--- a/src/construct/blank_line.rs
+++ b/src/construct/blank_line.rs
@@ -20,7 +20,7 @@
//!
//! ## Tokens
//!
-//! * [`SpaceOrTab`][crate::tokenizer::TokenType::SpaceOrTab]
+//! * [`SpaceOrTab`][crate::token::Token::SpaceOrTab]
//!
//! ## References
//!
diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs
index a3ffbc7..096ac0a 100644
--- a/src/construct/block_quote.rs
+++ b/src/construct/block_quote.rs
@@ -19,10 +19,10 @@
//!
//! ## Tokens
//!
-//! * [`BlockQuote`][TokenType::BlockQuote]
-//! * [`BlockQuoteMarker`][TokenType::BlockQuoteMarker]
-//! * [`BlockQuotePrefix`][TokenType::BlockQuotePrefix]
-//! * [`BlockQuoteWhitespace`][TokenType::BlockQuoteWhitespace]
+//! * [`BlockQuote`][Token::BlockQuote]
+//! * [`BlockQuoteMarker`][Token::BlockQuoteMarker]
+//! * [`BlockQuotePrefix`][Token::BlockQuotePrefix]
+//! * [`BlockQuoteWhitespace`][Token::BlockQuoteWhitespace]
//!
//! ## References
//!
@@ -35,7 +35,8 @@
use crate::constant::TAB_SIZE;
use crate::construct::partial_space_or_tab::space_or_tab_min_max;
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of block quote.
///
@@ -55,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.enter(TokenType::BlockQuote);
+ tokenizer.enter(Token::BlockQuote);
cont_before(tokenizer, code)
}
_ => cont_before(tokenizer, code),
@@ -82,10 +83,10 @@ pub fn cont(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.enter(TokenType::BlockQuotePrefix);
- tokenizer.enter(TokenType::BlockQuoteMarker);
+ tokenizer.enter(Token::BlockQuotePrefix);
+ tokenizer.enter(Token::BlockQuoteMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::BlockQuoteMarker);
+ tokenizer.exit(Token::BlockQuoteMarker);
(State::Fn(Box::new(cont_after)), None)
}
_ => (State::Nok, None),
@@ -101,20 +102,20 @@ fn cont_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn cont_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::VirtualSpace | Code::Char('\t' | ' ') => {
- tokenizer.enter(TokenType::BlockQuoteWhitespace);
+ tokenizer.enter(Token::BlockQuoteWhitespace);
tokenizer.consume(code);
- tokenizer.exit(TokenType::BlockQuoteWhitespace);
- tokenizer.exit(TokenType::BlockQuotePrefix);
+ tokenizer.exit(Token::BlockQuoteWhitespace);
+ tokenizer.exit(Token::BlockQuotePrefix);
(State::Ok, None)
}
_ => {
- tokenizer.exit(TokenType::BlockQuotePrefix);
+ tokenizer.exit(Token::BlockQuotePrefix);
(State::Ok, Some(vec![code]))
}
}
}
/// End of a block quote.
-pub fn end() -> Vec<TokenType> {
- vec![TokenType::BlockQuote]
+pub fn end() -> Vec<Token> {
+ vec![Token::BlockQuote]
}
diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs
index 0ccc574..55d321a 100644
--- a/src/construct/character_escape.rs
+++ b/src/construct/character_escape.rs
@@ -19,9 +19,9 @@
//!
//! ## Tokens
//!
-//! * [`CharacterEscape`][TokenType::CharacterEscape]
-//! * [`CharacterEscapeMarker`][TokenType::CharacterEscapeMarker]
-//! * [`CharacterEscapeValue`][TokenType::CharacterEscapeValue]
+//! * [`CharacterEscape`][Token::CharacterEscape]
+//! * [`CharacterEscapeMarker`][Token::CharacterEscapeMarker]
+//! * [`CharacterEscapeValue`][Token::CharacterEscapeValue]
//!
//! ## References
//!
@@ -33,7 +33,8 @@
//! [character_reference]: crate::construct::character_reference
//! [hard_break_escape]: crate::construct::hard_break_escape
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of a character escape.
///
@@ -45,10 +46,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('\\') => {
- tokenizer.enter(TokenType::CharacterEscape);
- tokenizer.enter(TokenType::CharacterEscapeMarker);
+ tokenizer.enter(Token::CharacterEscape);
+ tokenizer.enter(Token::CharacterEscapeMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterEscapeMarker);
+ tokenizer.exit(Token::CharacterEscapeMarker);
(State::Fn(Box::new(inside)), None)
}
_ => (State::Nok, None),
@@ -65,10 +66,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(char) if char.is_ascii_punctuation() => {
- tokenizer.enter(TokenType::CharacterEscapeValue);
+ tokenizer.enter(Token::CharacterEscapeValue);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterEscapeValue);
- tokenizer.exit(TokenType::CharacterEscape);
+ tokenizer.exit(Token::CharacterEscapeValue);
+ tokenizer.exit(Token::CharacterEscape);
(State::Ok, None)
}
_ => (State::Nok, None),
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index 65e49ca..04e89d2 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -42,12 +42,12 @@
//!
//! ## Tokens
//!
-//! * [`CharacterReference`][TokenType::CharacterReference]
-//! * [`CharacterReferenceMarker`][TokenType::CharacterReferenceMarker]
-//! * [`CharacterReferenceMarkerHexadecimal`][TokenType::CharacterReferenceMarkerHexadecimal]
-//! * [`CharacterReferenceMarkerNumeric`][TokenType::CharacterReferenceMarkerNumeric]
-//! * [`CharacterReferenceMarkerSemi`][TokenType::CharacterReferenceMarkerSemi]
-//! * [`CharacterReferenceValue`][TokenType::CharacterReferenceValue]
+//! * [`CharacterReference`][Token::CharacterReference]
+//! * [`CharacterReferenceMarker`][Token::CharacterReferenceMarker]
+//! * [`CharacterReferenceMarkerHexadecimal`][Token::CharacterReferenceMarkerHexadecimal]
+//! * [`CharacterReferenceMarkerNumeric`][Token::CharacterReferenceMarkerNumeric]
+//! * [`CharacterReferenceMarkerSemi`][Token::CharacterReferenceMarkerSemi]
+//! * [`CharacterReferenceValue`][Token::CharacterReferenceValue]
//!
//! ## References
//!
@@ -65,7 +65,8 @@ use crate::constant::{
CHARACTER_REFERENCE_DECIMAL_SIZE_MAX, CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX,
CHARACTER_REFERENCE_NAMED_SIZE_MAX, CHARACTER_REFERENCE_NAMES,
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Kind of a character reference.
#[derive(Debug, Clone, PartialEq)]
@@ -119,10 +120,10 @@ struct Info {
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('&') => {
- tokenizer.enter(TokenType::CharacterReference);
- tokenizer.enter(TokenType::CharacterReferenceMarker);
+ tokenizer.enter(Token::CharacterReference);
+ tokenizer.enter(Token::CharacterReferenceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarker);
+ tokenizer.exit(Token::CharacterReferenceMarker);
(State::Fn(Box::new(open)), None)
}
_ => (State::Nok, None),
@@ -143,12 +144,12 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
kind: Kind::Named,
};
if let Code::Char('#') = code {
- tokenizer.enter(TokenType::CharacterReferenceMarkerNumeric);
+ tokenizer.enter(Token::CharacterReferenceMarkerNumeric);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerNumeric);
+ tokenizer.exit(Token::CharacterReferenceMarkerNumeric);
(State::Fn(Box::new(|t, c| numeric(t, c, info))), None)
} else {
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ tokenizer.enter(Token::CharacterReferenceValue);
value(tokenizer, code, info)
}
}
@@ -162,14 +163,14 @@ fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// ```
fn numeric(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
if let Code::Char('x' | 'X') = code {
- tokenizer.enter(TokenType::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerHexadecimal);
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(Token::CharacterReferenceValue);
info.kind = Kind::Hexadecimal;
(State::Fn(Box::new(|t, c| value(t, c, info))), None)
} else {
- tokenizer.enter(TokenType::CharacterReferenceValue);
+ tokenizer.enter(Token::CharacterReferenceValue);
info.kind = Kind::Decimal;
value(tokenizer, code, info)
}
@@ -194,11 +195,11 @@ fn value(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult
{
(State::Nok, None)
} else {
- tokenizer.exit(TokenType::CharacterReferenceValue);
- tokenizer.enter(TokenType::CharacterReferenceMarkerSemi);
+ tokenizer.exit(Token::CharacterReferenceValue);
+ tokenizer.enter(Token::CharacterReferenceMarkerSemi);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CharacterReferenceMarkerSemi);
- tokenizer.exit(TokenType::CharacterReference);
+ tokenizer.exit(Token::CharacterReferenceMarkerSemi);
+ tokenizer.exit(Token::CharacterReference);
(State::Ok, None)
}
}
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index 05266ba..617979f 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -77,14 +77,14 @@
//!
//! ## Tokens
//!
-//! * [`CodeFenced`][TokenType::CodeFenced]
-//! * [`CodeFencedFence`][TokenType::CodeFencedFence]
-//! * [`CodeFencedFenceInfo`][TokenType::CodeFencedFenceInfo]
-//! * [`CodeFencedFenceMeta`][TokenType::CodeFencedFenceMeta]
-//! * [`CodeFencedFenceSequence`][TokenType::CodeFencedFenceSequence]
-//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`CodeFenced`][Token::CodeFenced]
+//! * [`CodeFencedFence`][Token::CodeFencedFence]
+//! * [`CodeFencedFenceInfo`][Token::CodeFencedFenceInfo]
+//! * [`CodeFencedFenceMeta`][Token::CodeFencedFenceMeta]
+//! * [`CodeFencedFenceSequence`][Token::CodeFencedFenceSequence]
+//! * [`CodeFlowChunk`][Token::CodeFlowChunk]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -103,7 +103,8 @@
use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE};
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
use crate::util::span::from_exit_event;
/// Kind of fences.
@@ -184,8 +185,8 @@ struct Info {
/// ~~~
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::CodeFenced);
- tokenizer.enter(TokenType::CodeFencedFence);
+ tokenizer.enter(Token::CodeFenced);
+ tokenizer.enter(Token::CodeFencedFence);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before_sequence_open)(tokenizer, code)
}
@@ -202,7 +203,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
let mut prefix = 0;
if let Some(event) = tail {
- if event.token_type == TokenType::SpaceOrTab {
+ if event.token_type == Token::SpaceOrTab {
let span = from_exit_event(&tokenizer.events, tokenizer.events.len() - 1);
prefix = span.end_index - span.start_index;
}
@@ -210,7 +211,7 @@ fn before_sequence_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
match code {
Code::Char('`' | '~') => {
- tokenizer.enter(TokenType::CodeFencedFenceSequence);
+ tokenizer.enter(Token::CodeFencedFenceSequence);
sequence_open(
tokenizer,
code,
@@ -245,7 +246,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State
)
}
_ if info.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => {
- tokenizer.exit(TokenType::CodeFencedFenceSequence);
+ tokenizer.exit(Token::CodeFencedFenceSequence);
tokenizer.attempt_opt(space_or_tab(), |t, c| info_before(t, c, info))(tokenizer, code)
}
_ => (State::Nok, None),
@@ -262,12 +263,12 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> State
fn info_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
_ => {
- tokenizer.enter(TokenType::CodeFencedFenceInfo);
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter(Token::CodeFencedFenceInfo);
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
info_inside(tokenizer, code, info, vec![])
}
}
@@ -288,14 +289,14 @@ fn info_inside(
) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::CodeFencedFenceInfo);
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::CodeFencedFenceInfo);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
Code::VirtualSpace | Code::Char('\t' | ' ') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::CodeFencedFenceInfo);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::CodeFencedFenceInfo);
tokenizer.attempt_opt(space_or_tab(), |t, c| meta_before(t, c, info))(tokenizer, code)
}
Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None),
@@ -320,12 +321,12 @@ fn info_inside(
fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
_ => {
- tokenizer.enter(TokenType::CodeFencedFenceMeta);
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter(Token::CodeFencedFenceMeta);
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
meta(tokenizer, code, info)
}
}
@@ -341,9 +342,9 @@ fn meta_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu
fn meta(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::CodeFencedFenceMeta);
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::CodeFencedFenceMeta);
+ tokenizer.exit(Token::CodeFencedFence);
at_break(tokenizer, code, info)
}
Code::Char('`') if info.kind == Kind::GraveAccent => (State::Nok, None),
@@ -390,9 +391,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult
fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| close_start(t, c, info))), None)
}
_ => unreachable!("expected eol"),
@@ -411,7 +412,7 @@ fn close_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu
/// | ~~~
/// ```
fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
- tokenizer.enter(TokenType::CodeFencedFence);
+ tokenizer.enter(Token::CodeFencedFence);
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), |t, c| {
close_before(t, c, info)
})(tokenizer, code)
@@ -431,7 +432,7 @@ fn close_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResu
fn close_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.enter(TokenType::CodeFencedFenceSequence);
+ tokenizer.enter(Token::CodeFencedFenceSequence);
close_sequence(tokenizer, code, info, 0)
}
_ => (State::Nok, None),
@@ -455,7 +456,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize
)
}
_ if size >= CODE_FENCED_SEQUENCE_SIZE_MIN && size >= info.size => {
- tokenizer.exit(TokenType::CodeFencedFenceSequence);
+ tokenizer.exit(Token::CodeFencedFenceSequence);
tokenizer.attempt_opt(space_or_tab(), close_sequence_after)(tokenizer, code)
}
_ => (State::Nok, None),
@@ -472,7 +473,7 @@ fn close_sequence(tokenizer: &mut Tokenizer, code: Code, info: Info, size: usize
fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFencedFence);
+ tokenizer.exit(Token::CodeFencedFence);
(State::Ok, Some(vec![code]))
}
_ => (State::Nok, None),
@@ -487,9 +488,9 @@ fn close_sequence_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
/// ~~~
/// ```
fn content_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| content_start(t, c, info))), None)
}
/// Before code content, definitely not before a closing fence.
@@ -518,7 +519,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe
at_break(tokenizer, code, info)
}
_ => {
- tokenizer.enter(TokenType::CodeFlowChunk);
+ tokenizer.enter(Token::CodeFlowChunk);
content_continue(tokenizer, code, info)
}
}
@@ -536,7 +537,7 @@ fn content_begin(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRe
fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFlowChunk);
+ tokenizer.exit(Token::CodeFlowChunk);
at_break(tokenizer, code, info)
}
_ => {
@@ -557,7 +558,7 @@ fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateF
/// ~~~|
/// ```
fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.exit(TokenType::CodeFenced);
+ tokenizer.exit(Token::CodeFenced);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index 9bdfd71..8966249 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -28,10 +28,10 @@
//!
//! ## Tokens
//!
-//! * [`CodeIndented`][TokenType::CodeIndented]
-//! * [`CodeFlowChunk`][TokenType::CodeFlowChunk]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`CodeIndented`][Token::CodeIndented]
+//! * [`CodeFlowChunk`][Token::CodeFlowChunk]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -47,7 +47,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::TAB_SIZE;
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of code (indented).
///
@@ -63,7 +64,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if tokenizer.interrupt {
(State::Nok, None)
} else {
- tokenizer.enter(TokenType::CodeIndented);
+ tokenizer.enter(Token::CodeIndented);
tokenizer.go(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), at_break)(tokenizer, code)
}
}
@@ -82,7 +83,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
Box::new(if ok { at_break } else { after })
})(tokenizer, code),
_ => {
- tokenizer.enter(TokenType::CodeFlowChunk);
+ tokenizer.enter(Token::CodeFlowChunk);
content(tokenizer, code)
}
}
@@ -98,7 +99,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::CodeFlowChunk);
+ tokenizer.exit(Token::CodeFlowChunk);
at_break(tokenizer, code)
}
_ => {
@@ -114,7 +115,7 @@ fn content(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// ab|
/// ```
fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.exit(TokenType::CodeIndented);
+ tokenizer.exit(Token::CodeIndented);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
@@ -130,9 +131,9 @@ fn further_start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
// To do: `nok` if lazy line.
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(further_start)), None)
}
_ => tokenizer.attempt(space_or_tab_min_max(TAB_SIZE, TAB_SIZE), |ok| {
diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs
index a6dc7eb..6df61b8 100644
--- a/src/construct/code_text.rs
+++ b/src/construct/code_text.rs
@@ -67,10 +67,10 @@
//!
//! ## Tokens
//!
-//! * [`CodeText`][TokenType::CodeText]
-//! * [`CodeTextData`][TokenType::CodeTextData]
-//! * [`CodeTextLineEnding`][TokenType::CodeTextLineEnding]
-//! * [`CodeTextSequence`][TokenType::CodeTextSequence]
+//! * [`CodeText`][Token::CodeText]
+//! * [`CodeTextData`][Token::CodeTextData]
+//! * [`CodeTextLineEnding`][Token::CodeTextLineEnding]
+//! * [`CodeTextSequence`][Token::CodeTextSequence]
//!
//! ## References
//!
@@ -83,7 +83,8 @@
//! [code_fenced]: crate::construct::code_fenced
//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of code (text).
///
@@ -100,11 +101,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('`')
if tokenizer.previous != Code::Char('`')
- || (len > 0
- && tokenizer.events[len - 1].token_type == TokenType::CharacterEscape) =>
+ || (len > 0 && tokenizer.events[len - 1].token_type == Token::CharacterEscape) =>
{
- tokenizer.enter(TokenType::CodeText);
- tokenizer.enter(TokenType::CodeTextSequence);
+ tokenizer.enter(Token::CodeText);
+ tokenizer.enter(Token::CodeTextSequence);
sequence_open(tokenizer, code, 0)
}
_ => (State::Nok, None),
@@ -124,7 +124,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnR
None,
)
} else {
- tokenizer.exit(TokenType::CodeTextSequence);
+ tokenizer.exit(Token::CodeTextSequence);
between(tokenizer, code, size)
}
}
@@ -139,20 +139,20 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe
match code {
Code::None => (State::Nok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::CodeTextLineEnding);
+ tokenizer.enter(Token::CodeTextLineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::CodeTextLineEnding);
+ tokenizer.exit(Token::CodeTextLineEnding);
(
State::Fn(Box::new(move |t, c| between(t, c, size_open))),
None,
)
}
Code::Char('`') => {
- tokenizer.enter(TokenType::CodeTextSequence);
+ tokenizer.enter(Token::CodeTextSequence);
sequence_close(tokenizer, code, size_open, 0)
}
_ => {
- tokenizer.enter(TokenType::CodeTextData);
+ tokenizer.enter(Token::CodeTextData);
data(tokenizer, code, size_open)
}
}
@@ -166,7 +166,7 @@ fn between(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnRe
fn data(tokenizer: &mut Tokenizer, code: Code, size_open: usize) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '`') => {
- tokenizer.exit(TokenType::CodeTextData);
+ tokenizer.exit(Token::CodeTextData);
between(tokenizer, code, size_open)
}
_ => {
@@ -198,16 +198,16 @@ fn sequence_close(
)
}
_ if size_open == size => {
- tokenizer.exit(TokenType::CodeTextSequence);
- tokenizer.exit(TokenType::CodeText);
+ tokenizer.exit(Token::CodeTextSequence);
+ tokenizer.exit(Token::CodeText);
(State::Ok, Some(vec![code]))
}
_ => {
let index = tokenizer.events.len();
- tokenizer.exit(TokenType::CodeTextSequence);
+ tokenizer.exit(Token::CodeTextSequence);
// Change the token type.
- tokenizer.events[index - 1].token_type = TokenType::CodeTextData;
- tokenizer.events[index].token_type = TokenType::CodeTextData;
+ tokenizer.events[index - 1].token_type = Token::CodeTextData;
+ tokenizer.events[index].token_type = Token::CodeTextData;
between(tokenizer, code, size_open)
}
}
diff --git a/src/construct/definition.rs b/src/construct/definition.rs
index db4a009..4d14653 100644
--- a/src/construct/definition.rs
+++ b/src/construct/definition.rs
@@ -59,21 +59,21 @@
//!
//! ## Tokens
//!
-//! * [`Definition`][TokenType::Definition]
-//! * [`DefinitionDestination`][TokenType::DefinitionDestination]
-//! * [`DefinitionDestinationLiteral`][TokenType::DefinitionDestinationLiteral]
-//! * [`DefinitionDestinationLiteralMarker`][TokenType::DefinitionDestinationLiteralMarker]
-//! * [`DefinitionDestinationRaw`][TokenType::DefinitionDestinationRaw]
-//! * [`DefinitionDestinationString`][TokenType::DefinitionDestinationString]
-//! * [`DefinitionLabel`][TokenType::DefinitionLabel]
-//! * [`DefinitionLabelMarker`][TokenType::DefinitionLabelMarker]
-//! * [`DefinitionLabelString`][TokenType::DefinitionLabelString]
-//! * [`DefinitionMarker`][TokenType::DefinitionMarker]
-//! * [`DefinitionTitle`][TokenType::DefinitionTitle]
-//! * [`DefinitionTitleMarker`][TokenType::DefinitionTitleMarker]
-//! * [`DefinitionTitleString`][TokenType::DefinitionTitleString]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`Definition`][Token::Definition]
+//! * [`DefinitionDestination`][Token::DefinitionDestination]
+//! * [`DefinitionDestinationLiteral`][Token::DefinitionDestinationLiteral]
+//! * [`DefinitionDestinationLiteralMarker`][Token::DefinitionDestinationLiteralMarker]
+//! * [`DefinitionDestinationRaw`][Token::DefinitionDestinationRaw]
+//! * [`DefinitionDestinationString`][Token::DefinitionDestinationString]
+//! * [`DefinitionLabel`][Token::DefinitionLabel]
+//! * [`DefinitionLabelMarker`][Token::DefinitionLabelMarker]
+//! * [`DefinitionLabelString`][Token::DefinitionLabelString]
+//! * [`DefinitionMarker`][Token::DefinitionMarker]
+//! * [`DefinitionTitle`][Token::DefinitionTitle]
+//! * [`DefinitionTitleMarker`][Token::DefinitionTitleMarker]
+//! * [`DefinitionTitleString`][Token::DefinitionTitleString]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -99,7 +99,8 @@ use crate::construct::{
partial_space_or_tab::{space_or_tab, space_or_tab_eol},
partial_title::{start as title, Options as TitleOptions},
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// At the start of a definition.
///
@@ -109,14 +110,14 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
let index = tokenizer.events.len();
let definition_before = index > 3
- && tokenizer.events[index - 1].token_type == TokenType::LineEnding
- && tokenizer.events[index - 3].token_type == TokenType::Definition;
+ && tokenizer.events[index - 1].token_type == Token::LineEnding
+ && tokenizer.events[index - 3].token_type == Token::Definition;
// Do not interrupt paragraphs (but do follow definitions).
if tokenizer.interrupt && !definition_before {
(State::Nok, None)
} else {
- tokenizer.enter(TokenType::Definition);
+ tokenizer.enter(Token::Definition);
// Note: arbitrary whitespace allowed even if code (indented) is on.
tokenizer.attempt_opt(space_or_tab(), before)(tokenizer, code)
}
@@ -135,9 +136,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
LabelOptions {
- label: TokenType::DefinitionLabel,
- marker: TokenType::DefinitionLabelMarker,
- string: TokenType::DefinitionLabelString,
+ label: Token::DefinitionLabel,
+ marker: Token::DefinitionLabelMarker,
+ string: Token::DefinitionLabelString,
},
)
},
@@ -155,9 +156,9 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn label_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(':') => {
- tokenizer.enter(TokenType::DefinitionMarker);
+ tokenizer.enter(Token::DefinitionMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::DefinitionMarker);
+ tokenizer.exit(Token::DefinitionMarker);
(
State::Fn(Box::new(
tokenizer.attempt_opt(space_or_tab_eol(), destination_before),
@@ -185,11 +186,11 @@ fn destination_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
c,
DestinationOptions {
limit: usize::MAX,
- destination: TokenType::DefinitionDestination,
- literal: TokenType::DefinitionDestinationLiteral,
- marker: TokenType::DefinitionDestinationLiteralMarker,
- raw: TokenType::DefinitionDestinationRaw,
- string: TokenType::DefinitionDestinationString,
+ destination: Token::DefinitionDestination,
+ literal: Token::DefinitionDestinationLiteral,
+ marker: Token::DefinitionDestinationLiteralMarker,
+ raw: Token::DefinitionDestinationRaw,
+ string: Token::DefinitionDestinationString,
},
)
},
@@ -228,7 +229,7 @@ fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn after_whitespace(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Definition);
+ tokenizer.exit(Token::Definition);
// You’d be interrupting.
tokenizer.interrupt = true;
(State::Ok, Some(vec![code]))
@@ -262,9 +263,9 @@ fn title_before_marker(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
TitleOptions {
- title: TokenType::DefinitionTitle,
- marker: TokenType::DefinitionTitleMarker,
- string: TokenType::DefinitionTitleString,
+ title: Token::DefinitionTitle,
+ marker: Token::DefinitionTitleMarker,
+ string: Token::DefinitionTitleString,
},
)
},
diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs
index 212d276..4fb87bf 100644
--- a/src/construct/hard_break_escape.rs
+++ b/src/construct/hard_break_escape.rs
@@ -26,8 +26,8 @@
//!
//! ## Tokens
//!
-//! * [`HardBreakEscape`][TokenType::HardBreakEscape]
-//! * [`HardBreakEscapeMarker`][TokenType::HardBreakEscapeMarker]
+//! * [`HardBreakEscape`][Token::HardBreakEscape]
+//! * [`HardBreakEscapeMarker`][Token::HardBreakEscapeMarker]
//!
//! ## References
//!
@@ -40,7 +40,8 @@
//! [hard_break_trailing]: crate::construct::hard_break_trailing
//! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of a hard break (escape).
///
@@ -50,10 +51,10 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('\\') => {
- tokenizer.enter(TokenType::HardBreakEscape);
- tokenizer.enter(TokenType::HardBreakEscapeMarker);
+ tokenizer.enter(Token::HardBreakEscape);
+ tokenizer.enter(Token::HardBreakEscapeMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::HardBreakEscapeMarker);
+ tokenizer.exit(Token::HardBreakEscapeMarker);
(State::Fn(Box::new(inside)), None)
}
_ => (State::Nok, None),
@@ -68,7 +69,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HardBreakEscape);
+ tokenizer.exit(Token::HardBreakEscape);
(State::Ok, Some(vec![code]))
}
_ => (State::Nok, None),
diff --git a/src/construct/hard_break_trailing.rs b/src/construct/hard_break_trailing.rs
index 35a7cab..6626675 100644
--- a/src/construct/hard_break_trailing.rs
+++ b/src/construct/hard_break_trailing.rs
@@ -26,8 +26,8 @@
//!
//! ## Tokens
//!
-//! * [`HardBreakTrailing`][TokenType::HardBreakTrailing]
-//! * [`HardBreakTrailingSpace`][TokenType::HardBreakTrailingSpace]
+//! * [`HardBreakTrailing`][Token::HardBreakTrailing]
+//! * [`HardBreakTrailingSpace`][Token::HardBreakTrailingSpace]
//!
//! ## References
//!
@@ -41,7 +41,8 @@
//! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element
use crate::constant::HARD_BREAK_PREFIX_SIZE_MIN;
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Start of a hard break (trailing).
///
@@ -52,8 +53,8 @@ use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(' ') => {
- tokenizer.enter(TokenType::HardBreakTrailing);
- tokenizer.enter(TokenType::HardBreakTrailingSpace);
+ tokenizer.enter(Token::HardBreakTrailing);
+ tokenizer.enter(Token::HardBreakTrailingSpace);
tokenizer.consume(code);
(State::Fn(Box::new(|t, c| inside(t, c, 1))), None)
}
@@ -79,8 +80,8 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, size: usize) -> StateFnResult {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')
if size >= HARD_BREAK_PREFIX_SIZE_MIN =>
{
- tokenizer.exit(TokenType::HardBreakTrailingSpace);
- tokenizer.exit(TokenType::HardBreakTrailing);
+ tokenizer.exit(Token::HardBreakTrailingSpace);
+ tokenizer.exit(Token::HardBreakTrailing);
(State::Ok, Some(vec![code]))
}
_ => (State::Nok, None),
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 9fa2ace..4a4992a 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -37,10 +37,10 @@
//!
//! ## Tokens
//!
-//! * [`HeadingAtx`][TokenType::HeadingAtx]
-//! * [`HeadingAtxSequence`][TokenType::HeadingAtxSequence]
-//! * [`HeadingAtxText`][TokenType::HeadingAtxText]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`HeadingAtx`][Token::HeadingAtx]
+//! * [`HeadingAtxSequence`][Token::HeadingAtxSequence]
+//! * [`HeadingAtxText`][Token::HeadingAtxText]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -56,9 +56,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE};
-use crate::tokenizer::{
- Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer,
-};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::edit_map::EditMap;
/// Start of a heading (atx).
@@ -67,7 +66,7 @@ use crate::util::edit_map::EditMap;
/// |## alpha
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::HeadingAtx);
+ tokenizer.enter(Token::HeadingAtx);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -79,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
/// ```
fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if Code::Char('#') == code {
- tokenizer.enter(TokenType::HeadingAtxSequence);
+ tokenizer.enter(Token::HeadingAtxSequence);
sequence_open(tokenizer, code, 0)
} else {
(State::Nok, None)
@@ -94,7 +93,7 @@ fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') if rank > 0 => {
- tokenizer.exit(TokenType::HeadingAtxSequence);
+ tokenizer.exit(Token::HeadingAtxSequence);
at_break(tokenizer, code)
}
Code::Char('#') if rank < HEADING_ATX_OPENING_FENCE_SIZE_MAX => {
@@ -107,7 +106,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR
)
}
_ if rank > 0 => {
- tokenizer.exit(TokenType::HeadingAtxSequence);
+ tokenizer.exit(Token::HeadingAtxSequence);
tokenizer.go(space_or_tab(), at_break)(tokenizer, code)
}
_ => (State::Nok, None),
@@ -126,7 +125,7 @@ fn sequence_open(tokenizer: &mut Tokenizer, code: Code, rank: usize) -> StateFnR
fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HeadingAtx);
+ tokenizer.exit(Token::HeadingAtx);
tokenizer.register_resolver("heading_atx".to_string(), Box::new(resolve));
// Feel free to interrupt.
tokenizer.interrupt = false;
@@ -136,11 +135,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
tokenizer.go(space_or_tab(), at_break)(tokenizer, code)
}
Code::Char('#') => {
- tokenizer.enter(TokenType::HeadingAtxSequence);
+ tokenizer.enter(Token::HeadingAtxSequence);
further_sequence(tokenizer, code)
}
Code::Char(_) => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::Text));
data(tokenizer, code)
}
}
@@ -157,7 +156,7 @@ fn further_sequence(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
tokenizer.consume(code);
(State::Fn(Box::new(further_sequence)), None)
} else {
- tokenizer.exit(TokenType::HeadingAtxSequence);
+ tokenizer.exit(Token::HeadingAtxSequence);
at_break(tokenizer, code)
}
}
@@ -171,7 +170,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
// Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text.
Code::None | Code::CarriageReturnLineFeed | Code::Char('\t' | '\n' | '\r' | ' ') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code)
}
_ => {
@@ -192,7 +191,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.token_type == TokenType::HeadingAtx {
+ if event.token_type == Token::HeadingAtx {
if event.event_type == EventType::Enter {
heading_start = Some(index);
} else if let Some(start) = data_start {
@@ -204,7 +203,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Enter,
- token_type: TokenType::HeadingAtxText,
+ token_type: Token::HeadingAtxText,
point: tokenizer.events[start].point.clone(),
index: tokenizer.events[start].index,
previous: None,
@@ -221,7 +220,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::HeadingAtxText,
+ token_type: Token::HeadingAtxText,
point: tokenizer.events[end].point.clone(),
index: tokenizer.events[end].index,
previous: None,
@@ -234,7 +233,7 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
data_start = None;
data_end = None;
}
- } else if heading_start.is_some() && event.token_type == TokenType::Data {
+ } else if heading_start.is_some() && event.token_type == Token::Data {
if event.event_type == EventType::Enter {
if data_start.is_none() {
data_start = Some(index);
diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index 440baa8..633f7de 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -40,9 +40,9 @@
//!
//! ## Tokens
//!
-//! * [`HeadingSetext`][TokenType::HeadingSetext]
-//! * [`HeadingSetextText`][TokenType::HeadingSetextText]
-//! * [`HeadingSetextUnderline`][TokenType::HeadingSetextUnderline]
+//! * [`HeadingSetext`][Token::HeadingSetext]
+//! * [`HeadingSetextText`][Token::HeadingSetextText]
+//! * [`HeadingSetextUnderline`][Token::HeadingSetextUnderline]
//!
//! ## References
//!
@@ -59,7 +59,8 @@
use crate::constant::TAB_SIZE;
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::{edit_map::EditMap, skip::opt_back as skip_opt_back};
/// Kind of underline.
@@ -120,14 +121,14 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
skip_opt_back(
&tokenizer.events,
index - 1,
- &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix],
+ &[Token::SpaceOrTab, Token::BlockQuotePrefix],
)
} else {
0
};
- let previous = skip_opt_back(&tokenizer.events, previous, &[TokenType::LineEnding]);
+ let previous = skip_opt_back(&tokenizer.events, previous, &[Token::LineEnding]);
let paragraph_before =
- previous > 1 && tokenizer.events[previous].token_type == TokenType::Paragraph;
+ previous > 1 && tokenizer.events[previous].token_type == Token::Paragraph;
println!(
"setext-start: {:?} {:?} {:?}",
@@ -152,7 +153,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(char) if char == '-' || char == '=' => {
- tokenizer.enter(TokenType::HeadingSetextUnderline);
+ tokenizer.enter(Token::HeadingSetextUnderline);
inside(tokenizer, code, Kind::from_char(char))
}
_ => (State::Nok, None),
@@ -184,7 +185,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult {
fn after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HeadingSetextUnderline);
+ tokenizer.exit(Token::HeadingSetextUnderline);
// Feel free to interrupt.
tokenizer.interrupt = false;
tokenizer.register_resolver("heading_setext".to_string(), Box::new(resolve));
@@ -206,27 +207,27 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
// Find paragraphs.
if event.event_type == EventType::Enter {
- if event.token_type == TokenType::Paragraph {
+ if event.token_type == Token::Paragraph {
paragraph_enter = Some(index);
}
- } else if event.token_type == TokenType::Paragraph {
+ } else if event.token_type == Token::Paragraph {
paragraph_exit = Some(index);
}
// We know this is preceded by a paragraph.
// Otherwise we don’t parse.
- else if event.token_type == TokenType::HeadingSetextUnderline {
+ else if event.token_type == Token::HeadingSetextUnderline {
let enter = paragraph_enter.take().unwrap();
let exit = paragraph_exit.take().unwrap();
// Change types of Enter:Paragraph, Exit:Paragraph.
- tokenizer.events[enter].token_type = TokenType::HeadingSetextText;
- tokenizer.events[exit].token_type = TokenType::HeadingSetextText;
+ tokenizer.events[enter].token_type = Token::HeadingSetextText;
+ tokenizer.events[exit].token_type = Token::HeadingSetextText;
// Add Enter:HeadingSetext, Exit:HeadingSetext.
let mut heading_enter = tokenizer.events[enter].clone();
- heading_enter.token_type = TokenType::HeadingSetext;
+ heading_enter.token_type = Token::HeadingSetext;
let mut heading_exit = tokenizer.events[index].clone();
- heading_exit.token_type = TokenType::HeadingSetext;
+ heading_exit.token_type = Token::HeadingSetext;
edit_map.add(enter, 0, vec![heading_enter]);
edit_map.add(index + 1, 0, vec![heading_exit]);
diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs
index be7a3a9..fde0a34 100644
--- a/src/construct/html_flow.rs
+++ b/src/construct/html_flow.rs
@@ -82,9 +82,9 @@
//!
//! ## Tokens
//!
-//! * [`HtmlFlow`][TokenType::HtmlFlow]
-//! * [`HtmlFlowData`][TokenType::HtmlFlowData]
-//! * [`LineEnding`][TokenType::LineEnding]
+//! * [`HtmlFlow`][Token::HtmlFlow]
+//! * [`HtmlFlowData`][Token::HtmlFlowData]
+//! * [`LineEnding`][Token::LineEnding]
//!
//! ## References
//!
@@ -102,7 +102,8 @@ use crate::constant::{HTML_BLOCK_NAMES, HTML_RAW_NAMES, HTML_RAW_SIZE_MAX, TAB_S
use crate::construct::{
blank_line::start as blank_line, partial_space_or_tab::space_or_tab_min_max,
};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
use crate::util::codes::{parse, serialize};
// To do: mark as concrete (block quotes or lists can’t “pierce” into HTML).
@@ -203,8 +204,8 @@ struct Info {
/// ```
///
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::HtmlFlow);
- tokenizer.enter(TokenType::HtmlFlowData);
+ tokenizer.enter(Token::HtmlFlow);
+ tokenizer.enter(Token::HtmlFlowData);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -776,7 +777,7 @@ fn continuation(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnRes
/// <x>|
/// ```
fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
- tokenizer.exit(TokenType::HtmlFlowData);
+ tokenizer.exit(Token::HtmlFlowData);
html_continue_start(tokenizer, code, info)
}
@@ -789,23 +790,23 @@ fn continuation_at_line_ending(tokenizer: &mut Tokenizer, code: Code, info: Info
fn html_continue_start(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None => {
- tokenizer.exit(TokenType::HtmlFlow);
+ tokenizer.exit(Token::HtmlFlow);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
}
// To do: do not allow lazy lines.
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(
State::Fn(Box::new(|t, c| html_continue_start(t, c, info))),
None,
)
}
_ => {
- tokenizer.enter(TokenType::HtmlFlowData);
+ tokenizer.enter(Token::HtmlFlowData);
continuation(tokenizer, code, info)
}
}
@@ -955,8 +956,8 @@ fn continuation_declaration_inside(
fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HtmlFlowData);
- tokenizer.exit(TokenType::HtmlFlow);
+ tokenizer.exit(Token::HtmlFlowData);
+ tokenizer.exit(Token::HtmlFlow);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
@@ -978,8 +979,8 @@ fn continuation_close(tokenizer: &mut Tokenizer, code: Code, info: Info) -> Stat
///
/// ```
fn blank_line_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(blank_line)), None)
}
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs
index 0926f48..cdd7c69 100644
--- a/src/construct/html_text.rs
+++ b/src/construct/html_text.rs
@@ -42,8 +42,8 @@
//!
//! ## Tokens
//!
-//! * [`HtmlText`][TokenType::HtmlText]
-//! * [`HtmlTextData`][TokenType::HtmlTextData]
+//! * [`HtmlText`][Token::HtmlText]
+//! * [`HtmlTextData`][Token::HtmlTextData]
//!
//! ## References
//!
@@ -55,7 +55,8 @@
//! [html-parsing]: https://html.spec.whatwg.org/multipage/parsing.html#parsing
use crate::construct::partial_space_or_tab::space_or_tab;
-use crate::tokenizer::{Code, State, StateFn, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFn, StateFnResult, Tokenizer};
use crate::util::codes::parse;
/// Start of HTML (text)
@@ -65,8 +66,8 @@ use crate::util::codes::parse;
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
if Code::Char('<') == code {
- tokenizer.enter(TokenType::HtmlText);
- tokenizer.enter(TokenType::HtmlTextData);
+ tokenizer.enter(Token::HtmlText);
+ tokenizer.enter(Token::HtmlTextData);
tokenizer.consume(code);
(State::Fn(Box::new(open)), None)
} else {
@@ -617,8 +618,8 @@ fn end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('>') => {
tokenizer.consume(code);
- tokenizer.exit(TokenType::HtmlTextData);
- tokenizer.exit(TokenType::HtmlText);
+ tokenizer.exit(Token::HtmlTextData);
+ tokenizer.exit(Token::HtmlText);
(State::Ok, None)
}
_ => (State::Nok, None),
@@ -641,10 +642,10 @@ fn at_line_ending(
) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::HtmlTextData);
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.exit(Token::HtmlTextData);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(
State::Fn(Box::new(|t, c| after_line_ending(t, c, return_state))),
None,
@@ -687,6 +688,6 @@ fn after_line_ending_prefix(
code: Code,
return_state: Box<StateFn>,
) -> StateFnResult {
- tokenizer.enter(TokenType::HtmlTextData);
+ tokenizer.enter(Token::HtmlTextData);
return_state(tokenizer, code)
}
diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs
index 3a40cc2..29ac6f9 100644
--- a/src/construct/label_end.rs
+++ b/src/construct/label_end.rs
@@ -102,28 +102,28 @@
//!
//! ## Tokens
//!
-//! * [`Data`][TokenType::Data]
-//! * [`Image`][TokenType::Image]
-//! * [`Label`][TokenType::Label]
-//! * [`LabelEnd`][TokenType::LabelEnd]
-//! * [`LabelMarker`][TokenType::LabelMarker]
-//! * [`LabelText`][TokenType::LabelText]
-//! * [`LineEnding`][TokenType::LineEnding]
-//! * [`Link`][TokenType::Link]
-//! * [`Reference`][TokenType::Reference]
-//! * [`ReferenceMarker`][TokenType::ReferenceMarker]
-//! * [`ReferenceString`][TokenType::ReferenceString]
-//! * [`Resource`][TokenType::Resource]
-//! * [`ResourceDestination`][TokenType::ResourceDestination]
-//! * [`ResourceDestinationLiteral`][TokenType::ResourceDestinationLiteral]
-//! * [`ResourceDestinationLiteralMarker`][TokenType::ResourceDestinationLiteralMarker]
-//! * [`ResourceDestinationRaw`][TokenType::ResourceDestinationRaw]
-//! * [`ResourceDestinationString`][TokenType::ResourceDestinationString]
-//! * [`ResourceMarker`][TokenType::ResourceMarker]
-//! * [`ResourceTitle`][TokenType::ResourceTitle]
-//! * [`ResourceTitleMarker`][TokenType::ResourceTitleMarker]
-//! * [`ResourceTitleString`][TokenType::ResourceTitleString]
-//! * [`SpaceOrTab`][TokenType::SpaceOrTab]
+//! * [`Data`][Token::Data]
+//! * [`Image`][Token::Image]
+//! * [`Label`][Token::Label]
+//! * [`LabelEnd`][Token::LabelEnd]
+//! * [`LabelMarker`][Token::LabelMarker]
+//! * [`LabelText`][Token::LabelText]
+//! * [`LineEnding`][Token::LineEnding]
+//! * [`Link`][Token::Link]
+//! * [`Reference`][Token::Reference]
+//! * [`ReferenceMarker`][Token::ReferenceMarker]
+//! * [`ReferenceString`][Token::ReferenceString]
+//! * [`Resource`][Token::Resource]
+//! * [`ResourceDestination`][Token::ResourceDestination]
+//! * [`ResourceDestinationLiteral`][Token::ResourceDestinationLiteral]
+//! * [`ResourceDestinationLiteralMarker`][Token::ResourceDestinationLiteralMarker]
+//! * [`ResourceDestinationRaw`][Token::ResourceDestinationRaw]
+//! * [`ResourceDestinationString`][Token::ResourceDestinationString]
+//! * [`ResourceMarker`][Token::ResourceMarker]
+//! * [`ResourceTitle`][Token::ResourceTitle]
+//! * [`ResourceTitleMarker`][Token::ResourceTitleMarker]
+//! * [`ResourceTitleString`][Token::ResourceTitleString]
+//! * [`SpaceOrTab`][Token::SpaceOrTab]
//!
//! ## References
//!
@@ -153,8 +153,9 @@ use crate::construct::{
partial_space_or_tab::space_or_tab_eol,
partial_title::{start as title, Options as TitleOptions},
};
+use crate::token::Token;
use crate::tokenizer::{
- Code, Event, EventType, LabelStart, Media, State, StateFnResult, TokenType, Tokenizer,
+ Code, Event, EventType, LabelStart, Media, State, StateFnResult, Tokenizer,
};
use crate::util::{
edit_map::EditMap,
@@ -198,7 +199,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
vec![
Event {
event_type: EventType::Enter,
- token_type: TokenType::Data,
+ token_type: Token::Data,
point: events[data_enter_index].point.clone(),
index: events[data_enter_index].index,
previous: None,
@@ -207,7 +208,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
},
Event {
event_type: EventType::Exit,
- token_type: TokenType::Data,
+ token_type: Token::Data,
point: events[data_exit_index].point.clone(),
index: events[data_exit_index].index,
previous: None,
@@ -229,7 +230,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
let group_enter_event = &events[group_enter_index];
// LabelLink:Exit or LabelImage:Exit.
let text_enter_index = media.start.0
- + (if group_enter_event.token_type == TokenType::LabelLink {
+ + (if group_enter_event.token_type == Token::LabelLink {
4
} else {
6
@@ -248,10 +249,10 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
vec![
Event {
event_type: EventType::Enter,
- token_type: if group_enter_event.token_type == TokenType::LabelLink {
- TokenType::Link
+ token_type: if group_enter_event.token_type == Token::LabelLink {
+ Token::Link
} else {
- TokenType::Image
+ Token::Image
},
point: group_enter_event.point.clone(),
index: group_enter_event.index,
@@ -261,7 +262,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
},
Event {
event_type: EventType::Enter,
- token_type: TokenType::Label,
+ token_type: Token::Label,
point: group_enter_event.point.clone(),
index: group_enter_event.index,
previous: None,
@@ -279,7 +280,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Enter,
- token_type: TokenType::LabelText,
+ token_type: Token::LabelText,
point: events[text_enter_index].point.clone(),
index: events[text_enter_index].index,
previous: None,
@@ -294,7 +295,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::LabelText,
+ token_type: Token::LabelText,
point: events[text_exit_index].point.clone(),
index: events[text_exit_index].index,
previous: None,
@@ -310,7 +311,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::Label,
+ token_type: Token::Label,
point: events[label_exit_index].point.clone(),
index: events[label_exit_index].index,
previous: None,
@@ -325,7 +326,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) -> Vec<Event> {
0,
vec![Event {
event_type: EventType::Exit,
- token_type: TokenType::Link,
+ token_type: Token::Link,
point: events[group_end_index].point.clone(),
index: events[group_end_index].index,
previous: None,
@@ -393,11 +394,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
},
};
- tokenizer.enter(TokenType::LabelEnd);
- tokenizer.enter(TokenType::LabelMarker);
+ tokenizer.enter(Token::LabelEnd);
+ tokenizer.enter(Token::LabelMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelMarker);
- tokenizer.exit(TokenType::LabelEnd);
+ tokenizer.exit(Token::LabelMarker);
+ tokenizer.exit(Token::LabelEnd);
return (State::Fn(Box::new(move |t, c| after(t, c, info))), None);
}
@@ -495,13 +496,13 @@ fn ok(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
left.remove(0);
tokenizer.label_start_list_loose.append(&mut left);
- let is_link = tokenizer.events[info.media.start.0].token_type == TokenType::LabelLink;
+ let is_link = tokenizer.events[info.media.start.0].token_type == Token::LabelLink;
if is_link {
let mut index = 0;
while index < tokenizer.label_start_stack.len() {
let label_start = &mut tokenizer.label_start_stack[index];
- if tokenizer.events[label_start.start.0].token_type == TokenType::LabelLink {
+ if tokenizer.events[label_start.start.0].token_type == Token::LabelLink {
label_start.inactive = true;
}
index += 1;
@@ -543,10 +544,10 @@ fn nok(tokenizer: &mut Tokenizer, _code: Code, label_start_index: usize) -> Stat
fn resource(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('(') => {
- tokenizer.enter(TokenType::Resource);
- tokenizer.enter(TokenType::ResourceMarker);
+ tokenizer.enter(Token::Resource);
+ tokenizer.enter(Token::ResourceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ResourceMarker);
+ tokenizer.exit(Token::ResourceMarker);
(State::Fn(Box::new(resource_start)), None)
}
_ => unreachable!("expected `(`"),
@@ -577,11 +578,11 @@ fn resource_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
c,
DestinationOptions {
limit: RESOURCE_DESTINATION_BALANCE_MAX,
- destination: TokenType::ResourceDestination,
- literal: TokenType::ResourceDestinationLiteral,
- marker: TokenType::ResourceDestinationLiteralMarker,
- raw: TokenType::ResourceDestinationRaw,
- string: TokenType::ResourceDestinationString,
+ destination: Token::ResourceDestination,
+ literal: Token::ResourceDestinationLiteral,
+ marker: Token::ResourceDestinationLiteralMarker,
+ raw: Token::ResourceDestinationRaw,
+ string: Token::ResourceDestinationString,
},
)
},
@@ -616,9 +617,9 @@ fn resource_between(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
TitleOptions {
- title: TokenType::ResourceTitle,
- marker: TokenType::ResourceTitleMarker,
- string: TokenType::ResourceTitleString,
+ title: Token::ResourceTitle,
+ marker: Token::ResourceTitleMarker,
+ string: Token::ResourceTitleString,
},
)
},
@@ -647,10 +648,10 @@ fn title_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn resource_end(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(')') => {
- tokenizer.enter(TokenType::ResourceMarker);
+ tokenizer.enter(Token::ResourceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ResourceMarker);
- tokenizer.exit(TokenType::Resource);
+ tokenizer.exit(Token::ResourceMarker);
+ tokenizer.exit(Token::Resource);
(State::Ok, None)
}
_ => (State::Nok, None),
@@ -670,9 +671,9 @@ fn full_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
t,
c,
LabelOptions {
- label: TokenType::Reference,
- marker: TokenType::ReferenceMarker,
- string: TokenType::ReferenceString,
+ label: Token::Reference,
+ marker: Token::ReferenceMarker,
+ string: Token::ReferenceString,
},
)
},
@@ -696,7 +697,7 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
while index > 0 {
index -= 1;
let event = &events[index];
- if event.token_type == TokenType::ReferenceString {
+ if event.token_type == Token::ReferenceString {
if event.event_type == EventType::Exit {
end = Some(event.index);
} else {
@@ -735,10 +736,10 @@ fn full_reference_after(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult
fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('[') => {
- tokenizer.enter(TokenType::Reference);
- tokenizer.enter(TokenType::ReferenceMarker);
+ tokenizer.enter(Token::Reference);
+ tokenizer.enter(Token::ReferenceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ReferenceMarker);
+ tokenizer.exit(Token::ReferenceMarker);
(State::Fn(Box::new(collapsed_reference_open)), None)
}
_ => (State::Nok, None),
@@ -755,10 +756,10 @@ fn collapsed_reference(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn collapsed_reference_open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char(']') => {
- tokenizer.enter(TokenType::ReferenceMarker);
+ tokenizer.enter(Token::ReferenceMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::ReferenceMarker);
- tokenizer.exit(TokenType::Reference);
+ tokenizer.exit(Token::ReferenceMarker);
+ tokenizer.exit(Token::Reference);
(State::Ok, None)
}
_ => (State::Nok, None),
diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs
index a45205a..f9b8300 100644
--- a/src/construct/label_start_image.rs
+++ b/src/construct/label_start_image.rs
@@ -15,9 +15,9 @@
//!
//! ## Tokens
//!
-//! * [`LabelImage`][TokenType::LabelImage]
-//! * [`LabelImageMarker`][TokenType::LabelImageMarker]
-//! * [`LabelMarker`][TokenType::LabelMarker]
+//! * [`LabelImage`][Token::LabelImage]
+//! * [`LabelImageMarker`][Token::LabelImageMarker]
+//! * [`LabelMarker`][Token::LabelMarker]
//!
//! ## References
//!
@@ -29,7 +29,8 @@
//! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element
use super::label_end::resolve_media;
-use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer};
/// Start of label (image) start.
///
@@ -39,10 +40,10 @@ use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokeni
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('!') => {
- tokenizer.enter(TokenType::LabelImage);
- tokenizer.enter(TokenType::LabelImageMarker);
+ tokenizer.enter(Token::LabelImage);
+ tokenizer.enter(Token::LabelImageMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelImageMarker);
+ tokenizer.exit(Token::LabelImageMarker);
(State::Fn(Box::new(open)), None)
}
_ => (State::Nok, None),
@@ -57,10 +58,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
pub fn open(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('[') => {
- tokenizer.enter(TokenType::LabelMarker);
+ tokenizer.enter(Token::LabelMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelMarker);
- tokenizer.exit(TokenType::LabelImage);
+ tokenizer.exit(Token::LabelMarker);
+ tokenizer.exit(Token::LabelImage);
let end = tokenizer.events.len() - 1;
tokenizer.label_start_stack.push(LabelStart {
start: (end - 5, end),
diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs
index 6c4d7ae..59729cc 100644
--- a/src/construct/label_start_link.rs
+++ b/src/construct/label_start_link.rs
@@ -15,8 +15,8 @@
//!
//! ## Tokens
//!
-//! * [`LabelLink`][TokenType::LabelLink]
-//! * [`LabelMarker`][TokenType::LabelMarker]
+//! * [`LabelLink`][Token::LabelLink]
+//! * [`LabelMarker`][Token::LabelMarker]
//!
//! ## References
//!
@@ -28,7 +28,8 @@
//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use super::label_end::resolve_media;
-use crate::tokenizer::{Code, LabelStart, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, LabelStart, State, StateFnResult, Tokenizer};
/// Start of label (link) start.
///
@@ -39,11 +40,11 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::Char('[') => {
let start = tokenizer.events.len();
- tokenizer.enter(TokenType::LabelLink);
- tokenizer.enter(TokenType::LabelMarker);
+ tokenizer.enter(Token::LabelLink);
+ tokenizer.enter(Token::LabelMarker);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LabelMarker);
- tokenizer.exit(TokenType::LabelLink);
+ tokenizer.exit(Token::LabelMarker);
+ tokenizer.exit(Token::LabelLink);
tokenizer.label_start_stack.push(LabelStart {
start: (start, tokenizer.events.len() - 1),
balanced: false,
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index ace174f..967e009 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -19,7 +19,7 @@
//!
//! ## Tokens
//!
-//! * [`Paragraph`][TokenType::Paragraph]
+//! * [`Paragraph`][Token::Paragraph]
//!
//! ## References
//!
@@ -32,9 +32,8 @@
//! [code_text]: crate::construct::code_text
//! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element
-use crate::tokenizer::{
- Code, ContentType, Event, EventType, State, StateFnResult, TokenType, Tokenizer,
-};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::{edit_map::EditMap, skip::opt as skip_opt};
/// Before a paragraph.
@@ -48,8 +47,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
unreachable!("unexpected eol/eof")
}
_ => {
- tokenizer.enter(TokenType::Paragraph);
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::Text));
+ tokenizer.enter(Token::Paragraph);
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::Text));
inside(tokenizer, code)
}
}
@@ -63,8 +62,8 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
fn inside(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
- tokenizer.exit(TokenType::Paragraph);
+ tokenizer.exit(Token::Data);
+ tokenizer.exit(Token::Paragraph);
tokenizer.register_resolver_before("paragraph".to_string(), Box::new(resolve));
// You’d be interrupting.
tokenizer.interrupt = true;
@@ -87,21 +86,21 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
while index < len {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == TokenType::Paragraph {
+ if event.event_type == EventType::Enter && event.token_type == Token::Paragraph {
// Exit:Paragraph
let mut exit_index = index + 3;
let mut enter_next_index =
- skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]);
+ skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]);
// Enter:Paragraph
enter_next_index = skip_opt(
&tokenizer.events,
enter_next_index,
- &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix],
+ &[Token::SpaceOrTab, Token::BlockQuotePrefix],
);
// Find future `Paragraphs`.
while enter_next_index < tokenizer.events.len()
- && tokenizer.events[enter_next_index].token_type == TokenType::Paragraph
+ && tokenizer.events[enter_next_index].token_type == Token::Paragraph
{
// Remove Exit:Paragraph, Enter:LineEnding, Exit:LineEnding, Enter:Paragraph.
edit_map.add(exit_index, 3, vec![]);
@@ -126,11 +125,11 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
// Potential next start.
exit_index = enter_next_index + 3;
enter_next_index =
- skip_opt(&tokenizer.events, exit_index + 1, &[TokenType::LineEnding]);
+ skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]);
enter_next_index = skip_opt(
&tokenizer.events,
enter_next_index,
- &[TokenType::SpaceOrTab, TokenType::BlockQuotePrefix],
+ &[Token::SpaceOrTab, Token::BlockQuotePrefix],
);
}
diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs
index 555ccaf..186665d 100644
--- a/src/construct/partial_data.rs
+++ b/src/construct/partial_data.rs
@@ -6,7 +6,8 @@
//! [string]: crate::content::string
//! [text]: crate::content::text
-use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, Event, EventType, State, StateFnResult, Tokenizer};
use crate::util::edit_map::EditMap;
/// At the beginning of data.
@@ -16,7 +17,7 @@ use crate::util::edit_map::EditMap;
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult {
if stop.contains(&code) {
- tokenizer.enter(TokenType::Data);
+ tokenizer.enter(Token::Data);
tokenizer.consume(code);
(State::Fn(Box::new(|t, c| data(t, c, stop))), None)
} else {
@@ -33,9 +34,9 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe
match code {
Code::None => (State::Ok, None),
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter(TokenType::LineEnding);
+ tokenizer.enter(Token::LineEnding);
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| at_break(t, c, stop))), None)
}
_ if stop.contains(&code) => {
@@ -43,7 +44,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnRe
(State::Ok, Some(vec![code]))
}
_ => {
- tokenizer.enter(TokenType::Data);
+ tokenizer.enter(Token::Data);
data(tokenizer, code, stop)
}
}
@@ -62,7 +63,7 @@ fn data(tokenizer: &mut Tokenizer, code: Code, stop: Vec<Code>) -> StateFnResult
};
if done {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, stop)
} else {
tokenizer.consume(code);
@@ -80,13 +81,13 @@ pub fn resolve(tokenizer: &mut Tokenizer) -> Vec<Event> {
while index < len {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == TokenType::Data {
+ if event.event_type == EventType::Enter && event.token_type == Token::Data {
let exit_index = index + 1;
let mut exit_far_index = exit_index;
// Find multiple `data` events.
while exit_far_index + 1 < len
- && tokenizer.events[exit_far_index + 1].token_type == TokenType::Data
+ && tokenizer.events[exit_far_index + 1].token_type == Token::Data
{
exit_far_index += 2;
}
diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs
index 31c13ec..daa968a 100644
--- a/src/construct/partial_destination.rs
+++ b/src/construct/partial_destination.rs
@@ -71,7 +71,8 @@
//! [label_end]: crate::construct::label_end
//! [sanitize_uri]: crate::util::sanitize_uri
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -79,15 +80,15 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole destination.
- pub destination: TokenType,
+ pub destination: Token,
/// Token for a literal (enclosed) destination.
- pub literal: TokenType,
+ pub literal: Token,
/// Token for a literal marker.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for a raw destination.
- pub raw: TokenType,
+ pub raw: Token,
/// Token for a the string.
- pub string: TokenType,
+ pub string: Token,
/// Maximum unbalanced parens.
pub limit: usize,
}
@@ -133,7 +134,7 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code, options: Options) -> StateFn
tokenizer.enter(info.options.destination.clone());
tokenizer.enter(info.options.raw.clone());
tokenizer.enter(info.options.string.clone());
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
raw(tokenizer, code, info)
}
}
@@ -154,7 +155,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn
(State::Ok, None)
} else {
tokenizer.enter(info.options.string.clone());
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
enclosed(tokenizer, code, info)
}
}
@@ -167,7 +168,7 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFn
fn enclosed(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char('>') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
enclosed_before(tokenizer, code, info)
}
@@ -221,7 +222,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
}
Code::Char(')') => {
if info.balance == 0 {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
tokenizer.exit(info.options.raw.clone());
tokenizer.exit(info.options.destination);
@@ -239,7 +240,7 @@ fn raw(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
if info.balance > 0 {
(State::Nok, None)
} else {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
tokenizer.exit(info.options.string.clone());
tokenizer.exit(info.options.raw.clone());
tokenizer.exit(info.options.destination);
diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs
index f201f60..f380c7d 100644
--- a/src/construct/partial_label.rs
+++ b/src/construct/partial_label.rs
@@ -61,7 +61,8 @@
use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::constant::LINK_REFERENCE_SIZE_MAX;
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -69,11 +70,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole label.
- pub label: TokenType,
+ pub label: Token,
/// Token for the markers.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for the string (inside the markers).
- pub string: TokenType,
+ pub string: Token,
}
/// State needed to parse labels.
@@ -144,7 +145,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
},
)(tokenizer, code),
_ => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -166,11 +167,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
fn label(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResult {
match code {
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r' | '[' | ']') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
_ if info.size > LINK_REFERENCE_SIZE_MAX => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::VirtualSpace | Code::Char('\t' | ' ') => {
diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs
index 5b1ec5e..78477de 100644
--- a/src/construct/partial_space_or_tab.rs
+++ b/src/construct/partial_space_or_tab.rs
@@ -5,7 +5,8 @@
//! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js)
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFn, StateFnResult, Tokenizer};
/// Options to parse `space_or_tab`.
#[derive(Debug)]
@@ -15,7 +16,7 @@ pub struct Options {
/// Maximum allowed characters (inclusive).
pub max: usize,
/// Token type to use for whitespace events.
- pub kind: TokenType,
+ pub kind: Token,
/// Connect this whitespace to the previous.
pub connect: bool,
/// Embedded content type to use.
@@ -67,7 +68,7 @@ pub fn space_or_tab() -> Box<StateFn> {
/// ```
pub fn space_or_tab_min_max(min: usize, max: usize) -> Box<StateFn> {
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min,
max,
content_type: None,
@@ -104,7 +105,7 @@ pub fn space_or_tab_eol_with_options(options: EolOptions) -> Box<StateFn> {
tokenizer.attempt(
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: info.options.content_type,
@@ -196,7 +197,7 @@ fn inside(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnResul
fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo) -> StateFnResult {
match code {
Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.enter_with_content(TokenType::LineEnding, info.options.content_type);
+ tokenizer.enter_with_content(Token::LineEnding, info.options.content_type);
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -206,7 +207,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo)
}
tokenizer.consume(code);
- tokenizer.exit(TokenType::LineEnding);
+ tokenizer.exit(Token::LineEnding);
(State::Fn(Box::new(|t, c| after_eol(t, c, info))), None)
}
_ if info.ok => (State::Ok, Some(vec![code])),
@@ -229,7 +230,7 @@ fn after_space_or_tab(tokenizer: &mut Tokenizer, code: Code, mut info: EolInfo)
fn after_eol(tokenizer: &mut Tokenizer, code: Code, info: EolInfo) -> StateFnResult {
tokenizer.attempt_opt(
space_or_tab_with_options(Options {
- kind: TokenType::SpaceOrTab,
+ kind: Token::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: info.options.content_type,
diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs
index 010f554..6303da8 100644
--- a/src/construct/partial_title.rs
+++ b/src/construct/partial_title.rs
@@ -32,7 +32,8 @@
use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::subtokenize::link;
-use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, ContentType, State, StateFnResult, Tokenizer};
/// Configuration.
///
@@ -40,11 +41,11 @@ use crate::tokenizer::{Code, ContentType, State, StateFnResult, TokenType, Token
#[derive(Debug)]
pub struct Options {
/// Token for the whole title.
- pub title: TokenType,
+ pub title: Token,
/// Token for the marker.
- pub marker: TokenType,
+ pub marker: Token,
/// Token for the string inside the quotes.
- pub string: TokenType,
+ pub string: Token,
}
/// Type of title.
@@ -204,7 +205,7 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
},
)(tokenizer, code),
_ => {
- tokenizer.enter_with_content(TokenType::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
if info.connect {
let index = tokenizer.events.len() - 1;
@@ -226,11 +227,11 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
fn title(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
match code {
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
- tokenizer.exit(TokenType::Data);
+ tokenizer.exit(Token::Data);
at_break(tokenizer, code, info)
}
Code::Char('\\') => {
diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs
index 28aca34..4d92f8d 100644
--- a/src/construct/thematic_break.rs
+++ b/src/construct/thematic_break.rs
@@ -35,8 +35,8 @@
//!
//! ## Tokens
//!
-//! * [`ThematicBreak`][TokenType::ThematicBreak]
-//! * [`ThematicBreakSequence`][TokenType::ThematicBreakSequence]
+//! * [`ThematicBreak`][Token::ThematicBreak]
+//! * [`ThematicBreakSequence`][Token::ThematicBreakSequence]
//!
//! ## References
//!
@@ -51,7 +51,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN};
-use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::token::Token;
+use crate::tokenizer::{Code, State, StateFnResult, Tokenizer};
/// Type of thematic break.
#[derive(Debug, PartialEq)]
@@ -134,7 +135,7 @@ struct Info {
/// |***
/// ```
pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
- tokenizer.enter(TokenType::ThematicBreak);
+ tokenizer.enter(Token::ThematicBreak);
// To do: allow arbitrary when code (indented) is turned off.
tokenizer.go(space_or_tab_min_max(0, TAB_SIZE - 1), before)(tokenizer, code)
}
@@ -170,13 +171,13 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult
Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')
if info.size >= THEMATIC_BREAK_MARKER_COUNT_MIN =>
{
- tokenizer.exit(TokenType::ThematicBreak);
+ tokenizer.exit(Token::ThematicBreak);
// Feel free to interrupt.
tokenizer.interrupt = false;
(State::Ok, Some(vec![code]))
}
Code::Char(char) if char == info.kind.as_char() => {
- tokenizer.enter(TokenType::ThematicBreakSequence);
+ tokenizer.enter(Token::ThematicBreakSequence);
sequence(tokenizer, code, info)
}
_ => (State::Nok, None),
@@ -198,7 +199,7 @@ fn sequence(tokenizer: &mut Tokenizer, code: Code, mut info: Info) -> StateFnRes
(State::Fn(Box::new(|t, c| sequence(t, c, info))), None)
}
_ => {
- tokenizer.exit(TokenType::ThematicBreakSequence);
+ tokenizer.exit(Token::ThematicBreakSequence);
tokenizer.attempt_opt(space_or_tab(), |t, c| at_break(t, c, info))(tokenizer, code)
}
}