aboutsummaryrefslogtreecommitdiffstats
path: root/src/construct
diff options
context:
space:
mode:
authorLibravatar Titus Wormer <tituswormer@gmail.com>2022-08-11 14:53:42 +0200
committerLibravatar Titus Wormer <tituswormer@gmail.com>2022-08-11 14:53:42 +0200
commitd87dc75500a2e73d9521135054b42c18e6eae987 (patch)
tree4bc3f4a72894db3ec5ed4069841c6be5be69713d /src/construct
parent6eb2f644057f371841fe25330a57ee185f91c7af (diff)
downloadmarkdown-rs-d87dc75500a2e73d9521135054b42c18e6eae987.tar.gz
markdown-rs-d87dc75500a2e73d9521135054b42c18e6eae987.tar.bz2
markdown-rs-d87dc75500a2e73d9521135054b42c18e6eae987.zip
Refactor to move some code to `event.rs`
Diffstat (limited to 'src/construct')
-rw-r--r--src/construct/attention.rs86
-rw-r--r--src/construct/autolink.rs66
-rw-r--r--src/construct/blank_line.rs6
-rw-r--r--src/construct/block_quote.rs32
-rw-r--r--src/construct/character_escape.rs19
-rw-r--r--src/construct/character_reference.rs46
-rw-r--r--src/construct/code_fenced.rs128
-rw-r--r--src/construct/code_indented.rs48
-rw-r--r--src/construct/code_text.rs54
-rw-r--r--src/construct/definition.rs112
-rw-r--r--src/construct/hard_break_escape.rs10
-rw-r--r--src/construct/heading_atx.rs64
-rw-r--r--src/construct/heading_setext.rs46
-rw-r--r--src/construct/html_flow.rs176
-rw-r--r--src/construct/html_text.rs172
-rw-r--r--src/construct/label_end.rs205
-rw-r--r--src/construct/label_start_image.rs18
-rw-r--r--src/construct/label_start_link.rs10
-rw-r--r--src/construct/list.rs108
-rw-r--r--src/construct/paragraph.rs31
-rw-r--r--src/construct/partial_bom.rs12
-rw-r--r--src/construct/partial_data.rs32
-rw-r--r--src/construct/partial_destination.rs42
-rw-r--r--src/construct/partial_label.rs38
-rw-r--r--src/construct/partial_non_lazy_continuation.rs10
-rw-r--r--src/construct/partial_space_or_tab.rs58
-rw-r--r--src/construct/partial_title.rs40
-rw-r--r--src/construct/partial_whitespace.rs40
-rw-r--r--src/construct/thematic_break.rs28
29 files changed, 878 insertions, 859 deletions
diff --git a/src/construct/attention.rs b/src/construct/attention.rs
index 7e873ca..ac2ef25 100644
--- a/src/construct/attention.rs
+++ b/src/construct/attention.rs
@@ -51,9 +51,9 @@
//! [html-em]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-em-element
//! [html-strong]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-strong-element
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{Event, EventType, Point, Tokenizer};
+use crate::event::{Event, Kind, Name, Point};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
use crate::unicode::PUNCTUATION;
use crate::util::slice::Slice;
@@ -120,8 +120,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'*' | b'_') if tokenizer.parse_state.constructs.attention => {
tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
- tokenizer.enter(Token::AttentionSequence);
- State::Retry(Name::AttentionInside)
+ tokenizer.enter(Name::AttentionSequence);
+ State::Retry(StateName::AttentionInside)
}
_ => State::Nok,
}
@@ -137,10 +137,10 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'*' | b'_') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => {
tokenizer.consume();
- State::Next(Name::AttentionInside)
+ State::Next(StateName::AttentionInside)
}
_ => {
- tokenizer.exit(Token::AttentionSequence);
+ tokenizer.exit(Name::AttentionSequence);
tokenizer.register_resolver("attention".to_string(), Box::new(resolve_attention));
tokenizer.tokenize_state.marker = b'\0';
State::Ok
@@ -159,10 +159,10 @@ fn resolve_attention(tokenizer: &mut Tokenizer) {
while start < tokenizer.events.len() {
let enter = &tokenizer.events[start];
- if enter.event_type == EventType::Enter {
+ if enter.kind == Kind::Enter {
balance += 1;
- if enter.token_type == Token::AttentionSequence {
+ if enter.name == Name::AttentionSequence {
let end = start + 1;
let exit = &tokenizer.events[end];
@@ -337,41 +337,41 @@ fn resolve_attention(tokenizer: &mut Tokenizer) {
0,
vec![
Event {
- event_type: EventType::Enter,
- token_type: if take == 1 {
- Token::Emphasis
+ kind: Kind::Enter,
+ name: if take == 1 {
+ Name::Emphasis
} else {
- Token::Strong
+ Name::Strong
},
point: seq_open_enter.clone(),
link: None,
},
Event {
- event_type: EventType::Enter,
- token_type: if take == 1 {
- Token::EmphasisSequence
+ kind: Kind::Enter,
+ name: if take == 1 {
+ Name::EmphasisSequence
} else {
- Token::StrongSequence
+ Name::StrongSequence
},
point: seq_open_enter.clone(),
link: None,
},
Event {
- event_type: EventType::Exit,
- token_type: if take == 1 {
- Token::EmphasisSequence
+ kind: Kind::Exit,
+ name: if take == 1 {
+ Name::EmphasisSequence
} else {
- Token::StrongSequence
+ Name::StrongSequence
},
point: seq_open_exit.clone(),
link: None,
},
Event {
- event_type: EventType::Enter,
- token_type: if take == 1 {
- Token::EmphasisText
+ kind: Kind::Enter,
+ name: if take == 1 {
+ Name::EmphasisText
} else {
- Token::StrongText
+ Name::StrongText
},
point: seq_open_exit.clone(),
link: None,
@@ -384,41 +384,41 @@ fn resolve_attention(tokenizer: &mut Tokenizer) {
0,
vec![
Event {
- event_type: EventType::Exit,
- token_type: if take == 1 {
- Token::EmphasisText
+ kind: Kind::Exit,
+ name: if take == 1 {
+ Name::EmphasisText
} else {
- Token::StrongText
+ Name::StrongText
},
point: seq_close_enter.clone(),
link: None,
},
Event {
- event_type: EventType::Enter,
- token_type: if take == 1 {
- Token::EmphasisSequence
+ kind: Kind::Enter,
+ name: if take == 1 {
+ Name::EmphasisSequence
} else {
- Token::StrongSequence
+ Name::StrongSequence
},
point: seq_close_enter.clone(),
link: None,
},
Event {
- event_type: EventType::Exit,
- token_type: if take == 1 {
- Token::EmphasisSequence
+ kind: Kind::Exit,
+ name: if take == 1 {
+ Name::EmphasisSequence
} else {
- Token::StrongSequence
+ Name::StrongSequence
},
point: seq_close_exit.clone(),
link: None,
},
Event {
- event_type: EventType::Exit,
- token_type: if take == 1 {
- Token::Emphasis
+ kind: Kind::Exit,
+ name: if take == 1 {
+ Name::Emphasis
} else {
- Token::Strong
+ Name::Strong
},
point: seq_close_exit.clone(),
link: None,
@@ -438,8 +438,8 @@ fn resolve_attention(tokenizer: &mut Tokenizer) {
let mut index = 0;
while index < sequences.len() {
let sequence = &sequences[index];
- tokenizer.events[sequence.event_index].token_type = Token::Data;
- tokenizer.events[sequence.event_index + 1].token_type = Token::Data;
+ tokenizer.events[sequence.event_index].name = Name::Data;
+ tokenizer.events[sequence.event_index + 1].name = Name::Data;
index += 1;
}
diff --git a/src/construct/autolink.rs b/src/construct/autolink.rs
index b635d96..c16b7c0 100644
--- a/src/construct/autolink.rs
+++ b/src/construct/autolink.rs
@@ -102,8 +102,8 @@
//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use crate::constant::{AUTOLINK_DOMAIN_SIZE_MAX, AUTOLINK_SCHEME_SIZE_MAX};
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of an autolink.
@@ -117,12 +117,12 @@ use crate::tokenizer::Tokenizer;
pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'<') if tokenizer.parse_state.constructs.autolink => {
- tokenizer.enter(Token::Autolink);
- tokenizer.enter(Token::AutolinkMarker);
+ tokenizer.enter(Name::Autolink);
+ tokenizer.enter(Name::AutolinkMarker);
tokenizer.consume();
- tokenizer.exit(Token::AutolinkMarker);
- tokenizer.enter(Token::AutolinkProtocol);
- State::Next(Name::AutolinkOpen)
+ tokenizer.exit(Name::AutolinkMarker);
+ tokenizer.enter(Name::AutolinkProtocol);
+ State::Next(StateName::AutolinkOpen)
}
_ => State::Nok,
}
@@ -141,9 +141,9 @@ pub fn open(tokenizer: &mut Tokenizer) -> State {
// ASCII alphabetic.
Some(b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::AutolinkSchemeOrEmailAtext)
+ State::Next(StateName::AutolinkSchemeOrEmailAtext)
}
- _ => State::Retry(Name::AutolinkEmailAtext),
+ _ => State::Retry(StateName::AutolinkEmailAtext),
}
}
@@ -161,9 +161,9 @@ pub fn scheme_or_email_atext(tokenizer: &mut Tokenizer) -> State {
Some(b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => {
// Count the previous alphabetical from `open` too.
tokenizer.tokenize_state.size = 1;
- State::Retry(Name::AutolinkSchemeInsideOrEmailAtext)
+ State::Retry(StateName::AutolinkSchemeInsideOrEmailAtext)
}
- _ => State::Retry(Name::AutolinkEmailAtext),
+ _ => State::Retry(StateName::AutolinkEmailAtext),
}
}
@@ -180,7 +180,7 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State {
Some(b':') => {
tokenizer.consume();
tokenizer.tokenize_state.size = 0;
- State::Next(Name::AutolinkUrlInside)
+ State::Next(StateName::AutolinkUrlInside)
}
// ASCII alphanumeric and `+`, `-`, and `.`.
Some(b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z')
@@ -188,11 +188,11 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State {
{
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
- State::Next(Name::AutolinkSchemeInsideOrEmailAtext)
+ State::Next(StateName::AutolinkSchemeInsideOrEmailAtext)
}
_ => {
tokenizer.tokenize_state.size = 0;
- State::Retry(Name::AutolinkEmailAtext)
+ State::Retry(StateName::AutolinkEmailAtext)
}
}
}
@@ -206,18 +206,18 @@ pub fn scheme_inside_or_email_atext(tokenizer: &mut Tokenizer) -> State {
pub fn url_inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'>') => {
- tokenizer.exit(Token::AutolinkProtocol);
- tokenizer.enter(Token::AutolinkMarker);
+ tokenizer.exit(Name::AutolinkProtocol);
+ tokenizer.enter(Name::AutolinkMarker);
tokenizer.consume();
- tokenizer.exit(Token::AutolinkMarker);
- tokenizer.exit(Token::Autolink);
+ tokenizer.exit(Name::AutolinkMarker);
+ tokenizer.exit(Name::Autolink);
State::Ok
}
// ASCII control, space, or `<`.
None | Some(b'\0'..=0x1F | b' ' | b'<' | 0x7F) => State::Nok,
Some(_) => {
tokenizer.consume();
- State::Next(Name::AutolinkUrlInside)
+ State::Next(StateName::AutolinkUrlInside)
}
}
}
@@ -232,7 +232,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'@') => {
tokenizer.consume();
- State::Next(Name::AutolinkEmailAtSignOrDot)
+ State::Next(StateName::AutolinkEmailAtSignOrDot)
}
// ASCII atext.
//
@@ -255,7 +255,7 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State {
b'#'..=b'\'' | b'*' | b'+' | b'-'..=b'9' | b'=' | b'?' | b'A'..=b'Z' | b'^'..=b'~',
) => {
tokenizer.consume();
- State::Next(Name::AutolinkEmailAtext)
+ State::Next(StateName::AutolinkEmailAtext)
}
_ => State::Nok,
}
@@ -270,7 +270,9 @@ pub fn email_atext(tokenizer: &mut Tokenizer) -> State {
pub fn email_at_sign_or_dot(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
// ASCII alphanumeric.
- Some(b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => State::Retry(Name::AutolinkEmailValue),
+ Some(b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => {
+ State::Retry(StateName::AutolinkEmailValue)
+ }
_ => State::Nok,
}
}
@@ -286,22 +288,22 @@ pub fn email_label(tokenizer: &mut Tokenizer) -> State {
Some(b'.') => {
tokenizer.tokenize_state.size = 0;
tokenizer.consume();
- State::Next(Name::AutolinkEmailAtSignOrDot)
+ State::Next(StateName::AutolinkEmailAtSignOrDot)
}
Some(b'>') => {
tokenizer.tokenize_state.size = 0;
let index = tokenizer.events.len();
- tokenizer.exit(Token::AutolinkProtocol);
+ tokenizer.exit(Name::AutolinkProtocol);
// Change the token type.
- tokenizer.events[index - 1].token_type = Token::AutolinkEmail;
- tokenizer.events[index].token_type = Token::AutolinkEmail;
- tokenizer.enter(Token::AutolinkMarker);
+ tokenizer.events[index - 1].name = Name::AutolinkEmail;
+ tokenizer.events[index].name = Name::AutolinkEmail;
+ tokenizer.enter(Name::AutolinkMarker);
tokenizer.consume();
- tokenizer.exit(Token::AutolinkMarker);
- tokenizer.exit(Token::Autolink);
+ tokenizer.exit(Name::AutolinkMarker);
+ tokenizer.exit(Name::Autolink);
State::Ok
}
- _ => State::Retry(Name::AutolinkEmailValue),
+ _ => State::Retry(StateName::AutolinkEmailValue),
}
}
@@ -320,9 +322,9 @@ pub fn email_value(tokenizer: &mut Tokenizer) -> State {
if tokenizer.tokenize_state.size < AUTOLINK_DOMAIN_SIZE_MAX =>
{
let name = if matches!(tokenizer.current, Some(b'-')) {
- Name::AutolinkEmailValue
+ StateName::AutolinkEmailValue
} else {
- Name::AutolinkEmailLabel
+ StateName::AutolinkEmailLabel
};
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
diff --git a/src/construct/blank_line.rs b/src/construct/blank_line.rs
index e8a06e9..2ea986d 100644
--- a/src/construct/blank_line.rs
+++ b/src/construct/blank_line.rs
@@ -33,7 +33,7 @@
//! [flow]: crate::content::flow
use crate::construct::partial_space_or_tab::space_or_tab;
-use crate::state::{Name, State};
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of a blank line.
@@ -50,8 +50,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::BlankLineAfter),
- State::Next(Name::BlankLineAfter),
+ State::Next(StateName::BlankLineAfter),
+ State::Next(StateName::BlankLineAfter),
)
}
diff --git a/src/construct/block_quote.rs b/src/construct/block_quote.rs
index 7b8ce82..a32375e 100644
--- a/src/construct/block_quote.rs
+++ b/src/construct/block_quote.rs
@@ -35,8 +35,8 @@
use crate::constant::TAB_SIZE;
use crate::construct::partial_space_or_tab::space_or_tab_min_max;
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of block quote.
@@ -56,7 +56,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
usize::MAX
},
);
- tokenizer.attempt(name, State::Next(Name::BlockQuoteBefore), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::BlockQuoteBefore), State::Nok)
} else {
State::Nok
}
@@ -71,10 +71,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
pub fn before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'>') => {
- tokenizer.enter(Token::BlockQuote);
- State::Retry(Name::BlockQuoteContBefore)
+ tokenizer.enter(Name::BlockQuote);
+ State::Retry(StateName::BlockQuoteContBefore)
}
- _ => State::Retry(Name::BlockQuoteContBefore),
+ _ => State::Retry(StateName::BlockQuoteContBefore),
}
}
@@ -95,7 +95,11 @@ pub fn cont_start(tokenizer: &mut Tokenizer) -> State {
usize::MAX
},
);
- tokenizer.attempt(name, State::Next(Name::BlockQuoteContBefore), State::Nok)
+ tokenizer.attempt(
+ name,
+ State::Next(StateName::BlockQuoteContBefore),
+ State::Nok,
+ )
}
/// After whitespace, before `>`.
@@ -108,11 +112,11 @@ pub fn cont_start(tokenizer: &mut Tokenizer) -> State {
pub fn cont_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'>') => {
- tokenizer.enter(Token::BlockQuotePrefix);
- tokenizer.enter(Token::BlockQuoteMarker);
+ tokenizer.enter(Name::BlockQuotePrefix);
+ tokenizer.enter(Name::BlockQuoteMarker);
tokenizer.consume();
- tokenizer.exit(Token::BlockQuoteMarker);
- State::Next(Name::BlockQuoteContAfter)
+ tokenizer.exit(Name::BlockQuoteMarker);
+ State::Next(StateName::BlockQuoteContAfter)
}
_ => State::Nok,
}
@@ -128,11 +132,11 @@ pub fn cont_before(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn cont_after(tokenizer: &mut Tokenizer) -> State {
if let Some(b'\t' | b' ') = tokenizer.current {
- tokenizer.enter(Token::SpaceOrTab);
+ tokenizer.enter(Name::SpaceOrTab);
tokenizer.consume();
- tokenizer.exit(Token::SpaceOrTab);
+ tokenizer.exit(Name::SpaceOrTab);
}
- tokenizer.exit(Token::BlockQuotePrefix);
+ tokenizer.exit(Name::BlockQuotePrefix);
State::Ok
}
diff --git a/src/construct/character_escape.rs b/src/construct/character_escape.rs
index c3d5458..0cd7126 100644
--- a/src/construct/character_escape.rs
+++ b/src/construct/character_escape.rs
@@ -33,8 +33,8 @@
//! [character_reference]: crate::construct::character_reference
//! [hard_break_escape]: crate::construct::hard_break_escape
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of a character escape.
@@ -46,11 +46,11 @@ use crate::tokenizer::Tokenizer;
pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\\') if tokenizer.parse_state.constructs.character_escape => {
- tokenizer.enter(Token::CharacterEscape);
- tokenizer.enter(Token::CharacterEscapeMarker);
+ tokenizer.enter(Name::CharacterEscape);
+ tokenizer.enter(Name::CharacterEscapeMarker);
tokenizer.consume();
- tokenizer.exit(Token::CharacterEscapeMarker);
- State::Next(Name::CharacterEscapeInside)
+ tokenizer.exit(Name::CharacterEscapeMarker);
+ State::Next(StateName::CharacterEscapeInside)
}
_ => State::Nok,
}
@@ -62,15 +62,14 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
/// > | a\*b
/// ^
/// ```
-// Name::CharacterEscapeInside
pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
// ASCII punctuation.
Some(b'!'..=b'/' | b':'..=b'@' | b'['..=b'`' | b'{'..=b'~') => {
- tokenizer.enter(Token::CharacterEscapeValue);
+ tokenizer.enter(Name::CharacterEscapeValue);
tokenizer.consume();
- tokenizer.exit(Token::CharacterEscapeValue);
- tokenizer.exit(Token::CharacterEscape);
+ tokenizer.exit(Name::CharacterEscapeValue);
+ tokenizer.exit(Name::CharacterEscape);
State::Ok
}
_ => State::Nok,
diff --git a/src/construct/character_reference.rs b/src/construct/character_reference.rs
index 435c115..0158acf 100644
--- a/src/construct/character_reference.rs
+++ b/src/construct/character_reference.rs
@@ -65,8 +65,8 @@ use crate::constant::{
CHARACTER_REFERENCES, CHARACTER_REFERENCE_DECIMAL_SIZE_MAX,
CHARACTER_REFERENCE_HEXADECIMAL_SIZE_MAX, CHARACTER_REFERENCE_NAMED_SIZE_MAX,
};
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
use crate::util::slice::Slice;
@@ -83,11 +83,11 @@ use crate::util::slice::Slice;
pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'&') if tokenizer.parse_state.constructs.character_reference => {
- tokenizer.enter(Token::CharacterReference);
- tokenizer.enter(Token::CharacterReferenceMarker);
+ tokenizer.enter(Name::CharacterReference);
+ tokenizer.enter(Name::CharacterReferenceMarker);
tokenizer.consume();
- tokenizer.exit(Token::CharacterReferenceMarker);
- State::Next(Name::CharacterReferenceOpen)
+ tokenizer.exit(Name::CharacterReferenceMarker);
+ State::Next(StateName::CharacterReferenceOpen)
}
_ => State::Nok,
}
@@ -104,17 +104,16 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
/// > | a&#x9;b
/// ^
/// ```
-// Name::CharacterReferenceOpen
pub fn open(tokenizer: &mut Tokenizer) -> State {
if let Some(b'#') = tokenizer.current {
- tokenizer.enter(Token::CharacterReferenceMarkerNumeric);
+ tokenizer.enter(Name::CharacterReferenceMarkerNumeric);
tokenizer.consume();
- tokenizer.exit(Token::CharacterReferenceMarkerNumeric);
- State::Next(Name::CharacterReferenceNumeric)
+ tokenizer.exit(Name::CharacterReferenceMarkerNumeric);
+ State::Next(StateName::CharacterReferenceNumeric)
} else {
tokenizer.tokenize_state.marker = b'&';
- tokenizer.enter(Token::CharacterReferenceValue);
- State::Retry(Name::CharacterReferenceValue)
+ tokenizer.enter(Name::CharacterReferenceValue);
+ State::Retry(StateName::CharacterReferenceValue)
}
}
@@ -127,19 +126,18 @@ pub fn open(tokenizer: &mut Tokenizer) -> State {
/// > | a&#x9;b
/// ^
/// ```
-// Name::CharacterReferenceNumeric
pub fn numeric(tokenizer: &mut Tokenizer) -> State {
if let Some(b'x' | b'X') = tokenizer.current {
- tokenizer.enter(Token::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(Name::CharacterReferenceMarkerHexadecimal);
tokenizer.consume();
- tokenizer.exit(Token::CharacterReferenceMarkerHexadecimal);
- tokenizer.enter(Token::CharacterReferenceValue);
+ tokenizer.exit(Name::CharacterReferenceMarkerHexadecimal);
+ tokenizer.enter(Name::CharacterReferenceValue);
tokenizer.tokenize_state.marker = b'x';
- State::Next(Name::CharacterReferenceValue)
+ State::Next(StateName::CharacterReferenceValue)
} else {
- tokenizer.enter(Token::CharacterReferenceValue);
+ tokenizer.enter(Name::CharacterReferenceValue);
tokenizer.tokenize_state.marker = b'#';
- State::Retry(Name::CharacterReferenceValue)
+ State::Retry(StateName::CharacterReferenceValue)
}
}
@@ -176,11 +174,11 @@ pub fn value(tokenizer: &mut Tokenizer) -> State {
}
}
- tokenizer.exit(Token::CharacterReferenceValue);
- tokenizer.enter(Token::CharacterReferenceMarkerSemi);
+ tokenizer.exit(Name::CharacterReferenceValue);
+ tokenizer.enter(Name::CharacterReferenceMarkerSemi);
tokenizer.consume();
- tokenizer.exit(Token::CharacterReferenceMarkerSemi);
- tokenizer.exit(Token::CharacterReference);
+ tokenizer.exit(Name::CharacterReferenceMarkerSemi);
+ tokenizer.exit(Name::CharacterReference);
tokenizer.tokenize_state.marker = 0;
tokenizer.tokenize_state.size = 0;
return State::Ok;
@@ -203,7 +201,7 @@ pub fn value(tokenizer: &mut Tokenizer) -> State {
if tokenizer.tokenize_state.size < max && test(&byte) {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
- return State::Next(Name::CharacterReferenceValue);
+ return State::Next(StateName::CharacterReferenceValue);
}
}
diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index 0ce8d02..9488f62 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -103,9 +103,9 @@
use crate::constant::{CODE_FENCED_SEQUENCE_SIZE_MIN, TAB_SIZE};
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{ContentType, Tokenizer};
+use crate::event::{Content, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
use crate::util::slice::{Position, Slice};
/// Start of fenced code.
@@ -118,8 +118,8 @@ use crate::util::slice::{Position, Slice};
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.parse_state.constructs.code_fenced {
- tokenizer.enter(Token::CodeFenced);
- tokenizer.enter(Token::CodeFencedFence);
+ tokenizer.enter(Name::CodeFenced);
+ tokenizer.enter(Name::CodeFencedFence);
let name = space_or_tab_min_max(
tokenizer,
0,
@@ -131,7 +131,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
);
tokenizer.attempt(
name,
- State::Next(Name::CodeFencedBeforeSequenceOpen),
+ State::Next(StateName::CodeFencedBeforeSequenceOpen),
State::Nok,
)
} else {
@@ -152,7 +152,7 @@ pub fn before_sequence_open(tokenizer: &mut Tokenizer) -> State {
let mut prefix = 0;
if let Some(event) = tail {
- if event.token_type == Token::SpaceOrTab {
+ if event.name == Name::SpaceOrTab {
prefix = Slice::from_position(
tokenizer.parse_state.bytes,
&Position::from_exit_event(&tokenizer.events, tokenizer.events.len() - 1),
@@ -164,8 +164,8 @@ pub fn before_sequence_open(tokenizer: &mut Tokenizer) -> State {
if let Some(b'`' | b'~') = tokenizer.current {
tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
tokenizer.tokenize_state.size_c = prefix;
- tokenizer.enter(Token::CodeFencedFenceSequence);
- State::Retry(Name::CodeFencedSequenceOpen)
+ tokenizer.enter(Name::CodeFencedFenceSequence);
+ State::Retry(StateName::CodeFencedSequenceOpen)
} else {
State::Nok
}
@@ -184,15 +184,15 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State {
Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
- State::Next(Name::CodeFencedSequenceOpen)
+ State::Next(StateName::CodeFencedSequenceOpen)
}
_ if tokenizer.tokenize_state.size >= CODE_FENCED_SEQUENCE_SIZE_MIN => {
- tokenizer.exit(Token::CodeFencedFenceSequence);
+ tokenizer.exit(Name::CodeFencedFenceSequence);
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::CodeFencedInfoBefore),
- State::Next(Name::CodeFencedInfoBefore),
+ State::Next(StateName::CodeFencedInfoBefore),
+ State::Next(StateName::CodeFencedInfoBefore),
)
}
_ => {
@@ -215,19 +215,19 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State {
pub fn info_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::CodeFencedFence);
+ tokenizer.exit(Name::CodeFencedFence);
// Do not form containers.
tokenizer.concrete = true;
tokenizer.check(
- Name::NonLazyContinuationStart,
- State::Next(Name::CodeFencedAtNonLazyBreak),
- State::Next(Name::CodeFencedAfter),
+ StateName::NonLazyContinuationStart,
+ State::Next(StateName::CodeFencedAtNonLazyBreak),
+ State::Next(StateName::CodeFencedAfter),
)
}
_ => {
- tokenizer.enter(Token::CodeFencedFenceInfo);
- tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
- State::Retry(Name::CodeFencedInfo)
+ tokenizer.enter(Name::CodeFencedFenceInfo);
+ tokenizer.enter_with_content(Name::Data, Some(Content::String));
+ State::Retry(StateName::CodeFencedInfo)
}
}
}
@@ -243,18 +243,18 @@ pub fn info_before(tokenizer: &mut Tokenizer) -> State {
pub fn info(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::Data);
- tokenizer.exit(Token::CodeFencedFenceInfo);
- State::Retry(Name::CodeFencedInfoBefore)
+ tokenizer.exit(Name::Data);
+ tokenizer.exit(Name::CodeFencedFenceInfo);
+ State::Retry(StateName::CodeFencedInfoBefore)
}
Some(b'\t' | b' ') => {
- tokenizer.exit(Token::Data);
- tokenizer.exit(Token::CodeFencedFenceInfo);
+ tokenizer.exit(Name::Data);
+ tokenizer.exit(Name::CodeFencedFenceInfo);
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::CodeFencedMetaBefore),
- State::Next(Name::CodeFencedMetaBefore),
+ State::Next(StateName::CodeFencedMetaBefore),
+ State::Next(StateName::CodeFencedMetaBefore),
)
}
Some(b'`') if tokenizer.tokenize_state.marker == b'`' => {
@@ -266,7 +266,7 @@ pub fn info(tokenizer: &mut Tokenizer) -> State {
}
Some(_) => {
tokenizer.consume();
- State::Next(Name::CodeFencedInfo)
+ State::Next(StateName::CodeFencedInfo)
}
}
}
@@ -281,11 +281,11 @@ pub fn info(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn meta_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- None | Some(b'\n') => State::Retry(Name::CodeFencedInfoBefore),
+ None | Some(b'\n') => State::Retry(StateName::CodeFencedInfoBefore),
_ => {
- tokenizer.enter(Token::CodeFencedFenceMeta);
- tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
- State::Retry(Name::CodeFencedMeta)
+ tokenizer.enter(Name::CodeFencedFenceMeta);
+ tokenizer.enter_with_content(Name::Data, Some(Content::String));
+ State::Retry(StateName::CodeFencedMeta)
}
}
}
@@ -301,9 +301,9 @@ pub fn meta_before(tokenizer: &mut Tokenizer) -> State {
pub fn meta(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::Data);
- tokenizer.exit(Token::CodeFencedFenceMeta);
- State::Retry(Name::CodeFencedInfoBefore)
+ tokenizer.exit(Name::Data);
+ tokenizer.exit(Name::CodeFencedFenceMeta);
+ State::Retry(StateName::CodeFencedInfoBefore)
}
Some(b'`') if tokenizer.tokenize_state.marker == b'`' => {
tokenizer.concrete = false;
@@ -314,7 +314,7 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State {
}
_ => {
tokenizer.consume();
- State::Next(Name::CodeFencedMeta)
+ State::Next(StateName::CodeFencedMeta)
}
}
}
@@ -330,9 +330,9 @@ pub fn meta(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn at_non_lazy_break(tokenizer: &mut Tokenizer) -> State {
tokenizer.attempt(
- Name::CodeFencedCloseBefore,
- State::Next(Name::CodeFencedAfter),
- State::Next(Name::CodeFencedContentBefore),
+ StateName::CodeFencedCloseBefore,
+ State::Next(StateName::CodeFencedAfter),
+ State::Next(StateName::CodeFencedContentBefore),
)
}
@@ -347,10 +347,10 @@ pub fn at_non_lazy_break(tokenizer: &mut Tokenizer) -> State {
pub fn close_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::CodeFencedCloseStart)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::CodeFencedCloseStart)
}
_ => unreachable!("expected eol"),
}
@@ -365,7 +365,7 @@ pub fn close_before(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn close_start(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::CodeFencedFence);
+ tokenizer.enter(Name::CodeFencedFence);
let name = space_or_tab_min_max(
tokenizer,
0,
@@ -377,7 +377,7 @@ pub fn close_start(tokenizer: &mut Tokenizer) -> State {
);
tokenizer.attempt(
name,
- State::Next(Name::CodeFencedBeforeSequenceClose),
+ State::Next(StateName::CodeFencedBeforeSequenceClose),
State::Nok,
)
}
@@ -393,8 +393,8 @@ pub fn close_start(tokenizer: &mut Tokenizer) -> State {
pub fn before_sequence_close(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => {
- tokenizer.enter(Token::CodeFencedFenceSequence);
- State::Retry(Name::CodeFencedSequenceClose)
+ tokenizer.enter(Name::CodeFencedFenceSequence);
+ State::Retry(StateName::CodeFencedSequenceClose)
}
_ => State::Nok,
}
@@ -413,18 +413,18 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State {
Some(b'`' | b'~') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => {
tokenizer.tokenize_state.size_b += 1;
tokenizer.consume();
- State::Next(Name::CodeFencedSequenceClose)
+ State::Next(StateName::CodeFencedSequenceClose)
}
_ if tokenizer.tokenize_state.size_b >= CODE_FENCED_SEQUENCE_SIZE_MIN
&& tokenizer.tokenize_state.size_b >= tokenizer.tokenize_state.size =>
{
tokenizer.tokenize_state.size_b = 0;
- tokenizer.exit(Token::CodeFencedFenceSequence);
+ tokenizer.exit(Name::CodeFencedFenceSequence);
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::CodeFencedAfterSequenceClose),
- State::Next(Name::CodeFencedAfterSequenceClose),
+ State::Next(StateName::CodeFencedAfterSequenceClose),
+ State::Next(StateName::CodeFencedAfterSequenceClose),
)
}
_ => {
@@ -445,7 +445,7 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State {
pub fn sequence_close_after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::CodeFencedFence);
+ tokenizer.exit(Name::CodeFencedFence);
State::Ok
}
_ => State::Nok,
@@ -461,10 +461,10 @@ pub fn sequence_close_after(tokenizer: &mut Tokenizer) -> State {
/// | ~~~
/// ```
pub fn content_before(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::CodeFencedContentStart)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::CodeFencedContentStart)
}
/// Before code content, definitely not before a closing fence.
///
@@ -478,7 +478,7 @@ pub fn content_start(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_min_max(tokenizer, 0, tokenizer.tokenize_state.size_c);
tokenizer.attempt(
name,
- State::Next(Name::CodeFencedBeforeContentChunk),
+ State::Next(StateName::CodeFencedBeforeContentChunk),
State::Nok,
)
}
@@ -494,13 +494,13 @@ pub fn content_start(tokenizer: &mut Tokenizer) -> State {
pub fn before_content_chunk(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => tokenizer.check(
- Name::NonLazyContinuationStart,
- State::Next(Name::CodeFencedAtNonLazyBreak),
- State::Next(Name::CodeFencedAfter),
+ StateName::NonLazyContinuationStart,
+ State::Next(StateName::CodeFencedAtNonLazyBreak),
+ State::Next(StateName::CodeFencedAfter),
),
_ => {
- tokenizer.enter(Token::CodeFlowChunk);
- State::Retry(Name::CodeFencedContentChunk)
+ tokenizer.enter(Name::CodeFlowChunk);
+ State::Retry(StateName::CodeFencedContentChunk)
}
}
}
@@ -516,12 +516,12 @@ pub fn before_content_chunk(tokenizer: &mut Tokenizer) -> State {
pub fn content_chunk(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::CodeFlowChunk);
- State::Retry(Name::CodeFencedBeforeContentChunk)
+ tokenizer.exit(Name::CodeFlowChunk);
+ State::Retry(StateName::CodeFencedBeforeContentChunk)
}
_ => {
tokenizer.consume();
- State::Next(Name::CodeFencedContentChunk)
+ State::Next(StateName::CodeFencedContentChunk)
}
}
}
@@ -535,7 +535,7 @@ pub fn content_chunk(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.exit(Token::CodeFenced);
+ tokenizer.exit(Name::CodeFenced);
tokenizer.tokenize_state.marker = 0;
tokenizer.tokenize_state.size_c = 0;
tokenizer.tokenize_state.size = 0;
diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index f442f27..7297759 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -47,8 +47,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::TAB_SIZE;
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of code (indented).
@@ -64,9 +64,13 @@ use crate::tokenizer::Tokenizer;
pub fn start(tokenizer: &mut Tokenizer) -> State {
// Do not interrupt paragraphs.
if !tokenizer.interrupt && tokenizer.parse_state.constructs.code_indented {
- tokenizer.enter(Token::CodeIndented);
+ tokenizer.enter(Name::CodeIndented);
let name = space_or_tab_min_max(tokenizer, TAB_SIZE, TAB_SIZE);
- tokenizer.attempt(name, State::Next(Name::CodeIndentedAtBreak), State::Nok)
+ tokenizer.attempt(
+ name,
+ State::Next(StateName::CodeIndentedAtBreak),
+ State::Nok,
+ )
} else {
State::Nok
}
@@ -80,15 +84,15 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn at_break(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- None => State::Retry(Name::CodeIndentedAfter),
+ None => State::Retry(StateName::CodeIndentedAfter),
Some(b'\n') => tokenizer.attempt(
- Name::CodeIndentedFurtherStart,
- State::Next(Name::CodeIndentedAtBreak),
- State::Next(Name::CodeIndentedAfter),
+ StateName::CodeIndentedFurtherStart,
+ State::Next(StateName::CodeIndentedAtBreak),
+ State::Next(StateName::CodeIndentedAfter),
),
_ => {
- tokenizer.enter(Token::CodeFlowChunk);
- State::Retry(Name::CodeIndentedInside)
+ tokenizer.enter(Name::CodeFlowChunk);
+ State::Retry(StateName::CodeIndentedInside)
}
}
}
@@ -102,12 +106,12 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::CodeFlowChunk);
- State::Retry(Name::CodeIndentedAtBreak)
+ tokenizer.exit(Name::CodeFlowChunk);
+ State::Retry(StateName::CodeIndentedAtBreak)
}
_ => {
tokenizer.consume();
- State::Next(Name::CodeIndentedInside)
+ State::Next(StateName::CodeIndentedInside)
}
}
}
@@ -119,7 +123,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.exit(Token::CodeIndented);
+ tokenizer.exit(Name::CodeIndented);
// Feel free to interrupt.
tokenizer.interrupt = false;
State::Ok
@@ -135,17 +139,17 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
pub fn further_start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') if !tokenizer.lazy => {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::CodeIndentedFurtherStart)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::CodeIndentedFurtherStart)
}
_ if !tokenizer.lazy => {
let name = space_or_tab_min_max(tokenizer, TAB_SIZE, TAB_SIZE);
tokenizer.attempt(
name,
- State::Next(Name::CodeIndentedFurtherEnd),
- State::Next(Name::CodeIndentedFurtherBegin),
+ State::Next(StateName::CodeIndentedFurtherEnd),
+ State::Next(StateName::CodeIndentedFurtherBegin),
)
}
_ => State::Nok,
@@ -174,8 +178,8 @@ pub fn further_begin(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::CodeIndentedFurtherAfter),
- State::Next(Name::CodeIndentedFurtherAfter),
+ State::Next(StateName::CodeIndentedFurtherAfter),
+ State::Next(StateName::CodeIndentedFurtherAfter),
)
}
@@ -188,7 +192,7 @@ pub fn further_begin(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn further_after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- Some(b'\n') => State::Retry(Name::CodeIndentedFurtherStart),
+ Some(b'\n') => State::Retry(StateName::CodeIndentedFurtherStart),
_ => State::Nok,
}
}
diff --git a/src/construct/code_text.rs b/src/construct/code_text.rs
index f626cfb..f48f63c 100644
--- a/src/construct/code_text.rs
+++ b/src/construct/code_text.rs
@@ -83,8 +83,8 @@
//! [code_fenced]: crate::construct::code_fenced
//! [html-code]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-code-element
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of code (text).
@@ -101,12 +101,12 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.parse_state.constructs.code_text
&& (tokenizer.previous != Some(b'`')
|| (!tokenizer.events.is_empty()
- && tokenizer.events[tokenizer.events.len() - 1].token_type
- == Token::CharacterEscape)) =>
+ && tokenizer.events[tokenizer.events.len() - 1].name
+ == Name::CharacterEscape)) =>
{
- tokenizer.enter(Token::CodeText);
- tokenizer.enter(Token::CodeTextSequence);
- State::Retry(Name::CodeTextSequenceOpen)
+ tokenizer.enter(Name::CodeText);
+ tokenizer.enter(Name::CodeTextSequence);
+ State::Retry(StateName::CodeTextSequenceOpen)
}
_ => State::Nok,
}
@@ -122,10 +122,10 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State {
if let Some(b'`') = tokenizer.current {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
- State::Next(Name::CodeTextSequenceOpen)
+ State::Next(StateName::CodeTextSequenceOpen)
} else {
- tokenizer.exit(Token::CodeTextSequence);
- State::Retry(Name::CodeTextBetween)
+ tokenizer.exit(Name::CodeTextSequence);
+ State::Retry(StateName::CodeTextBetween)
}
}
@@ -142,18 +142,18 @@ pub fn between(tokenizer: &mut Tokenizer) -> State {
State::Nok
}
Some(b'\n') => {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::CodeTextBetween)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::CodeTextBetween)
}
Some(b'`') => {
- tokenizer.enter(Token::CodeTextSequence);
- State::Retry(Name::CodeTextSequenceClose)
+ tokenizer.enter(Name::CodeTextSequence);
+ State::Retry(StateName::CodeTextSequenceClose)
}
_ => {
- tokenizer.enter(Token::CodeTextData);
- State::Retry(Name::CodeTextData)
+ tokenizer.enter(Name::CodeTextData);
+ State::Retry(StateName::CodeTextData)
}
}
}
@@ -167,12 +167,12 @@ pub fn between(tokenizer: &mut Tokenizer) -> State {
pub fn data(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n' | b'`') => {
- tokenizer.exit(Token::CodeTextData);
- State::Retry(Name::CodeTextBetween)
+ tokenizer.exit(Name::CodeTextData);
+ State::Retry(StateName::CodeTextBetween)
}
_ => {
tokenizer.consume();
- State::Next(Name::CodeTextData)
+ State::Next(StateName::CodeTextData)
}
}
}
@@ -188,23 +188,23 @@ pub fn sequence_close(tokenizer: &mut Tokenizer) -> State {
Some(b'`') => {
tokenizer.tokenize_state.size_b += 1;
tokenizer.consume();
- State::Next(Name::CodeTextSequenceClose)
+ State::Next(StateName::CodeTextSequenceClose)
}
_ => {
if tokenizer.tokenize_state.size == tokenizer.tokenize_state.size_b {
- tokenizer.exit(Token::CodeTextSequence);
- tokenizer.exit(Token::CodeText);
+ tokenizer.exit(Name::CodeTextSequence);
+ tokenizer.exit(Name::CodeText);
tokenizer.tokenize_state.size = 0;
tokenizer.tokenize_state.size_b = 0;
State::Ok
} else {
let index = tokenizer.events.len();
- tokenizer.exit(Token::CodeTextSequence);
+ tokenizer.exit(Name::CodeTextSequence);
// More or less accents: mark as data.
- tokenizer.events[index - 1].token_type = Token::CodeTextData;
- tokenizer.events[index].token_type = Token::CodeTextData;
+ tokenizer.events[index - 1].name = Name::CodeTextData;
+ tokenizer.events[index].name = Name::CodeTextData;
tokenizer.tokenize_state.size_b = 0;
- State::Retry(Name::CodeTextBetween)
+ State::Retry(StateName::CodeTextBetween)
}
}
}
diff --git a/src/construct/definition.rs b/src/construct/definition.rs
index 394375f..2533a1c 100644
--- a/src/construct/definition.rs
+++ b/src/construct/definition.rs
@@ -94,8 +94,8 @@
//! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_eol};
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
use crate::util::skip::opt_back as skip_opt_back;
@@ -112,19 +112,19 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
&& tokenizer.events[skip_opt_back(
&tokenizer.events,
tokenizer.events.len() - 1,
- &[Token::LineEnding, Token::SpaceOrTab],
+ &[Name::LineEnding, Name::SpaceOrTab],
)]
- .token_type
- == Token::Definition);
+ .name
+ == Name::Definition);
if possible && tokenizer.parse_state.constructs.definition {
- tokenizer.enter(Token::Definition);
+ tokenizer.enter(Name::Definition);
// Note: arbitrary whitespace allowed even if code (indented) is on.
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::DefinitionBefore),
- State::Next(Name::DefinitionBefore),
+ State::Next(StateName::DefinitionBefore),
+ State::Next(StateName::DefinitionBefore),
)
} else {
State::Nok
@@ -140,12 +140,12 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
pub fn before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'[') => {
- tokenizer.tokenize_state.token_1 = Token::DefinitionLabel;
- tokenizer.tokenize_state.token_2 = Token::DefinitionLabelMarker;
- tokenizer.tokenize_state.token_3 = Token::DefinitionLabelString;
+ tokenizer.tokenize_state.token_1 = Name::DefinitionLabel;
+ tokenizer.tokenize_state.token_2 = Name::DefinitionLabelMarker;
+ tokenizer.tokenize_state.token_3 = Name::DefinitionLabelString;
tokenizer.attempt(
- Name::LabelStart,
- State::Next(Name::DefinitionLabelAfter),
+ StateName::LabelStart,
+ State::Next(StateName::DefinitionLabelAfter),
State::Nok,
)
}
@@ -160,16 +160,16 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn label_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
match tokenizer.current {
Some(b':') => {
- tokenizer.enter(Token::DefinitionMarker);
+ tokenizer.enter(Name::DefinitionMarker);
tokenizer.consume();
- tokenizer.exit(Token::DefinitionMarker);
- State::Next(Name::DefinitionMarkerAfter)
+ tokenizer.exit(Name::DefinitionMarker);
+ State::Next(StateName::DefinitionMarkerAfter)
}
_ => State::Nok,
}
@@ -185,8 +185,8 @@ pub fn marker_after(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_eol(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::DefinitionDestinationBefore),
- State::Next(Name::DefinitionDestinationBefore),
+ State::Next(StateName::DefinitionDestinationBefore),
+ State::Next(StateName::DefinitionDestinationBefore),
)
}
@@ -197,16 +197,16 @@ pub fn marker_after(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn destination_before(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::DefinitionDestination;
- tokenizer.tokenize_state.token_2 = Token::DefinitionDestinationLiteral;
- tokenizer.tokenize_state.token_3 = Token::DefinitionDestinationLiteralMarker;
- tokenizer.tokenize_state.token_4 = Token::DefinitionDestinationRaw;
- tokenizer.tokenize_state.token_5 = Token::DefinitionDestinationString;
+ tokenizer.tokenize_state.token_1 = Name::DefinitionDestination;
+ tokenizer.tokenize_state.token_2 = Name::DefinitionDestinationLiteral;
+ tokenizer.tokenize_state.token_3 = Name::DefinitionDestinationLiteralMarker;
+ tokenizer.tokenize_state.token_4 = Name::DefinitionDestinationRaw;
+ tokenizer.tokenize_state.token_5 = Name::DefinitionDestinationString;
tokenizer.tokenize_state.size_b = usize::MAX;
tokenizer.attempt(
- Name::DestinationStart,
- State::Next(Name::DefinitionDestinationAfter),
- State::Next(Name::DefinitionDestinationMissing),
+ StateName::DestinationStart,
+ State::Next(StateName::DefinitionDestinationAfter),
+ State::Next(StateName::DefinitionDestinationMissing),
)
}
@@ -217,26 +217,26 @@ pub fn destination_before(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn destination_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
- tokenizer.tokenize_state.token_4 = Token::Data;
- tokenizer.tokenize_state.token_5 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
+ tokenizer.tokenize_state.token_4 = Name::Data;
+ tokenizer.tokenize_state.token_5 = Name::Data;
tokenizer.tokenize_state.size_b = 0;
tokenizer.attempt(
- Name::DefinitionTitleBefore,
- State::Next(Name::DefinitionAfter),
- State::Next(Name::DefinitionAfter),
+ StateName::DefinitionTitleBefore,
+ State::Next(StateName::DefinitionAfter),
+ State::Next(StateName::DefinitionAfter),
)
}
/// Without destination.
pub fn destination_missing(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
- tokenizer.tokenize_state.token_4 = Token::Data;
- tokenizer.tokenize_state.token_5 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
+ tokenizer.tokenize_state.token_4 = Name::Data;
+ tokenizer.tokenize_state.token_5 = Name::Data;
tokenizer.tokenize_state.size_b = 0;
State::Nok
}
@@ -253,8 +253,8 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::DefinitionAfterWhitespace),
- State::Next(Name::DefinitionAfterWhitespace),
+ State::Next(StateName::DefinitionAfterWhitespace),
+ State::Next(StateName::DefinitionAfterWhitespace),
)
}
@@ -269,7 +269,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
pub fn after_whitespace(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::Definition);
+ tokenizer.exit(Name::Definition);
// You’d be interrupting.
tokenizer.interrupt = true;
State::Ok
@@ -290,7 +290,7 @@ pub fn title_before(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_eol(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::DefinitionTitleBeforeMarker),
+ State::Next(StateName::DefinitionTitleBeforeMarker),
State::Nok,
)
}
@@ -303,12 +303,12 @@ pub fn title_before(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn title_before_marker(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::DefinitionTitle;
- tokenizer.tokenize_state.token_2 = Token::DefinitionTitleMarker;
- tokenizer.tokenize_state.token_3 = Token::DefinitionTitleString;
+ tokenizer.tokenize_state.token_1 = Name::DefinitionTitle;
+ tokenizer.tokenize_state.token_2 = Name::DefinitionTitleMarker;
+ tokenizer.tokenize_state.token_3 = Name::DefinitionTitleString;
tokenizer.attempt(
- Name::TitleStart,
- State::Next(Name::DefinitionTitleAfter),
+ StateName::TitleStart,
+ State::Next(StateName::DefinitionTitleAfter),
State::Nok,
)
}
@@ -320,14 +320,14 @@ pub fn title_before_marker(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn title_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::DefinitionTitleAfterOptionalWhitespace),
- State::Next(Name::DefinitionTitleAfterOptionalWhitespace),
+ State::Next(StateName::DefinitionTitleAfterOptionalWhitespace),
+ State::Next(StateName::DefinitionTitleAfterOptionalWhitespace),
)
}
diff --git a/src/construct/hard_break_escape.rs b/src/construct/hard_break_escape.rs
index 482648f..9af5b83 100644
--- a/src/construct/hard_break_escape.rs
+++ b/src/construct/hard_break_escape.rs
@@ -39,8 +39,8 @@
//! [hard_break_trailing]: crate::construct::partial_whitespace
//! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of a hard break (escape).
@@ -53,9 +53,9 @@ use crate::tokenizer::Tokenizer;
pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\\') if tokenizer.parse_state.constructs.hard_break_escape => {
- tokenizer.enter(Token::HardBreakEscape);
+ tokenizer.enter(Name::HardBreakEscape);
tokenizer.consume();
- State::Next(Name::HardBreakEscapeAfter)
+ State::Next(StateName::HardBreakEscapeAfter)
}
_ => State::Nok,
}
@@ -71,7 +71,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
pub fn after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => {
- tokenizer.exit(Token::HardBreakEscape);
+ tokenizer.exit(Name::HardBreakEscape);
State::Ok
}
_ => State::Nok,
diff --git a/src/construct/heading_atx.rs b/src/construct/heading_atx.rs
index 12bc5b1..e856ac3 100644
--- a/src/construct/heading_atx.rs
+++ b/src/construct/heading_atx.rs
@@ -56,9 +56,9 @@
use crate::constant::{HEADING_ATX_OPENING_FENCE_SIZE_MAX, TAB_SIZE};
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{ContentType, Event, EventType, Tokenizer};
+use crate::event::{Content, Event, Kind, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
/// Start of a heading (atx).
///
@@ -68,7 +68,7 @@ use crate::tokenizer::{ContentType, Event, EventType, Tokenizer};
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.parse_state.constructs.heading_atx {
- tokenizer.enter(Token::HeadingAtx);
+ tokenizer.enter(Name::HeadingAtx);
let name = space_or_tab_min_max(
tokenizer,
0,
@@ -78,7 +78,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
usize::MAX
},
);
- tokenizer.attempt(name, State::Next(Name::HeadingAtxBefore), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::HeadingAtxBefore), State::Nok)
} else {
State::Nok
}
@@ -92,8 +92,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn before(tokenizer: &mut Tokenizer) -> State {
if Some(b'#') == tokenizer.current {
- tokenizer.enter(Token::HeadingAtxSequence);
- State::Retry(Name::HeadingAtxSequenceOpen)
+ tokenizer.enter(Name::HeadingAtxSequence);
+ State::Retry(StateName::HeadingAtxSequenceOpen)
} else {
State::Nok
}
@@ -109,19 +109,19 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') if tokenizer.tokenize_state.size > 0 => {
tokenizer.tokenize_state.size = 0;
- tokenizer.exit(Token::HeadingAtxSequence);
- State::Retry(Name::HeadingAtxAtBreak)
+ tokenizer.exit(Name::HeadingAtxSequence);
+ State::Retry(StateName::HeadingAtxAtBreak)
}
Some(b'#') if tokenizer.tokenize_state.size < HEADING_ATX_OPENING_FENCE_SIZE_MAX => {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
- State::Next(Name::HeadingAtxSequenceOpen)
+ State::Next(StateName::HeadingAtxSequenceOpen)
}
_ if tokenizer.tokenize_state.size > 0 => {
tokenizer.tokenize_state.size = 0;
- tokenizer.exit(Token::HeadingAtxSequence);
+ tokenizer.exit(Name::HeadingAtxSequence);
let name = space_or_tab(tokenizer);
- tokenizer.attempt(name, State::Next(Name::HeadingAtxAtBreak), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::HeadingAtxAtBreak), State::Nok)
}
_ => {
tokenizer.tokenize_state.size = 0;
@@ -139,7 +139,7 @@ pub fn sequence_open(tokenizer: &mut Tokenizer) -> State {
pub fn at_break(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::HeadingAtx);
+ tokenizer.exit(Name::HeadingAtx);
tokenizer.register_resolver("heading_atx".to_string(), Box::new(resolve));
// Feel free to interrupt.
tokenizer.interrupt = false;
@@ -147,15 +147,15 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
}
Some(b'\t' | b' ') => {
let name = space_or_tab(tokenizer);
- tokenizer.attempt(name, State::Next(Name::HeadingAtxAtBreak), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::HeadingAtxAtBreak), State::Nok)
}
Some(b'#') => {
- tokenizer.enter(Token::HeadingAtxSequence);
- State::Retry(Name::HeadingAtxSequenceFurther)
+ tokenizer.enter(Name::HeadingAtxSequence);
+ State::Retry(StateName::HeadingAtxSequenceFurther)
}
Some(_) => {
- tokenizer.enter_with_content(Token::Data, Some(ContentType::Text));
- State::Retry(Name::HeadingAtxData)
+ tokenizer.enter_with_content(Name::Data, Some(Content::Text));
+ State::Retry(StateName::HeadingAtxData)
}
}
}
@@ -171,10 +171,10 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
pub fn sequence_further(tokenizer: &mut Tokenizer) -> State {
if let Some(b'#') = tokenizer.current {
tokenizer.consume();
- State::Next(Name::HeadingAtxSequenceFurther)
+ State::Next(StateName::HeadingAtxSequenceFurther)
} else {
- tokenizer.exit(Token::HeadingAtxSequence);
- State::Retry(Name::HeadingAtxAtBreak)
+ tokenizer.exit(Name::HeadingAtxSequence);
+ State::Retry(StateName::HeadingAtxAtBreak)
}
}
@@ -188,12 +188,12 @@ pub fn data(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
// Note: `#` for closing sequence must be preceded by whitespace, otherwise it’s just text.
None | Some(b'\t' | b'\n' | b' ') => {
- tokenizer.exit(Token::Data);
- State::Retry(Name::HeadingAtxAtBreak)
+ tokenizer.exit(Name::Data);
+ State::Retry(StateName::HeadingAtxAtBreak)
}
_ => {
tokenizer.consume();
- State::Next(Name::HeadingAtxData)
+ State::Next(StateName::HeadingAtxData)
}
}
}
@@ -208,8 +208,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.token_type == Token::HeadingAtx {
- if event.event_type == EventType::Enter {
+ if event.name == Name::HeadingAtx {
+ if event.kind == Kind::Enter {
heading_inside = true;
} else {
if let Some(start) = data_start {
@@ -220,8 +220,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
start,
0,
vec![Event {
- event_type: EventType::Enter,
- token_type: Token::HeadingAtxText,
+ kind: Kind::Enter,
+ name: Name::HeadingAtxText,
point: tokenizer.events[start].point.clone(),
link: None,
}],
@@ -234,8 +234,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
end + 1,
0,
vec![Event {
- event_type: EventType::Exit,
- token_type: Token::HeadingAtxText,
+ kind: Kind::Exit,
+ name: Name::HeadingAtxText,
point: tokenizer.events[end].point.clone(),
link: None,
}],
@@ -246,8 +246,8 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
data_start = None;
data_end = None;
}
- } else if heading_inside && event.token_type == Token::Data {
- if event.event_type == EventType::Enter {
+ } else if heading_inside && event.name == Name::Data {
+ if event.kind == Kind::Enter {
if data_start.is_none() {
data_start = Some(index);
}
diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index 8b45fff..3a24f9f 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -59,9 +59,9 @@
use crate::constant::TAB_SIZE;
use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{EventType, Tokenizer};
+use crate::event::{Kind, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
use crate::util::skip::opt_back as skip_opt_back;
/// At a line ending, presumably an underline.
@@ -79,10 +79,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
&& tokenizer.events[skip_opt_back(
&tokenizer.events,
tokenizer.events.len() - 1,
- &[Token::LineEnding, Token::SpaceOrTab],
+ &[Name::LineEnding, Name::SpaceOrTab],
)]
- .token_type
- == Token::Paragraph)
+ .name
+ == Name::Paragraph)
{
let name = space_or_tab_min_max(
tokenizer,
@@ -94,7 +94,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
},
);
- tokenizer.attempt(name, State::Next(Name::HeadingSetextBefore), State::Nok)
+ tokenizer.attempt(
+ name,
+ State::Next(StateName::HeadingSetextBefore),
+ State::Nok,
+ )
} else {
State::Nok
}
@@ -111,8 +115,8 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'-' | b'=') => {
tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
- tokenizer.enter(Token::HeadingSetextUnderline);
- State::Retry(Name::HeadingSetextInside)
+ tokenizer.enter(Name::HeadingSetextUnderline);
+ State::Retry(StateName::HeadingSetextInside)
}
_ => State::Nok,
}
@@ -129,16 +133,16 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'-' | b'=') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => {
tokenizer.consume();
- State::Next(Name::HeadingSetextInside)
+ State::Next(StateName::HeadingSetextInside)
}
_ => {
tokenizer.tokenize_state.marker = 0;
- tokenizer.exit(Token::HeadingSetextUnderline);
+ tokenizer.exit(Name::HeadingSetextUnderline);
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::HeadingSetextAfter),
- State::Next(Name::HeadingSetextAfter),
+ State::Next(StateName::HeadingSetextAfter),
+ State::Next(StateName::HeadingSetextAfter),
)
}
}
@@ -173,28 +177,28 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
let event = &tokenizer.events[index];
// Find paragraphs.
- if event.event_type == EventType::Enter {
- if event.token_type == Token::Paragraph {
+ if event.kind == Kind::Enter {
+ if event.name == Name::Paragraph {
paragraph_enter = Some(index);
}
- } else if event.token_type == Token::Paragraph {
+ } else if event.name == Name::Paragraph {
paragraph_exit = Some(index);
}
// We know this is preceded by a paragraph.
// Otherwise we don’t parse.
- else if event.token_type == Token::HeadingSetextUnderline {
+ else if event.name == Name::HeadingSetextUnderline {
let enter = paragraph_enter.take().unwrap();
let exit = paragraph_exit.take().unwrap();
// Change types of Enter:Paragraph, Exit:Paragraph.
- tokenizer.events[enter].token_type = Token::HeadingSetextText;
- tokenizer.events[exit].token_type = Token::HeadingSetextText;
+ tokenizer.events[enter].name = Name::HeadingSetextText;
+ tokenizer.events[exit].name = Name::HeadingSetextText;
// Add Enter:HeadingSetext, Exit:HeadingSetext.
let mut heading_enter = tokenizer.events[enter].clone();
- heading_enter.token_type = Token::HeadingSetext;
+ heading_enter.name = Name::HeadingSetext;
let mut heading_exit = tokenizer.events[index].clone();
- heading_exit.token_type = Token::HeadingSetext;
+ heading_exit.name = Name::HeadingSetext;
tokenizer.map.add(enter, 0, vec![heading_enter]);
tokenizer.map.add(index + 1, 0, vec![heading_exit]);
diff --git a/src/construct/html_flow.rs b/src/construct/html_flow.rs
index c9f8610..8e5321f 100644
--- a/src/construct/html_flow.rs
+++ b/src/construct/html_flow.rs
@@ -104,8 +104,8 @@ use crate::constant::{
use crate::construct::partial_space_or_tab::{
space_or_tab_with_options, Options as SpaceOrTabOptions,
};
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
use crate::util::slice::Slice;
@@ -132,11 +132,11 @@ const COMPLETE: u8 = 7;
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.parse_state.constructs.html_flow {
- tokenizer.enter(Token::HtmlFlow);
+ tokenizer.enter(Name::HtmlFlow);
let name = space_or_tab_with_options(
tokenizer,
SpaceOrTabOptions {
- kind: Token::HtmlFlowData,
+ kind: Name::HtmlFlowData,
min: 0,
max: if tokenizer.parse_state.constructs.code_indented {
TAB_SIZE - 1
@@ -148,7 +148,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
},
);
- tokenizer.attempt(name, State::Next(Name::HtmlFlowBefore), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::HtmlFlowBefore), State::Nok)
} else {
State::Nok
}
@@ -162,9 +162,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn before(tokenizer: &mut Tokenizer) -> State {
if Some(b'<') == tokenizer.current {
- tokenizer.enter(Token::HtmlFlowData);
+ tokenizer.enter(Name::HtmlFlowData);
tokenizer.consume();
- State::Next(Name::HtmlFlowOpen)
+ State::Next(StateName::HtmlFlowOpen)
} else {
State::Nok
}
@@ -184,13 +184,13 @@ pub fn open(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'!') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowDeclarationOpen)
+ State::Next(StateName::HtmlFlowDeclarationOpen)
}
Some(b'/') => {
tokenizer.consume();
tokenizer.tokenize_state.seen = true;
tokenizer.tokenize_state.start = tokenizer.point.index;
- State::Next(Name::HtmlFlowTagCloseStart)
+ State::Next(StateName::HtmlFlowTagCloseStart)
}
Some(b'?') => {
tokenizer.tokenize_state.marker = INSTRUCTION;
@@ -199,12 +199,12 @@ pub fn open(tokenizer: &mut Tokenizer) -> State {
tokenizer.concrete = true;
// While we’re in an instruction instead of a declaration, we’re on a `?`
// right now, so we do need to search for `>`, similar to declarations.
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
}
// ASCII alphabetical.
Some(b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.tokenize_state.start = tokenizer.point.index;
- State::Retry(Name::HtmlFlowTagName)
+ State::Retry(StateName::HtmlFlowTagName)
}
_ => State::Nok,
}
@@ -225,19 +225,19 @@ pub fn declaration_open(tokenizer: &mut Tokenizer) -> State {
Some(b'-') => {
tokenizer.consume();
tokenizer.tokenize_state.marker = COMMENT;
- State::Next(Name::HtmlFlowCommentOpenInside)
+ State::Next(StateName::HtmlFlowCommentOpenInside)
}
Some(b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
tokenizer.tokenize_state.marker = DECLARATION;
// Do not form containers.
tokenizer.concrete = true;
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
}
Some(b'[') => {
tokenizer.consume();
tokenizer.tokenize_state.marker = CDATA;
- State::Next(Name::HtmlFlowCdataOpenInside)
+ State::Next(StateName::HtmlFlowCdataOpenInside)
}
_ => State::Nok,
}
@@ -254,7 +254,7 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State {
tokenizer.consume();
// Do not form containers.
tokenizer.concrete = true;
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
} else {
tokenizer.tokenize_state.marker = 0;
State::Nok
@@ -276,9 +276,9 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.size = 0;
// Do not form containers.
tokenizer.concrete = true;
- State::Next(Name::HtmlFlowContinuation)
+ State::Next(StateName::HtmlFlowContinuation)
} else {
- State::Next(Name::HtmlFlowCdataOpenInside)
+ State::Next(StateName::HtmlFlowCdataOpenInside)
}
} else {
tokenizer.tokenize_state.marker = 0;
@@ -296,7 +296,7 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State {
pub fn tag_close_start(tokenizer: &mut Tokenizer) -> State {
if let Some(b'A'..=b'Z' | b'a'..=b'z') = tokenizer.current {
tokenizer.consume();
- State::Next(Name::HtmlFlowTagName)
+ State::Next(StateName::HtmlFlowTagName)
} else {
tokenizer.tokenize_state.seen = false;
tokenizer.tokenize_state.start = 0;
@@ -335,17 +335,17 @@ pub fn tag_name(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.marker = RAW;
// Do not form containers.
tokenizer.concrete = true;
- State::Retry(Name::HtmlFlowContinuation)
+ State::Retry(StateName::HtmlFlowContinuation)
} else if HTML_BLOCK_NAMES.contains(&name.as_str()) {
tokenizer.tokenize_state.marker = BASIC;
if slash {
tokenizer.consume();
- State::Next(Name::HtmlFlowBasicSelfClosing)
+ State::Next(StateName::HtmlFlowBasicSelfClosing)
} else {
// Do not form containers.
tokenizer.concrete = true;
- State::Retry(Name::HtmlFlowContinuation)
+ State::Retry(StateName::HtmlFlowContinuation)
}
} else {
tokenizer.tokenize_state.marker = COMPLETE;
@@ -355,16 +355,16 @@ pub fn tag_name(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.marker = 0;
State::Nok
} else if closing_tag {
- State::Retry(Name::HtmlFlowCompleteClosingTagAfter)
+ State::Retry(StateName::HtmlFlowCompleteClosingTagAfter)
} else {
- State::Retry(Name::HtmlFlowCompleteAttributeNameBefore)
+ State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore)
}
}
}
// ASCII alphanumerical and `-`.
Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowTagName)
+ State::Next(StateName::HtmlFlowTagName)
}
Some(_) => {
tokenizer.tokenize_state.seen = false;
@@ -384,7 +384,7 @@ pub fn basic_self_closing(tokenizer: &mut Tokenizer) -> State {
tokenizer.consume();
// Do not form containers.
tokenizer.concrete = true;
- State::Next(Name::HtmlFlowContinuation)
+ State::Next(StateName::HtmlFlowContinuation)
} else {
tokenizer.tokenize_state.marker = 0;
State::Nok
@@ -401,9 +401,9 @@ pub fn complete_closing_tag_after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteClosingTagAfter)
+ State::Next(StateName::HtmlFlowCompleteClosingTagAfter)
}
- _ => State::Retry(Name::HtmlFlowCompleteEnd),
+ _ => State::Retry(StateName::HtmlFlowCompleteEnd),
}
}
@@ -430,18 +430,18 @@ pub fn complete_attribute_name_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeNameBefore)
+ State::Next(StateName::HtmlFlowCompleteAttributeNameBefore)
}
Some(b'/') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteEnd)
+ State::Next(StateName::HtmlFlowCompleteEnd)
}
// ASCII alphanumerical and `:` and `_`.
Some(b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeName)
+ State::Next(StateName::HtmlFlowCompleteAttributeName)
}
- _ => State::Retry(Name::HtmlFlowCompleteEnd),
+ _ => State::Retry(StateName::HtmlFlowCompleteEnd),
}
}
@@ -460,9 +460,9 @@ pub fn complete_attribute_name(tokenizer: &mut Tokenizer) -> State {
// ASCII alphanumerical and `-`, `.`, `:`, and `_`.
Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeName)
+ State::Next(StateName::HtmlFlowCompleteAttributeName)
}
- _ => State::Retry(Name::HtmlFlowCompleteAttributeNameAfter),
+ _ => State::Retry(StateName::HtmlFlowCompleteAttributeNameAfter),
}
}
@@ -479,13 +479,13 @@ pub fn complete_attribute_name_after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeNameAfter)
+ State::Next(StateName::HtmlFlowCompleteAttributeNameAfter)
}
Some(b'=') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeValueBefore)
+ State::Next(StateName::HtmlFlowCompleteAttributeValueBefore)
}
- _ => State::Retry(Name::HtmlFlowCompleteAttributeNameBefore),
+ _ => State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore),
}
}
@@ -506,14 +506,14 @@ pub fn complete_attribute_value_before(tokenizer: &mut Tokenizer) -> State {
}
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeValueBefore)
+ State::Next(StateName::HtmlFlowCompleteAttributeValueBefore)
}
Some(b'"' | b'\'') => {
tokenizer.tokenize_state.marker_b = tokenizer.current.unwrap();
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeValueQuoted)
+ State::Next(StateName::HtmlFlowCompleteAttributeValueQuoted)
}
- _ => State::Retry(Name::HtmlFlowCompleteAttributeValueUnquoted),
+ _ => State::Retry(StateName::HtmlFlowCompleteAttributeValueUnquoted),
}
}
@@ -535,11 +535,11 @@ pub fn complete_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State {
Some(b'"' | b'\'') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker_b => {
tokenizer.tokenize_state.marker_b = 0;
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeValueQuotedAfter)
+ State::Next(StateName::HtmlFlowCompleteAttributeValueQuotedAfter)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeValueQuoted)
+ State::Next(StateName::HtmlFlowCompleteAttributeValueQuoted)
}
}
}
@@ -553,11 +553,11 @@ pub fn complete_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State {
pub fn complete_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\t' | b'\n' | b' ' | b'"' | b'\'' | b'/' | b'<' | b'=' | b'>' | b'`') => {
- State::Retry(Name::HtmlFlowCompleteAttributeNameAfter)
+ State::Retry(StateName::HtmlFlowCompleteAttributeNameAfter)
}
Some(_) => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAttributeValueUnquoted)
+ State::Next(StateName::HtmlFlowCompleteAttributeValueUnquoted)
}
}
}
@@ -571,7 +571,7 @@ pub fn complete_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn complete_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State {
if let Some(b'\t' | b' ' | b'/' | b'>') = tokenizer.current {
- State::Retry(Name::HtmlFlowCompleteAttributeNameBefore)
+ State::Retry(StateName::HtmlFlowCompleteAttributeNameBefore)
} else {
tokenizer.tokenize_state.marker = 0;
State::Nok
@@ -587,7 +587,7 @@ pub fn complete_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State
pub fn complete_end(tokenizer: &mut Tokenizer) -> State {
if let Some(b'>') = tokenizer.current {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAfter)
+ State::Next(StateName::HtmlFlowCompleteAfter)
} else {
tokenizer.tokenize_state.marker = 0;
State::Nok
@@ -605,11 +605,11 @@ pub fn complete_after(tokenizer: &mut Tokenizer) -> State {
None | Some(b'\n') => {
// Do not form containers.
tokenizer.concrete = true;
- State::Retry(Name::HtmlFlowContinuation)
+ State::Retry(StateName::HtmlFlowContinuation)
}
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowCompleteAfter)
+ State::Next(StateName::HtmlFlowCompleteAfter)
}
Some(_) => {
tokenizer.tokenize_state.marker = 0;
@@ -630,41 +630,41 @@ pub fn continuation(tokenizer: &mut Tokenizer) -> State {
if tokenizer.tokenize_state.marker == BASIC
|| tokenizer.tokenize_state.marker == COMPLETE =>
{
- tokenizer.exit(Token::HtmlFlowData);
+ tokenizer.exit(Name::HtmlFlowData);
tokenizer.check(
- Name::HtmlFlowBlankLineBefore,
- State::Next(Name::HtmlFlowContinuationAfter),
- State::Next(Name::HtmlFlowContinuationStart),
+ StateName::HtmlFlowBlankLineBefore,
+ State::Next(StateName::HtmlFlowContinuationAfter),
+ State::Next(StateName::HtmlFlowContinuationStart),
)
}
// Note: important that this is after the basic/complete case.
None | Some(b'\n') => {
- tokenizer.exit(Token::HtmlFlowData);
- State::Retry(Name::HtmlFlowContinuationStart)
+ tokenizer.exit(Name::HtmlFlowData);
+ State::Retry(StateName::HtmlFlowContinuationStart)
}
Some(b'-') if tokenizer.tokenize_state.marker == COMMENT => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationCommentInside)
+ State::Next(StateName::HtmlFlowContinuationCommentInside)
}
Some(b'<') if tokenizer.tokenize_state.marker == RAW => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationRawTagOpen)
+ State::Next(StateName::HtmlFlowContinuationRawTagOpen)
}
Some(b'>') if tokenizer.tokenize_state.marker == DECLARATION => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationClose)
+ State::Next(StateName::HtmlFlowContinuationClose)
}
Some(b'?') if tokenizer.tokenize_state.marker == INSTRUCTION => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
}
Some(b']') if tokenizer.tokenize_state.marker == CDATA => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationCdataInside)
+ State::Next(StateName::HtmlFlowContinuationCdataInside)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuation)
+ State::Next(StateName::HtmlFlowContinuation)
}
}
}
@@ -678,9 +678,9 @@ pub fn continuation(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn continuation_start(tokenizer: &mut Tokenizer) -> State {
tokenizer.check(
- Name::NonLazyContinuationStart,
- State::Next(Name::HtmlFlowContinuationStartNonLazy),
- State::Next(Name::HtmlFlowContinuationAfter),
+ StateName::NonLazyContinuationStart,
+ State::Next(StateName::HtmlFlowContinuationStartNonLazy),
+ State::Next(StateName::HtmlFlowContinuationAfter),
)
}
@@ -694,10 +694,10 @@ pub fn continuation_start(tokenizer: &mut Tokenizer) -> State {
pub fn continuation_start_non_lazy(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::HtmlFlowContinuationBefore)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::HtmlFlowContinuationBefore)
}
_ => unreachable!("expected eol"),
}
@@ -712,10 +712,10 @@ pub fn continuation_start_non_lazy(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn continuation_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- None | Some(b'\n') => State::Retry(Name::HtmlFlowContinuationStart),
+ None | Some(b'\n') => State::Retry(StateName::HtmlFlowContinuationStart),
_ => {
- tokenizer.enter(Token::HtmlFlowData);
- State::Retry(Name::HtmlFlowContinuation)
+ tokenizer.enter(Name::HtmlFlowData);
+ State::Retry(StateName::HtmlFlowContinuation)
}
}
}
@@ -730,9 +730,9 @@ pub fn continuation_comment_inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'-') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
}
- _ => State::Retry(Name::HtmlFlowContinuation),
+ _ => State::Retry(StateName::HtmlFlowContinuation),
}
}
@@ -747,9 +747,9 @@ pub fn continuation_raw_tag_open(tokenizer: &mut Tokenizer) -> State {
Some(b'/') => {
tokenizer.consume();
tokenizer.tokenize_state.start = tokenizer.point.index;
- State::Next(Name::HtmlFlowContinuationRawEndTag)
+ State::Next(StateName::HtmlFlowContinuationRawEndTag)
}
- _ => State::Retry(Name::HtmlFlowContinuation),
+ _ => State::Retry(StateName::HtmlFlowContinuation),
}
}
@@ -774,20 +774,20 @@ pub fn continuation_raw_end_tag(tokenizer: &mut Tokenizer) -> State {
if HTML_RAW_NAMES.contains(&name.as_str()) {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationClose)
+ State::Next(StateName::HtmlFlowContinuationClose)
} else {
- State::Retry(Name::HtmlFlowContinuation)
+ State::Retry(StateName::HtmlFlowContinuation)
}
}
Some(b'A'..=b'Z' | b'a'..=b'z')
if tokenizer.point.index - tokenizer.tokenize_state.start < HTML_RAW_SIZE_MAX =>
{
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationRawEndTag)
+ State::Next(StateName::HtmlFlowContinuationRawEndTag)
}
_ => {
tokenizer.tokenize_state.start = 0;
- State::Retry(Name::HtmlFlowContinuation)
+ State::Retry(StateName::HtmlFlowContinuation)
}
}
}
@@ -802,9 +802,9 @@ pub fn continuation_cdata_inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b']') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
}
- _ => State::Retry(Name::HtmlFlowContinuation),
+ _ => State::Retry(StateName::HtmlFlowContinuation),
}
}
@@ -826,13 +826,13 @@ pub fn continuation_declaration_inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'>') => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationClose)
+ State::Next(StateName::HtmlFlowContinuationClose)
}
Some(b'-') if tokenizer.tokenize_state.marker == COMMENT => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationDeclarationInside)
+ State::Next(StateName::HtmlFlowContinuationDeclarationInside)
}
- _ => State::Retry(Name::HtmlFlowContinuation),
+ _ => State::Retry(StateName::HtmlFlowContinuation),
}
}
@@ -845,12 +845,12 @@ pub fn continuation_declaration_inside(tokenizer: &mut Tokenizer) -> State {
pub fn continuation_close(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::HtmlFlowData);
- State::Retry(Name::HtmlFlowContinuationAfter)
+ tokenizer.exit(Name::HtmlFlowData);
+ State::Retry(StateName::HtmlFlowContinuationAfter)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlFlowContinuationClose)
+ State::Next(StateName::HtmlFlowContinuationClose)
}
}
}
@@ -862,7 +862,7 @@ pub fn continuation_close(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn continuation_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.exit(Token::HtmlFlow);
+ tokenizer.exit(Name::HtmlFlow);
tokenizer.tokenize_state.marker = 0;
// Feel free to interrupt.
tokenizer.interrupt = false;
@@ -879,8 +879,8 @@ pub fn continuation_after(tokenizer: &mut Tokenizer) -> State {
/// |
/// ```
pub fn blank_line_before(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::BlankLineStart)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::BlankLineStart)
}
diff --git a/src/construct/html_text.rs b/src/construct/html_text.rs
index dd14e70..27e92f5 100644
--- a/src/construct/html_text.rs
+++ b/src/construct/html_text.rs
@@ -56,8 +56,8 @@
use crate::constant::HTML_CDATA_PREFIX;
use crate::construct::partial_space_or_tab::space_or_tab;
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of HTML (text)
@@ -68,10 +68,10 @@ use crate::tokenizer::Tokenizer;
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if Some(b'<') == tokenizer.current && tokenizer.parse_state.constructs.html_text {
- tokenizer.enter(Token::HtmlText);
- tokenizer.enter(Token::HtmlTextData);
+ tokenizer.enter(Name::HtmlText);
+ tokenizer.enter(Name::HtmlTextData);
tokenizer.consume();
- State::Next(Name::HtmlTextOpen)
+ State::Next(StateName::HtmlTextOpen)
} else {
State::Nok
}
@@ -91,20 +91,20 @@ pub fn open(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'!') => {
tokenizer.consume();
- State::Next(Name::HtmlTextDeclarationOpen)
+ State::Next(StateName::HtmlTextDeclarationOpen)
}
Some(b'/') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagCloseStart)
+ State::Next(StateName::HtmlTextTagCloseStart)
}
Some(b'?') => {
tokenizer.consume();
- State::Next(Name::HtmlTextInstruction)
+ State::Next(StateName::HtmlTextInstruction)
}
// ASCII alphabetical.
Some(b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpen)
+ State::Next(StateName::HtmlTextTagOpen)
}
_ => State::Nok,
}
@@ -124,16 +124,16 @@ pub fn declaration_open(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'-') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCommentOpenInside)
+ State::Next(StateName::HtmlTextCommentOpenInside)
}
// ASCII alphabetical.
Some(b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextDeclaration)
+ State::Next(StateName::HtmlTextDeclaration)
}
Some(b'[') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCdataOpenInside)
+ State::Next(StateName::HtmlTextCdataOpenInside)
}
_ => State::Nok,
}
@@ -149,7 +149,7 @@ pub fn comment_open_inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'-') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCommentStart)
+ State::Next(StateName::HtmlTextCommentStart)
}
_ => State::Nok,
}
@@ -173,9 +173,9 @@ pub fn comment_start(tokenizer: &mut Tokenizer) -> State {
Some(b'>') => State::Nok,
Some(b'-') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCommentStartDash)
+ State::Next(StateName::HtmlTextCommentStartDash)
}
- _ => State::Retry(Name::HtmlTextComment),
+ _ => State::Retry(StateName::HtmlTextComment),
}
}
@@ -195,7 +195,7 @@ pub fn comment_start(tokenizer: &mut Tokenizer) -> State {
pub fn comment_start_dash(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'>') => State::Nok,
- _ => State::Retry(Name::HtmlTextComment),
+ _ => State::Retry(StateName::HtmlTextComment),
}
}
@@ -209,17 +209,17 @@ pub fn comment(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None => State::Nok,
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextComment),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextComment),
State::Nok,
),
Some(b'-') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCommentClose)
+ State::Next(StateName::HtmlTextCommentClose)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlTextComment)
+ State::Next(StateName::HtmlTextComment)
}
}
}
@@ -234,9 +234,9 @@ pub fn comment_close(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'-') => {
tokenizer.consume();
- State::Next(Name::HtmlTextEnd)
+ State::Next(StateName::HtmlTextEnd)
}
- _ => State::Retry(Name::HtmlTextComment),
+ _ => State::Retry(StateName::HtmlTextComment),
}
}
@@ -253,9 +253,9 @@ pub fn cdata_open_inside(tokenizer: &mut Tokenizer) -> State {
if tokenizer.tokenize_state.size == HTML_CDATA_PREFIX.len() {
tokenizer.tokenize_state.size = 0;
- State::Next(Name::HtmlTextCdata)
+ State::Next(StateName::HtmlTextCdata)
} else {
- State::Next(Name::HtmlTextCdataOpenInside)
+ State::Next(StateName::HtmlTextCdataOpenInside)
}
} else {
State::Nok
@@ -272,17 +272,17 @@ pub fn cdata(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None => State::Nok,
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextCdata),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextCdata),
State::Nok,
),
Some(b']') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCdataClose)
+ State::Next(StateName::HtmlTextCdataClose)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlTextCdata)
+ State::Next(StateName::HtmlTextCdata)
}
}
}
@@ -297,9 +297,9 @@ pub fn cdata_close(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b']') => {
tokenizer.consume();
- State::Next(Name::HtmlTextCdataEnd)
+ State::Next(StateName::HtmlTextCdataEnd)
}
- _ => State::Retry(Name::HtmlTextCdata),
+ _ => State::Retry(StateName::HtmlTextCdata),
}
}
@@ -311,9 +311,9 @@ pub fn cdata_close(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn cdata_end(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- Some(b'>') => State::Retry(Name::HtmlTextEnd),
- Some(b']') => State::Retry(Name::HtmlTextCdataClose),
- _ => State::Retry(Name::HtmlTextCdata),
+ Some(b'>') => State::Retry(StateName::HtmlTextEnd),
+ Some(b']') => State::Retry(StateName::HtmlTextCdataClose),
+ _ => State::Retry(StateName::HtmlTextCdata),
}
}
@@ -325,15 +325,15 @@ pub fn cdata_end(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn declaration(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- None | Some(b'>') => State::Retry(Name::HtmlTextEnd),
+ None | Some(b'>') => State::Retry(StateName::HtmlTextEnd),
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextDeclaration),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextDeclaration),
State::Nok,
),
_ => {
tokenizer.consume();
- State::Next(Name::HtmlTextDeclaration)
+ State::Next(StateName::HtmlTextDeclaration)
}
}
}
@@ -348,17 +348,17 @@ pub fn instruction(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None => State::Nok,
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextInstruction),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextInstruction),
State::Nok,
),
Some(b'?') => {
tokenizer.consume();
- State::Next(Name::HtmlTextInstructionClose)
+ State::Next(StateName::HtmlTextInstructionClose)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlTextInstruction)
+ State::Next(StateName::HtmlTextInstruction)
}
}
}
@@ -371,8 +371,8 @@ pub fn instruction(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn instruction_close(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- Some(b'>') => State::Retry(Name::HtmlTextEnd),
- _ => State::Retry(Name::HtmlTextInstruction),
+ Some(b'>') => State::Retry(StateName::HtmlTextEnd),
+ _ => State::Retry(StateName::HtmlTextInstruction),
}
}
@@ -387,7 +387,7 @@ pub fn tag_close_start(tokenizer: &mut Tokenizer) -> State {
// ASCII alphabetical.
Some(b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagClose)
+ State::Next(StateName::HtmlTextTagClose)
}
_ => State::Nok,
}
@@ -404,9 +404,9 @@ pub fn tag_close(tokenizer: &mut Tokenizer) -> State {
// ASCII alphanumerical and `-`.
Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagClose)
+ State::Next(StateName::HtmlTextTagClose)
}
- _ => State::Retry(Name::HtmlTextTagCloseBetween),
+ _ => State::Retry(StateName::HtmlTextTagCloseBetween),
}
}
@@ -419,15 +419,15 @@ pub fn tag_close(tokenizer: &mut Tokenizer) -> State {
pub fn tag_close_between(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextTagCloseBetween),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextTagCloseBetween),
State::Nok,
),
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagCloseBetween)
+ State::Next(StateName::HtmlTextTagCloseBetween)
}
- _ => State::Retry(Name::HtmlTextEnd),
+ _ => State::Retry(StateName::HtmlTextEnd),
}
}
@@ -442,9 +442,9 @@ pub fn tag_open(tokenizer: &mut Tokenizer) -> State {
// ASCII alphanumerical and `-`.
Some(b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpen)
+ State::Next(StateName::HtmlTextTagOpen)
}
- Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(Name::HtmlTextTagOpenBetween),
+ Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(StateName::HtmlTextTagOpenBetween),
_ => State::Nok,
}
}
@@ -458,24 +458,24 @@ pub fn tag_open(tokenizer: &mut Tokenizer) -> State {
pub fn tag_open_between(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextTagOpenBetween),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextTagOpenBetween),
State::Nok,
),
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenBetween)
+ State::Next(StateName::HtmlTextTagOpenBetween)
}
Some(b'/') => {
tokenizer.consume();
- State::Next(Name::HtmlTextEnd)
+ State::Next(StateName::HtmlTextEnd)
}
// ASCII alphabetical and `:` and `_`.
Some(b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeName)
+ State::Next(StateName::HtmlTextTagOpenAttributeName)
}
- _ => State::Retry(Name::HtmlTextEnd),
+ _ => State::Retry(StateName::HtmlTextEnd),
}
}
@@ -490,9 +490,9 @@ pub fn tag_open_attribute_name(tokenizer: &mut Tokenizer) -> State {
// ASCII alphabetical and `-`, `.`, `:`, and `_`.
Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeName)
+ State::Next(StateName::HtmlTextTagOpenAttributeName)
}
- _ => State::Retry(Name::HtmlTextTagOpenAttributeNameAfter),
+ _ => State::Retry(StateName::HtmlTextTagOpenAttributeNameAfter),
}
}
@@ -506,19 +506,19 @@ pub fn tag_open_attribute_name(tokenizer: &mut Tokenizer) -> State {
pub fn tag_open_attribute_name_after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextTagOpenAttributeNameAfter),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextTagOpenAttributeNameAfter),
State::Nok,
),
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeNameAfter)
+ State::Next(StateName::HtmlTextTagOpenAttributeNameAfter)
}
Some(b'=') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueBefore)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueBefore)
}
- _ => State::Retry(Name::HtmlTextTagOpenBetween),
+ _ => State::Retry(StateName::HtmlTextTagOpenBetween),
}
}
@@ -533,22 +533,22 @@ pub fn tag_open_attribute_value_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'<' | b'=' | b'>' | b'`') => State::Nok,
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextTagOpenAttributeValueBefore),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextTagOpenAttributeValueBefore),
State::Nok,
),
Some(b'\t' | b' ') => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueBefore)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueBefore)
}
Some(b'"' | b'\'') => {
tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueQuoted)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted)
}
Some(_) => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueUnquoted)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueUnquoted)
}
}
}
@@ -566,18 +566,18 @@ pub fn tag_open_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State {
State::Nok
}
Some(b'\n') => tokenizer.attempt(
- Name::HtmlTextLineEndingBefore,
- State::Next(Name::HtmlTextTagOpenAttributeValueQuoted),
+ StateName::HtmlTextLineEndingBefore,
+ State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted),
State::Nok,
),
Some(b'"' | b'\'') if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker => {
tokenizer.tokenize_state.marker = 0;
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueQuotedAfter)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueQuotedAfter)
}
_ => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueQuoted)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueQuoted)
}
}
}
@@ -591,10 +591,10 @@ pub fn tag_open_attribute_value_quoted(tokenizer: &mut Tokenizer) -> State {
pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'"' | b'\'' | b'<' | b'=' | b'`') => State::Nok,
- Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(Name::HtmlTextTagOpenBetween),
+ Some(b'\t' | b'\n' | b' ' | b'/' | b'>') => State::Retry(StateName::HtmlTextTagOpenBetween),
Some(_) => {
tokenizer.consume();
- State::Next(Name::HtmlTextTagOpenAttributeValueUnquoted)
+ State::Next(StateName::HtmlTextTagOpenAttributeValueUnquoted)
}
}
}
@@ -608,7 +608,7 @@ pub fn tag_open_attribute_value_unquoted(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn tag_open_attribute_value_quoted_after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
- Some(b'\t' | b'\n' | b' ' | b'>' | b'/') => State::Retry(Name::HtmlTextTagOpenBetween),
+ Some(b'\t' | b'\n' | b' ' | b'>' | b'/') => State::Retry(StateName::HtmlTextTagOpenBetween),
_ => State::Nok,
}
}
@@ -623,8 +623,8 @@ pub fn end(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'>') => {
tokenizer.consume();
- tokenizer.exit(Token::HtmlTextData);
- tokenizer.exit(Token::HtmlText);
+ tokenizer.exit(Name::HtmlTextData);
+ tokenizer.exit(Name::HtmlText);
State::Ok
}
_ => State::Nok,
@@ -644,11 +644,11 @@ pub fn end(tokenizer: &mut Tokenizer) -> State {
pub fn line_ending_before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => {
- tokenizer.exit(Token::HtmlTextData);
- tokenizer.enter(Token::LineEnding);
+ tokenizer.exit(Name::HtmlTextData);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::HtmlTextLineEndingAfter)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::HtmlTextLineEndingAfter)
}
_ => unreachable!("expected eol"),
}
@@ -668,8 +668,8 @@ pub fn line_ending_after(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::HtmlTextLineEndingAfterPrefix),
- State::Next(Name::HtmlTextLineEndingAfterPrefix),
+ State::Next(StateName::HtmlTextLineEndingAfterPrefix),
+ State::Next(StateName::HtmlTextLineEndingAfterPrefix),
)
}
@@ -684,6 +684,6 @@ pub fn line_ending_after(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn line_ending_after_prefix(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::HtmlTextData);
+ tokenizer.enter(Name::HtmlTextData);
State::Ok
}
diff --git a/src/construct/label_end.rs b/src/construct/label_end.rs
index 0607077..61f378d 100644
--- a/src/construct/label_end.rs
+++ b/src/construct/label_end.rs
@@ -148,9 +148,10 @@
use crate::constant::RESOURCE_DESTINATION_BALANCE_MAX;
use crate::construct::partial_space_or_tab::space_or_tab_eol;
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{Event, EventType, Media, Tokenizer};
+use crate::event::{Event, Kind, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::{Media, Tokenizer};
+
use crate::util::{
normalize_identifier::normalize_identifier,
skip,
@@ -195,15 +196,15 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
// Mark as balanced if the info is inactive.
if label_start.inactive {
- return State::Retry(Name::LabelEndNok);
+ return State::Retry(StateName::LabelEndNok);
}
- tokenizer.enter(Token::LabelEnd);
- tokenizer.enter(Token::LabelMarker);
+ tokenizer.enter(Name::LabelEnd);
+ tokenizer.enter(Name::LabelMarker);
tokenizer.consume();
- tokenizer.exit(Token::LabelMarker);
- tokenizer.exit(Token::LabelEnd);
- return State::Next(Name::LabelEndAfter);
+ tokenizer.exit(Name::LabelMarker);
+ tokenizer.exit(Name::LabelEnd);
+ return State::Next(StateName::LabelEndAfter);
}
}
@@ -240,29 +241,29 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
// Resource (`[asd](fgh)`)?
Some(b'(') => tokenizer.attempt(
- Name::LabelEndResourceStart,
- State::Next(Name::LabelEndOk),
+ StateName::LabelEndResourceStart,
+ State::Next(StateName::LabelEndOk),
State::Next(if defined {
- Name::LabelEndOk
+ StateName::LabelEndOk
} else {
- Name::LabelEndNok
+ StateName::LabelEndNok
}),
),
// Full (`[asd][fgh]`) or collapsed (`[asd][]`) reference?
Some(b'[') => tokenizer.attempt(
- Name::LabelEndReferenceFull,
- State::Next(Name::LabelEndOk),
+ StateName::LabelEndReferenceFull,
+ State::Next(StateName::LabelEndOk),
State::Next(if defined {
- Name::LabelEndReferenceNotFull
+ StateName::LabelEndReferenceNotFull
} else {
- Name::LabelEndNok
+ StateName::LabelEndNok
}),
),
// Shortcut (`[asd]`) reference?
_ => State::Retry(if defined {
- Name::LabelEndOk
+ StateName::LabelEndOk
} else {
- Name::LabelEndNok
+ StateName::LabelEndNok
}),
}
}
@@ -279,9 +280,9 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn reference_not_full(tokenizer: &mut Tokenizer) -> State {
tokenizer.attempt(
- Name::LabelEndReferenceCollapsed,
- State::Next(Name::LabelEndOk),
- State::Next(Name::LabelEndNok),
+ StateName::LabelEndReferenceCollapsed,
+ State::Next(StateName::LabelEndOk),
+ State::Next(StateName::LabelEndNok),
)
}
@@ -311,13 +312,13 @@ pub fn ok(tokenizer: &mut Tokenizer) -> State {
.label_start_list_loose
.append(&mut left);
- let is_link = tokenizer.events[label_start.start.0].token_type == Token::LabelLink;
+ let is_link = tokenizer.events[label_start.start.0].name == Name::LabelLink;
if is_link {
let mut index = 0;
while index < tokenizer.tokenize_state.label_start_stack.len() {
let label_start = &mut tokenizer.tokenize_state.label_start_stack[index];
- if tokenizer.events[label_start.start.0].token_type == Token::LabelLink {
+ if tokenizer.events[label_start.start.0].name == Name::LabelLink {
label_start.inactive = true;
}
index += 1;
@@ -367,11 +368,11 @@ pub fn nok(tokenizer: &mut Tokenizer) -> State {
pub fn resource_start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'(') => {
- tokenizer.enter(Token::Resource);
- tokenizer.enter(Token::ResourceMarker);
+ tokenizer.enter(Name::Resource);
+ tokenizer.enter(Name::ResourceMarker);
tokenizer.consume();
- tokenizer.exit(Token::ResourceMarker);
- State::Next(Name::LabelEndResourceBefore)
+ tokenizer.exit(Name::ResourceMarker);
+ State::Next(StateName::LabelEndResourceBefore)
}
_ => unreachable!("expected `(`"),
}
@@ -387,8 +388,8 @@ pub fn resource_before(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_eol(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::LabelEndResourceOpen),
- State::Next(Name::LabelEndResourceOpen),
+ State::Next(StateName::LabelEndResourceOpen),
+ State::Next(StateName::LabelEndResourceOpen),
)
}
@@ -400,19 +401,19 @@ pub fn resource_before(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn resource_open(tokenizer: &mut Tokenizer) -> State {
if let Some(b')') = tokenizer.current {
- State::Retry(Name::LabelEndResourceEnd)
+ State::Retry(StateName::LabelEndResourceEnd)
} else {
- tokenizer.tokenize_state.token_1 = Token::ResourceDestination;
- tokenizer.tokenize_state.token_2 = Token::ResourceDestinationLiteral;
- tokenizer.tokenize_state.token_3 = Token::ResourceDestinationLiteralMarker;
- tokenizer.tokenize_state.token_4 = Token::ResourceDestinationRaw;
- tokenizer.tokenize_state.token_5 = Token::ResourceDestinationString;
+ tokenizer.tokenize_state.token_1 = Name::ResourceDestination;
+ tokenizer.tokenize_state.token_2 = Name::ResourceDestinationLiteral;
+ tokenizer.tokenize_state.token_3 = Name::ResourceDestinationLiteralMarker;
+ tokenizer.tokenize_state.token_4 = Name::ResourceDestinationRaw;
+ tokenizer.tokenize_state.token_5 = Name::ResourceDestinationString;
tokenizer.tokenize_state.size_b = RESOURCE_DESTINATION_BALANCE_MAX;
tokenizer.attempt(
- Name::DestinationStart,
- State::Next(Name::LabelEndResourceDestinationAfter),
- State::Next(Name::LabelEndResourceDestinationMissing),
+ StateName::DestinationStart,
+ State::Next(StateName::LabelEndResourceDestinationAfter),
+ State::Next(StateName::LabelEndResourceDestinationMissing),
)
}
}
@@ -424,27 +425,27 @@ pub fn resource_open(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn resource_destination_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
- tokenizer.tokenize_state.token_4 = Token::Data;
- tokenizer.tokenize_state.token_5 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
+ tokenizer.tokenize_state.token_4 = Name::Data;
+ tokenizer.tokenize_state.token_5 = Name::Data;
tokenizer.tokenize_state.size_b = 0;
let name = space_or_tab_eol(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::LabelEndResourceBetween),
- State::Next(Name::LabelEndResourceEnd),
+ State::Next(StateName::LabelEndResourceBetween),
+ State::Next(StateName::LabelEndResourceEnd),
)
}
/// Without destination.
pub fn resource_destination_missing(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
- tokenizer.tokenize_state.token_4 = Token::Data;
- tokenizer.tokenize_state.token_5 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
+ tokenizer.tokenize_state.token_4 = Name::Data;
+ tokenizer.tokenize_state.token_5 = Name::Data;
tokenizer.tokenize_state.size_b = 0;
State::Nok
}
@@ -458,16 +459,16 @@ pub fn resource_destination_missing(tokenizer: &mut Tokenizer) -> State {
pub fn resource_between(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'"' | b'\'' | b'(') => {
- tokenizer.tokenize_state.token_1 = Token::ResourceTitle;
- tokenizer.tokenize_state.token_2 = Token::ResourceTitleMarker;
- tokenizer.tokenize_state.token_3 = Token::ResourceTitleString;
+ tokenizer.tokenize_state.token_1 = Name::ResourceTitle;
+ tokenizer.tokenize_state.token_2 = Name::ResourceTitleMarker;
+ tokenizer.tokenize_state.token_3 = Name::ResourceTitleString;
tokenizer.attempt(
- Name::TitleStart,
- State::Next(Name::LabelEndResourceTitleAfter),
+ StateName::TitleStart,
+ State::Next(StateName::LabelEndResourceTitleAfter),
State::Nok,
)
}
- _ => State::Retry(Name::LabelEndResourceEnd),
+ _ => State::Retry(StateName::LabelEndResourceEnd),
}
}
@@ -478,14 +479,14 @@ pub fn resource_between(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn resource_title_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
let name = space_or_tab_eol(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::LabelEndResourceEnd),
- State::Next(Name::LabelEndResourceEnd),
+ State::Next(StateName::LabelEndResourceEnd),
+ State::Next(StateName::LabelEndResourceEnd),
)
}
@@ -498,10 +499,10 @@ pub fn resource_title_after(tokenizer: &mut Tokenizer) -> State {
pub fn resource_end(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b')') => {
- tokenizer.enter(Token::ResourceMarker);
+ tokenizer.enter(Name::ResourceMarker);
tokenizer.consume();
- tokenizer.exit(Token::ResourceMarker);
- tokenizer.exit(Token::Resource);
+ tokenizer.exit(Name::ResourceMarker);
+ tokenizer.exit(Name::Resource);
State::Ok
}
_ => State::Nok,
@@ -517,12 +518,12 @@ pub fn resource_end(tokenizer: &mut Tokenizer) -> State {
pub fn reference_full(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'[') => {
- tokenizer.tokenize_state.token_1 = Token::Reference;
- tokenizer.tokenize_state.token_2 = Token::ReferenceMarker;
- tokenizer.tokenize_state.token_3 = Token::ReferenceString;
+ tokenizer.tokenize_state.token_1 = Name::Reference;
+ tokenizer.tokenize_state.token_2 = Name::ReferenceMarker;
+ tokenizer.tokenize_state.token_3 = Name::ReferenceString;
tokenizer.attempt(
- Name::LabelStart,
- State::Next(Name::LabelEndReferenceFullAfter),
+ StateName::LabelStart,
+ State::Next(StateName::LabelEndReferenceFullAfter),
State::Nok,
)
}
@@ -537,9 +538,9 @@ pub fn reference_full(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn reference_full_after(tokenizer: &mut Tokenizer) -> State {
- tokenizer.tokenize_state.token_1 = Token::Data;
- tokenizer.tokenize_state.token_2 = Token::Data;
- tokenizer.tokenize_state.token_3 = Token::Data;
+ tokenizer.tokenize_state.token_1 = Name::Data;
+ tokenizer.tokenize_state.token_2 = Name::Data;
+ tokenizer.tokenize_state.token_3 = Name::Data;
if tokenizer
.parse_state
@@ -553,7 +554,7 @@ pub fn reference_full_after(tokenizer: &mut Tokenizer) -> State {
skip::to_back(
&tokenizer.events,
tokenizer.events.len() - 1,
- &[Token::ReferenceString],
+ &[Name::ReferenceString],
),
),
)
@@ -577,11 +578,11 @@ pub fn reference_full_after(tokenizer: &mut Tokenizer) -> State {
pub fn reference_collapsed(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'[') => {
- tokenizer.enter(Token::Reference);
- tokenizer.enter(Token::ReferenceMarker);
+ tokenizer.enter(Name::Reference);
+ tokenizer.enter(Name::ReferenceMarker);
tokenizer.consume();
- tokenizer.exit(Token::ReferenceMarker);
- State::Next(Name::LabelEndReferenceCollapsedOpen)
+ tokenizer.exit(Name::ReferenceMarker);
+ State::Next(StateName::LabelEndReferenceCollapsedOpen)
}
_ => State::Nok,
}
@@ -598,10 +599,10 @@ pub fn reference_collapsed(tokenizer: &mut Tokenizer) -> State {
pub fn reference_collapsed_open(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b']') => {
- tokenizer.enter(Token::ReferenceMarker);
+ tokenizer.enter(Name::ReferenceMarker);
tokenizer.consume();
- tokenizer.exit(Token::ReferenceMarker);
- tokenizer.exit(Token::Reference);
+ tokenizer.exit(Name::ReferenceMarker);
+ tokenizer.exit(Name::Reference);
State::Ok
}
_ => State::Nok,
@@ -633,14 +634,14 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
data_exit_index - data_enter_index + 1,
vec![
Event {
- event_type: EventType::Enter,
- token_type: Token::Data,
+ kind: Kind::Enter,
+ name: Name::Data,
point: events[data_enter_index].point.clone(),
link: None,
},
Event {
- event_type: EventType::Exit,
- token_type: Token::Data,
+ kind: Kind::Exit,
+ name: Name::Data,
point: events[data_exit_index].point.clone(),
link: None,
},
@@ -659,7 +660,7 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
let group_enter_event = &events[group_enter_index];
// LabelLink:Exit or LabelImage:Exit.
let text_enter_index = media.start.0
- + (if group_enter_event.token_type == Token::LabelLink {
+ + (if group_enter_event.name == Name::LabelLink {
4
} else {
6
@@ -677,18 +678,18 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
0,
vec![
Event {
- event_type: EventType::Enter,
- token_type: if group_enter_event.token_type == Token::LabelLink {
- Token::Link
+ kind: Kind::Enter,
+ name: if group_enter_event.name == Name::LabelLink {
+ Name::Link
} else {
- Token::Image
+ Name::Image
},
point: group_enter_event.point.clone(),
link: None,
},
Event {
- event_type: EventType::Enter,
- token_type: Token::Label,
+ kind: Kind::Enter,
+ name: Name::Label,
point: group_enter_event.point.clone(),
link: None,
},
@@ -702,8 +703,8 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
text_enter_index,
0,
vec![Event {
- event_type: EventType::Enter,
- token_type: Token::LabelText,
+ kind: Kind::Enter,
+ name: Name::LabelText,
point: events[text_enter_index].point.clone(),
link: None,
}],
@@ -714,8 +715,8 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
text_exit_index,
0,
vec![Event {
- event_type: EventType::Exit,
- token_type: Token::LabelText,
+ kind: Kind::Exit,
+ name: Name::LabelText,
point: events[text_exit_index].point.clone(),
link: None,
}],
@@ -727,8 +728,8 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
label_exit_index + 1,
0,
vec![Event {
- event_type: EventType::Exit,
- token_type: Token::Label,
+ kind: Kind::Exit,
+ name: Name::Label,
point: events[label_exit_index].point.clone(),
link: None,
}],
@@ -739,11 +740,11 @@ pub fn resolve_media(tokenizer: &mut Tokenizer) {
group_end_index + 1,
0,
vec![Event {
- event_type: EventType::Exit,
- token_type: if group_enter_event.token_type == Token::LabelLink {
- Token::Link
+ kind: Kind::Exit,
+ name: if group_enter_event.name == Name::LabelLink {
+ Name::Link
} else {
- Token::Image
+ Name::Image
},
point: events[group_end_index].point.clone(),
link: None,
diff --git a/src/construct/label_start_image.rs b/src/construct/label_start_image.rs
index 7703ba4..2f7c0bf 100644
--- a/src/construct/label_start_image.rs
+++ b/src/construct/label_start_image.rs
@@ -29,8 +29,8 @@
//! [html-img]: https://html.spec.whatwg.org/multipage/embedded-content.html#the-img-element
use super::label_end::resolve_media;
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::{LabelStart, Tokenizer};
/// Start of label (image) start.
@@ -42,11 +42,11 @@ use crate::tokenizer::{LabelStart, Tokenizer};
pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'!') if tokenizer.parse_state.constructs.label_start_image => {
- tokenizer.enter(Token::LabelImage);
- tokenizer.enter(Token::LabelImageMarker);
+ tokenizer.enter(Name::LabelImage);
+ tokenizer.enter(Name::LabelImageMarker);
tokenizer.consume();
- tokenizer.exit(Token::LabelImageMarker);
- State::Next(Name::LabelStartImageOpen)
+ tokenizer.exit(Name::LabelImageMarker);
+ State::Next(StateName::LabelStartImageOpen)
}
_ => State::Nok,
}
@@ -61,10 +61,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
pub fn open(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'[') => {
- tokenizer.enter(Token::LabelMarker);
+ tokenizer.enter(Name::LabelMarker);
tokenizer.consume();
- tokenizer.exit(Token::LabelMarker);
- tokenizer.exit(Token::LabelImage);
+ tokenizer.exit(Name::LabelMarker);
+ tokenizer.exit(Name::LabelImage);
tokenizer.tokenize_state.label_start_stack.push(LabelStart {
start: (tokenizer.events.len() - 6, tokenizer.events.len() - 1),
balanced: false,
diff --git a/src/construct/label_start_link.rs b/src/construct/label_start_link.rs
index 3ca51bf..456a4e9 100644
--- a/src/construct/label_start_link.rs
+++ b/src/construct/label_start_link.rs
@@ -28,8 +28,8 @@
//! [html-a]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-a-element
use super::label_end::resolve_media;
+use crate::event::Name;
use crate::state::State;
-use crate::token::Token;
use crate::tokenizer::{LabelStart, Tokenizer};
/// Start of label (link) start.
@@ -42,11 +42,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'[') if tokenizer.parse_state.constructs.label_start_link => {
let start = tokenizer.events.len();
- tokenizer.enter(Token::LabelLink);
- tokenizer.enter(Token::LabelMarker);
+ tokenizer.enter(Name::LabelLink);
+ tokenizer.enter(Name::LabelMarker);
tokenizer.consume();
- tokenizer.exit(Token::LabelMarker);
- tokenizer.exit(Token::LabelLink);
+ tokenizer.exit(Name::LabelMarker);
+ tokenizer.exit(Name::LabelLink);
tokenizer.tokenize_state.label_start_stack.push(LabelStart {
start: (start, tokenizer.events.len() - 1),
balanced: false,
diff --git a/src/construct/list.rs b/src/construct/list.rs
index 516cec7..ded77d0 100644
--- a/src/construct/list.rs
+++ b/src/construct/list.rs
@@ -46,9 +46,9 @@
use crate::constant::{LIST_ITEM_VALUE_SIZE_MAX, TAB_SIZE};
use crate::construct::partial_space_or_tab::space_or_tab_min_max;
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{EventType, Tokenizer};
+use crate::event::{Kind, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
use crate::util::{
skip,
slice::{Position, Slice},
@@ -62,7 +62,7 @@ use crate::util::{
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.parse_state.constructs.list {
- tokenizer.enter(Token::ListItem);
+ tokenizer.enter(Name::ListItem);
let name = space_or_tab_min_max(
tokenizer,
0,
@@ -72,7 +72,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
usize::MAX
},
);
- tokenizer.attempt(name, State::Next(Name::ListBefore), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::ListBefore), State::Nok)
} else {
State::Nok
}
@@ -88,14 +88,14 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
// Unordered.
Some(b'*' | b'-') => tokenizer.check(
- Name::ThematicBreakStart,
- State::Next(Name::ListNok),
- State::Next(Name::ListBeforeUnordered),
+ StateName::ThematicBreakStart,
+ State::Next(StateName::ListNok),
+ State::Next(StateName::ListBeforeUnordered),
),
- Some(b'+') => State::Retry(Name::ListBeforeUnordered),
+ Some(b'+') => State::Retry(StateName::ListBeforeUnordered),
// Ordered.
- Some(b'0'..=b'9') if !tokenizer.interrupt => State::Retry(Name::ListBeforeOrdered),
- Some(b'1') => State::Retry(Name::ListBeforeOrdered),
+ Some(b'0'..=b'9') if !tokenizer.interrupt => State::Retry(StateName::ListBeforeOrdered),
+ Some(b'1') => State::Retry(StateName::ListBeforeOrdered),
_ => State::Nok,
}
}
@@ -109,8 +109,8 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn before_unordered(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::ListItemPrefix);
- State::Retry(Name::ListMarker)
+ tokenizer.enter(Name::ListItemPrefix);
+ State::Retry(StateName::ListMarker)
}
/// Start of an ordered list item.
@@ -120,9 +120,9 @@ pub fn before_unordered(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn before_ordered(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::ListItemPrefix);
- tokenizer.enter(Token::ListItemValue);
- State::Retry(Name::ListValue)
+ tokenizer.enter(Name::ListItemPrefix);
+ tokenizer.enter(Name::ListItemValue);
+ State::Retry(StateName::ListValue)
}
/// In an ordered list item value.
@@ -134,13 +134,13 @@ pub fn before_ordered(tokenizer: &mut Tokenizer) -> State {
pub fn value(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'.' | b')') if !tokenizer.interrupt || tokenizer.tokenize_state.size < 2 => {
- tokenizer.exit(Token::ListItemValue);
- State::Retry(Name::ListMarker)
+ tokenizer.exit(Name::ListItemValue);
+ State::Retry(StateName::ListMarker)
}
Some(b'0'..=b'9') if tokenizer.tokenize_state.size + 1 < LIST_ITEM_VALUE_SIZE_MAX => {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
- State::Next(Name::ListValue)
+ State::Next(StateName::ListValue)
}
_ => {
tokenizer.tokenize_state.size = 0;
@@ -158,10 +158,10 @@ pub fn value(tokenizer: &mut Tokenizer) -> State {
/// ^
/// ```
pub fn marker(tokenizer: &mut Tokenizer) -> State {
- tokenizer.enter(Token::ListItemMarker);
+ tokenizer.enter(Name::ListItemMarker);
tokenizer.consume();
- tokenizer.exit(Token::ListItemMarker);
- State::Next(Name::ListMarkerAfter)
+ tokenizer.exit(Name::ListItemMarker);
+ State::Next(StateName::ListMarkerAfter)
}
/// After a list item marker.
@@ -175,9 +175,9 @@ pub fn marker(tokenizer: &mut Tokenizer) -> State {
pub fn marker_after(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.size = 1;
tokenizer.check(
- Name::BlankLineStart,
- State::Next(Name::ListAfter),
- State::Next(Name::ListMarkerAfterFilled),
+ StateName::BlankLineStart,
+ State::Next(StateName::ListAfter),
+ State::Next(StateName::ListMarkerAfterFilled),
)
}
@@ -192,9 +192,9 @@ pub fn marker_after_filled(tokenizer: &mut Tokenizer) -> State {
// Attempt to parse up to the largest allowed indent, `nok` if there is more whitespace.
tokenizer.attempt(
- Name::ListWhitespace,
- State::Next(Name::ListAfter),
- State::Next(Name::ListPrefixOther),
+ StateName::ListWhitespace,
+ State::Next(StateName::ListAfter),
+ State::Next(StateName::ListPrefixOther),
)
}
@@ -206,7 +206,11 @@ pub fn marker_after_filled(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn whitespace(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_min_max(tokenizer, 1, TAB_SIZE);
- tokenizer.attempt(name, State::Next(Name::ListWhitespaceAfter), State::Nok)
+ tokenizer.attempt(
+ name,
+ State::Next(StateName::ListWhitespaceAfter),
+ State::Nok,
+ )
}
/// After acceptable whitespace.
@@ -232,10 +236,10 @@ pub fn whitespace_after(tokenizer: &mut Tokenizer) -> State {
pub fn prefix_other(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\t' | b' ') => {
- tokenizer.enter(Token::SpaceOrTab);
+ tokenizer.enter(Name::SpaceOrTab);
tokenizer.consume();
- tokenizer.exit(Token::SpaceOrTab);
- State::Next(Name::ListAfter)
+ tokenizer.exit(Name::SpaceOrTab);
+ State::Next(StateName::ListAfter)
}
_ => State::Nok,
}
@@ -257,7 +261,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
let start = skip::to_back(
&tokenizer.events,
tokenizer.events.len() - 1,
- &[Token::ListItem],
+ &[Name::ListItem],
);
let mut prefix = Slice::from_position(
tokenizer.parse_state.bytes,
@@ -278,7 +282,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
container.blank_initial = blank;
container.size = prefix;
- tokenizer.exit(Token::ListItemPrefix);
+ tokenizer.exit(Name::ListItemPrefix);
tokenizer.register_resolver_before("list_item".to_string(), Box::new(resolve_list_item));
State::Ok
}
@@ -293,9 +297,9 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn cont_start(tokenizer: &mut Tokenizer) -> State {
tokenizer.check(
- Name::BlankLineStart,
- State::Next(Name::ListContBlank),
- State::Next(Name::ListContFilled),
+ StateName::BlankLineStart,
+ State::Next(StateName::ListContBlank),
+ State::Next(StateName::ListContFilled),
)
}
@@ -317,7 +321,7 @@ pub fn cont_blank(tokenizer: &mut Tokenizer) -> State {
} else {
let name = space_or_tab_min_max(tokenizer, 0, size);
// Consume, optionally, at most `size`.
- tokenizer.attempt(name, State::Next(Name::ListOk), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::ListOk), State::Nok)
}
}
@@ -337,7 +341,7 @@ pub fn cont_filled(tokenizer: &mut Tokenizer) -> State {
// Consume exactly `size`.
let name = space_or_tab_min_max(tokenizer, size, size);
- tokenizer.attempt(name, State::Next(Name::ListOk), State::Nok)
+ tokenizer.attempt(name, State::Next(StateName::ListOk), State::Nok)
}
/// A state fn to yield [`State::Ok`].
@@ -361,10 +365,10 @@ pub fn resolve_list_item(tokenizer: &mut Tokenizer) {
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.token_type == Token::ListItem {
- if event.event_type == EventType::Enter {
- let end = skip::opt(&tokenizer.events, index, &[Token::ListItem]) - 1;
- let marker = skip::to(&tokenizer.events, index, &[Token::ListItemMarker]);
+ if event.name == Name::ListItem {
+ if event.kind == Kind::Enter {
+ let end = skip::opt(&tokenizer.events, index, &[Name::ListItem]) - 1;
+ let marker = skip::to(&tokenizer.events, index, &[Name::ListItemMarker]);
// Guaranteed to be a valid ASCII byte.
let marker = Slice::from_index(
tokenizer.parse_state.bytes,
@@ -384,10 +388,10 @@ pub fn resolve_list_item(tokenizer: &mut Tokenizer) {
&tokenizer.events,
previous.3 + 1,
&[
- Token::SpaceOrTab,
- Token::LineEnding,
- Token::BlankLineEnding,
- Token::BlockQuotePrefix,
+ Name::SpaceOrTab,
+ Name::LineEnding,
+ Name::BlankLineEnding,
+ Name::BlockQuotePrefix,
],
);
@@ -441,12 +445,12 @@ pub fn resolve_list_item(tokenizer: &mut Tokenizer) {
let list_item = &lists[index];
let mut list_start = tokenizer.events[list_item.2].clone();
let mut list_end = tokenizer.events[list_item.3].clone();
- let token_type = match list_item.0 {
- b'.' | b')' => Token::ListOrdered,
- _ => Token::ListUnordered,
+ let name = match list_item.0 {
+ b'.' | b')' => Name::ListOrdered,
+ _ => Name::ListUnordered,
};
- list_start.token_type = token_type.clone();
- list_end.token_type = token_type;
+ list_start.name = name.clone();
+ list_end.name = name;
tokenizer.map.add(list_item.2, 0, vec![list_start]);
tokenizer.map.add(list_item.3 + 1, 0, vec![list_end]);
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index dec25b8..b605c0f 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -32,9 +32,9 @@
//! [code_text]: crate::construct::code_text
//! [html]: https://html.spec.whatwg.org/multipage/grouping-content.html#the-p-element
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{ContentType, EventType, Tokenizer};
+use crate::event::{Content, Kind, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
use crate::util::skip::opt as skip_opt;
/// Before a paragraph.
@@ -47,9 +47,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => unreachable!("unexpected eol/eof"),
_ => {
- tokenizer.enter(Token::Paragraph);
- tokenizer.enter_with_content(Token::Data, Some(ContentType::Text));
- State::Retry(Name::ParagraphInside)
+ tokenizer.enter(Name::Paragraph);
+ tokenizer.enter_with_content(Name::Data, Some(Content::Text));
+ State::Retry(StateName::ParagraphInside)
}
}
}
@@ -63,8 +63,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::Data);
- tokenizer.exit(Token::Paragraph);
+ tokenizer.exit(Name::Data);
+ tokenizer.exit(Name::Paragraph);
tokenizer.register_resolver_before("paragraph".to_string(), Box::new(resolve));
// You’d be interrupting.
tokenizer.interrupt = true;
@@ -72,7 +72,7 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
}
_ => {
tokenizer.consume();
- State::Next(Name::ParagraphInside)
+ State::Next(StateName::ParagraphInside)
}
}
}
@@ -85,21 +85,21 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == Token::Paragraph {
+ if event.kind == Kind::Enter && event.name == Name::Paragraph {
// Exit:Paragraph
let mut exit_index = index + 3;
let mut enter_next_index =
- skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]);
+ skip_opt(&tokenizer.events, exit_index + 1, &[Name::LineEnding]);
// Enter:Paragraph
enter_next_index = skip_opt(
&tokenizer.events,
enter_next_index,
- &[Token::SpaceOrTab, Token::BlockQuotePrefix],
+ &[Name::SpaceOrTab, Name::BlockQuotePrefix],
);
// Find future `Paragraphs`.
while enter_next_index < tokenizer.events.len()
- && tokenizer.events[enter_next_index].token_type == Token::Paragraph
+ && tokenizer.events[enter_next_index].name == Name::Paragraph
{
// Remove Exit:Paragraph, Enter:LineEnding, Exit:LineEnding, Enter:Paragraph.
tokenizer.map.add(exit_index, 3, vec![]);
@@ -123,12 +123,11 @@ pub fn resolve(tokenizer: &mut Tokenizer) {
// Potential next start.
exit_index = enter_next_index + 3;
- enter_next_index =
- skip_opt(&tokenizer.events, exit_index + 1, &[Token::LineEnding]);
+ enter_next_index = skip_opt(&tokenizer.events, exit_index + 1, &[Name::LineEnding]);
enter_next_index = skip_opt(
&tokenizer.events,
enter_next_index,
- &[Token::SpaceOrTab, Token::BlockQuotePrefix],
+ &[Name::SpaceOrTab, Name::BlockQuotePrefix],
);
}
diff --git a/src/construct/partial_bom.rs b/src/construct/partial_bom.rs
index cca0770..74ca04e 100644
--- a/src/construct/partial_bom.rs
+++ b/src/construct/partial_bom.rs
@@ -10,8 +10,8 @@
//!
//! * [`micromark/lib/preprocess.js` in `micromark`](https://github.com/micromark/micromark/blob/ed23453/packages/micromark/dev/lib/preprocess.js#L54-L60)
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
@@ -24,8 +24,8 @@ const BOM: [u8; 3] = [0xEF, 0xBB, 0xBF];
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.current == Some(BOM[0]) {
- tokenizer.enter(Token::ByteOrderMark);
- State::Retry(Name::BomInside)
+ tokenizer.enter(Name::ByteOrderMark);
+ State::Retry(StateName::BomInside)
} else {
State::Nok
}
@@ -42,11 +42,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.size += 1;
tokenizer.consume();
if tokenizer.tokenize_state.size == BOM.len() {
- tokenizer.exit(Token::ByteOrderMark);
+ tokenizer.exit(Name::ByteOrderMark);
tokenizer.tokenize_state.size = 0;
State::Ok
} else {
- State::Next(Name::BomInside)
+ State::Next(StateName::BomInside)
}
} else {
tokenizer.tokenize_state.size = 0;
diff --git a/src/construct/partial_data.rs b/src/construct/partial_data.rs
index c05aaa5..fda021e 100644
--- a/src/construct/partial_data.rs
+++ b/src/construct/partial_data.rs
@@ -6,9 +6,9 @@
//! [string]: crate::content::string
//! [text]: crate::content::text
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{EventType, Tokenizer};
+use crate::event::{Kind, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
/// At the beginning of data.
///
@@ -20,11 +20,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
// Make sure to eat the first `markers`.
Some(byte) if tokenizer.tokenize_state.markers.contains(&byte) => {
- tokenizer.enter(Token::Data);
+ tokenizer.enter(Name::Data);
tokenizer.consume();
- State::Next(Name::DataInside)
+ State::Next(StateName::DataInside)
}
- _ => State::Retry(Name::DataAtBreak),
+ _ => State::Retry(StateName::DataAtBreak),
}
}
@@ -38,18 +38,18 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None => State::Ok,
Some(b'\n') => {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::DataAtBreak)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::DataAtBreak)
}
Some(byte) if tokenizer.tokenize_state.markers.contains(&byte) => {
tokenizer.register_resolver_before("data".to_string(), Box::new(resolve_data));
State::Ok
}
_ => {
- tokenizer.enter(Token::Data);
- State::Retry(Name::DataInside)
+ tokenizer.enter(Name::Data);
+ State::Retry(StateName::DataInside)
}
}
}
@@ -68,11 +68,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
};
if done {
- tokenizer.exit(Token::Data);
- State::Retry(Name::DataAtBreak)
+ tokenizer.exit(Name::Data);
+ State::Retry(StateName::DataAtBreak)
} else {
tokenizer.consume();
- State::Next(Name::DataInside)
+ State::Next(StateName::DataInside)
}
}
@@ -85,13 +85,13 @@ pub fn resolve_data(tokenizer: &mut Tokenizer) {
while index < len {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Enter && event.token_type == Token::Data {
+ if event.kind == Kind::Enter && event.name == Name::Data {
let exit_index = index + 1;
let mut exit_far_index = exit_index;
// Find multiple `data` events.
while exit_far_index + 1 < len
- && tokenizer.events[exit_far_index + 1].token_type == Token::Data
+ && tokenizer.events[exit_far_index + 1].name == Name::Data
{
exit_far_index += 2;
}
diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs
index 5aa0539..c1c1e10 100644
--- a/src/construct/partial_destination.rs
+++ b/src/construct/partial_destination.rs
@@ -71,9 +71,9 @@
//! [label_end]: crate::construct::label_end
//! [sanitize_uri]: crate::util::sanitize_uri
-use crate::state::{Name, State};
-use crate::token::Token;
-use crate::tokenizer::{ContentType, Tokenizer};
+use crate::event::{Content, Name};
+use crate::state::{Name as StateName, State};
+use crate::tokenizer::Tokenizer;
/// Before a destination.
///
@@ -91,7 +91,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
tokenizer.enter(tokenizer.tokenize_state.token_3.clone());
tokenizer.consume();
tokenizer.exit(tokenizer.tokenize_state.token_3.clone());
- State::Next(Name::DestinationEnclosedBefore)
+ State::Next(StateName::DestinationEnclosedBefore)
}
// ASCII control, space, closing paren, but *not* `\0`.
None | Some(0x01..=0x1F | b' ' | b')' | 0x7F) => State::Nok,
@@ -99,8 +99,8 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
tokenizer.enter(tokenizer.tokenize_state.token_1.clone());
tokenizer.enter(tokenizer.tokenize_state.token_4.clone());
tokenizer.enter(tokenizer.tokenize_state.token_5.clone());
- tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
- State::Retry(Name::DestinationRaw)
+ tokenizer.enter_with_content(Name::Data, Some(Content::String));
+ State::Retry(StateName::DestinationRaw)
}
}
}
@@ -121,8 +121,8 @@ pub fn enclosed_before(tokenizer: &mut Tokenizer) -> State {
State::Ok
} else {
tokenizer.enter(tokenizer.tokenize_state.token_5.clone());
- tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
- State::Retry(Name::DestinationEnclosed)
+ tokenizer.enter_with_content(Name::Data, Some(Content::String));
+ State::Retry(StateName::DestinationEnclosed)
}
}
@@ -136,17 +136,17 @@ pub fn enclosed(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n' | b'<') => State::Nok,
Some(b'>') => {
- tokenizer.exit(Token::Data);
+ tokenizer.exit(Name::Data);
tokenizer.exit(tokenizer.tokenize_state.token_5.clone());
- State::Retry(Name::DestinationEnclosedBefore)
+ State::Retry(StateName::DestinationEnclosedBefore)
}
Some(b'\\') => {
tokenizer.consume();
- State::Next(Name::DestinationEnclosedEscape)
+ State::Next(StateName::DestinationEnclosedEscape)
}
_ => {
tokenizer.consume();
- State::Next(Name::DestinationEnclosed)
+ State::Next(StateName::DestinationEnclosed)
}
}
}
@@ -161,9 +161,9 @@ pub fn enclosed_escape(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'<' | b'>' | b'\\') => {
tokenizer.consume();
- State::Next(Name::DestinationEnclosed)
+ State::Next(StateName::DestinationEnclosed)
}
- _ => State::Retry(Name::DestinationEnclosed),
+ _ => State::Retry(StateName::DestinationEnclosed),
}
}
@@ -176,7 +176,7 @@ pub fn enclosed_escape(tokenizer: &mut Tokenizer) -> State {
pub fn raw(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\t' | b'\n' | b' ' | b')') if tokenizer.tokenize_state.size == 0 => {
- tokenizer.exit(Token::Data);
+ tokenizer.exit(Name::Data);
tokenizer.exit(tokenizer.tokenize_state.token_5.clone());
tokenizer.exit(tokenizer.tokenize_state.token_4.clone());
tokenizer.exit(tokenizer.tokenize_state.token_1.clone());
@@ -186,7 +186,7 @@ pub fn raw(tokenizer: &mut Tokenizer) -> State {
Some(b'(') if tokenizer.tokenize_state.size < tokenizer.tokenize_state.size_b => {
tokenizer.consume();
tokenizer.tokenize_state.size += 1;
- State::Next(Name::DestinationRaw)
+ State::Next(StateName::DestinationRaw)
}
// ASCII control (but *not* `\0`) and space and `(`.
None | Some(0x01..=0x1F | b' ' | b'(' | 0x7F) => {
@@ -196,15 +196,15 @@ pub fn raw(tokenizer: &mut Tokenizer) -> State {
Some(b')') => {
tokenizer.consume();
tokenizer.tokenize_state.size -= 1;
- State::Next(Name::DestinationRaw)
+ State::Next(StateName::DestinationRaw)
}
Some(b'\\') => {
tokenizer.consume();
- State::Next(Name::DestinationRawEscape)
+ State::Next(StateName::DestinationRawEscape)
}
Some(_) => {
tokenizer.consume();
- State::Next(Name::DestinationRaw)
+ State::Next(StateName::DestinationRaw)
}
}
}
@@ -219,8 +219,8 @@ pub fn raw_escape(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'(' | b')' | b'\\') => {
tokenizer.consume();
- State::Next(Name::DestinationRaw)
+ State::Next(StateName::DestinationRaw)
}
- _ => State::Retry(Name::DestinationRaw),
+ _ => State::Retry(StateName::DestinationRaw),
}
}
diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs
index 6e7c843..85769eb 100644
--- a/src/construct/partial_label.rs
+++ b/src/construct/partial_label.rs
@@ -58,12 +58,12 @@
//! [code_text]: crate::construct::code_text
//! [link_reference_size_max]: crate::constant::LINK_REFERENCE_SIZE_MAX
-use super::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
use crate::constant::LINK_REFERENCE_SIZE_MAX;
-use crate::state::{Name, State};
+use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
+use crate::event::{Content, Name};
+use crate::state::{Name as StateName, State};
use crate::subtokenize::link;
-use crate::token::Token;
-use crate::tokenizer::{ContentType, Tokenizer};
+use crate::tokenizer::Tokenizer;
/// Before a label.
///
@@ -79,7 +79,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
tokenizer.consume();
tokenizer.exit(tokenizer.tokenize_state.token_2.clone());
tokenizer.enter(tokenizer.tokenize_state.token_3.clone());
- State::Next(Name::LabelAtBreak)
+ State::Next(StateName::LabelAtBreak)
}
_ => State::Nok,
}
@@ -106,14 +106,14 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_eol_with_options(
tokenizer,
EolOptions {
- content_type: Some(ContentType::String),
+ content_type: Some(Content::String),
connect: tokenizer.tokenize_state.connect,
},
);
tokenizer.attempt(
name,
- State::Next(Name::LabelEolAfter),
- State::Next(Name::LabelAtBlankLine),
+ State::Next(StateName::LabelEolAfter),
+ State::Next(StateName::LabelAtBlankLine),
)
}
Some(b']') => {
@@ -128,7 +128,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
State::Ok
}
_ => {
- tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Name::Data, Some(Content::String));
if tokenizer.tokenize_state.connect {
let index = tokenizer.events.len() - 1;
@@ -137,7 +137,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.connect = true;
}
- State::Retry(Name::LabelInside)
+ State::Retry(StateName::LabelInside)
}
}
}
@@ -152,7 +152,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn eol_after(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.connect = true;
- State::Retry(Name::LabelAtBreak)
+ State::Retry(StateName::LabelAtBreak)
}
/// In a label, at a blank line.
@@ -178,13 +178,13 @@ pub fn at_blank_line(tokenizer: &mut Tokenizer) -> State {
pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n' | b'[' | b']') => {
- tokenizer.exit(Token::Data);
- State::Retry(Name::LabelAtBreak)
+ tokenizer.exit(Name::Data);
+ State::Retry(StateName::LabelAtBreak)
}
Some(byte) => {
if tokenizer.tokenize_state.size > LINK_REFERENCE_SIZE_MAX {
- tokenizer.exit(Token::Data);
- State::Retry(Name::LabelAtBreak)
+ tokenizer.exit(Name::Data);
+ State::Retry(StateName::LabelAtBreak)
} else {
tokenizer.consume();
tokenizer.tokenize_state.size += 1;
@@ -192,9 +192,9 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.seen = true;
}
State::Next(if matches!(byte, b'\\') {
- Name::LabelEscape
+ StateName::LabelEscape
} else {
- Name::LabelInside
+ StateName::LabelInside
})
}
}
@@ -212,8 +212,8 @@ pub fn escape(tokenizer: &mut Tokenizer) -> State {
Some(b'[' | b'\\' | b']') => {
tokenizer.consume();
tokenizer.tokenize_state.size += 1;
- State::Next(Name::LabelInside)
+ State::Next(StateName::LabelInside)
}
- _ => State::Retry(Name::LabelInside),
+ _ => State::Retry(StateName::LabelInside),
}
}
diff --git a/src/construct/partial_non_lazy_continuation.rs b/src/construct/partial_non_lazy_continuation.rs
index 497c81e..76854c8 100644
--- a/src/construct/partial_non_lazy_continuation.rs
+++ b/src/construct/partial_non_lazy_continuation.rs
@@ -10,8 +10,8 @@
//! [code_indented]: crate::construct::code_indented
//! [html_flow]: crate::construct::html_flow
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of continuation.
@@ -24,10 +24,10 @@ use crate::tokenizer::Tokenizer;
pub fn start(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'\n') => {
- tokenizer.enter(Token::LineEnding);
+ tokenizer.enter(Name::LineEnding);
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::NonLazyContinuationAfter)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::NonLazyContinuationAfter)
}
_ => State::Nok,
}
diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs
index a8e8f06..3fc9484 100644
--- a/src/construct/partial_space_or_tab.rs
+++ b/src/construct/partial_space_or_tab.rs
@@ -4,10 +4,10 @@
//!
//! * [`micromark-factory-space/index.js` in `micromark`](https://github.com/micromark/micromark/blob/main/packages/micromark-factory-space/dev/index.js)
-use crate::state::{Name, State};
+use crate::event::{Content, Name};
+use crate::state::{Name as StateName, State};
use crate::subtokenize::link;
-use crate::token::Token;
-use crate::tokenizer::{ContentType, Tokenizer};
+use crate::tokenizer::Tokenizer;
/// Options to parse `space_or_tab`.
#[derive(Debug)]
@@ -17,11 +17,11 @@ pub struct Options {
/// Maximum allowed bytes (inclusive).
pub max: usize,
/// Token type to use for whitespace events.
- pub kind: Token,
+ pub kind: Name,
/// Connect this whitespace to the previous.
pub connect: bool,
/// Embedded content type to use.
- pub content_type: Option<ContentType>,
+ pub content_type: Option<Content>,
}
/// Options to parse `space_or_tab` and one optional eol, but no blank line.
@@ -30,7 +30,7 @@ pub struct EolOptions {
/// Connect this whitespace to the previous.
pub connect: bool,
/// Embedded content type to use.
- pub content_type: Option<ContentType>,
+ pub content_type: Option<Content>,
}
/// One or more `space_or_tab`.
@@ -38,7 +38,7 @@ pub struct EolOptions {
/// ```bnf
/// space_or_tab ::= 1*( ' ' '\t' )
/// ```
-pub fn space_or_tab(tokenizer: &mut Tokenizer) -> Name {
+pub fn space_or_tab(tokenizer: &mut Tokenizer) -> StateName {
space_or_tab_min_max(tokenizer, 1, usize::MAX)
}
@@ -47,11 +47,11 @@ pub fn space_or_tab(tokenizer: &mut Tokenizer) -> Name {
/// ```bnf
/// space_or_tab_min_max ::= x*y( ' ' '\t' )
/// ```
-pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -> Name {
+pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -> StateName {
space_or_tab_with_options(
tokenizer,
Options {
- kind: Token::SpaceOrTab,
+ kind: Name::SpaceOrTab,
min,
max,
content_type: None,
@@ -61,13 +61,13 @@ pub fn space_or_tab_min_max(tokenizer: &mut Tokenizer, min: usize, max: usize) -
}
/// `space_or_tab`, with the given options.
-pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> Name {
+pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) -> StateName {
tokenizer.tokenize_state.space_or_tab_connect = options.connect;
tokenizer.tokenize_state.space_or_tab_content_type = options.content_type;
tokenizer.tokenize_state.space_or_tab_min = options.min;
tokenizer.tokenize_state.space_or_tab_max = options.max;
tokenizer.tokenize_state.space_or_tab_token = options.kind;
- Name::SpaceOrTabStart
+ StateName::SpaceOrTabStart
}
/// `space_or_tab`, or optionally `space_or_tab`, one `eol`, and
@@ -76,7 +76,7 @@ pub fn space_or_tab_with_options(tokenizer: &mut Tokenizer, options: Options) ->
/// ```bnf
/// space_or_tab_eol ::= 1*( ' ' '\t' ) | 0*( ' ' '\t' ) eol 0*( ' ' '\t' )
/// ```
-pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> Name {
+pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> StateName {
space_or_tab_eol_with_options(
tokenizer,
EolOptions {
@@ -87,10 +87,10 @@ pub fn space_or_tab_eol(tokenizer: &mut Tokenizer) -> Name {
}
/// `space_or_tab_eol`, with the given options.
-pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> Name {
+pub fn space_or_tab_eol_with_options(tokenizer: &mut Tokenizer, options: EolOptions) -> StateName {
tokenizer.tokenize_state.space_or_tab_eol_content_type = options.content_type;
tokenizer.tokenize_state.space_or_tab_eol_connect = options.connect;
- Name::SpaceOrTabEolStart
+ StateName::SpaceOrTabEolStart
}
/// Before `space_or_tab`.
@@ -114,9 +114,9 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.space_or_tab_connect = true;
}
- State::Retry(Name::SpaceOrTabInside)
+ State::Retry(StateName::SpaceOrTabInside)
}
- _ => State::Retry(Name::SpaceOrTabAfter),
+ _ => State::Retry(StateName::SpaceOrTabAfter),
}
}
@@ -134,11 +134,11 @@ pub fn inside(tokenizer: &mut Tokenizer) -> State {
{
tokenizer.consume();
tokenizer.tokenize_state.space_or_tab_size += 1;
- State::Next(Name::SpaceOrTabInside)
+ State::Next(StateName::SpaceOrTabInside)
}
_ => {
tokenizer.exit(tokenizer.tokenize_state.space_or_tab_token.clone());
- State::Retry(Name::SpaceOrTabAfter)
+ State::Retry(StateName::SpaceOrTabAfter)
}
}
}
@@ -162,7 +162,7 @@ pub fn after(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.space_or_tab_size = 0;
tokenizer.tokenize_state.space_or_tab_max = 0;
tokenizer.tokenize_state.space_or_tab_min = 0;
- tokenizer.tokenize_state.space_or_tab_token = Token::SpaceOrTab;
+ tokenizer.tokenize_state.space_or_tab_token = Name::SpaceOrTab;
state
}
@@ -170,7 +170,7 @@ pub fn eol_start(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_with_options(
tokenizer,
Options {
- kind: Token::SpaceOrTab,
+ kind: Name::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: tokenizer
@@ -183,8 +183,8 @@ pub fn eol_start(tokenizer: &mut Tokenizer) -> State {
tokenizer.attempt(
name,
- State::Next(Name::SpaceOrTabEolAfterFirst),
- State::Next(Name::SpaceOrTabEolAtEol),
+ State::Next(StateName::SpaceOrTabEolAfterFirst),
+ State::Next(StateName::SpaceOrTabEolAtEol),
)
}
@@ -199,7 +199,7 @@ pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.space_or_tab_eol_connect = true;
}
- State::Retry(Name::SpaceOrTabEolAtEol)
+ State::Retry(StateName::SpaceOrTabEolAtEol)
}
/// `space_or_tab_eol`: after optionally first `space_or_tab`.
@@ -212,7 +212,7 @@ pub fn eol_after_first(tokenizer: &mut Tokenizer) -> State {
pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State {
if let Some(b'\n') = tokenizer.current {
tokenizer.enter_with_content(
- Token::LineEnding,
+ Name::LineEnding,
tokenizer
.tokenize_state
.space_or_tab_eol_content_type
@@ -231,8 +231,8 @@ pub fn eol_at_eol(tokenizer: &mut Tokenizer) -> State {
}
tokenizer.consume();
- tokenizer.exit(Token::LineEnding);
- State::Next(Name::SpaceOrTabEolAfterEol)
+ tokenizer.exit(Name::LineEnding);
+ State::Next(StateName::SpaceOrTabEolAfterEol)
} else {
let ok = tokenizer.tokenize_state.space_or_tab_eol_ok;
tokenizer.tokenize_state.space_or_tab_eol_content_type = None;
@@ -258,7 +258,7 @@ pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_with_options(
tokenizer,
Options {
- kind: Token::SpaceOrTab,
+ kind: Name::SpaceOrTab,
min: 1,
max: usize::MAX,
content_type: tokenizer
@@ -270,8 +270,8 @@ pub fn eol_after_eol(tokenizer: &mut Tokenizer) -> State {
);
tokenizer.attempt(
name,
- State::Next(Name::SpaceOrTabEolAfterMore),
- State::Next(Name::SpaceOrTabEolAfterMore),
+ State::Next(StateName::SpaceOrTabEolAfterMore),
+ State::Next(StateName::SpaceOrTabEolAfterMore),
)
}
diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs
index 11c28bd..be06c02 100644
--- a/src/construct/partial_title.rs
+++ b/src/construct/partial_title.rs
@@ -31,10 +31,10 @@
//! [label_end]: crate::construct::label_end
use crate::construct::partial_space_or_tab::{space_or_tab_eol_with_options, EolOptions};
-use crate::state::{Name, State};
+use crate::event::{Content, Name};
+use crate::state::{Name as StateName, State};
use crate::subtokenize::link;
-use crate::token::Token;
-use crate::tokenizer::{ContentType, Tokenizer};
+use crate::tokenizer::Tokenizer;
/// Before a title.
///
@@ -51,7 +51,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
tokenizer.enter(tokenizer.tokenize_state.token_2.clone());
tokenizer.consume();
tokenizer.exit(tokenizer.tokenize_state.token_2.clone());
- State::Next(Name::TitleBegin)
+ State::Next(StateName::TitleBegin)
}
_ => State::Nok,
}
@@ -80,7 +80,7 @@ pub fn begin(tokenizer: &mut Tokenizer) -> State {
}
_ => {
tokenizer.enter(tokenizer.tokenize_state.token_3.clone());
- State::Retry(Name::TitleAtBreak)
+ State::Retry(StateName::TitleAtBreak)
}
}
}
@@ -102,25 +102,25 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
let name = space_or_tab_eol_with_options(
tokenizer,
EolOptions {
- content_type: Some(ContentType::String),
+ content_type: Some(Content::String),
connect: tokenizer.tokenize_state.connect,
},
);
tokenizer.attempt(
name,
- State::Next(Name::TitleAfterEol),
- State::Next(Name::TitleAtBlankLine),
+ State::Next(StateName::TitleAfterEol),
+ State::Next(StateName::TitleAtBlankLine),
)
}
Some(b'"' | b'\'' | b')')
if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker =>
{
tokenizer.exit(tokenizer.tokenize_state.token_3.clone());
- State::Retry(Name::TitleBegin)
+ State::Retry(StateName::TitleBegin)
}
Some(_) => {
- tokenizer.enter_with_content(Token::Data, Some(ContentType::String));
+ tokenizer.enter_with_content(Name::Data, Some(Content::String));
if tokenizer.tokenize_state.connect {
let index = tokenizer.events.len() - 1;
@@ -129,7 +129,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.connect = true;
}
- State::Retry(Name::TitleInside)
+ State::Retry(StateName::TitleInside)
}
}
}
@@ -143,7 +143,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
/// ```
pub fn after_eol(tokenizer: &mut Tokenizer) -> State {
tokenizer.tokenize_state.connect = true;
- State::Retry(Name::TitleAtBreak)
+ State::Retry(StateName::TitleAtBreak)
}
/// In a title, at a blank line.
@@ -169,21 +169,21 @@ pub fn at_blank_line(tokenizer: &mut Tokenizer) -> State {
pub fn inside(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
None | Some(b'\n') => {
- tokenizer.exit(Token::Data);
- State::Retry(Name::TitleAtBreak)
+ tokenizer.exit(Name::Data);
+ State::Retry(StateName::TitleAtBreak)
}
Some(b'"' | b'\'' | b')')
if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker =>
{
- tokenizer.exit(Token::Data);
- State::Retry(Name::TitleAtBreak)
+ tokenizer.exit(Name::Data);
+ State::Retry(StateName::TitleAtBreak)
}
Some(byte) => {
tokenizer.consume();
State::Next(if matches!(byte, b'\\') {
- Name::TitleEscape
+ StateName::TitleEscape
} else {
- Name::TitleInside
+ StateName::TitleInside
})
}
}
@@ -199,8 +199,8 @@ pub fn escape(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'"' | b'\'' | b')') => {
tokenizer.consume();
- State::Next(Name::TitleInside)
+ State::Next(StateName::TitleInside)
}
- _ => State::Retry(Name::TitleInside),
+ _ => State::Retry(StateName::TitleInside),
}
}
diff --git a/src/construct/partial_whitespace.rs b/src/construct/partial_whitespace.rs
index 0905e10..688e7fd 100644
--- a/src/construct/partial_whitespace.rs
+++ b/src/construct/partial_whitespace.rs
@@ -46,8 +46,8 @@
//! [html]: https://html.spec.whatwg.org/multipage/text-level-semantics.html#the-br-element
use crate::constant::HARD_BREAK_PREFIX_SIZE_MIN;
-use crate::token::Token;
-use crate::tokenizer::{Event, EventType, Tokenizer};
+use crate::event::{Event, Kind, Name};
+use crate::tokenizer::Tokenizer;
use crate::util::slice::{Position, Slice};
/// Resolve whitespace.
@@ -57,12 +57,12 @@ pub fn resolve_whitespace(tokenizer: &mut Tokenizer, hard_break: bool, trim_whol
while index < tokenizer.events.len() {
let event = &tokenizer.events[index];
- if event.event_type == EventType::Exit && event.token_type == Token::Data {
+ if event.kind == Kind::Exit && event.name == Name::Data {
let trim_start = (trim_whole && index == 1)
- || (index > 1 && tokenizer.events[index - 2].token_type == Token::LineEnding);
+ || (index > 1 && tokenizer.events[index - 2].name == Name::LineEnding);
let trim_end = (trim_whole && index == tokenizer.events.len() - 1)
|| (index + 1 < tokenizer.events.len()
- && tokenizer.events[index + 1].token_type == Token::LineEnding);
+ && tokenizer.events[index + 1].name == Name::LineEnding);
trim_data(tokenizer, index, trim_start, trim_end, hard_break);
}
@@ -98,21 +98,21 @@ fn trim_data(
}
let diff = slice.bytes.len() - index;
- let token_type = if hard_break
+ let name = if hard_break
&& spaces_only
&& diff >= HARD_BREAK_PREFIX_SIZE_MIN
&& exit_index + 1 < tokenizer.events.len()
{
- Token::HardBreakTrailing
+ Name::HardBreakTrailing
} else {
- Token::SpaceOrTab
+ Name::SpaceOrTab
};
// The whole data is whitespace.
// We can be very fast: we only change the token types.
if index == 0 {
- tokenizer.events[exit_index - 1].token_type = token_type.clone();
- tokenizer.events[exit_index].token_type = token_type;
+ tokenizer.events[exit_index - 1].name = name.clone();
+ tokenizer.events[exit_index].name = name;
return;
}
@@ -128,14 +128,14 @@ fn trim_data(
0,
vec![
Event {
- event_type: EventType::Enter,
- token_type: token_type.clone(),
+ kind: Kind::Enter,
+ name: name.clone(),
point: enter_point.clone(),
link: None,
},
Event {
- event_type: EventType::Exit,
- token_type,
+ kind: Kind::Exit,
+ name,
point: exit_point,
link: None,
},
@@ -159,8 +159,8 @@ fn trim_data(
// The whole data is whitespace.
// We can be very fast: we only change the token types.
if index == slice.bytes.len() {
- tokenizer.events[exit_index - 1].token_type = Token::SpaceOrTab;
- tokenizer.events[exit_index].token_type = Token::SpaceOrTab;
+ tokenizer.events[exit_index - 1].name = Name::SpaceOrTab;
+ tokenizer.events[exit_index].name = Name::SpaceOrTab;
return;
}
@@ -176,14 +176,14 @@ fn trim_data(
0,
vec![
Event {
- event_type: EventType::Enter,
- token_type: Token::SpaceOrTab,
+ kind: Kind::Enter,
+ name: Name::SpaceOrTab,
point: enter_point,
link: None,
},
Event {
- event_type: EventType::Exit,
- token_type: Token::SpaceOrTab,
+ kind: Kind::Exit,
+ name: Name::SpaceOrTab,
point: exit_point.clone(),
link: None,
},
diff --git a/src/construct/thematic_break.rs b/src/construct/thematic_break.rs
index fc71d73..8e3c4f4 100644
--- a/src/construct/thematic_break.rs
+++ b/src/construct/thematic_break.rs
@@ -50,8 +50,8 @@
use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
use crate::constant::{TAB_SIZE, THEMATIC_BREAK_MARKER_COUNT_MIN};
-use crate::state::{Name, State};
-use crate::token::Token;
+use crate::event::Name;
+use crate::state::{Name as StateName, State};
use crate::tokenizer::Tokenizer;
/// Start of a thematic break.
@@ -62,7 +62,7 @@ use crate::tokenizer::Tokenizer;
/// ```
pub fn start(tokenizer: &mut Tokenizer) -> State {
if tokenizer.parse_state.constructs.thematic_break {
- tokenizer.enter(Token::ThematicBreak);
+ tokenizer.enter(Name::ThematicBreak);
let name = space_or_tab_min_max(
tokenizer,
0,
@@ -73,7 +73,11 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
},
);
- tokenizer.attempt(name, State::Next(Name::ThematicBreakBefore), State::Nok)
+ tokenizer.attempt(
+ name,
+ State::Next(StateName::ThematicBreakBefore),
+ State::Nok,
+ )
} else {
State::Nok
}
@@ -89,7 +93,7 @@ pub fn before(tokenizer: &mut Tokenizer) -> State {
match tokenizer.current {
Some(b'*' | b'-' | b'_') => {
tokenizer.tokenize_state.marker = tokenizer.current.unwrap();
- State::Retry(Name::ThematicBreakAtBreak)
+ State::Retry(StateName::ThematicBreakAtBreak)
}
_ => State::Nok,
}
@@ -106,7 +110,7 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
None | Some(b'\n') if tokenizer.tokenize_state.size >= THEMATIC_BREAK_MARKER_COUNT_MIN => {
tokenizer.tokenize_state.marker = 0;
tokenizer.tokenize_state.size = 0;
- tokenizer.exit(Token::ThematicBreak);
+ tokenizer.exit(Name::ThematicBreak);
// Feel free to interrupt.
tokenizer.interrupt = false;
State::Ok
@@ -114,8 +118,8 @@ pub fn at_break(tokenizer: &mut Tokenizer) -> State {
Some(b'*' | b'-' | b'_')
if tokenizer.current.unwrap() == tokenizer.tokenize_state.marker =>
{
- tokenizer.enter(Token::ThematicBreakSequence);
- State::Retry(Name::ThematicBreakSequence)
+ tokenizer.enter(Name::ThematicBreakSequence);
+ State::Retry(StateName::ThematicBreakSequence)
}
_ => {
tokenizer.tokenize_state.marker = 0;
@@ -138,15 +142,15 @@ pub fn sequence(tokenizer: &mut Tokenizer) -> State {
{
tokenizer.consume();
tokenizer.tokenize_state.size += 1;
- State::Next(Name::ThematicBreakSequence)
+ State::Next(StateName::ThematicBreakSequence)
}
_ => {
- tokenizer.exit(Token::ThematicBreakSequence);
+ tokenizer.exit(Name::ThematicBreakSequence);
let name = space_or_tab(tokenizer);
tokenizer.attempt(
name,
- State::Next(Name::ThematicBreakAtBreak),
- State::Next(Name::ThematicBreakAtBreak),
+ State::Next(StateName::ThematicBreakAtBreak),
+ State::Next(StateName::ThematicBreakAtBreak),
)
}
}