aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r--src/tokenizer.rs32
1 files changed, 5 insertions, 27 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index c1bb61b..de27d12 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -25,7 +25,6 @@ pub enum TokenType {
AutolinkProtocol,
AutolinkEmail,
BlankLineEnding,
- BlankLineWhitespace,
CharacterEscape,
CharacterEscapeMarker,
CharacterEscapeValue,
@@ -38,12 +37,10 @@ pub enum TokenType {
CodeFenced,
CodeFencedFence,
CodeFencedFenceSequence,
- CodeFencedFenceWhitespace,
CodeFencedFenceInfo,
CodeFencedFenceMeta,
CodeFlowChunk,
CodeIndented,
- CodeIndentedPrefixWhitespace,
CodeText,
CodeTextSequence,
CodeTextLineEnding,
@@ -81,7 +78,6 @@ pub enum TokenType {
Paragraph,
ThematicBreak,
ThematicBreakSequence,
- ThematicBreakWhitespace,
Whitespace,
// Chunks are tokenizer, but unraveled by `subtokenize`.
@@ -114,7 +110,7 @@ pub struct Point {
/// 1-indexed line number.
pub line: usize,
/// 1-indexed column number.
- /// Note that this is increases up to a tab stop for tabs.
+ /// This is increases up to a tab stop for tabs.
/// Some editors count tabs as 1 character, so this position is not always
/// the same as editors.
pub column: usize,
@@ -485,32 +481,14 @@ impl Tokenizer {
)
}
- pub fn attempt_3(
- &mut self,
- a: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
- b: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
- c: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
- done: impl FnOnce(bool) -> Box<StateFn> + 'static,
- ) -> Box<StateFn> {
- self.call_multiple(
- false,
- Some(Box::new(a)),
- Some(Box::new(b)),
- Some(Box::new(c)),
- None,
- None,
- None,
- None,
- done,
- )
- }
-
- pub fn attempt_4(
+ #[allow(clippy::too_many_arguments, clippy::many_single_char_names)]
+ pub fn attempt_5(
&mut self,
a: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
b: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
c: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
d: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
+ e: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
done: impl FnOnce(bool) -> Box<StateFn> + 'static,
) -> Box<StateFn> {
self.call_multiple(
@@ -519,7 +497,7 @@ impl Tokenizer {
Some(Box::new(b)),
Some(Box::new(c)),
Some(Box::new(d)),
- None,
+ Some(Box::new(e)),
None,
None,
done,