aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
authorLibravatar Titus Wormer <tituswormer@gmail.com>2022-06-16 13:34:05 +0200
committerLibravatar Titus Wormer <tituswormer@gmail.com>2022-06-16 13:34:05 +0200
commitef14d6581848ba5052d3389bb61fc96645551eef (patch)
treec1b2de69814950c5d2813b6b5798998c45635394 /src/tokenizer.rs
parent7350acc692a79d9d4cf56afbc53ac3c9f2a6237c (diff)
downloadmarkdown-rs-ef14d6581848ba5052d3389bb61fc96645551eef.tar.gz
markdown-rs-ef14d6581848ba5052d3389bb61fc96645551eef.tar.bz2
markdown-rs-ef14d6581848ba5052d3389bb61fc96645551eef.zip
Refactor to reorder thing alphabetically
Diffstat (limited to '')
-rw-r--r--src/tokenizer.rs28
1 files changed, 7 insertions, 21 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index da45ee5..0aae480 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -24,66 +24,52 @@ pub enum TokenType {
AutolinkMarker,
AutolinkProtocol,
AutolinkEmail,
-
AtxHeading,
AtxHeadingSequence,
AtxHeadingWhitespace,
AtxHeadingText,
-
+ BlankLineEnding,
+ BlankLineWhitespace,
CharacterEscape,
CharacterEscapeMarker,
CharacterEscapeValue,
-
CharacterReference,
CharacterReferenceMarker,
CharacterReferenceMarkerNumeric,
CharacterReferenceMarkerHexadecimal,
CharacterReferenceMarkerSemi,
CharacterReferenceValue,
-
CodeFenced,
CodeFencedFence,
CodeFencedFenceSequence,
CodeFencedFenceWhitespace,
CodeFencedFenceInfo,
CodeFencedFenceMeta,
-
+ CodeFlowChunk,
CodeIndented,
CodeIndentedPrefixWhitespace,
-
CodeText,
CodeTextSequence,
CodeTextLineEnding,
CodeTextData,
-
- CodeFlowChunk,
-
+ Content,
Data,
-
HardBreakEscape,
HardBreakEscapeMarker,
HardBreakTrailing,
HardBreakTrailingSpace,
-
HtmlFlow,
HtmlFlowData,
-
HtmlText,
HtmlTextData,
-
+ LineEnding,
+ Paragraph,
ThematicBreak,
ThematicBreakSequence,
ThematicBreakWhitespace,
-
Whitespace,
- LineEnding,
- BlankLineEnding,
- BlankLineWhitespace,
-
- Content,
-
- Paragraph,
+ // Chunks are tokenizer, but unraveled by `subtokenize`.
ChunkContent,
ChunkString,
ChunkText,