diff options
author | Titus Wormer <tituswormer@gmail.com> | 2022-06-16 13:34:05 +0200 |
---|---|---|
committer | Titus Wormer <tituswormer@gmail.com> | 2022-06-16 13:34:05 +0200 |
commit | ef14d6581848ba5052d3389bb61fc96645551eef (patch) | |
tree | c1b2de69814950c5d2813b6b5798998c45635394 /src/tokenizer.rs | |
parent | 7350acc692a79d9d4cf56afbc53ac3c9f2a6237c (diff) | |
download | markdown-rs-ef14d6581848ba5052d3389bb61fc96645551eef.tar.gz markdown-rs-ef14d6581848ba5052d3389bb61fc96645551eef.tar.bz2 markdown-rs-ef14d6581848ba5052d3389bb61fc96645551eef.zip |
Refactor to reorder thing alphabetically
Diffstat (limited to '')
-rw-r--r-- | src/tokenizer.rs | 28 |
1 files changed, 7 insertions, 21 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index da45ee5..0aae480 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -24,66 +24,52 @@ pub enum TokenType { AutolinkMarker, AutolinkProtocol, AutolinkEmail, - AtxHeading, AtxHeadingSequence, AtxHeadingWhitespace, AtxHeadingText, - + BlankLineEnding, + BlankLineWhitespace, CharacterEscape, CharacterEscapeMarker, CharacterEscapeValue, - CharacterReference, CharacterReferenceMarker, CharacterReferenceMarkerNumeric, CharacterReferenceMarkerHexadecimal, CharacterReferenceMarkerSemi, CharacterReferenceValue, - CodeFenced, CodeFencedFence, CodeFencedFenceSequence, CodeFencedFenceWhitespace, CodeFencedFenceInfo, CodeFencedFenceMeta, - + CodeFlowChunk, CodeIndented, CodeIndentedPrefixWhitespace, - CodeText, CodeTextSequence, CodeTextLineEnding, CodeTextData, - - CodeFlowChunk, - + Content, Data, - HardBreakEscape, HardBreakEscapeMarker, HardBreakTrailing, HardBreakTrailingSpace, - HtmlFlow, HtmlFlowData, - HtmlText, HtmlTextData, - + LineEnding, + Paragraph, ThematicBreak, ThematicBreakSequence, ThematicBreakWhitespace, - Whitespace, - LineEnding, - BlankLineEnding, - BlankLineWhitespace, - - Content, - - Paragraph, + // Chunks are tokenizer, but unraveled by `subtokenize`. ChunkContent, ChunkString, ChunkText, |