From ef14d6581848ba5052d3389bb61fc96645551eef Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Thu, 16 Jun 2022 13:34:05 +0200 Subject: Refactor to reorder thing alphabetically --- src/tokenizer.rs | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) (limited to 'src/tokenizer.rs') diff --git a/src/tokenizer.rs b/src/tokenizer.rs index da45ee5..0aae480 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -24,66 +24,52 @@ pub enum TokenType { AutolinkMarker, AutolinkProtocol, AutolinkEmail, - AtxHeading, AtxHeadingSequence, AtxHeadingWhitespace, AtxHeadingText, - + BlankLineEnding, + BlankLineWhitespace, CharacterEscape, CharacterEscapeMarker, CharacterEscapeValue, - CharacterReference, CharacterReferenceMarker, CharacterReferenceMarkerNumeric, CharacterReferenceMarkerHexadecimal, CharacterReferenceMarkerSemi, CharacterReferenceValue, - CodeFenced, CodeFencedFence, CodeFencedFenceSequence, CodeFencedFenceWhitespace, CodeFencedFenceInfo, CodeFencedFenceMeta, - + CodeFlowChunk, CodeIndented, CodeIndentedPrefixWhitespace, - CodeText, CodeTextSequence, CodeTextLineEnding, CodeTextData, - - CodeFlowChunk, - + Content, Data, - HardBreakEscape, HardBreakEscapeMarker, HardBreakTrailing, HardBreakTrailingSpace, - HtmlFlow, HtmlFlowData, - HtmlText, HtmlTextData, - + LineEnding, + Paragraph, ThematicBreak, ThematicBreakSequence, ThematicBreakWhitespace, - Whitespace, - LineEnding, - BlankLineEnding, - BlankLineWhitespace, - - Content, - - Paragraph, + // Chunks are tokenizer, but unraveled by `subtokenize`. ChunkContent, ChunkString, ChunkText, -- cgit