diff options
Diffstat (limited to '')
-rw-r--r-- | src/tokenizer.rs | 28 |
1 files changed, 7 insertions, 21 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs index da45ee5..0aae480 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -24,66 +24,52 @@ pub enum TokenType { AutolinkMarker, AutolinkProtocol, AutolinkEmail, - AtxHeading, AtxHeadingSequence, AtxHeadingWhitespace, AtxHeadingText, - + BlankLineEnding, + BlankLineWhitespace, CharacterEscape, CharacterEscapeMarker, CharacterEscapeValue, - CharacterReference, CharacterReferenceMarker, CharacterReferenceMarkerNumeric, CharacterReferenceMarkerHexadecimal, CharacterReferenceMarkerSemi, CharacterReferenceValue, - CodeFenced, CodeFencedFence, CodeFencedFenceSequence, CodeFencedFenceWhitespace, CodeFencedFenceInfo, CodeFencedFenceMeta, - + CodeFlowChunk, CodeIndented, CodeIndentedPrefixWhitespace, - CodeText, CodeTextSequence, CodeTextLineEnding, CodeTextData, - - CodeFlowChunk, - + Content, Data, - HardBreakEscape, HardBreakEscapeMarker, HardBreakTrailing, HardBreakTrailingSpace, - HtmlFlow, HtmlFlowData, - HtmlText, HtmlTextData, - + LineEnding, + Paragraph, ThematicBreak, ThematicBreakSequence, ThematicBreakWhitespace, - Whitespace, - LineEnding, - BlankLineEnding, - BlankLineWhitespace, - - Content, - - Paragraph, + // Chunks are tokenizer, but unraveled by `subtokenize`. ChunkContent, ChunkString, ChunkText, |