aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r--src/tokenizer.rs28
1 files changed, 7 insertions, 21 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index da45ee5..0aae480 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -24,66 +24,52 @@ pub enum TokenType {
AutolinkMarker,
AutolinkProtocol,
AutolinkEmail,
-
AtxHeading,
AtxHeadingSequence,
AtxHeadingWhitespace,
AtxHeadingText,
-
+ BlankLineEnding,
+ BlankLineWhitespace,
CharacterEscape,
CharacterEscapeMarker,
CharacterEscapeValue,
-
CharacterReference,
CharacterReferenceMarker,
CharacterReferenceMarkerNumeric,
CharacterReferenceMarkerHexadecimal,
CharacterReferenceMarkerSemi,
CharacterReferenceValue,
-
CodeFenced,
CodeFencedFence,
CodeFencedFenceSequence,
CodeFencedFenceWhitespace,
CodeFencedFenceInfo,
CodeFencedFenceMeta,
-
+ CodeFlowChunk,
CodeIndented,
CodeIndentedPrefixWhitespace,
-
CodeText,
CodeTextSequence,
CodeTextLineEnding,
CodeTextData,
-
- CodeFlowChunk,
-
+ Content,
Data,
-
HardBreakEscape,
HardBreakEscapeMarker,
HardBreakTrailing,
HardBreakTrailingSpace,
-
HtmlFlow,
HtmlFlowData,
-
HtmlText,
HtmlTextData,
-
+ LineEnding,
+ Paragraph,
ThematicBreak,
ThematicBreakSequence,
ThematicBreakWhitespace,
-
Whitespace,
- LineEnding,
- BlankLineEnding,
- BlankLineWhitespace,
-
- Content,
-
- Paragraph,
+ // Chunks are tokenizer, but unraveled by `subtokenize`.
ChunkContent,
ChunkString,
ChunkText,