aboutsummaryrefslogtreecommitdiffstats
path: root/src/tokenizer.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tokenizer.rs')
-rw-r--r--src/tokenizer.rs2
1 files changed, 0 insertions, 2 deletions
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 9884986..c1bb61b 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -48,7 +48,6 @@ pub enum TokenType {
CodeTextSequence,
CodeTextLineEnding,
CodeTextData,
- Content,
Data,
Definition,
DefinitionLabel,
@@ -86,7 +85,6 @@ pub enum TokenType {
Whitespace,
// Chunks are tokenizer, but unraveled by `subtokenize`.
- ChunkContent,
ChunkString,
ChunkText,
}