From 17f4eec55ad0a5f74aedbcff6c2f0119ad52e584 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Fri, 10 Jun 2022 16:47:43 +0200 Subject: Add text content type * Add character reference and character escapes in text * Add recursive subtokenization --- src/tokenizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/tokenizer.rs') diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 1746a19..4d235ed 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -63,10 +63,10 @@ pub enum TokenType { BlankLineWhitespace, Content, - ContentChunk, Paragraph, + ChunkContent, ChunkString, ChunkText, } -- cgit