From 0450e7c2b12bd3ef53e0cffb60a3dd860325b478 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Mon, 4 Jul 2022 15:21:11 +0200 Subject: Add support for unicode punctuation --- src/tokenizer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/tokenizer.rs') diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9d870c9..1fa94d7 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -11,9 +11,9 @@ //! [`attempt`]: Tokenizer::attempt //! [`check`]: Tokenizer::check -use std::collections::HashMap; use crate::constant::TAB_SIZE; use crate::parser::ParseState; +use std::collections::HashMap; /// Semantic label of a span. // To do: figure out how to share this so extensions can add their own stuff, -- cgit