From f7e5fb852dc9c416b9eeb1f0d4f2d51ba5b68456 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Thu, 28 Jul 2022 16:48:00 +0200 Subject: Refactor to work on `char`s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, a custom char implementation was used. This was easier to work with, as sometimes “virtual” characters are injected, or characters are ignored. This replaces that with working on actual `char`s. In the hope of in the future working on `u8`s, even. This simplifies the state machine somewhat, as only `\n` is fed, regardless of whether it was a CRLF, CR, or LF. It also feeds `' '` instead of virtual spaces. The BOM, if present, is now available as a `ByteOrderMark` event. --- src/token.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'src/token.rs') diff --git a/src/token.rs b/src/token.rs index a0479e1..db3bffc 100644 --- a/src/token.rs +++ b/src/token.rs @@ -157,6 +157,17 @@ pub enum Token { /// | b /// ``` BlockQuotePrefix, + /// Byte order mark. + /// + /// ## Info + /// + /// * **Context**: + /// optional first event + /// * **Content model**: + /// void + /// * **Construct**: + /// [`document`][crate::content::document] + ByteOrderMark, /// Whole character escape. /// /// ## Info @@ -1822,13 +1833,14 @@ pub enum Token { } /// List of void tokens, used to make sure everything is working good. -pub const VOID_TOKENS: [Token; 39] = [ +pub const VOID_TOKENS: [Token; 40] = [ Token::AttentionSequence, Token::AutolinkEmail, Token::AutolinkMarker, Token::AutolinkProtocol, Token::BlankLineEnding, Token::BlockQuoteMarker, + Token::ByteOrderMark, Token::CharacterEscapeMarker, Token::CharacterEscapeValue, Token::CharacterReferenceMarker, -- cgit