From f7e5fb852dc9c416b9eeb1f0d4f2d51ba5b68456 Mon Sep 17 00:00:00 2001 From: Titus Wormer Date: Thu, 28 Jul 2022 16:48:00 +0200 Subject: Refactor to work on `char`s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, a custom char implementation was used. This was easier to work with, as sometimes “virtual” characters are injected, or characters are ignored. This replaces that with working on actual `char`s. In the hope of in the future working on `u8`s, even. This simplifies the state machine somewhat, as only `\n` is fed, regardless of whether it was a CRLF, CR, or LF. It also feeds `' '` instead of virtual spaces. The BOM, if present, is now available as a `ByteOrderMark` event. --- src/content/flow.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'src/content/flow.rs') diff --git a/src/content/flow.rs b/src/content/flow.rs index ea09cd9..09c4e2c 100644 --- a/src/content/flow.rs +++ b/src/content/flow.rs @@ -27,7 +27,7 @@ use crate::construct::{ thematic_break::start as thematic_break, }; use crate::token::Token; -use crate::tokenizer::{Code, State, Tokenizer}; +use crate::tokenizer::{State, Tokenizer}; /// Before flow. /// @@ -41,7 +41,7 @@ use crate::tokenizer::{Code, State, Tokenizer}; /// ``` pub fn start(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Code::None => State::Ok, + None => State::Ok, _ => tokenizer.attempt(blank_line, |ok| { Box::new(if ok { blank_line_after } else { initial_before }) })(tokenizer), @@ -62,7 +62,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State { /// ``` fn initial_before(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Code::None => State::Ok, + None => State::Ok, _ => tokenizer.attempt_n( vec![ Box::new(code_indented), @@ -87,8 +87,8 @@ fn initial_before(tokenizer: &mut Tokenizer) -> State { /// ``` fn blank_line_after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Code::None => State::Ok, - Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { + None => State::Ok, + Some('\n') => { tokenizer.enter(Token::BlankLineEnding); tokenizer.consume(); tokenizer.exit(Token::BlankLineEnding); @@ -111,8 +111,8 @@ fn blank_line_after(tokenizer: &mut Tokenizer) -> State { /// ``` fn after(tokenizer: &mut Tokenizer) -> State { match tokenizer.current { - Code::None => State::Ok, - Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => { + None => State::Ok, + Some('\n') => { tokenizer.enter(Token::LineEnding); tokenizer.consume(); tokenizer.exit(Token::LineEnding); -- cgit