From f7e5fb852dc9c416b9eeb1f0d4f2d51ba5b68456 Mon Sep 17 00:00:00 2001
From: Titus Wormer <tituswormer@gmail.com>
Date: Thu, 28 Jul 2022 16:48:00 +0200
Subject: Refactor to work on `char`s
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously, a custom char implementation was used.
This was easier to work with, as sometimes “virtual” characters are injected,
or characters are ignored.

This replaces that with working on actual `char`s.
In the hope of in the future working on `u8`s, even.

This simplifies the state machine somewhat, as only `\n` is fed, regardless of
whether it was a CRLF, CR, or LF.
It also feeds `' '` instead of virtual spaces.

The BOM, if present, is now available as a `ByteOrderMark` event.
---
 src/construct/code_indented.rs | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

(limited to 'src/construct/code_indented.rs')

diff --git a/src/construct/code_indented.rs b/src/construct/code_indented.rs
index 015c4a0..bb1615c 100644
--- a/src/construct/code_indented.rs
+++ b/src/construct/code_indented.rs
@@ -48,7 +48,7 @@
 use super::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
 use crate::constant::TAB_SIZE;
 use crate::token::Token;
-use crate::tokenizer::{Code, State, Tokenizer};
+use crate::tokenizer::{State, Tokenizer};
 
 /// Start of code (indented).
 ///
@@ -78,11 +78,10 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
 /// ```
 fn at_break(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
-        Code::None => after(tokenizer),
-        Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => tokenizer
-            .attempt(further_start, |ok| {
-                Box::new(if ok { at_break } else { after })
-            })(tokenizer),
+        None => after(tokenizer),
+        Some('\n') => tokenizer.attempt(further_start, |ok| {
+            Box::new(if ok { at_break } else { after })
+        })(tokenizer),
         _ => {
             tokenizer.enter(Token::CodeFlowChunk);
             content(tokenizer)
@@ -98,7 +97,7 @@ fn at_break(tokenizer: &mut Tokenizer) -> State {
 /// ```
 fn content(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
-        Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
+        None | Some('\n') => {
             tokenizer.exit(Token::CodeFlowChunk);
             at_break(tokenizer)
         }
@@ -134,7 +133,7 @@ fn further_start(tokenizer: &mut Tokenizer) -> State {
         State::Nok
     } else {
         match tokenizer.current {
-            Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
+            Some('\n') => {
                 tokenizer.enter(Token::LineEnding);
                 tokenizer.consume();
                 tokenizer.exit(Token::LineEnding);
@@ -178,7 +177,7 @@ fn further_begin(tokenizer: &mut Tokenizer) -> State {
 /// ```
 fn further_after(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
-        Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => further_start(tokenizer),
+        Some('\n') => further_start(tokenizer),
         _ => State::Nok,
     }
 }
-- 
cgit