Refactor to work on `char`s

Previously, a custom char implementation was used.
This was easier to work with, as sometimes “virtual” characters are injected,
or characters are ignored.

This replaces that with working on actual `char`s.
In the hope of in the future working on `u8`s, even.

This simplifies the state machine somewhat, as only `\n` is fed, regardless of
whether it was a CRLF, CR, or LF.
It also feeds `' '` instead of virtual spaces.

The BOM, if present, is now available as a `ByteOrderMark` event.

1 files changed, 5 insertions, 8 deletions

diff --git a/src/construct/partial_space_or_tab.rs b/src/construct/partial_space_or_tab.rs
index 5f1a917..6070ffe 100644
--- a/src/construct/partial_space_or_tab.rs
+++ b/src/construct/partial_space_or_tab.rs
@@ -6,7 +6,7 @@
 
 use crate::subtokenize::link;
 use crate::token::Token;
-use crate::tokenizer::{Code, ContentType, State, StateFn, Tokenizer};
+use crate::tokenizer::{ContentType, State, StateFn, Tokenizer};
 
 /// Options to parse `space_or_tab`.
 #[derive(Debug)]
@@ -134,7 +134,7 @@ pub fn space_or_tab_eol_with_options(options: EolOptions) -> Box<StateFn> {
 /// ```
 fn start(tokenizer: &mut Tokenizer, mut info: Info) -> State {
     match tokenizer.current {
-        Code::VirtualSpace | Code::Char('\t' | ' ') if info.options.max > 0 => {
+        Some('\t' | ' ') if info.options.max > 0 => {
             tokenizer
                 .enter_with_content(info.options.kind.clone(), info.options.content_type.clone());
 
@@ -165,7 +165,7 @@ fn start(tokenizer: &mut Tokenizer, mut info: Info) -> State {
 /// ```
 fn inside(tokenizer: &mut Tokenizer, mut info: Info) -> State {
     match tokenizer.current {
-        Code::VirtualSpace | Code::Char('\t' | ' ') if info.size < info.options.max => {
+        Some('\t' | ' ') if info.size < info.options.max => {
             tokenizer.consume();
             info.size += 1;
             State::Fn(Box::new(|t| inside(t, info)))
@@ -190,7 +190,7 @@ fn inside(tokenizer: &mut Tokenizer, mut info: Info) -> State {
 /// ```
 fn after_space_or_tab(tokenizer: &mut Tokenizer, mut info: EolInfo) -> State {
     match tokenizer.current {
-        Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
+        Some('\n') => {
             tokenizer.enter_with_content(Token::LineEnding, info.options.content_type.clone());
 
             if info.connect {
@@ -239,10 +239,7 @@ fn after_eol(tokenizer: &mut Tokenizer, info: EolInfo) -> State {
 /// ```
 fn after_more_space_or_tab(tokenizer: &mut Tokenizer) -> State {
     // Blank line not allowed.
-    if matches!(
-        tokenizer.current,
-        Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r')
-    ) {
+    if matches!(tokenizer.current, None | Some('\n')) {
         State::Nok
     } else {
         State::Ok