Refactor to work on `char`s

Previously, a custom char implementation was used.
This was easier to work with, as sometimes “virtual” characters are injected,
or characters are ignored.

This replaces that with working on actual `char`s.
In the hope of in the future working on `u8`s, even.

This simplifies the state machine somewhat, as only `\n` is fed, regardless of
whether it was a CRLF, CR, or LF.
It also feeds `' '` instead of virtual spaces.

The BOM, if present, is now available as a `ByteOrderMark` event.

1 files changed, 4 insertions, 4 deletions

diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index a0f7545..f9dd3f7 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -60,7 +60,7 @@
 use crate::constant::TAB_SIZE;
 use crate::construct::partial_space_or_tab::{space_or_tab, space_or_tab_min_max};
 use crate::token::Token;
-use crate::tokenizer::{Code, EventType, State, Tokenizer};
+use crate::tokenizer::{EventType, State, Tokenizer};
 use crate::util::skip::opt_back as skip_opt_back;
 
 /// Kind of underline.
@@ -148,7 +148,7 @@ pub fn start(tokenizer: &mut Tokenizer) -> State {
 /// ```
 fn before(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
-        Code::Char(char) if char == '-' || char == '=' => {
+        Some(char) if matches!(char, '-' | '=') => {
             tokenizer.enter(Token::HeadingSetextUnderline);
             inside(tokenizer, Kind::from_char(char))
         }
@@ -165,7 +165,7 @@ fn before(tokenizer: &mut Tokenizer) -> State {
 /// ```
 fn inside(tokenizer: &mut Tokenizer, kind: Kind) -> State {
     match tokenizer.current {
-        Code::Char(char) if char == kind.as_char() => {
+        Some(char) if char == kind.as_char() => {
             tokenizer.consume();
             State::Fn(Box::new(move |t| inside(t, kind)))
         }
@@ -185,7 +185,7 @@ fn inside(tokenizer: &mut Tokenizer, kind: Kind) -> State {
 /// ```
 fn after(tokenizer: &mut Tokenizer) -> State {
     match tokenizer.current {
-        Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
+        None | Some('\n') => {
             // Feel free to interrupt.
             tokenizer.interrupt = false;
             tokenizer.register_resolver("heading_setext".to_string(), Box::new(resolve));