Fix support for deep subtokenization

*   Fix a couple of forgotten line ending handling in html (text)
*   Fix missing initial case for html (text) not having a `<` 😬
*   Add line ending handling to `text` construct

1 files changed, 14 insertions, 7 deletions

diff --git a/src/content/text.rs b/src/content/text.rs
index 73c2d55..433d030 100644
--- a/src/content/text.rs
+++ b/src/content/text.rs
@@ -52,12 +52,19 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// |qwe
 /// ```
 fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
-    if let Code::None = code {
-        (State::Ok, None)
-    } else {
-        tokenizer.enter(TokenType::Data);
-        tokenizer.consume(code);
-        (State::Fn(Box::new(in_data)), None)
+    match code {
+        Code::None => (State::Ok, None),
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+            tokenizer.enter(TokenType::LineEnding);
+            tokenizer.consume(code);
+            tokenizer.exit(TokenType::LineEnding);
+            (State::Fn(Box::new(start)), None)
+        }
+        _ => {
+            tokenizer.enter(TokenType::Data);
+            tokenizer.consume(code);
+            (State::Fn(Box::new(in_data)), None)
+        }
     }
 }
 
@@ -73,7 +80,7 @@ fn in_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
             (State::Ok, None)
         }
         // To do: somehow get these markers from constructs.
-        Code::Char('&' | '\\' | '<') => {
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n' | '&' | '\\' | '<') => {
             tokenizer.exit(TokenType::Data);
             start(tokenizer, code)
         }