Add support for line endings in string

1 files changed, 23 insertions, 9 deletions

diff --git a/src/content/string.rs b/src/content/string.rs
index f591cd7..efb6e60 100644
--- a/src/content/string.rs
+++ b/src/content/string.rs
@@ -43,13 +43,28 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// |qwe
 /// ```
 fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
-    if let Code::None = code {
-        (State::Ok, None)
-    } else {
-        tokenizer.enter(TokenType::Data);
-        tokenizer.consume(code);
-        (State::Fn(Box::new(in_data)), None)
+    match code {
+        Code::None => (State::Ok, None),
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+            tokenizer.enter(TokenType::LineEnding);
+            tokenizer.consume(code);
+            tokenizer.exit(TokenType::LineEnding);
+            (State::Fn(Box::new(start)), None)
+        }
+        _ => {
+            tokenizer.enter(TokenType::Data);
+            tokenizer.consume(code);
+            (State::Fn(Box::new(in_data)), None)
+        }
     }
+
+    // if let Code::None = code {
+    //     (State::Ok, None)
+    // } else {
+    //     tokenizer.enter(TokenType::Data);
+    //     tokenizer.consume(code);
+    //     (State::Fn(Box::new(in_data)), None)
+    // }
 }
 
 /// In data.
@@ -59,10 +74,9 @@ fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// ```
 fn in_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
     match code {
-        // To do: line endings.
-        Code::None => {
+        Code::None | Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
             tokenizer.exit(TokenType::Data);
-            (State::Ok, None)
+            before_data(tokenizer, code)
         }
         // To do: somehow get these markers from constructs.
         Code::Char('&' | '\\') => {