Add support for line endings in string

8 files changed, 133 insertions, 64 deletions

diff --git a/src/construct/heading_setext.rs b/src/construct/heading_setext.rs
index 64647cb..579fa71 100644
--- a/src/construct/heading_setext.rs
+++ b/src/construct/heading_setext.rs
@@ -52,7 +52,7 @@
 use crate::constant::TAB_SIZE;
 use crate::construct::partial_space_or_tab::space_or_tab_opt;
 use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
-use crate::util::span::from_exit_event;
+use crate::util::{link::link, span::from_exit_event};
 
 /// Kind of underline.
 #[derive(Debug, Clone, PartialEq)]
@@ -133,16 +133,12 @@ fn text_continue(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 
     match code {
         Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => {
-            let next = tokenizer.events.len();
-            let previous = next - 2;
-
             tokenizer.enter(TokenType::LineEnding);
+            let index = tokenizer.events.len() - 1;
+            link(&mut tokenizer.events, index);
             tokenizer.consume(code);
             tokenizer.exit(TokenType::LineEnding);
 
-            tokenizer.events[previous].next = Some(next);
-            tokenizer.events[next].previous = Some(previous);
-
             (
                 State::Fn(Box::new(tokenizer.go(space_or_tab_opt(), text_line_start))),
                 None,
@@ -160,27 +156,20 @@ fn text_continue(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// ==
 /// ```
 fn text_line_start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
-    let next = tokenizer.events.len() - 2;
-    let previous = next - 2;
+    let index = tokenizer.events.len() - 2;
 
     // Link the whitespace, if it exists.
-    if tokenizer.events[next].token_type == TokenType::Whitespace {
-        tokenizer.events[previous].next = Some(next);
-        tokenizer.events[next].previous = Some(previous);
+    if tokenizer.events[index].token_type == TokenType::Whitespace {
+        link(&mut tokenizer.events, index);
     }
 
     match code {
         // Blank lines not allowed.
         Code::None | Code::CarriageReturnLineFeed | Code::Char('\n' | '\r') => (State::Nok, None),
         _ => {
-            let next = tokenizer.events.len();
-            let previous = next - 2;
-
             tokenizer.enter(TokenType::ChunkText);
-
-            tokenizer.events[previous].next = Some(next);
-            tokenizer.events[next].previous = Some(previous);
-
+            let index = tokenizer.events.len() - 1;
+            link(&mut tokenizer.events, index);
             text_inside(tokenizer, code)
         }
     }
diff --git a/src/construct/paragraph.rs b/src/construct/paragraph.rs
index fa18f28..b00188d 100644
--- a/src/construct/paragraph.rs
+++ b/src/construct/paragraph.rs
@@ -35,6 +35,7 @@ use crate::construct::{
     partial_space_or_tab::space_or_tab_min_max, thematic_break::start as thematic_break,
 };
 use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::util::link::link;
 
 /// Before a paragraph.
 ///
@@ -83,9 +84,8 @@ fn at_line_ending(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
     tokenizer.consume(code);
     tokenizer.exit(TokenType::ChunkText);
     tokenizer.enter(TokenType::ChunkText);
-    let next_index = tokenizer.events.len() - 1;
-    tokenizer.events[next_index - 2].next = Some(next_index);
-    tokenizer.events[next_index].previous = Some(next_index - 2);
+    let index = tokenizer.events.len() - 1;
+    link(&mut tokenizer.events, index);
     (State::Fn(Box::new(inside)), None)
 }
 
diff --git a/src/construct/partial_destination.rs b/src/construct/partial_destination.rs
index bc95055..901a10d 100644
--- a/src/construct/partial_destination.rs
+++ b/src/construct/partial_destination.rs
@@ -18,8 +18,8 @@
 //! They are counted with a counter that starts at `0`, and is incremented
 //! every time `(` occurs and decremented every time `)` occurs.
 //! If `)` is found when the counter is `0`, the destination closes immediately
-//! after it.
-//! Escaped parens do not count.
+//! before it.
+//! Escaped parens do not count in balancing.
 //!
 //! It is recommended to use the enclosed variant of destinations, as it allows
 //! arbitrary parens, and also allows for whitespace and other characters in
@@ -68,7 +68,6 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
             tokenizer.enter(TokenType::DefinitionDestination);
             tokenizer.enter(TokenType::DefinitionDestinationRaw);
             tokenizer.enter(TokenType::DefinitionDestinationString);
-            // To do: link.
             tokenizer.enter(TokenType::ChunkString);
             raw(tokenizer, code, 0)
         }
@@ -90,7 +89,6 @@ fn enclosed_before(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
         (State::Ok, None)
     } else {
         tokenizer.enter(TokenType::DefinitionDestinationString);
-        // To do: link.
         tokenizer.enter(TokenType::ChunkString);
         enclosed(tokenizer, code)
     }
diff --git a/src/construct/partial_label.rs b/src/construct/partial_label.rs
index 4997390..55efd13 100644
--- a/src/construct/partial_label.rs
+++ b/src/construct/partial_label.rs
@@ -56,7 +56,9 @@
 // To do: pass token types in.
 
 use crate::constant::LINK_REFERENCE_SIZE_MAX;
+use crate::construct::partial_space_or_tab::space_or_tab_opt;
 use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::util::link::link;
 
 /// Before a label.
 ///
@@ -71,7 +73,10 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
             tokenizer.consume(code);
             tokenizer.exit(TokenType::DefinitionLabelMarker);
             tokenizer.enter(TokenType::DefinitionLabelData);
-            (State::Fn(Box::new(|t, c| at_break(t, c, false, 0))), None)
+            (
+                State::Fn(Box::new(|t, c| at_break(t, c, false, 0, false))),
+                None,
+            )
         }
         _ => (State::Nok, None),
     }
@@ -83,7 +88,13 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// [|a]
 /// [a|]
 /// ```
-fn at_break(tokenizer: &mut Tokenizer, code: Code, data: bool, size: usize) -> StateFnResult {
+fn at_break(
+    tokenizer: &mut Tokenizer,
+    code: Code,
+    data: bool,
+    size: usize,
+    connect: bool,
+) -> StateFnResult {
     match code {
         Code::None | Code::Char('[') => (State::Nok, None),
         Code::Char(']') if !data => (State::Nok, None),
@@ -96,24 +107,57 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, data: bool, size: usize) -> S
             tokenizer.exit(TokenType::DefinitionLabel);
             (State::Ok, None)
         }
-        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
-            tokenizer.enter(TokenType::LineEnding);
-            tokenizer.consume(code);
-            tokenizer.exit(TokenType::LineEnding);
-            // To do: limit blank lines.
-            (
-                State::Fn(Box::new(move |t, c| at_break(t, c, data, size))),
-                None,
-            )
-        }
         _ => {
             tokenizer.enter(TokenType::ChunkString);
-            // To do: link.
+
+            if connect {
+                let index = tokenizer.events.len() - 1;
+                link(&mut tokenizer.events, index);
+            }
+
             label(tokenizer, code, data, size)
         }
     }
 }
 
+/// After a line ending.
+///
+/// ```markdown
+/// [a
+/// |b]
+/// ```
+fn line_start(
+    tokenizer: &mut Tokenizer,
+    code: Code,
+    data: bool,
+    size: usize,
+    connect: bool,
+) -> StateFnResult {
+    tokenizer.go(space_or_tab_opt(), move |t, c| {
+        line_begin(t, c, data, size, connect)
+    })(tokenizer, code)
+}
+
+/// After a line ending, after optional whitespace.
+///
+/// ```markdown
+/// [a
+/// |b]
+/// ```
+fn line_begin(
+    tokenizer: &mut Tokenizer,
+    code: Code,
+    data: bool,
+    size: usize,
+    connect: bool,
+) -> StateFnResult {
+    match code {
+        // Blank line not allowed.
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => (State::Nok, None),
+        _ => at_break(tokenizer, code, data, size, connect),
+    }
+}
+
 /// In a label, in text.
 ///
 /// ```markdown
@@ -121,13 +165,21 @@ fn at_break(tokenizer: &mut Tokenizer, code: Code, data: bool, size: usize) -> S
 /// ```
 fn label(tokenizer: &mut Tokenizer, code: Code, data: bool, size: usize) -> StateFnResult {
     match code {
-        Code::None | Code::CarriageReturnLineFeed | Code::Char('\r' | '\n' | '[' | ']') => {
+        Code::None | Code::Char('[' | ']') => {
             tokenizer.exit(TokenType::ChunkString);
-            at_break(tokenizer, code, data, size)
+            at_break(tokenizer, code, data, size, true)
         }
         _ if size > LINK_REFERENCE_SIZE_MAX => {
             tokenizer.exit(TokenType::ChunkString);
-            at_break(tokenizer, code, data, size)
+            at_break(tokenizer, code, data, size, true)
+        }
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+            tokenizer.consume(code);
+            tokenizer.exit(TokenType::ChunkString);
+            (
+                State::Fn(Box::new(move |t, c| line_start(t, c, data, size + 1, true))),
+                None,
+            )
         }
         Code::VirtualSpace | Code::Char('\t' | ' ') => {
             tokenizer.consume(code);
diff --git a/src/construct/partial_title.rs b/src/construct/partial_title.rs
index 0669c8e..322a3e6 100644
--- a/src/construct/partial_title.rs
+++ b/src/construct/partial_title.rs
@@ -35,6 +35,7 @@
 
 use crate::construct::partial_space_or_tab::space_or_tab_opt;
 use crate::tokenizer::{Code, State, StateFnResult, TokenType, Tokenizer};
+use crate::util::link::link;
 
 /// Type of title.
 #[derive(Debug, Clone, PartialEq)]
@@ -102,7 +103,7 @@ fn begin(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult {
         }
         _ => {
             tokenizer.enter(TokenType::DefinitionTitleString);
-            at_break(tokenizer, code, kind)
+            at_break(tokenizer, code, kind, false)
         }
     }
 }
@@ -115,22 +116,19 @@ fn begin(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult {
 /// (a|
 /// b)
 /// ```
-fn at_break(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult {
+fn at_break(tokenizer: &mut Tokenizer, code: Code, kind: Kind, connect: bool) -> StateFnResult {
     match code {
         Code::Char(char) if char == kind_to_marker(&kind) => {
             tokenizer.exit(TokenType::DefinitionTitleString);
             begin(tokenizer, code, kind)
         }
         Code::None => (State::Nok, None),
-        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
-            tokenizer.enter(TokenType::LineEnding);
-            tokenizer.consume(code);
-            tokenizer.exit(TokenType::LineEnding);
-            (State::Fn(Box::new(|t, c| line_start(t, c, kind))), None)
-        }
         _ => {
-            // To do: link.
             tokenizer.enter(TokenType::ChunkString);
+            if connect {
+                let index = tokenizer.events.len() - 1;
+                link(&mut tokenizer.events, index);
+            }
             title(tokenizer, code, kind)
         }
     }
@@ -156,7 +154,7 @@ fn line_begin(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResul
     match code {
         // Blank line not allowed.
         Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => (State::Nok, None),
-        _ => at_break(tokenizer, code, kind),
+        _ => at_break(tokenizer, code, kind, true),
     }
 }
 
@@ -169,11 +167,20 @@ fn title(tokenizer: &mut Tokenizer, code: Code, kind: Kind) -> StateFnResult {
     match code {
         Code::Char(char) if char == kind_to_marker(&kind) => {
             tokenizer.exit(TokenType::ChunkString);
-            at_break(tokenizer, code, kind)
+            at_break(tokenizer, code, kind, true)
         }
-        Code::None | Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+        Code::None => {
+            tokenizer.exit(TokenType::ChunkString);
+            at_break(tokenizer, code, kind, true)
+        }
+        // To do: limit blank lines.
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+            tokenizer.consume(code);
             tokenizer.exit(TokenType::ChunkString);
-            at_break(tokenizer, code, kind)
+            (
+                State::Fn(Box::new(move |t, c| line_start(t, c, kind))),
+                None,
+            )
         }
         Code::Char('\\') => {
             tokenizer.consume(code);
diff --git a/src/content/string.rs b/src/content/string.rs
index f591cd7..efb6e60 100644
--- a/src/content/string.rs
+++ b/src/content/string.rs
@@ -43,13 +43,28 @@ pub fn start(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// |qwe
 /// ```
 fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
-    if let Code::None = code {
-        (State::Ok, None)
-    } else {
-        tokenizer.enter(TokenType::Data);
-        tokenizer.consume(code);
-        (State::Fn(Box::new(in_data)), None)
+    match code {
+        Code::None => (State::Ok, None),
+        Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
+            tokenizer.enter(TokenType::LineEnding);
+            tokenizer.consume(code);
+            tokenizer.exit(TokenType::LineEnding);
+            (State::Fn(Box::new(start)), None)
+        }
+        _ => {
+            tokenizer.enter(TokenType::Data);
+            tokenizer.consume(code);
+            (State::Fn(Box::new(in_data)), None)
+        }
     }
+
+    // if let Code::None = code {
+    //     (State::Ok, None)
+    // } else {
+    //     tokenizer.enter(TokenType::Data);
+    //     tokenizer.consume(code);
+    //     (State::Fn(Box::new(in_data)), None)
+    // }
 }
 
 /// In data.
@@ -59,10 +74,9 @@ fn before_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
 /// ```
 fn in_data(tokenizer: &mut Tokenizer, code: Code) -> StateFnResult {
     match code {
-        // To do: line endings.
-        Code::None => {
+        Code::None | Code::CarriageReturnLineFeed | Code::Char('\r' | '\n') => {
             tokenizer.exit(TokenType::Data);
-            (State::Ok, None)
+            before_data(tokenizer, code)
         }
         // To do: somehow get these markers from constructs.
         Code::Char('&' | '\\') => {
diff --git a/src/util/link.rs b/src/util/link.rs
new file mode 100644
index 0000000..917ce4d
--- /dev/null
+++ b/src/util/link.rs
@@ -0,0 +1,8 @@
+//! To do.
+
+use crate::tokenizer::Event;
+
+pub fn link(events: &mut [Event], index: usize) {
+    events[index - 2].next = Some(index);
+    events[index].previous = Some(index - 2);
+}
diff --git a/src/util/mod.rs b/src/util/mod.rs
index c3db267..5439c62 100644
--- a/src/util/mod.rs
+++ b/src/util/mod.rs
@@ -2,5 +2,6 @@
 
 pub mod decode_character_reference;
 pub mod encode;
+pub mod link;
 pub mod sanitize_uri;
 pub mod span;