Fix closing of flow when exiting containers

8 files changed, 122 insertions, 97 deletions

diff --git a/src/construct/code_fenced.rs b/src/construct/code_fenced.rs
index e2165a9..c7b2334 100644
--- a/src/construct/code_fenced.rs
+++ b/src/construct/code_fenced.rs
@@ -568,6 +568,7 @@ fn content_continue(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateF
 /// console.log('1')
 /// ~~~|
 /// ```
+#[allow(clippy::needless_pass_by_value)]
 fn after(tokenizer: &mut Tokenizer, code: Code, info: Info) -> StateFnResult {
     tokenizer.exit(Token::CodeFenced);
     // Feel free to interrupt.
diff --git a/src/content/document.rs b/src/content/document.rs
index feffb62..2934890 100644
--- a/src/content/document.rs
+++ b/src/content/document.rs
@@ -155,7 +155,7 @@ fn document_continue(
     //   assert(point, 'could not find previous flow chunk')
 
     let size = info.continued;
-    exit_containers(tokenizer, &mut info, size);
+    info = exit_containers(tokenizer, info, size);
 
     //   // Fix positions.
     //   let index = indexBeforeExits
@@ -195,8 +195,7 @@ fn check_new_containers(
     // step 1 before creating the new block as a child of the last matched
     // block.
     if info.continued == info.stack.len() {
-        println!("  to do: interrupt ({:?})?", tokenizer.interrupt);
-        //   // No need to `check` whether there’s a container, of `exitContainers`
+        //   // No need to `check` whether there’s a container, if `exitContainers`
         //   // would be moot.
         //   // We can instead immediately `attempt` to parse one.
         //   if (!childFlow) {
@@ -211,6 +210,7 @@ fn check_new_containers(
             return flow_start(tokenizer, code, info);
         }
 
+        println!("  to do: interrupt ({:?})?", tokenizer.interrupt);
         //   // If we do have flow, it could still be a blank line,
         //   // but we’d be interrupting it w/ a new container if there’s a current
         //   // construct.
@@ -237,17 +237,26 @@ fn there_is_a_new_container(
     name: String,
 ) -> StateFnResult {
     println!("there_is_a_new_container");
-    println!("  todo: close_flow");
-    // if (childFlow) closeFlow()
     let size = info.continued;
-    exit_containers(tokenizer, &mut info, size);
+    info = exit_containers(tokenizer, info, size);
     info.stack.push(name);
     info.continued += 1;
     document_continued(tokenizer, code, info)
 }
 
 /// Exit open containers.
-fn exit_containers(tokenizer: &mut Tokenizer, info: &mut DocumentInfo, size: usize) {
+fn exit_containers(tokenizer: &mut Tokenizer, mut info: DocumentInfo, size: usize) -> DocumentInfo {
+    if info.stack.len() > size {
+        println!("closing flow. To do: are these resulting exits okay?");
+        let index_before = tokenizer.events.len();
+        let result = tokenizer.flush(info.next);
+        info.next = Box::new(flow); // This is weird but Rust needs a function there.
+        assert!(matches!(result.0, State::Ok));
+        assert!(result.1.is_none());
+        let shift = tokenizer.events.len() - index_before;
+        info.last_line_ending_index = info.last_line_ending_index.map(|d| d + shift);
+    }
+
     while info.stack.len() > size {
         let name = info.stack.pop().unwrap();
 
@@ -281,6 +290,8 @@ fn exit_containers(tokenizer: &mut Tokenizer, info: &mut DocumentInfo, size: usi
         while index < token_types.len() {
             let token_type = &token_types[index];
 
+            println!("injected exit for `{:?}`", token_type);
+
             info.map.add(
                 insert_index,
                 0,
@@ -314,6 +325,8 @@ fn exit_containers(tokenizer: &mut Tokenizer, info: &mut DocumentInfo, size: usi
             index += 1;
         }
     }
+
+    info
 }
 
 fn there_is_no_new_container(
@@ -373,14 +386,15 @@ fn container_continue(
 }
 
 fn flow_start(tokenizer: &mut Tokenizer, code: Code, mut info: DocumentInfo) -> StateFnResult {
-    println!("flow_start");
-    let next = info.next;
-    info.next = Box::new(flow); // This is weird but Rust needs a function there.
+    println!("flow_start {:?}", code);
 
     let size = info.continued;
-    exit_containers(tokenizer, &mut info, size);
+    info = exit_containers(tokenizer, info, size);
 
-    tokenizer.go_until(next, eof_eol, move |(state, remainder)| {
+    let state = info.next;
+    info.next = Box::new(flow); // This is weird but Rust needs a function there.
+
+    tokenizer.go_until(state, eof_eol, move |(state, remainder)| {
         (
             State::Fn(Box::new(move |t, c| flow_end(t, c, info, state))),
             remainder,
@@ -414,10 +428,15 @@ fn flow_end(
         info.last_line_ending_index = None;
     }
 
+    println!(
+        "set `last_line_ending_index` to {:?}",
+        info.last_line_ending_index
+    );
+
     match result {
         State::Ok => {
             println!("State::Ok");
-            exit_containers(tokenizer, &mut info, 0);
+            info = exit_containers(tokenizer, info, 0);
             tokenizer.events = info.map.consume(&mut tokenizer.events);
             (State::Ok, Some(vec![code]))
         }
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index efd8068..dcbcb09 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -413,12 +413,7 @@ impl<'a> Tokenizer<'a> {
             vec![],
             |result: (Vec<Code>, Vec<Code>), ok, tokenizer: &mut Tokenizer, _state| {
                 if ok {
-                    feed_impl(
-                        tokenizer,
-                        &if ok { result.1 } else { result.0 },
-                        after,
-                        false,
-                    )
+                    feed_impl(tokenizer, &if ok { result.1 } else { result.0 }, after)
                 } else {
                     (State::Nok, None)
                 }
@@ -468,7 +463,7 @@ impl<'a> Tokenizer<'a> {
             vec![],
             |result: (Vec<Code>, Vec<Code>), ok, tokenizer: &mut Tokenizer, _state| {
                 tokenizer.free(previous);
-                feed_impl(tokenizer, &result.0, done(ok), false)
+                feed_impl(tokenizer, &result.0, done(ok))
             },
         )
     }
@@ -508,7 +503,7 @@ impl<'a> Tokenizer<'a> {
                     codes,
                     tokenizer.point
                 );
-                feed_impl(tokenizer, &codes, done(ok), false)
+                feed_impl(tokenizer, &codes, done(ok))
             },
         )
     }
@@ -556,9 +551,16 @@ impl<'a> Tokenizer<'a> {
     ) -> StateFnResult {
         assert!(!self.drained, "cannot feed after drain");
 
-        let result = feed_impl(self, codes, start, drain);
+        let mut result = feed_impl(self, codes, start);
 
         if drain {
+            let func = match result.0 {
+                State::Fn(func) => func,
+                _ => unreachable!("expected next state"),
+            };
+
+            result = flush_impl(self, func);
+
             self.drained = true;
 
             while !self.resolvers.is_empty() {
@@ -569,6 +571,14 @@ impl<'a> Tokenizer<'a> {
 
         result
     }
+
+    /// To do.
+    pub fn flush(
+        &mut self,
+        start: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
+    ) -> StateFnResult {
+        flush_impl(self, start)
+    }
 }
 
 /// Internal utility to wrap states to also capture codes.
@@ -635,7 +645,6 @@ fn feed_impl(
     tokenizer: &mut Tokenizer,
     codes: &[Code],
     start: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
-    drain: bool,
 ) -> StateFnResult {
     let codes = codes;
     let mut state = State::Fn(Box::new(start));
@@ -665,10 +674,17 @@ fn feed_impl(
         }
     }
 
-    // Yield to a higher loop if we shouldn’t feed EOFs.
-    if !drain {
-        return check_statefn_result((state, Some(codes[index..].to_vec())));
-    }
+    // Yield to a higher loop.
+    check_statefn_result((state, Some(codes[index..].to_vec())))
+}
+
+/// To do.
+fn flush_impl(
+    tokenizer: &mut Tokenizer,
+    start: impl FnOnce(&mut Tokenizer, Code) -> StateFnResult + 'static,
+) -> StateFnResult {
+    let mut state = State::Fn(Box::new(start));
+    tokenizer.consumed = true;
 
     loop {
         // Feed EOF.
diff --git a/tests/block_quote.rs b/tests/block_quote.rs
index 02256e1..af9e9d9 100644
--- a/tests/block_quote.rs
+++ b/tests/block_quote.rs
@@ -47,12 +47,11 @@ fn block_quote() {
         "should support setext headings underlines in block quotes"
     );
 
-    // To do: block quote (lazy, setext underline)
-    // assert_eq!(
-    //     micromark("> a\n---"),
-    //     "<blockquote>\n<p>a</p>\n</blockquote>\n<hr />",
-    //     "should not support lazy setext headings underlines in block quotes"
-    // );
+    assert_eq!(
+        micromark("> a\n---"),
+        "<blockquote>\n<p>a</p>\n</blockquote>\n<hr />",
+        "should not support lazy setext headings underlines in block quotes"
+    );
 
     // To do: list.
     // assert_eq!(
@@ -156,26 +155,25 @@ fn block_quote() {
         "should not support interrupting a blank line in a block quotes w/ paragraphs"
     );
 
-    // To do: block quote (multi, lazy).
+    // To do: block quote (lazy).
     // assert_eq!(
     //     micromark("> > > a\nb"),
     //     "<blockquote>\n<blockquote>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</blockquote>\n</blockquote>",
     //     "should not support interrupting many block quotes w/ paragraphs (1)"
     // );
 
-    // To do: block quote (multi, lazy).
+    // To do: block quote (lazy).
     // assert_eq!(
     //     micromark(">>> a\n> b\n>>c"),
     //     "<blockquote>\n<blockquote>\n<blockquote>\n<p>a\nb\nc</p>\n</blockquote>\n</blockquote>\n</blockquote>",
     //     "should not support interrupting many block quotes w/ paragraphs (2)"
     // );
 
-    // To do: block quote (exit flow if container exits).
-    // assert_eq!(
-    //     micromark(">     a\n\n>    b"),
-    //     "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<blockquote>\n<p>b</p>\n</blockquote>",
-    //     "should support 5 spaces for indented code, not 4"
-    // );
+    assert_eq!(
+        micromark(">     a\n\n>    b"),
+        "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<blockquote>\n<p>b</p>\n</blockquote>",
+        "should support 5 spaces for indented code, not 4"
+    );
 
     // To do: turning things off.
     // assert_eq!(
diff --git a/tests/code_fenced.rs b/tests/code_fenced.rs
index 84d0d83..29ef62a 100644
--- a/tests/code_fenced.rs
+++ b/tests/code_fenced.rs
@@ -234,7 +234,7 @@ fn code_fenced() {
     //     "should not support a closing sequence w/ too much indent, regardless of opening sequence (2)"
     // );
 
-    // To do: blockquote (some bug).
+    // To do: blockquote (some bug, perhaps compiler).
     // assert_eq!(
     //     micromark("> ```a\nb"),
     //     "<blockquote>\n<pre><code class=\"language-a\"></code></pre>\n</blockquote>\n<p>b</p>",
@@ -247,7 +247,7 @@ fn code_fenced() {
         "should not support lazyness (2)"
     );
 
-    // To do: blockquote (lazy).
+    // To do: blockquote (some bug, also compiler).
     // assert_eq!(
     //     micromark("> ```a\n```"),
     //     "<blockquote>\n<pre><code class=\"language-a\"></code></pre>\n</blockquote>\n<pre><code></code></pre>\n",
diff --git a/tests/code_indented.rs b/tests/code_indented.rs
index d7cf181..ba1b483 100644
--- a/tests/code_indented.rs
+++ b/tests/code_indented.rs
@@ -76,12 +76,11 @@ fn code_indented() {
         "should support trailing whitespace"
     );
 
-    // To do: blockquote (some bug).
-    // assert_eq!(
-    //     micromark(">     a\nb"),
-    //     "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<p>b</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark(">     a\nb"),
+        "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<p>b</p>",
+        "should not support lazyness (1)"
+    );
 
     // To do: blockquote (lazy).
     // assert_eq!(
diff --git a/tests/definition.rs b/tests/definition.rs
index ca8b97c..ffb3460 100644
--- a/tests/definition.rs
+++ b/tests/definition.rs
@@ -192,12 +192,11 @@ fn definition() {
         "should support definitions after definitions"
     );
 
-    // To do: block quote (some bug).
-    // assert_eq!(
-    //     micromark("> [foo]: /url\n\n[foo]"),
-    //     "<blockquote>\n</blockquote>\n<p><a href=\"/url\">foo</a></p>",
-    //     "should support definitions in block quotes"
-    // );
+    assert_eq!(
+        micromark("> [foo]: /url\n\n[foo]"),
+        "<blockquote>\n</blockquote>\n<p><a href=\"/url\">foo</a></p>",
+        "should support definitions in block quotes"
+    );
 
     // Extra
     assert_eq!(
diff --git a/tests/html_flow.rs b/tests/html_flow.rs
index 796ced4..71e7bbe 100644
--- a/tests/html_flow.rs
+++ b/tests/html_flow.rs
@@ -171,12 +171,11 @@ p {color:blue;}
         "should support blank lines in raw"
     );
 
-    // To do: block quote (lazy).
-    // assert_eq!(
-    //     micromark_with_options("> <script>\na", DANGER),
-    //     "<blockquote>\n<script>\n</blockquote>\n<p>a</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <script>\na", DANGER),
+        "<blockquote>\n<script>\n</blockquote>\n<p>a</p>",
+        "should not support lazyness (1)"
+    );
 
     assert_eq!(
         micromark_with_options("> a\n<script>", DANGER),
@@ -270,12 +269,11 @@ fn html_flow_2_comment() {
         "should support blank lines in comments"
     );
 
-    // To do: blockquote (lazy).
-    // assert_eq!(
-    //     micromark_with_options("> <!--\na", DANGER),
-    //     "<blockquote>\n<!--\n</blockquote>\n<p>a</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <!--\na", DANGER),
+        "<blockquote>\n<!--\n</blockquote>\n<p>a</p>",
+        "should not support lazyness (1)"
+    );
 
     assert_eq!(
         micromark_with_options("> a\n<!--", DANGER),
@@ -317,12 +315,11 @@ fn html_flow_3_instruction() {
         "should support blank lines in instructions"
     );
 
-    // To do: blockquote (lazy).
-    // assert_eq!(
-    //     micromark_with_options("> <?\na", DANGER),
-    //     "<blockquote>\n<?\n</blockquote>\n<p>a</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <?\na", DANGER),
+        "<blockquote>\n<?\n</blockquote>\n<p>a</p>",
+        "should not support lazyness (1)"
+    );
 
     assert_eq!(
         micromark_with_options("> a\n<?", DANGER),
@@ -372,12 +369,11 @@ fn html_flow_4_declaration() {
         "should support blank lines in declarations"
     );
 
-    // To do: blockquote (lazy).
-    // assert_eq!(
-    //     micromark_with_options("> <!a\nb", DANGER),
-    //     "<blockquote>\n<!a\n</blockquote>\n<p>b</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <!a\nb", DANGER),
+        "<blockquote>\n<!a\n</blockquote>\n<p>b</p>",
+        "should not support lazyness (1)"
+    );
 
     assert_eq!(
         micromark_with_options("> a\n<!b", DANGER),
@@ -436,12 +432,11 @@ fn html_flow_5_cdata() {
         "should support blank lines in cdata"
     );
 
-    // To do: blockquote (lazy).
-    // assert_eq!(
-    //     micromark_with_options("> <![CDATA[\na", DANGER),
-    //     "<blockquote>\n<![CDATA[\n</blockquote>\n<p>a</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <![CDATA[\na", DANGER),
+        "<blockquote>\n<![CDATA[\n</blockquote>\n<p>a</p>",
+        "should not support lazyness (1)"
+    );
 
     assert_eq!(
         micromark_with_options("> a\n<![CDATA[", DANGER),
@@ -715,12 +710,11 @@ okay.",
         "should not support blank lines in basic"
     );
 
-    // To do: block quote (some bug).
-    // assert_eq!(
-    //     micromark_with_options("> <div\na", DANGER),
-    //     "<blockquote>\n<div\n</blockquote>\n<p>a</p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <div\na", DANGER),
+        "<blockquote>\n<div\n</blockquote>\n<p>a</p>",
+        "should not support lazyness (1)"
+    );
 
     assert_eq!(
         micromark_with_options("> a\n<div", DANGER),
@@ -1019,12 +1013,11 @@ fn html_flow_7_complete() {
         "should not support blank lines in complete"
     );
 
-    // To do: containers: close flow when closing container.
-    // assert_eq!(
-    //     micromark_with_options("> <a>\n*bar*", DANGER),
-    //     "<blockquote>\n<a>\n</blockquote>\n<p><em>bar</em></p>",
-    //     "should not support lazyness (1)"
-    // );
+    assert_eq!(
+        micromark_with_options("> <a>\n*bar*", DANGER),
+        "<blockquote>\n<a>\n</blockquote>\n<p><em>bar</em></p>",
+        "should not support lazyness (1)"
+    );
 
     // To do: blockquote (lazy).
     // assert_eq!(