Fix to prefer flow over definitions, setext headings

An undocumented part of CommonMark is how to deal with things in definition
labels or definition titles (which both can span multiple lines).
Can flow (or containers?) interrupt them?
They can according to the `cmark` reference parser, so this was implemented here.

This adds a new `Content` content type, which houses zero or more definitions,
and then zero-or-one paragraphs.
Content can be followed by a setext heading underline, which either turns
into a setext heading when the content ends in a paragraph, or turns into
the start of the following paragraph when it is followed by content that
starts with a paragraph, or turns into a stray paragraph.

1 files changed, 16 insertions, 7 deletions

diff --git a/src/parser.rs b/src/parser.rs
index 3a7713a..c69eb38 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -49,16 +49,25 @@ pub fn parse<'a>(value: &'a str, options: &'a Options) -> Result<(Vec<Event>, &'
         (parse_state.bytes.len(), 0),
         State::Next(StateName::DocumentStart),
     );
-    tokenizer.flush(state, true)?;
-
+    let mut result = tokenizer.flush(state, true)?;
     let mut events = tokenizer.events;
 
-    let footnote = tokenizer.tokenize_state.gfm_footnote_definitions;
-    let normal = tokenizer.tokenize_state.definitions;
-    parse_state.gfm_footnote_definitions = footnote;
-    parse_state.definitions = normal;
+    parse_state
+        .gfm_footnote_definitions
+        .append(&mut result.gfm_footnote_definitions);
+    parse_state.definitions.append(&mut result.definitions);
+
+    loop {
+        let mut result = subtokenize(&mut events, &parse_state, &None)?;
+        parse_state
+            .gfm_footnote_definitions
+            .append(&mut result.gfm_footnote_definitions);
+        parse_state.definitions.append(&mut result.definitions);
 
-    while !(subtokenize(&mut events, &parse_state)?) {}
+        if result.done {
+            break;
+        }
+    }
 
     Ok((events, parse_state.bytes))
 }