From 74d2688aa329f0a41c2a92034c3454ed9299e71a Mon Sep 17 00:00:00 2001
From: Titus Wormer <tituswormer@gmail.com>
Date: Wed, 14 Sep 2022 16:21:42 +0200
Subject: Fix to prefer flow over definitions, setext headings

An undocumented part of CommonMark is how to deal with things in definition
labels or definition titles (which both can span multiple lines).
Can flow (or containers?) interrupt them?
They can according to the `cmark` reference parser, so this was implemented here.

This adds a new `Content` content type, which houses zero or more definitions,
and then zero-or-one paragraphs.
Content can be followed by a setext heading underline, which either turns
into a setext heading when the content ends in a paragraph, or turns into
the start of the following paragraph when it is followed by content that
starts with a paragraph, or turns into a stray paragraph.
---
 src/tokenizer.rs | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

(limited to 'src/tokenizer.rs')

diff --git a/src/tokenizer.rs b/src/tokenizer.rs
index 5095abb..8441f7e 100644
--- a/src/tokenizer.rs
+++ b/src/tokenizer.rs
@@ -12,6 +12,7 @@ use crate::event::{Content, Event, Kind, Link, Name, Point, VOID_EVENTS};
 use crate::parser::ParseState;
 use crate::resolve::{call as call_resolve, Name as ResolveName};
 use crate::state::{call, State};
+use crate::subtokenize::Subresult;
 use crate::util::{char::format_byte_opt, constant::TAB_SIZE, edit_map::EditMap};
 use alloc::{boxed::Box, string::String, vec, vec::Vec};
 
@@ -609,23 +610,35 @@ impl<'a> Tokenizer<'a> {
     }
 
     /// Flush.
-    pub fn flush(&mut self, state: State, resolve: bool) -> Result<(), String> {
+    pub fn flush(&mut self, state: State, resolve: bool) -> Result<Subresult, String> {
         let to = (self.point.index, self.point.vs);
         let state = push_impl(self, to, to, state, true);
-        let result = state.to_result();
 
-        if resolve && result.is_ok() {
+        state.to_result()?;
+
+        let mut value = Subresult {
+            done: false,
+            gfm_footnote_definitions: self.tokenize_state.gfm_footnote_definitions.split_off(0),
+            definitions: self.tokenize_state.definitions.split_off(0),
+        };
+
+        if resolve {
             let resolvers = self.resolvers.split_off(0);
             let mut index = 0;
             while index < resolvers.len() {
-                call_resolve(self, resolvers[index]);
+                if let Some(mut result) = call_resolve(self, resolvers[index])? {
+                    value
+                        .gfm_footnote_definitions
+                        .append(&mut result.gfm_footnote_definitions);
+                    value.definitions.append(&mut result.definitions);
+                }
                 index += 1;
             }
 
             self.map.consume(&mut self.events);
         }
 
-        result
+        Ok(value)
     }
 }
 
-- 
cgit